encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BUILD.gn: Remove use of exec_script 

Chromium's BUILD files try to avoid uses of exec_script when possible
because they slow down every GN invocation. In preparation for building
Dawn inside Chromium, the calls to exec_script for the code generator
are removed.

In GN, the generator now outputs a "JSON tarball", a dictionnary mapping
filenames to content. This allows us to use the "depfile" feature of GN
to avoid the exec_script call to gather the script's inputs.

Outputs of the generator are now listed in the BUILD.gn files. To keep
it in sync with the generator, GN outputs a file containing "expected
outputs" that is checked by the code generator.

Finally the dawn_generator GN template doesn't create a target anymore,
but users are expected to gather outputs using get_target_outputs.
This commit is contained in:
Corentin Wallez 2018-08-16 15:32:35 +02:00 committed by Corentin Wallez
parent d273d33a63
commit 59e7fad99b
3 changed files with 276 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

267
BUILD.gn
View File

@ -21,96 +21,104 @@ import("//testing/test.gni")
# Template to wrap the Dawn code generator
###############################################################################
jinja2_python_path = rebase_path("${dawn_jinja2_dir}/..")
# Template to help with invocation of the Dawn code generator, looks like this:
#
# dawn_generator("my_target_gen") {
# # Which generator target to output
# target = "my_target"
# # The list of expected outputs, generation fails if there's a mismatch
# outputs = [
# "MyTarget.cpp",
# "MyTarget.h",
# ]
# }
#
# Using the generated files is done like so:
#
# shared_library("my_target") {
# deps = [ ":my_target_gen "]
# sources = get_target_outputs(":my_target_gen")
# }
#
template("dawn_generator") {
generator = "generator/main.py"
json = "dawn.json"
template_dir = "generator/templates"
target = invoker.target
common_args = [
rebase_path(json, root_build_dir),
# The base arguments for the generator: from this dawn.json, generate this
# target using templates in this directory.
generator_args = [
rebase_path("dawn.json", root_build_dir),
"--template-dir",
rebase_path(template_dir, root_build_dir),
"--output-dir",
rebase_path(target_gen_dir, root_build_dir),
"--extra-python-path",
jinja2_python_path,
rebase_path("generator/templates", root_build_dir),
"--targets",
target,
invoker.target,
]
# Gather the inputs and outputs of the code generator.
# TODO(cwallez@chromium.org): These exec_script are cheap but are done on
# every GN invocation. However we can't use a depfile because the generator
# can have multiple outputs which ninja doesn't support in depfiles.
# We could avoid exec_script by making the code generator generate a single
# output such as a tarball:
# - The outputs would be the tarball
# - The inputs could be handled via a depfile
# Then another action would depend on the generator target and decompress the
# tarball.
script_inputs = exec_script(generator,
common_args + [
"--print-dependencies",
"--gn",
],
"list lines",
[ json ])
script_outputs = exec_script(generator,
common_args + [
"--print-outputs",
"--gn",
],
"list lines",
[ json ])
# Use the Jinja2 version pulled from the DEPS file. We do it so we don't
# have version problems, and users don't have to install Jinja2.
jinja2_python_path = rebase_path("${dawn_jinja2_dir}/..")
generator_args += [
"--extra-python-path",
jinja2_python_path,
]
rebased_outputs = []
foreach(path, script_outputs) {
rebased_outputs +=
[ root_build_dir + "/" + rebase_path(path, root_build_dir) ]
# For build parallelism GN wants to know the exact inputs and outputs of
# action targets like we use for our code generator. We avoid asking the
# generator about its inputs by using the "depfile" feature of GN/Ninja.
#
# A ninja limitation is that the depfile is a subset of Makefile that can
# contain a single target, so we output a single "JSON-tarball" instead.
json_tarball = "${target_gen_dir}/${target_name}.json_tarball"
json_tarball_depfile = "${json_tarball}.d"
generator_args += [
"--output-json-tarball",
rebase_path(json_tarball, root_build_dir),
"--depfile",
rebase_path(json_tarball_depfile, root_build_dir),
]
# After the JSON tarball is created we need an action target to extract it
# with a list of its outputs. The invoker provided a list of expected
# outputs. To make sure the list is in sync between the generator and the
# build files, we write it to a file and ask the generator to assert it is
# correct.
expected_outputs_file = "${target_gen_dir}/${target_name}.expected_outputs"
write_file(expected_outputs_file, invoker.outputs)
generator_args += [
"--expected-outputs-file",
rebase_path(expected_outputs_file, root_build_dir),
]
# The code generator invocation that will write the JSON tarball, check the
# outputs are what's expected and write a depfile for Ninja.
action("${target_name}_json_tarball") {
script = "generator/main.py"
outputs = [
json_tarball,
]
depfile = json_tarball_depfile
args = generator_args
}
# Make an action to execute the code generator
action("${target_name}_gen") {
script = generator
inputs = script_inputs
outputs = rebased_outputs
args = common_args
}
# Extract the JSON tarball into the target_gen_dir
action("${target_name}") {
script = "generator/extract_json.py"
args = [
rebase_path(json_tarball, root_build_dir),
rebase_path(target_gen_dir, root_build_dir),
]
if (defined(invoker.target_type)) {
# Make a target with a custom target_type that will contain the outputs of
# the code generator in sources.
target(invoker.target_type, target_name) {
deps = [
":${target_name}_gen",
]
sources = script_outputs
deps = [
":${target_name}_json_tarball",
]
inputs = [
json_tarball,
]
# Forward variables from the invoker. deps, configs and source are
# special cased because the invoker's must be added to the lists already
# present in this target
forward_variables_from(invoker,
"*",
[
"deps",
"target",
"configs",
"sources",
])
if (defined(invoker.deps)) {
deps += invoker.deps
}
if (defined(invoker.sources)) {
sources += invoker.sources
}
if (defined(invoker.configs)) {
configs += invoker.configs
}
# The expected output list is relative to the target_gen_dir but action
# target outputs are from the root dir so we need to rebase them.
outputs = []
foreach(source, invoker.outputs) {
outputs += [ "${target_gen_dir}/${source}" ]
}
}
}
@ -197,28 +205,48 @@ static_library("dawn_common") {
# Dawn headers and libdawn.so
###############################################################################
dawn_generator("dawn_headers") {
dawn_generator("dawn_headers_gen") {
target = "dawn_headers"
target_type = "source_set"
outputs = [
"dawn/dawncpp.h",
"dawn/dawn.h",
"dawn/dawncpp_traits.h",
]
}
source_set("dawn_headers") {
public_configs = [ ":libdawn_public" ]
sources = [
deps = [
":dawn_headers_gen",
]
sources = get_target_outputs(":dawn_headers_gen")
sources += [
"src/include/dawn/EnumClassBitmasks.h",
"src/include/dawn/dawn_export.h",
"src/include/dawn/dawn_wsi.h",
]
}
dawn_generator("libdawn") {
dawn_generator("libdawn_gen") {
target = "libdawn"
target_type = "shared_library"
defines = [ "DAWN_IMPLEMENTATION" ]
outputs = [
"dawn/dawncpp.cpp",
"dawn/dawn.c",
]
}
shared_library("libdawn") {
public_deps = [
":dawn_headers",
]
defines = [ "DAWN_IMPLEMENTATION" ]
deps = [
":libdawn_gen",
]
sources = get_target_outputs(":libdawn_gen")
# Tell dependents where to find this shared library
if (is_mac) {
ldflags = [
@ -243,13 +271,14 @@ config("libdawn_native_internal") {
}
}
dawn_generator("libdawn_native_utils") {
dawn_generator("libdawn_native_utils_gen") {
target = "dawn_native_utils"
target_type = "source_set"
configs = [ ":libdawn_native_internal" ]
deps = [
":dawn_headers",
outputs = [
"dawn_native/ProcTable.cpp",
"dawn_native/dawn_structs_autogen.h",
"dawn_native/dawn_structs_autogen.cpp",
"dawn_native/ValidationUtils_autogen.h",
"dawn_native/ValidationUtils_autogen.cpp",
]
}
@ -278,7 +307,7 @@ source_set("libdawn_native_headers") {
source_set("libdawn_native_sources") {
deps = [
":dawn_common",
":libdawn_native_utils",
":libdawn_native_utils_gen",
"third_party:spirv_cross",
]
@ -292,7 +321,8 @@ source_set("libdawn_native_sources") {
libs = []
sources = [
sources = get_target_outputs(":libdawn_native_utils_gen")
sources += [
"src/dawn_native/BindGroup.cpp",
"src/dawn_native/BindGroup.h",
"src/dawn_native/BindGroupLayout.cpp",
@ -591,28 +621,28 @@ source_set("libdawn_wire_headers") {
]
}
# The meat of the compilation for libdawn_wire so that we can cheaply have
# shared_library / static_library / component versions of it.
dawn_generator("libdawn_wire_sources") {
dawn_generator("libdawn_wire_gen") {
target = "dawn_wire"
target_type = "source_set"
configs = [ ":dawn_internal" ]
deps = [
":dawn_common",
":libdawn_wire_headers",
]
defines = [ "DAWN_WIRE_IMPLEMENTATION" ]
sources = [
"src/dawn_wire/WireCmd.h",
outputs = [
"dawn_wire/WireServer.cpp",
"dawn_wire/WireCmd_autogen.h",
"dawn_wire/WireClient.cpp",
"dawn_wire/WireCmd_autogen.cpp",
]
}
shared_library("libdawn_wire") {
deps = [
":libdawn_wire_sources",
":dawn_common",
":libdawn_wire_gen",
":libdawn_wire_headers",
]
configs += [ ":dawn_internal" ]
defines = [ "DAWN_WIRE_IMPLEMENTATION" ]
sources = get_target_outputs(":libdawn_wire_gen")
sources += [ "src/dawn_wire/WireCmd.h" ]
#Make headers publically visible
public_deps = [
":libdawn_wire_headers",
@ -646,7 +676,6 @@ static_library("dawn_utils") {
"src/utils/TerribleCommandBuffer.cpp",
"src/utils/TerribleCommandBuffer.h",
]
libs = []
deps = [
":dawn_common",
":libdawn_native",
@ -654,6 +683,7 @@ static_library("dawn_utils") {
"third_party:glfw",
"third_party:libshaderc",
]
libs = []
if (dawn_enable_d3d12) {
sources += [ "src/utils/D3D12Binding.cpp" ]
@ -691,14 +721,11 @@ static_library("dawn_utils") {
# Dawn test targets
###############################################################################
dawn_generator("mock_dawn") {
testonly = true
dawn_generator("mock_dawn_gen") {
target = "mock_dawn"
target_type = "source_set"
deps = [
":dawn_headers",
"third_party:gmock",
outputs = [
"mock/mock_dawn.h",
"mock/mock_dawn.cpp",
]
}
@ -711,16 +738,18 @@ test("dawn_unittests") {
deps = [
":dawn_common",
":dawn_headers",
":dawn_utils",
":libdawn",
":libdawn_native_sources",
":libdawn_wire",
":mock_dawn",
":mock_dawn_gen",
"third_party:gmock",
"third_party:gtest",
]
sources = [
sources = get_target_outputs(":mock_dawn_gen")
sources += [
"src/tests/UnittestsMain.cpp",
"src/tests/unittests/BitSetIteratorTests.cpp",
"src/tests/unittests/CommandAllocatorTests.cpp",

35
generator/extract_json.py Normal file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env python2
# Copyright 2018 The Dawn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os, sys, json
if __name__ == "__main__":
if len(sys.argv) != 3:
print("Usage: extract_json.py JSON DIR")
sys.exit(1)
with open(sys.argv[1]) as f:
files = json.loads(f.read())
output_dir = sys.argv[2]
for (name, content) in files.iteritems():
output_file = output_dir + os.path.sep + name
directory = os.path.dirname(output_file)
if not os.path.exists(directory):
os.makedirs(directory)
with open(output_file, 'w') as outfile:
outfile.write(content)

137
generator/main.py
View File

@ -280,21 +280,20 @@ class PreprocessingLoader(jinja2.BaseLoader):
FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
def do_renders(renders, template_dir, output_dir):
FileOutput = namedtuple('FileOutput', ['name', 'content'])
def do_renders(renders, template_dir):
env = jinja2.Environment(loader=PreprocessingLoader(template_dir), trim_blocks=True, line_comment_prefix='//*')
outputs = []
for render in renders:
params = {}
for param_dict in render.params_dicts:
params.update(param_dict)
output = env.get_template(render.template).render(**params)
content = env.get_template(render.template).render(**params)
outputs.append(FileOutput(render.output, content))
output_file = output_dir + os.path.sep + render.output
directory = os.path.dirname(output_file)
if not os.path.exists(directory):
os.makedirs(directory)
with open(output_file, 'w') as outfile:
outfile.write(output)
return outputs
#############################################################
# MAIN SOMETHING WHATEVER
@ -389,37 +388,7 @@ def js_native_methods(types, typ):
def debug(text):
print(text)
def main():
targets = ['dawn_headers', 'libdawn', 'mock_dawn', 'dawn_wire', "dawn_native_utils"]
parser = argparse.ArgumentParser(
description = 'Generates code for various target for Dawn.',
formatter_class = argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('json', metavar='DAWN_JSON', nargs=1, type=str, help ='The DAWN JSON definition to use.')
parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.')
parser.add_argument('-o', '--output-dir', default=None, type=str, help='Output directory for the generated source files.')
parser.add_argument('-T', '--targets', default=None, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(targets))
parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2')
parser.add_argument('--print-dependencies', action='store_true', help='Prints a space separated list of file dependencies, used for CMake integration')
parser.add_argument('--print-outputs', action='store_true', help='Prints a space separated list of file outputs, used for CMake integration')
parser.add_argument('--gn', action='store_true', help='Make the printing of dependencies/outputs GN-friendly')
args = parser.parse_args()
if args.targets != None:
targets = args.targets.split(',')
with open(args.json[0]) as f:
loaded_json = json.loads(f.read())
# A fake api_params to avoid parsing the JSON when just querying dependencies and outputs
api_params = {
'types': {}
}
if not args.print_outputs and not args.print_dependencies:
api_params = parse_json(loaded_json)
def get_renders_for_targets(api_params, targets):
base_params = {
'enumerate': enumerate,
'format': format,
@ -491,7 +460,61 @@ def main():
renders.append(FileRender('dawn_wire/WireClient.cpp', 'dawn_wire/WireClient.cpp', wire_params))
renders.append(FileRender('dawn_wire/WireServer.cpp', 'dawn_wire/WireServer.cpp', wire_params))
output_separator = '\n' if args.gn else ';'
return renders
def output_to_files(outputs, output_dir):
for output in outputs:
output_file = output_dir + os.path.sep + output.name
directory = os.path.dirname(output_file)
if not os.path.exists(directory):
os.makedirs(directory)
with open(output_file, 'w') as outfile:
outfile.write(output.content)
def output_to_json(outputs, output_json):
json_root = {}
for output in outputs:
json_root[output.name] = output.content
with open(output_json, 'w') as f:
f.write(json.dumps(json_root))
def output_depfile(depfile, output, dependencies):
with open(depfile, 'w') as f:
f.write(output + ": " + " ".join(dependencies))
def main():
allowed_targets = ['dawn_headers', 'libdawn', 'mock_dawn', 'dawn_wire', "dawn_native_utils"]
parser = argparse.ArgumentParser(
description = 'Generates code for various target for Dawn.',
formatter_class = argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('json', metavar='DAWN_JSON', nargs=1, type=str, help ='The DAWN JSON definition to use.')
parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.')
parser.add_argument('-T', '--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets))
# Arguments used only for the GN build
parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2')
parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).')
parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball')
parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match")
# Arguments used only for the CMake build
parser.add_argument('-o', '--output-dir', default=None, type=str, help='Output directory for the generated source files.')
parser.add_argument('--print-dependencies', action='store_true', help='Prints a space separated list of file dependencies, used for CMake integration')
parser.add_argument('--print-outputs', action='store_true', help='Prints a space separated list of file outputs, used for CMake integration')
args = parser.parse_args()
# Load and parse the API json file
with open(args.json[0]) as f:
loaded_json = json.loads(f.read())
api_params = parse_json(loaded_json)
targets = args.targets.split(',')
renders = get_renders_for_targets(api_params, targets)
# Print outputs and dependencies for CMake
if args.print_dependencies:
dependencies = set(
[os.path.abspath(args.template_dir + os.path.sep + render.template) for render in renders] +
@ -499,7 +522,7 @@ def main():
[os.path.realpath(__file__)]
)
dependencies = [dependency.replace('\\', '/') for dependency in dependencies]
sys.stdout.write(output_separator.join(dependencies))
sys.stdout.write(';'.join(dependencies))
return 0
if args.print_outputs:
@ -507,10 +530,36 @@ def main():
[os.path.abspath(args.output_dir + os.path.sep + render.output) for render in renders]
)
outputs = [output.replace('\\', '/') for output in outputs]
sys.stdout.write(output_separator.join(outputs))
sys.stdout.write(';'.join(outputs))
return 0
do_renders(renders, args.template_dir, args.output_dir)
# The caller wants to assert that the outputs are what it expects.
# Load the file and compare with our renders. GN-only.
if args.expected_outputs_file != None:
with open(args.expected_outputs_file) as f:
expected = set([line.strip() for line in f.readlines()])
actual = set()
actual.update([render.output for render in renders])
if actual != expected:
print("Wrong expected outputs, caller expected:\n " + repr(list(expected)))
print("Actual output:\n " + repr(list(actual)))
return 1
outputs = do_renders(renders, args.template_dir)
# CMake only: output all the files directly.
if args.output_dir != None:
output_to_files(outputs, args.output_dir)
# GN only: output the tarball and its depfile
if args.output_json_tarball != None:
output_to_json(outputs, args.output_json_tarball)
dependencies = [args.template_dir + os.path.sep + render.template for render in renders]
dependencies.append(args.json[0])
output_depfile(args.depfile, args.output_json_tarball, dependencies)
if __name__ == '__main__':
sys.exit(main())