diff --git a/BUILD.gn b/BUILD.gn index 265aa5fac1..77febcb040 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -21,96 +21,104 @@ import("//testing/test.gni") # Template to wrap the Dawn code generator ############################################################################### -jinja2_python_path = rebase_path("${dawn_jinja2_dir}/..") - +# Template to help with invocation of the Dawn code generator, looks like this: +# +# dawn_generator("my_target_gen") { +# # Which generator target to output +# target = "my_target" +# # The list of expected outputs, generation fails if there's a mismatch +# outputs = [ +# "MyTarget.cpp", +# "MyTarget.h", +# ] +# } +# +# Using the generated files is done like so: +# +# shared_library("my_target") { +# deps = [ ":my_target_gen "] +# sources = get_target_outputs(":my_target_gen") +# } +# template("dawn_generator") { - generator = "generator/main.py" - json = "dawn.json" - template_dir = "generator/templates" - - target = invoker.target - - common_args = [ - rebase_path(json, root_build_dir), + # The base arguments for the generator: from this dawn.json, generate this + # target using templates in this directory. + generator_args = [ + rebase_path("dawn.json", root_build_dir), "--template-dir", - rebase_path(template_dir, root_build_dir), - "--output-dir", - rebase_path(target_gen_dir, root_build_dir), - "--extra-python-path", - jinja2_python_path, + rebase_path("generator/templates", root_build_dir), "--targets", - target, + invoker.target, ] - # Gather the inputs and outputs of the code generator. - # TODO(cwallez@chromium.org): These exec_script are cheap but are done on - # every GN invocation. However we can't use a depfile because the generator - # can have multiple outputs which ninja doesn't support in depfiles. - # We could avoid exec_script by making the code generator generate a single - # output such as a tarball: - # - The outputs would be the tarball - # - The inputs could be handled via a depfile - # Then another action would depend on the generator target and decompress the - # tarball. - script_inputs = exec_script(generator, - common_args + [ - "--print-dependencies", - "--gn", - ], - "list lines", - [ json ]) - script_outputs = exec_script(generator, - common_args + [ - "--print-outputs", - "--gn", - ], - "list lines", - [ json ]) + # Use the Jinja2 version pulled from the DEPS file. We do it so we don't + # have version problems, and users don't have to install Jinja2. + jinja2_python_path = rebase_path("${dawn_jinja2_dir}/..") + generator_args += [ + "--extra-python-path", + jinja2_python_path, + ] - rebased_outputs = [] - foreach(path, script_outputs) { - rebased_outputs += - [ root_build_dir + "/" + rebase_path(path, root_build_dir) ] + # For build parallelism GN wants to know the exact inputs and outputs of + # action targets like we use for our code generator. We avoid asking the + # generator about its inputs by using the "depfile" feature of GN/Ninja. + # + # A ninja limitation is that the depfile is a subset of Makefile that can + # contain a single target, so we output a single "JSON-tarball" instead. + json_tarball = "${target_gen_dir}/${target_name}.json_tarball" + json_tarball_depfile = "${json_tarball}.d" + + generator_args += [ + "--output-json-tarball", + rebase_path(json_tarball, root_build_dir), + "--depfile", + rebase_path(json_tarball_depfile, root_build_dir), + ] + + # After the JSON tarball is created we need an action target to extract it + # with a list of its outputs. The invoker provided a list of expected + # outputs. To make sure the list is in sync between the generator and the + # build files, we write it to a file and ask the generator to assert it is + # correct. + expected_outputs_file = "${target_gen_dir}/${target_name}.expected_outputs" + write_file(expected_outputs_file, invoker.outputs) + + generator_args += [ + "--expected-outputs-file", + rebase_path(expected_outputs_file, root_build_dir), + ] + + # The code generator invocation that will write the JSON tarball, check the + # outputs are what's expected and write a depfile for Ninja. + action("${target_name}_json_tarball") { + script = "generator/main.py" + outputs = [ + json_tarball, + ] + depfile = json_tarball_depfile + args = generator_args } - # Make an action to execute the code generator - action("${target_name}_gen") { - script = generator - inputs = script_inputs - outputs = rebased_outputs - args = common_args - } + # Extract the JSON tarball into the target_gen_dir + action("${target_name}") { + script = "generator/extract_json.py" + args = [ + rebase_path(json_tarball, root_build_dir), + rebase_path(target_gen_dir, root_build_dir), + ] - if (defined(invoker.target_type)) { - # Make a target with a custom target_type that will contain the outputs of - # the code generator in sources. - target(invoker.target_type, target_name) { - deps = [ - ":${target_name}_gen", - ] - sources = script_outputs + deps = [ + ":${target_name}_json_tarball", + ] + inputs = [ + json_tarball, + ] - # Forward variables from the invoker. deps, configs and source are - # special cased because the invoker's must be added to the lists already - # present in this target - forward_variables_from(invoker, - "*", - [ - "deps", - "target", - "configs", - "sources", - ]) - - if (defined(invoker.deps)) { - deps += invoker.deps - } - if (defined(invoker.sources)) { - sources += invoker.sources - } - if (defined(invoker.configs)) { - configs += invoker.configs - } + # The expected output list is relative to the target_gen_dir but action + # target outputs are from the root dir so we need to rebase them. + outputs = [] + foreach(source, invoker.outputs) { + outputs += [ "${target_gen_dir}/${source}" ] } } } @@ -197,28 +205,48 @@ static_library("dawn_common") { # Dawn headers and libdawn.so ############################################################################### -dawn_generator("dawn_headers") { +dawn_generator("dawn_headers_gen") { target = "dawn_headers" - target_type = "source_set" + outputs = [ + "dawn/dawncpp.h", + "dawn/dawn.h", + "dawn/dawncpp_traits.h", + ] +} +source_set("dawn_headers") { public_configs = [ ":libdawn_public" ] - sources = [ + deps = [ + ":dawn_headers_gen", + ] + + sources = get_target_outputs(":dawn_headers_gen") + sources += [ "src/include/dawn/EnumClassBitmasks.h", "src/include/dawn/dawn_export.h", "src/include/dawn/dawn_wsi.h", ] } -dawn_generator("libdawn") { +dawn_generator("libdawn_gen") { target = "libdawn" - target_type = "shared_library" - - defines = [ "DAWN_IMPLEMENTATION" ] + outputs = [ + "dawn/dawncpp.cpp", + "dawn/dawn.c", + ] +} +shared_library("libdawn") { public_deps = [ ":dawn_headers", ] + defines = [ "DAWN_IMPLEMENTATION" ] + deps = [ + ":libdawn_gen", + ] + sources = get_target_outputs(":libdawn_gen") + # Tell dependents where to find this shared library if (is_mac) { ldflags = [ @@ -243,13 +271,14 @@ config("libdawn_native_internal") { } } -dawn_generator("libdawn_native_utils") { +dawn_generator("libdawn_native_utils_gen") { target = "dawn_native_utils" - target_type = "source_set" - - configs = [ ":libdawn_native_internal" ] - deps = [ - ":dawn_headers", + outputs = [ + "dawn_native/ProcTable.cpp", + "dawn_native/dawn_structs_autogen.h", + "dawn_native/dawn_structs_autogen.cpp", + "dawn_native/ValidationUtils_autogen.h", + "dawn_native/ValidationUtils_autogen.cpp", ] } @@ -278,7 +307,7 @@ source_set("libdawn_native_headers") { source_set("libdawn_native_sources") { deps = [ ":dawn_common", - ":libdawn_native_utils", + ":libdawn_native_utils_gen", "third_party:spirv_cross", ] @@ -292,7 +321,8 @@ source_set("libdawn_native_sources") { libs = [] - sources = [ + sources = get_target_outputs(":libdawn_native_utils_gen") + sources += [ "src/dawn_native/BindGroup.cpp", "src/dawn_native/BindGroup.h", "src/dawn_native/BindGroupLayout.cpp", @@ -591,28 +621,28 @@ source_set("libdawn_wire_headers") { ] } -# The meat of the compilation for libdawn_wire so that we can cheaply have -# shared_library / static_library / component versions of it. -dawn_generator("libdawn_wire_sources") { +dawn_generator("libdawn_wire_gen") { target = "dawn_wire" - target_type = "source_set" - - configs = [ ":dawn_internal" ] - deps = [ - ":dawn_common", - ":libdawn_wire_headers", - ] - defines = [ "DAWN_WIRE_IMPLEMENTATION" ] - sources = [ - "src/dawn_wire/WireCmd.h", + outputs = [ + "dawn_wire/WireServer.cpp", + "dawn_wire/WireCmd_autogen.h", + "dawn_wire/WireClient.cpp", + "dawn_wire/WireCmd_autogen.cpp", ] } shared_library("libdawn_wire") { deps = [ - ":libdawn_wire_sources", + ":dawn_common", + ":libdawn_wire_gen", + ":libdawn_wire_headers", ] + configs += [ ":dawn_internal" ] + defines = [ "DAWN_WIRE_IMPLEMENTATION" ] + sources = get_target_outputs(":libdawn_wire_gen") + sources += [ "src/dawn_wire/WireCmd.h" ] + #Make headers publically visible public_deps = [ ":libdawn_wire_headers", @@ -646,7 +676,6 @@ static_library("dawn_utils") { "src/utils/TerribleCommandBuffer.cpp", "src/utils/TerribleCommandBuffer.h", ] - libs = [] deps = [ ":dawn_common", ":libdawn_native", @@ -654,6 +683,7 @@ static_library("dawn_utils") { "third_party:glfw", "third_party:libshaderc", ] + libs = [] if (dawn_enable_d3d12) { sources += [ "src/utils/D3D12Binding.cpp" ] @@ -691,14 +721,11 @@ static_library("dawn_utils") { # Dawn test targets ############################################################################### -dawn_generator("mock_dawn") { - testonly = true +dawn_generator("mock_dawn_gen") { target = "mock_dawn" - target_type = "source_set" - - deps = [ - ":dawn_headers", - "third_party:gmock", + outputs = [ + "mock/mock_dawn.h", + "mock/mock_dawn.cpp", ] } @@ -711,16 +738,18 @@ test("dawn_unittests") { deps = [ ":dawn_common", + ":dawn_headers", ":dawn_utils", ":libdawn", ":libdawn_native_sources", ":libdawn_wire", - ":mock_dawn", + ":mock_dawn_gen", "third_party:gmock", "third_party:gtest", ] - sources = [ + sources = get_target_outputs(":mock_dawn_gen") + sources += [ "src/tests/UnittestsMain.cpp", "src/tests/unittests/BitSetIteratorTests.cpp", "src/tests/unittests/CommandAllocatorTests.cpp", diff --git a/generator/extract_json.py b/generator/extract_json.py new file mode 100644 index 0000000000..2cc4b3d6f6 --- /dev/null +++ b/generator/extract_json.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python2 +# Copyright 2018 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os, sys, json + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: extract_json.py JSON DIR") + sys.exit(1) + + with open(sys.argv[1]) as f: + files = json.loads(f.read()) + + output_dir = sys.argv[2] + + for (name, content) in files.iteritems(): + output_file = output_dir + os.path.sep + name + + directory = os.path.dirname(output_file) + if not os.path.exists(directory): + os.makedirs(directory) + + with open(output_file, 'w') as outfile: + outfile.write(content) diff --git a/generator/main.py b/generator/main.py index 93805ba35e..5935e486e7 100644 --- a/generator/main.py +++ b/generator/main.py @@ -280,21 +280,20 @@ class PreprocessingLoader(jinja2.BaseLoader): FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts']) -def do_renders(renders, template_dir, output_dir): +FileOutput = namedtuple('FileOutput', ['name', 'content']) + +def do_renders(renders, template_dir): env = jinja2.Environment(loader=PreprocessingLoader(template_dir), trim_blocks=True, line_comment_prefix='//*') + + outputs = [] for render in renders: params = {} for param_dict in render.params_dicts: params.update(param_dict) - output = env.get_template(render.template).render(**params) + content = env.get_template(render.template).render(**params) + outputs.append(FileOutput(render.output, content)) - output_file = output_dir + os.path.sep + render.output - directory = os.path.dirname(output_file) - if not os.path.exists(directory): - os.makedirs(directory) - - with open(output_file, 'w') as outfile: - outfile.write(output) + return outputs ############################################################# # MAIN SOMETHING WHATEVER @@ -389,37 +388,7 @@ def js_native_methods(types, typ): def debug(text): print(text) -def main(): - targets = ['dawn_headers', 'libdawn', 'mock_dawn', 'dawn_wire', "dawn_native_utils"] - - parser = argparse.ArgumentParser( - description = 'Generates code for various target for Dawn.', - formatter_class = argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument('json', metavar='DAWN_JSON', nargs=1, type=str, help ='The DAWN JSON definition to use.') - parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.') - parser.add_argument('-o', '--output-dir', default=None, type=str, help='Output directory for the generated source files.') - parser.add_argument('-T', '--targets', default=None, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(targets)) - parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2') - parser.add_argument('--print-dependencies', action='store_true', help='Prints a space separated list of file dependencies, used for CMake integration') - parser.add_argument('--print-outputs', action='store_true', help='Prints a space separated list of file outputs, used for CMake integration') - parser.add_argument('--gn', action='store_true', help='Make the printing of dependencies/outputs GN-friendly') - - args = parser.parse_args() - - if args.targets != None: - targets = args.targets.split(',') - - with open(args.json[0]) as f: - loaded_json = json.loads(f.read()) - - # A fake api_params to avoid parsing the JSON when just querying dependencies and outputs - api_params = { - 'types': {} - } - if not args.print_outputs and not args.print_dependencies: - api_params = parse_json(loaded_json) - +def get_renders_for_targets(api_params, targets): base_params = { 'enumerate': enumerate, 'format': format, @@ -491,7 +460,61 @@ def main(): renders.append(FileRender('dawn_wire/WireClient.cpp', 'dawn_wire/WireClient.cpp', wire_params)) renders.append(FileRender('dawn_wire/WireServer.cpp', 'dawn_wire/WireServer.cpp', wire_params)) - output_separator = '\n' if args.gn else ';' + return renders + +def output_to_files(outputs, output_dir): + for output in outputs: + output_file = output_dir + os.path.sep + output.name + directory = os.path.dirname(output_file) + if not os.path.exists(directory): + os.makedirs(directory) + + with open(output_file, 'w') as outfile: + outfile.write(output.content) + +def output_to_json(outputs, output_json): + json_root = {} + for output in outputs: + json_root[output.name] = output.content + + with open(output_json, 'w') as f: + f.write(json.dumps(json_root)) + +def output_depfile(depfile, output, dependencies): + with open(depfile, 'w') as f: + f.write(output + ": " + " ".join(dependencies)) + +def main(): + allowed_targets = ['dawn_headers', 'libdawn', 'mock_dawn', 'dawn_wire', "dawn_native_utils"] + + parser = argparse.ArgumentParser( + description = 'Generates code for various target for Dawn.', + formatter_class = argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument('json', metavar='DAWN_JSON', nargs=1, type=str, help ='The DAWN JSON definition to use.') + parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.') + parser.add_argument('-T', '--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets)) + # Arguments used only for the GN build + parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2') + parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).') + parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball') + parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match") + # Arguments used only for the CMake build + parser.add_argument('-o', '--output-dir', default=None, type=str, help='Output directory for the generated source files.') + parser.add_argument('--print-dependencies', action='store_true', help='Prints a space separated list of file dependencies, used for CMake integration') + parser.add_argument('--print-outputs', action='store_true', help='Prints a space separated list of file outputs, used for CMake integration') + + args = parser.parse_args() + + # Load and parse the API json file + with open(args.json[0]) as f: + loaded_json = json.loads(f.read()) + api_params = parse_json(loaded_json) + + targets = args.targets.split(',') + renders = get_renders_for_targets(api_params, targets) + + # Print outputs and dependencies for CMake if args.print_dependencies: dependencies = set( [os.path.abspath(args.template_dir + os.path.sep + render.template) for render in renders] + @@ -499,7 +522,7 @@ def main(): [os.path.realpath(__file__)] ) dependencies = [dependency.replace('\\', '/') for dependency in dependencies] - sys.stdout.write(output_separator.join(dependencies)) + sys.stdout.write(';'.join(dependencies)) return 0 if args.print_outputs: @@ -507,10 +530,36 @@ def main(): [os.path.abspath(args.output_dir + os.path.sep + render.output) for render in renders] ) outputs = [output.replace('\\', '/') for output in outputs] - sys.stdout.write(output_separator.join(outputs)) + sys.stdout.write(';'.join(outputs)) return 0 - do_renders(renders, args.template_dir, args.output_dir) + # The caller wants to assert that the outputs are what it expects. + # Load the file and compare with our renders. GN-only. + if args.expected_outputs_file != None: + with open(args.expected_outputs_file) as f: + expected = set([line.strip() for line in f.readlines()]) + + actual = set() + actual.update([render.output for render in renders]) + + if actual != expected: + print("Wrong expected outputs, caller expected:\n " + repr(list(expected))) + print("Actual output:\n " + repr(list(actual))) + return 1 + + outputs = do_renders(renders, args.template_dir) + + # CMake only: output all the files directly. + if args.output_dir != None: + output_to_files(outputs, args.output_dir) + + # GN only: output the tarball and its depfile + if args.output_json_tarball != None: + output_to_json(outputs, args.output_json_tarball) + + dependencies = [args.template_dir + os.path.sep + render.template for render in renders] + dependencies.append(args.json[0]) + output_depfile(args.depfile, args.output_json_tarball, dependencies) if __name__ == '__main__': sys.exit(main())