diff --git a/generator/dawn_generator.gni b/generator/dawn_generator.gni index 19b2e23dcf..3ab81e7b20 100644 --- a/generator/dawn_generator.gni +++ b/generator/dawn_generator.gni @@ -13,6 +13,7 @@ # limitations under the License. import("../scripts/dawn_overrides_with_defaults.gni") +import("generator_lib.gni") # Template to help invoking Dawn code generators based on generator_lib # @@ -42,94 +43,10 @@ import("../scripts/dawn_overrides_with_defaults.gni") # } # template("dawn_generator") { - generator_args = [] - if (defined(invoker.args)) { - generator_args += invoker.args - } - - generator_args += [ - "--template-dir", - rebase_path("${dawn_root}/generator/templates", root_build_dir), - ] - - # Use the Jinja2 version pulled from the DEPS file. We do it so we don't - # have version problems, and users don't have to install Jinja2. - jinja2_python_path = rebase_path("${dawn_jinja2_dir}/..") - generator_args += [ - "--extra-python-path", - jinja2_python_path, - ] - - # Chooses either the default gen_dir or the custom one required by the - # invoker. This allows moving the definition of code generators in different - # BUILD.gn files without changing the location of generated file. Without - # this generated headers could cause issues when old headers aren't removed. - gen_dir = target_gen_dir - if (defined(invoker.custom_gen_dir)) { - gen_dir = invoker.custom_gen_dir - } - - # For build parallelism GN wants to know the exact inputs and outputs of - # action targets like we use for our code generator. We avoid asking the - # generator about its inputs by using the "depfile" feature of GN/Ninja. - # - # A ninja limitation is that the depfile is a subset of Makefile that can - # contain a single target, so we output a single "JSON-tarball" instead. - json_tarball = "${gen_dir}/${target_name}.json_tarball" - json_tarball_depfile = "${json_tarball}.d" - - generator_args += [ - "--output-json-tarball", - rebase_path(json_tarball, root_build_dir), - "--depfile", - rebase_path(json_tarball_depfile, root_build_dir), - ] - - # After the JSON tarball is created we need an action target to extract it - # with a list of its outputs. The invoker provided a list of expected - # outputs. To make sure the list is in sync between the generator and the - # build files, we write it to a file and ask the generator to assert it is - # correct. - expected_outputs_file = "${gen_dir}/${target_name}.expected_outputs" - write_file(expected_outputs_file, invoker.outputs) - - generator_args += [ - "--expected-outputs-file", - rebase_path(expected_outputs_file, root_build_dir), - ] - - # The code generator invocation that will write the JSON tarball, check the - # outputs are what's expected and write a depfile for Ninja. - action("${target_name}_json_tarball") { - script = invoker.script - outputs = [ - json_tarball, - ] - depfile = json_tarball_depfile - args = generator_args - } - - # Extract the JSON tarball into the gen_dir - action(target_name) { - script = "${dawn_root}/generator/extract_json.py" - args = [ - rebase_path(json_tarball, root_build_dir), - rebase_path(gen_dir, root_build_dir), - ] - - deps = [ - ":${target_name}_json_tarball", - ] - inputs = [ - json_tarball, - ] - - # The expected output list is relative to the gen_dir but action - # target outputs are from the root dir so we need to rebase them. - outputs = [] - foreach(source, invoker.outputs) { - outputs += [ "${gen_dir}/${source}" ] - } + generator_lib_action(target_name) { + forward_variables_from(invoker, "*") + generator_lib_dir = "${dawn_root}/generator" + jinja2_path = dawn_jinja2_dir } } diff --git a/generator/generator_lib.gni b/generator/generator_lib.gni new file mode 100644 index 0000000000..d3698f2d4c --- /dev/null +++ b/generator/generator_lib.gni @@ -0,0 +1,147 @@ +# Copyright 2019 The Dawn Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Template to help invoking code generators based on generator_lib.py +# Internal use only, this should only be called from templates implementing +# generator-specific actions. +# +# Variables: +# script: Path to generator script. +# +# args: List of extra command-line arguments passed to the generator. +# +# outputs: List of expected outputs, generation will fail if there is a +# mistmatch. +# +# generator_lib_dir: directory where generator_lib.py is located. +# +# custom_gen_dir: Optional custom target gen dir. Defaults to $target_gen_dir +# but allows output files to not depend on the location of the BUILD.gn +# that generates them. +# +# template_dir: Optional template root directory. Defaults to +# "${generator_lib_dir}/templates". +# +# jinja2_path: Optional Jinja2 installation path. +# +# root_dir: Optional root source dir for Python dependencies +# computation. Defaults to "${generator_lib_dir}/..". Any dependency +# outside of this directory is considered a system file and will be +# omitted. +# +template("generator_lib_action") { + _generator_args = [] + if (defined(invoker.args)) { + _generator_args += invoker.args + } + + assert(defined(invoker.generator_lib_dir), + "generator_lib_dir must be defined before calling this action!") + + _template_dir = "${invoker.generator_lib_dir}/templates" + if (defined(invoker.template_dir)) { + _template_dir = invoker.template_dir + } + _generator_args += [ + "--template-dir", + rebase_path(_template_dir), + ] + + if (defined(invoker.root_dir)) { + _generator_args += [ + "--root-dir", + rebase_path(_root_dir, root_build_dir), + ] + } + + if (defined(invoker.jinja2_path)) { + _generator_args += [ + "--jinja2-path", + rebase_path(invoker.jinja2_path), + ] + } + + # Chooses either the default gen_dir or the custom one required by the + # invoker. This allows moving the definition of code generators in different + # BUILD.gn files without changing the location of generated file. Without + # this generated headers could cause issues when old headers aren't removed. + _gen_dir = target_gen_dir + if (defined(invoker.custom_gen_dir)) { + _gen_dir = invoker.custom_gen_dir + } + + # For build parallelism GN wants to know the exact inputs and outputs of + # action targets like we use for our code generator. We avoid asking the + # generator about its inputs by using the "depfile" feature of GN/Ninja. + # + # A ninja limitation is that the depfile is a subset of Makefile that can + # contain a single target, so we output a single "JSON-tarball" instead. + _json_tarball = "${_gen_dir}/${target_name}.json_tarball" + _json_tarball_target = "${target_name}__json_tarball" + _json_tarball_depfile = "${_json_tarball}.d" + + _generator_args += [ + "--output-json-tarball", + rebase_path(_json_tarball, root_build_dir), + "--depfile", + rebase_path(_json_tarball_depfile, root_build_dir), + ] + + # After the JSON tarball is created we need an action target to extract it + # with a list of its outputs. The invoker provided a list of expected + # outputs. To make sure the list is in sync between the generator and the + # build files, we write it to a file and ask the generator to assert it is + # correct. + _expected_outputs_file = "${_gen_dir}/${target_name}.expected_outputs" + write_file(_expected_outputs_file, invoker.outputs) + + _generator_args += [ + "--expected-outputs-file", + rebase_path(_expected_outputs_file, root_build_dir), + ] + + # The code generator invocation that will write the JSON tarball, check the + # outputs are what's expected and write a depfile for Ninja. + action(_json_tarball_target) { + script = invoker.script + outputs = [ + _json_tarball, + ] + depfile = _json_tarball_depfile + args = _generator_args + } + + # Extract the JSON tarball into the gen_dir + action(target_name) { + script = "${invoker.generator_lib_dir}/extract_json.py" + args = [ + rebase_path(_json_tarball, root_build_dir), + rebase_path(_gen_dir, root_build_dir), + ] + + deps = [ + ":${_json_tarball_target}", + ] + inputs = [ + _json_tarball, + ] + + # The expected output list is relative to the gen_dir but action + # target outputs are from the root dir so we need to rebase them. + outputs = [] + foreach(source, invoker.outputs) { + outputs += [ "${_gen_dir}/${source}" ] + } + } +} diff --git a/generator/generator_lib.py b/generator/generator_lib.py index a1723ab583..fc355a9076 100644 --- a/generator/generator_lib.py +++ b/generator/generator_lib.py @@ -13,31 +13,77 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Module to create generators that render multiple Jinja2 templates for GN. + +A helper module that can be used to create generator scripts (clients) +that expand one or more Jinja2 templates, without outputs usable from +GN and Ninja build-based systems. See generator_lib.gni as well. + +Clients should create a Generator sub-class, then call run_generator() +with a proper derived class instance. + +Clients specify a list of FileRender operations, each one of them will +output a file into a temporary output directory through Jinja2 expansion. +All temporary output files are then grouped and written to into a single JSON +file, that acts as a convenient single GN output target. Use extract_json.py +to extract the output files from the JSON tarball in another GN action. + +--depfile can be used to specify an output Ninja dependency file for the +JSON tarball, to ensure it is regenerated any time one of its dependencies +changes. + +Finally, --expected-output-files can be used to check the list of generated +output files. +""" + import argparse, json, os, re, sys from collections import namedtuple +# A FileRender represents a single Jinja2 template render operation: +# +# template: Jinja2 template name, relative to --template-dir path. +# +# output: Output file path, relative to temporary output directory. +# +# params_dicts: iterable of (name:string -> value:string) dictionaries. +# All of them will be merged before being sent as Jinja2 template +# expansion parameters. +# +# Example: +# FileRender('api.c', 'src/project_api.c', [{'PROJECT_VERSION': '1.0.0'}]) +# +FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts']) + # The interface that must be implemented by generators. class Generator: def get_description(self): + """Return generator description for --help.""" return "" def add_commandline_arguments(self, parser): + """Add generator-specific argparse arguments.""" pass def get_file_renders(self, args): + """Return the list of FileRender objects to process.""" return [] def get_dependencies(self, args): + """Return a list of extra input dependencies.""" return [] -FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts']) - -# Try using an additional python path from the arguments if present. This -# isn't done through the regular argparse because PreprocessingLoader uses -# jinja2 in the global scope before "main" gets to run. -kExtraPythonPath = '--extra-python-path' -if kExtraPythonPath in sys.argv: - path = sys.argv[sys.argv.index(kExtraPythonPath) + 1] +# Allow custom Jinja2 installation path through an additional python +# path from the arguments if present. This isn't done through the regular +# argparse because PreprocessingLoader uses jinja2 in the global scope before +# "main" gets to run. +# +# NOTE: If this argument appears several times, this only uses the first +# value, while argparse would typically keep the last one! +kJinja2Path = '--jinja2-path' +jinja2_path_argv_index = sys.argv.index(kJinja2Path) +if jinja2_path_argv_index >= 0: + # Add parent path for the import to succeed. + path = os.path.join(sys.argv[jinja2_path_argv_index + 1], os.pardir) sys.path.insert(1, path) import jinja2 @@ -123,9 +169,12 @@ def _do_renders(renders, template_dir): return outputs # Compute the list of imported, non-system Python modules. -# It assumes that any path outside of Dawn's root directory is system. -def _compute_python_dependencies(): - dawn_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) +# It assumes that any path outside of the root directory is system. +def _compute_python_dependencies(root_dir = None): + if not root_dir: + # Assume this script is under generator/ by default. + root_dir = os.path.join(os.path.dirname(__file__), os.pardir) + root_dir = os.path.abspath(root_dir) module_paths = (module.__file__ for module in sys.modules.values() if module and hasattr(module, '__file__')) @@ -134,7 +183,7 @@ def _compute_python_dependencies(): for path in module_paths: path = os.path.abspath(path) - if not path.startswith(dawn_root): + if not path.startswith(root_dir): continue if (path.endswith('.pyc') @@ -153,10 +202,11 @@ def run_generator(generator): generator.add_commandline_arguments(parser); parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.') - parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2') + parser.add_argument(kJinja2Path, default=None, type=str, help='Additional python path to set before loading Jinja2') parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).') parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball') parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match") + parser.add_argument('--root-dir', default=None, type=str, help='Optional source root directory for Python dependency computations') args = parser.parse_args() @@ -168,20 +218,13 @@ def run_generator(generator): with open(args.expected_outputs_file) as f: expected = set([line.strip() for line in f.readlines()]) - actual = set() - actual.update([render.output for render in renders]) + actual = {render.output for render in renders} if actual != expected: - print("Wrong expected outputs, caller expected:\n " + repr(list(expected))) - print("Actual output:\n " + repr(list(actual))) + print("Wrong expected outputs, caller expected:\n " + repr(sorted(expected))) + print("Actual output:\n " + repr(sorted(actual))) return 1 - # Add a any extra Python path before importing Jinja2 so invokers can point - # to a checkout of Jinja2 and note require it to be installed on the system - if args.extra_python_path != None: - sys.path.insert(1, args.extra_python_path) - import jinja2 - outputs = _do_renders(renders, args.template_dir) # Output the tarball and its depfile @@ -197,7 +240,7 @@ def run_generator(generator): if args.depfile != None: dependencies = generator.get_dependencies(args) dependencies += [args.template_dir + os.path.sep + render.template for render in renders] - dependencies += _compute_python_dependencies() + dependencies += _compute_python_dependencies(args.root_dir) with open(args.depfile, 'w') as f: f.write(args.output_json_tarball + ": " + " ".join(dependencies))