[generator_lib]: Make generator_lib.py easier to reuse.

This CL tries to make generator_lib.py easier to reuse in
other projects (e.g. non Dawn-related), by doing the following:

- Removing dawn-specific variables from the script and
  replacing them by path-relative defaults, or through
  additional command-line parameters (e.g. --root-dir can
  now be used to pass the root source directory for Python
  dependency computations).

- Move project-agnostic processing from dawn_generator
  GN template into a new generator_lib:generator_lib_action
  template. The new generator_lib.gni file does not
  contain Dawn-specific settings and can be reused more
  easily outside of Dawn.

+ Replace --extra-python-path with --jinja2-path to be
  more explicit.

+ Add a few documentation comments in the Python scripts.

R=cwallez@chromium.org

Bug: NONE
Change-Id: I3e89f4bc32bdb6a019d251473222c6ce5cdc5f9f
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/8280
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>

generator/dawn_generator.gni

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import("../scripts/dawn_overrides_with_defaults.gni")
+import("generator_lib.gni")
 
 # Template to help invoking Dawn code generators based on generator_lib
 #
@@ -42,94 +43,10 @@ import("../scripts/dawn_overrides_with_defaults.gni")
 #   }
 #
 template("dawn_generator") {
-  generator_args = []
+  generator_lib_action(target_name) {
-  if (defined(invoker.args)) {
+    forward_variables_from(invoker, "*")
-    generator_args += invoker.args
+    generator_lib_dir = "${dawn_root}/generator"
-  }
+    jinja2_path = dawn_jinja2_dir
-
-  generator_args += [
-    "--template-dir",
-    rebase_path("${dawn_root}/generator/templates", root_build_dir),
-  ]
-
-  # Use the Jinja2 version pulled from the DEPS file. We do it so we don't
-  # have version problems, and users don't have to install Jinja2.
-  jinja2_python_path = rebase_path("${dawn_jinja2_dir}/..")
-  generator_args += [
-    "--extra-python-path",
-    jinja2_python_path,
-  ]
-
-  # Chooses either the default gen_dir or the custom one required by the
-  # invoker. This allows moving the definition of code generators in different
-  # BUILD.gn files without changing the location of generated file. Without
-  # this generated headers could cause issues when old headers aren't removed.
-  gen_dir = target_gen_dir
-  if (defined(invoker.custom_gen_dir)) {
-    gen_dir = invoker.custom_gen_dir
-  }
-
-  # For build parallelism GN wants to know the exact inputs and outputs of
-  # action targets like we use for our code generator. We avoid asking the
-  # generator about its inputs by using the "depfile" feature of GN/Ninja.
-  #
-  # A ninja limitation is that the depfile is a subset of Makefile that can
-  # contain a single target, so we output a single "JSON-tarball" instead.
-  json_tarball = "${gen_dir}/${target_name}.json_tarball"
-  json_tarball_depfile = "${json_tarball}.d"
-
-  generator_args += [
-    "--output-json-tarball",
-    rebase_path(json_tarball, root_build_dir),
-    "--depfile",
-    rebase_path(json_tarball_depfile, root_build_dir),
-  ]
-
-  # After the JSON tarball is created we need an action target to extract it
-  # with a list of its outputs. The invoker provided a list of expected
-  # outputs. To make sure the list is in sync between the generator and the
-  # build files, we write it to a file and ask the generator to assert it is
-  # correct.
-  expected_outputs_file = "${gen_dir}/${target_name}.expected_outputs"
-  write_file(expected_outputs_file, invoker.outputs)
-
-  generator_args += [
-    "--expected-outputs-file",
-    rebase_path(expected_outputs_file, root_build_dir),
-  ]
-
-  # The code generator invocation that will write the JSON tarball, check the
-  # outputs are what's expected and write a depfile for Ninja.
-  action("${target_name}_json_tarball") {
-    script = invoker.script
-    outputs = [
-      json_tarball,
-    ]
-    depfile = json_tarball_depfile
-    args = generator_args
-  }
-
-  # Extract the JSON tarball into the gen_dir
-  action(target_name) {
-    script = "${dawn_root}/generator/extract_json.py"
-    args = [
-      rebase_path(json_tarball, root_build_dir),
-      rebase_path(gen_dir, root_build_dir),
-    ]
-
-    deps = [
-      ":${target_name}_json_tarball",
-    ]
-    inputs = [
-      json_tarball,
-    ]
-
-    # The expected output list is relative to the gen_dir but action
-    # target outputs are from the root dir so we need to rebase them.
-    outputs = []
-    foreach(source, invoker.outputs) {
-      outputs += [ "${gen_dir}/${source}" ]
-    }
   }
 }
 

generator/generator_lib.gni

@@ -0,0 +1,147 @@
+# Copyright 2019 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Template to help invoking code generators based on generator_lib.py
+# Internal use only, this should only be called from templates implementing
+# generator-specific actions.
+#
+# Variables:
+#   script: Path to generator script.
+#
+#   args: List of extra command-line arguments passed to the generator.
+#
+#   outputs: List of expected outputs, generation will fail if there is a
+#     mistmatch.
+#
+#   generator_lib_dir: directory where generator_lib.py is located.
+#
+#   custom_gen_dir: Optional custom target gen dir. Defaults to $target_gen_dir
+#     but allows output files to not depend on the location of the BUILD.gn
+#     that generates them.
+#
+#   template_dir: Optional template root directory. Defaults to
+#     "${generator_lib_dir}/templates".
+#
+#   jinja2_path: Optional Jinja2 installation path.
+#
+#   root_dir: Optional root source dir for Python dependencies
+#     computation. Defaults to "${generator_lib_dir}/..". Any dependency
+#     outside of this directory is considered a system file and will be
+#     omitted.
+#
+template("generator_lib_action") {
+  _generator_args = []
+  if (defined(invoker.args)) {
+    _generator_args += invoker.args
+  }
+
+  assert(defined(invoker.generator_lib_dir),
+         "generator_lib_dir must be defined before calling this action!")
+
+  _template_dir = "${invoker.generator_lib_dir}/templates"
+  if (defined(invoker.template_dir)) {
+    _template_dir = invoker.template_dir
+  }
+  _generator_args += [
+    "--template-dir",
+    rebase_path(_template_dir),
+  ]
+
+  if (defined(invoker.root_dir)) {
+    _generator_args += [
+      "--root-dir",
+      rebase_path(_root_dir, root_build_dir),
+    ]
+  }
+
+  if (defined(invoker.jinja2_path)) {
+    _generator_args += [
+      "--jinja2-path",
+      rebase_path(invoker.jinja2_path),
+    ]
+  }
+
+  # Chooses either the default gen_dir or the custom one required by the
+  # invoker. This allows moving the definition of code generators in different
+  # BUILD.gn files without changing the location of generated file. Without
+  # this generated headers could cause issues when old headers aren't removed.
+  _gen_dir = target_gen_dir
+  if (defined(invoker.custom_gen_dir)) {
+    _gen_dir = invoker.custom_gen_dir
+  }
+
+  # For build parallelism GN wants to know the exact inputs and outputs of
+  # action targets like we use for our code generator. We avoid asking the
+  # generator about its inputs by using the "depfile" feature of GN/Ninja.
+  #
+  # A ninja limitation is that the depfile is a subset of Makefile that can
+  # contain a single target, so we output a single "JSON-tarball" instead.
+  _json_tarball = "${_gen_dir}/${target_name}.json_tarball"
+  _json_tarball_target = "${target_name}__json_tarball"
+  _json_tarball_depfile = "${_json_tarball}.d"
+
+  _generator_args += [
+    "--output-json-tarball",
+    rebase_path(_json_tarball, root_build_dir),
+    "--depfile",
+    rebase_path(_json_tarball_depfile, root_build_dir),
+  ]
+
+  # After the JSON tarball is created we need an action target to extract it
+  # with a list of its outputs. The invoker provided a list of expected
+  # outputs. To make sure the list is in sync between the generator and the
+  # build files, we write it to a file and ask the generator to assert it is
+  # correct.
+  _expected_outputs_file = "${_gen_dir}/${target_name}.expected_outputs"
+  write_file(_expected_outputs_file, invoker.outputs)
+
+  _generator_args += [
+    "--expected-outputs-file",
+    rebase_path(_expected_outputs_file, root_build_dir),
+  ]
+
+  # The code generator invocation that will write the JSON tarball, check the
+  # outputs are what's expected and write a depfile for Ninja.
+  action(_json_tarball_target) {
+    script = invoker.script
+    outputs = [
+      _json_tarball,
+    ]
+    depfile = _json_tarball_depfile
+    args = _generator_args
+  }
+
+  # Extract the JSON tarball into the gen_dir
+  action(target_name) {
+    script = "${invoker.generator_lib_dir}/extract_json.py"
+    args = [
+      rebase_path(_json_tarball, root_build_dir),
+      rebase_path(_gen_dir, root_build_dir),
+    ]
+
+    deps = [
+      ":${_json_tarball_target}",
+    ]
+    inputs = [
+      _json_tarball,
+    ]
+
+    # The expected output list is relative to the gen_dir but action
+    # target outputs are from the root dir so we need to rebase them.
+    outputs = []
+    foreach(source, invoker.outputs) {
+      outputs += [ "${_gen_dir}/${source}" ]
+    }
+  }
+}

generator/generator_lib.py

@@ -13,31 +13,77 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Module to create generators that render multiple Jinja2 templates for GN.
+
+A helper module that can be used to create generator scripts (clients)
+that expand one or more Jinja2 templates, without outputs usable from
+GN and Ninja build-based systems. See generator_lib.gni as well.
+
+Clients should create a Generator sub-class, then call run_generator()
+with a proper derived class instance.
+
+Clients specify a list of FileRender operations, each one of them will
+output a file into a temporary output directory through Jinja2 expansion.
+All temporary output files are then grouped and written to into a single JSON
+file, that acts as a convenient single GN output target. Use extract_json.py
+to extract the output files from the JSON tarball in another GN action.
+
+--depfile can be used to specify an output Ninja dependency file for the
+JSON tarball, to ensure it is regenerated any time one of its dependencies
+changes.
+
+Finally, --expected-output-files can be used to check the list of generated
+output files.
+"""
+
 import argparse, json, os, re, sys
 from collections import namedtuple
 
+# A FileRender represents a single Jinja2 template render operation:
+#
+#   template: Jinja2 template name, relative to --template-dir path.
+#
+#   output: Output file path, relative to temporary output directory.
+#
+#   params_dicts: iterable of (name:string -> value:string) dictionaries.
+#       All of them will be merged before being sent as Jinja2 template
+#       expansion parameters.
+#
+# Example:
+#   FileRender('api.c', 'src/project_api.c', [{'PROJECT_VERSION': '1.0.0'}])
+#
+FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
+
 # The interface that must be implemented by generators.
 class Generator:
     def get_description(self):
+        """Return generator description for --help."""
         return ""
 
     def add_commandline_arguments(self, parser):
+        """Add generator-specific argparse arguments."""
         pass
 
     def get_file_renders(self, args):
+        """Return the list of FileRender objects to process."""
         return []
 
     def get_dependencies(self, args):
+        """Return a list of extra input dependencies."""
         return []
 
-FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
+# Allow custom Jinja2 installation path through an additional python
-
+# path from the arguments if present. This isn't done through the regular
-# Try using an additional python path from the arguments if present. This
+# argparse because PreprocessingLoader uses jinja2 in the global scope before
-# isn't done through the regular argparse because PreprocessingLoader uses
+# "main" gets to run.
-# jinja2 in the global scope before "main" gets to run.
+#
-kExtraPythonPath = '--extra-python-path'
+# NOTE: If this argument appears several times, this only uses the first
-if kExtraPythonPath in sys.argv:
+#       value, while argparse would typically keep the last one!
-    path = sys.argv[sys.argv.index(kExtraPythonPath) + 1]
+kJinja2Path = '--jinja2-path'
+jinja2_path_argv_index = sys.argv.index(kJinja2Path)
+if jinja2_path_argv_index >= 0:
+    # Add parent path for the import to succeed.
+    path = os.path.join(sys.argv[jinja2_path_argv_index + 1], os.pardir)
     sys.path.insert(1, path)
 
 import jinja2
@@ -123,9 +169,12 @@ def _do_renders(renders, template_dir):
     return outputs
 
 # Compute the list of imported, non-system Python modules.
-# It assumes that any path outside of Dawn's root directory is system.
+# It assumes that any path outside of the root directory is system.
-def _compute_python_dependencies():
+def _compute_python_dependencies(root_dir = None):
-    dawn_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+    if not root_dir:
+        # Assume this script is under generator/ by default.
+        root_dir = os.path.join(os.path.dirname(__file__), os.pardir)
+    root_dir = os.path.abspath(root_dir)
 
     module_paths = (module.__file__ for module in sys.modules.values()
                                     if module and hasattr(module, '__file__'))
@@ -134,7 +183,7 @@ def _compute_python_dependencies():
     for path in module_paths:
         path = os.path.abspath(path)
 
-        if not path.startswith(dawn_root):
+        if not path.startswith(root_dir):
             continue
 
         if (path.endswith('.pyc')
@@ -153,10 +202,11 @@ def run_generator(generator):
 
     generator.add_commandline_arguments(parser);
     parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.')
-    parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2')
+    parser.add_argument(kJinja2Path, default=None, type=str, help='Additional python path to set before loading Jinja2')
     parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).')
     parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball')
     parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match")
+    parser.add_argument('--root-dir', default=None, type=str, help='Optional source root directory for Python dependency computations')
 
     args = parser.parse_args()
 
@@ -168,20 +218,13 @@ def run_generator(generator):
         with open(args.expected_outputs_file) as f:
             expected = set([line.strip() for line in f.readlines()])
 
-        actual = set()
+        actual = {render.output for render in renders}
-        actual.update([render.output for render in renders])
 
         if actual != expected:
-            print("Wrong expected outputs, caller expected:\n    " + repr(list(expected)))
+            print("Wrong expected outputs, caller expected:\n    " + repr(sorted(expected)))
-            print("Actual output:\n    " + repr(list(actual)))
+            print("Actual output:\n    " + repr(sorted(actual)))
             return 1
 
-    # Add a any extra Python path before importing Jinja2 so invokers can point
-    # to a checkout of Jinja2 and note require it to be installed on the system
-    if args.extra_python_path != None:
-        sys.path.insert(1, args.extra_python_path)
-    import jinja2
-
     outputs = _do_renders(renders, args.template_dir)
 
     # Output the tarball and its depfile
@@ -197,7 +240,7 @@ def run_generator(generator):
     if args.depfile != None:
         dependencies = generator.get_dependencies(args)
         dependencies += [args.template_dir + os.path.sep + render.template for render in renders]
-        dependencies += _compute_python_dependencies()
+        dependencies += _compute_python_dependencies(args.root_dir)
 
         with open(args.depfile, 'w') as f:
             f.write(args.output_json_tarball + ": " + " ".join(dependencies))