encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[generator_lib]: Make generator_lib.py easier to reuse. 

This CL tries to make generator_lib.py easier to reuse in
other projects (e.g. non Dawn-related), by doing the following:

- Removing dawn-specific variables from the script and
  replacing them by path-relative defaults, or through
  additional command-line parameters (e.g. --root-dir can
  now be used to pass the root source directory for Python
  dependency computations).

- Move project-agnostic processing from dawn_generator
  GN template into a new generator_lib:generator_lib_action
  template. The new generator_lib.gni file does not
  contain Dawn-specific settings and can be reused more
  easily outside of Dawn.

+ Replace --extra-python-path with --jinja2-path to be
  more explicit.

+ Add a few documentation comments in the Python scripts.

R=cwallez@chromium.org

Bug: NONE
Change-Id: I3e89f4bc32bdb6a019d251473222c6ce5cdc5f9f
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/8280
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
This commit is contained in:
David 'Digit' Turner 2019-06-24 14:31:06 +00:00 committed by Commit Bot service account
parent 4ec2c1060e
commit 5dee3e826b
3 changed files with 219 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
generator/dawn_generator.gni
View File

@ -13,6 +13,7 @@
# limitations under the License.
import("../scripts/dawn_overrides_with_defaults.gni")
import("generator_lib.gni")
# Template to help invoking Dawn code generators based on generator_lib
#
@ -42,94 +43,10 @@ import("../scripts/dawn_overrides_with_defaults.gni")
# }
#
template("dawn_generator") {
generator_args = []
if (defined(invoker.args)) {
generator_args += invoker.args
}
generator_args += [
"--template-dir",
rebase_path("${dawn_root}/generator/templates", root_build_dir),
]
# Use the Jinja2 version pulled from the DEPS file. We do it so we don't
# have version problems, and users don't have to install Jinja2.
jinja2_python_path = rebase_path("${dawn_jinja2_dir}/..")
generator_args += [
"--extra-python-path",
jinja2_python_path,
]
# Chooses either the default gen_dir or the custom one required by the
# invoker. This allows moving the definition of code generators in different
# BUILD.gn files without changing the location of generated file. Without
# this generated headers could cause issues when old headers aren't removed.
gen_dir = target_gen_dir
if (defined(invoker.custom_gen_dir)) {
gen_dir = invoker.custom_gen_dir
}
# For build parallelism GN wants to know the exact inputs and outputs of
# action targets like we use for our code generator. We avoid asking the
# generator about its inputs by using the "depfile" feature of GN/Ninja.
#
# A ninja limitation is that the depfile is a subset of Makefile that can
# contain a single target, so we output a single "JSON-tarball" instead.
json_tarball = "${gen_dir}/${target_name}.json_tarball"
json_tarball_depfile = "${json_tarball}.d"
generator_args += [
"--output-json-tarball",
rebase_path(json_tarball, root_build_dir),
"--depfile",
rebase_path(json_tarball_depfile, root_build_dir),
]
# After the JSON tarball is created we need an action target to extract it
# with a list of its outputs. The invoker provided a list of expected
# outputs. To make sure the list is in sync between the generator and the
# build files, we write it to a file and ask the generator to assert it is
# correct.
expected_outputs_file = "${gen_dir}/${target_name}.expected_outputs"
write_file(expected_outputs_file, invoker.outputs)
generator_args += [
"--expected-outputs-file",
rebase_path(expected_outputs_file, root_build_dir),
]
# The code generator invocation that will write the JSON tarball, check the
# outputs are what's expected and write a depfile for Ninja.
action("${target_name}_json_tarball") {
script = invoker.script
outputs = [
json_tarball,
]
depfile = json_tarball_depfile
args = generator_args
}
# Extract the JSON tarball into the gen_dir
action(target_name) {
script = "${dawn_root}/generator/extract_json.py"
args = [
rebase_path(json_tarball, root_build_dir),
rebase_path(gen_dir, root_build_dir),
]
deps = [
":${target_name}_json_tarball",
]
inputs = [
json_tarball,
]
# The expected output list is relative to the gen_dir but action
# target outputs are from the root dir so we need to rebase them.
outputs = []
foreach(source, invoker.outputs) {
outputs += [ "${gen_dir}/${source}" ]
}
generator_lib_action(target_name) {
forward_variables_from(invoker, "*")
generator_lib_dir = "${dawn_root}/generator"
jinja2_path = dawn_jinja2_dir
}
}

147
generator/generator_lib.gni Normal file
View File

@ -0,0 +1,147 @@
# Copyright 2019 The Dawn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Template to help invoking code generators based on generator_lib.py
# Internal use only, this should only be called from templates implementing
# generator-specific actions.
#
# Variables:
# script: Path to generator script.
#
# args: List of extra command-line arguments passed to the generator.
#
# outputs: List of expected outputs, generation will fail if there is a
# mistmatch.
#
# generator_lib_dir: directory where generator_lib.py is located.
#
# custom_gen_dir: Optional custom target gen dir. Defaults to $target_gen_dir
# but allows output files to not depend on the location of the BUILD.gn
# that generates them.
#
# template_dir: Optional template root directory. Defaults to
# "${generator_lib_dir}/templates".
#
# jinja2_path: Optional Jinja2 installation path.
#
# root_dir: Optional root source dir for Python dependencies
# computation. Defaults to "${generator_lib_dir}/..". Any dependency
# outside of this directory is considered a system file and will be
# omitted.
#
template("generator_lib_action") {
_generator_args = []
if (defined(invoker.args)) {
_generator_args += invoker.args
}
assert(defined(invoker.generator_lib_dir),
"generator_lib_dir must be defined before calling this action!")
_template_dir = "${invoker.generator_lib_dir}/templates"
if (defined(invoker.template_dir)) {
_template_dir = invoker.template_dir
}
_generator_args += [
"--template-dir",
rebase_path(_template_dir),
]
if (defined(invoker.root_dir)) {
_generator_args += [
"--root-dir",
rebase_path(_root_dir, root_build_dir),
]
}
if (defined(invoker.jinja2_path)) {
_generator_args += [
"--jinja2-path",
rebase_path(invoker.jinja2_path),
]
}
# Chooses either the default gen_dir or the custom one required by the
# invoker. This allows moving the definition of code generators in different
# BUILD.gn files without changing the location of generated file. Without
# this generated headers could cause issues when old headers aren't removed.
_gen_dir = target_gen_dir
if (defined(invoker.custom_gen_dir)) {
_gen_dir = invoker.custom_gen_dir
}
# For build parallelism GN wants to know the exact inputs and outputs of
# action targets like we use for our code generator. We avoid asking the
# generator about its inputs by using the "depfile" feature of GN/Ninja.
#
# A ninja limitation is that the depfile is a subset of Makefile that can
# contain a single target, so we output a single "JSON-tarball" instead.
_json_tarball = "${_gen_dir}/${target_name}.json_tarball"
_json_tarball_target = "${target_name}__json_tarball"
_json_tarball_depfile = "${_json_tarball}.d"
_generator_args += [
"--output-json-tarball",
rebase_path(_json_tarball, root_build_dir),
"--depfile",
rebase_path(_json_tarball_depfile, root_build_dir),
]
# After the JSON tarball is created we need an action target to extract it
# with a list of its outputs. The invoker provided a list of expected
# outputs. To make sure the list is in sync between the generator and the
# build files, we write it to a file and ask the generator to assert it is
# correct.
_expected_outputs_file = "${_gen_dir}/${target_name}.expected_outputs"
write_file(_expected_outputs_file, invoker.outputs)
_generator_args += [
"--expected-outputs-file",
rebase_path(_expected_outputs_file, root_build_dir),
]
# The code generator invocation that will write the JSON tarball, check the
# outputs are what's expected and write a depfile for Ninja.
action(_json_tarball_target) {
script = invoker.script
outputs = [
_json_tarball,
]
depfile = _json_tarball_depfile
args = _generator_args
}
# Extract the JSON tarball into the gen_dir
action(target_name) {
script = "${invoker.generator_lib_dir}/extract_json.py"
args = [
rebase_path(_json_tarball, root_build_dir),
rebase_path(_gen_dir, root_build_dir),
]
deps = [
":${_json_tarball_target}",
]
inputs = [
_json_tarball,
]
# The expected output list is relative to the gen_dir but action
# target outputs are from the root dir so we need to rebase them.
outputs = []
foreach(source, invoker.outputs) {
outputs += [ "${_gen_dir}/${source}" ]
}
}
}

91
generator/generator_lib.py
View File

@ -13,31 +13,77 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module to create generators that render multiple Jinja2 templates for GN.
A helper module that can be used to create generator scripts (clients)
that expand one or more Jinja2 templates, without outputs usable from
GN and Ninja build-based systems. See generator_lib.gni as well.
Clients should create a Generator sub-class, then call run_generator()
with a proper derived class instance.
Clients specify a list of FileRender operations, each one of them will
output a file into a temporary output directory through Jinja2 expansion.
All temporary output files are then grouped and written to into a single JSON
file, that acts as a convenient single GN output target. Use extract_json.py
to extract the output files from the JSON tarball in another GN action.
--depfile can be used to specify an output Ninja dependency file for the
JSON tarball, to ensure it is regenerated any time one of its dependencies
changes.
Finally, --expected-output-files can be used to check the list of generated
output files.
"""
import argparse, json, os, re, sys
from collections import namedtuple
# A FileRender represents a single Jinja2 template render operation:
#
# template: Jinja2 template name, relative to --template-dir path.
#
# output: Output file path, relative to temporary output directory.
#
# params_dicts: iterable of (name:string -> value:string) dictionaries.
# All of them will be merged before being sent as Jinja2 template
# expansion parameters.
#
# Example:
# FileRender('api.c', 'src/project_api.c', [{'PROJECT_VERSION': '1.0.0'}])
#
FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
# The interface that must be implemented by generators.
class Generator:
def get_description(self):
"""Return generator description for --help."""
return ""
def add_commandline_arguments(self, parser):
"""Add generator-specific argparse arguments."""
pass
def get_file_renders(self, args):
"""Return the list of FileRender objects to process."""
return []
def get_dependencies(self, args):
"""Return a list of extra input dependencies."""
return []
FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
# Try using an additional python path from the arguments if present. This
# isn't done through the regular argparse because PreprocessingLoader uses
# jinja2 in the global scope before "main" gets to run.
kExtraPythonPath = '--extra-python-path'
if kExtraPythonPath in sys.argv:
path = sys.argv[sys.argv.index(kExtraPythonPath) + 1]
# Allow custom Jinja2 installation path through an additional python
# path from the arguments if present. This isn't done through the regular
# argparse because PreprocessingLoader uses jinja2 in the global scope before
# "main" gets to run.
#
# NOTE: If this argument appears several times, this only uses the first
# value, while argparse would typically keep the last one!
kJinja2Path = '--jinja2-path'
jinja2_path_argv_index = sys.argv.index(kJinja2Path)
if jinja2_path_argv_index >= 0:
# Add parent path for the import to succeed.
path = os.path.join(sys.argv[jinja2_path_argv_index + 1], os.pardir)
sys.path.insert(1, path)
import jinja2
@ -123,9 +169,12 @@ def _do_renders(renders, template_dir):
return outputs
# Compute the list of imported, non-system Python modules.
# It assumes that any path outside of Dawn's root directory is system.
def _compute_python_dependencies():
dawn_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
# It assumes that any path outside of the root directory is system.
def _compute_python_dependencies(root_dir = None):
if not root_dir:
# Assume this script is under generator/ by default.
root_dir = os.path.join(os.path.dirname(__file__), os.pardir)
root_dir = os.path.abspath(root_dir)
module_paths = (module.__file__ for module in sys.modules.values()
if module and hasattr(module, '__file__'))
@ -134,7 +183,7 @@ def _compute_python_dependencies():
for path in module_paths:
path = os.path.abspath(path)
if not path.startswith(dawn_root):
if not path.startswith(root_dir):
continue
if (path.endswith('.pyc')
@ -153,10 +202,11 @@ def run_generator(generator):
generator.add_commandline_arguments(parser);
parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.')
parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2')
parser.add_argument(kJinja2Path, default=None, type=str, help='Additional python path to set before loading Jinja2')
parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).')
parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball')
parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match")
parser.add_argument('--root-dir', default=None, type=str, help='Optional source root directory for Python dependency computations')
args = parser.parse_args()
@ -168,20 +218,13 @@ def run_generator(generator):
with open(args.expected_outputs_file) as f:
expected = set([line.strip() for line in f.readlines()])
actual = set()
actual.update([render.output for render in renders])
actual = {render.output for render in renders}
if actual != expected:
print("Wrong expected outputs, caller expected:\n " + repr(list(expected)))
print("Actual output:\n " + repr(list(actual)))
print("Wrong expected outputs, caller expected:\n " + repr(sorted(expected)))
print("Actual output:\n " + repr(sorted(actual)))
return 1
# Add a any extra Python path before importing Jinja2 so invokers can point
# to a checkout of Jinja2 and note require it to be installed on the system
if args.extra_python_path != None:
sys.path.insert(1, args.extra_python_path)
import jinja2
outputs = _do_renders(renders, args.template_dir)
# Output the tarball and its depfile
@ -197,7 +240,7 @@ def run_generator(generator):
if args.depfile != None:
dependencies = generator.get_dependencies(args)
dependencies += [args.template_dir + os.path.sep + render.template for render in renders]
dependencies += _compute_python_dependencies()
dependencies += _compute_python_dependencies(args.root_dir)
with open(args.depfile, 'w') as f:
f.write(args.output_json_tarball + ": " + " ".join(dependencies))