Split off the reusable part of the code generator

This is in preparation for using it to generate code for replacing glad
with our own OpenGL function pointer loading code.

BUG=dawn:165

Change-Id: Ic3e774ab207e85a1491f299ad06131c8095416ae
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/7781
Reviewed-by: Austin Eng <enga@chromium.org>
Reviewed-by: Kai Ninomiya <kainino@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>

generator/dawn_generator.gni

@@ -44,6 +44,7 @@ template("dawn_generator") {
   # The base arguments for the generator: from this dawn.json, generate this
   # target using templates in this directory.
   generator_args = [
+    "--dawn-json",
     rebase_path("${dawn_root}/dawn.json", root_build_dir),
     "--wire-json",
     rebase_path("${dawn_root}/dawn_wire.json", root_build_dir),

generator/generator_lib.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python2
+# Copyright 2019 The Dawn Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse, json, os, re, sys
+from collections import namedtuple
+
+# The interface that must be implemented by generators.
+class Generator:
+    def get_description(self):
+        return ""
+
+    def add_commandline_arguments(self, parser):
+        pass
+
+    def get_file_renders(self, args):
+        return []
+
+    def get_dependencies(self, args):
+        return []
+
+FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
+
+# Try using an additional python path from the arguments if present. This
+# isn't done through the regular argparse because PreprocessingLoader uses
+# jinja2 in the global scope before "main" gets to run.
+kExtraPythonPath = '--extra-python-path'
+if kExtraPythonPath in sys.argv:
+    path = sys.argv[sys.argv.index(kExtraPythonPath) + 1]
+    sys.path.insert(1, path)
+
+import jinja2
+
+# A custom Jinja2 template loader that removes the extra indentation
+# of the template blocks so that the output is correctly indented
+class PreprocessingLoader(jinja2.BaseLoader):
+    def __init__(self, path):
+        self.path = path
+
+    def get_source(self, environment, template):
+        path = os.path.join(self.path, template)
+        if not os.path.exists(path):
+            raise jinja2.TemplateNotFound(template)
+        mtime = os.path.getmtime(path)
+        with open(path) as f:
+            source = self.preprocess(f.read())
+        return source, path, lambda: mtime == os.path.getmtime(path)
+
+    blockstart = re.compile('{%-?\s*(if|for|block)[^}]*%}')
+    blockend = re.compile('{%-?\s*end(if|for|block)[^}]*%}')
+
+    def preprocess(self, source):
+        lines = source.split('\n')
+
+        # Compute the current indentation level of the template blocks and remove their indentation
+        result = []
+        indentation_level = 0
+
+        for line in lines:
+            # The capture in the regex adds one element per block start or end so we divide by two
+            # there is also an extra line chunk corresponding to the line end, so we substract it.
+            numends = (len(self.blockend.split(line)) - 1) // 2
+            indentation_level -= numends
+
+            result.append(self.remove_indentation(line, indentation_level))
+
+            numstarts = (len(self.blockstart.split(line)) - 1) // 2
+            indentation_level += numstarts
+
+        return '\n'.join(result) + '\n'
+
+    def remove_indentation(self, line, n):
+        for _ in range(n):
+            if line.startswith(' '):
+                line = line[4:]
+            elif line.startswith('\t'):
+                line = line[1:]
+            else:
+                assert(line.strip() == '')
+        return line
+
+_FileOutput = namedtuple('FileOutput', ['name', 'content'])
+
+def _do_renders(renders, template_dir):
+    loader = PreprocessingLoader(template_dir)
+    env = jinja2.Environment(loader=loader, lstrip_blocks=True, trim_blocks=True, line_comment_prefix='//*')
+
+    outputs = []
+    for render in renders:
+        params = {}
+        for param_dict in render.params_dicts:
+            params.update(param_dict)
+        content = env.get_template(render.template).render(**params)
+        outputs.append(_FileOutput(render.output, content))
+
+    return outputs
+
+# Compute the list of imported, non-system Python modules.
+# It assumes that any path outside of Dawn's root directory is system.
+def _compute_python_dependencies():
+    dawn_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+
+    module_paths = (module.__file__ for module in sys.modules.values()
+                                    if module and hasattr(module, '__file__'))
+
+    paths = set()
+    for path in module_paths:
+        path = os.path.abspath(path)
+
+        if not path.startswith(dawn_root):
+            continue
+
+        if (path.endswith('.pyc')
+                or (path.endswith('c') and not os.path.splitext(path)[1])):
+            path = path[:-1]
+
+        paths.add(path)
+
+    return paths
+
+def run_generator(generator):
+    parser = argparse.ArgumentParser(
+        description = generator.get_description(),
+        formatter_class = argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    generator.add_commandline_arguments(parser);
+    parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.')
+    parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2')
+    parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).')
+    parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball')
+    parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match")
+
+    args = parser.parse_args()
+
+    renders = generator.get_file_renders(args);
+
+    # The caller wants to assert that the outputs are what it expects.
+    # Load the file and compare with our renders.
+    if args.expected_outputs_file != None:
+        with open(args.expected_outputs_file) as f:
+            expected = set([line.strip() for line in f.readlines()])
+
+        actual = set()
+        actual.update([render.output for render in renders])
+
+        if actual != expected:
+            print("Wrong expected outputs, caller expected:\n    " + repr(list(expected)))
+            print("Actual output:\n    " + repr(list(actual)))
+            return 1
+
+    # Add a any extra Python path before importing Jinja2 so invokers can point
+    # to a checkout of Jinja2 and note require it to be installed on the system
+    if args.extra_python_path != None:
+        sys.path.insert(1, args.extra_python_path)
+    import jinja2
+
+    outputs = _do_renders(renders, args.template_dir)
+
+    # Output the tarball and its depfile
+    if args.output_json_tarball != None:
+        json_root = {}
+        for output in outputs:
+            json_root[output.name] = output.content
+
+        with open(args.output_json_tarball, 'w') as f:
+            f.write(json.dumps(json_root))
+
+    # Output a list of all dependencies for the tarball for Ninja.
+    if args.depfile != None:
+        dependencies = generator.get_dependencies(args)
+        dependencies += [args.template_dir + os.path.sep + render.template for render in renders]
+        dependencies += _compute_python_dependencies()
+
+        with open(args.depfile, 'w') as f:
+            f.write(args.output_json_tarball + ": " + " ".join(dependencies))

generator/main.py

@@ -13,18 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-############################################################
+import json, os, sys
-# COMMON
-############################################################
 from collections import namedtuple
-from common import Name
+
 import common
+from common import Name
+from generator_lib import Generator, run_generator, FileRender
 import wire_cmd
 
 ############################################################
 # PARSE
 ############################################################
-import json
 
 def is_native_method(method):
     return method.return_type.category == "natively defined" or \
@@ -122,98 +121,8 @@ def parse_json(json):
     }
 
 #############################################################
-# OUTPUT
+# Generator
 #############################################################
-import re, os, sys
-from collections import OrderedDict
-
-kExtraPythonPath = '--extra-python-path'
-
-# Try using an additional python path from the arguments if present. This
-# isn't done through the regular argparse because PreprocessingLoader uses
-# jinja2 in the global scope before "main" gets to run.
-if kExtraPythonPath in sys.argv:
-    path = sys.argv[sys.argv.index(kExtraPythonPath) + 1]
-    sys.path.insert(1, path)
-
-import jinja2
-
-# A custom Jinja2 template loader that removes the extra indentation
-# of the template blocks so that the output is correctly indented
-class PreprocessingLoader(jinja2.BaseLoader):
-    def __init__(self, path):
-        self.path = path
-
-    def get_source(self, environment, template):
-        path = os.path.join(self.path, template)
-        if not os.path.exists(path):
-            raise jinja2.TemplateNotFound(template)
-        mtime = os.path.getmtime(path)
-        with open(path) as f:
-            source = self.preprocess(f.read())
-        return source, path, lambda: mtime == os.path.getmtime(path)
-
-    blockstart = re.compile('{%-?\s*(if|for|block)[^}]*%}')
-    blockend = re.compile('{%-?\s*end(if|for|block)[^}]*%}')
-
-    def preprocess(self, source):
-        lines = source.split('\n')
-
-        # Compute the current indentation level of the template blocks and remove their indentation
-        result = []
-        indentation_level = 0
-
-        for line in lines:
-            # The capture in the regex adds one element per block start or end so we divide by two
-            # there is also an extra line chunk corresponding to the line end, so we substract it.
-            numends = (len(self.blockend.split(line)) - 1) // 2
-            indentation_level -= numends
-
-            line = self.remove_indentation(line, indentation_level)
-
-            # Manually perform the lstrip_blocks jinja2 env options as it available starting from 2.7
-            # and Travis only has Jinja 2.6
-            if line.lstrip().startswith('{%'):
-                line = line.lstrip()
-
-            result.append(line)
-
-            numstarts = (len(self.blockstart.split(line)) - 1) // 2
-            indentation_level += numstarts
-
-        return '\n'.join(result) + '\n'
-
-    def remove_indentation(self, line, n):
-        for _ in range(n):
-            if line.startswith(' '):
-                line = line[4:]
-            elif line.startswith('\t'):
-                line = line[1:]
-            else:
-                assert(line.strip() == '')
-        return line
-
-FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
-
-FileOutput = namedtuple('FileOutput', ['name', 'content'])
-
-def do_renders(renders, template_dir):
-    env = jinja2.Environment(loader=PreprocessingLoader(template_dir), trim_blocks=True, line_comment_prefix='//*')
-
-    outputs = []
-    for render in renders:
-        params = {}
-        for param_dict in render.params_dicts:
-            params.update(param_dict)
-        content = env.get_template(render.template).render(**params)
-        outputs.append(FileOutput(render.output, content))
-
-    return outputs
-
-#############################################################
-# MAIN SOMETHING WHATEVER
-#############################################################
-import argparse, sys
 
 def as_varName(*names):
     return names[0].camelCase() + ''.join([name.CamelCase() for name in names[1:]])
@@ -320,7 +229,29 @@ def do_assert(expr):
     assert expr
     return ''
 
-def get_renders_for_targets(api_params, wire_json, targets):
+class MultiGeneratorFromDawnJSON(Generator):
+    def get_description(self):
+        return 'Generates code for various target from Dawn.json.'
+
+    def add_commandline_arguments(self, parser):
+        allowed_targets = ['dawn_headers', 'libdawn', 'mock_dawn', 'dawn_wire', "dawn_native_utils"]
+
+        parser.add_argument('--dawn-json', required=True, type=str, help ='The DAWN JSON definition to use.')
+        parser.add_argument('--wire-json', default=None, type=str, help='The DAWN WIRE JSON definition to use.')
+        parser.add_argument('-T', '--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets))
+
+    def get_file_renders(self, args):
+        with open(args.dawn_json) as f:
+            loaded_json = json.loads(f.read())
+        api_params = parse_json(loaded_json)
+
+        targets = args.targets.split(',')
+
+        wire_json = None
+        if args.wire_json:
+            with open(args.wire_json) as f:
+                wire_json = json.loads(f.read())
+
         base_params = {
             'enumerate': enumerate,
             'format': format,
@@ -406,78 +337,11 @@ def get_renders_for_targets(api_params, wire_json, targets):
 
         return renders
 
-def output_to_json(outputs, output_json):
+    def get_dependencies(self, args):
-    json_root = {}
+        deps = [os.path.abspath(args.dawn_json)]
-    for output in outputs:
+        if args.wire_json != None:
-        json_root[output.name] = output.content
+            deps += [os.path.abspath(args.wire_json)]
-
+        return deps
-    with open(output_json, 'w') as f:
-        f.write(json.dumps(json_root))
-
-def output_depfile(depfile, output, dependencies):
-    with open(depfile, 'w') as f:
-        f.write(output + ": " + " ".join(dependencies))
-
-def main():
-    allowed_targets = ['dawn_headers', 'libdawn', 'mock_dawn', 'dawn_wire', "dawn_native_utils"]
-
-    parser = argparse.ArgumentParser(
-        description = 'Generates code for various target for Dawn.',
-        formatter_class = argparse.ArgumentDefaultsHelpFormatter
-    )
-    parser.add_argument('json', metavar='DAWN_JSON', nargs=1, type=str, help ='The DAWN JSON definition to use.')
-    parser.add_argument('--wire-json', default=None, type=str, help='The DAWN WIRE JSON definition to use.')
-    parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.')
-    parser.add_argument('-T', '--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets))
-    parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2')
-    parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).')
-    parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball')
-    parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match")
-
-    args = parser.parse_args()
-
-    # Load and parse the API json file
-    with open(args.json[0]) as f:
-        loaded_json = json.loads(f.read())
-    api_params = parse_json(loaded_json)
-
-    targets = args.targets.split(',')
-    dependencies = [
-        os.path.join(os.path.abspath(os.path.dirname(__file__)), "common.py")
-    ]
-
-    loaded_wire_json = None
-    if args.wire_json:
-      with open(args.wire_json) as f:
-          loaded_wire_json = json.loads(f.read())
-      dependencies.append(args.wire_json)
-
-    renders = get_renders_for_targets(api_params, loaded_wire_json, targets)
-
-    # The caller wants to assert that the outputs are what it expects.
-    # Load the file and compare with our renders.
-    if args.expected_outputs_file != None:
-        with open(args.expected_outputs_file) as f:
-            expected = set([line.strip() for line in f.readlines()])
-
-        actual = set()
-        actual.update([render.output for render in renders])
-
-        if actual != expected:
-            print("Wrong expected outputs, caller expected:\n    " + repr(list(expected)))
-            print("Actual output:\n    " + repr(list(actual)))
-            return 1
-
-    outputs = do_renders(renders, args.template_dir)
-
-    # Output the tarball and its depfile
-    if args.output_json_tarball != None:
-        output_to_json(outputs, args.output_json_tarball)
-
-        dependencies += [args.template_dir + os.path.sep + render.template for render in renders]
-        dependencies.append(args.json[0])
-        dependencies.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), "wire_cmd.py"))
-        output_depfile(args.depfile, args.output_json_tarball, dependencies)
 
 if __name__ == '__main__':
-    sys.exit(main())
+    sys.exit(run_generator(MultiGeneratorFromDawnJSON()))