#!/usr/bin/env python2 # Copyright 2019 The Dawn Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse, json, os, re, sys from collections import namedtuple # The interface that must be implemented by generators. class Generator: def get_description(self): return "" def add_commandline_arguments(self, parser): pass def get_file_renders(self, args): return [] def get_dependencies(self, args): return [] FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts']) # Try using an additional python path from the arguments if present. This # isn't done through the regular argparse because PreprocessingLoader uses # jinja2 in the global scope before "main" gets to run. kExtraPythonPath = '--extra-python-path' if kExtraPythonPath in sys.argv: path = sys.argv[sys.argv.index(kExtraPythonPath) + 1] sys.path.insert(1, path) import jinja2 # A custom Jinja2 template loader that removes the extra indentation # of the template blocks so that the output is correctly indented class _PreprocessingLoader(jinja2.BaseLoader): def __init__(self, path): self.path = path def get_source(self, environment, template): path = os.path.join(self.path, template) if not os.path.exists(path): raise jinja2.TemplateNotFound(template) mtime = os.path.getmtime(path) with open(path) as f: source = self.preprocess(f.read()) return source, path, lambda: mtime == os.path.getmtime(path) blockstart = re.compile('{%-?\s*(if|for|block)[^}]*%}') blockend = re.compile('{%-?\s*end(if|for|block)[^}]*%}') def preprocess(self, source): lines = source.split('\n') # Compute the current indentation level of the template blocks and remove their indentation result = [] indentation_level = 0 for line in lines: # The capture in the regex adds one element per block start or end so we divide by two # there is also an extra line chunk corresponding to the line end, so we substract it. numends = (len(self.blockend.split(line)) - 1) // 2 indentation_level -= numends result.append(self.remove_indentation(line, indentation_level)) numstarts = (len(self.blockstart.split(line)) - 1) // 2 indentation_level += numstarts return '\n'.join(result) + '\n' def remove_indentation(self, line, n): for _ in range(n): if line.startswith(' '): line = line[4:] elif line.startswith('\t'): line = line[1:] else: assert(line.strip() == '') return line _FileOutput = namedtuple('FileOutput', ['name', 'content']) def _do_renders(renders, template_dir): loader = _PreprocessingLoader(template_dir) env = jinja2.Environment(loader=loader, lstrip_blocks=True, trim_blocks=True, line_comment_prefix='//*') def do_assert(expr): assert expr return '' def debug(text): print(text) base_params = { 'enumerate': enumerate, 'format': format, 'len': len, 'debug': debug, 'assert': do_assert, } outputs = [] for render in renders: params = {} params.update(base_params) for param_dict in render.params_dicts: params.update(param_dict) content = env.get_template(render.template).render(**params) outputs.append(_FileOutput(render.output, content)) return outputs # Compute the list of imported, non-system Python modules. # It assumes that any path outside of Dawn's root directory is system. def _compute_python_dependencies(): dawn_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) module_paths = (module.__file__ for module in sys.modules.values() if module and hasattr(module, '__file__')) paths = set() for path in module_paths: path = os.path.abspath(path) if not path.startswith(dawn_root): continue if (path.endswith('.pyc') or (path.endswith('c') and not os.path.splitext(path)[1])): path = path[:-1] paths.add(path) return paths def run_generator(generator): parser = argparse.ArgumentParser( description = generator.get_description(), formatter_class = argparse.ArgumentDefaultsHelpFormatter ) generator.add_commandline_arguments(parser); parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.') parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2') parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).') parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball') parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match") args = parser.parse_args() renders = generator.get_file_renders(args); # The caller wants to assert that the outputs are what it expects. # Load the file and compare with our renders. if args.expected_outputs_file != None: with open(args.expected_outputs_file) as f: expected = set([line.strip() for line in f.readlines()]) actual = set() actual.update([render.output for render in renders]) if actual != expected: print("Wrong expected outputs, caller expected:\n " + repr(list(expected))) print("Actual output:\n " + repr(list(actual))) return 1 # Add a any extra Python path before importing Jinja2 so invokers can point # to a checkout of Jinja2 and note require it to be installed on the system if args.extra_python_path != None: sys.path.insert(1, args.extra_python_path) import jinja2 outputs = _do_renders(renders, args.template_dir) # Output the tarball and its depfile if args.output_json_tarball != None: json_root = {} for output in outputs: json_root[output.name] = output.content with open(args.output_json_tarball, 'w') as f: f.write(json.dumps(json_root)) # Output a list of all dependencies for the tarball for Ninja. if args.depfile != None: dependencies = generator.get_dependencies(args) dependencies += [args.template_dir + os.path.sep + render.template for render in renders] dependencies += _compute_python_dependencies() with open(args.depfile, 'w') as f: f.write(args.output_json_tarball + ": " + " ".join(dependencies))