encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Split off the reusable part of the code generator 

This is in preparation for using it to generate code for replacing glad
with our own OpenGL function pointer loading code.

BUG=dawn:165

Change-Id: Ic3e774ab207e85a1491f299ad06131c8095416ae
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/7781
Reviewed-by: Austin Eng <enga@chromium.org>
Reviewed-by: Kai Ninomiya <kainino@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
This commit is contained in:
Corentin Wallez 2019-06-07 08:59:17 +00:00 committed by Commit Bot service account
parent 120f5d9062
commit 0c38e92187
3 changed files with 288 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
generator/dawn_generator.gni
View File

@ -44,6 +44,7 @@ template("dawn_generator") {
# The base arguments for the generator: from this dawn.json, generate this
# target using templates in this directory.
generator_args = [
"--dawn-json",
rebase_path("${dawn_root}/dawn.json", root_build_dir),
"--wire-json",
rebase_path("${dawn_root}/dawn_wire.json", root_build_dir),

187
generator/generator_lib.py Normal file
View File

@ -0,0 +1,187 @@
#!/usr/bin/env python2
# Copyright 2019 The Dawn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse, json, os, re, sys
from collections import namedtuple
# The interface that must be implemented by generators.
class Generator:
def get_description(self):
return ""
def add_commandline_arguments(self, parser):
pass
def get_file_renders(self, args):
return []
def get_dependencies(self, args):
return []
FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
# Try using an additional python path from the arguments if present. This
# isn't done through the regular argparse because PreprocessingLoader uses
# jinja2 in the global scope before "main" gets to run.
kExtraPythonPath = '--extra-python-path'
if kExtraPythonPath in sys.argv:
path = sys.argv[sys.argv.index(kExtraPythonPath) + 1]
sys.path.insert(1, path)
import jinja2
# A custom Jinja2 template loader that removes the extra indentation
# of the template blocks so that the output is correctly indented
class PreprocessingLoader(jinja2.BaseLoader):
def __init__(self, path):
self.path = path
def get_source(self, environment, template):
path = os.path.join(self.path, template)
if not os.path.exists(path):
raise jinja2.TemplateNotFound(template)
mtime = os.path.getmtime(path)
with open(path) as f:
source = self.preprocess(f.read())
return source, path, lambda: mtime == os.path.getmtime(path)
blockstart = re.compile('{%-?\s*(if|for|block)[^}]*%}')
blockend = re.compile('{%-?\s*end(if|for|block)[^}]*%}')
def preprocess(self, source):
lines = source.split('\n')
# Compute the current indentation level of the template blocks and remove their indentation
result = []
indentation_level = 0
for line in lines:
# The capture in the regex adds one element per block start or end so we divide by two
# there is also an extra line chunk corresponding to the line end, so we substract it.
numends = (len(self.blockend.split(line)) - 1) // 2
indentation_level -= numends
result.append(self.remove_indentation(line, indentation_level))
numstarts = (len(self.blockstart.split(line)) - 1) // 2
indentation_level += numstarts
return '\n'.join(result) + '\n'
def remove_indentation(self, line, n):
for _ in range(n):
if line.startswith(' '):
line = line[4:]
elif line.startswith('\t'):
line = line[1:]
else:
assert(line.strip() == '')
return line
_FileOutput = namedtuple('FileOutput', ['name', 'content'])
def _do_renders(renders, template_dir):
loader = PreprocessingLoader(template_dir)
env = jinja2.Environment(loader=loader, lstrip_blocks=True, trim_blocks=True, line_comment_prefix='//*')
outputs = []
for render in renders:
params = {}
for param_dict in render.params_dicts:
params.update(param_dict)
content = env.get_template(render.template).render(**params)
outputs.append(_FileOutput(render.output, content))
return outputs
# Compute the list of imported, non-system Python modules.
# It assumes that any path outside of Dawn's root directory is system.
def _compute_python_dependencies():
dawn_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
module_paths = (module.__file__ for module in sys.modules.values()
if module and hasattr(module, '__file__'))
paths = set()
for path in module_paths:
path = os.path.abspath(path)
if not path.startswith(dawn_root):
continue
if (path.endswith('.pyc')
or (path.endswith('c') and not os.path.splitext(path)[1])):
path = path[:-1]
paths.add(path)
return paths
def run_generator(generator):
parser = argparse.ArgumentParser(
description = generator.get_description(),
formatter_class = argparse.ArgumentDefaultsHelpFormatter
)
generator.add_commandline_arguments(parser);
parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.')
parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2')
parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).')
parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball')
parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match")
args = parser.parse_args()
renders = generator.get_file_renders(args);
# The caller wants to assert that the outputs are what it expects.
# Load the file and compare with our renders.
if args.expected_outputs_file != None:
with open(args.expected_outputs_file) as f:
expected = set([line.strip() for line in f.readlines()])
actual = set()
actual.update([render.output for render in renders])
if actual != expected:
print("Wrong expected outputs, caller expected:\n " + repr(list(expected)))
print("Actual output:\n " + repr(list(actual)))
return 1
# Add a any extra Python path before importing Jinja2 so invokers can point
# to a checkout of Jinja2 and note require it to be installed on the system
if args.extra_python_path != None:
sys.path.insert(1, args.extra_python_path)
import jinja2
outputs = _do_renders(renders, args.template_dir)
# Output the tarball and its depfile
if args.output_json_tarball != None:
json_root = {}
for output in outputs:
json_root[output.name] = output.content
with open(args.output_json_tarball, 'w') as f:
f.write(json.dumps(json_root))
# Output a list of all dependencies for the tarball for Ninja.
if args.depfile != None:
dependencies = generator.get_dependencies(args)
dependencies += [args.template_dir + os.path.sep + render.template for render in renders]
dependencies += _compute_python_dependencies()
with open(args.depfile, 'w') as f:
f.write(args.output_json_tarball + ": " + " ".join(dependencies))

336
generator/main.py
View File

@ -13,18 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
############################################################
# COMMON
############################################################
import json, os, sys
from collections import namedtuple
from common import Name
import common
from common import Name
from generator_lib import Generator, run_generator, FileRender
import wire_cmd
############################################################
# PARSE
############################################################
import json
def is_native_method(method):
return method.return_type.category == "natively defined" or \
@ -122,98 +121,8 @@ def parse_json(json):
}
#############################################################
# OUTPUT
# Generator
#############################################################
import re, os, sys
from collections import OrderedDict
kExtraPythonPath = '--extra-python-path'
# Try using an additional python path from the arguments if present. This
# isn't done through the regular argparse because PreprocessingLoader uses
# jinja2 in the global scope before "main" gets to run.
if kExtraPythonPath in sys.argv:
path = sys.argv[sys.argv.index(kExtraPythonPath) + 1]
sys.path.insert(1, path)
import jinja2
# A custom Jinja2 template loader that removes the extra indentation
# of the template blocks so that the output is correctly indented
class PreprocessingLoader(jinja2.BaseLoader):
def __init__(self, path):
self.path = path
def get_source(self, environment, template):
path = os.path.join(self.path, template)
if not os.path.exists(path):
raise jinja2.TemplateNotFound(template)
mtime = os.path.getmtime(path)
with open(path) as f:
source = self.preprocess(f.read())
return source, path, lambda: mtime == os.path.getmtime(path)
blockstart = re.compile('{%-?\s*(if|for|block)[^}]*%}')
blockend = re.compile('{%-?\s*end(if|for|block)[^}]*%}')
def preprocess(self, source):
lines = source.split('\n')
# Compute the current indentation level of the template blocks and remove their indentation
result = []
indentation_level = 0
for line in lines:
# The capture in the regex adds one element per block start or end so we divide by two
# there is also an extra line chunk corresponding to the line end, so we substract it.
numends = (len(self.blockend.split(line)) - 1) // 2
indentation_level -= numends
line = self.remove_indentation(line, indentation_level)
# Manually perform the lstrip_blocks jinja2 env options as it available starting from 2.7
# and Travis only has Jinja 2.6
if line.lstrip().startswith('{%'):
line = line.lstrip()
result.append(line)
numstarts = (len(self.blockstart.split(line)) - 1) // 2
indentation_level += numstarts
return '\n'.join(result) + '\n'
def remove_indentation(self, line, n):
for _ in range(n):
if line.startswith(' '):
line = line[4:]
elif line.startswith('\t'):
line = line[1:]
else:
assert(line.strip() == '')
return line
FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
FileOutput = namedtuple('FileOutput', ['name', 'content'])
def do_renders(renders, template_dir):
env = jinja2.Environment(loader=PreprocessingLoader(template_dir), trim_blocks=True, line_comment_prefix='//*')
outputs = []
for render in renders:
params = {}
for param_dict in render.params_dicts:
params.update(param_dict)
content = env.get_template(render.template).render(**params)
outputs.append(FileOutput(render.output, content))
return outputs
#############################################################
# MAIN SOMETHING WHATEVER
#############################################################
import argparse, sys
def as_varName(*names):
return names[0].camelCase() + ''.join([name.CamelCase() for name in names[1:]])
@ -320,164 +229,119 @@ def do_assert(expr):
assert expr
return ''
def get_renders_for_targets(api_params, wire_json, targets):
base_params = {
'enumerate': enumerate,
'format': format,
'len': len,
'debug': debug,
'assert': do_assert,
class MultiGeneratorFromDawnJSON(Generator):
def get_description(self):
return 'Generates code for various target from Dawn.json.'
'Name': lambda name: Name(name),
def add_commandline_arguments(self, parser):
allowed_targets = ['dawn_headers', 'libdawn', 'mock_dawn', 'dawn_wire', "dawn_native_utils"]
'as_annotated_cType': lambda arg: annotated(as_cType(arg.type.name), arg),
'as_annotated_cppType': lambda arg: annotated(as_cppType(arg.type.name), arg),
'as_cEnum': as_cEnum,
'as_cppEnum': as_cppEnum,
'as_cMethod': as_cMethod,
'as_MethodSuffix': as_MethodSuffix,
'as_cProc': as_cProc,
'as_cType': as_cType,
'as_cppType': as_cppType,
'convert_cType_to_cppType': convert_cType_to_cppType,
'as_varName': as_varName,
'decorate': decorate,
}
parser.add_argument('--dawn-json', required=True, type=str, help ='The DAWN JSON definition to use.')
parser.add_argument('--wire-json', default=None, type=str, help='The DAWN WIRE JSON definition to use.')
parser.add_argument('-T', '--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets))
renders = []
def get_file_renders(self, args):
with open(args.dawn_json) as f:
loaded_json = json.loads(f.read())
api_params = parse_json(loaded_json)
c_params = {'native_methods': lambda typ: c_native_methods(api_params['types'], typ)}
cpp_params = {'native_methods': lambda typ: cpp_native_methods(api_params['types'], typ)}
targets = args.targets.split(',')
if 'dawn_headers' in targets:
renders.append(FileRender('api.h', 'dawn/dawn.h', [base_params, api_params, c_params]))
renders.append(FileRender('apicpp.h', 'dawn/dawncpp.h', [base_params, api_params, cpp_params]))
wire_json = None
if args.wire_json:
with open(args.wire_json) as f:
wire_json = json.loads(f.read())
if 'libdawn' in targets:
additional_params = {'native_methods': lambda typ: cpp_native_methods(api_params['types'], typ)}
renders.append(FileRender('api.c', 'dawn/dawn.c', [base_params, api_params, c_params]))
renders.append(FileRender('apicpp.cpp', 'dawn/dawncpp.cpp', [base_params, api_params, cpp_params]))
base_params = {
'enumerate': enumerate,
'format': format,
'len': len,
'debug': debug,
'assert': do_assert,
if 'mock_dawn' in targets:
renders.append(FileRender('mock_api.h', 'mock/mock_dawn.h', [base_params, api_params, c_params]))
renders.append(FileRender('mock_api.cpp', 'mock/mock_dawn.cpp', [base_params, api_params, c_params]))
'Name': lambda name: Name(name),
if 'dawn_native_utils' in targets:
frontend_params = [
base_params,
api_params,
c_params,
{
'as_frontendType': lambda typ: as_frontendType(typ), # TODO as_frontendType and friends take a Type and not a Name :(
'as_annotated_frontendType': lambda arg: annotated(as_frontendType(arg.type), arg)
}
]
'as_annotated_cType': lambda arg: annotated(as_cType(arg.type.name), arg),
'as_annotated_cppType': lambda arg: annotated(as_cppType(arg.type.name), arg),
'as_cEnum': as_cEnum,
'as_cppEnum': as_cppEnum,
'as_cMethod': as_cMethod,
'as_MethodSuffix': as_MethodSuffix,
'as_cProc': as_cProc,
'as_cType': as_cType,
'as_cppType': as_cppType,
'convert_cType_to_cppType': convert_cType_to_cppType,
'as_varName': as_varName,
'decorate': decorate,
}
renders.append(FileRender('dawn_native/ValidationUtils.h', 'dawn_native/ValidationUtils_autogen.h', frontend_params))
renders.append(FileRender('dawn_native/ValidationUtils.cpp', 'dawn_native/ValidationUtils_autogen.cpp', frontend_params))
renders.append(FileRender('dawn_native/api_structs.h', 'dawn_native/dawn_structs_autogen.h', frontend_params))
renders.append(FileRender('dawn_native/api_structs.cpp', 'dawn_native/dawn_structs_autogen.cpp', frontend_params))
renders.append(FileRender('dawn_native/ProcTable.cpp', 'dawn_native/ProcTable.cpp', frontend_params))
renders = []
if 'dawn_wire' in targets:
additional_params = wire_cmd.compute_wire_params(api_params, wire_json)
c_params = {'native_methods': lambda typ: c_native_methods(api_params['types'], typ)}
cpp_params = {'native_methods': lambda typ: cpp_native_methods(api_params['types'], typ)}
wire_params = [
base_params,
api_params,
c_params,
{
'as_wireType': lambda typ: typ.name.CamelCase() + '*' if typ.category == 'object' else as_cppType(typ.name)
},
additional_params
]
renders.append(FileRender('dawn_wire/WireCmd.h', 'dawn_wire/WireCmd_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/WireCmd.cpp', 'dawn_wire/WireCmd_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/client/ApiObjects.h', 'dawn_wire/client/ApiObjects_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/client/ApiProcs.cpp', 'dawn_wire/client/ApiProcs_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/client/ApiProcs.h', 'dawn_wire/client/ApiProcs_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/client/ClientBase.h', 'dawn_wire/client/ClientBase_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/client/ClientHandlers.cpp', 'dawn_wire/client/ClientHandlers_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/client/ClientPrototypes.inc', 'dawn_wire/client/ClientPrototypes_autogen.inc', wire_params))
renders.append(FileRender('dawn_wire/server/ServerBase.h', 'dawn_wire/server/ServerBase_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/server/ServerDoers.cpp', 'dawn_wire/server/ServerDoers_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/server/ServerHandlers.cpp', 'dawn_wire/server/ServerHandlers_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/server/ServerPrototypes.inc', 'dawn_wire/server/ServerPrototypes_autogen.inc', wire_params))
if 'dawn_headers' in targets:
renders.append(FileRender('api.h', 'dawn/dawn.h', [base_params, api_params, c_params]))
renders.append(FileRender('apicpp.h', 'dawn/dawncpp.h', [base_params, api_params, cpp_params]))
return renders
if 'libdawn' in targets:
additional_params = {'native_methods': lambda typ: cpp_native_methods(api_params['types'], typ)}
renders.append(FileRender('api.c', 'dawn/dawn.c', [base_params, api_params, c_params]))
renders.append(FileRender('apicpp.cpp', 'dawn/dawncpp.cpp', [base_params, api_params, cpp_params]))
def output_to_json(outputs, output_json):
json_root = {}
for output in outputs:
json_root[output.name] = output.content
if 'mock_dawn' in targets:
renders.append(FileRender('mock_api.h', 'mock/mock_dawn.h', [base_params, api_params, c_params]))
renders.append(FileRender('mock_api.cpp', 'mock/mock_dawn.cpp', [base_params, api_params, c_params]))
with open(output_json, 'w') as f:
f.write(json.dumps(json_root))
if 'dawn_native_utils' in targets:
frontend_params = [
base_params,
api_params,
c_params,
{
'as_frontendType': lambda typ: as_frontendType(typ), # TODO as_frontendType and friends take a Type and not a Name :(
'as_annotated_frontendType': lambda arg: annotated(as_frontendType(arg.type), arg)
}
]
def output_depfile(depfile, output, dependencies):
with open(depfile, 'w') as f:
f.write(output + ": " + " ".join(dependencies))
renders.append(FileRender('dawn_native/ValidationUtils.h', 'dawn_native/ValidationUtils_autogen.h', frontend_params))
renders.append(FileRender('dawn_native/ValidationUtils.cpp', 'dawn_native/ValidationUtils_autogen.cpp', frontend_params))
renders.append(FileRender('dawn_native/api_structs.h', 'dawn_native/dawn_structs_autogen.h', frontend_params))
renders.append(FileRender('dawn_native/api_structs.cpp', 'dawn_native/dawn_structs_autogen.cpp', frontend_params))
renders.append(FileRender('dawn_native/ProcTable.cpp', 'dawn_native/ProcTable.cpp', frontend_params))
def main():
allowed_targets = ['dawn_headers', 'libdawn', 'mock_dawn', 'dawn_wire', "dawn_native_utils"]
if 'dawn_wire' in targets:
additional_params = wire_cmd.compute_wire_params(api_params, wire_json)
parser = argparse.ArgumentParser(
description = 'Generates code for various target for Dawn.',
formatter_class = argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('json', metavar='DAWN_JSON', nargs=1, type=str, help ='The DAWN JSON definition to use.')
parser.add_argument('--wire-json', default=None, type=str, help='The DAWN WIRE JSON definition to use.')
parser.add_argument('-t', '--template-dir', default='templates', type=str, help='Directory with template files.')
parser.add_argument('-T', '--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets))
parser.add_argument(kExtraPythonPath, default=None, type=str, help='Additional python path to set before loading Jinja2')
parser.add_argument('--output-json-tarball', default=None, type=str, help='Name of the "JSON tarball" to create (tar is too annoying to use in python).')
parser.add_argument('--depfile', default=None, type=str, help='Name of the Ninja depfile to create for the JSON tarball')
parser.add_argument('--expected-outputs-file', default=None, type=str, help="File to compare outputs with and fail if it doesn't match")
wire_params = [
base_params,
api_params,
c_params,
{
'as_wireType': lambda typ: typ.name.CamelCase() + '*' if typ.category == 'object' else as_cppType(typ.name)
},
additional_params
]
renders.append(FileRender('dawn_wire/WireCmd.h', 'dawn_wire/WireCmd_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/WireCmd.cpp', 'dawn_wire/WireCmd_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/client/ApiObjects.h', 'dawn_wire/client/ApiObjects_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/client/ApiProcs.cpp', 'dawn_wire/client/ApiProcs_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/client/ApiProcs.h', 'dawn_wire/client/ApiProcs_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/client/ClientBase.h', 'dawn_wire/client/ClientBase_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/client/ClientHandlers.cpp', 'dawn_wire/client/ClientHandlers_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/client/ClientPrototypes.inc', 'dawn_wire/client/ClientPrototypes_autogen.inc', wire_params))
renders.append(FileRender('dawn_wire/server/ServerBase.h', 'dawn_wire/server/ServerBase_autogen.h', wire_params))
renders.append(FileRender('dawn_wire/server/ServerDoers.cpp', 'dawn_wire/server/ServerDoers_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/server/ServerHandlers.cpp', 'dawn_wire/server/ServerHandlers_autogen.cpp', wire_params))
renders.append(FileRender('dawn_wire/server/ServerPrototypes.inc', 'dawn_wire/server/ServerPrototypes_autogen.inc', wire_params))
args = parser.parse_args()
return renders
# Load and parse the API json file
with open(args.json[0]) as f:
loaded_json = json.loads(f.read())
api_params = parse_json(loaded_json)
targets = args.targets.split(',')
dependencies = [
os.path.join(os.path.abspath(os.path.dirname(__file__)), "common.py")
]
loaded_wire_json = None
if args.wire_json:
with open(args.wire_json) as f:
loaded_wire_json = json.loads(f.read())
dependencies.append(args.wire_json)
renders = get_renders_for_targets(api_params, loaded_wire_json, targets)
# The caller wants to assert that the outputs are what it expects.
# Load the file and compare with our renders.
if args.expected_outputs_file != None:
with open(args.expected_outputs_file) as f:
expected = set([line.strip() for line in f.readlines()])
actual = set()
actual.update([render.output for render in renders])
if actual != expected:
print("Wrong expected outputs, caller expected:\n " + repr(list(expected)))
print("Actual output:\n " + repr(list(actual)))
return 1
outputs = do_renders(renders, args.template_dir)
# Output the tarball and its depfile
if args.output_json_tarball != None:
output_to_json(outputs, args.output_json_tarball)
dependencies += [args.template_dir + os.path.sep + render.template for render in renders]
dependencies.append(args.json[0])
dependencies.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), "wire_cmd.py"))
output_depfile(args.depfile, args.output_json_tarball, dependencies)
def get_dependencies(self, args):
deps = [os.path.abspath(args.dawn_json)]
if args.wire_json != None:
deps += [os.path.abspath(args.wire_json)]
return deps
if __name__ == '__main__':
sys.exit(main())
sys.exit(run_generator(MultiGeneratorFromDawnJSON()))