diff --git a/BUILD.gn b/BUILD.gn index 7b3e78a93e..ab34144a5c 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -33,7 +33,7 @@ config("libdawn_native_internal") { } } -dawn_generator("libdawn_native_utils_gen") { +dawn_json_generator("libdawn_native_utils_gen") { target = "dawn_native_utils" outputs = [ "dawn_native/ProcTable.cpp", @@ -394,7 +394,7 @@ dawn_component("libdawn_native") { # libdawn_wire ############################################################################### -dawn_generator("libdawn_wire_gen") { +dawn_json_generator("libdawn_wire_gen") { target = "dawn_wire" outputs = [ "dawn_wire/WireCmd_autogen.h", @@ -524,7 +524,7 @@ static_library("dawn_utils") { # Dawn test targets ############################################################################### -dawn_generator("mock_dawn_gen") { +dawn_json_generator("mock_dawn_gen") { target = "mock_dawn" outputs = [ "mock/mock_dawn.h", diff --git a/generator/common.py b/generator/common.py deleted file mode 100644 index 430da453b0..0000000000 --- a/generator/common.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2017 The Dawn Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import namedtuple - -class Name: - def __init__(self, name, native=False): - self.native = native - if native: - self.chunks = [name] - else: - self.chunks = name.split(' ') - - def CamelChunk(self, chunk): - return chunk[0].upper() + chunk[1:] - - def canonical_case(self): - return (' '.join(self.chunks)).lower() - - def concatcase(self): - return ''.join(self.chunks) - - def camelCase(self): - return self.chunks[0] + ''.join([self.CamelChunk(chunk) for chunk in self.chunks[1:]]) - - def CamelCase(self): - return ''.join([self.CamelChunk(chunk) for chunk in self.chunks]) - - def SNAKE_CASE(self): - return '_'.join([chunk.upper() for chunk in self.chunks]) - - def snake_case(self): - return '_'.join(self.chunks) - -class Type: - def __init__(self, name, json_data, native=False): - self.json_data = json_data - self.dict_name = name - self.name = Name(name, native=native) - self.category = json_data['category'] - -EnumValue = namedtuple('EnumValue', ['name', 'value']) -class EnumType(Type): - def __init__(self, name, json_data): - Type.__init__(self, name, json_data) - self.values = [EnumValue(Name(m['name']), m['value']) for m in self.json_data['values']] - -BitmaskValue = namedtuple('BitmaskValue', ['name', 'value']) -class BitmaskType(Type): - def __init__(self, name, json_data): - Type.__init__(self, name, json_data) - self.values = [BitmaskValue(Name(m['name']), m['value']) for m in self.json_data['values']] - self.full_mask = 0 - for value in self.values: - self.full_mask = self.full_mask | value.value - -class NativeType(Type): - def __init__(self, name, json_data): - Type.__init__(self, name, json_data, native=True) - -class NativelyDefined(Type): - def __init__(self, name, json_data): - Type.__init__(self, name, json_data) - -# Methods and structures are both "records", so record members correspond to -# method arguments or structure members. -class RecordMember: - def __init__(self, name, typ, annotation, optional, is_return_value): - self.name = name - self.type = typ - self.annotation = annotation - self.length = None - self.optional = optional - self.is_return_value = is_return_value - self.handle_type = None - - def set_handle_type(self, handle_type): - assert self.type.dict_name == "ObjectHandle" - self.handle_type = handle_type - -Method = namedtuple('Method', ['name', 'return_type', 'arguments']) -class ObjectType(Type): - def __init__(self, name, json_data): - Type.__init__(self, name, json_data) - self.methods = [] - self.native_methods = [] - self.built_type = None - -class Record: - def __init__(self, name): - self.name = Name(name) - self.members = [] - self.has_dawn_object = False - - def update_metadata(self): - def has_dawn_object(member): - if isinstance(member.type, ObjectType): - return True - elif isinstance(member.type, StructureType): - return member.type.has_dawn_object - else: - return False - - self.has_dawn_object = any(has_dawn_object(member) for member in self.members) - -class StructureType(Record, Type): - def __init__(self, name, json_data): - Record.__init__(self, name) - Type.__init__(self, name, json_data) - self.extensible = json_data.get("extensible", False) - -class Command(Record): - def __init__(self, name, members=None): - Record.__init__(self, name) - self.members = members or [] - self.derived_object = None - self.derived_method = None - -def linked_record_members(json_data, types): - members = [] - members_by_name = {} - for m in json_data: - member = RecordMember(Name(m['name']), types[m['type']], - m.get('annotation', 'value'), m.get('optional', False), - m.get('is_return_value', False)) - handle_type = m.get('handle_type') - if handle_type: - member.set_handle_type(types[handle_type]) - members.append(member) - members_by_name[member.name.canonical_case()] = member - - for (member, m) in zip(members, json_data): - if member.annotation != 'value': - if not 'length' in m: - if member.type.category != 'object': - member.length = "constant" - member.constant_length = 1 - else: - assert(False) - elif m['length'] == 'strlen': - member.length = 'strlen' - else: - member.length = members_by_name[m['length']] - - return members diff --git a/generator/dawn_generator.gni b/generator/dawn_generator.gni index 4cfb62c4fc..d5f84c6212 100644 --- a/generator/dawn_generator.gni +++ b/generator/dawn_generator.gni @@ -14,19 +14,20 @@ import("../scripts/dawn_overrides_with_defaults.gni") -############################################################################### -# Template to wrap the Dawn code generator -############################################################################### - -# Template to help with invocation of the Dawn code generator, looks like this: +# Template to help invoking Dawn code generators based on generator_lib # # dawn_generator("my_target_gen") { -# # Which generator target to output -# target = "my_target" +# # The script and generator specific arguments +# script = [ "my_awesome_generator.py" ] +# args = [ +# "--be-awesome", +# "yes" +# ] +# # # The list of expected outputs, generation fails if there's a mismatch # outputs = [ -# "MyTarget.cpp", -# "MyTarget.h", +# "MyAwesomeTarget.cpp", +# "MyAwesomeTarget.h", # ] # # # Optional, use a custom generated file directory. @@ -41,18 +42,10 @@ import("../scripts/dawn_overrides_with_defaults.gni") # } # template("dawn_generator") { - # The base arguments for the generator: from this dawn.json, generate this - # target using templates in this directory. - generator_args = [ - "--dawn-json", - rebase_path("${dawn_root}/dawn.json", root_build_dir), - "--wire-json", - rebase_path("${dawn_root}/dawn_wire.json", root_build_dir), - "--template-dir", - rebase_path("${dawn_root}/generator/templates", root_build_dir), - "--targets", - invoker.target, - ] + generator_args = [] + if (defined(invoker.args)) { + generator_args += invoker.args + } # Use the Jinja2 version pulled from the DEPS file. We do it so we don't # have version problems, and users don't have to install Jinja2. @@ -103,7 +96,7 @@ template("dawn_generator") { # The code generator invocation that will write the JSON tarball, check the # outputs are what's expected and write a depfile for Ninja. action("${target_name}_json_tarball") { - script = "${dawn_root}/generator/main.py" + script = invoker.script outputs = [ json_tarball, ] @@ -112,7 +105,7 @@ template("dawn_generator") { } # Extract the JSON tarball into the gen_dir - action("${target_name}") { + action(target_name) { script = "${dawn_root}/generator/extract_json.py" args = [ rebase_path(json_tarball, root_build_dir), @@ -134,3 +127,33 @@ template("dawn_generator") { } } } + +# Helper generator for calling the generator from dawn.json +# +# dawn_json_generator("my_target_gen") { +# # Which generator target to output +# target = "my_target" +# +# # Also supports `outputs` and `custom_gen_dir` like dawn_generator. +# } +template("dawn_json_generator") { + dawn_generator(target_name) { + + script = "${dawn_root}/generator/dawn_json_generator.py" + + # The base arguments for the generator: from this dawn.json, generate this + # target using templates in this directory. + args = [ + "--dawn-json", + rebase_path("${dawn_root}/dawn.json", root_build_dir), + "--wire-json", + rebase_path("${dawn_root}/dawn_wire.json", root_build_dir), + "--template-dir", + rebase_path("${dawn_root}/generator/templates", root_build_dir), + "--targets", + invoker.target, + ] + + forward_variables_from(invoker, "*", ["target"]) + } +} diff --git a/generator/main.py b/generator/dawn_json_generator.py similarity index 60% rename from generator/main.py rename to generator/dawn_json_generator.py index 593ce9a9ba..9eb4c2e0f8 100644 --- a/generator/main.py +++ b/generator/dawn_json_generator.py @@ -16,10 +16,155 @@ import json, os, sys from collections import namedtuple -import common -from common import Name from generator_lib import Generator, run_generator, FileRender -import wire_cmd + +############################################################ +# OBJECT MODEL +############################################################ + +class Name: + def __init__(self, name, native=False): + self.native = native + if native: + self.chunks = [name] + else: + self.chunks = name.split(' ') + + def CamelChunk(self, chunk): + return chunk[0].upper() + chunk[1:] + + def canonical_case(self): + return (' '.join(self.chunks)).lower() + + def concatcase(self): + return ''.join(self.chunks) + + def camelCase(self): + return self.chunks[0] + ''.join([self.CamelChunk(chunk) for chunk in self.chunks[1:]]) + + def CamelCase(self): + return ''.join([self.CamelChunk(chunk) for chunk in self.chunks]) + + def SNAKE_CASE(self): + return '_'.join([chunk.upper() for chunk in self.chunks]) + + def snake_case(self): + return '_'.join(self.chunks) + +def concat_names(*names): + return ' '.join([name.canonical_case() for name in names]) + +class Type: + def __init__(self, name, json_data, native=False): + self.json_data = json_data + self.dict_name = name + self.name = Name(name, native=native) + self.category = json_data['category'] + +EnumValue = namedtuple('EnumValue', ['name', 'value']) +class EnumType(Type): + def __init__(self, name, json_data): + Type.__init__(self, name, json_data) + self.values = [EnumValue(Name(m['name']), m['value']) for m in self.json_data['values']] + +BitmaskValue = namedtuple('BitmaskValue', ['name', 'value']) +class BitmaskType(Type): + def __init__(self, name, json_data): + Type.__init__(self, name, json_data) + self.values = [BitmaskValue(Name(m['name']), m['value']) for m in self.json_data['values']] + self.full_mask = 0 + for value in self.values: + self.full_mask = self.full_mask | value.value + +class NativeType(Type): + def __init__(self, name, json_data): + Type.__init__(self, name, json_data, native=True) + +class NativelyDefined(Type): + def __init__(self, name, json_data): + Type.__init__(self, name, json_data) + +# Methods and structures are both "records", so record members correspond to +# method arguments or structure members. +class RecordMember: + def __init__(self, name, typ, annotation, optional, is_return_value): + self.name = name + self.type = typ + self.annotation = annotation + self.length = None + self.optional = optional + self.is_return_value = is_return_value + self.handle_type = None + + def set_handle_type(self, handle_type): + assert self.type.dict_name == "ObjectHandle" + self.handle_type = handle_type + +Method = namedtuple('Method', ['name', 'return_type', 'arguments']) +class ObjectType(Type): + def __init__(self, name, json_data): + Type.__init__(self, name, json_data) + self.methods = [] + self.native_methods = [] + self.built_type = None + +class Record: + def __init__(self, name): + self.name = Name(name) + self.members = [] + self.has_dawn_object = False + + def update_metadata(self): + def has_dawn_object(member): + if isinstance(member.type, ObjectType): + return True + elif isinstance(member.type, StructureType): + return member.type.has_dawn_object + else: + return False + + self.has_dawn_object = any(has_dawn_object(member) for member in self.members) + +class StructureType(Record, Type): + def __init__(self, name, json_data): + Record.__init__(self, name) + Type.__init__(self, name, json_data) + self.extensible = json_data.get("extensible", False) + +class Command(Record): + def __init__(self, name, members=None): + Record.__init__(self, name) + self.members = members or [] + self.derived_object = None + self.derived_method = None + +def linked_record_members(json_data, types): + members = [] + members_by_name = {} + for m in json_data: + member = RecordMember(Name(m['name']), types[m['type']], + m.get('annotation', 'value'), m.get('optional', False), + m.get('is_return_value', False)) + handle_type = m.get('handle_type') + if handle_type: + member.set_handle_type(types[handle_type]) + members.append(member) + members_by_name[member.name.canonical_case()] = member + + for (member, m) in zip(members, json_data): + if member.annotation != 'value': + if not 'length' in m: + if member.type.category != 'object': + member.length = "constant" + member.constant_length = 1 + else: + assert(False) + elif m['length'] == 'strlen': + member.length = 'strlen' + else: + member.length = members_by_name[m['length']] + + return members ############################################################ # PARSE @@ -31,15 +176,15 @@ def is_native_method(method): def link_object(obj, types): def make_method(json_data): - arguments = common.linked_record_members(json_data.get('args', []), types) - return common.Method(Name(json_data['name']), types[json_data.get('returns', 'void')], arguments) + arguments = linked_record_members(json_data.get('args', []), types) + return Method(Name(json_data['name']), types[json_data.get('returns', 'void')], arguments) methods = [make_method(m) for m in obj.json_data.get('methods', [])] obj.methods = [method for method in methods if not is_native_method(method)] obj.native_methods = [method for method in methods if is_native_method(method)] def link_structure(struct, types): - struct.members = common.linked_record_members(struct.json_data['members'], types) + struct.members = linked_record_members(struct.json_data['members'], types) # Sort structures so that if struct A has struct B as a member, then B is listed before A # This is a form of topological sort where we try to keep the order reasonably similar to the @@ -79,12 +224,12 @@ def topo_sort_structure(structs): def parse_json(json): category_to_parser = { - 'bitmask': common.BitmaskType, - 'enum': common.EnumType, - 'native': common.NativeType, - 'natively defined': common.NativelyDefined, - 'object': common.ObjectType, - 'structure': common.StructureType, + 'bitmask': BitmaskType, + 'enum': EnumType, + 'native': NativeType, + 'natively defined': NativelyDefined, + 'object': ObjectType, + 'structure': StructureType, } types = {} @@ -120,6 +265,67 @@ def parse_json(json): 'by_category': by_category } +############################################################ +# WIRE STUFF +############################################################ + +# Create wire commands from api methods +def compute_wire_params(api_params, wire_json): + wire_params = api_params.copy() + types = wire_params['types'] + + commands = [] + return_commands = [] + + # Generate commands from object methods + for api_object in wire_params['by_category']['object']: + for method in api_object.methods: + command_name = concat_names(api_object.name, method.name) + command_suffix = Name(command_name).CamelCase() + + # Only object return values or void are supported. Other methods must be handwritten. + if method.return_type.category != 'object' and method.return_type.name.canonical_case() != 'void': + assert(command_suffix in wire_json['special items']['client_handwritten_commands']) + continue + + if command_suffix in wire_json['special items']['client_side_commands']: + continue + + # Create object method commands by prepending "self" + members = [RecordMember(Name('self'), types[api_object.dict_name], 'value', False, False)] + members += method.arguments + + # Client->Server commands that return an object return the result object handle + if method.return_type.category == 'object': + result = RecordMember(Name('result'), types['ObjectHandle'], 'value', False, True) + result.set_handle_type(method.return_type) + members.append(result) + + command = Command(command_name, members) + command.derived_object = api_object + command.derived_method = method + commands.append(command) + + for (name, json_data) in wire_json['commands'].items(): + commands.append(Command(name, linked_record_members(json_data, types))) + + for (name, json_data) in wire_json['return commands'].items(): + return_commands.append(Command(name, linked_record_members(json_data, types))) + + wire_params['cmd_records'] = { + 'command': commands, + 'return command': return_commands + } + + for commands in wire_params['cmd_records'].values(): + for command in commands: + command.update_metadata() + commands.sort(key=lambda c: c.name.canonical_case()) + + wire_params.update(wire_json.get('special items', {})) + + return wire_params + ############################################################# # Generator ############################################################# @@ -215,20 +421,10 @@ def cpp_native_methods(types, typ): def c_native_methods(types, typ): return cpp_native_methods(types, typ) + [ - common.Method(Name('reference'), types['void'], []), - common.Method(Name('release'), types['void'], []), + Method(Name('reference'), types['void'], []), + Method(Name('release'), types['void'], []), ] -def js_native_methods(types, typ): - return cpp_native_methods(types, typ) - -def debug(text): - print(text) - -def do_assert(expr): - assert expr - return '' - class MultiGeneratorFromDawnJSON(Generator): def get_description(self): return 'Generates code for various target from Dawn.json.' @@ -238,7 +434,7 @@ class MultiGeneratorFromDawnJSON(Generator): parser.add_argument('--dawn-json', required=True, type=str, help ='The DAWN JSON definition to use.') parser.add_argument('--wire-json', default=None, type=str, help='The DAWN WIRE JSON definition to use.') - parser.add_argument('-T', '--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets)) + parser.add_argument('--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets)) def get_file_renders(self, args): with open(args.dawn_json) as f: @@ -253,12 +449,6 @@ class MultiGeneratorFromDawnJSON(Generator): wire_json = json.loads(f.read()) base_params = { - 'enumerate': enumerate, - 'format': format, - 'len': len, - 'debug': debug, - 'assert': do_assert, - 'Name': lambda name: Name(name), 'as_annotated_cType': lambda arg: annotated(as_cType(arg.type.name), arg), @@ -311,7 +501,7 @@ class MultiGeneratorFromDawnJSON(Generator): renders.append(FileRender('dawn_native/ProcTable.cpp', 'dawn_native/ProcTable.cpp', frontend_params)) if 'dawn_wire' in targets: - additional_params = wire_cmd.compute_wire_params(api_params, wire_json) + additional_params = compute_wire_params(api_params, wire_json) wire_params = [ base_params, diff --git a/generator/generator_lib.py b/generator/generator_lib.py index 63744124c9..efe449d493 100644 --- a/generator/generator_lib.py +++ b/generator/generator_lib.py @@ -96,9 +96,25 @@ def _do_renders(renders, template_dir): loader = PreprocessingLoader(template_dir) env = jinja2.Environment(loader=loader, lstrip_blocks=True, trim_blocks=True, line_comment_prefix='//*') + def do_assert(expr): + assert expr + return '' + + def debug(text): + print(text) + + base_params = { + 'enumerate': enumerate, + 'format': format, + 'len': len, + 'debug': debug, + 'assert': do_assert, + } + outputs = [] for render in renders: params = {} + params.update(base_params) for param_dict in render.params_dicts: params.update(param_dict) content = env.get_template(render.template).render(**params) diff --git a/generator/wire_cmd.py b/generator/wire_cmd.py deleted file mode 100644 index e6c8f0fe28..0000000000 --- a/generator/wire_cmd.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2019 The Dawn Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import namedtuple -from common import Name -import common - -def concat_names(*names): - return ' '.join([name.canonical_case() for name in names]) - -# Create wire commands from api methods -def compute_wire_params(api_params, wire_json): - wire_params = api_params.copy() - types = wire_params['types'] - - commands = [] - return_commands = [] - - # Generate commands from object methods - for api_object in wire_params['by_category']['object']: - for method in api_object.methods: - command_name = concat_names(api_object.name, method.name) - command_suffix = Name(command_name).CamelCase() - - # Only object return values or void are supported. Other methods must be handwritten. - if method.return_type.category != 'object' and method.return_type.name.canonical_case() != 'void': - assert(command_suffix in wire_json['special items']['client_handwritten_commands']) - continue - - if command_suffix in wire_json['special items']['client_side_commands']: - continue - - # Create object method commands by prepending "self" - members = [common.RecordMember(Name('self'), types[api_object.dict_name], 'value', False, False)] - members += method.arguments - - # Client->Server commands that return an object return the result object handle - if method.return_type.category == 'object': - result = common.RecordMember(Name('result'), types['ObjectHandle'], 'value', False, True) - result.set_handle_type(method.return_type) - members.append(result) - - command = common.Command(command_name, members) - command.derived_object = api_object - command.derived_method = method - commands.append(command) - - for (name, json_data) in wire_json['commands'].items(): - commands.append(common.Command(name, common.linked_record_members(json_data, types))) - - for (name, json_data) in wire_json['return commands'].items(): - return_commands.append(common.Command(name, common.linked_record_members(json_data, types))) - - wire_params['cmd_records'] = { - 'command': commands, - 'return command': return_commands - } - - for commands in wire_params['cmd_records'].values(): - for command in commands: - command.update_metadata() - commands.sort(key=lambda c: c.name.canonical_case()) - - wire_params.update(wire_json.get('special items', {})) - - return wire_params diff --git a/src/dawn/BUILD.gn b/src/dawn/BUILD.gn index a53ea52ccb..7e2efade3e 100644 --- a/src/dawn/BUILD.gn +++ b/src/dawn/BUILD.gn @@ -21,7 +21,7 @@ import("${dawn_root}/generator/dawn_generator.gni") # Dawn headers ############################################################################### -dawn_generator("dawn_headers_gen") { +dawn_json_generator("dawn_headers_gen") { target = "dawn_headers" # Generate as if we were in the main BUILD.gn because that was historically @@ -51,7 +51,7 @@ source_set("dawn_headers") { # libdawn ############################################################################### -dawn_generator("libdawn_gen") { +dawn_json_generator("libdawn_gen") { target = "libdawn" outputs = [ "dawn/dawncpp.cpp",