encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Generators: main.py is now dawn_json_generator.py 

This merges all the files for main.py together again except
generator_lib.py because there doesn't seeem to be a good way to
separate the pure dawn.json generators and the dawn wire generator.

Also updates the GN templates to make it easier to define new generators
based on generator_lib, and move the Jinja2 template "stdlib" to
generator_lib.py

BUG=dawn:165

Change-Id: I1b5b2ef0a59cb142e214f3af9a58048a88ae949a
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/7880
Commit-Queue: Kai Ninomiya <kainino@chromium.org>
Reviewed-by: Austin Eng <enga@chromium.org>
Reviewed-by: Kai Ninomiya <kainino@chromium.org>
This commit is contained in:
Corentin Wallez 2019-06-11 18:03:05 +00:00 committed by Commit Bot service account
parent 88860d1ced
commit 031fbbbaa1
7 changed files with 289 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
BUILD.gn
View File

@ -33,7 +33,7 @@ config("libdawn_native_internal") {
}
}
dawn_generator("libdawn_native_utils_gen") {
dawn_json_generator("libdawn_native_utils_gen") {
target = "dawn_native_utils"
outputs = [
"dawn_native/ProcTable.cpp",
@ -394,7 +394,7 @@ dawn_component("libdawn_native") {
# libdawn_wire
###############################################################################
dawn_generator("libdawn_wire_gen") {
dawn_json_generator("libdawn_wire_gen") {
target = "dawn_wire"
outputs = [
"dawn_wire/WireCmd_autogen.h",
@ -524,7 +524,7 @@ static_library("dawn_utils") {
# Dawn test targets
###############################################################################
dawn_generator("mock_dawn_gen") {
dawn_json_generator("mock_dawn_gen") {
target = "mock_dawn"
outputs = [
"mock/mock_dawn.h",

156
generator/common.py
View File

@ -1,156 +0,0 @@
# Copyright 2017 The Dawn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import namedtuple
class Name:
def __init__(self, name, native=False):
self.native = native
if native:
self.chunks = [name]
else:
self.chunks = name.split(' ')
def CamelChunk(self, chunk):
return chunk[0].upper() + chunk[1:]
def canonical_case(self):
return (' '.join(self.chunks)).lower()
def concatcase(self):
return ''.join(self.chunks)
def camelCase(self):
return self.chunks[0] + ''.join([self.CamelChunk(chunk) for chunk in self.chunks[1:]])
def CamelCase(self):
return ''.join([self.CamelChunk(chunk) for chunk in self.chunks])
def SNAKE_CASE(self):
return '_'.join([chunk.upper() for chunk in self.chunks])
def snake_case(self):
return '_'.join(self.chunks)
class Type:
def __init__(self, name, json_data, native=False):
self.json_data = json_data
self.dict_name = name
self.name = Name(name, native=native)
self.category = json_data['category']
EnumValue = namedtuple('EnumValue', ['name', 'value'])
class EnumType(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data)
self.values = [EnumValue(Name(m['name']), m['value']) for m in self.json_data['values']]
BitmaskValue = namedtuple('BitmaskValue', ['name', 'value'])
class BitmaskType(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data)
self.values = [BitmaskValue(Name(m['name']), m['value']) for m in self.json_data['values']]
self.full_mask = 0
for value in self.values:
self.full_mask = self.full_mask | value.value
class NativeType(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data, native=True)
class NativelyDefined(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data)
# Methods and structures are both "records", so record members correspond to
# method arguments or structure members.
class RecordMember:
def __init__(self, name, typ, annotation, optional, is_return_value):
self.name = name
self.type = typ
self.annotation = annotation
self.length = None
self.optional = optional
self.is_return_value = is_return_value
self.handle_type = None
def set_handle_type(self, handle_type):
assert self.type.dict_name == "ObjectHandle"
self.handle_type = handle_type
Method = namedtuple('Method', ['name', 'return_type', 'arguments'])
class ObjectType(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data)
self.methods = []
self.native_methods = []
self.built_type = None
class Record:
def __init__(self, name):
self.name = Name(name)
self.members = []
self.has_dawn_object = False
def update_metadata(self):
def has_dawn_object(member):
if isinstance(member.type, ObjectType):
return True
elif isinstance(member.type, StructureType):
return member.type.has_dawn_object
else:
return False
self.has_dawn_object = any(has_dawn_object(member) for member in self.members)
class StructureType(Record, Type):
def __init__(self, name, json_data):
Record.__init__(self, name)
Type.__init__(self, name, json_data)
self.extensible = json_data.get("extensible", False)
class Command(Record):
def __init__(self, name, members=None):
Record.__init__(self, name)
self.members = members or []
self.derived_object = None
self.derived_method = None
def linked_record_members(json_data, types):
members = []
members_by_name = {}
for m in json_data:
member = RecordMember(Name(m['name']), types[m['type']],
m.get('annotation', 'value'), m.get('optional', False),
m.get('is_return_value', False))
handle_type = m.get('handle_type')
if handle_type:
member.set_handle_type(types[handle_type])
members.append(member)
members_by_name[member.name.canonical_case()] = member
for (member, m) in zip(members, json_data):
if member.annotation != 'value':
if not 'length' in m:
if member.type.category != 'object':
member.length = "constant"
member.constant_length = 1
else:
assert(False)
elif m['length'] == 'strlen':
member.length = 'strlen'
else:
member.length = members_by_name[m['length']]
return members

69
generator/dawn_generator.gni
View File

@ -14,19 +14,20 @@
import("../scripts/dawn_overrides_with_defaults.gni")
###############################################################################
# Template to wrap the Dawn code generator
###############################################################################
# Template to help with invocation of the Dawn code generator, looks like this:
# Template to help invoking Dawn code generators based on generator_lib
#
# dawn_generator("my_target_gen") {
# # Which generator target to output
# target = "my_target"
# # The script and generator specific arguments
# script = [ "my_awesome_generator.py" ]
# args = [
# "--be-awesome",
# "yes"
# ]
#
# # The list of expected outputs, generation fails if there's a mismatch
# outputs = [
# "MyTarget.cpp",
# "MyTarget.h",
# "MyAwesomeTarget.cpp",
# "MyAwesomeTarget.h",
# ]
#
# # Optional, use a custom generated file directory.
@ -41,18 +42,10 @@ import("../scripts/dawn_overrides_with_defaults.gni")
# }
#
template("dawn_generator") {
# The base arguments for the generator: from this dawn.json, generate this
# target using templates in this directory.
generator_args = [
"--dawn-json",
rebase_path("${dawn_root}/dawn.json", root_build_dir),
"--wire-json",
rebase_path("${dawn_root}/dawn_wire.json", root_build_dir),
"--template-dir",
rebase_path("${dawn_root}/generator/templates", root_build_dir),
"--targets",
invoker.target,
]
generator_args = []
if (defined(invoker.args)) {
generator_args += invoker.args
}
# Use the Jinja2 version pulled from the DEPS file. We do it so we don't
# have version problems, and users don't have to install Jinja2.
@ -103,7 +96,7 @@ template("dawn_generator") {
# The code generator invocation that will write the JSON tarball, check the
# outputs are what's expected and write a depfile for Ninja.
action("${target_name}_json_tarball") {
script = "${dawn_root}/generator/main.py"
script = invoker.script
outputs = [
json_tarball,
]
@ -112,7 +105,7 @@ template("dawn_generator") {
}
# Extract the JSON tarball into the gen_dir
action("${target_name}") {
action(target_name) {
script = "${dawn_root}/generator/extract_json.py"
args = [
rebase_path(json_tarball, root_build_dir),
@ -134,3 +127,33 @@ template("dawn_generator") {
}
}
}
# Helper generator for calling the generator from dawn.json
#
# dawn_json_generator("my_target_gen") {
# # Which generator target to output
# target = "my_target"
#
# # Also supports `outputs` and `custom_gen_dir` like dawn_generator.
# }
template("dawn_json_generator") {
dawn_generator(target_name) {
script = "${dawn_root}/generator/dawn_json_generator.py"
# The base arguments for the generator: from this dawn.json, generate this
# target using templates in this directory.
args = [
"--dawn-json",
rebase_path("${dawn_root}/dawn.json", root_build_dir),
"--wire-json",
rebase_path("${dawn_root}/dawn_wire.json", root_build_dir),
"--template-dir",
rebase_path("${dawn_root}/generator/templates", root_build_dir),
"--targets",
invoker.target,
]
forward_variables_from(invoker, "*", ["target"])
}
}

254
generator/main.py → generator/dawn_json_generator.py
View File

@ -16,10 +16,155 @@
import json, os, sys
from collections import namedtuple
import common
from common import Name
from generator_lib import Generator, run_generator, FileRender
import wire_cmd
############################################################
# OBJECT MODEL
############################################################
class Name:
def __init__(self, name, native=False):
self.native = native
if native:
self.chunks = [name]
else:
self.chunks = name.split(' ')
def CamelChunk(self, chunk):
return chunk[0].upper() + chunk[1:]
def canonical_case(self):
return (' '.join(self.chunks)).lower()
def concatcase(self):
return ''.join(self.chunks)
def camelCase(self):
return self.chunks[0] + ''.join([self.CamelChunk(chunk) for chunk in self.chunks[1:]])
def CamelCase(self):
return ''.join([self.CamelChunk(chunk) for chunk in self.chunks])
def SNAKE_CASE(self):
return '_'.join([chunk.upper() for chunk in self.chunks])
def snake_case(self):
return '_'.join(self.chunks)
def concat_names(*names):
return ' '.join([name.canonical_case() for name in names])
class Type:
def __init__(self, name, json_data, native=False):
self.json_data = json_data
self.dict_name = name
self.name = Name(name, native=native)
self.category = json_data['category']
EnumValue = namedtuple('EnumValue', ['name', 'value'])
class EnumType(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data)
self.values = [EnumValue(Name(m['name']), m['value']) for m in self.json_data['values']]
BitmaskValue = namedtuple('BitmaskValue', ['name', 'value'])
class BitmaskType(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data)
self.values = [BitmaskValue(Name(m['name']), m['value']) for m in self.json_data['values']]
self.full_mask = 0
for value in self.values:
self.full_mask = self.full_mask | value.value
class NativeType(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data, native=True)
class NativelyDefined(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data)
# Methods and structures are both "records", so record members correspond to
# method arguments or structure members.
class RecordMember:
def __init__(self, name, typ, annotation, optional, is_return_value):
self.name = name
self.type = typ
self.annotation = annotation
self.length = None
self.optional = optional
self.is_return_value = is_return_value
self.handle_type = None
def set_handle_type(self, handle_type):
assert self.type.dict_name == "ObjectHandle"
self.handle_type = handle_type
Method = namedtuple('Method', ['name', 'return_type', 'arguments'])
class ObjectType(Type):
def __init__(self, name, json_data):
Type.__init__(self, name, json_data)
self.methods = []
self.native_methods = []
self.built_type = None
class Record:
def __init__(self, name):
self.name = Name(name)
self.members = []
self.has_dawn_object = False
def update_metadata(self):
def has_dawn_object(member):
if isinstance(member.type, ObjectType):
return True
elif isinstance(member.type, StructureType):
return member.type.has_dawn_object
else:
return False
self.has_dawn_object = any(has_dawn_object(member) for member in self.members)
class StructureType(Record, Type):
def __init__(self, name, json_data):
Record.__init__(self, name)
Type.__init__(self, name, json_data)
self.extensible = json_data.get("extensible", False)
class Command(Record):
def __init__(self, name, members=None):
Record.__init__(self, name)
self.members = members or []
self.derived_object = None
self.derived_method = None
def linked_record_members(json_data, types):
members = []
members_by_name = {}
for m in json_data:
member = RecordMember(Name(m['name']), types[m['type']],
m.get('annotation', 'value'), m.get('optional', False),
m.get('is_return_value', False))
handle_type = m.get('handle_type')
if handle_type:
member.set_handle_type(types[handle_type])
members.append(member)
members_by_name[member.name.canonical_case()] = member
for (member, m) in zip(members, json_data):
if member.annotation != 'value':
if not 'length' in m:
if member.type.category != 'object':
member.length = "constant"
member.constant_length = 1
else:
assert(False)
elif m['length'] == 'strlen':
member.length = 'strlen'
else:
member.length = members_by_name[m['length']]
return members
############################################################
# PARSE
@ -31,15 +176,15 @@ def is_native_method(method):
def link_object(obj, types):
def make_method(json_data):
arguments = common.linked_record_members(json_data.get('args', []), types)
return common.Method(Name(json_data['name']), types[json_data.get('returns', 'void')], arguments)
arguments = linked_record_members(json_data.get('args', []), types)
return Method(Name(json_data['name']), types[json_data.get('returns', 'void')], arguments)
methods = [make_method(m) for m in obj.json_data.get('methods', [])]
obj.methods = [method for method in methods if not is_native_method(method)]
obj.native_methods = [method for method in methods if is_native_method(method)]
def link_structure(struct, types):
struct.members = common.linked_record_members(struct.json_data['members'], types)
struct.members = linked_record_members(struct.json_data['members'], types)
# Sort structures so that if struct A has struct B as a member, then B is listed before A
# This is a form of topological sort where we try to keep the order reasonably similar to the
@ -79,12 +224,12 @@ def topo_sort_structure(structs):
def parse_json(json):
category_to_parser = {
'bitmask': common.BitmaskType,
'enum': common.EnumType,
'native': common.NativeType,
'natively defined': common.NativelyDefined,
'object': common.ObjectType,
'structure': common.StructureType,
'bitmask': BitmaskType,
'enum': EnumType,
'native': NativeType,
'natively defined': NativelyDefined,
'object': ObjectType,
'structure': StructureType,
}
types = {}
@ -120,6 +265,67 @@ def parse_json(json):
'by_category': by_category
}
############################################################
# WIRE STUFF
############################################################
# Create wire commands from api methods
def compute_wire_params(api_params, wire_json):
wire_params = api_params.copy()
types = wire_params['types']
commands = []
return_commands = []
# Generate commands from object methods
for api_object in wire_params['by_category']['object']:
for method in api_object.methods:
command_name = concat_names(api_object.name, method.name)
command_suffix = Name(command_name).CamelCase()
# Only object return values or void are supported. Other methods must be handwritten.
if method.return_type.category != 'object' and method.return_type.name.canonical_case() != 'void':
assert(command_suffix in wire_json['special items']['client_handwritten_commands'])
continue
if command_suffix in wire_json['special items']['client_side_commands']:
continue
# Create object method commands by prepending "self"
members = [RecordMember(Name('self'), types[api_object.dict_name], 'value', False, False)]
members += method.arguments
# Client->Server commands that return an object return the result object handle
if method.return_type.category == 'object':
result = RecordMember(Name('result'), types['ObjectHandle'], 'value', False, True)
result.set_handle_type(method.return_type)
members.append(result)
command = Command(command_name, members)
command.derived_object = api_object
command.derived_method = method
commands.append(command)
for (name, json_data) in wire_json['commands'].items():
commands.append(Command(name, linked_record_members(json_data, types)))
for (name, json_data) in wire_json['return commands'].items():
return_commands.append(Command(name, linked_record_members(json_data, types)))
wire_params['cmd_records'] = {
'command': commands,
'return command': return_commands
}
for commands in wire_params['cmd_records'].values():
for command in commands:
command.update_metadata()
commands.sort(key=lambda c: c.name.canonical_case())
wire_params.update(wire_json.get('special items', {}))
return wire_params
#############################################################
# Generator
#############################################################
@ -215,20 +421,10 @@ def cpp_native_methods(types, typ):
def c_native_methods(types, typ):
return cpp_native_methods(types, typ) + [
common.Method(Name('reference'), types['void'], []),
common.Method(Name('release'), types['void'], []),
Method(Name('reference'), types['void'], []),
Method(Name('release'), types['void'], []),
]
def js_native_methods(types, typ):
return cpp_native_methods(types, typ)
def debug(text):
print(text)
def do_assert(expr):
assert expr
return ''
class MultiGeneratorFromDawnJSON(Generator):
def get_description(self):
return 'Generates code for various target from Dawn.json.'
@ -238,7 +434,7 @@ class MultiGeneratorFromDawnJSON(Generator):
parser.add_argument('--dawn-json', required=True, type=str, help ='The DAWN JSON definition to use.')
parser.add_argument('--wire-json', default=None, type=str, help='The DAWN WIRE JSON definition to use.')
parser.add_argument('-T', '--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets))
parser.add_argument('--targets', required=True, type=str, help='Comma-separated subset of targets to output. Available targets: ' + ', '.join(allowed_targets))
def get_file_renders(self, args):
with open(args.dawn_json) as f:
@ -253,12 +449,6 @@ class MultiGeneratorFromDawnJSON(Generator):
wire_json = json.loads(f.read())
base_params = {
'enumerate': enumerate,
'format': format,
'len': len,
'debug': debug,
'assert': do_assert,
'Name': lambda name: Name(name),
'as_annotated_cType': lambda arg: annotated(as_cType(arg.type.name), arg),
@ -311,7 +501,7 @@ class MultiGeneratorFromDawnJSON(Generator):
renders.append(FileRender('dawn_native/ProcTable.cpp', 'dawn_native/ProcTable.cpp', frontend_params))
if 'dawn_wire' in targets:
additional_params = wire_cmd.compute_wire_params(api_params, wire_json)
additional_params = compute_wire_params(api_params, wire_json)
wire_params = [
base_params,

16
generator/generator_lib.py
View File

@ -96,9 +96,25 @@ def _do_renders(renders, template_dir):
loader = PreprocessingLoader(template_dir)
env = jinja2.Environment(loader=loader, lstrip_blocks=True, trim_blocks=True, line_comment_prefix='//*')
def do_assert(expr):
assert expr
return ''
def debug(text):
print(text)
base_params = {
'enumerate': enumerate,
'format': format,
'len': len,
'debug': debug,
'assert': do_assert,
}
outputs = []
for render in renders:
params = {}
params.update(base_params)
for param_dict in render.params_dicts:
params.update(param_dict)
content = env.get_template(render.template).render(**params)

77
generator/wire_cmd.py
View File

@ -1,77 +0,0 @@
# Copyright 2019 The Dawn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import namedtuple
from common import Name
import common
def concat_names(*names):
return ' '.join([name.canonical_case() for name in names])
# Create wire commands from api methods
def compute_wire_params(api_params, wire_json):
wire_params = api_params.copy()
types = wire_params['types']
commands = []
return_commands = []
# Generate commands from object methods
for api_object in wire_params['by_category']['object']:
for method in api_object.methods:
command_name = concat_names(api_object.name, method.name)
command_suffix = Name(command_name).CamelCase()
# Only object return values or void are supported. Other methods must be handwritten.
if method.return_type.category != 'object' and method.return_type.name.canonical_case() != 'void':
assert(command_suffix in wire_json['special items']['client_handwritten_commands'])
continue
if command_suffix in wire_json['special items']['client_side_commands']:
continue
# Create object method commands by prepending "self"
members = [common.RecordMember(Name('self'), types[api_object.dict_name], 'value', False, False)]
members += method.arguments
# Client->Server commands that return an object return the result object handle
if method.return_type.category == 'object':
result = common.RecordMember(Name('result'), types['ObjectHandle'], 'value', False, True)
result.set_handle_type(method.return_type)
members.append(result)
command = common.Command(command_name, members)
command.derived_object = api_object
command.derived_method = method
commands.append(command)
for (name, json_data) in wire_json['commands'].items():
commands.append(common.Command(name, common.linked_record_members(json_data, types)))
for (name, json_data) in wire_json['return commands'].items():
return_commands.append(common.Command(name, common.linked_record_members(json_data, types)))
wire_params['cmd_records'] = {
'command': commands,
'return command': return_commands
}
for commands in wire_params['cmd_records'].values():
for command in commands:
command.update_metadata()
commands.sort(key=lambda c: c.name.canonical_case())
wire_params.update(wire_json.get('special items', {}))
return wire_params

4
src/dawn/BUILD.gn
View File

@ -21,7 +21,7 @@ import("${dawn_root}/generator/dawn_generator.gni")
# Dawn headers
###############################################################################
dawn_generator("dawn_headers_gen") {
dawn_json_generator("dawn_headers_gen") {
target = "dawn_headers"
# Generate as if we were in the main BUILD.gn because that was historically
@ -51,7 +51,7 @@ source_set("dawn_headers") {
# libdawn
###############################################################################
dawn_generator("libdawn_gen") {
dawn_json_generator("libdawn_gen") {
target = "libdawn"
outputs = [
"dawn/dawncpp.cpp",