encounter
/
dawn-cmake
mirror of https://github.com/encounter/dawn-cmake.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Delete the remove_stale_autogen_files mechanism. 

Previously when moving around directories for generated files, Dawn ran
into an issue where stale files where #included instead of the new ones,
causing compilation failures. To get around this a
remove_stale_autogen_files mechanism was added that scans the gen/
directory for files not in an allow-list of directories.

This mechanism is now causing problems for bringing up Dawn standalone
tests on Android as these test also generate files in Dawn's gen/
directories, and their files get deleted by remove_stale_autogen_files.

We are not foresseing any additional shuffling of directories and it's
safe to expect that all stale files have been removed from CI builder
caches at this time. So remove_stale_autogen_files can go. This is what
this CL does.

Fixed: dawn:1543
Change-Id: I7dbf1eae6c55b7659f3837b6d4a565052001ce57
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/103040
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
This commit is contained in:
Corentin Wallez 2022-09-20 17:28:33 +00:00 committed by Dawn LUCI CQ
parent eebb7d5e52
commit 30f51b94da
7 changed files with 0 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
generator/BUILD.gn
View File

@ -1,63 +0,0 @@
# Copyright 2019 The Dawn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import("../scripts/dawn_overrides_with_defaults.gni")
import("dawn_generator.gni")
# The list of directories in which to check for stale autogenerated files.
# It should include the list of all directories in which we ever generated
# files but we can't just put dawn_gen_root because there are more than
# autogenerated sources there.
_stale_dirs = [
"dawn",
"dawn/native",
"dawn/wire",
"mock",
"src",
]
_allowed_output_dirs_file =
"${dawn_gen_root}/removed_stale_autogen_files.allowed_output_dirs"
write_file(_allowed_output_dirs_file, dawn_allowed_gen_output_dirs)
_stale_dirs_file = "${dawn_gen_root}/removed_stale_autogen_files.stale_dirs"
write_file(_stale_dirs_file, _stale_dirs)
_stamp_file = "${dawn_gen_root}/removed_stale_autogen_files.stamp"
# An action that removes autogenerated files that aren't in allowed directories
# see dawn_generator.gni for more details.
action("remove_stale_autogen_files") {
script = "remove_files.py"
args = [
"--root-dir",
rebase_path(dawn_gen_root, root_build_dir),
"--allowed-output-dirs-file",
rebase_path(_allowed_output_dirs_file, root_build_dir),
"--stale-dirs-file",
rebase_path(_stale_dirs_file, root_build_dir),
"--stamp",
rebase_path(_stamp_file, root_build_dir),
]
# Have the "list of file" inputs as a dependency so that the action reruns
# as soon as they change.
inputs = [
_allowed_output_dirs_file,
_stale_dirs_file,
]
# Output a stamp file so we don't re-run this action on every build.
outputs = [ _stamp_file ]
}

40
generator/dawn_generator.gni
View File

@ -15,41 +15,6 @@
import("../scripts/dawn_overrides_with_defaults.gni")
import("generator_lib.gni")
# Dawn used to put autogenerated files in a lot of different places. When we
# started to move them around, some compilation issues arised because some
# stale include files stayed in the build directory and were picked up.
# To counter this, now Dawn does the following:
#
# 1. The generated output file directory structure has to match the structure
# of the source tree, starting at dawn_gen_root (gen/ or
# gen/third_party/dawn depending on where we are).
# 2. include and dawn_gen_root/include has to match the structure of
# the source tree too.
# 3. Dawn files must use include relative to src/ or include such as
# "dawn/dawn.h" or "dawn/native/backend/BackendStuff.h".
#
# The allowed list below ensure 1). Include directory rules for Dawn ensure 3)
# and 2) is something we need to enforce in code review.
#
# However GN's toolchains automatically add some include directories for us
# which breaks 3) slightly. To avoid stale headers in for example
# dawn_gen_root/src/dawn/dawn/ to be picked up (instead of
# dawn_gen_root/src/dawn), we have a special action that removes files in
# disallowed gen directories.
dawn_allowed_gen_output_dirs = [
"src/dawn/",
"src/dawn/common/",
"src/dawn/native/",
"src/dawn/native/opengl/",
"src/dawn/wire/client/",
"src/dawn/wire/server/",
"src/dawn/wire/",
"include/dawn/",
"emscripten-bits/",
"webgpu-headers/",
]
# Template to help invoking Dawn code generators based on generator_lib
#
# dawn_generator("my_target_gen") {
@ -84,12 +49,7 @@ template("dawn_generator") {
# Force Dawn's autogenerated file structure to mirror exactly the source
# tree but start at ${dawn_gen_root} instead of ${dawn_root}
allowed_output_dirs = dawn_allowed_gen_output_dirs
custom_gen_dir = dawn_gen_root
# Make sure that we delete stale autogenerated file in directories that are
# no longer used by code generation to avoid include conflicts.
deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
}
}

18
generator/generator_lib.gni
View File

@ -37,11 +37,6 @@
#
# jinja2_path: Optional Jinja2 installation path.
#
# allowed_output_dirs: Optional list of directories that are the only
# directories in which files of `outputs` are allowed to be (and not
# in children directories). Generation will fail if an output isn't
# in a directory in the list.
#
# root_dir: Optional root source dir for Python dependencies
# computation. Defaults to "${generator_lib_dir}/..". Any dependency
# outside of this directory is considered a system file and will be
@ -118,19 +113,6 @@ template("generator_lib_action") {
rebase_path(_expected_outputs_file, root_build_dir),
]
# Check that all of the outputs are in a directory that's allowed. This is
# useful to keep the list of directories in sink with other parts of the
# build.
if (defined(invoker.allowed_output_dirs)) {
_allowed_output_dirs_file = "${_gen_dir}/${target_name}.allowed_output_dirs"
write_file(_allowed_output_dirs_file, invoker.allowed_output_dirs)
_generator_args += [
"--allowed-output-dirs-file",
rebase_path(_allowed_output_dirs_file, root_build_dir),
]
}
# The code generator invocation that will write the JSON tarball, check the
# outputs are what's expected and write a depfile for Ninja.
action(_json_tarball_target) {

32
generator/generator_lib.py
View File

@ -258,12 +258,6 @@ def run_generator(generator):
type=str,
help=('Optional source root directory for Python dependency '
'computations'))
parser.add_argument(
'--allowed-output-dirs-file',
default=None,
type=str,
help=("File containing a list of allowed directories where files "
"can be output."))
parser.add_argument(
'--print-cmake-dependencies',
default=False,
@ -326,32 +320,6 @@ def run_generator(generator):
outputs = _do_renders(renders, args.template_dir)
# The caller wants to assert that the outputs are only in specific
# directories.
if args.allowed_output_dirs_file != None:
with open(args.allowed_output_dirs_file) as f:
allowed_dirs = set([line.strip() for line in f.readlines()])
for directory in allowed_dirs:
if not directory.endswith('/'):
print('Allowed directory entry "{}" doesn\'t '
'end with /'.format(directory))
return 1
def check_in_subdirectory(path, directory):
return path.startswith(
directory) and not '/' in path[len(directory):]
for render in renders:
if not any(
check_in_subdirectory(render.output, directory)
for directory in allowed_dirs):
print('Output file "{}" is not in the allowed directory '
'list below:'.format(render.output))
for directory in sorted(allowed_dirs):
print(' "{}"'.format(directory))
return 1
# Output the JSON tarball
if args.output_json_tarball != None:
json_root = {}

91
generator/remove_files.py
View File

@ -1,91 +0,0 @@
#!/usr/bin/env python3
# Copyright 2019 The Dawn Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse, glob, os, sys
def check_in_subdirectory(path, directory):
return path.startswith(directory) and not '/' in path[len(directory):]
def check_is_allowed(path, allowed_dirs):
return any(
check_in_subdirectory(path, directory) for directory in allowed_dirs)
def get_all_files_in_dir(find_directory):
result = []
for (directory, _, files) in os.walk(find_directory):
result += [os.path.join(directory, filename) for filename in files]
return result
def run():
# Parse command line arguments
parser = argparse.ArgumentParser(
description="Removes stale autogenerated files from gen/ directories.")
parser.add_argument(
'--root-dir',
type=str,
help='The root directory, all other paths in files are relative to it.'
)
parser.add_argument(
'--allowed-output-dirs-file',
type=str,
help='The file containing a list of allowed directories')
parser.add_argument(
'--stale-dirs-file',
type=str,
help=
'The file containing a list of directories to check for stale files')
parser.add_argument('--stamp',
type=str,
help='A stamp written once this script completes')
args = parser.parse_args()
root_dir = args.root_dir
stamp_file = args.stamp
# Load the list of allowed and stale directories
with open(args.allowed_output_dirs_file) as f:
allowed_dirs = set(
[os.path.join(root_dir, line.strip()) for line in f.readlines()])
for directory in allowed_dirs:
if not directory.endswith('/'):
print('Allowed directory entry "{}" doesn\'t end with /'.format(
directory))
return 1
with open(args.stale_dirs_file) as f:
stale_dirs = set([line.strip() for line in f.readlines()])
# Remove all files in stale dirs that aren't in the allowed dirs.
for stale_dir in stale_dirs:
stale_dir = os.path.join(root_dir, stale_dir)
for candidate in get_all_files_in_dir(stale_dir):
if not check_is_allowed(candidate, allowed_dirs):
os.remove(candidate)
# Finished! Write the stamp file so ninja knows to not run this again.
with open(stamp_file, "w") as f:
f.write("")
return 0
if __name__ == "__main__":
sys.exit(run())

2
src/tint/fuzzers/BUILD.gn
View File

@ -33,8 +33,6 @@ if (build_with_chromium) {
rebase_path(fuzzer_corpus_wgsl_dir, root_build_dir),
]
outputs = [ fuzzer_corpus_wgsl_stamp ]
deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
}
tint_fuzzer_common_libfuzzer_options = [

1
src/tint/fuzzers/tint_ast_fuzzer/BUILD.gn
View File

@ -23,7 +23,6 @@ if (build_with_chromium) {
sources = [ "protobufs/tint_ast_fuzzer.proto" ]
generate_python = false
use_protobuf_full = true
deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
}
source_set("tint_ast_fuzzer") {