Delete the remove_stale_autogen_files mechanism.

Previously when moving around directories for generated files, Dawn ran
into an issue where stale files where #included instead of the new ones,
causing compilation failures. To get around this a
remove_stale_autogen_files mechanism was added that scans the gen/
directory for files not in an allow-list of directories.

This mechanism is now causing problems for bringing up Dawn standalone
tests on Android as these test also generate files in Dawn's gen/
directories, and their files get deleted by remove_stale_autogen_files.

We are not foresseing any additional shuffling of directories and it's
safe to expect that all stale files have been removed from CI builder
caches at this time. So remove_stale_autogen_files can go. This is what
this CL does.

Fixed: dawn:1543
Change-Id: I7dbf1eae6c55b7659f3837b6d4a565052001ce57
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/103040
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>

generator/BUILD.gn

@@ -1,63 +0,0 @@
-# Copyright 2019 The Dawn Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import("../scripts/dawn_overrides_with_defaults.gni")
-import("dawn_generator.gni")
-
-# The list of directories in which to check for stale autogenerated files.
-# It should include the list of all directories in which we ever generated
-# files but we can't just put dawn_gen_root because there are more than
-# autogenerated sources there.
-_stale_dirs = [
-  "dawn",
-  "dawn/native",
-  "dawn/wire",
-  "mock",
-  "src",
-]
-
-_allowed_output_dirs_file =
-    "${dawn_gen_root}/removed_stale_autogen_files.allowed_output_dirs"
-write_file(_allowed_output_dirs_file, dawn_allowed_gen_output_dirs)
-
-_stale_dirs_file = "${dawn_gen_root}/removed_stale_autogen_files.stale_dirs"
-write_file(_stale_dirs_file, _stale_dirs)
-
-_stamp_file = "${dawn_gen_root}/removed_stale_autogen_files.stamp"
-
-# An action that removes autogenerated files that aren't in allowed directories
-# see dawn_generator.gni for more details.
-action("remove_stale_autogen_files") {
-  script = "remove_files.py"
-  args = [
-    "--root-dir",
-    rebase_path(dawn_gen_root, root_build_dir),
-    "--allowed-output-dirs-file",
-    rebase_path(_allowed_output_dirs_file, root_build_dir),
-    "--stale-dirs-file",
-    rebase_path(_stale_dirs_file, root_build_dir),
-    "--stamp",
-    rebase_path(_stamp_file, root_build_dir),
-  ]
-
-  # Have the "list of file" inputs as a dependency so that the action reruns
-  # as soon as they change.
-  inputs = [
-    _allowed_output_dirs_file,
-    _stale_dirs_file,
-  ]
-
-  # Output a stamp file so we don't re-run this action on every build.
-  outputs = [ _stamp_file ]
-}

generator/dawn_generator.gni

@@ -15,41 +15,6 @@
 import("../scripts/dawn_overrides_with_defaults.gni")
 import("generator_lib.gni")
 
-# Dawn used to put autogenerated files in a lot of different places. When we
-# started to move them around, some compilation issues arised because some
-# stale include files stayed in the build directory and were picked up.
-# To counter this, now Dawn does the following:
-#
-#  1. The generated output file directory structure has to match the structure
-#    of the source tree, starting at dawn_gen_root (gen/ or
-#    gen/third_party/dawn depending on where we are).
-#  2. include and dawn_gen_root/include has to match the structure of
-#    the source tree too.
-#  3. Dawn files must use include relative to src/ or include such as
-#    "dawn/dawn.h" or "dawn/native/backend/BackendStuff.h".
-#
-# The allowed list below ensure 1). Include directory rules for Dawn ensure 3)
-# and 2) is something we need to enforce in code review.
-#
-# However GN's toolchains automatically add some include directories for us
-# which breaks 3) slightly. To avoid stale headers in for example
-# dawn_gen_root/src/dawn/dawn/ to be picked up (instead of
-# dawn_gen_root/src/dawn), we have a special action that removes files in
-# disallowed gen directories.
-
-dawn_allowed_gen_output_dirs = [
-  "src/dawn/",
-  "src/dawn/common/",
-  "src/dawn/native/",
-  "src/dawn/native/opengl/",
-  "src/dawn/wire/client/",
-  "src/dawn/wire/server/",
-  "src/dawn/wire/",
-  "include/dawn/",
-  "emscripten-bits/",
-  "webgpu-headers/",
-]
-
 # Template to help invoking Dawn code generators based on generator_lib
 #
 #   dawn_generator("my_target_gen") {
@@ -84,12 +49,7 @@ template("dawn_generator") {
 
     # Force Dawn's autogenerated file structure to mirror exactly the source
     # tree but start at ${dawn_gen_root} instead of ${dawn_root}
-    allowed_output_dirs = dawn_allowed_gen_output_dirs
     custom_gen_dir = dawn_gen_root
-
-    # Make sure that we delete stale autogenerated file in directories that are
-    # no longer used by code generation to avoid include conflicts.
-    deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
   }
 }
 

generator/generator_lib.gni

@@ -37,11 +37,6 @@
 #
 #   jinja2_path: Optional Jinja2 installation path.
 #
-#   allowed_output_dirs: Optional list of directories that are the only
-#     directories in which files of `outputs` are allowed to be (and not
-#     in children directories). Generation will fail if an output isn't
-#     in a directory in the list.
-#
 #   root_dir: Optional root source dir for Python dependencies
 #     computation. Defaults to "${generator_lib_dir}/..". Any dependency
 #     outside of this directory is considered a system file and will be
@@ -118,19 +113,6 @@ template("generator_lib_action") {
     rebase_path(_expected_outputs_file, root_build_dir),
   ]
 
-  # Check that all of the outputs are in a directory that's allowed. This is
-  # useful to keep the list of directories in sink with other parts of the
-  # build.
-  if (defined(invoker.allowed_output_dirs)) {
-    _allowed_output_dirs_file = "${_gen_dir}/${target_name}.allowed_output_dirs"
-    write_file(_allowed_output_dirs_file, invoker.allowed_output_dirs)
-
-    _generator_args += [
-      "--allowed-output-dirs-file",
-      rebase_path(_allowed_output_dirs_file, root_build_dir),
-    ]
-  }
-
   # The code generator invocation that will write the JSON tarball, check the
   # outputs are what's expected and write a depfile for Ninja.
   action(_json_tarball_target) {

generator/generator_lib.py

@@ -258,12 +258,6 @@ def run_generator(generator):
         type=str,
         help=('Optional source root directory for Python dependency '
               'computations'))
-    parser.add_argument(
-        '--allowed-output-dirs-file',
-        default=None,
-        type=str,
-        help=("File containing a list of allowed directories where files "
-              "can be output."))
     parser.add_argument(
         '--print-cmake-dependencies',
         default=False,
@@ -326,32 +320,6 @@ def run_generator(generator):
 
     outputs = _do_renders(renders, args.template_dir)
 
-    # The caller wants to assert that the outputs are only in specific
-    # directories.
-    if args.allowed_output_dirs_file != None:
-        with open(args.allowed_output_dirs_file) as f:
-            allowed_dirs = set([line.strip() for line in f.readlines()])
-
-        for directory in allowed_dirs:
-            if not directory.endswith('/'):
-                print('Allowed directory entry "{}" doesn\'t '
-                      'end with /'.format(directory))
-                return 1
-
-        def check_in_subdirectory(path, directory):
-            return path.startswith(
-                directory) and not '/' in path[len(directory):]
-
-        for render in renders:
-            if not any(
-                    check_in_subdirectory(render.output, directory)
-                    for directory in allowed_dirs):
-                print('Output file "{}" is not in the allowed directory '
-                      'list below:'.format(render.output))
-                for directory in sorted(allowed_dirs):
-                    print('    "{}"'.format(directory))
-                return 1
-
     # Output the JSON tarball
     if args.output_json_tarball != None:
         json_root = {}

generator/remove_files.py

@@ -1,91 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2019 The Dawn Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse, glob, os, sys
-
-
-def check_in_subdirectory(path, directory):
-    return path.startswith(directory) and not '/' in path[len(directory):]
-
-
-def check_is_allowed(path, allowed_dirs):
-    return any(
-        check_in_subdirectory(path, directory) for directory in allowed_dirs)
-
-
-def get_all_files_in_dir(find_directory):
-    result = []
-    for (directory, _, files) in os.walk(find_directory):
-        result += [os.path.join(directory, filename) for filename in files]
-    return result
-
-
-def run():
-    # Parse command line arguments
-    parser = argparse.ArgumentParser(
-        description="Removes stale autogenerated files from gen/ directories.")
-    parser.add_argument(
-        '--root-dir',
-        type=str,
-        help='The root directory, all other paths in files are relative to it.'
-    )
-    parser.add_argument(
-        '--allowed-output-dirs-file',
-        type=str,
-        help='The file containing a list of allowed directories')
-    parser.add_argument(
-        '--stale-dirs-file',
-        type=str,
-        help=
-        'The file containing a list of directories to check for stale files')
-    parser.add_argument('--stamp',
-                        type=str,
-                        help='A stamp written once this script completes')
-    args = parser.parse_args()
-
-    root_dir = args.root_dir
-    stamp_file = args.stamp
-
-    # Load the list of allowed and stale directories
-    with open(args.allowed_output_dirs_file) as f:
-        allowed_dirs = set(
-            [os.path.join(root_dir, line.strip()) for line in f.readlines()])
-
-    for directory in allowed_dirs:
-        if not directory.endswith('/'):
-            print('Allowed directory entry "{}" doesn\'t end with /'.format(
-                directory))
-            return 1
-
-    with open(args.stale_dirs_file) as f:
-        stale_dirs = set([line.strip() for line in f.readlines()])
-
-    # Remove all files in stale dirs that aren't in the allowed dirs.
-    for stale_dir in stale_dirs:
-        stale_dir = os.path.join(root_dir, stale_dir)
-
-        for candidate in get_all_files_in_dir(stale_dir):
-            if not check_is_allowed(candidate, allowed_dirs):
-                os.remove(candidate)
-
-    # Finished! Write the stamp file so ninja knows to not run this again.
-    with open(stamp_file, "w") as f:
-        f.write("")
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(run())

src/tint/fuzzers/BUILD.gn

@@ -33,8 +33,6 @@ if (build_with_chromium) {
       rebase_path(fuzzer_corpus_wgsl_dir, root_build_dir),
     ]
     outputs = [ fuzzer_corpus_wgsl_stamp ]
-
-    deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
   }
 
   tint_fuzzer_common_libfuzzer_options = [

src/tint/fuzzers/tint_ast_fuzzer/BUILD.gn

@@ -23,7 +23,6 @@ if (build_with_chromium) {
     sources = [ "protobufs/tint_ast_fuzzer.proto" ]
     generate_python = false
     use_protobuf_full = true
-    deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
   }
 
   source_set("tint_ast_fuzzer") {