From db6ade73b2786a61da234a65c1c3e38c35066e42 Mon Sep 17 00:00:00 2001 From: Ryan Harrison Date: Thu, 5 Mar 2020 15:58:27 +0000 Subject: [PATCH] Convert dependency management to using gclient BUG=tint:12 Change-Id: I9c6b9cdd584f9a2e6c360c9a457069670146da7c Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/16482 Reviewed-by: dan sinclair --- README.md | 28 ++++- standalone.gclient | 12 ++ tools/git-sync-deps | 293 -------------------------------------------- 3 files changed, 38 insertions(+), 295 deletions(-) create mode 100644 standalone.gclient delete mode 100755 tools/git-sync-deps diff --git a/README.md b/README.md index 59bd60e1fa..10a9e69525 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,37 @@ This is not an officially supported Google product. * `TINT_BUILD_SPV_PARSER` : enable the SPIR-V input parser ## Building +Tint uses Chromium dependency management so you need to [install depot_tools] and add it to your PATH. +[install depot_tools]: http://commondatastorage.googleapis.com/chrome-infra-docs/flat/depot_tools/docs/html/depot_tools_tutorial.html#_setting_up + +### Getting source & dependencies + +```sh +# Clone the repo as "tint" +git clone https://dawn.googlesource.com/tint tint && cd tint + +# Bootstrap the gclient configuration +cp standalone.gclient .gclient + +# Fetch external dependencies and toolchains with gclient +gclient sync ``` -./tools/git-sync-deps + +### Compiling using CMake + Ninja +```sh mkdir -p out/Debug cd out/Debug cmake -GNinja ../.. -ninja +ninja # or autoninja +``` + +### Compiling using CMake + make +```sh +mkdir -p out/Debug +cd out/Debug +cmake ../.. +make # -j N for N-way parallel build ``` ## Issues diff --git a/standalone.gclient b/standalone.gclient new file mode 100644 index 0000000000..5ddec6a575 --- /dev/null +++ b/standalone.gclient @@ -0,0 +1,12 @@ +# Copy this file to /.gclient to bootstrap gclient in a +# standalone checkout of Tint. + +solutions = [ + { "name" : ".", + "url" : "https://dawn.googlesource.com/tint", + "deps_file" : "DEPS", + "managed" : False, + "custom_deps": { + }, + }, +] diff --git a/tools/git-sync-deps b/tools/git-sync-deps deleted file mode 100755 index a562f262a9..0000000000 --- a/tools/git-sync-deps +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/env python -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Parse a DEPS file and git checkout all of the dependencies. - -Args: - An optional list of deps_os values. - - --with-swiftshader Checkout Swiftshader dependencies - --with-clspv Checkout clspv dependencies - --with-dxc Checkout dxc dependencies - -Environment Variables: - GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of - ['git', 'git.exe', 'git.bat'] in your default path. - - GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, - will use the file ../DEPS relative to this script's directory. - - GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. - -Git Config: - To disable syncing of a single repository: - cd path/to/repository - git config sync-deps.disable true - - To re-enable sync: - cd path/to/repository - git config --unset sync-deps.disable -""" - -import os -import re -import subprocess -import sys -import threading -from builtins import bytes - -with_clspv = False -with_dxc = False -with_swiftshader = False - -def git_executable(): - """Find the git executable. - - Returns: - A string suitable for passing to subprocess functions, or None. - """ - envgit = os.environ.get('GIT_EXECUTABLE') - searchlist = ['git', 'git.exe', 'git.bat'] - if envgit: - searchlist.insert(0, envgit) - with open(os.devnull, 'w') as devnull: - for git in searchlist: - try: - subprocess.call([git, '--version'], stdout=devnull) - except (OSError,): - continue - return git - return None - - -DEFAULT_DEPS_PATH = os.path.normpath( - os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) - - -def usage(deps_file_path = None): - sys.stderr.write( - 'Usage: run to grab dependencies, with optional platform support:\n') - sys.stderr.write(' %s %s' % (sys.executable, __file__)) - if deps_file_path: - parsed_deps = parse_file_to_dict(deps_file_path) - if 'deps_os' in parsed_deps: - for deps_os in parsed_deps['deps_os']: - sys.stderr.write(' [%s]' % deps_os) - sys.stderr.write('\n\n') - sys.stderr.write(__doc__) - - -def git_repository_sync_is_disabled(git, directory): - try: - disable = subprocess.check_output( - [git, 'config', 'sync-deps.disable'], cwd=directory) - return disable.lower().strip() in ['true', '1', 'yes', 'on'] - except subprocess.CalledProcessError: - return False - - -def is_git_toplevel(git, directory): - """Return true iff the directory is the top level of a Git repository. - - Args: - git (string) the git executable - - directory (string) the path into which the repository - is expected to be checked out. - """ - try: - toplevel = subprocess.check_output( - [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() - return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel) - except subprocess.CalledProcessError: - return False - - -def status(directory, checkoutable): - def truncate(s, length): - return s if len(s) <= length else s[:(length - 3)] + '...' - dlen = 36 - directory = truncate(directory, dlen) - checkoutable = truncate(checkoutable, 40) - sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) - - -def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): - """Checkout (and clone if needed) a Git repository. - - Args: - git (string) the git executable - - repo (string) the location of the repository, suitable - for passing to `git clone`. - - checkoutable (string) a tag, branch, or commit, suitable for - passing to `git checkout` - - directory (string) the path into which the repository - should be checked out. - - verbose (boolean) - - Raises an exception if any calls to git fail. - """ - if verbose: - status(directory, checkoutable) - - if not os.path.isdir(directory): - subprocess.check_call( - [git, 'clone', '--quiet', repo, directory]) - - if not is_git_toplevel(git, directory): - # if the directory exists, but isn't a git repo, you will modify - # the parent repostory, which isn't what you want. - sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) - return - - # Check to see if this repo is disabled. Quick return. - if git_repository_sync_is_disabled(git, directory): - sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) - return - - with open(os.devnull, 'w') as devnull: - # If this fails, we will fetch before trying again. Don't spam user - # with error infomation. - if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable], - cwd=directory, stderr=devnull): - # if this succeeds, skip slow `git fetch`. - return - - # If the repo has changed, always force use of the correct repo. - # If origin already points to repo, this is a quick no-op. - subprocess.check_call( - [git, 'remote', 'set-url', 'origin', repo], cwd=directory) - - subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) - - subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory) - - -def parse_file_to_dict(path): - dictionary = {} - contents = open(path).read() - # Need to convert Var() to vars[], so that the DEPS is actually Python. Var() - # comes from Autoroller using gclient which has a slightly different DEPS - # format. - contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents) - exec(contents, dictionary) - return dictionary - - -def git_sync_deps(deps_file_path, command_line_os_requests, verbose): - """Grab dependencies, with optional platform support. - - Args: - deps_file_path (string) Path to the DEPS file. - - command_line_os_requests (list of strings) Can be empty list. - List of strings that should each be a key in the deps_os - dictionary in the DEPS file. - - Raises git Exceptions. - """ - git = git_executable() - assert git - - deps_file_directory = os.path.dirname(deps_file_path) - deps_file = parse_file_to_dict(deps_file_path) - dependencies = deps_file['deps'].copy() - os_specific_dependencies = deps_file.get('deps_os', dict()) - if 'all' in command_line_os_requests: - for value in os_specific_dependencies.values(): - dependencies.update(value) - else: - for os_name in command_line_os_requests: - # Add OS-specific dependencies - if os_name in os_specific_dependencies: - dependencies.update(os_specific_dependencies[os_name]) - for directory in dependencies: - for other_dir in dependencies: - if directory.startswith(other_dir + '/'): - raise Exception('%r is parent of %r' % (other_dir, directory)) - list_of_arg_lists = [] - for directory in sorted(dependencies): - if '@' in dependencies[directory]: - repo, checkoutable = dependencies[directory].split('@', 1) - else: - raise Exception("please specify commit or tag") - - if not with_clspv and directory is 'third_party/clspv': - continue - - if not with_clspv and directory is 'third_party/clspv-llvm': - continue - - if not with_dxc and directory is 'third_party/dxc': - continue - - if not with_swiftshader and directory is 'third_party/swiftshader': - continue - - relative_directory = os.path.join(deps_file_directory, directory) - - list_of_arg_lists.append( - (git, repo, checkoutable, relative_directory, verbose)) - - multithread(git_checkout_to_directory, list_of_arg_lists) - - for directory in deps_file.get('recursedeps', []): - recursive_path = os.path.join(deps_file_directory, directory, 'DEPS') - git_sync_deps(recursive_path, command_line_os_requests, verbose) - - -def multithread(function, list_of_arg_lists): - # for args in list_of_arg_lists: - # function(*args) - # return - threads = [] - for args in list_of_arg_lists: - thread = threading.Thread(None, function, None, args) - thread.start() - threads.append(thread) - for thread in threads: - thread.join() - - -def main(argv): - deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) - verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) - - if '--help' in argv or '-h' in argv: - usage(deps_file_path) - return 1 - - if '--with-clspv' in argv: - with_clspv = True - - if '--with-dxc' in argv: - with_dxc = True - - if '--with-swiftshader' in argv: - with_swiftshader = True - - git_sync_deps(deps_file_path, argv, verbose) - # subprocess.check_call( - # [sys.executable, - # os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) - return 0 - - -if __name__ == '__main__': - exit(main(sys.argv[1:]))