Convert dependency management to using gclient
BUG=tint:12 Change-Id: I9c6b9cdd584f9a2e6c360c9a457069670146da7c Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/16482 Reviewed-by: dan sinclair <dsinclair@google.com>
This commit is contained in:
Ryan Harrison
dan sinclair
parent
707705ddf4
commit
db6ade73b2
28
README.md
28
README.md
|
|
@ -14,13 +14,37 @@ This is not an officially supported Google product.
|
||||||
* `TINT_BUILD_SPV_PARSER` : enable the SPIR-V input parser
|
* `TINT_BUILD_SPV_PARSER` : enable the SPIR-V input parser
|
||||||
|
|
||||||
## Building
|
## Building
|
||||||
|
Tint uses Chromium dependency management so you need to [install depot_tools] and add it to your PATH.
|
||||||
|
|
||||||
|
[install depot_tools]: http://commondatastorage.googleapis.com/chrome-infra-docs/flat/depot_tools/docs/html/depot_tools_tutorial.html#_setting_up
|
||||||
|
|
||||||
|
### Getting source & dependencies
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Clone the repo as "tint"
|
||||||
|
git clone https://dawn.googlesource.com/tint tint && cd tint
|
||||||
|
|
||||||
|
# Bootstrap the gclient configuration
|
||||||
|
cp standalone.gclient .gclient
|
||||||
|
|
||||||
|
# Fetch external dependencies and toolchains with gclient
|
||||||
|
gclient sync
|
||||||
```
|
```
|
||||||
./tools/git-sync-deps
|
|
||||||
|
### Compiling using CMake + Ninja
|
||||||
|
```sh
|
||||||
mkdir -p out/Debug
|
mkdir -p out/Debug
|
||||||
cd out/Debug
|
cd out/Debug
|
||||||
cmake -GNinja ../..
|
cmake -GNinja ../..
|
||||||
ninja
|
ninja # or autoninja
|
||||||
|
```
|
||||||
|
|
||||||
|
### Compiling using CMake + make
|
||||||
|
```sh
|
||||||
|
mkdir -p out/Debug
|
||||||
|
cd out/Debug
|
||||||
|
cmake ../..
|
||||||
|
make # -j N for N-way parallel build
|
||||||
```
|
```
|
||||||
|
|
||||||
## Issues
|
## Issues
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
# Copy this file to <tint clone dir>/.gclient to bootstrap gclient in a
|
||||||
|
# standalone checkout of Tint.
|
||||||
|
|
||||||
|
solutions = [
|
||||||
|
{ "name" : ".",
|
||||||
|
"url" : "https://dawn.googlesource.com/tint",
|
||||||
|
"deps_file" : "DEPS",
|
||||||
|
"managed" : False,
|
||||||
|
"custom_deps": {
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
@ -1,293 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# Copyright 2019 Google LLC
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
"""Parse a DEPS file and git checkout all of the dependencies.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
An optional list of deps_os values.
|
|
||||||
|
|
||||||
--with-swiftshader Checkout Swiftshader dependencies
|
|
||||||
--with-clspv Checkout clspv dependencies
|
|
||||||
--with-dxc Checkout dxc dependencies
|
|
||||||
|
|
||||||
Environment Variables:
|
|
||||||
GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of
|
|
||||||
['git', 'git.exe', 'git.bat'] in your default path.
|
|
||||||
|
|
||||||
GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
|
|
||||||
will use the file ../DEPS relative to this script's directory.
|
|
||||||
|
|
||||||
GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
|
|
||||||
|
|
||||||
Git Config:
|
|
||||||
To disable syncing of a single repository:
|
|
||||||
cd path/to/repository
|
|
||||||
git config sync-deps.disable true
|
|
||||||
|
|
||||||
To re-enable sync:
|
|
||||||
cd path/to/repository
|
|
||||||
git config --unset sync-deps.disable
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import threading
|
|
||||||
from builtins import bytes
|
|
||||||
|
|
||||||
with_clspv = False
|
|
||||||
with_dxc = False
|
|
||||||
with_swiftshader = False
|
|
||||||
|
|
||||||
def git_executable():
|
|
||||||
"""Find the git executable.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
A string suitable for passing to subprocess functions, or None.
|
|
||||||
"""
|
|
||||||
envgit = os.environ.get('GIT_EXECUTABLE')
|
|
||||||
searchlist = ['git', 'git.exe', 'git.bat']
|
|
||||||
if envgit:
|
|
||||||
searchlist.insert(0, envgit)
|
|
||||||
with open(os.devnull, 'w') as devnull:
|
|
||||||
for git in searchlist:
|
|
||||||
try:
|
|
||||||
subprocess.call([git, '--version'], stdout=devnull)
|
|
||||||
except (OSError,):
|
|
||||||
continue
|
|
||||||
return git
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_DEPS_PATH = os.path.normpath(
|
|
||||||
os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
|
|
||||||
|
|
||||||
|
|
||||||
def usage(deps_file_path = None):
|
|
||||||
sys.stderr.write(
|
|
||||||
'Usage: run to grab dependencies, with optional platform support:\n')
|
|
||||||
sys.stderr.write(' %s %s' % (sys.executable, __file__))
|
|
||||||
if deps_file_path:
|
|
||||||
parsed_deps = parse_file_to_dict(deps_file_path)
|
|
||||||
if 'deps_os' in parsed_deps:
|
|
||||||
for deps_os in parsed_deps['deps_os']:
|
|
||||||
sys.stderr.write(' [%s]' % deps_os)
|
|
||||||
sys.stderr.write('\n\n')
|
|
||||||
sys.stderr.write(__doc__)
|
|
||||||
|
|
||||||
|
|
||||||
def git_repository_sync_is_disabled(git, directory):
|
|
||||||
try:
|
|
||||||
disable = subprocess.check_output(
|
|
||||||
[git, 'config', 'sync-deps.disable'], cwd=directory)
|
|
||||||
return disable.lower().strip() in ['true', '1', 'yes', 'on']
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def is_git_toplevel(git, directory):
|
|
||||||
"""Return true iff the directory is the top level of a Git repository.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
git (string) the git executable
|
|
||||||
|
|
||||||
directory (string) the path into which the repository
|
|
||||||
is expected to be checked out.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
toplevel = subprocess.check_output(
|
|
||||||
[git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
|
|
||||||
return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel)
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def status(directory, checkoutable):
|
|
||||||
def truncate(s, length):
|
|
||||||
return s if len(s) <= length else s[:(length - 3)] + '...'
|
|
||||||
dlen = 36
|
|
||||||
directory = truncate(directory, dlen)
|
|
||||||
checkoutable = truncate(checkoutable, 40)
|
|
||||||
sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable))
|
|
||||||
|
|
||||||
|
|
||||||
def git_checkout_to_directory(git, repo, checkoutable, directory, verbose):
|
|
||||||
"""Checkout (and clone if needed) a Git repository.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
git (string) the git executable
|
|
||||||
|
|
||||||
repo (string) the location of the repository, suitable
|
|
||||||
for passing to `git clone`.
|
|
||||||
|
|
||||||
checkoutable (string) a tag, branch, or commit, suitable for
|
|
||||||
passing to `git checkout`
|
|
||||||
|
|
||||||
directory (string) the path into which the repository
|
|
||||||
should be checked out.
|
|
||||||
|
|
||||||
verbose (boolean)
|
|
||||||
|
|
||||||
Raises an exception if any calls to git fail.
|
|
||||||
"""
|
|
||||||
if verbose:
|
|
||||||
status(directory, checkoutable)
|
|
||||||
|
|
||||||
if not os.path.isdir(directory):
|
|
||||||
subprocess.check_call(
|
|
||||||
[git, 'clone', '--quiet', repo, directory])
|
|
||||||
|
|
||||||
if not is_git_toplevel(git, directory):
|
|
||||||
# if the directory exists, but isn't a git repo, you will modify
|
|
||||||
# the parent repostory, which isn't what you want.
|
|
||||||
sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Check to see if this repo is disabled. Quick return.
|
|
||||||
if git_repository_sync_is_disabled(git, directory):
|
|
||||||
sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory)
|
|
||||||
return
|
|
||||||
|
|
||||||
with open(os.devnull, 'w') as devnull:
|
|
||||||
# If this fails, we will fetch before trying again. Don't spam user
|
|
||||||
# with error infomation.
|
|
||||||
if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable],
|
|
||||||
cwd=directory, stderr=devnull):
|
|
||||||
# if this succeeds, skip slow `git fetch`.
|
|
||||||
return
|
|
||||||
|
|
||||||
# If the repo has changed, always force use of the correct repo.
|
|
||||||
# If origin already points to repo, this is a quick no-op.
|
|
||||||
subprocess.check_call(
|
|
||||||
[git, 'remote', 'set-url', 'origin', repo], cwd=directory)
|
|
||||||
|
|
||||||
subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
|
|
||||||
|
|
||||||
subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_file_to_dict(path):
|
|
||||||
dictionary = {}
|
|
||||||
contents = open(path).read()
|
|
||||||
# Need to convert Var() to vars[], so that the DEPS is actually Python. Var()
|
|
||||||
# comes from Autoroller using gclient which has a slightly different DEPS
|
|
||||||
# format.
|
|
||||||
contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents)
|
|
||||||
exec(contents, dictionary)
|
|
||||||
return dictionary
|
|
||||||
|
|
||||||
|
|
||||||
def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
|
|
||||||
"""Grab dependencies, with optional platform support.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
deps_file_path (string) Path to the DEPS file.
|
|
||||||
|
|
||||||
command_line_os_requests (list of strings) Can be empty list.
|
|
||||||
List of strings that should each be a key in the deps_os
|
|
||||||
dictionary in the DEPS file.
|
|
||||||
|
|
||||||
Raises git Exceptions.
|
|
||||||
"""
|
|
||||||
git = git_executable()
|
|
||||||
assert git
|
|
||||||
|
|
||||||
deps_file_directory = os.path.dirname(deps_file_path)
|
|
||||||
deps_file = parse_file_to_dict(deps_file_path)
|
|
||||||
dependencies = deps_file['deps'].copy()
|
|
||||||
os_specific_dependencies = deps_file.get('deps_os', dict())
|
|
||||||
if 'all' in command_line_os_requests:
|
|
||||||
for value in os_specific_dependencies.values():
|
|
||||||
dependencies.update(value)
|
|
||||||
else:
|
|
||||||
for os_name in command_line_os_requests:
|
|
||||||
# Add OS-specific dependencies
|
|
||||||
if os_name in os_specific_dependencies:
|
|
||||||
dependencies.update(os_specific_dependencies[os_name])
|
|
||||||
for directory in dependencies:
|
|
||||||
for other_dir in dependencies:
|
|
||||||
if directory.startswith(other_dir + '/'):
|
|
||||||
raise Exception('%r is parent of %r' % (other_dir, directory))
|
|
||||||
list_of_arg_lists = []
|
|
||||||
for directory in sorted(dependencies):
|
|
||||||
if '@' in dependencies[directory]:
|
|
||||||
repo, checkoutable = dependencies[directory].split('@', 1)
|
|
||||||
else:
|
|
||||||
raise Exception("please specify commit or tag")
|
|
||||||
|
|
||||||
if not with_clspv and directory is 'third_party/clspv':
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not with_clspv and directory is 'third_party/clspv-llvm':
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not with_dxc and directory is 'third_party/dxc':
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not with_swiftshader and directory is 'third_party/swiftshader':
|
|
||||||
continue
|
|
||||||
|
|
||||||
relative_directory = os.path.join(deps_file_directory, directory)
|
|
||||||
|
|
||||||
list_of_arg_lists.append(
|
|
||||||
(git, repo, checkoutable, relative_directory, verbose))
|
|
||||||
|
|
||||||
multithread(git_checkout_to_directory, list_of_arg_lists)
|
|
||||||
|
|
||||||
for directory in deps_file.get('recursedeps', []):
|
|
||||||
recursive_path = os.path.join(deps_file_directory, directory, 'DEPS')
|
|
||||||
git_sync_deps(recursive_path, command_line_os_requests, verbose)
|
|
||||||
|
|
||||||
|
|
||||||
def multithread(function, list_of_arg_lists):
|
|
||||||
# for args in list_of_arg_lists:
|
|
||||||
# function(*args)
|
|
||||||
# return
|
|
||||||
threads = []
|
|
||||||
for args in list_of_arg_lists:
|
|
||||||
thread = threading.Thread(None, function, None, args)
|
|
||||||
thread.start()
|
|
||||||
threads.append(thread)
|
|
||||||
for thread in threads:
|
|
||||||
thread.join()
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv):
|
|
||||||
deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
|
|
||||||
verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
|
|
||||||
|
|
||||||
if '--help' in argv or '-h' in argv:
|
|
||||||
usage(deps_file_path)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
if '--with-clspv' in argv:
|
|
||||||
with_clspv = True
|
|
||||||
|
|
||||||
if '--with-dxc' in argv:
|
|
||||||
with_dxc = True
|
|
||||||
|
|
||||||
if '--with-swiftshader' in argv:
|
|
||||||
with_swiftshader = True
|
|
||||||
|
|
||||||
git_sync_deps(deps_file_path, argv, verbose)
|
|
||||||
# subprocess.check_call(
|
|
||||||
# [sys.executable,
|
|
||||||
# os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
exit(main(sys.argv[1:]))
|
|
||||||
Loading…
Reference in New Issue