# Copyright (c) 2015, Google Inc. # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY # SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION # OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """Extracts archives.""" import hashlib import optparse import os import os.path import tarfile import shutil import sys import zipfile def CheckedJoin(output, path): """ CheckedJoin returns os.path.join(output, path). It checks that the resulting path is under output, but shouldn't be used on untrusted input. """ path = os.path.normpath(path) if os.path.isabs(path) or path.startswith("."): raise ValueError(path) return os.path.join(output, path) class FileEntry(object): def __init__(self, path, mode, fileobj): self.path = path self.mode = mode self.fileobj = fileobj class SymlinkEntry(object): def __init__(self, path, mode, target): self.path = path self.mode = mode self.target = target def IterateZip(path): """ IterateZip opens the zip file at path and returns a generator of entry objects for each file in it. """ with zipfile.ZipFile(path, "r") as zip_file: for info in zip_file.infolist(): if info.filename.endswith("/"): continue yield FileEntry(info.filename, None, zip_file.open(info)) def IterateTar(path, compression): """ IterateTar opens the tar.gz or tar.bz2 file at path and returns a generator of entry objects for each file in it. """ with tarfile.open(path, "r:" + compression) as tar_file: for info in tar_file: if info.isdir(): pass elif info.issym(): yield SymlinkEntry(info.name, None, info.linkname) elif info.isfile(): yield FileEntry(info.name, info.mode, tar_file.extractfile(info)) else: raise ValueError('Unknown entry type "%s"' % (info.name, )) def main(args): parser = optparse.OptionParser(usage="Usage: %prog ARCHIVE OUTPUT") parser.add_option( "--no-prefix", dest="no_prefix", action="store_true", help="Do not remove a prefix from paths in the archive.", ) options, args = parser.parse_args(args) if len(args) != 2: parser.print_help() return 1 archive, output = args if not os.path.exists(archive): # Skip archives that weren't downloaded. return 0 with open(archive, "rb") as f: sha256 = hashlib.sha256() while True: chunk = f.read(1024 * 1024) if not chunk: break sha256.update(chunk) digest = sha256.hexdigest() stamp_path = os.path.join(output, ".dawn_archive_digest") if os.path.exists(stamp_path): with open(stamp_path) as f: if f.read().strip() == digest: print("Already up-to-date.") return 0 if archive.endswith(".zip"): entries = IterateZip(archive) elif archive.endswith(".tar.gz"): entries = IterateTar(archive, "gz") elif archive.endswith(".tar.bz2"): entries = IterateTar(archive, "bz2") else: raise ValueError(archive) try: if os.path.exists(output): print("Removing %s" % (output, )) shutil.rmtree(output) print("Extracting %s to %s" % (archive, output)) prefix = None num_extracted = 0 for entry in entries: # Even on Windows, zip files must always use forward slashes. if "\\" in entry.path or entry.path.startswith("/"): raise ValueError(entry.path) if not options.no_prefix: new_prefix, rest = entry.path.split("/", 1) # Ensure the archive is consistent. if prefix is None: prefix = new_prefix if prefix != new_prefix: raise ValueError((prefix, new_prefix)) else: rest = entry.path # Extract the file into the output directory. fixed_path = CheckedJoin(output, rest) if not os.path.isdir(os.path.dirname(fixed_path)): os.makedirs(os.path.dirname(fixed_path)) if isinstance(entry, FileEntry): with open(fixed_path, "wb") as out: shutil.copyfileobj(entry.fileobj, out) elif isinstance(entry, SymlinkEntry): os.symlink(entry.target, fixed_path) else: raise TypeError("unknown entry type") # Fix up permissions if needbe. # TODO(davidben): To be extra tidy, this should only track the execute bit # as in git. if entry.mode is not None: os.chmod(fixed_path, entry.mode) # Print every 100 files, so bots do not time out on large archives. num_extracted += 1 if num_extracted % 100 == 0: print("Extracted %d files..." % (num_extracted, )) finally: entries.close() with open(stamp_path, "w") as f: f.write(digest) print("Done. Extracted %d files." % (num_extracted, )) return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))