From 28f16a7cfcb7d7a2874ccd740dd555bacdd19b29 Mon Sep 17 00:00:00 2001 From: Luke Street Date: Sun, 23 Jun 2024 22:34:51 -0600 Subject: [PATCH] Sync with latest dtk-template --- .github/workflows/build.yml | 38 +- .vscode/settings.json | 3 +- configure.py | 104 +++-- ldscript.lcf | 44 -- tools/decompctx.py | 85 ++-- tools/download_tool.py | 43 +- tools/ninja_syntax.py | 145 ++++--- tools/project.py | 793 +++++++++++++++++++++++------------- tools/transform_dep.py | 6 +- tools/upload_progress.py | 15 +- 10 files changed, 790 insertions(+), 486 deletions(-) delete mode 100644 ldscript.lcf mode change 100644 => 100755 tools/transform_dep.py mode change 100644 => 100755 tools/upload_progress.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c540bba..8d704d3f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,36 +6,52 @@ on: jobs: build: - runs-on: ubuntu-latest container: ghcr.io/primedecomp/build:main + + runs-on: ubuntu-latest strategy: fail-fast: false matrix: version: [GM8E01_00] # GM8E01_01, GM8E01_48 + steps: + # Checkout the repository (shallow clone) - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 submodules: recursive + + # Set Git config - name: Git config run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + # Copy the original files to the workspace - name: Prepare run: cp -R /orig . + + # Build the project - name: Build run: | - python configure.py --map --version ${{matrix.version}} --compilers /compilers - ninja all_source build/${{matrix.version}}/progress.json + python configure.py --map --version ${{ matrix.version }} \ + --binutils /binutils --compilers /compilers + ninja all_source build/${{ matrix.version }}/progress.json + + # Upload progress if we're on the main branch - name: Upload progress - if: github.ref == 'refs/heads/main' && matrix.version == 'GM8E01_00' + if: github.ref == 'refs/heads/main' continue-on-error: true env: - PROGRESS_API_KEY: ${{secrets.PROGRESS_API_KEY}} + PROGRESS_SLUG: prime + PROGRESS_API_KEY: ${{ secrets.PROGRESS_API_KEY }} run: | - python tools/upload_progress.py -b https://progress.decomp.club/ -p prime -v ${{matrix.version}} \ - build/${{matrix.version}}/progress.json + python tools/upload_progress.py -b https://progress.decomp.club/ \ + -p $PROGRESS_SLUG -v ${{ matrix.version }} \ + build/${{ matrix.version }}/progress.json + + # Upload map files - name: Upload map - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: ${{matrix.version}}_maps - path: build/${{matrix.version}}/**/*.MAP + name: ${{ matrix.version }}_maps + path: build/${{ matrix.version }}/**/*.MAP diff --git a/.vscode/settings.json b/.vscode/settings.json index 1937059d..aae06086 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -48,5 +48,6 @@ ], "C/C++ Include Guard.Auto Update Path Blocklist": [ "include/zlib" - ] + ], + "cmake.configureOnOpen": false } diff --git a/configure.py b/configure.py index 4c9d421b..f3215f58 100755 --- a/configure.py +++ b/configure.py @@ -12,10 +12,11 @@ # Append --help to see available options. ### -import sys import argparse - +import sys from pathlib import Path +from typing import Any, Dict, List + from tools.project import ( Object, ProjectConfig, @@ -38,104 +39,127 @@ VERSIONS = [ # "R3MP01_00", # mp-v3.629 Trilogy PAL ] -if len(VERSIONS) > 1: - versions_str = ", ".join(VERSIONS[:-1]) + f" or {VERSIONS[-1]}" -else: - versions_str = VERSIONS[0] - parser = argparse.ArgumentParser() parser.add_argument( "mode", + choices=["configure", "progress"], default="configure", - help="configure or progress (default: configure)", + help="script mode (default: configure)", nargs="?", ) parser.add_argument( + "-v", "--version", - dest="version", + choices=VERSIONS, + type=str.upper, default=VERSIONS[DEFAULT_VERSION], - help=f"version to build ({versions_str})", + help="version to build", ) parser.add_argument( "--build-dir", - dest="build_dir", + metavar="DIR", type=Path, default=Path("build"), help="base build directory (default: build)", ) +parser.add_argument( + "--binutils", + metavar="BINARY", + type=Path, + help="path to binutils (optional)", +) parser.add_argument( "--compilers", - dest="compilers", + metavar="DIR", type=Path, help="path to compilers (optional)", ) parser.add_argument( "--map", - dest="map", action="store_true", help="generate map file(s)", ) +parser.add_argument( + "--no-asm", + action="store_true", + help="don't incorporate .s files from asm directory", +) parser.add_argument( "--debug", - dest="debug", action="store_true", help="build with debug info (non-matching)", ) if not is_windows(): parser.add_argument( "--wrapper", - dest="wrapper", + metavar="BINARY", type=Path, help="path to wibo or wine (optional)", ) parser.add_argument( - "--build-dtk", - dest="build_dtk", + "--dtk", + metavar="BINARY | DIR", type=Path, - help="path to decomp-toolkit source (optional)", + help="path to decomp-toolkit binary or source (optional)", ) parser.add_argument( "--sjiswrap", - dest="sjiswrap", + metavar="EXE", type=Path, help="path to sjiswrap.exe (optional)", ) parser.add_argument( "--verbose", - dest="verbose", action="store_true", help="print verbose output", ) +parser.add_argument( + "--non-matching", + dest="non_matching", + action="store_true", + help="builds equivalent (but non-matching) or modded objects", +) args = parser.parse_args() config = ProjectConfig() -config.version = args.version -if config.version not in VERSIONS: - sys.exit(f"Invalid version '{config.version}', expected {versions_str}") +config.version = str(args.version) version_num = VERSIONS.index(config.version) # Apply arguments config.build_dir = args.build_dir -config.build_dtk_path = args.build_dtk +config.dtk_path = args.dtk +config.binutils_path = args.binutils config.compilers_path = args.compilers config.debug = args.debug config.generate_map = args.map +config.non_matching = args.non_matching config.sjiswrap_path = args.sjiswrap if not is_windows(): config.wrapper = args.wrapper +if args.no_asm: + config.asm_dir = None # Tool versions +config.binutils_tag = "2.42-1" config.compilers_tag = "20231018" -config.dtk_tag = "v0.7.6" +config.dtk_tag = "v0.9.2" config.sjiswrap_tag = "v1.1.1" -config.wibo_tag = "0.6.9" +config.wibo_tag = "0.6.11" # Project config.config_path = Path("config") / config.version / "config.yml" config.check_sha_path = Path("config") / config.version / "build.sha1" +config.asflags = [ + "-mgekko", + "--strip-local-absolute", + "-I include", + f"-I build/{config.version}/include", + f"--defsym version={version_num}", +] config.ldflags = [ "-fp hardware", "-nodefaults", + "-warn off", ] config.progress_all = False @@ -151,24 +175,29 @@ config.build_rels = False # Base flags, common to most GC/Wii games. # Generally leave untouched, with overrides added below. cflags_base = [ - "-proc gekko", "-nodefaults", - "-Cpp_exceptions off", - "-RTTI off", - "-fp hard", - "-fp_contract on", - "-O4,p", - "-maxerrors 1", + "-proc gekko", + "-align powerpc", "-enum int", + "-fp hardware", + "-Cpp_exceptions off", + # "-W all", + "-O4,p", "-inline auto", - "-str reuse", + '-pragma "cats off"', + '-pragma "warn_notinlined off"', + "-maxerrors 1", "-nosyspath", + "-RTTI off", + "-fp_contract on", + "-str reuse", + "-multibyte", "-i include", "-i extern/musyx/include", "-i libc", f"-i build/{config.version}/include", - "-DPRIME1", f"-DVERSION={version_num}", + "-DPRIME1", "-DNONMATCHING=0", ] @@ -298,8 +327,9 @@ def Rel(lib_name, objects): } -Matching = True -NonMatching = False +Matching = True # Object matches and should be linked +NonMatching = False # Object does not match and should not be linked +Equivalent = config.non_matching # Object should be linked when configured with --non-matching config.warn_missing_config = True config.warn_missing_source = False diff --git a/ldscript.lcf b/ldscript.lcf deleted file mode 100644 index 70870cf0..00000000 --- a/ldscript.lcf +++ /dev/null @@ -1,44 +0,0 @@ -MEMORY -{ - text : origin = 0x80003100 -} - -SECTIONS -{ - GROUP: - { - .init ALIGN(0x20):{} - extab ALIGN(0x20):{} - extabindex ALIGN(0x20):{} - .text ALIGN(0x20):{TRK_MINNOW_DOLPHIN.a} - .ctors ALIGN(0x20):{} - .dtors ALIGN(0x20):{} - .rodata ALIGN(0x20):{} - .data ALIGN(0x20):{} - .bss ALIGN(0x20):{} - .sdata ALIGN(0x20):{} - .sbss ALIGN(0x20):{} - .sdata2 ALIGN(0x20):{} - .sbss2 ALIGN(0x20):{} - .stack ALIGN(0x100):{} - } > text - - /* Stack size upped from the default of 65535 */ - _stack_addr = (_f_sbss2 + SIZEOF(.sbss2) + 65535 + 3073 /* Retro Addition */ + 0x7) & ~0x7; - _stack_end = _f_sbss2 + SIZEOF(.sbss2); - _db_stack_addr = (_stack_addr + 0x2000); - _db_stack_end = _stack_addr; - __ArenaLo = (_db_stack_addr + 0x1f) & ~0x1f; - __ArenaHi = 0x81700000; -} - -FORCEFILES -{ -} - -FORCEACTIVE -{ - OSInitMessageQueue - OSSendMessage - OSReceiveMessage -} diff --git a/tools/decompctx.py b/tools/decompctx.py index d2dc5224..d4aeb8b9 100755 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -13,6 +13,7 @@ import argparse import os import re +from typing import List script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.abspath(os.path.join(script_dir, "..")) @@ -24,54 +25,63 @@ include_dirs = [ ] include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$') -guard_pattern = re.compile(r'^#ifndef\s+(.*)$') +guard_pattern = re.compile(r"^#ifndef\s+(.*)$") defines = set() -def import_h_file(in_file: str, r_path: str) -> str: + +def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str: rel_path = os.path.join(root_dir, r_path, in_file) if os.path.exists(rel_path): - return import_c_file(rel_path) + return import_c_file(rel_path, deps) for include_dir in include_dirs: inc_path = os.path.join(include_dir, in_file) if os.path.exists(inc_path): - return import_c_file(inc_path) + return import_c_file(inc_path, deps) else: print("Failed to locate", in_file) return "" -def import_c_file(in_file) -> str: + +def import_c_file(in_file: str, deps: List[str]) -> str: in_file = os.path.relpath(in_file, root_dir) - out_text = '' + deps.append(in_file) + out_text = "" try: - with open(in_file, encoding="utf-8") as file: - out_text += process_file(in_file, list(file)) + with open(in_file, encoding="utf-8") as file: + out_text += process_file(in_file, list(file), deps) except Exception: - with open(in_file) as file: - out_text += process_file(in_file, list(file)) + with open(in_file) as file: + out_text += process_file(in_file, list(file), deps) return out_text -def process_file(in_file: str, lines) -> str: - out_text = '' + +def process_file(in_file: str, lines: List[str], deps: List[str]) -> str: + out_text = "" for idx, line in enumerate(lines): - guard_match = guard_pattern.match(line.strip()) - if idx == 0: - if guard_match: - if guard_match[1] in defines: - break - defines.add(guard_match[1]) - print("Processing file", in_file) - include_match = include_pattern.match(line.strip()) - if include_match and not include_match[1].endswith(".s"): - out_text += f"/* \"{in_file}\" line {idx} \"{include_match[1]}\" */\n" - out_text += import_h_file(include_match[1], os.path.dirname(in_file)) - out_text += f"/* end \"{include_match[1]}\" */\n" - else: - out_text += line + guard_match = guard_pattern.match(line.strip()) + if idx == 0: + if guard_match: + if guard_match[1] in defines: + break + defines.add(guard_match[1]) + print("Processing file", in_file) + include_match = include_pattern.match(line.strip()) + if include_match and not include_match[1].endswith(".s"): + out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n' + out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps) + out_text += f'/* end "{include_match[1]}" */\n' + else: + out_text += line return out_text + +def sanitize_path(path: str) -> str: + return path.replace("\\", "/").replace(" ", "\\ ") + + def main(): parser = argparse.ArgumentParser( description="""Create a context file which can be used for decomp.me""" @@ -80,13 +90,32 @@ def main(): "c_file", help="""File from which to create context""", ) + parser.add_argument( + "-o", + "--output", + help="""Output file""", + default="ctx.c", + ) + parser.add_argument( + "-d", + "--depfile", + help="""Dependency file""", + ) args = parser.parse_args() - output = import_c_file(args.c_file) + deps = [] + output = import_c_file(args.c_file, deps) - with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f: + with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f: f.write(output) + if args.depfile: + with open(os.path.join(root_dir, args.depfile), "w", encoding="utf-8") as f: + f.write(sanitize_path(args.output) + ":") + for dep in deps: + path = sanitize_path(dep) + f.write(f" \\\n\t{path}") + if __name__ == "__main__": main() diff --git a/tools/download_tool.py b/tools/download_tool.py index fef42d6e..7b386a4b 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -18,11 +18,29 @@ import shutil import stat import urllib.request import zipfile - +from typing import Callable, Dict from pathlib import Path -def dtk_url(tag): +def binutils_url(tag): + uname = platform.uname() + system = uname.system.lower() + arch = uname.machine.lower() + if system == "darwin": + system = "macos" + arch = "universal" + elif arch == "amd64": + arch = "x86_64" + + repo = "https://github.com/encounter/gc-wii-binutils" + return f"{repo}/releases/download/{tag}/{system}-{arch}.zip" + + +def compilers_url(tag: str) -> str: + return f"https://files.decomp.dev/compilers_{tag}.zip" + + +def dtk_url(tag: str) -> str: uname = platform.uname() suffix = "" system = uname.system.lower() @@ -38,29 +56,26 @@ def dtk_url(tag): return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" -def sjiswrap_url(tag): +def sjiswrap_url(tag: str) -> str: repo = "https://github.com/encounter/sjiswrap" return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe" -def wibo_url(tag): +def wibo_url(tag: str) -> str: repo = "https://github.com/decompals/wibo" return f"{repo}/releases/download/{tag}/wibo" -def compilers_url(tag): - return f"https://files.decomp.dev/compilers_{tag}.zip" - - -TOOLS = { +TOOLS: Dict[str, Callable[[str], str]] = { + "binutils": binutils_url, + "compilers": compilers_url, "dtk": dtk_url, "sjiswrap": sjiswrap_url, "wibo": wibo_url, - "compilers": compilers_url, } -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("tool", help="Tool name") parser.add_argument("output", type=Path, help="output file path") @@ -77,7 +92,11 @@ def main(): data = io.BytesIO(response.read()) with zipfile.ZipFile(data) as f: f.extractall(output) - output.touch(mode=0o755) + # Make all files executable + for root, _, files in os.walk(output): + for name in files: + os.chmod(os.path.join(root, name), 0o755) + output.touch(mode=0o755) # Update dir modtime else: with open(output, "wb") as f: shutil.copyfileobj(response, f) diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py index ffd88a01..7306ee1d 100644 --- a/tools/ninja_syntax.py +++ b/tools/ninja_syntax.py @@ -21,50 +21,67 @@ use Python. import re import textwrap +import os +from io import StringIO +from pathlib import Path +from typing import Dict, List, Match, Optional, Tuple, Union + +NinjaPath = Union[str, Path] +NinjaPaths = Union[ + List[str], + List[Path], + List[NinjaPath], + List[Optional[str]], + List[Optional[Path]], + List[Optional[NinjaPath]], +] +NinjaPathOrPaths = Union[NinjaPath, NinjaPaths] -def escape_path(word): +def escape_path(word: str) -> str: return word.replace("$ ", "$$ ").replace(" ", "$ ").replace(":", "$:") class Writer(object): - def __init__(self, output, width=78): + def __init__(self, output: StringIO, width: int = 78) -> None: self.output = output self.width = width - def newline(self): + def newline(self) -> None: self.output.write("\n") - def comment(self, text): + def comment(self, text: str) -> None: for line in textwrap.wrap( text, self.width - 2, break_long_words=False, break_on_hyphens=False ): self.output.write("# " + line + "\n") - def variable(self, key, value, indent=0): - if value is None: - return - if isinstance(value, list): - value = " ".join(filter(None, value)) # Filter out empty strings. + def variable( + self, + key: str, + value: Optional[NinjaPathOrPaths], + indent: int = 0, + ) -> None: + value = " ".join(serialize_paths(value)) self._line("%s = %s" % (key, value), indent) - def pool(self, name, depth): + def pool(self, name: str, depth: int) -> None: self._line("pool %s" % name) - self.variable("depth", depth, indent=1) + self.variable("depth", str(depth), indent=1) def rule( self, - name, - command, - description=None, - depfile=None, - generator=False, - pool=None, - restat=False, - rspfile=None, - rspfile_content=None, - deps=None, - ): + name: str, + command: str, + description: Optional[str] = None, + depfile: Optional[NinjaPath] = None, + generator: bool = False, + pool: Optional[str] = None, + restat: bool = False, + rspfile: Optional[NinjaPath] = None, + rspfile_content: Optional[NinjaPath] = None, + deps: Optional[NinjaPathOrPaths] = None, + ) -> None: self._line("rule %s" % name) self.variable("command", command, indent=1) if description: @@ -86,32 +103,39 @@ class Writer(object): def build( self, - outputs, - rule, - inputs=None, - implicit=None, - order_only=None, - variables=None, - implicit_outputs=None, - pool=None, - dyndep=None, - ): - outputs = as_list(outputs) + outputs: NinjaPathOrPaths, + rule: str, + inputs: Optional[NinjaPathOrPaths] = None, + implicit: Optional[NinjaPathOrPaths] = None, + order_only: Optional[NinjaPathOrPaths] = None, + variables: Optional[ + Union[ + List[Tuple[str, Optional[NinjaPathOrPaths]]], + Dict[str, Optional[NinjaPathOrPaths]], + ] + ] = None, + implicit_outputs: Optional[NinjaPathOrPaths] = None, + pool: Optional[str] = None, + dyndep: Optional[NinjaPath] = None, + ) -> List[str]: + outputs = serialize_paths(outputs) out_outputs = [escape_path(x) for x in outputs] - all_inputs = [escape_path(x) for x in as_list(inputs)] + all_inputs = [escape_path(x) for x in serialize_paths(inputs)] if implicit: - implicit = [escape_path(x) for x in as_list(implicit)] + implicit = [escape_path(x) for x in serialize_paths(implicit)] all_inputs.append("|") - all_inputs.extend(implicit) + all_inputs.extend(map(str, implicit)) if order_only: - order_only = [escape_path(x) for x in as_list(order_only)] + order_only = [escape_path(x) for x in serialize_paths(order_only)] all_inputs.append("||") - all_inputs.extend(order_only) + all_inputs.extend(map(str, order_only)) if implicit_outputs: - implicit_outputs = [escape_path(x) for x in as_list(implicit_outputs)] + implicit_outputs = [ + escape_path(x) for x in serialize_paths(implicit_outputs) + ] out_outputs.append("|") - out_outputs.extend(implicit_outputs) + out_outputs.extend(map(str, implicit_outputs)) self._line( "build %s: %s" % (" ".join(out_outputs), " ".join([rule] + all_inputs)) @@ -119,7 +143,7 @@ class Writer(object): if pool is not None: self._line(" pool = %s" % pool) if dyndep is not None: - self._line(" dyndep = %s" % dyndep) + self._line(" dyndep = %s" % serialize_path(dyndep)) if variables: if isinstance(variables, dict): @@ -132,16 +156,16 @@ class Writer(object): return outputs - def include(self, path): + def include(self, path: str) -> None: self._line("include %s" % path) - def subninja(self, path): + def subninja(self, path: str) -> None: self._line("subninja %s" % path) - def default(self, paths): - self._line("default %s" % " ".join(as_list(paths))) + def default(self, paths: NinjaPathOrPaths) -> None: + self._line("default %s" % " ".join(serialize_paths(paths))) - def _count_dollars_before_index(self, s, i): + def _count_dollars_before_index(self, s: str, i: int) -> int: """Returns the number of '$' characters right in front of s[i].""" dollar_count = 0 dollar_index = i - 1 @@ -150,7 +174,7 @@ class Writer(object): dollar_index -= 1 return dollar_count - def _line(self, text, indent=0): + def _line(self, text: str, indent: int = 0) -> None: """Write 'text' word-wrapped at self.width characters.""" leading_space = " " * indent while len(leading_space) + len(text) > self.width: @@ -187,19 +211,26 @@ class Writer(object): self.output.write(leading_space + text + "\n") - def close(self): + def close(self) -> None: self.output.close() -def as_list(input): - if input is None: - return [] +def serialize_path(input: Optional[NinjaPath]) -> str: + if not input: + return "" + if isinstance(input, Path): + return str(input).replace("/", os.sep) + else: + return str(input) + + +def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]: if isinstance(input, list): - return input - return [input] + return [serialize_path(path) for path in input if path] + return [serialize_path(input)] if input else [] -def escape(string): +def escape(string: str) -> str: """Escape a string such that it can be embedded into a Ninja file without further interpretation.""" assert "\n" not in string, "Ninja syntax does not allow newlines" @@ -207,17 +238,17 @@ def escape(string): return string.replace("$", "$$") -def expand(string, vars, local_vars={}): +def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str: """Expand a string containing $vars as Ninja would. Note: doesn't handle the full Ninja variable syntax, but it's enough to make configure.py's use of it work. """ - def exp(m): + def exp(m: Match[str]) -> str: var = m.group(1) if var == "$": return "$" return local_vars.get(var, vars.get(var, "")) - return re.sub(r"\$(\$|\w*)", exp, string) \ No newline at end of file + return re.sub(r"\$(\$|\w*)", exp, string) diff --git a/tools/project.py b/tools/project.py index 8ccfde1a..15795f1f 100644 --- a/tools/project.py +++ b/tools/project.py @@ -12,13 +12,15 @@ import io import json +import math import os import platform import sys -import math - from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple, Union + from . import ninja_syntax +from .ninja_syntax import serialize_path if sys.platform == "cygwin": sys.exit( @@ -28,51 +30,86 @@ if sys.platform == "cygwin": ) +class Object: + def __init__(self, completed: bool, name: str, **options: Any) -> None: + self.name = name + self.base_name = Path(name).with_suffix("") + self.completed = completed + self.options: Dict[str, Any] = { + "add_to_all": True, + "asflags": None, + "extra_asflags": None, + "cflags": None, + "extra_cflags": None, + "mw_version": None, + "shift_jis": None, + "source": name, + } + self.options.update(options) + + class ProjectConfig: - def __init__(self): + def __init__(self) -> None: # Paths - self.build_dir = Path("build") - self.src_dir = Path("src") - self.tools_dir = Path("tools") + self.build_dir: Path = Path("build") # Output build files + self.src_dir: Path = Path("src") # C/C++/asm source files + self.tools_dir: Path = Path("tools") # Python scripts + self.asm_dir: Optional[Path] = Path( + "asm" + ) # Override incomplete objects (for modding) # Tooling - self.dtk_tag = None # Git tag - self.build_dtk_path = None # If None, download - self.compilers_tag = None # 1 - self.compilers_path = None # If None, download - self.wibo_tag = None # Git tag - self.wrapper = None # If None, download wibo on Linux - self.sjiswrap_tag = None # Git tag - self.sjiswrap_path = None # If None, download + self.binutils_tag: Optional[str] = None # Git tag + self.binutils_path: Optional[Path] = None # If None, download + self.dtk_tag: Optional[str] = None # Git tag + self.dtk_path: Optional[Path] = None # If None, download + self.compilers_tag: Optional[str] = None # 1 + self.compilers_path: Optional[Path] = None # If None, download + self.wibo_tag: Optional[str] = None # Git tag + self.wrapper: Optional[Path] = None # If None, download wibo on Linux + self.sjiswrap_tag: Optional[str] = None # Git tag + self.sjiswrap_path: Optional[Path] = None # If None, download # Project config - self.build_rels = True # Build REL files - self.check_sha_path = None # Path to version.sha1 - self.config_path = None # Path to config.yml - self.debug = False # Build with debug info - self.generate_map = False # Generate map file(s) - self.ldflags = None # Linker flags - self.libs = None # List of libraries - self.linker_version = None # mwld version - self.version = None # Version name - self.warn_missing_config = False # Warn on missing unit configuration - self.warn_missing_source = False # Warn on missing source file - self.rel_strip_partial = True # Generate PLFs with -strip_partial - self.rel_empty_file = None # Path to empty.c for generating empty RELs + self.non_matching: bool = False + self.build_rels: bool = True # Build REL files + self.check_sha_path: Optional[Path] = None # Path to version.sha1 + self.config_path: Optional[Path] = None # Path to config.yml + self.debug: bool = False # Build with debug info + self.generate_map: bool = False # Generate map file(s) + self.asflags: Optional[List[str]] = None # Assembler flags + self.ldflags: Optional[List[str]] = None # Linker flags + self.libs: Optional[List[Dict[str, Any]]] = None # List of libraries + self.linker_version: Optional[str] = None # mwld version + self.version: Optional[str] = None # Version name + self.warn_missing_config: bool = False # Warn on missing unit configuration + self.warn_missing_source: bool = False # Warn on missing source file + self.rel_strip_partial: bool = True # Generate PLFs with -strip_partial + self.rel_empty_file: Optional[ + str + ] = None # Object name for generating empty RELs + self.shift_jis = ( + True # Convert source files from UTF-8 to Shift JIS automatically + ) + self.reconfig_deps: Optional[List[Path]] = ( + None # Additional re-configuration dependency files + ) # Progress output and progress.json config - self.progress_all = True # Include combined "all" category - self.progress_modules = True # Include combined "modules" category - self.progress_each_module = True # Include individual modules, disable for large numbers of modules + self.progress_all: bool = True # Include combined "all" category + self.progress_modules: bool = True # Include combined "modules" category + self.progress_each_module: bool = ( + True # Include individual modules, disable for large numbers of modules + ) # Progress fancy printing - self.progress_use_fancy = False - self.progress_code_fancy_frac = 0 - self.progress_code_fancy_item = "" - self.progress_data_fancy_frac = 0 - self.progress_data_fancy_item = "" + self.progress_use_fancy: bool = False + self.progress_code_fancy_frac: int = 0 + self.progress_code_fancy_item: str = "" + self.progress_data_fancy_frac: int = 0 + self.progress_data_fancy_item: str = "" - def validate(self): + def validate(self) -> None: required_attrs = [ "build_dir", "src_dir", @@ -88,33 +125,18 @@ class ProjectConfig: if getattr(self, attr) is None: sys.exit(f"ProjectConfig.{attr} missing") - def find_object(self, name): - for lib in self.libs: + def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]: + for lib in self.libs or {}: for obj in lib["objects"]: if obj.name == name: - return [lib, obj] + return lib, obj return None - def out_path(self): - return self.build_dir / self.version + def out_path(self) -> Path: + return self.build_dir / str(self.version) -class Object: - def __init__(self, completed, name, **options): - self.name = name - self.completed = completed - self.options = { - "add_to_all": True, - "cflags": None, - "extra_cflags": None, - "mw_version": None, - "shiftjis": True, - "source": name, - } - self.options.update(options) - - -def is_windows(): +def is_windows() -> bool: return os.name == "nt" @@ -124,36 +146,25 @@ CHAIN = "cmd /c " if is_windows() else "" EXE = ".exe" if is_windows() else "" -# Replace forward slashes with backslashes on Windows -def os_str(value): - return str(value).replace("/", os.sep) - - -# Replace backslashes with forward slashes on Windows -def unix_str(value): - return str(value).replace(os.sep, "/") - - -# Stringify paths for ninja_syntax -def path(value): - if value is None: - return None - elif isinstance(value, list): - return list(map(os_str, filter(lambda x: x is not None, value))) +def make_flags_str(cflags: Union[str, List[str]]) -> str: + if isinstance(cflags, list): + return " ".join(cflags) else: - return [os_str(value)] + return cflags # Load decomp-toolkit generated config.json -def load_build_config(config, build_config_path): +def load_build_config( + config: ProjectConfig, build_config_path: Path +) -> Optional[Dict[str, Any]]: if not build_config_path.is_file(): return None - def versiontuple(v): + def versiontuple(v: str) -> Tuple[int, ...]: return tuple(map(int, (v.split(".")))) f = open(build_config_path, "r", encoding="utf-8") - build_config = json.load(f) + build_config: Dict[str, Any] = json.load(f) config_version = build_config.get("version") if not config_version: # Invalid config.json @@ -161,7 +172,7 @@ def load_build_config(config, build_config_path): os.remove(build_config_path) return None - dtk_version = config.dtk_tag[1:] # Strip v + dtk_version = str(config.dtk_tag)[1:] # Strip v if versiontuple(config_version) < versiontuple(dtk_version): # Outdated config.json f.close() @@ -173,14 +184,16 @@ def load_build_config(config, build_config_path): # Generate build.ninja and objdiff.json -def generate_build(config): +def generate_build(config: ProjectConfig) -> None: build_config = load_build_config(config, config.out_path() / "config.json") generate_build_ninja(config, build_config) generate_objdiff_config(config, build_config) # Generate build.ninja -def generate_build_ninja(config, build_config): +def generate_build_ninja( + config: ProjectConfig, build_config: Optional[Dict[str, Any]] +) -> None: config.validate() out = io.StringIO() @@ -188,9 +201,9 @@ def generate_build_ninja(config, build_config): n.variable("ninja_required_version", "1.3") n.newline() - configure_script = os.path.relpath(os.path.abspath(sys.argv[0])) - python_lib = os.path.relpath(__file__) - python_lib_dir = os.path.dirname(python_lib) + configure_script = Path(os.path.relpath(os.path.abspath(sys.argv[0]))) + python_lib = Path(os.path.relpath(__file__)) + python_lib_dir = python_lib.parent n.comment("The arguments passed to configure.py, for rerunning it.") n.variable("configure_args", sys.argv[1:]) n.variable("python", f'"{sys.executable}"') @@ -200,13 +213,15 @@ def generate_build_ninja(config, build_config): # Variables ### n.comment("Variables") - ldflags = " ".join(config.ldflags) + ldflags = " ".join(config.ldflags or []) if config.generate_map: ldflags += " -mapunused" if config.debug: ldflags += " -g" n.variable("ldflags", ldflags) - n.variable("mw_version", config.linker_version) + if not config.linker_version: + sys.exit("ProjectConfig.linker_version missing") + n.variable("mw_version", Path(config.linker_version)) n.newline() ### @@ -215,6 +230,7 @@ def generate_build_ninja(config, build_config): n.comment("Tooling") build_path = config.out_path() + progress_path = build_path / "progress.json" build_tools_path = config.build_dir / "tools" download_tool = config.tools_dir / "download_tool.py" n.rule( @@ -223,20 +239,31 @@ def generate_build_ninja(config, build_config): description="TOOL $out", ) - if config.build_dtk_path: + decompctx = config.tools_dir / "decompctx.py" + n.rule( + name="decompctx", + command=f"$python {decompctx} $in -o $out -d $out.d", + description="CTX $in", + depfile="$out.d", + deps="gcc", + ) + + if config.dtk_path is not None and config.dtk_path.is_file(): + dtk = config.dtk_path + elif config.dtk_path is not None: dtk = build_tools_path / "release" / f"dtk{EXE}" n.rule( name="cargo", command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", description="CARGO $bin", - depfile=path(Path("$target") / "release" / "$bin.d"), + depfile=Path("$target") / "release" / "$bin.d", deps="gcc", ) n.build( - outputs=path(dtk), + outputs=dtk, rule="cargo", - inputs=path(config.build_dtk_path / "Cargo.toml"), - implicit=path(config.build_dtk_path / "Cargo.lock"), + inputs=config.dtk_path / "Cargo.toml", + implicit=config.dtk_path / "Cargo.lock", variables={ "bin": "dtk", "target": build_tools_path, @@ -245,9 +272,9 @@ def generate_build_ninja(config, build_config): elif config.dtk_tag: dtk = build_tools_path / f"dtk{EXE}" n.build( - outputs=path(dtk), + outputs=dtk, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "dtk", "tag": config.dtk_tag, @@ -261,9 +288,9 @@ def generate_build_ninja(config, build_config): elif config.sjiswrap_tag: sjiswrap = build_tools_path / "sjiswrap.exe" n.build( - outputs=path(sjiswrap), + outputs=sjiswrap, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "sjiswrap", "tag": config.sjiswrap_tag, @@ -274,7 +301,7 @@ def generate_build_ninja(config, build_config): # Only add an implicit dependency on wibo if we download it wrapper = config.wrapper - wrapper_implicit = None + wrapper_implicit: Optional[Path] = None if ( config.wibo_tag is not None and sys.platform == "linux" @@ -284,33 +311,53 @@ def generate_build_ninja(config, build_config): wrapper = build_tools_path / "wibo" wrapper_implicit = wrapper n.build( - outputs=path(wrapper), + outputs=wrapper, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "wibo", "tag": config.wibo_tag, }, ) if not is_windows() and wrapper is None: - wrapper = "wine" + wrapper = Path("wine") wrapper_cmd = f"{wrapper} " if wrapper else "" - compilers_implicit = None + compilers_implicit: Optional[Path] = None if config.compilers_path: compilers = config.compilers_path elif config.compilers_tag: compilers = config.build_dir / "compilers" compilers_implicit = compilers n.build( - outputs=path(compilers), + outputs=compilers, rule="download_tool", - implicit=path(download_tool), + implicit=download_tool, variables={ "tool": "compilers", "tag": config.compilers_tag, }, ) + else: + sys.exit("ProjectConfig.compilers_tag missing") + + binutils_implicit = None + if config.binutils_path: + binutils = config.binutils_path + elif config.binutils_tag: + binutils = config.build_dir / "binutils" + binutils_implicit = binutils + n.build( + outputs=binutils, + rule="download_tool", + implicit=download_tool, + variables={ + "tool": "binutils", + "tag": config.binutils_tag, + }, + ) + else: + sys.exit("ProjectConfig.binutils_tag missing") n.newline() @@ -322,16 +369,24 @@ def generate_build_ninja(config, build_config): # MWCC mwcc = compiler_path / "mwcceppc.exe" mwcc_cmd = f"{wrapper_cmd}{mwcc} $cflags -MMD -c $in -o $basedir" - mwcc_implicit = [compilers_implicit or mwcc, wrapper_implicit] + mwcc_implicit: List[Optional[Path]] = [compilers_implicit or mwcc, wrapper_implicit] # MWCC with UTF-8 to Shift JIS wrapper mwcc_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir" - mwcc_sjis_implicit = [*mwcc_implicit, sjiswrap] + mwcc_sjis_implicit: List[Optional[Path]] = [*mwcc_implicit, sjiswrap] # MWLD mwld = compiler_path / "mwldeppc.exe" mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp" - mwld_implicit = [compilers_implicit or mwld, wrapper_implicit] + mwld_implicit: List[Optional[Path]] = [compilers_implicit or mwld, wrapper_implicit] + + # GNU as + gnu_as = binutils / f"powerpc-eabi-as{EXE}" + gnu_as_cmd = ( + f"{CHAIN}{gnu_as} $asflags -o $out $in -MD $out.d" + + f" && {dtk} elf fixup $out $out" + ) + gnu_as_implicit = [binutils_implicit or gnu_as, dtk] if os.name != "nt": transform_dep = config.tools_dir / "transform_dep.py" @@ -358,17 +413,6 @@ def generate_build_ninja(config, build_config): ) n.newline() - n.comment("Generate REL(s)") - makerel_rsp = build_path / "makerel.rsp" - n.rule( - name="makerel", - command=f"{dtk} rel make -w -c $config @{makerel_rsp}", - description="REL", - rspfile=path(makerel_rsp), - rspfile_content="$in_newline", - ) - n.newline() - n.comment("MWCC build") n.rule( name="mwcc", @@ -389,6 +433,16 @@ def generate_build_ninja(config, build_config): ) n.newline() + n.comment("Assemble asm") + n.rule( + name="as", + command=gnu_as_cmd, + description="AS $out", + depfile="$out.d", + deps="gcc", + ) + n.newline() + n.comment("Host build") n.variable("host_cflags", "-I include -Wno-trigraphs") n.variable( @@ -411,66 +465,67 @@ def generate_build_ninja(config, build_config): # Source files ### n.comment("Source files") + build_asm_path = build_path / "mod" build_src_path = build_path / "src" build_host_path = build_path / "host" build_config_path = build_path / "config.json" - def map_path(path): + def map_path(path: Path) -> Path: return path.parent / (path.name + ".MAP") class LinkStep: - def __init__(self, config): - self.name = config["name"] - self.module_id = config["module_id"] - self.ldscript = config["ldscript"] + def __init__(self, config: Dict[str, Any]) -> None: + self.name: str = config["name"] + self.module_id: int = config["module_id"] + self.ldscript: Optional[Path] = Path(config["ldscript"]) self.entry = config["entry"] - self.inputs = [] + self.inputs: List[str] = [] - def add(self, obj): - self.inputs.append(obj) + def add(self, obj: Path) -> None: + self.inputs.append(serialize_path(obj)) - def output(self): + def output(self) -> Path: if self.module_id == 0: return build_path / f"{self.name}.dol" else: return build_path / self.name / f"{self.name}.rel" - def partial_output(self): + def partial_output(self) -> Path: if self.module_id == 0: return build_path / f"{self.name}.elf" else: return build_path / self.name / f"{self.name}.plf" - def write(self, n): + def write(self, n: ninja_syntax.Writer) -> None: n.comment(f"Link {self.name}") if self.module_id == 0: elf_path = build_path / f"{self.name}.elf" dol_path = build_path / f"{self.name}.dol" - elf_ldflags = f"$ldflags -lcf {self.ldscript}" + elf_ldflags = f"$ldflags -lcf {serialize_path(self.ldscript)}" if config.generate_map: elf_map = map_path(elf_path) - elf_ldflags += f" -map {elf_map}" + elf_ldflags += f" -map {serialize_path(elf_map)}" else: elf_map = None n.build( - outputs=path(elf_path), + outputs=elf_path, rule="link", - inputs=path(self.inputs), - implicit=path([self.ldscript, *mwld_implicit]), - implicit_outputs=path(elf_map), + inputs=self.inputs, + implicit=[self.ldscript, *mwld_implicit], + implicit_outputs=elf_map, variables={"ldflags": elf_ldflags}, ) n.build( - outputs=path(dol_path), + outputs=dol_path, rule="elf2dol", - inputs=path(elf_path), - implicit=path(dtk), + inputs=elf_path, + implicit=dtk, ) else: preplf_path = build_path / self.name / f"{self.name}.preplf" plf_path = build_path / self.name / f"{self.name}.plf" - preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" - plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}" + preplf_ldflags = "$ldflags -sdata 0 -sdata2 0 -r" + plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {serialize_path(self.ldscript)}" if self.entry: plf_ldflags += f" -m {self.entry}" # -strip_partial is only valid with -m @@ -478,44 +533,140 @@ def generate_build_ninja(config, build_config): plf_ldflags += " -strip_partial" if config.generate_map: preplf_map = map_path(preplf_path) - preplf_ldflags += f" -map {preplf_map}" + preplf_ldflags += f" -map {serialize_path(preplf_map)}" plf_map = map_path(plf_path) - plf_ldflags += f" -map {plf_map}" + plf_ldflags += f" -map {serialize_path(plf_map)}" else: preplf_map = None plf_map = None n.build( - outputs=path(preplf_path), + outputs=preplf_path, rule="link", - inputs=path(self.inputs), - implicit=path(mwld_implicit), - implicit_outputs=path(preplf_map), + inputs=self.inputs, + implicit=mwld_implicit, + implicit_outputs=preplf_map, variables={"ldflags": preplf_ldflags}, ) n.build( - outputs=path(plf_path), + outputs=plf_path, rule="link", - inputs=path(self.inputs), - implicit=path([self.ldscript, preplf_path, *mwld_implicit]), - implicit_outputs=path(plf_map), + inputs=self.inputs, + implicit=[self.ldscript, preplf_path, *mwld_implicit], + implicit_outputs=plf_map, variables={"ldflags": plf_ldflags}, ) n.newline() + link_outputs: List[Path] = [] if build_config: - link_steps = [] - used_compiler_versions = set() - source_inputs = [] - host_source_inputs = [] - source_added = set() + link_steps: List[LinkStep] = [] + used_compiler_versions: Set[str] = set() + source_inputs: List[Path] = [] + host_source_inputs: List[Path] = [] + source_added: Set[Path] = set() - def make_cflags_str(cflags): - if isinstance(cflags, list): - return " ".join(cflags) - else: - return cflags + def c_build( + obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + ) -> Optional[Path]: + cflags_str = make_flags_str(options["cflags"]) + if options["extra_cflags"] is not None: + extra_cflags_str = make_flags_str(options["extra_cflags"]) + cflags_str += " " + extra_cflags_str + used_compiler_versions.add(options["mw_version"]) - def add_unit(build_obj, link_step): + src_obj_path = build_src_path / f"{obj.base_name}.o" + src_base_path = build_src_path / obj.base_name + + # Avoid creating duplicate build rules + if src_obj_path in source_added: + return src_obj_path + source_added.add(src_obj_path) + + shift_jis = options["shift_jis"] + if shift_jis is None: + shift_jis = config.shift_jis + + # Add MWCC build rule + n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") + n.build( + outputs=src_obj_path, + rule="mwcc_sjis" if shift_jis else "mwcc", + inputs=src_path, + variables={ + "mw_version": Path(options["mw_version"]), + "cflags": cflags_str, + "basedir": os.path.dirname(src_base_path), + "basefile": src_base_path, + }, + implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit, + ) + + # Add ctx build rule + ctx_path = build_src_path / f"{obj.base_name}.ctx" + n.build( + outputs=ctx_path, + rule="decompctx", + inputs=src_path, + implicit=decompctx, + ) + + # Add host build rule + if options.get("host", False): + host_obj_path = build_host_path / f"{obj.base_name}.o" + host_base_path = build_host_path / obj.base_name + n.build( + outputs=host_obj_path, + rule="host_cc" if src_path.suffix == ".c" else "host_cpp", + inputs=src_path, + variables={ + "basedir": os.path.dirname(host_base_path), + "basefile": host_base_path, + }, + ) + if options["add_to_all"]: + host_source_inputs.append(host_obj_path) + n.newline() + + if options["add_to_all"]: + source_inputs.append(src_obj_path) + + return src_obj_path + + def asm_build( + obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + ) -> Optional[Path]: + asflags = options["asflags"] or config.asflags + if asflags is None: + sys.exit("ProjectConfig.asflags missing") + asflags_str = make_flags_str(asflags) + if options["extra_asflags"] is not None: + extra_asflags_str = make_flags_str(options["extra_asflags"]) + asflags_str += " " + extra_asflags_str + + asm_obj_path = build_asm_path / f"{obj.base_name}.o" + + # Avoid creating duplicate build rules + if asm_obj_path in source_added: + return asm_obj_path + source_added.add(asm_obj_path) + + # Add assembler build rule + n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") + n.build( + outputs=asm_obj_path, + rule="as", + inputs=src_path, + variables={"asflags": asflags_str}, + implicit=gnu_as_implicit, + ) + n.newline() + + if options["add_to_all"]: + source_inputs.append(asm_obj_path) + + return asm_obj_path + + def add_unit(build_obj, link_step: LinkStep): obj_path, obj_name = build_obj["object"], build_obj["name"] result = config.find_object(obj_name) if not result: @@ -526,71 +677,50 @@ def generate_build_ninja(config, build_config): lib, obj = result lib_name = lib["lib"] - src_dir = Path(lib.get("src_dir", config.src_dir)) - options = obj.options - completed = obj.completed + # Use object options, then library options + options = lib.copy() + for key, value in obj.options.items(): + if value is not None or key not in options: + options[key] = value - unit_src_path = src_dir / options["source"] + unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"] - if not unit_src_path.exists(): - if config.warn_missing_source or completed: + unit_asm_path: Optional[Path] = None + if config.asm_dir is not None: + unit_asm_path = ( + Path(lib.get("asm_dir", config.asm_dir)) / options["source"] + ).with_suffix(".s") + + link_built_obj = obj.completed + built_obj_path: Optional[Path] = None + if unit_src_path.exists(): + if unit_src_path.suffix in (".c", ".cp", ".cpp"): + # Add MWCC & host build rules + built_obj_path = c_build(obj, options, lib_name, unit_src_path) + elif unit_src_path.suffix == ".s": + # Add assembler build rule + built_obj_path = asm_build(obj, options, lib_name, unit_src_path) + else: + sys.exit(f"Unknown source file type {unit_src_path}") + else: + if config.warn_missing_source or obj.completed: print(f"Missing source file {unit_src_path}") + link_built_obj = False + + # Assembly overrides + if unit_asm_path is not None and unit_asm_path.exists(): + link_built_obj = True + built_obj_path = asm_build(obj, options, lib_name, unit_asm_path) + + if link_built_obj and built_obj_path is not None: + # Use the source-built object + link_step.add(built_obj_path) + elif obj_path is not None: + # Use the original (extracted) object link_step.add(obj_path) - return - - mw_version = options["mw_version"] or lib["mw_version"] - cflags_str = make_cflags_str(options["cflags"] or lib["cflags"]) - if options["extra_cflags"] is not None: - extra_cflags_str = make_cflags_str(options["extra_cflags"]) - cflags_str += " " + extra_cflags_str - used_compiler_versions.add(mw_version) - - base_object = Path(obj.name).with_suffix("") - src_obj_path = build_src_path / f"{base_object}.o" - src_base_path = build_src_path / base_object - - if src_obj_path not in source_added: - source_added.add(src_obj_path) - - n.comment(f"{obj_name}: {lib_name} (linked {completed})") - n.build( - outputs=path(src_obj_path), - rule="mwcc_sjis" if options["shiftjis"] else "mwcc", - inputs=path(unit_src_path), - variables={ - "mw_version": path(Path(mw_version)), - "cflags": cflags_str, - "basedir": os.path.dirname(src_base_path), - "basefile": path(src_base_path), - }, - implicit=path( - mwcc_sjis_implicit if options["shiftjis"] else mwcc_implicit - ), - ) - - if lib["host"]: - host_obj_path = build_host_path / f"{base_object}.o" - host_base_path = build_host_path / base_object - n.build( - outputs=path(host_obj_path), - rule="host_cc" if unit_src_path.suffix == ".c" else "host_cpp", - inputs=path(unit_src_path), - variables={ - "basedir": os.path.dirname(host_base_path), - "basefile": path(host_base_path), - }, - ) - if options["add_to_all"]: - host_source_inputs.append(host_obj_path) - n.newline() - - if options["add_to_all"]: - source_inputs.append(src_obj_path) - - if completed: - obj_path = src_obj_path - link_step.add(obj_path) + else: + sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}") # Add DOL link step link_step = LinkStep(build_config) @@ -626,7 +756,7 @@ def generate_build_ninja(config, build_config): sys.exit(f"Compiler {mw_path} does not exist") # Check if linker exists - mw_path = compilers / config.linker_version / "mwldeppc.exe" + mw_path = compilers / str(config.linker_version) / "mwldeppc.exe" if config.compilers_path and not os.path.exists(mw_path): sys.exit(f"Linker {mw_path} does not exist") @@ -635,13 +765,25 @@ def generate_build_ninja(config, build_config): ### for step in link_steps: step.write(n) + link_outputs.append(step.output()) n.newline() ### # Generate RELs ### + n.comment("Generate REL(s)") + flags = "-w" + if len(build_config["links"]) > 1: + flags += " -q" + n.rule( + name="makerel", + command=f"{dtk} rel make {flags} -c $config $names @$rspfile", + description="REL", + rspfile="$rspfile", + rspfile_content="$in_newline", + ) generated_rels = [] - for link in build_config["links"]: + for idx, link in enumerate(build_config["links"]): # Map module names to link steps link_steps_local = list( filter( @@ -655,7 +797,7 @@ def generate_build_ninja(config, build_config): rels_to_generate = list( filter( lambda step: step.module_id != 0 - and not step.name in generated_rels, + and step.name not in generated_rels, link_steps_local, ) ) @@ -668,15 +810,23 @@ def generate_build_ninja(config, build_config): rels_to_generate, ) ) - n.comment("Generate RELs") + rel_names = list( + map( + lambda step: step.name, + link_steps_local, + ) + ) + rel_names_arg = " ".join(map(lambda name: f"-n {name}", rel_names)) n.build( - outputs=path(rel_outputs), + outputs=rel_outputs, rule="makerel", - inputs=path( - list(map(lambda step: step.partial_output(), link_steps_local)) - ), - implicit=path([dtk, config.config_path]), - variables={"config": path(config.config_path)}, + inputs=list(map(lambda step: step.partial_output(), link_steps_local)), + implicit=[dtk, config.config_path], + variables={ + "config": config.config_path, + "rspfile": config.out_path() / f"rel{idx}.rsp", + "names": rel_names_arg, + }, ) n.newline() @@ -687,7 +837,7 @@ def generate_build_ninja(config, build_config): n.build( outputs="all_source", rule="phony", - inputs=path(source_inputs), + inputs=source_inputs, ) n.newline() @@ -698,7 +848,7 @@ def generate_build_ninja(config, build_config): n.build( outputs="all_source_host", rule="phony", - inputs=path(host_source_inputs), + inputs=host_source_inputs, ) n.newline() @@ -714,10 +864,10 @@ def generate_build_ninja(config, build_config): description="CHECK $in", ) n.build( - outputs=path(ok_path), + outputs=ok_path, rule="check", - inputs=path(config.check_sha_path), - implicit=path([dtk, *map(lambda step: step.output(), link_steps)]), + inputs=config.check_sha_path, + implicit=[dtk, *link_outputs], ) n.newline() @@ -725,16 +875,15 @@ def generate_build_ninja(config, build_config): # Calculate progress ### n.comment("Calculate progress") - progress_path = build_path / "progress.json" n.rule( name="progress", command=f"$python {configure_script} $configure_args progress", description="PROGRESS", ) n.build( - outputs=path(progress_path), + outputs=progress_path, rule="progress", - implicit=path([ok_path, configure_script, python_lib, config.config_path]), + implicit=[ok_path, configure_script, python_lib, config.config_path], ) ### @@ -750,7 +899,7 @@ def generate_build_ninja(config, build_config): description=f"DIFF {dol_elf_path}", ) n.build( - inputs=path([config.config_path, dol_elf_path]), + inputs=[config.config_path, dol_elf_path], outputs="dol_diff", rule="dol_diff", ) @@ -768,10 +917,10 @@ def generate_build_ninja(config, build_config): description=f"APPLY {dol_elf_path}", ) n.build( - inputs=path([config.config_path, dol_elf_path]), + inputs=[config.config_path, dol_elf_path], outputs="dol_apply", rule="dol_apply", - implicit=path([ok_path]), + implicit=[ok_path], ) n.build( outputs="apply", @@ -792,11 +941,11 @@ def generate_build_ninja(config, build_config): deps="gcc", ) n.build( - inputs=path(config.config_path), - outputs=path(build_config_path), + inputs=config.config_path, + outputs=build_config_path, rule="split", - implicit=path(dtk), - variables={"out_dir": path(build_path)}, + implicit=dtk, + variables={"out_dir": build_path}, ) n.newline() @@ -813,14 +962,13 @@ def generate_build_ninja(config, build_config): n.build( outputs="build.ninja", rule="configure", - implicit=path( - [ - build_config_path, - configure_script, - python_lib, - Path(python_lib_dir) / "ninja_syntax.py", - ] - ), + implicit=[ + build_config_path, + configure_script, + python_lib, + python_lib_dir / "ninja_syntax.py", + *(config.reconfig_deps or []) + ], ) n.newline() @@ -829,9 +977,12 @@ def generate_build_ninja(config, build_config): ### n.comment("Default rule") if build_config: - n.default(path(progress_path)) + if config.non_matching: + n.default(link_outputs) + else: + n.default(progress_path) else: - n.default(path(build_config_path)) + n.default(build_config_path) # Write build.ninja with open("build.ninja", "w", encoding="utf-8") as f: @@ -840,12 +991,14 @@ def generate_build_ninja(config, build_config): # Generate objdiff.json -def generate_objdiff_config(config, build_config): +def generate_objdiff_config( + config: ProjectConfig, build_config: Optional[Dict[str, Any]] +) -> None: if not build_config: return - objdiff_config = { - "min_version": "0.4.3", + objdiff_config: Dict[str, Any] = { + "min_version": "1.0.0", "custom_make": "ninja", "build_target": False, "watch_patterns": [ @@ -863,18 +1016,50 @@ def generate_objdiff_config(config, build_config): "units": [], } + # decomp.me compiler name mapping + # Commented out versions have not been added to decomp.me yet + COMPILER_MAP = { + "GC/1.0": "mwcc_233_144", + "GC/1.1": "mwcc_233_159", + "GC/1.2.5": "mwcc_233_163", + "GC/1.2.5e": "mwcc_233_163e", + "GC/1.2.5n": "mwcc_233_163n", + "GC/1.3.2": "mwcc_242_81", + "GC/1.3.2r": "mwcc_242_81r", + "GC/2.0": "mwcc_247_92", + "GC/2.5": "mwcc_247_105", + "GC/2.6": "mwcc_247_107", + "GC/2.7": "mwcc_247_108", + "GC/3.0a3": "mwcc_41_51213", + "GC/3.0a3.2": "mwcc_41_60126", + "GC/3.0a3.3": "mwcc_41_60209", + "GC/3.0a3.4": "mwcc_42_60308", + "GC/3.0a5": "mwcc_42_60422", + "GC/3.0a5.2": "mwcc_41_60831", + "GC/3.0": "mwcc_41_60831", + "Wii/1.0RC1": "mwcc_42_140", + "Wii/0x4201_127": "mwcc_42_142", + "Wii/1.0a": "mwcc_42_142", + "Wii/1.0": "mwcc_43_145", + "Wii/1.1": "mwcc_43_151", + "Wii/1.3": "mwcc_43_172", + "Wii/1.5": "mwcc_43_188", + "Wii/1.6": "mwcc_43_202", + "Wii/1.7": "mwcc_43_213", + } + build_path = config.out_path() - def add_unit(build_obj, module_name): + def add_unit(build_obj: Dict[str, Any], module_name: str) -> None: if build_obj["autogenerated"]: # Skip autogenerated objects return obj_path, obj_name = build_obj["object"], build_obj["name"] base_object = Path(obj_name).with_suffix("") - unit_config = { - "name": unix_str(Path(module_name) / base_object), - "target_path": unix_str(obj_path), + unit_config: Dict[str, Any] = { + "name": Path(module_name) / base_object, + "target_path": obj_path, } result = config.find_object(obj_name) @@ -885,14 +1070,21 @@ def generate_objdiff_config(config, build_config): lib, obj = result src_dir = Path(lib.get("src_dir", config.src_dir)) - unit_src_path = src_dir / obj.options["source"] + # Use object options, then library options + options = lib.copy() + for key, value in obj.options.items(): + if value is not None or key not in options: + options[key] = value + + unit_src_path = src_dir / str(options["source"]) if not unit_src_path.exists(): objdiff_config["units"].append(unit_config) return - cflags = obj.options["cflags"] or lib["cflags"] - src_obj_path = build_path / "src" / f"{base_object}.o" + cflags = options["cflags"] + src_obj_path = build_path / "src" / f"{obj.base_name}.o" + src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx" reverse_fn_order = False if type(cflags) is list: @@ -905,9 +1097,32 @@ def generate_objdiff_config(config, build_config): elif value == "nodeferred": reverse_fn_order = False - unit_config["base_path"] = unix_str(src_obj_path) + # Filter out include directories + def keep_flag(flag): + return not flag.startswith("-i ") and not flag.startswith("-I ") + + cflags = list(filter(keep_flag, cflags)) + + # Add appropriate lang flag + if unit_src_path.suffix in (".cp", ".cpp"): + cflags.insert(0, "-lang=c++") + else: + cflags.insert(0, "-lang=c") + + unit_config["base_path"] = src_obj_path unit_config["reverse_fn_order"] = reverse_fn_order unit_config["complete"] = obj.completed + compiler_version = COMPILER_MAP.get(options["mw_version"]) + if compiler_version is None: + print(f"Missing scratch compiler mapping for {options['mw_version']}") + else: + unit_config["scratch"] = { + "platform": "gc_wii", + "compiler": compiler_version, + "c_flags": make_flags_str(cflags), + "ctx_path": src_ctx_path, + "build_ctx": True, + } objdiff_config["units"].append(unit_config) # Add DOL units @@ -921,32 +1136,36 @@ def generate_objdiff_config(config, build_config): # Write objdiff.json with open("objdiff.json", "w", encoding="utf-8") as w: - json.dump(objdiff_config, w, indent=4) + + def unix_path(input: Any) -> str: + return str(input).replace(os.sep, "/") if input else "" + + json.dump(objdiff_config, w, indent=4, default=unix_path) # Calculate, print and write progress to progress.json -def calculate_progress(config): +def calculate_progress(config: ProjectConfig) -> None: out_path = config.out_path() build_config = load_build_config(config, out_path / "config.json") if not build_config: return class ProgressUnit: - def __init__(self, name): - self.name = name - self.code_total = 0 - self.code_fancy_frac = config.progress_code_fancy_frac - self.code_fancy_item = config.progress_code_fancy_item - self.code_progress = 0 - self.data_total = 0 - self.data_fancy_frac = config.progress_data_fancy_frac - self.data_fancy_item = config.progress_data_fancy_item - self.data_progress = 0 - self.objects_progress = 0 - self.objects_total = 0 - self.objects = set() + def __init__(self, name: str) -> None: + self.name: str = name + self.code_total: int = 0 + self.code_fancy_frac: int = config.progress_code_fancy_frac + self.code_fancy_item: str = config.progress_code_fancy_item + self.code_progress: int = 0 + self.data_total: int = 0 + self.data_fancy_frac: int = config.progress_data_fancy_frac + self.data_fancy_item: str = config.progress_data_fancy_item + self.data_progress: int = 0 + self.objects_progress: int = 0 + self.objects_total: int = 0 + self.objects: Set[Object] = set() - def add(self, build_obj): + def add(self, build_obj: Dict[str, Any]) -> None: self.code_total += build_obj["code_size"] self.data_total += build_obj["data_size"] @@ -973,10 +1192,10 @@ def calculate_progress(config): if include_object: self.objects_progress += 1 - def code_frac(self): + def code_frac(self) -> float: return self.code_progress / self.code_total - def data_frac(self): + def data_frac(self) -> float: return self.data_progress / self.data_total # Add DOL units @@ -989,7 +1208,7 @@ def calculate_progress(config): # Add REL units rels_progress = ProgressUnit("Modules") if config.progress_modules else None - modules_progress = [] + modules_progress: List[ProgressUnit] = [] for module in build_config["modules"]: progress = ProgressUnit(module["name"]) modules_progress.append(progress) @@ -1003,7 +1222,7 @@ def calculate_progress(config): # Print human-readable progress print("Progress:") - def print_category(unit): + def print_category(unit: Optional[ProgressUnit]) -> None: if unit is None: return @@ -1016,7 +1235,7 @@ def calculate_progress(config): print(f" Data: {unit.data_progress} / {unit.data_total} bytes") if config.progress_use_fancy: print( - "\nYou have {} out of {} {} and collected {} out of {} {}.".format( + "\nYou have {} out of {} {} and {} out of {} {}.".format( math.floor(code_frac * unit.code_fancy_frac), unit.code_fancy_frac, unit.code_fancy_item, @@ -1037,9 +1256,9 @@ def calculate_progress(config): print_category(progress) # Generate and write progress.json - progress_json = {} + progress_json: Dict[str, Any] = {} - def add_category(category, unit): + def add_category(category: str, unit: ProgressUnit) -> None: progress_json[category] = { "code": unit.code_progress, "code/total": unit.code_total, diff --git a/tools/transform_dep.py b/tools/transform_dep.py old mode 100644 new mode 100755 index 86bd2ecb..124de04b --- a/tools/transform_dep.py +++ b/tools/transform_dep.py @@ -25,7 +25,7 @@ def in_wsl() -> bool: return "microsoft-standard" in uname().release -def import_d_file(in_file) -> str: +def import_d_file(in_file: str) -> str: out_text = "" with open(in_file) as file: @@ -60,7 +60,7 @@ def import_d_file(in_file) -> str: return out_text -def main(): +def main() -> None: parser = argparse.ArgumentParser( description="""Transform a .d file from Wine paths to normal paths""" ) @@ -81,4 +81,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/upload_progress.py b/tools/upload_progress.py old mode 100644 new mode 100755 index 673bb3d5..dc61d156 --- a/tools/upload_progress.py +++ b/tools/upload_progress.py @@ -51,7 +51,7 @@ if __name__ == "__main__": args = parser.parse_args() api_key = args.api_key or os.environ.get("PROGRESS_API_KEY") if not api_key: - raise "API key required" + raise KeyError("API key required") url = generate_url(args) entries = [] @@ -68,9 +68,12 @@ if __name__ == "__main__": print("Publishing entry to", url) json.dump(entries[0], sys.stdout, indent=4) print() - r = requests.post(url, json={ - "api_key": api_key, - "entries": entries, - }) + r = requests.post( + url, + json={ + "api_key": api_key, + "entries": entries, + }, + ) r.raise_for_status() - print("Done!") \ No newline at end of file + print("Done!")