Assembly modding support & objdiff + decomp.me integration (#15)

* Add initial asm build support

* WIP decomp.me and links

README.example.md

@@ -73,7 +73,7 @@ Linux:
 ------
 - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages).
 - For non-x86(_64) platforms: Install wine from your package manager.
-  - For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.
+  - For x86(_64), [wibo](https://github.com/decompals/wibo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.
 
 Building
 ========

config/GAMEID/config.example.yml

@@ -63,6 +63,11 @@ symbols_known: false
 # this can be set to false.
 fill_gaps: true
 
+# (optional) By default, emitted objects will "export" all symbols (force active).
+# This is useful to prevent the linker from removing any symbols.
+# Individual symbols can be excluded using `noexport` in the symbols file.
+export_all: true
+
 # (optional) Custom template for `ldscript.lcf`. Avoid unless necessary.
 # See https://github.com/encounter/decomp-toolkit/blob/main/assets/ldscript.lcf
 ldscript_template: config/GAMEID/module/ldscript.tpl
@@ -99,7 +104,14 @@ modules:
   # See https://github.com/encounter/decomp-toolkit/blob/main/assets/ldscript_partial.lcf
   ldscript_template: config/GAMEID/module/ldscript.tpl
 
+  # (optional) By default, every REL is linked with every other REL.
+  # Some games link RELs individually, so the module IDs are not unique.
+  # To support this, `links` overrides which other modules are included in this module's analysis.
+  # The DOL is always included, and does not need to be specified.
+  links: [module2] # This module will be linked with the DOL and "module2".
+
 # (optional) Configuration for asset extraction.
+# For modules, this goes in the module's configuration above.
 extract:
 
 - # The symbol name to extract.

configure.py

@@ -56,6 +56,12 @@ parser.add_argument(
     default=Path("build"),
     help="base build directory (default: build)",
 )
+parser.add_argument(
+    "--binutils",
+    dest="binutils",
+    type=Path,
+    help="path to binutils (optional)",
+)
 parser.add_argument(
     "--compilers",
     dest="compilers",
@@ -110,6 +116,7 @@ version_num = VERSIONS.index(config.version)
 # Apply arguments
 config.build_dir = args.build_dir
 config.build_dtk_path = args.build_dtk
+config.binutils_path = args.binutils
 config.compilers_path = args.compilers
 config.debug = args.debug
 config.generate_map = args.map
@@ -118,14 +125,22 @@ if not is_windows():
     config.wrapper = args.wrapper
 
 # Tool versions
+config.binutils_tag = "2.42-1"
 config.compilers_tag = "20231018"
-config.dtk_tag = "v0.6.2"
+config.dtk_tag = "v0.7.4"
 config.sjiswrap_tag = "v1.1.1"
-config.wibo_tag = "0.6.9"
+config.wibo_tag = "0.6.11"
 
 # Project
 config.config_path = Path("config") / config.version / "config.yml"
 config.check_sha_path = Path("config") / config.version / "build.sha1"
+config.asflags = [
+    "-mgekko",
+    "--strip-local-absolute",
+    "-I include",
+    f"-I build/{config.version}/include",
+    f"--defsym version={version_num}",
+]
 config.ldflags = [
     "-fp hardware",
     "-nodefaults",

docs/dependencies.md

@@ -1,6 +1,8 @@
-# Dependencies
+Dependencies
+============
 
-## Windows:
+Windows:
+--------
 
 On Windows, it's **highly recommended** to use native tooling. WSL or msys2 are **not** required.  
 When running under WSL, [objdiff](#diffing) is unable to get filesystem notifications for automatic rebuilds.
@@ -10,8 +12,8 @@ When running under WSL, [objdiff](#diffing) is unable to get filesystem notifica
 - Download [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`.
   - Quick install via pip: `pip install ninja`
 
-## macOS:
+macOS:
-
+------
 - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages):
   ```
   brew install ninja
@@ -26,8 +28,8 @@ After OS upgrades, if macOS complains about `Wine Crossover.app` being unverifie
 sudo xattr -rd com.apple.quarantine '/Applications/Wine Crossover.app'
 ```
 
-## Linux:
+Linux:
-
+------
 - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages).
 - For non-x86(_64) platforms: Install wine from your package manager.
-  - For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.
+  - For x86(_64), [wibo](https://github.com/decompals/wibo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.

docs/symbols.md

@@ -33,3 +33,5 @@ All attributes are optional, and are separated by spaces.
 - `hidden` Marked as "hidden" in the generated object. (Only used for extab)
 - `force_active` Marked as ["exported"](comment_section.md) in the generated object, and added to `FORCEACTIVE` in the generated `ldscript.lcf`. Prevents the symbol from being deadstripped.
 - `noreloc` Prevents the _contents_ of the symbol from being interpreted as addresses. Used for objects containing data that look like pointers, but aren't.
+- `noexport` When `export_all` is enabled, this excludes the symbol from being exported. Used for symbols that are affected by linker `-code_merging`.
+- `stripped` Indicates a symbol that was stripped by the linker. Used for symbols that affect the [common BSS inflation bug](common_bss.md), despite not existing in the final binary.

tools/decompctx.py

@@ -13,45 +13,50 @@
 import argparse
 import os
 import re
-from typing import List, Set
+from typing import List
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.abspath(os.path.join(script_dir, ".."))
 src_dir = os.path.join(root_dir, "src")
-include_dir = os.path.join(root_dir, "include")
+include_dirs = [
+    os.path.join(root_dir, "include"),
+    # Add additional include directories here
+]
 
 include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$')
 guard_pattern = re.compile(r"^#ifndef\s+(.*)$")
 
-defines: Set[str] = set()
+defines = set()
 
 
-def import_h_file(in_file: str, r_path: str) -> str:
+def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str:
     rel_path = os.path.join(root_dir, r_path, in_file)
-    inc_path = os.path.join(include_dir, in_file)
     if os.path.exists(rel_path):
-        return import_c_file(rel_path)
+        return import_c_file(rel_path, deps)
-    elif os.path.exists(inc_path):
+    for include_dir in include_dirs:
-        return import_c_file(inc_path)
+        inc_path = os.path.join(include_dir, in_file)
+        if os.path.exists(inc_path):
+            return import_c_file(inc_path, deps)
     else:
         print("Failed to locate", in_file)
-        exit(1)
+        return ""
 
 
-def import_c_file(in_file: str) -> str:
+def import_c_file(in_file: str, deps: List[str]) -> str:
     in_file = os.path.relpath(in_file, root_dir)
+    deps.append(in_file)
     out_text = ""
 
     try:
         with open(in_file, encoding="utf-8") as file:
-            out_text += process_file(in_file, list(file))
+            out_text += process_file(in_file, list(file), deps)
     except Exception:
         with open(in_file) as file:
-            out_text += process_file(in_file, list(file))
+            out_text += process_file(in_file, list(file), deps)
     return out_text
 
 
-def process_file(in_file: str, lines: List[str]) -> str:
+def process_file(in_file: str, lines: List[str], deps: List[str]) -> str:
     out_text = ""
     for idx, line in enumerate(lines):
         guard_match = guard_pattern.match(line.strip())
@@ -64,7 +69,7 @@ def process_file(in_file: str, lines: List[str]) -> str:
         include_match = include_pattern.match(line.strip())
         if include_match and not include_match[1].endswith(".s"):
             out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n'
-            out_text += import_h_file(include_match[1], os.path.dirname(in_file))
+            out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps)
             out_text += f'/* end "{include_match[1]}" */\n'
         else:
             out_text += line
@@ -72,7 +77,11 @@ def process_file(in_file: str, lines: List[str]) -> str:
     return out_text
 
 
-def main() -> None:
+def sanitize_path(path: str) -> str:
+    return path.replace("\\", "/").replace(" ", "\\ ")
+
+
+def main():
     parser = argparse.ArgumentParser(
         description="""Create a context file which can be used for decomp.me"""
     )
@@ -80,13 +89,32 @@ def main() -> None:
         "c_file",
         help="""File from which to create context""",
     )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="""Output file""",
+        default="ctx.c",
+    )
+    parser.add_argument(
+        "-d",
+        "--depfile",
+        help="""Dependency file""",
+    )
     args = parser.parse_args()
 
-    output = import_c_file(args.c_file)
+    deps = []
+    output = import_c_file(args.c_file, deps)
 
-    with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f:
+    with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f:
         f.write(output)
 
+    if args.depfile:
+        with open(os.path.join(root_dir, args.depfile), "w", encoding="utf-8") as f:
+            f.write(sanitize_path(args.output) + ":")
+            for dep in deps:
+                path = sanitize_path(dep)
+                f.write(f" \\\n\t{path}")
+
 
 if __name__ == "__main__":
     main()

tools/download_tool.py

@@ -22,6 +22,24 @@ from typing import Callable, Dict
 from pathlib import Path
 
 
+def binutils_url(tag):
+    uname = platform.uname()
+    system = uname.system.lower()
+    arch = uname.machine.lower()
+    if system == "darwin":
+        system = "macos"
+        arch = "universal"
+    elif arch == "amd64":
+        arch = "x86_64"
+
+    repo = "https://github.com/encounter/gc-wii-binutils"
+    return f"{repo}/releases/download/{tag}/{system}-{arch}.zip"
+
+
+def compilers_url(tag: str) -> str:
+    return f"https://files.decomp.dev/compilers_{tag}.zip"
+
+
 def dtk_url(tag: str) -> str:
     uname = platform.uname()
     suffix = ""
@@ -48,15 +66,12 @@ def wibo_url(tag: str) -> str:
     return f"{repo}/releases/download/{tag}/wibo"
 
 
-def compilers_url(tag: str) -> str:
-    return f"https://files.decomp.dev/compilers_{tag}.zip"
-
-
 TOOLS: Dict[str, Callable[[str], str]] = {
+    "binutils": binutils_url,
+    "compilers": compilers_url,
     "dtk": dtk_url,
     "sjiswrap": sjiswrap_url,
     "wibo": wibo_url,
-    "compilers": compilers_url,
 }
 
 
@@ -77,7 +92,11 @@ def main() -> None:
             data = io.BytesIO(response.read())
             with zipfile.ZipFile(data) as f:
                 f.extractall(output)
-            output.touch(mode=0o755)
+            # Make all files executable
+            for root, _, files in os.walk(output):
+                for name in files:
+                    os.chmod(os.path.join(root, name), 0o755)
+            output.touch(mode=0o755)  # Update dir modtime
         else:
             with open(output, "wb") as f:
                 shutil.copyfileobj(response, f)

tools/project.py

@@ -32,41 +32,53 @@ if sys.platform == "cygwin":
 class Object:
     def __init__(self, completed: bool, name: str, **options: Any) -> None:
         self.name = name
+        self.base_name = Path(name).with_suffix("")
         self.completed = completed
         self.options: Dict[str, Any] = {
             "add_to_all": True,
+            "asflags": None,
+            "extra_asflags": None,
             "cflags": None,
             "extra_cflags": None,
             "mw_version": None,
-            "shiftjis": True,
+            "shift_jis": None,
             "source": name,
         }
         self.options.update(options)
 
 
+PathLike = Union[str, os.PathLike]
+
+
 class ProjectConfig:
     def __init__(self) -> None:
         # Paths
-        self.build_dir: Path = Path("build")
+        self.build_dir: PathLike = Path("build")  # Output build files
-        self.src_dir: Path = Path("src")
+        self.src_dir: PathLike = Path("src")  # C/C++/asm source files
-        self.tools_dir: Path = Path("tools")
+        self.tools_dir: PathLike = Path("tools")  # Python scripts
+        self.asm_dir: PathLike = Path(
+            "asm"
+        )  # Override incomplete objects (for modding)
 
         # Tooling
+        self.binutils_tag: Optional[str] = None  # Git tag
+        self.binutils_path: Optional[PathLike] = None  # If None, download
         self.dtk_tag: Optional[str] = None  # Git tag
-        self.build_dtk_path: Optional[Path] = None  # If None, download
+        self.build_dtk_path: Optional[PathLike] = None  # If None, download
         self.compilers_tag: Optional[str] = None  # 1
-        self.compilers_path: Optional[Path] = None  # If None, download
+        self.compilers_path: Optional[PathLike] = None  # If None, download
         self.wibo_tag: Optional[str] = None  # Git tag
-        self.wrapper: Optional[Path] = None  # If None, download wibo on Linux
+        self.wrapper: Optional[PathLike] = None  # If None, download wibo on Linux
         self.sjiswrap_tag: Optional[str] = None  # Git tag
-        self.sjiswrap_path: Optional[Path] = None  # If None, download
+        self.sjiswrap_path: Optional[PathLike] = None  # If None, download
 
         # Project config
         self.build_rels: bool = True  # Build REL files
-        self.check_sha_path: Optional[Path] = None  # Path to version.sha1
+        self.check_sha_path: Optional[PathLike] = None  # Path to version.sha1
-        self.config_path: Optional[Path] = None  # Path to config.yml
+        self.config_path: Optional[PathLike] = None  # Path to config.yml
         self.debug: bool = False  # Build with debug info
         self.generate_map: bool = False  # Generate map file(s)
+        self.asflags: Optional[List[str]] = None  # Assembler flags
         self.ldflags: Optional[List[str]] = None  # Linker flags
         self.libs: Optional[List[Dict[str, Any]]] = None  # List of libraries
         self.linker_version: Optional[str] = None  # mwld version
@@ -74,9 +86,12 @@ class ProjectConfig:
         self.warn_missing_config: bool = False  # Warn on missing unit configuration
         self.warn_missing_source: bool = False  # Warn on missing source file
         self.rel_strip_partial: bool = True  # Generate PLFs with -strip_partial
-        self.rel_empty_file: Optional[
+        self.rel_empty_file: Optional[PathLike] = (
-            Path
+            None  # Path to empty.c for generating empty RELs
-        ] = None  # Path to empty.c for generating empty RELs
+        )
+        self.shift_jis = (
+            True  # Convert source files from UTF-8 to Shift JIS automatically
+        )
 
         # Progress output and progress.json config
         self.progress_all: bool = True  # Include combined "all" category
@@ -129,6 +144,13 @@ CHAIN = "cmd /c " if is_windows() else ""
 EXE = ".exe" if is_windows() else ""
 
 
+def make_flags_str(cflags: Union[str, List[str]]) -> str:
+    if isinstance(cflags, list):
+        return " ".join(cflags)
+    else:
+        return cflags
+
+
 # Load decomp-toolkit generated config.json
 def load_build_config(
     config: ProjectConfig, build_config_path: Path
@@ -213,6 +235,15 @@ def generate_build_ninja(
         description="TOOL $out",
     )
 
+    decompctx = config.tools_dir / "decompctx.py"
+    n.rule(
+        name="decompctx",
+        command=f"$python {decompctx} $in -o $out -d $out.d",
+        description="CTX $in",
+        depfile="$out.d",
+        deps="gcc",
+    )
+
     if config.build_dtk_path:
         dtk = build_tools_path / "release" / f"dtk{EXE}"
         n.rule(
@@ -304,6 +335,24 @@ def generate_build_ninja(
     else:
         sys.exit("ProjectConfig.compilers_tag missing")
 
+    binutils_implicit = None
+    if config.binutils_path:
+        binutils = config.binutils_path
+    elif config.binutils_tag:
+        binutils = config.build_dir / "binutils"
+        binutils_implicit = binutils
+        n.build(
+            outputs=binutils,
+            rule="download_tool",
+            implicit=download_tool,
+            variables={
+                "tool": "binutils",
+                "tag": config.binutils_tag,
+            },
+        )
+    else:
+        sys.exit("ProjectConfig.binutils_tag missing")
+
     n.newline()
 
     ###
@@ -325,6 +374,14 @@ def generate_build_ninja(
     mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp"
     mwld_implicit: List[Optional[Path]] = [compilers_implicit or mwld, wrapper_implicit]
 
+    # GNU as
+    gnu_as = binutils / f"powerpc-eabi-as{EXE}"
+    gnu_as_cmd = (
+        f"{CHAIN}{gnu_as} $asflags -o $out $in -MD $out.d"
+        + f" && {dtk} elf fixup $out $out"
+    )
+    gnu_as_implicit = [binutils_implicit or gnu_as, dtk]
+
     if os.name != "nt":
         transform_dep = config.tools_dir / "transform_dep.py"
         mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d"
@@ -350,17 +407,6 @@ def generate_build_ninja(
     )
     n.newline()
 
-    n.comment("Generate REL(s)")
-    makerel_rsp = build_path / "makerel.rsp"
-    n.rule(
-        name="makerel",
-        command=f"{dtk} rel make -w -c $config @{makerel_rsp}",
-        description="REL",
-        rspfile=makerel_rsp,
-        rspfile_content="$in_newline",
-    )
-    n.newline()
-
     n.comment("MWCC build")
     n.rule(
         name="mwcc",
@@ -381,6 +427,16 @@ def generate_build_ninja(
     )
     n.newline()
 
+    n.comment("Assemble asm")
+    n.rule(
+        name="as",
+        command=gnu_as_cmd,
+        description="AS $out",
+        depfile="$out.d",
+        deps="gcc",
+    )
+    n.newline()
+
     n.comment("Host build")
     n.variable("host_cflags", "-I include -Wno-trigraphs")
     n.variable(
@@ -403,6 +459,7 @@ def generate_build_ninja(
     # Source files
     ###
     n.comment("Source files")
+    build_asm_path = build_path / "mod"
     build_src_path = build_path / "src"
     build_host_path = build_path / "host"
     build_config_path = build_path / "config.json"
@@ -501,13 +558,109 @@ def generate_build_ninja(
         host_source_inputs: List[Path] = []
         source_added: Set[Path] = set()
 
-        def make_cflags_str(cflags: Union[str, List[str]]) -> str:
+        def c_build(
-            if isinstance(cflags, list):
+            obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path
-                return " ".join(cflags)
+        ) -> Optional[Path]:
-            else:
-                return cflags
 
-        def add_unit(build_obj: Dict[str, Any], link_step: LinkStep) -> None:
+            cflags_str = make_flags_str(options["cflags"])
+            if options["extra_cflags"] is not None:
+                extra_cflags_str = make_flags_str(options["extra_cflags"])
+                cflags_str += " " + extra_cflags_str
+            used_compiler_versions.add(options["mw_version"])
+
+            src_obj_path = build_src_path / f"{obj.base_name}.o"
+            src_base_path = build_src_path / obj.base_name
+
+            # Avoid creating duplicate build rules
+            if src_obj_path in source_added:
+                return src_obj_path
+            source_added.add(src_obj_path)
+
+            shift_jis = options["shift_jis"]
+            if shift_jis is None:
+                shift_jis = config.shift_jis
+
+            # Add MWCC build rule
+            n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})")
+            n.build(
+                outputs=src_obj_path,
+                rule="mwcc_sjis" if shift_jis else "mwcc",
+                inputs=src_path,
+                variables={
+                    "mw_version": Path(options["mw_version"]),
+                    "cflags": cflags_str,
+                    "basedir": os.path.dirname(src_base_path),
+                    "basefile": src_base_path,
+                },
+                implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit,
+            )
+
+            # Add ctx build rule
+            ctx_path = build_src_path / f"{obj.base_name}.ctx"
+            n.build(
+                outputs=ctx_path,
+                rule="decompctx",
+                inputs=src_path,
+                implicit=decompctx,
+            )
+
+            # Add host build rule
+            if options.get("host", False):
+                host_obj_path = build_host_path / f"{obj.base_name}.o"
+                host_base_path = build_host_path / obj.base_name
+                n.build(
+                    outputs=host_obj_path,
+                    rule="host_cc" if src_path.suffix == ".c" else "host_cpp",
+                    inputs=src_path,
+                    variables={
+                        "basedir": os.path.dirname(host_base_path),
+                        "basefile": host_base_path,
+                    },
+                )
+                if options["add_to_all"]:
+                    host_source_inputs.append(host_obj_path)
+            n.newline()
+
+            if options["add_to_all"]:
+                source_inputs.append(src_obj_path)
+
+            return src_obj_path
+
+        def asm_build(
+            obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path
+        ) -> Optional[Path]:
+            asflags = options["asflags"] or config.asflags
+            if asflags is None:
+                sys.exit("ProjectConfig.asflags missing")
+            asflags_str = make_flags_str(asflags)
+            if options["extra_asflags"] is not None:
+                extra_asflags_str = make_flags_str(options["extra_asflags"])
+                asflags_str += " " + extra_asflags_str
+
+            asm_obj_path = build_asm_path / f"{obj.base_name}.o"
+
+            # Avoid creating duplicate build rules
+            if asm_obj_path in source_added:
+                return asm_obj_path
+            source_added.add(asm_obj_path)
+
+            # Add assembler build rule
+            n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})")
+            n.build(
+                outputs=asm_obj_path,
+                rule="as",
+                inputs=src_path,
+                variables={"asflags": asflags_str},
+                implicit=gnu_as_implicit,
+            )
+            n.newline()
+
+            if options["add_to_all"]:
+                source_inputs.append(asm_obj_path)
+
+            return asm_obj_path
+
+        def add_unit(build_obj, link_step: LinkStep):
             obj_path, obj_name = build_obj["object"], build_obj["name"]
             result = config.find_object(obj_name)
             if not result:
@@ -518,71 +671,47 @@ def generate_build_ninja(
 
             lib, obj = result
             lib_name = lib["lib"]
-            src_dir = Path(lib.get("src_dir", config.src_dir))
 
-            options = obj.options
+            # Use object options, then library options
-            completed = obj.completed
+            options = lib.copy()
+            for key, value in obj.options.items():
+                if value is not None or key not in options:
+                    options[key] = value
 
-            unit_src_path = src_dir / str(options["source"])
+            unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"]
+            if config.asm_dir is not None:
+                unit_asm_path = (
+                    Path(lib.get("asm_dir", config.asm_dir)) / options["source"]
+                ).with_suffix(".s")
 
-            if not unit_src_path.exists():
+            link_built_obj = obj.completed
-                if config.warn_missing_source or completed:
+            if unit_src_path.exists():
+                if unit_src_path.suffix in (".c", ".cp", ".cpp"):
+                    # Add MWCC & host build rules
+                    built_obj_path = c_build(obj, options, lib_name, unit_src_path)
+                elif unit_src_path.suffix == ".s":
+                    # Add assembler build rule
+                    built_obj_path = asm_build(obj, options, lib_name, unit_src_path)
+                else:
+                    sys.exit(f"Unknown source file type {unit_src_path}")
+            else:
+                if config.warn_missing_source or obj.completed:
                     print(f"Missing source file {unit_src_path}")
+                link_built_obj = False
+
+            # Assembly overrides
+            if unit_asm_path is not None and unit_asm_path.exists():
+                link_built_obj = True
+                built_obj_path = asm_build(obj, options, lib_name, unit_asm_path)
+
+            if link_built_obj and built_obj_path is not None:
+                # Use the source-built object
+                link_step.add(built_obj_path)
+            elif obj_path is not None:
+                # Use the original (extracted) object
                 link_step.add(obj_path)
-                return
+            else:
-
+                sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}")
-            mw_version = options["mw_version"] or lib["mw_version"]
-            cflags_str = make_cflags_str(options["cflags"] or lib["cflags"])
-            if options["extra_cflags"] is not None:
-                extra_cflags_str = make_cflags_str(options["extra_cflags"])
-                cflags_str += " " + extra_cflags_str
-            used_compiler_versions.add(mw_version)
-
-            base_object = Path(obj.name).with_suffix("")
-            src_obj_path = build_src_path / f"{base_object}.o"
-            src_base_path = build_src_path / base_object
-
-            if src_obj_path not in source_added:
-                source_added.add(src_obj_path)
-
-                n.comment(f"{obj_name}: {lib_name} (linked {completed})")
-                n.build(
-                    outputs=src_obj_path,
-                    rule="mwcc_sjis" if options["shiftjis"] else "mwcc",
-                    inputs=unit_src_path,
-                    variables={
-                        "mw_version": Path(mw_version),
-                        "cflags": cflags_str,
-                        "basedir": os.path.dirname(src_base_path),
-                        "basefile": src_base_path,
-                    },
-                    implicit=mwcc_sjis_implicit
-                    if options["shiftjis"]
-                    else mwcc_implicit,
-                )
-
-                if lib["host"]:
-                    host_obj_path = build_host_path / f"{base_object}.o"
-                    host_base_path = build_host_path / base_object
-                    n.build(
-                        outputs=host_obj_path,
-                        rule="host_cc" if unit_src_path.suffix == ".c" else "host_cpp",
-                        inputs=unit_src_path,
-                        variables={
-                            "basedir": os.path.dirname(host_base_path),
-                            "basefile": host_base_path,
-                        },
-                    )
-                    if options["add_to_all"]:
-                        host_source_inputs.append(host_obj_path)
-                n.newline()
-
-                if options["add_to_all"]:
-                    source_inputs.append(src_obj_path)
-
-            if completed:
-                obj_path = src_obj_path
-            link_step.add(obj_path)
 
         # Add DOL link step
         link_step = LinkStep(build_config)
@@ -632,8 +761,19 @@ def generate_build_ninja(
         ###
         # Generate RELs
         ###
-        generated_rels: List[str] = []
+        n.comment("Generate REL(s)")
-        for link in build_config["links"]:
+        flags = "-w"
+        if len(build_config["links"]) > 1:
+            flags += " -q"
+        n.rule(
+            name="makerel",
+            command=f"{dtk} rel make {flags} -c $config $names @$rspfile",
+            description="REL",
+            rspfile="$rspfile",
+            rspfile_content="$in_newline",
+        )
+        generated_rels = []
+        for idx, link in enumerate(build_config["links"]):
             # Map module names to link steps
             link_steps_local = list(
                 filter(
@@ -660,13 +800,23 @@ def generate_build_ninja(
                     rels_to_generate,
                 )
             )
-            n.comment("Generate RELs")
+            rel_names = list(
+                map(
+                    lambda step: step.name,
+                    link_steps_local,
+                )
+            )
+            rel_names_arg = " ".join(map(lambda name: f"-n {name}", rel_names))
             n.build(
                 outputs=rel_outputs,
                 rule="makerel",
                 inputs=list(map(lambda step: step.partial_output(), link_steps_local)),
                 implicit=[dtk, config.config_path],
-                variables={"config": config.config_path},
+                variables={
+                    "config": config.config_path,
+                    "rspfile": config.out_path() / f"rel{idx}.rsp",
+                    "names": rel_names_arg,
+                },
             )
             n.newline()
 
@@ -834,7 +984,7 @@ def generate_objdiff_config(
         return
 
     objdiff_config: Dict[str, Any] = {
-        "min_version": "0.4.3",
+        "min_version": "1.0.0",
         "custom_make": "ninja",
         "build_target": False,
         "watch_patterns": [
@@ -852,6 +1002,38 @@ def generate_objdiff_config(
         "units": [],
     }
 
+    # decomp.me compiler name mapping
+    # Commented out versions have not been added to decomp.me yet
+    COMPILER_MAP = {
+        "GC/1.0": "mwcc_233_144",
+        "GC/1.1": "mwcc_233_159",
+        "GC/1.2.5": "mwcc_233_163",
+        "GC/1.2.5e": "mwcc_233_163e",
+        "GC/1.2.5n": "mwcc_233_163n",
+        "GC/1.3.2": "mwcc_242_81",
+        "GC/1.3.2r": "mwcc_242_81r",
+        "GC/2.0": "mwcc_247_92",
+        "GC/2.5": "mwcc_247_105",
+        "GC/2.6": "mwcc_247_107",
+        "GC/2.7": "mwcc_247_108",
+        "GC/3.0": "mwcc_41_60831",
+        # "GC/3.0a3": "mwcc_41_51213",
+        "GC/3.0a3.2": "mwcc_41_60126",
+        # "GC/3.0a3.3": "mwcc_41_60209",
+        # "GC/3.0a3.4": "mwcc_42_60308",
+        # "GC/3.0a5": "mwcc_42_60422",
+        "GC/3.0a5.2": "mwcc_41_60831",
+        "Wii/0x4201_127": "mwcc_42_142",
+        # "Wii/1.0": "mwcc_43_145",
+        # "Wii/1.0RC1": "mwcc_42_140",
+        "Wii/1.0a": "mwcc_42_142",
+        "Wii/1.1": "mwcc_43_151",
+        "Wii/1.3": "mwcc_43_172",
+        # "Wii/1.5": "mwcc_43_188",
+        "Wii/1.6": "mwcc_43_202",
+        "Wii/1.7": "mwcc_43_213",
+    }
+
     build_path = config.out_path()
 
     def add_unit(build_obj: Dict[str, Any], module_name: str) -> None:
@@ -874,14 +1056,21 @@ def generate_objdiff_config(
         lib, obj = result
         src_dir = Path(lib.get("src_dir", config.src_dir))
 
-        unit_src_path = src_dir / str(obj.options["source"])
+        # Use object options, then library options
+        options = lib.copy()
+        for key, value in obj.options.items():
+            if value is not None or key not in options:
+                options[key] = value
+
+        unit_src_path = src_dir / str(options["source"])
 
         if not unit_src_path.exists():
             objdiff_config["units"].append(unit_config)
             return
 
-        cflags = obj.options["cflags"] or lib["cflags"]
+        cflags = options["cflags"]
-        src_obj_path = build_path / "src" / f"{base_object}.o"
+        src_obj_path = build_path / "src" / f"{obj.base_name}.o"
+        src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx"
 
         reverse_fn_order = False
         if type(cflags) is list:
@@ -894,9 +1083,32 @@ def generate_objdiff_config(
                     elif value == "nodeferred":
                         reverse_fn_order = False
 
+            # Filter out include directories
+            def keep_flag(flag):
+                return not flag.startswith("-i ") and not flag.startswith("-I ")
+
+            cflags = list(filter(keep_flag, cflags))
+
+            # Add appropriate lang flag
+            if unit_src_path.suffix in (".cp", ".cpp"):
+                cflags.insert(0, "-lang=c++")
+            else:
+                cflags.insert(0, "-lang=c")
+
         unit_config["base_path"] = src_obj_path
         unit_config["reverse_fn_order"] = reverse_fn_order
         unit_config["complete"] = obj.completed
+        compiler_version = COMPILER_MAP.get(options["mw_version"])
+        if compiler_version is None:
+            print(f"Missing scratch compiler mapping for {options['mw_version']}")
+        else:
+            unit_config["scratch"] = {
+                "platform": "gc_wii",
+                "compiler": compiler_version,
+                "c_flags": make_flags_str(cflags),
+                "ctx_path": src_ctx_path,
+                "build_ctx": True,
+            }
         objdiff_config["units"].append(unit_config)
 
     # Add DOL units