Sync various changes from other repos & minor updates

.gitattributes

@@ -9,5 +9,5 @@
 *.sh   text eol=lf
 *.sha1 text eol=lf
 
-# DTK keeps these files with LF
+# decomp-toolkit writes files with LF
 config/**/*.txt  text eol=lf

.gitignore

@@ -9,3 +9,4 @@ objdiff.json
 orig/*/*
 !orig/*/.gitkeep
 /*.txt
+ctx.c

README.md

@@ -29,10 +29,16 @@ References
 
 Projects using this structure:
 - [zeldaret/tww](https://github.com/zeldaret/tww)
+- [PrimeDecomp/prime](https://github.com/PrimeDecomp/prime)
 - [PrimeDecomp/echoes](https://github.com/PrimeDecomp/echoes)
 - [DarkRTA/rb3](https://github.com/DarkRTA/rb3)
+- [InputEvelution/sadx-dtk](https://github.com/InputEvelution/sadx-dtk)
 - [InputEvelution/wp](https://github.com/InputEvelution/wp)
+- [lepelog/ss-dtk](https://github.com/lepelog/ss-dtk)
+- [NWPlayer123/AnimalCrossing-dtk](https://github.com/NWPlayer123/AnimalCrossing-dtk)
+- [Rainchus/mp4-dtk](https://github.com/Rainchus/mp4-dtk)
 - [Rainchus/ttyd_dtk](https://github.com/Rainchus/ttyd_dtk)
+- [Sage-of-Mirrors/zmansion](https://github.com/Sage-of-Mirrors/zmansion)
 
 Features
 --------

configure.py

@@ -117,10 +117,10 @@ if not is_windows():
     config.wrapper = args.wrapper
 
 # Tool versions
-config.compilers_tag = "20230715"
+config.compilers_tag = "20231018"
-config.dtk_tag = "v0.5.7"
+config.dtk_tag = "v0.6.0"
 config.sjiswrap_tag = "v1.1.1"
-config.wibo_tag = "0.6.3"
+config.wibo_tag = "0.6.9"
 
 # Project
 config.config_path = Path("config") / config.version / "config.yml"
@@ -128,7 +128,7 @@ config.check_sha_path = Path("config") / config.version / "build.sha1"
 config.ldflags = [
     "-fp hardware",
     "-nodefaults",
-    "-listclosure",
+    # "-listclosure", # Uncomment for Wii linkers
 ]
 
 # Base flags, common to most GC/Wii games.
@@ -151,8 +151,7 @@ cflags_base = [
     "-fp_contract on",
     "-str reuse",
     "-i include",
-    "-i libc",
+	"-multibyte", # For Wii compilers, replace with `-enc SJIS`
-	"-enc SJIS",
     f"-DVERSION={version_num}",
 ]
 
@@ -179,14 +178,14 @@ cflags_rel = [
     "-sdata2 0",
 ]
 
-config.linker_version = "Wii/1.3"
+config.linker_version = "GC/1.3.2"
 
 
 # Helper function for Dolphin libraries
 def DolphinLib(lib_name, objects):
     return {
         "lib": lib_name,
-        "mw_version": "Wii/1.1",
+        "mw_version": "GC/1.2.5n",
         "cflags": cflags_base,
         "host": False,
         "objects": objects,
@@ -197,7 +196,7 @@ def DolphinLib(lib_name, objects):
 def Rel(lib_name, objects):
     return {
         "lib": lib_name,
-        "mw_version": "Wii/1.3",
+        "mw_version": "GC/1.3.2",
         "cflags": cflags_rel,
         "host": True,
         "objects": objects,

tools/decompctx.py

@@ -84,4 +84,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

tools/download_tool.py

@@ -16,7 +16,6 @@ import os
 import platform
 import shutil
 import stat
-import sys
 import urllib.request
 import zipfile
 
@@ -52,6 +51,7 @@ def wibo_url(tag):
 def compilers_url(tag):
     return f"https://files.decomp.dev/compilers_{tag}.zip"
 
+
 TOOLS = {
     "dtk": dtk_url,
     "sjiswrap": sjiswrap_url,
@@ -86,4 +86,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

tools/ninja_syntax.py

@@ -220,4 +220,4 @@ def expand(string, vars, local_vars={}):
             return "$"
         return local_vars.get(var, vars.get(var, ""))
 
-    return re.sub(r"\$(\$|\w*)", exp, string)
+    return re.sub(r"\$(\$|\w*)", exp, string)

tools/project.py

@@ -56,13 +56,13 @@ class ProjectConfig:
         self.version = None  # Version name
         self.warn_missing_config = False  # Warn on missing unit configuration
         self.warn_missing_source = False  # Warn on missing source file
+        self.rel_strip_partial = True  # Generate PLFs with -strip_partial
+        self.rel_empty_file = None  # Path to empty.c for generating empty RELs
 
         # Progress output and progress.json config
         self.progress_all = True  # Include combined "all" category
         self.progress_modules = True  # Include combined "modules" category
-        self.progress_each_module = (
+        self.progress_each_module = True  # Include individual modules, disable for large numbers of modules
-            True  # Include individual modules, disable for large numbers of modules
-        )
 
     def validate(self):
         required_attrs = [
@@ -98,6 +98,7 @@ class Object:
         self.options = {
             "add_to_all": True,
             "cflags": None,
+            "extra_cflags": None,
             "mw_version": None,
             "shiftjis": True,
             "source": name,
@@ -461,7 +462,12 @@ def generate_build_ninja(config, build_config):
                 preplf_path = build_path / self.name / f"{self.name}.preplf"
                 plf_path = build_path / self.name / f"{self.name}.plf"
                 preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r"
-                plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}"
+                plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}"
+                if self.entry:
+                    plf_ldflags += f" -m {self.entry}"
+                    # -strip_partial is only valid with -m
+                    if config.rel_strip_partial:
+                        plf_ldflags += " -strip_partial"
                 if config.generate_map:
                     preplf_map = map_path(preplf_path)
                     preplf_ldflags += f" -map {preplf_map}"
@@ -495,6 +501,12 @@ def generate_build_ninja(config, build_config):
         host_source_inputs = []
         source_added = set()
 
+        def make_cflags_str(cflags):
+            if isinstance(cflags, list):
+                return " ".join(cflags)
+            else:
+                return cflags
+
         def add_unit(build_obj, link_step):
             obj_path, obj_name = build_obj["object"], build_obj["name"]
             result = config.find_object(obj_name)
@@ -512,17 +524,16 @@ def generate_build_ninja(config, build_config):
 
             unit_src_path = config.src_dir / options["source"]
             if not unit_src_path.exists():
-                if config.warn_missing_source:
+                if config.warn_missing_source or completed:
                     print(f"Missing source file {unit_src_path}")
                 link_step.add(obj_path)
                 return
 
             mw_version = options["mw_version"] or lib["mw_version"]
-            cflags = options["cflags"] or lib["cflags"]
+            cflags_str = make_cflags_str(options["cflags"] or lib["cflags"])
-            if type(cflags) is list:
+            if options["extra_cflags"] is not None:
-                cflags_str = " ".join(cflags)
+                extra_cflags_str = make_cflags_str(options["extra_cflags"])
-            else:
+                cflags_str += " " + extra_cflags_str
-                cflags_str = str(cflags)
             used_compiler_versions.add(mw_version)
 
             base_object = Path(obj.name).with_suffix("")
@@ -583,6 +594,18 @@ def generate_build_ninja(config, build_config):
                 module_link_step = LinkStep(module)
                 for unit in module["units"]:
                     add_unit(unit, module_link_step)
+                # Add empty object to empty RELs
+                if len(module_link_step.inputs) == 0:
+                    if not config.rel_empty_file:
+                        sys.exit("ProjectConfig.rel_empty_file missing")
+                    add_unit(
+                        {
+                            "object": None,
+                            "name": config.rel_empty_file,
+                            "autogenerated": True,
+                        },
+                        module_link_step,
+                    )
                 link_steps.append(module_link_step)
         n.newline()
 
@@ -607,18 +630,41 @@ def generate_build_ninja(config, build_config):
         ###
         # Generate RELs
         ###
-        rel_outputs = list(
+        generated_rels = []
-            map(
+        for link in build_config["links"]:
-                lambda step: step.output(),
+            # Map module names to link steps
-                filter(lambda step: step.module_id != 0, link_steps),
+            link_steps_local = list(
+                filter(
+                    lambda step: step.name in link["modules"],
+                    link_steps,
+                )
+            )
+            link_steps_local.sort(key=lambda step: step.module_id)
+            # RELs can be the output of multiple link steps,
+            # so we need to filter out duplicates
+            rels_to_generate = list(
+                filter(
+                    lambda step: step.module_id != 0
+                    and not step.name in generated_rels,
+                    link_steps_local,
+                )
+            )
+            if len(rels_to_generate) == 0:
+                continue
+            generated_rels.extend(map(lambda step: step.name, rels_to_generate))
+            rel_outputs = list(
+                map(
+                    lambda step: step.output(),
+                    rels_to_generate,
+                )
             )
-        )
-        if len(rel_outputs) > 0:
             n.comment("Generate RELs")
             n.build(
                 outputs=path(rel_outputs),
                 rule="makerel",
-                inputs=path(list(map(lambda step: step.partial_output(), link_steps))),
+                inputs=path(
+                    list(map(lambda step: step.partial_output(), link_steps_local))
+                ),
                 implicit=path([dtk, config.config_path]),
                 variables={"config": path(config.config_path)},
             )
@@ -798,6 +844,7 @@ def generate_objdiff_config(config, build_config):
             "*.cpp",
             "*.h",
             "*.hpp",
+            "*.inc",
             "*.py",
             "*.yml",
             "*.txt",
@@ -940,6 +987,9 @@ def calculate_progress(config):
     print("Progress:")
 
     def print_category(unit):
+        if unit is None:
+            return
+
         code_frac = unit.code_frac()
         data_frac = unit.data_frac()
         print(
@@ -979,4 +1029,4 @@ def calculate_progress(config):
             for progress in modules_progress:
                 add_category(progress.name, progress)
     with open(out_path / "progress.json", "w", encoding="utf-8") as w:
-        json.dump(progress_json, w, indent=4)
+        json.dump(progress_json, w, indent=4)

tools/transform_dep.py

@@ -81,4 +81,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

tools/upload_progress.py

@@ -73,4 +73,4 @@ if __name__ == "__main__":
         "entries": entries,
     })
     r.raise_for_status()
-    print("Done!")
+    print("Done!")