diff --git a/.gitattributes b/.gitattributes index 9545db5..47d4301 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,5 +9,5 @@ *.sh text eol=lf *.sha1 text eol=lf -# DTK keeps these files with LF +# decomp-toolkit writes files with LF config/**/*.txt text eol=lf diff --git a/.gitignore b/.gitignore index d37c144..5f9962b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ objdiff.json orig/*/* !orig/*/.gitkeep /*.txt +ctx.c diff --git a/README.md b/README.md index 6d77cac..a9fbe93 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,16 @@ References Projects using this structure: - [zeldaret/tww](https://github.com/zeldaret/tww) +- [PrimeDecomp/prime](https://github.com/PrimeDecomp/prime) - [PrimeDecomp/echoes](https://github.com/PrimeDecomp/echoes) - [DarkRTA/rb3](https://github.com/DarkRTA/rb3) +- [InputEvelution/sadx-dtk](https://github.com/InputEvelution/sadx-dtk) - [InputEvelution/wp](https://github.com/InputEvelution/wp) +- [lepelog/ss-dtk](https://github.com/lepelog/ss-dtk) +- [NWPlayer123/AnimalCrossing-dtk](https://github.com/NWPlayer123/AnimalCrossing-dtk) +- [Rainchus/mp4-dtk](https://github.com/Rainchus/mp4-dtk) - [Rainchus/ttyd_dtk](https://github.com/Rainchus/ttyd_dtk) +- [Sage-of-Mirrors/zmansion](https://github.com/Sage-of-Mirrors/zmansion) Features -------- diff --git a/configure.py b/configure.py index 01db743..0691381 100644 --- a/configure.py +++ b/configure.py @@ -117,10 +117,10 @@ if not is_windows(): config.wrapper = args.wrapper # Tool versions -config.compilers_tag = "20230715" -config.dtk_tag = "v0.5.7" +config.compilers_tag = "20231018" +config.dtk_tag = "v0.6.0" config.sjiswrap_tag = "v1.1.1" -config.wibo_tag = "0.6.3" +config.wibo_tag = "0.6.9" # Project config.config_path = Path("config") / config.version / "config.yml" @@ -128,7 +128,7 @@ config.check_sha_path = Path("config") / config.version / "build.sha1" config.ldflags = [ "-fp hardware", "-nodefaults", - "-listclosure", + # "-listclosure", # Uncomment for Wii linkers ] # Base flags, common to most GC/Wii games. @@ -151,8 +151,7 @@ cflags_base = [ "-fp_contract on", "-str reuse", "-i include", - "-i libc", - "-enc SJIS", + "-multibyte", # For Wii compilers, replace with `-enc SJIS` f"-DVERSION={version_num}", ] @@ -179,14 +178,14 @@ cflags_rel = [ "-sdata2 0", ] -config.linker_version = "Wii/1.3" +config.linker_version = "GC/1.3.2" # Helper function for Dolphin libraries def DolphinLib(lib_name, objects): return { "lib": lib_name, - "mw_version": "Wii/1.1", + "mw_version": "GC/1.2.5n", "cflags": cflags_base, "host": False, "objects": objects, @@ -197,7 +196,7 @@ def DolphinLib(lib_name, objects): def Rel(lib_name, objects): return { "lib": lib_name, - "mw_version": "Wii/1.3", + "mw_version": "GC/1.3.2", "cflags": cflags_rel, "host": True, "objects": objects, diff --git a/tools/decompctx.py b/tools/decompctx.py index c42b182..3b27d6c 100644 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -84,4 +84,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/download_tool.py b/tools/download_tool.py index 681c65b..fef42d6 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -16,7 +16,6 @@ import os import platform import shutil import stat -import sys import urllib.request import zipfile @@ -52,6 +51,7 @@ def wibo_url(tag): def compilers_url(tag): return f"https://files.decomp.dev/compilers_{tag}.zip" + TOOLS = { "dtk": dtk_url, "sjiswrap": sjiswrap_url, @@ -86,4 +86,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py index ffd88a0..5f706df 100644 --- a/tools/ninja_syntax.py +++ b/tools/ninja_syntax.py @@ -220,4 +220,4 @@ def expand(string, vars, local_vars={}): return "$" return local_vars.get(var, vars.get(var, "")) - return re.sub(r"\$(\$|\w*)", exp, string) \ No newline at end of file + return re.sub(r"\$(\$|\w*)", exp, string) diff --git a/tools/project.py b/tools/project.py index 28a8d32..bcdeba0 100644 --- a/tools/project.py +++ b/tools/project.py @@ -56,13 +56,13 @@ class ProjectConfig: self.version = None # Version name self.warn_missing_config = False # Warn on missing unit configuration self.warn_missing_source = False # Warn on missing source file + self.rel_strip_partial = True # Generate PLFs with -strip_partial + self.rel_empty_file = None # Path to empty.c for generating empty RELs # Progress output and progress.json config self.progress_all = True # Include combined "all" category self.progress_modules = True # Include combined "modules" category - self.progress_each_module = ( - True # Include individual modules, disable for large numbers of modules - ) + self.progress_each_module = True # Include individual modules, disable for large numbers of modules def validate(self): required_attrs = [ @@ -98,6 +98,7 @@ class Object: self.options = { "add_to_all": True, "cflags": None, + "extra_cflags": None, "mw_version": None, "shiftjis": True, "source": name, @@ -461,7 +462,12 @@ def generate_build_ninja(config, build_config): preplf_path = build_path / self.name / f"{self.name}.preplf" plf_path = build_path / self.name / f"{self.name}.plf" preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" - plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}" + plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}" + if self.entry: + plf_ldflags += f" -m {self.entry}" + # -strip_partial is only valid with -m + if config.rel_strip_partial: + plf_ldflags += " -strip_partial" if config.generate_map: preplf_map = map_path(preplf_path) preplf_ldflags += f" -map {preplf_map}" @@ -495,6 +501,12 @@ def generate_build_ninja(config, build_config): host_source_inputs = [] source_added = set() + def make_cflags_str(cflags): + if isinstance(cflags, list): + return " ".join(cflags) + else: + return cflags + def add_unit(build_obj, link_step): obj_path, obj_name = build_obj["object"], build_obj["name"] result = config.find_object(obj_name) @@ -512,17 +524,16 @@ def generate_build_ninja(config, build_config): unit_src_path = config.src_dir / options["source"] if not unit_src_path.exists(): - if config.warn_missing_source: + if config.warn_missing_source or completed: print(f"Missing source file {unit_src_path}") link_step.add(obj_path) return mw_version = options["mw_version"] or lib["mw_version"] - cflags = options["cflags"] or lib["cflags"] - if type(cflags) is list: - cflags_str = " ".join(cflags) - else: - cflags_str = str(cflags) + cflags_str = make_cflags_str(options["cflags"] or lib["cflags"]) + if options["extra_cflags"] is not None: + extra_cflags_str = make_cflags_str(options["extra_cflags"]) + cflags_str += " " + extra_cflags_str used_compiler_versions.add(mw_version) base_object = Path(obj.name).with_suffix("") @@ -583,6 +594,18 @@ def generate_build_ninja(config, build_config): module_link_step = LinkStep(module) for unit in module["units"]: add_unit(unit, module_link_step) + # Add empty object to empty RELs + if len(module_link_step.inputs) == 0: + if not config.rel_empty_file: + sys.exit("ProjectConfig.rel_empty_file missing") + add_unit( + { + "object": None, + "name": config.rel_empty_file, + "autogenerated": True, + }, + module_link_step, + ) link_steps.append(module_link_step) n.newline() @@ -607,18 +630,41 @@ def generate_build_ninja(config, build_config): ### # Generate RELs ### - rel_outputs = list( - map( - lambda step: step.output(), - filter(lambda step: step.module_id != 0, link_steps), + generated_rels = [] + for link in build_config["links"]: + # Map module names to link steps + link_steps_local = list( + filter( + lambda step: step.name in link["modules"], + link_steps, + ) + ) + link_steps_local.sort(key=lambda step: step.module_id) + # RELs can be the output of multiple link steps, + # so we need to filter out duplicates + rels_to_generate = list( + filter( + lambda step: step.module_id != 0 + and not step.name in generated_rels, + link_steps_local, + ) + ) + if len(rels_to_generate) == 0: + continue + generated_rels.extend(map(lambda step: step.name, rels_to_generate)) + rel_outputs = list( + map( + lambda step: step.output(), + rels_to_generate, + ) ) - ) - if len(rel_outputs) > 0: n.comment("Generate RELs") n.build( outputs=path(rel_outputs), rule="makerel", - inputs=path(list(map(lambda step: step.partial_output(), link_steps))), + inputs=path( + list(map(lambda step: step.partial_output(), link_steps_local)) + ), implicit=path([dtk, config.config_path]), variables={"config": path(config.config_path)}, ) @@ -798,6 +844,7 @@ def generate_objdiff_config(config, build_config): "*.cpp", "*.h", "*.hpp", + "*.inc", "*.py", "*.yml", "*.txt", @@ -940,6 +987,9 @@ def calculate_progress(config): print("Progress:") def print_category(unit): + if unit is None: + return + code_frac = unit.code_frac() data_frac = unit.data_frac() print( @@ -979,4 +1029,4 @@ def calculate_progress(config): for progress in modules_progress: add_category(progress.name, progress) with open(out_path / "progress.json", "w", encoding="utf-8") as w: - json.dump(progress_json, w, indent=4) \ No newline at end of file + json.dump(progress_json, w, indent=4) diff --git a/tools/transform_dep.py b/tools/transform_dep.py index 86bd2ec..61d66f8 100644 --- a/tools/transform_dep.py +++ b/tools/transform_dep.py @@ -81,4 +81,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/upload_progress.py b/tools/upload_progress.py index 673bb3d..6ec8f08 100644 --- a/tools/upload_progress.py +++ b/tools/upload_progress.py @@ -73,4 +73,4 @@ if __name__ == "__main__": "entries": entries, }) r.raise_for_status() - print("Done!") \ No newline at end of file + print("Done!")