Sync various changes from other repos & minor updates

This commit is contained in:
Luke Street 2023-11-21 20:14:13 -05:00
parent 3be5e73054
commit 1bcf33638f
10 changed files with 90 additions and 34 deletions

2
.gitattributes vendored
View File

@ -9,5 +9,5 @@
*.sh text eol=lf *.sh text eol=lf
*.sha1 text eol=lf *.sha1 text eol=lf
# DTK keeps these files with LF # decomp-toolkit writes files with LF
config/**/*.txt text eol=lf config/**/*.txt text eol=lf

1
.gitignore vendored
View File

@ -9,3 +9,4 @@ objdiff.json
orig/*/* orig/*/*
!orig/*/.gitkeep !orig/*/.gitkeep
/*.txt /*.txt
ctx.c

View File

@ -29,10 +29,16 @@ References
Projects using this structure: Projects using this structure:
- [zeldaret/tww](https://github.com/zeldaret/tww) - [zeldaret/tww](https://github.com/zeldaret/tww)
- [PrimeDecomp/prime](https://github.com/PrimeDecomp/prime)
- [PrimeDecomp/echoes](https://github.com/PrimeDecomp/echoes) - [PrimeDecomp/echoes](https://github.com/PrimeDecomp/echoes)
- [DarkRTA/rb3](https://github.com/DarkRTA/rb3) - [DarkRTA/rb3](https://github.com/DarkRTA/rb3)
- [InputEvelution/sadx-dtk](https://github.com/InputEvelution/sadx-dtk)
- [InputEvelution/wp](https://github.com/InputEvelution/wp) - [InputEvelution/wp](https://github.com/InputEvelution/wp)
- [lepelog/ss-dtk](https://github.com/lepelog/ss-dtk)
- [NWPlayer123/AnimalCrossing-dtk](https://github.com/NWPlayer123/AnimalCrossing-dtk)
- [Rainchus/mp4-dtk](https://github.com/Rainchus/mp4-dtk)
- [Rainchus/ttyd_dtk](https://github.com/Rainchus/ttyd_dtk) - [Rainchus/ttyd_dtk](https://github.com/Rainchus/ttyd_dtk)
- [Sage-of-Mirrors/zmansion](https://github.com/Sage-of-Mirrors/zmansion)
Features Features
-------- --------

View File

@ -117,10 +117,10 @@ if not is_windows():
config.wrapper = args.wrapper config.wrapper = args.wrapper
# Tool versions # Tool versions
config.compilers_tag = "20230715" config.compilers_tag = "20231018"
config.dtk_tag = "v0.5.7" config.dtk_tag = "v0.6.0"
config.sjiswrap_tag = "v1.1.1" config.sjiswrap_tag = "v1.1.1"
config.wibo_tag = "0.6.3" config.wibo_tag = "0.6.9"
# Project # Project
config.config_path = Path("config") / config.version / "config.yml" config.config_path = Path("config") / config.version / "config.yml"
@ -128,7 +128,7 @@ config.check_sha_path = Path("config") / config.version / "build.sha1"
config.ldflags = [ config.ldflags = [
"-fp hardware", "-fp hardware",
"-nodefaults", "-nodefaults",
"-listclosure", # "-listclosure", # Uncomment for Wii linkers
] ]
# Base flags, common to most GC/Wii games. # Base flags, common to most GC/Wii games.
@ -151,8 +151,7 @@ cflags_base = [
"-fp_contract on", "-fp_contract on",
"-str reuse", "-str reuse",
"-i include", "-i include",
"-i libc", "-multibyte", # For Wii compilers, replace with `-enc SJIS`
"-enc SJIS",
f"-DVERSION={version_num}", f"-DVERSION={version_num}",
] ]
@ -179,14 +178,14 @@ cflags_rel = [
"-sdata2 0", "-sdata2 0",
] ]
config.linker_version = "Wii/1.3" config.linker_version = "GC/1.3.2"
# Helper function for Dolphin libraries # Helper function for Dolphin libraries
def DolphinLib(lib_name, objects): def DolphinLib(lib_name, objects):
return { return {
"lib": lib_name, "lib": lib_name,
"mw_version": "Wii/1.1", "mw_version": "GC/1.2.5n",
"cflags": cflags_base, "cflags": cflags_base,
"host": False, "host": False,
"objects": objects, "objects": objects,
@ -197,7 +196,7 @@ def DolphinLib(lib_name, objects):
def Rel(lib_name, objects): def Rel(lib_name, objects):
return { return {
"lib": lib_name, "lib": lib_name,
"mw_version": "Wii/1.3", "mw_version": "GC/1.3.2",
"cflags": cflags_rel, "cflags": cflags_rel,
"host": True, "host": True,
"objects": objects, "objects": objects,

View File

@ -84,4 +84,4 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@ -16,7 +16,6 @@ import os
import platform import platform
import shutil import shutil
import stat import stat
import sys
import urllib.request import urllib.request
import zipfile import zipfile
@ -52,6 +51,7 @@ def wibo_url(tag):
def compilers_url(tag): def compilers_url(tag):
return f"https://files.decomp.dev/compilers_{tag}.zip" return f"https://files.decomp.dev/compilers_{tag}.zip"
TOOLS = { TOOLS = {
"dtk": dtk_url, "dtk": dtk_url,
"sjiswrap": sjiswrap_url, "sjiswrap": sjiswrap_url,
@ -86,4 +86,4 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@ -220,4 +220,4 @@ def expand(string, vars, local_vars={}):
return "$" return "$"
return local_vars.get(var, vars.get(var, "")) return local_vars.get(var, vars.get(var, ""))
return re.sub(r"\$(\$|\w*)", exp, string) return re.sub(r"\$(\$|\w*)", exp, string)

View File

@ -56,13 +56,13 @@ class ProjectConfig:
self.version = None # Version name self.version = None # Version name
self.warn_missing_config = False # Warn on missing unit configuration self.warn_missing_config = False # Warn on missing unit configuration
self.warn_missing_source = False # Warn on missing source file self.warn_missing_source = False # Warn on missing source file
self.rel_strip_partial = True # Generate PLFs with -strip_partial
self.rel_empty_file = None # Path to empty.c for generating empty RELs
# Progress output and progress.json config # Progress output and progress.json config
self.progress_all = True # Include combined "all" category self.progress_all = True # Include combined "all" category
self.progress_modules = True # Include combined "modules" category self.progress_modules = True # Include combined "modules" category
self.progress_each_module = ( self.progress_each_module = True # Include individual modules, disable for large numbers of modules
True # Include individual modules, disable for large numbers of modules
)
def validate(self): def validate(self):
required_attrs = [ required_attrs = [
@ -98,6 +98,7 @@ class Object:
self.options = { self.options = {
"add_to_all": True, "add_to_all": True,
"cflags": None, "cflags": None,
"extra_cflags": None,
"mw_version": None, "mw_version": None,
"shiftjis": True, "shiftjis": True,
"source": name, "source": name,
@ -461,7 +462,12 @@ def generate_build_ninja(config, build_config):
preplf_path = build_path / self.name / f"{self.name}.preplf" preplf_path = build_path / self.name / f"{self.name}.preplf"
plf_path = build_path / self.name / f"{self.name}.plf" plf_path = build_path / self.name / f"{self.name}.plf"
preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r"
plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}" plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r1 -lcf {self.ldscript}"
if self.entry:
plf_ldflags += f" -m {self.entry}"
# -strip_partial is only valid with -m
if config.rel_strip_partial:
plf_ldflags += " -strip_partial"
if config.generate_map: if config.generate_map:
preplf_map = map_path(preplf_path) preplf_map = map_path(preplf_path)
preplf_ldflags += f" -map {preplf_map}" preplf_ldflags += f" -map {preplf_map}"
@ -495,6 +501,12 @@ def generate_build_ninja(config, build_config):
host_source_inputs = [] host_source_inputs = []
source_added = set() source_added = set()
def make_cflags_str(cflags):
if isinstance(cflags, list):
return " ".join(cflags)
else:
return cflags
def add_unit(build_obj, link_step): def add_unit(build_obj, link_step):
obj_path, obj_name = build_obj["object"], build_obj["name"] obj_path, obj_name = build_obj["object"], build_obj["name"]
result = config.find_object(obj_name) result = config.find_object(obj_name)
@ -512,17 +524,16 @@ def generate_build_ninja(config, build_config):
unit_src_path = config.src_dir / options["source"] unit_src_path = config.src_dir / options["source"]
if not unit_src_path.exists(): if not unit_src_path.exists():
if config.warn_missing_source: if config.warn_missing_source or completed:
print(f"Missing source file {unit_src_path}") print(f"Missing source file {unit_src_path}")
link_step.add(obj_path) link_step.add(obj_path)
return return
mw_version = options["mw_version"] or lib["mw_version"] mw_version = options["mw_version"] or lib["mw_version"]
cflags = options["cflags"] or lib["cflags"] cflags_str = make_cflags_str(options["cflags"] or lib["cflags"])
if type(cflags) is list: if options["extra_cflags"] is not None:
cflags_str = " ".join(cflags) extra_cflags_str = make_cflags_str(options["extra_cflags"])
else: cflags_str += " " + extra_cflags_str
cflags_str = str(cflags)
used_compiler_versions.add(mw_version) used_compiler_versions.add(mw_version)
base_object = Path(obj.name).with_suffix("") base_object = Path(obj.name).with_suffix("")
@ -583,6 +594,18 @@ def generate_build_ninja(config, build_config):
module_link_step = LinkStep(module) module_link_step = LinkStep(module)
for unit in module["units"]: for unit in module["units"]:
add_unit(unit, module_link_step) add_unit(unit, module_link_step)
# Add empty object to empty RELs
if len(module_link_step.inputs) == 0:
if not config.rel_empty_file:
sys.exit("ProjectConfig.rel_empty_file missing")
add_unit(
{
"object": None,
"name": config.rel_empty_file,
"autogenerated": True,
},
module_link_step,
)
link_steps.append(module_link_step) link_steps.append(module_link_step)
n.newline() n.newline()
@ -607,18 +630,41 @@ def generate_build_ninja(config, build_config):
### ###
# Generate RELs # Generate RELs
### ###
rel_outputs = list( generated_rels = []
map( for link in build_config["links"]:
lambda step: step.output(), # Map module names to link steps
filter(lambda step: step.module_id != 0, link_steps), link_steps_local = list(
filter(
lambda step: step.name in link["modules"],
link_steps,
)
)
link_steps_local.sort(key=lambda step: step.module_id)
# RELs can be the output of multiple link steps,
# so we need to filter out duplicates
rels_to_generate = list(
filter(
lambda step: step.module_id != 0
and not step.name in generated_rels,
link_steps_local,
)
)
if len(rels_to_generate) == 0:
continue
generated_rels.extend(map(lambda step: step.name, rels_to_generate))
rel_outputs = list(
map(
lambda step: step.output(),
rels_to_generate,
)
) )
)
if len(rel_outputs) > 0:
n.comment("Generate RELs") n.comment("Generate RELs")
n.build( n.build(
outputs=path(rel_outputs), outputs=path(rel_outputs),
rule="makerel", rule="makerel",
inputs=path(list(map(lambda step: step.partial_output(), link_steps))), inputs=path(
list(map(lambda step: step.partial_output(), link_steps_local))
),
implicit=path([dtk, config.config_path]), implicit=path([dtk, config.config_path]),
variables={"config": path(config.config_path)}, variables={"config": path(config.config_path)},
) )
@ -798,6 +844,7 @@ def generate_objdiff_config(config, build_config):
"*.cpp", "*.cpp",
"*.h", "*.h",
"*.hpp", "*.hpp",
"*.inc",
"*.py", "*.py",
"*.yml", "*.yml",
"*.txt", "*.txt",
@ -940,6 +987,9 @@ def calculate_progress(config):
print("Progress:") print("Progress:")
def print_category(unit): def print_category(unit):
if unit is None:
return
code_frac = unit.code_frac() code_frac = unit.code_frac()
data_frac = unit.data_frac() data_frac = unit.data_frac()
print( print(
@ -979,4 +1029,4 @@ def calculate_progress(config):
for progress in modules_progress: for progress in modules_progress:
add_category(progress.name, progress) add_category(progress.name, progress)
with open(out_path / "progress.json", "w", encoding="utf-8") as w: with open(out_path / "progress.json", "w", encoding="utf-8") as w:
json.dump(progress_json, w, indent=4) json.dump(progress_json, w, indent=4)

View File

@ -81,4 +81,4 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@ -73,4 +73,4 @@ if __name__ == "__main__":
"entries": entries, "entries": entries,
}) })
r.raise_for_status() r.raise_for_status()
print("Done!") print("Done!")