commit 5b8fbc6cd5f96ee77ec4c4369e492371656f279b Author: Luke Street Date: Wed Oct 4 23:26:02 2023 -0400 Initial commit diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9545db5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Explicitly declare text files +*.py text + +# Enforce platform-specific encodings +*.bat text eol=crlf +*.sh text eol=lf +*.sha1 text eol=lf + +# DTK keeps these files with LF +config/**/*.txt text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d37c144 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +__pycache__ +.idea +.vscode +.ninja_* +*.exe +build +build.ninja +objdiff.json +orig/*/* +!orig/*/.gitkeep +/*.txt diff --git a/.vscode.example/c_cpp_properties.json b/.vscode.example/c_cpp_properties.json new file mode 100644 index 0000000..60e51b8 --- /dev/null +++ b/.vscode.example/c_cpp_properties.json @@ -0,0 +1,22 @@ +{ + "configurations": [ + { + "name": "Linux", + "includePath": [ + "${workspaceFolder}/include/**" + ], + "cStandard": "c99", + "cppStandard": "c++98", + "intelliSenseMode": "linux-clang-x86", + "compilerPath": "", + "configurationProvider": "ms-vscode.makefile-tools", + "browse": { + "path": [ + "${workspaceFolder}/include" + ], + "limitSymbolsToIncludedHeaders": true + } + } + ], + "version": 4 +} diff --git a/.vscode.example/settings.json b/.vscode.example/settings.json new file mode 100644 index 0000000..60e7e27 --- /dev/null +++ b/.vscode.example/settings.json @@ -0,0 +1,26 @@ +{ + "[c]": { + "files.encoding": "utf8", + "editor.defaultFormatter": "xaver.clang-format" + }, + "[cpp]": { + "files.encoding": "utf8", + "editor.defaultFormatter": "xaver.clang-format" + }, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "files.associations": { + "*.inc": "cpp" + }, + "search.useIgnoreFiles": false, + "search.exclude": { + "build/*/config.json": true, + "build/**/*.MAP": true, + "build.ninja": true, + ".ninja_*": true, + "objdiff.json": true + } +} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..af8690a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Luke Street + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.example.md b/README.example.md new file mode 100644 index 0000000..af8f9a1 --- /dev/null +++ b/README.example.md @@ -0,0 +1,114 @@ +Some Game +[![Build Status]][actions] ![Progress] ![DOL Progress] ![RELs Progress] [![Discord Badge]][discord] +============= + + +[Build Status]: https://github.com/zeldaret/tww/actions/workflows/build.yml/badge.svg +[actions]: https://github.com/zeldaret/tww/actions/workflows/build.yml + +[Progress]: https://img.shields.io/endpoint?label=Code&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Ftww%2FGZLE01%2Fall%2F%3Fmode%3Dshield%26measure%3Dcode + +[DOL Progress]: https://img.shields.io/endpoint?label=DOL&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Ftww%2FGZLE01%2Fdol%2F%3Fmode%3Dshield%26measure%3Dcode + +[RELs Progress]: https://img.shields.io/endpoint?label=RELs&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Ftww%2FGZLE01%2Fmodules%2F%3Fmode%3Dshield%26measure%3Dcode + +[Discord Badge]: https://img.shields.io/discord/727908905392275526?color=%237289DA&logo=discord&logoColor=%23FFFFFF +[discord]: https://discord.gg/hKx3FJJgrV + +A work-in-progress decompilation of Some Game. + +This repository does **not** contain any game assets or assembly whatsoever. An existing copy of the game is required. + +Supported versions: + +- `GAMEID`: Rev 0 (USA) + +Dependencies +============ + +Windows: +-------- + +On Windows, it's **highly recommended** to use native tooling. WSL or msys2 are **not** required. +When running under WSL, [objdiff](#diffing) is unable to get filesystem notifications for automatic rebuilds. + +- Install [Python](https://www.python.org/downloads/) and add it to `%PATH%`. + - Also available from the [Windows Store](https://apps.microsoft.com/store/detail/python-311/9NRWMJP3717K). +- Download [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`. + - Quick install via pip: `pip install ninja` + +macOS: +------ +- Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages): + ``` + brew install ninja + ``` +- Install [wine-crossover](https://github.com/Gcenx/homebrew-wine): + ``` + brew install --cask --no-quarantine gcenx/wine/wine-crossover + ``` + +After OS upgrades, if macOS complains about `Wine Crossover.app` being unverified, you can unquarantine it using: +```sh +sudo xattr -rd com.apple.quarantine '/Applications/Wine Crossover.app' +``` + +Linux: +------ +- Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages). +- For non-x86(_64) platforms: Install wine from your package manager. + - For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used. + +Building +======== + +- Clone the repository: + ``` + git clone https://github.com/my/repo.git + ``` +- Using [Dolphin Emulator](https://dolphin-emu.org/), extract your game to `orig/GAMEID`. +![](assets/dolphin-extract.png) + - To save space, the only necessary files are the following. Any others can be deleted. + - `sys/main.dol` + - `files/rels/*.rel` +- Configure: + ``` + python configure.py + ``` + To use a version other than `GAMEID` (USA), specify it with `--version`. +- Build: + ``` + ninja + ``` + +Visual Studio Code +================== + +If desired, use the recommended Visual Studio Code settings by renaming the `.vscode.example` directory to `.vscode`. + +Diffing +======= + +Once the initial build succeeds, an `objdiff.json` should exist in the project root. + +Download the latest release from [encounter/objdiff](https://github.com/encounter/objdiff). Under project settings, set `Project directory`. The configuration should be loaded automatically. + +Select an object from the left sidebar to begin diffing. Changes to the project will rebuild automatically: changes to source files, headers, `configure.py`, `splits.txt` or `symbols.txt`. + +![](assets/objdiff.png) diff --git a/README.md b/README.md new file mode 100644 index 0000000..6d77cac --- /dev/null +++ b/README.md @@ -0,0 +1,65 @@ +decomp-toolkit Project Template +=============================== + +If starting a new GameCube / Wii decompilation project, this repository can be used as a scaffold. + +See [decomp-toolkit](https://github.com/encounter/decomp-toolkit) for background on the concept and more information on the tooling used. + +Documentation +------------- + +- [Dependencies](docs/dependencies.md) +- [Getting Started](docs/getting_started.md) +- [`symbols.txt`](docs/symbols.md) +- [`splits.txt`](docs/splits.md) + +General: +- [Common BSS](docs/common_bss.md) +- [`.comment` section](docs/comment_section.md) + +References +-------- + +- [Discord: GC/Wii Decompilation](https://discord.gg/hKx3FJJgrV) (Come to `#dtk` for help!) +- [objdiff](https://github.com/encounter/objdiff) (Local diffing tool) +- [decomp.me](https://decomp.me) (Collaborate on matches) +- [frogress](https://github.com/decompals/frogress) (Decompilation progress API) +- [wibo](https://github.com/decompals/wibo) (Minimal Win32 wrapper for Linux) +- [sjiswrap](https://github.com/encounter/sjiswrap) (UTF-8 to Shift JIS wrapper) + +Projects using this structure: +- [zeldaret/tww](https://github.com/zeldaret/tww) +- [PrimeDecomp/echoes](https://github.com/PrimeDecomp/echoes) +- [DarkRTA/rb3](https://github.com/DarkRTA/rb3) +- [InputEvelution/wp](https://github.com/InputEvelution/wp) +- [Rainchus/ttyd_dtk](https://github.com/Rainchus/ttyd_dtk) + +Features +-------- +- Few external dependencies: Just `python` for the generator and `ninja` for the build system. See [Dependencies](docs/dependencies.md). +- Simple configuration: Everything lives in `config.yml`, `symbols.txt`, and `splits.txt`. +- Multi-version support: Separate configurations for each game version, and a `configure.py --version` flag to switch between them. +- Feature-rich analyzer: Many time-consuming tasks are automated, allowing you to focus on the decompilation itself. See [Analyzer features](https://github.com/encounter/decomp-toolkit#analyzer-features). +- REL support: RELs each have their own `symbols.txt` and `splits.txt`, and will automatically be built and linked against the main binary. +- No manual assembly: decomp-toolkit handles splitting the DOL into relocatable objects based on the configuration. No game assets are committed to the repository. +- Progress calculation and upload script for [frogress](https://github.com/decompals/frogress). +- Integration with [objdiff](https://github.com/encounter/objdiff) for a diffing workflow. +- (TODO) CI workflow template for GitHub Actions. + +Project structure +----------------- + +- `configure.py` - Project configuration and generator script. +- `config/[GAMEID]` - Configuration files for each game version. +- `config/[GAMEID]/build.sha1` - SHA-1 hashes for each built artifact, for final verification. +- `build/` - Build artifacts generated by the the build process. Ignored by `.gitignore`. +- `orig/[GAMEID]` - Original game files, extracted from the disc. Ignored by `.gitignore`. +- `orig/[GAMEID]/.gitkeep` - Empty checked-in file to ensure the directory is created on clone. +- `src/` - C/C++ source files. +- `include/` - C/C++ header files. +- `tools/` - Scripts shared between projects. + +Temporary, delete when done: +- `config/GAMEID/config.example.yml` - Example configuration file and documentation. +- `docs/` - Documentation for decomp-toolkit configuration. +- `README.md` - This file, replace with your own. For a template, see [`README.example.md`](README.example.md). \ No newline at end of file diff --git a/assets/dolphin-extract.png b/assets/dolphin-extract.png new file mode 100644 index 0000000..67a3aa3 Binary files /dev/null and b/assets/dolphin-extract.png differ diff --git a/assets/objdiff.png b/assets/objdiff.png new file mode 100644 index 0000000..080e335 Binary files /dev/null and b/assets/objdiff.png differ diff --git a/config/GAMEID/build.sha1 b/config/GAMEID/build.sha1 new file mode 100644 index 0000000..8bcde68 --- /dev/null +++ b/config/GAMEID/build.sha1 @@ -0,0 +1,2 @@ +0123456789abcdef0123456789abcdef01234567 build/GAMEID/main.dol +0123456789abcdef0123456789abcdef01234567 build/GAMEID/module/module.rel diff --git a/config/GAMEID/config.example.yml b/config/GAMEID/config.example.yml new file mode 100644 index 0000000..1f0022d --- /dev/null +++ b/config/GAMEID/config.example.yml @@ -0,0 +1,100 @@ +# Path to the main.dol file. +object: orig/GAMEID/sys/main.dol +# (optional) SHA-1 hash of the main.dol file for verification. +hash: 0123456789abcdef0123456789abcdef01234567 +# (optional) Name override. Defaults to "main". +name: main + +# (optional) Path to the symbols.txt file. +# This file will be created if it does not exist. +# See docs/symbols.md for more information. +symbols: config/GAMEID/symbols.txt +# (optional) Path to the splits.txt file. +# This file will be created if it does not exist. +# See docs/splits.md for more information. +splits: config/GAMEID/splits.txt + +# (optional) Path to the DOL's .map file. +# This should only used for initial analysis, and generating the symbols and splits files. +# Once those files are generated, remove this to avoid conflicts. +map: orig/GAMEID/files/main.MAP +# (optional) Start address of common BSS symbols, if any. +# Useful along with `map`, but not required otherwise, since common BSS +# is marked in the splits file. +common_start: 0x80001234 + +# (optional) Version used to generate `.comment` sections in the split objects. +# If not specified, no `.comment` sections will be generated. +# See docs/comment_section.md for more information. +mw_comment_version: 8 + +# (optional) Path to `selfile.sel` for Wii games with RSO files. +selfile: orig/GAMEID/files/selfile.sel +# (optional) SHA-1 hash of the `selfile.sel` file for verification. +selfile_hash: 0123456789abcdef0123456789abcdef01234567 + +# (optional) When enabled, function boundary analysis will be skipped. +# Only valid _after_ initial analysis has been performed and +# the symbols and splits files have been generated. +quick_analysis: false + +# (optional) When enabled, the analyzer will attempt to detect sizes +# and data types of objects based on code usage and alignment. +detect_objects: true + +# (optional) When enabled, the analyzer will attempt to detect strings, +# wide strings, and string tables. +detect_strings: true + +# (optional) Whether to write disassembly to the split output directory. +# While not used in the build process, the disassembly is useful +# for reading and usage with other tools, like decomp.me. +write_asm: true + +# (optional) If symbols are _fully_ known (e.g. from a complete map file), +# this can be set to true to skip most analysis steps, and ensure new +# symbols are not created by the analyzer. +# If you're not sure, leave this false. +symbols_known: false + +# (optional) Whether to create `gap_` symbols to prevent the linker from +# adjusting the alignment / address of symbols. +# When alignments are fully known (e.g. from a complete map file), +# this can be set to false. +fill_gaps: true + +# (optional) Custom template for `ldscript.lcf`. Avoid unless necessary. +# See https://github.com/encounter/decomp-toolkit/blob/main/assets/ldscript.lcf +ldscript_template: config/GAMEID/module/ldscript.tpl + +# (optional) Configuration for modules. +modules: + +- # Path to the module. + object: orig/GAMEID/files/module.rel + + # (optional) SHA-1 hash of the module for verification. + hash: 0123456789abcdef0123456789abcdef01234567 + + # (optional) Name of the module. Defaults to the module's filename. + name: module + + # (optional) Path to the module's symbols.txt file. + # This file will be created if it does not exist. + # See docs/symbols.md for more information. + symbols: config/GAMEID/module/symbols.txt + # (optional) Path to the module's splits.txt file. + # This file will be created if it does not exist. + # See docs/splits.md for more information. + splits: config/GAMEID/module/splits.txt + + # (optional) Path to the module's .map file. + # See `map` above for more information. + map: orig/GAMEID/files/module.MAP + + # (optional) Mark symbols as "force active" / "exported". + force_active: [] + + # (optional) Custom template for `ldscript.lcf`, if needed. + # See https://github.com/encounter/decomp-toolkit/blob/main/assets/ldscript_partial.lcf + ldscript_template: config/GAMEID/module/ldscript.tpl diff --git a/config/GAMEID/config.yml b/config/GAMEID/config.yml new file mode 100644 index 0000000..e48180d --- /dev/null +++ b/config/GAMEID/config.yml @@ -0,0 +1,12 @@ +# See config.example.yml for documentation. +object: orig/GAMEID/sys/main.dol +hash: 0123456789abcdef0123456789abcdef01234567 +symbols: config/GAMEID/symbols.txt +splits: config/GAMEID/splits.txt +mw_comment_version: 8 + +modules: +- object: orig/GAMEID/files/module.rel + hash: 0123456789abcdef0123456789abcdef01234567 + symbols: config/GAMEID/module/symbols.txt + splits: config/GAMEID/module/splits.txt diff --git a/config/GAMEID/splits.txt b/config/GAMEID/splits.txt new file mode 100644 index 0000000..a2e4ff3 --- /dev/null +++ b/config/GAMEID/splits.txt @@ -0,0 +1 @@ +// Intentionally empty. Initial analysis will generate this automatically. \ No newline at end of file diff --git a/config/GAMEID/symbols.txt b/config/GAMEID/symbols.txt new file mode 100644 index 0000000..a2e4ff3 --- /dev/null +++ b/config/GAMEID/symbols.txt @@ -0,0 +1 @@ +// Intentionally empty. Initial analysis will generate this automatically. \ No newline at end of file diff --git a/configure.py b/configure.py new file mode 100644 index 0000000..9dd3bda --- /dev/null +++ b/configure.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 + +### +# Generates build files for the project. +# This file also includes the project configuration, +# such as compiler flags and the object matching status. +# +# Usage: +# python3 configure.py +# ninja +# +# Append --help to see available options. +### + +import sys +import argparse + +from pathlib import Path +from tools.project import ( + Object, + ProjectConfig, + calculate_progress, + generate_build, + is_windows, +) + +# Game versions +DEFAULT_VERSION = 0 +VERSIONS = [ + "GAMEID", # 0 +] + +if len(VERSIONS) > 1: + versions_str = ", ".join(VERSIONS[:-1]) + f" or {VERSIONS[-1]}" +else: + versions_str = VERSIONS[0] + +parser = argparse.ArgumentParser() +parser.add_argument( + "mode", + default="configure", + help="configure or progress (default: configure)", + nargs="?", +) +parser.add_argument( + "--version", + dest="version", + default=VERSIONS[DEFAULT_VERSION], + help=f"version to build ({versions_str})", +) +parser.add_argument( + "--build-dir", + dest="build_dir", + type=Path, + default=Path("build"), + help="base build directory (default: build)", +) +parser.add_argument( + "--compilers", + dest="compilers", + type=Path, + help="path to compilers (optional)", +) +parser.add_argument( + "--map", + dest="map", + action="store_true", + help="generate map file(s)", +) +parser.add_argument( + "--debug", + dest="debug", + action="store_true", + help="build with debug info (non-matching)", +) +if not is_windows(): + parser.add_argument( + "--wrapper", + dest="wrapper", + type=Path, + help="path to wibo or wine (optional)", + ) +parser.add_argument( + "--build-dtk", + dest="build_dtk", + type=Path, + help="path to decomp-toolkit source (optional)", +) +parser.add_argument( + "--sjiswrap", + dest="sjiswrap", + type=Path, + help="path to sjiswrap.exe (optional)", +) +parser.add_argument( + "--verbose", + dest="verbose", + action="store_true", + help="print verbose output", +) +args = parser.parse_args() + +config = ProjectConfig() +config.version = args.version.upper() +if config.version not in VERSIONS: + sys.exit(f"Invalid version '{config.version}', expected {versions_str}") +version_num = VERSIONS.index(config.version) + +# Apply arguments +config.build_dir = args.build_dir +config.build_dtk_path = args.build_dtk +config.compilers_path = args.compilers +config.debug = args.debug +config.generate_map = args.map +config.sjiswrap_path = args.sjiswrap +if not is_windows(): + config.wrapper = args.wrapper + +# Tool versions +config.compilers_tag = "1" +config.dtk_tag = "v0.5.5" +config.sjiswrap_tag = "v1.1.1" +config.wibo_tag = "0.6.3" + +# Project +config.config_path = Path("config") / config.version / "config.yml" +config.check_sha_path = Path("config") / config.version / "build.sha1" +config.ldflags = [ + "-fp hardware", + "-nodefaults", + "-listclosure", +] + +# Base flags, common to most GC/Wii games. +# Generally leave untouched, with overrides added below. +cflags_base = [ + "-nodefaults", + "-proc gekko", + "-align powerpc", + "-enum int", + "-fp hardware", + "-Cpp_exceptions off", + # "-W all", + "-O4,p", + "-inline auto", + '-pragma "cats off"', + '-pragma "warn_notinlined off"', + "-maxerrors 1", + "-nosyspath", + "-RTTI off", + "-fp_contract on", + "-str reuse", + "-i include", + "-i libc", + "-enc SJIS", + f"-DVERSION={version_num}", +] + +# Debug flags +if config.debug: + cflags_base.extend(["-sym on", "-DDEBUG=1"]) +else: + cflags_base.append("-DNDEBUG=1") + +# Metrowerks library flags +cflags_runtime = [ + *cflags_base, + "-use_lmw_stmw on", + "-str reuse,pool,readonly", + "-gccinc", + "-common off", + "-inline auto", +] + +# REL flags +cflags_rel = [ + *cflags_base, + "-sdata 0", + "-sdata2 0", +] + +config.linker_version = "Wii/1.3" + + +# Helper function for Dolphin libraries +def DolphinLib(lib_name, objects): + return { + "lib": lib_name, + "mw_version": "Wii/1.1", + "cflags": cflags_base, + "host": False, + "objects": objects, + } + + +# Helper function for REL script objects +def Rel(lib_name, objects): + return { + "lib": lib_name, + "mw_version": "Wii/1.3", + "cflags": cflags_rel, + "host": True, + "objects": objects, + } + + +Matching = True +NonMatching = False + +config.warn_missing_config = True +config.warn_missing_source = False +config.libs = [ + { + "lib": "Runtime.PPCEABI.H", + "mw_version": config.linker_version, + "cflags": cflags_runtime, + "host": False, + "objects": [ + Object(NonMatching, "Runtime.PPCEABI.H/global_destructor_chain.c"), + Object(NonMatching, "Runtime.PPCEABI.H/__init_cpp_exceptions.cpp"), + ], + }, +] + +if args.mode == "configure": + # Write build.ninja and objdiff.json + generate_build(config) +elif args.mode == "progress": + # Print progress and write progress.json + config.progress_each_module = args.verbose + calculate_progress(config) +else: + sys.exit("Unknown mode: " + args.mode) diff --git a/docs/comment_section.md b/docs/comment_section.md new file mode 100644 index 0000000..2e63b2e --- /dev/null +++ b/docs/comment_section.md @@ -0,0 +1,106 @@ +# CodeWarrior `.comment` section + +Files built with `mwcc` contain a `.comment` section: + +``` +$ powerpc-eabi-readelf -We object.o + +Section Headers: + [Nr] Name Type Addr Off Size ES Flg Lk Inf Al + [ 0] NULL 00000000 000000 000000 00 0 0 0 + [ 1] .text PROGBITS 00000000 000034 000708 00 AX 0 0 4 + ... + [16] .comment PROGBITS 00000000 00153b 0001b4 01 0 0 1 +``` + +The `.comment` section contains information that `mwld` uses during linking, primarily symbol alignment and a "force active" / export flag. + +If missing, `mwld` will **not** adjust the alignment of symbols or remove any unused symbols. + +This behavior is quite useful in some cases. When we split our program into objects, we're working from the final post-aligned, post-stripped result, and don't want the linker to make any changes. Most decompilation projects rely on this behavior unintentionally, since their generated objects don't contain a `.comment` section. (For example, objects built with `powerpc-eabi-as`.) + +However, we need the `.comment` section for some purposes: +- Reproducing the [common BSS inflation bug](common_bss.md#inflation-bug) requires the `.comment` section present, due to the above. The linker inflates the size of the first common BSS symbol in a TU, but won't actually move any data around unless the `.comment` section is present. +- In newer versions of the linker, using common BSS at all _without_ a valid `.comment` section will cause an internal linker error. + +When the `.comment` section is generated, decomp-toolkit will mark all global symbols as "exported" to prevent any deadstripping, since the presence of the `.comment` section itself enables deadstripping. + +Generating the `.comment` section and setting the "export" flag is also useful to prevent the linker from removing entire objects. A missing `.comment` section will prevent the removal of unused symbols _inside_ of an object, but the linker will still remove the entire object itself if it thinks it's unused. + +## Contents + +The contents of this section follow a very simple format: + +### Header + +`[0x0 size: 0xB]` Magic: `43 6F 64 65 57 61 72 72 69 6F 72` ("CodeWarrior") + +`[0xB size: 0x1]` Version(?): `XX` + +It's not known whether this field actually affects `mwld` in any way, but it's configurable for completeness sake. (See `mw_comment_version` in [`config.example.yml`](/config/GAMEID/config.example.yml).) + +Known values: +- `08` - CodeWarrior for GameCube 1.0+ +- `0A` - CodeWarrior for GameCube 1.3.2+ +- `0B`, `0C` - CodeWarrior for GameCube 2.7+ (difference unknown) +- `0E`, `0F` - CodeWarrior for GameCube 3.0a3+ (difference unknown) + +`[0xC size: 0x4]` Compiler version: `XX XX XX XX` + +First 3 bytes are major, minor, and patch version numbers. +4th byte is unknown, but is always `01`. + +Example: `Version 2.3.3 build 144` -> `02 03 00 01` +Often the `.exe`'s properties (which `--help` reads from) and the internal version number (here) will differ. + +`[0x10 size: 1]` Pool data: `XX` + +- `00` - Data pooling disabled +- `01` - Data pooling enabled + +`[0x11 size: 1]` Float type: `XX` + +- `00` - Floating point disabled +- `01` - Software floating point +- `02` - Hardware floating point + +`[0x12 size: 2]` Processor type: `00 16` (Gekko) + +`[0x14 size: 1]` Unknown, always `2C`. Possibly the start of symbol entries. + +`[0x15 size: 1]` "Quirk" flags: `XX` + +Bitfield of miscellaneous flags. Known flags: +- `01` - "Incompatible return small structs" +- `02` - "Incompatible SFPE double params" +- `04` - "Unsafe global reg vars" + +`[0x16 size: 22]` Padding until `0x2C` + +### Symbol entry + +At `0x2C` is the first symbol entry. There is one 8 byte entry per ELF symbol. + +This includes the "null" ELF symbol, so the first entry will be all 0's. + +`[0x0 size: 4]` Alignment: `XX XX XX XX` + +`[0x4 size: 1]` Visibility flags(?): `XX` + +Known values: +- `00` - Default +- `0D` - Weak +- `0E` - Unknown, also weak? + +`[0x5 size: 1]` Active flags(?): `XX` + +Known values: +- `00` - Default +- `08` - Force active / export. Prevents the symbol from being deadstripped. + When applied on a section symbol, the entire section is kept as-is. This is used + by `mwcc` when data pooling is triggered (indicated by a symbol like `...data.0`), likely to prevent the hard-coded section-relative offsets from breaking. + Can also be set using `#pragma force_active on` or `__declspec(export)`. +- `10` - Unknown +- `20` - Unknown + +`[0x6 size: 2]` Padding(?): `00 00` \ No newline at end of file diff --git a/docs/common_bss.md b/docs/common_bss.md new file mode 100644 index 0000000..948da8a --- /dev/null +++ b/docs/common_bss.md @@ -0,0 +1,69 @@ +# Common BSS + +When passed the `-common on` flag, `mwcc` will generate global BSS symbols as **common**. The linker deduplicates common symbols with the same name, and allocates an area at the **end** of `.bss` for them. + +This is a legacy feature, allowing uninitialized global variables to be defined in headers without linker errors: + +```c +// foo.h +int foo; +``` + +With `-common on`, any TU that includes `foo.h` will define `foo` as a **common** symbol. The linker will deduplicate `foo` across TUs, similar to weak symbols. Common symbols are then generated at the **end** of `.bss`, after all other `.bss` symbols. + +With `-common off`, `foo` would be defined as a **global** symbol, and the linker would error out with a duplicate symbol error if `foo.h` was included in multiple TUs. + +In `splits.txt`, common BSS can be defined with the `common` attribute: + +``` +foo.cpp: + .text start:0x80047E5C end:0x8004875C + .ctors start:0x803A54C4 end:0x803A54C8 + .data start:0x803B1B40 end:0x803B1B60 + .bss start:0x803DF828 end:0x803DFA8C + .bss start:0x8040D4AC end:0x8040D4D8 common +``` + +As shown above, a file can contain both regular `.bss` and common `.bss`. Marking common `.bss` appropriately is important for determining the final link order. + +## Detection + +Example from Pikmin 2: +``` +00016e60 00000c 805069c0 1 .bss utilityU.a PSMainSide_CreaturePrm.cpp +00016e60 00000c 805069c0 4 @3464 utilityU.a PSMainSide_CreaturePrm.cpp +00016e6c 000048 805069cc 4 saoVVOutput_direction___Q214JStudio_JStage14TAdaptor_light JSystem.a object-light.cpp +00016eb4 0000d0 80506a14 4 saoVVOutput___Q214JStudio_JStage14TAdaptor_actor JSystem.a object-actor.cpp +``` + +In this example, we see a symbol from `utilityU.a PSMainSide_CreaturePrm.cpp`. We know that this file is very close to the _end_ of the link order. Afterwards, there's a symbol from `JSystem.a object-light.cpp`, which is very close to the _beginning_ of the link order. + +A file can't be both at the beginning and end of the link order, so it's a strong indication that `saoVVOutput_direction___Q214JStudio_JStage14TAdaptor_light` marks the beginning of the common BSS section. + +One other indication from this example is the lack of a `.bss` section symbol from `JSystem.a object-actor.cpp` and any following files in the link order. Section symbols aren't generated for common BSS. + +Without a map, it's harder to tell if there's a common BSS section, but guesses can be made. When looking at XREFs in Ghidra, if a symbol is close to the _end_ of `.bss`, but has XREFs from various addresses close to the _beginning_ of `.text`, it could be an indication of common BSS. + +For games built with older versions of the linker, the inflation bug (described below) can also be used to detect common BSS. + +## Inflation bug + +In older versions of the linker (<= GC 2.6?), when calculating the size of common symbols, the linker will accidentally set the size of the first common symbol in a TU to the size of the _entire_ common section in that TU. + +Example from Pikmin 2: + +``` +# Section Addr | Size | Addr | Alignment | Name | File +00017260 000188 80506dc0 4 mPadList__10JUTGamePad JSystem.a JUTGamePad.cpp +000173e8 000030 80506f48 4 mPadStatus__10JUTGamePad JSystem.a JUTGamePad.cpp +00017418 0000c0 80506f78 4 mPadButton__10JUTGamePad JSystem.a JUTGamePad.cpp +000174d8 000040 80507038 4 mPadMStick__10JUTGamePad JSystem.a JUTGamePad.cpp +00017518 000040 80507078 4 mPadSStick__10JUTGamePad JSystem.a JUTGamePad.cpp +00017558 00000c 805070b8 4 sPatternList__19JUTGamePadLongPress JSystem.a JUTGamePad.cpp +``` + +In this example, `mPadList__10JUTGamePad` is the first common symbol in the TU, and was inflated to include the size of all other common symbols in the TU. In reality, it's only supposed to be `0xC` bytes, given `0x188 - 0x30 - 0xC0 - 0x40 - 0x40 - 0xC`. + +This can be useful to determine if symbols are in the same TU without a map: if a `.bss` symbol is much larger than expected, it could be the first common symbol in a TU. One can subtract the sizes of following symbols to find the true size of the symbol, along with the end of the TU's common symbols. + +To reproduce this behavior, the `.comment` section must be present in the object. See [`.comment` section](comment_section.md) for more details. diff --git a/docs/dependencies.md b/docs/dependencies.md new file mode 100644 index 0000000..b8b531b --- /dev/null +++ b/docs/dependencies.md @@ -0,0 +1,33 @@ +# Dependencies + +## Windows: + +On Windows, it's **highly recommended** to use native tooling. WSL or msys2 are **not** required. +When running under WSL, [objdiff](#diffing) is unable to get filesystem notifications for automatic rebuilds. + +- Install [Python](https://www.python.org/downloads/) and add it to `%PATH%`. + - Also available from the [Windows Store](https://apps.microsoft.com/store/detail/python-311/9NRWMJP3717K). +- Download [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`. + - Quick install via pip: `pip install ninja` + +## macOS: + +- Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages): + ``` + brew install ninja + ``` +- Install [wine-crossover](https://github.com/Gcenx/homebrew-wine): + ``` + brew install --cask --no-quarantine gcenx/wine/wine-crossover + ``` + +After OS upgrades, if macOS complains about `Wine Crossover.app` being unverified, you can unquarantine it using: +```sh +sudo xattr -rd com.apple.quarantine '/Applications/Wine Crossover.app' +``` + +## Linux: + +- Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages). +- For non-x86(_64) platforms: Install wine from your package manager. + - For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used. diff --git a/docs/getting_started.md b/docs/getting_started.md new file mode 100644 index 0000000..b084c5e --- /dev/null +++ b/docs/getting_started.md @@ -0,0 +1,124 @@ +# Getting Started + +See [Dependencies](dependencies.md) first. + +Clone this template repository. + +Rename `orig/GAMEID` to the game's ID. (For example, `GLZE01` for _The Legend of Zelda: The Wind Waker_.) + +Extract your game to `orig/[GAMEID]`. In Dolphin, use "Extract Entire Disc" for GameCube games, or use "Data Partition" -> "Extract Entire Partition" for Wii games. + +Rename `config/GAMEID` to the game's ID and modify `config/[GAMEID]/config.yml` appropriately, using [`config.example.yml`](/config/GAMEID/config.example.yml) as a reference. If the game doesn't use RELs, the `modules` list in `config.yml` can be removed. + +Generate a `config/[GAMEID]/build.sha1` file for verification. This file is a list of SHA-1 hashes for each build artifact. One possible way: + +```shell +$ dtk shasum orig/[GAMEID]/sys/main.dol orig/[GAMEID]/files/*.rel > config/[GAMEID]/build.sha1 +``` + +Then, modify the paths in `config/[GAMEID]/build.sha1` to point to the `build` directory instead of `orig`. The DOL will be built at `build/[GAMEID]/main.dol`, and modules will be built at `build/[GAMEID]/[module_name]/[module_name].rel`. + +Update `VERSIONS` in [`configure.py`](/configure.py) with the game ID. + +Run `python configure.py` to generate the initial `build.ninja`. + +Run `ninja` to perform initial analysis. + +If all goes well, the initial `symbols.txt` and `splits.txt` should be automatically generated. Though it's likely it won't build yet. See [Post-analysis](#post-analysis) for next steps. + +## Using a `.map` + +If the game has `.map` files matching the DOL (and RELs, if applicable), they can be used to fill out `symbols.txt` and `splits.txt` automatically during the initial analysis. + +Add the `map` key to `config.yml`, pointing to the `.map` file from the game disc. (For example, `orig/[GAMEID]/files/main.map`.) For RELs, add a `map` key to each module in `config.yml`. + +If the game uses [common BSS](common_bss.md), be sure to set `common_start` as well. (See [`config.example.yml`](/config/GAMEID/config.example.yml).) Otherwise, the final link order may fail to be determined. + +Once the initial analysis is completed, `symbols.txt` and `splits.txt` will be generated from the map information. **Remove** the `map` fields from `config.yml` to avoid conflicts. + +## Post-analysis + +After the initial analysis, `symbols.txt` and `splits.txt` will be generated. These files can be modified to adjust symbols and split points. + +If the game uses C++ exceptions, it's required to set up a split for the `__init_cpp_exceptions.cpp` file. This differs between linker versions. + +Often indicated by the following error: + +``` +# runtime sources 'global_destructor_chain.c' and +# '__init_cpp_exceptions.cpp' both need to be updated to latest version. +``` + +### GC 1.0 - 2.6 linkers: + +```yaml +# splits.txt +Runtime.PPCEABI.H/__init_cpp_exceptions.cpp: + .text start:0x803294EC end:0x80329568 + .ctors start:0x80338680 end:0x80338684 + .dtors start:0x80338820 end:0x80338828 + .sdata start:0x803F67F0 end:0x803F67F8 +``` + +`.text`: +Find the following symbols in `symbols.txt`: +``` +GetR2__Fv = .text:0x803294EC; // type:function size:0x8 scope:local align:4 +__fini_cpp_exceptions = .text:0x803294F4; // type:function size:0x34 scope:global align:4 +__init_cpp_exceptions = .text:0x80329528; // type:function size:0x40 scope:global align:4 +``` +The split end is the address of `__init_cpp_exceptions` + size. + +`.ctors`: +Find the address of `__init_cpp_exception_reference` or `_ctors` in symbols.txt. +Always size 4. + +`.dtors`: +Look for the address of `__destroy_global_chain_reference` or `_dtors` in symbols.txt. +If `__fini_cpp_exceptions_reference` is present, it's size 8, otherwise size 4 + +`.sdata`: +Find the following symbol in `symbols.txt`: +``` +fragmentID = .sdata:0x803F67F0; // type:object size:0x4 scope:local align:4 data:4byte +``` +The split end includes any inter-TU padding, so it's usually size 8. + +### GC 2.7+ and Wii linkers: + +```yaml +# splits.txt +Runtime.PPCEABI.H/__init_cpp_exceptions.cpp: + .text start:0x80345C34 end:0x80345CA4 + .ctors start:0x803A54A0 end:0x803A54A4 rename:.ctors$10 + .dtors start:0x803A56A0 end:0x803A56A4 rename:.dtors$10 + .dtors start:0x803A56A4 end:0x803A56A8 rename:.dtors$15 + .sdata start:0x80418CA8 end:0x80418CB0 +``` + +`.text`: +Find the following symbols in `symbols.txt`: +``` +__fini_cpp_exceptions = .text:0x80345C34; // type:function size:0x34 scope:global +__init_cpp_exceptions = .text:0x80345C68; // type:function size:0x3C scope:global +``` +The split end is the address of `__init_cpp_exceptions` + size. + +`.ctors$10`: +Find the address of `__init_cpp_exception_reference` or `_ctors` in symbols.txt. +Always size 4. + +`.dtors$10`: +Look for the address of `__destroy_global_chain_reference` or `_dtors` in symbols.txt. +Always size 4. + +`.dtors$15`: +Look for the address of `__fini_cpp_exceptions_reference` in symbols.txt. +Always size 4. + +`.sdata`: +Find the following symbol in `symbols.txt`: +``` +fragmentID = .sdata:0x80418CA8; // type:object size:0x4 scope:local data:4byte +``` +The split end includes any inter-TU padding, so it's usually size 8. \ No newline at end of file diff --git a/docs/splits.md b/docs/splits.md new file mode 100644 index 0000000..2fa4739 --- /dev/null +++ b/docs/splits.md @@ -0,0 +1,42 @@ +# `splits.txt` + +This file contains file splits for a module. + +Example: + +``` +path/to/file.cpp: + .text start:0x80047E5C end:0x8004875C + .ctors start:0x803A54C4 end:0x803A54C8 + .data start:0x803B1B40 end:0x803B1B60 + .bss start:0x803DF828 end:0x803DFA8C + .bss start:0x8040D4AC end:0x8040D4D8 common +``` + +## Format + +``` +path/to/file.cpp: [file attributes] + section [section attributes] + ... +``` + +- `path/to/file.cpp` The name of the source file, usually relative to `src`. The file does **not** need to exist to start. + This corresponds to an entry in `configure.py` for specifying compiler flags and other options. + +### File attributes + +- `comment:` Overrides the `mw_comment_version` setting in [`config.yml`](/config/GAMEID/config.example.yml) for this file. See [Comment section](comment_section.md). + +`comment:0` is used to disable `.comment` section generation for a file that wasn't compiled with `mwcc`. +Example: `TRK_MINNOW_DOLPHIN/ppc/Export/targsupp.s: comment:0` +This file was assembled and only contains label symbols. Generating a `.comment` section for it will crash `mwld`. + +### Section attributes + +- `start:` The start address of the section within the file. For DOLs, this is the absolute address (e.g. `0x80001234`). For RELs, this is the section-relative address (e.g. `0x1234`). +- `end:` The end address of the section within the file. +- `align:` Specifies the alignment of the section. If not specified, the default alignment for the section is used. +- `rename:` Writes this section under a different name when generating the split object. Used for `.ctors$10`, etc. +- `common` Only valid for `.bss`. See [Common BSS](common_bss.md). +- `skip` Skips this data when writing the object file. Used for ignoring data that's linker-generated. diff --git a/docs/symbols.md b/docs/symbols.md new file mode 100644 index 0000000..cfcffff --- /dev/null +++ b/docs/symbols.md @@ -0,0 +1,35 @@ +# `symbols.txt` + +This file contains all symbols for a module, one per line. + +Example line: +``` +__dt__13mDoExt_bckAnmFv = .text:0x800DD2EC; // type:function size:0x5C scope:global align:4 +``` + +## Format + +Numbers can be written as decimal or hexadecimal. Hexadecimal numbers must be prefixed with `0x`. + +Comment lines starting with `//` or `#` are permitted, but are currently **not** preserved when updating the file. + +``` +symbol_name = section:address; // [attributes] +``` + +- `symbol_name` - The name of the symbol. (For C++, this is the mangled name, e.g. `__dt__13mDoExt_bckAnmFv`) +- `section` - The section the symbol is in. +- `address` - The address of the symbol. For DOLs, this is the absolute address (e.g. `0x80001234`). For RELs, this is the section-relative address (e.g. `0x1234`). + +### Attributes + +All attributes are optional, and are separated by spaces. + +- `type:` The symbol type. `function`, `object`, or `label`. +- `size:` The size of the symbol. +- `scope:` The symbol's visibility. `global` (default), `local` or `weak`. +- `align:` The symbol's alignment. +- `data:` The data type used when writing disassembly. `byte`, `2byte`, `4byte`, `8byte`, `float`, `double`, `string`, `wstring`, `string_table`, or `wstring_table`. +- `hidden` Marked as "hidden" in the generated object. (Only used for extab) +- `force_active` Marked as ["exported"](comment_section.md) in the generated object, and added to `FORCEACTIVE` in the generated `ldscript.lcf`. Prevents the symbol from being deadstripped. +- `noreloc` Prevents the _contents_ of the symbol from being interpreted as addresses. Used for objects containing data that look like pointers, but aren't. \ No newline at end of file diff --git a/orig/GAMEID/.gitkeep b/orig/GAMEID/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/decompctx.py b/tools/decompctx.py new file mode 100644 index 0000000..c42b182 --- /dev/null +++ b/tools/decompctx.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +### +# Generates a ctx.c file, usable for "Context" on https://decomp.me. +# +# Usage: +# python3 tools/decompctx.py src/file.cpp +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import argparse +import os +import re + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.abspath(os.path.join(script_dir, "..")) +src_dir = os.path.join(root_dir, "src") +include_dir = os.path.join(root_dir, "include") + +include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$') +guard_pattern = re.compile(r'^#ifndef\s+(.*)$') + +defines = set() + +def import_h_file(in_file: str, r_path: str) -> str: + rel_path = os.path.join(root_dir, r_path, in_file) + inc_path = os.path.join(include_dir, in_file) + if os.path.exists(rel_path): + return import_c_file(rel_path) + elif os.path.exists(inc_path): + return import_c_file(inc_path) + else: + print("Failed to locate", in_file) + exit(1) + +def import_c_file(in_file) -> str: + in_file = os.path.relpath(in_file, root_dir) + out_text = '' + + try: + with open(in_file, encoding="utf-8") as file: + out_text += process_file(in_file, list(file)) + except Exception: + with open(in_file) as file: + out_text += process_file(in_file, list(file)) + return out_text + +def process_file(in_file: str, lines) -> str: + out_text = '' + for idx, line in enumerate(lines): + guard_match = guard_pattern.match(line.strip()) + if idx == 0: + if guard_match: + if guard_match[1] in defines: + break + defines.add(guard_match[1]) + print("Processing file", in_file) + include_match = include_pattern.match(line.strip()) + if include_match and not include_match[1].endswith(".s"): + out_text += f"/* \"{in_file}\" line {idx} \"{include_match[1]}\" */\n" + out_text += import_h_file(include_match[1], os.path.dirname(in_file)) + out_text += f"/* end \"{include_match[1]}\" */\n" + else: + out_text += line + + return out_text + +def main(): + parser = argparse.ArgumentParser( + description="""Create a context file which can be used for decomp.me""" + ) + parser.add_argument( + "c_file", + help="""File from which to create context""", + ) + args = parser.parse_args() + + output = import_c_file(args.c_file) + + with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f: + f.write(output) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/download_tool.py b/tools/download_tool.py new file mode 100644 index 0000000..2709966 --- /dev/null +++ b/tools/download_tool.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 + +### +# Downloads various tools from GitHub releases. +# +# Usage: +# python3 tools/download_tool.py wibo build/tools/wibo --tag 1.0.0 +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import argparse +import io +import os +import platform +import shutil +import stat +import sys +import urllib.request +import zipfile + +from pathlib import Path + + +def dtk_url(tag): + uname = platform.uname() + suffix = "" + system = uname.system.lower() + if system == "darwin": + system = "macos" + elif system == "windows": + suffix = ".exe" + arch = uname.machine.lower() + if arch == "amd64": + arch = "x86_64" + + repo = "https://github.com/encounter/decomp-toolkit" + return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" + + +def sjiswrap_url(tag): + repo = "https://github.com/encounter/sjiswrap" + return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe" + + +def wibo_url(tag): + repo = "https://github.com/decompals/wibo" + return f"{repo}/releases/download/{tag}/wibo" + + +def compilers_url(tag): + if tag == "1": + return "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip" + else: + sys.exit("Unknown compilers tag %s" % tag) + + +TOOLS = { + "dtk": dtk_url, + "sjiswrap": sjiswrap_url, + "wibo": wibo_url, + "compilers": compilers_url, +} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("tool", help="Tool name") + parser.add_argument("output", type=Path, help="output file path") + parser.add_argument("--tag", help="GitHub tag", required=True) + args = parser.parse_args() + + url = TOOLS[args.tool](args.tag) + output = Path(args.output) + + print(f"Downloading {url} to {output}") + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req) as response: + if url.endswith(".zip"): + data = io.BytesIO(response.read()) + with zipfile.ZipFile(data) as f: + f.extractall(output) + output.touch(mode=0o755) + else: + with open(output, "wb") as f: + shutil.copyfileobj(response, f) + st = os.stat(output) + os.chmod(output, st.st_mode | stat.S_IEXEC) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py new file mode 100644 index 0000000..ffd88a0 --- /dev/null +++ b/tools/ninja_syntax.py @@ -0,0 +1,223 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Python module for generating .ninja files. + +Note that this is emphatically not a required piece of Ninja; it's +just a helpful utility for build-file-generation systems that already +use Python. +""" + +import re +import textwrap + + +def escape_path(word): + return word.replace("$ ", "$$ ").replace(" ", "$ ").replace(":", "$:") + + +class Writer(object): + def __init__(self, output, width=78): + self.output = output + self.width = width + + def newline(self): + self.output.write("\n") + + def comment(self, text): + for line in textwrap.wrap( + text, self.width - 2, break_long_words=False, break_on_hyphens=False + ): + self.output.write("# " + line + "\n") + + def variable(self, key, value, indent=0): + if value is None: + return + if isinstance(value, list): + value = " ".join(filter(None, value)) # Filter out empty strings. + self._line("%s = %s" % (key, value), indent) + + def pool(self, name, depth): + self._line("pool %s" % name) + self.variable("depth", depth, indent=1) + + def rule( + self, + name, + command, + description=None, + depfile=None, + generator=False, + pool=None, + restat=False, + rspfile=None, + rspfile_content=None, + deps=None, + ): + self._line("rule %s" % name) + self.variable("command", command, indent=1) + if description: + self.variable("description", description, indent=1) + if depfile: + self.variable("depfile", depfile, indent=1) + if generator: + self.variable("generator", "1", indent=1) + if pool: + self.variable("pool", pool, indent=1) + if restat: + self.variable("restat", "1", indent=1) + if rspfile: + self.variable("rspfile", rspfile, indent=1) + if rspfile_content: + self.variable("rspfile_content", rspfile_content, indent=1) + if deps: + self.variable("deps", deps, indent=1) + + def build( + self, + outputs, + rule, + inputs=None, + implicit=None, + order_only=None, + variables=None, + implicit_outputs=None, + pool=None, + dyndep=None, + ): + outputs = as_list(outputs) + out_outputs = [escape_path(x) for x in outputs] + all_inputs = [escape_path(x) for x in as_list(inputs)] + + if implicit: + implicit = [escape_path(x) for x in as_list(implicit)] + all_inputs.append("|") + all_inputs.extend(implicit) + if order_only: + order_only = [escape_path(x) for x in as_list(order_only)] + all_inputs.append("||") + all_inputs.extend(order_only) + if implicit_outputs: + implicit_outputs = [escape_path(x) for x in as_list(implicit_outputs)] + out_outputs.append("|") + out_outputs.extend(implicit_outputs) + + self._line( + "build %s: %s" % (" ".join(out_outputs), " ".join([rule] + all_inputs)) + ) + if pool is not None: + self._line(" pool = %s" % pool) + if dyndep is not None: + self._line(" dyndep = %s" % dyndep) + + if variables: + if isinstance(variables, dict): + iterator = iter(variables.items()) + else: + iterator = iter(variables) + + for key, val in iterator: + self.variable(key, val, indent=1) + + return outputs + + def include(self, path): + self._line("include %s" % path) + + def subninja(self, path): + self._line("subninja %s" % path) + + def default(self, paths): + self._line("default %s" % " ".join(as_list(paths))) + + def _count_dollars_before_index(self, s, i): + """Returns the number of '$' characters right in front of s[i].""" + dollar_count = 0 + dollar_index = i - 1 + while dollar_index > 0 and s[dollar_index] == "$": + dollar_count += 1 + dollar_index -= 1 + return dollar_count + + def _line(self, text, indent=0): + """Write 'text' word-wrapped at self.width characters.""" + leading_space = " " * indent + while len(leading_space) + len(text) > self.width: + # The text is too wide; wrap if possible. + + # Find the rightmost space that would obey our width constraint and + # that's not an escaped space. + available_space = self.width - len(leading_space) - len(" $") + space = available_space + while True: + space = text.rfind(" ", 0, space) + if space < 0 or self._count_dollars_before_index(text, space) % 2 == 0: + break + + if space < 0: + # No such space; just use the first unescaped space we can find. + space = available_space - 1 + while True: + space = text.find(" ", space + 1) + if ( + space < 0 + or self._count_dollars_before_index(text, space) % 2 == 0 + ): + break + if space < 0: + # Give up on breaking. + break + + self.output.write(leading_space + text[0:space] + " $\n") + text = text[space + 1 :] + + # Subsequent lines are continuations, so indent them. + leading_space = " " * (indent + 2) + + self.output.write(leading_space + text + "\n") + + def close(self): + self.output.close() + + +def as_list(input): + if input is None: + return [] + if isinstance(input, list): + return input + return [input] + + +def escape(string): + """Escape a string such that it can be embedded into a Ninja file without + further interpretation.""" + assert "\n" not in string, "Ninja syntax does not allow newlines" + # We only have one special metacharacter: '$'. + return string.replace("$", "$$") + + +def expand(string, vars, local_vars={}): + """Expand a string containing $vars as Ninja would. + + Note: doesn't handle the full Ninja variable syntax, but it's enough + to make configure.py's use of it work. + """ + + def exp(m): + var = m.group(1) + if var == "$": + return "$" + return local_vars.get(var, vars.get(var, "")) + + return re.sub(r"\$(\$|\w*)", exp, string) \ No newline at end of file diff --git a/tools/project.py b/tools/project.py new file mode 100644 index 0000000..28a8d32 --- /dev/null +++ b/tools/project.py @@ -0,0 +1,982 @@ +### +# decomp-toolkit project generator +# Generates build.ninja and objdiff.json. +# +# This generator is intentionally project-agnostic +# and shared between multiple projects. Any configuration +# specific to a project should be added to `configure.py`. +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import io +import json +import os +import platform +import sys + +from pathlib import Path +from . import ninja_syntax + +if sys.platform == "cygwin": + sys.exit( + f"Cygwin/MSYS2 is not supported." + f"\nPlease use native Windows Python instead." + f"\n(Current path: {sys.executable})" + ) + + +class ProjectConfig: + def __init__(self): + # Paths + self.build_dir = Path("build") + self.src_dir = Path("src") + self.tools_dir = Path("tools") + + # Tooling + self.dtk_tag = None # Git tag + self.build_dtk_path = None # If None, download + self.compilers_tag = None # 1 + self.compilers_path = None # If None, download + self.wibo_tag = None # Git tag + self.wrapper = None # If None, download wibo on Linux + self.sjiswrap_tag = None # Git tag + self.sjiswrap_path = None # If None, download + + # Project config + self.build_rels = True # Build REL files + self.check_sha_path = None # Path to version.sha1 + self.config_path = None # Path to config.yml + self.debug = False # Build with debug info + self.generate_map = False # Generate map file(s) + self.ldflags = None # Linker flags + self.libs = None # List of libraries + self.linker_version = None # mwld version + self.version = None # Version name + self.warn_missing_config = False # Warn on missing unit configuration + self.warn_missing_source = False # Warn on missing source file + + # Progress output and progress.json config + self.progress_all = True # Include combined "all" category + self.progress_modules = True # Include combined "modules" category + self.progress_each_module = ( + True # Include individual modules, disable for large numbers of modules + ) + + def validate(self): + required_attrs = [ + "build_dir", + "src_dir", + "tools_dir", + "check_sha_path", + "config_path", + "ldflags", + "linker_version", + "libs", + "version", + ] + for attr in required_attrs: + if getattr(self, attr) is None: + sys.exit(f"ProjectConfig.{attr} missing") + + def find_object(self, name): + for lib in self.libs: + for obj in lib["objects"]: + if obj.name == name: + return [lib, obj] + return None + + def out_path(self): + return self.build_dir / self.version + + +class Object: + def __init__(self, completed, name, **options): + self.name = name + self.completed = completed + self.options = { + "add_to_all": True, + "cflags": None, + "mw_version": None, + "shiftjis": True, + "source": name, + } + self.options.update(options) + + +def is_windows(): + return os.name == "nt" + + +# On Windows, we need this to use && in commands +CHAIN = "cmd /c " if is_windows() else "" +# Native executable extension +EXE = ".exe" if is_windows() else "" + + +# Replace forward slashes with backslashes on Windows +def os_str(value): + return str(value).replace("/", os.sep) + + +# Replace backslashes with forward slashes on Windows +def unix_str(value): + return str(value).replace(os.sep, "/") + + +# Stringify paths for ninja_syntax +def path(value): + if value is None: + return None + elif isinstance(value, list): + return list(map(os_str, filter(lambda x: x is not None, value))) + else: + return [os_str(value)] + + +# Load decomp-toolkit generated config.json +def load_build_config(config, build_config_path): + if not build_config_path.is_file(): + return None + + def versiontuple(v): + return tuple(map(int, (v.split(".")))) + + f = open(build_config_path, "r", encoding="utf-8") + build_config = json.load(f) + config_version = build_config.get("version") + if not config_version: + # Invalid config.json + f.close() + os.remove(build_config_path) + return None + + dtk_version = config.dtk_tag[1:] # Strip v + if versiontuple(config_version) < versiontuple(dtk_version): + # Outdated config.json + f.close() + os.remove(build_config_path) + return None + + f.close() + return build_config + + +# Generate build.ninja and objdiff.json +def generate_build(config): + build_config = load_build_config(config, config.out_path() / "config.json") + generate_build_ninja(config, build_config) + generate_objdiff_config(config, build_config) + + +# Generate build.ninja +def generate_build_ninja(config, build_config): + config.validate() + + out = io.StringIO() + n = ninja_syntax.Writer(out) + n.variable("ninja_required_version", "1.3") + n.newline() + + configure_script = os.path.relpath(os.path.abspath(sys.argv[0])) + python_lib = os.path.relpath(__file__) + python_lib_dir = os.path.dirname(python_lib) + n.comment("The arguments passed to configure.py, for rerunning it.") + n.variable("configure_args", sys.argv[1:]) + n.variable("python", f'"{sys.executable}"') + n.newline() + + ### + # Variables + ### + n.comment("Variables") + ldflags = " ".join(config.ldflags) + if config.generate_map: + ldflags += " -mapunused" + if config.debug: + ldflags += " -g" + n.variable("ldflags", ldflags) + n.variable("mw_version", config.linker_version) + n.newline() + + ### + # Tooling + ### + n.comment("Tooling") + + build_path = config.out_path() + build_tools_path = config.build_dir / "tools" + download_tool = config.tools_dir / "download_tool.py" + n.rule( + name="download_tool", + command=f"$python {download_tool} $tool $out --tag $tag", + description="TOOL $out", + ) + + if config.build_dtk_path: + dtk = build_tools_path / "release" / f"dtk{EXE}" + n.rule( + name="cargo", + command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", + description="CARGO $bin", + depfile=path(Path("$target") / "release" / "$bin.d"), + deps="gcc", + ) + n.build( + outputs=path(dtk), + rule="cargo", + inputs=path(config.build_dtk_path / "Cargo.toml"), + implicit=path(config.build_dtk_path / "Cargo.lock"), + variables={ + "bin": "dtk", + "target": build_tools_path, + }, + ) + elif config.dtk_tag: + dtk = build_tools_path / f"dtk{EXE}" + n.build( + outputs=path(dtk), + rule="download_tool", + implicit=path(download_tool), + variables={ + "tool": "dtk", + "tag": config.dtk_tag, + }, + ) + else: + sys.exit("ProjectConfig.dtk_tag missing") + + if config.sjiswrap_path: + sjiswrap = config.sjiswrap_path + elif config.sjiswrap_tag: + sjiswrap = build_tools_path / "sjiswrap.exe" + n.build( + outputs=path(sjiswrap), + rule="download_tool", + implicit=path(download_tool), + variables={ + "tool": "sjiswrap", + "tag": config.sjiswrap_tag, + }, + ) + else: + sys.exit("ProjectConfig.sjiswrap_tag missing") + + # Only add an implicit dependency on wibo if we download it + wrapper = config.wrapper + wrapper_implicit = None + if ( + config.wibo_tag is not None + and sys.platform == "linux" + and platform.machine() in ("i386", "x86_64") + and config.wrapper is None + ): + wrapper = build_tools_path / "wibo" + wrapper_implicit = wrapper + n.build( + outputs=path(wrapper), + rule="download_tool", + implicit=path(download_tool), + variables={ + "tool": "wibo", + "tag": config.wibo_tag, + }, + ) + if not is_windows() and wrapper is None: + wrapper = "wine" + wrapper_cmd = f"{wrapper} " if wrapper else "" + + compilers_implicit = None + if config.compilers_path: + compilers = config.compilers_path + elif config.compilers_tag: + compilers = config.build_dir / "compilers" + compilers_implicit = compilers + n.build( + outputs=path(compilers), + rule="download_tool", + implicit=path(download_tool), + variables={ + "tool": "compilers", + "tag": config.compilers_tag, + }, + ) + + n.newline() + + ### + # Build rules + ### + compiler_path = compilers / "$mw_version" + + # MWCC + mwcc = compiler_path / "mwcceppc.exe" + mwcc_cmd = f"{wrapper_cmd}{mwcc} $cflags -MMD -c $in -o $basedir" + mwcc_implicit = [compilers_implicit or mwcc, wrapper_implicit] + + # MWCC with UTF-8 to Shift JIS wrapper + mwcc_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir" + mwcc_sjis_implicit = [*mwcc_implicit, sjiswrap] + + # MWLD + mwld = compiler_path / "mwldeppc.exe" + mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp" + mwld_implicit = [compilers_implicit or mwld, wrapper_implicit] + + if os.name != "nt": + transform_dep = config.tools_dir / "transform_dep.py" + mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_sjis_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_implicit.append(transform_dep) + mwcc_sjis_implicit.append(transform_dep) + + n.comment("Link ELF file") + n.rule( + name="link", + command=mwld_cmd, + description="LINK $out", + rspfile="$out.rsp", + rspfile_content="$in_newline", + ) + n.newline() + + n.comment("Generate DOL") + n.rule( + name="elf2dol", + command=f"{dtk} elf2dol $in $out", + description="DOL $out", + ) + n.newline() + + n.comment("Generate REL(s)") + makerel_rsp = build_path / "makerel.rsp" + n.rule( + name="makerel", + command=f"{dtk} rel make -w -c $config @{makerel_rsp}", + description="REL", + rspfile=path(makerel_rsp), + rspfile_content="$in_newline", + ) + n.newline() + + n.comment("MWCC build") + n.rule( + name="mwcc", + command=mwcc_cmd, + description="MWCC $out", + depfile="$basefile.d", + deps="gcc", + ) + n.newline() + + n.comment("MWCC build (with UTF-8 to Shift JIS wrapper)") + n.rule( + name="mwcc_sjis", + command=mwcc_sjis_cmd, + description="MWCC $out", + depfile="$basefile.d", + deps="gcc", + ) + n.newline() + + n.comment("Host build") + n.variable("host_cflags", "-I include -Wno-trigraphs") + n.variable( + "host_cppflags", + "-std=c++98 -I include -fno-exceptions -fno-rtti -D_CRT_SECURE_NO_WARNINGS -Wno-trigraphs -Wno-c++11-extensions", + ) + n.rule( + name="host_cc", + command="clang $host_cflags -c -o $out $in", + description="CC $out", + ) + n.rule( + name="host_cpp", + command="clang++ $host_cppflags -c -o $out $in", + description="CXX $out", + ) + n.newline() + + ### + # Source files + ### + n.comment("Source files") + build_src_path = build_path / "src" + build_host_path = build_path / "host" + build_config_path = build_path / "config.json" + + def map_path(path): + return path.parent / (path.name + ".MAP") + + class LinkStep: + def __init__(self, config): + self.name = config["name"] + self.module_id = config["module_id"] + self.ldscript = config["ldscript"] + self.entry = config["entry"] + self.inputs = [] + + def add(self, obj): + self.inputs.append(obj) + + def output(self): + if self.module_id == 0: + return build_path / f"{self.name}.dol" + else: + return build_path / self.name / f"{self.name}.rel" + + def partial_output(self): + if self.module_id == 0: + return build_path / f"{self.name}.elf" + else: + return build_path / self.name / f"{self.name}.plf" + + def write(self, n): + n.comment(f"Link {self.name}") + if self.module_id == 0: + elf_path = build_path / f"{self.name}.elf" + dol_path = build_path / f"{self.name}.dol" + elf_ldflags = f"$ldflags -lcf {self.ldscript}" + if config.generate_map: + elf_map = map_path(elf_path) + elf_ldflags += f" -map {elf_map}" + else: + elf_map = None + n.build( + outputs=path(elf_path), + rule="link", + inputs=path(self.inputs), + implicit=path([self.ldscript, *mwld_implicit]), + implicit_outputs=path(elf_map), + variables={"ldflags": elf_ldflags}, + ) + n.build( + outputs=path(dol_path), + rule="elf2dol", + inputs=path(elf_path), + implicit=path(dtk), + ) + else: + preplf_path = build_path / self.name / f"{self.name}.preplf" + plf_path = build_path / self.name / f"{self.name}.plf" + preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r" + plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}" + if config.generate_map: + preplf_map = map_path(preplf_path) + preplf_ldflags += f" -map {preplf_map}" + plf_map = map_path(plf_path) + plf_ldflags += f" -map {plf_map}" + else: + preplf_map = None + plf_map = None + n.build( + outputs=path(preplf_path), + rule="link", + inputs=path(self.inputs), + implicit=path(mwld_implicit), + implicit_outputs=path(preplf_map), + variables={"ldflags": preplf_ldflags}, + ) + n.build( + outputs=path(plf_path), + rule="link", + inputs=path(self.inputs), + implicit=path([self.ldscript, preplf_path, *mwld_implicit]), + implicit_outputs=path(plf_map), + variables={"ldflags": plf_ldflags}, + ) + n.newline() + + if build_config: + link_steps = [] + used_compiler_versions = set() + source_inputs = [] + host_source_inputs = [] + source_added = set() + + def add_unit(build_obj, link_step): + obj_path, obj_name = build_obj["object"], build_obj["name"] + result = config.find_object(obj_name) + if not result: + if config.warn_missing_config and not build_obj["autogenerated"]: + print(f"Missing configuration for {obj_name}") + link_step.add(obj_path) + return + + lib, obj = result + lib_name = lib["lib"] + + options = obj.options + completed = obj.completed + + unit_src_path = config.src_dir / options["source"] + if not unit_src_path.exists(): + if config.warn_missing_source: + print(f"Missing source file {unit_src_path}") + link_step.add(obj_path) + return + + mw_version = options["mw_version"] or lib["mw_version"] + cflags = options["cflags"] or lib["cflags"] + if type(cflags) is list: + cflags_str = " ".join(cflags) + else: + cflags_str = str(cflags) + used_compiler_versions.add(mw_version) + + base_object = Path(obj.name).with_suffix("") + src_obj_path = build_src_path / f"{base_object}.o" + src_base_path = build_src_path / base_object + + if src_obj_path not in source_added: + source_added.add(src_obj_path) + + n.comment(f"{obj_name}: {lib_name} (linked {completed})") + n.build( + outputs=path(src_obj_path), + rule="mwcc_sjis" if options["shiftjis"] else "mwcc", + inputs=path(unit_src_path), + variables={ + "mw_version": path(Path(mw_version)), + "cflags": cflags_str, + "basedir": os.path.dirname(src_base_path), + "basefile": path(src_base_path), + }, + implicit=path( + mwcc_sjis_implicit if options["shiftjis"] else mwcc_implicit + ), + ) + + if lib["host"]: + host_obj_path = build_host_path / f"{base_object}.o" + host_base_path = build_host_path / base_object + n.build( + outputs=path(host_obj_path), + rule="host_cc" if unit_src_path.suffix == ".c" else "host_cpp", + inputs=path(unit_src_path), + variables={ + "basedir": os.path.dirname(host_base_path), + "basefile": path(host_base_path), + }, + ) + if options["add_to_all"]: + host_source_inputs.append(host_obj_path) + n.newline() + + if options["add_to_all"]: + source_inputs.append(src_obj_path) + + if completed: + obj_path = src_obj_path + link_step.add(obj_path) + + # Add DOL link step + link_step = LinkStep(build_config) + for unit in build_config["units"]: + add_unit(unit, link_step) + link_steps.append(link_step) + + if config.build_rels: + # Add REL link steps + for module in build_config["modules"]: + module_link_step = LinkStep(module) + for unit in module["units"]: + add_unit(unit, module_link_step) + link_steps.append(module_link_step) + n.newline() + + # Check if all compiler versions exist + for mw_version in used_compiler_versions: + mw_path = compilers / mw_version / "mwcceppc.exe" + if config.compilers_path and not os.path.exists(mw_path): + sys.exit(f"Compiler {mw_path} does not exist") + + # Check if linker exists + mw_path = compilers / config.linker_version / "mwldeppc.exe" + if config.compilers_path and not os.path.exists(mw_path): + sys.exit(f"Linker {mw_path} does not exist") + + ### + # Link + ### + for step in link_steps: + step.write(n) + n.newline() + + ### + # Generate RELs + ### + rel_outputs = list( + map( + lambda step: step.output(), + filter(lambda step: step.module_id != 0, link_steps), + ) + ) + if len(rel_outputs) > 0: + n.comment("Generate RELs") + n.build( + outputs=path(rel_outputs), + rule="makerel", + inputs=path(list(map(lambda step: step.partial_output(), link_steps))), + implicit=path([dtk, config.config_path]), + variables={"config": path(config.config_path)}, + ) + n.newline() + + ### + # Helper rule for building all source files + ### + n.comment("Build all source files") + n.build( + outputs="all_source", + rule="phony", + inputs=path(source_inputs), + ) + n.newline() + + ### + # Helper rule for building all source files, with a host compiler + ### + n.comment("Build all source files with a host compiler") + n.build( + outputs="all_source_host", + rule="phony", + inputs=path(host_source_inputs), + ) + n.newline() + + ### + # Check hash + ### + n.comment("Check hash") + ok_path = build_path / "ok" + quiet = "-q " if len(link_steps) > 3 else "" + n.rule( + name="check", + command=f"{dtk} shasum {quiet} -c $in -o $out", + description="CHECK $in", + ) + n.build( + outputs=path(ok_path), + rule="check", + inputs=path(config.check_sha_path), + implicit=path([dtk, *map(lambda step: step.output(), link_steps)]), + ) + n.newline() + + ### + # Calculate progress + ### + n.comment("Calculate progress") + progress_path = build_path / "progress.json" + n.rule( + name="progress", + command=f"$python {configure_script} $configure_args progress", + description="PROGRESS", + ) + n.build( + outputs=path(progress_path), + rule="progress", + implicit=path([ok_path, configure_script, python_lib, config.config_path]), + ) + + ### + # Helper tools + ### + # TODO: make these rules work for RELs too + dol_link_step = link_steps[0] + dol_elf_path = dol_link_step.partial_output() + n.comment("Check for mismatching symbols") + n.rule( + name="dol_diff", + command=f"{dtk} -L error dol diff $in", + description=f"DIFF {dol_elf_path}", + ) + n.build( + inputs=path([config.config_path, dol_elf_path]), + outputs="dol_diff", + rule="dol_diff", + ) + n.build( + outputs="diff", + rule="phony", + inputs="dol_diff", + ) + n.newline() + + n.comment("Apply symbols from linked ELF") + n.rule( + name="dol_apply", + command=f"{dtk} dol apply $in", + description=f"APPLY {dol_elf_path}", + ) + n.build( + inputs=path([config.config_path, dol_elf_path]), + outputs="dol_apply", + rule="dol_apply", + implicit=path([ok_path]), + ) + n.build( + outputs="apply", + rule="phony", + inputs="dol_apply", + ) + n.newline() + + ### + # Split DOL + ### + n.comment("Split DOL into relocatable objects") + n.rule( + name="split", + command=f"{dtk} dol split $in $out_dir", + description="SPLIT $in", + depfile="$out_dir/dep", + deps="gcc", + ) + n.build( + inputs=path(config.config_path), + outputs=path(build_config_path), + rule="split", + implicit=path(dtk), + variables={"out_dir": path(build_path)}, + ) + n.newline() + + ### + # Regenerate on change + ### + n.comment("Reconfigure on change") + n.rule( + name="configure", + command=f"$python {configure_script} $configure_args", + generator=True, + description=f"RUN {configure_script}", + ) + n.build( + outputs="build.ninja", + rule="configure", + implicit=path( + [ + build_config_path, + configure_script, + python_lib, + Path(python_lib_dir) / "ninja_syntax.py", + ] + ), + ) + n.newline() + + ### + # Default rule + ### + n.comment("Default rule") + if build_config: + n.default(path(progress_path)) + else: + n.default(path(build_config_path)) + + # Write build.ninja + with open("build.ninja", "w", encoding="utf-8") as f: + f.write(out.getvalue()) + out.close() + + +# Generate objdiff.json +def generate_objdiff_config(config, build_config): + if not build_config: + return + + objdiff_config = { + "min_version": "0.4.3", + "custom_make": "ninja", + "build_target": False, + "watch_patterns": [ + "*.c", + "*.cp", + "*.cpp", + "*.h", + "*.hpp", + "*.py", + "*.yml", + "*.txt", + "*.json", + ], + "units": [], + } + + build_path = config.out_path() + + def add_unit(build_obj, module_name): + if build_obj["autogenerated"]: + # Skip autogenerated objects + return + + obj_path, obj_name = build_obj["object"], build_obj["name"] + base_object = Path(obj_name).with_suffix("") + unit_config = { + "name": unix_str(Path(module_name) / base_object), + "target_path": unix_str(obj_path), + } + + result = config.find_object(obj_name) + if not result: + objdiff_config["units"].append(unit_config) + return + + lib, obj = result + unit_src_path = config.src_dir / obj.options["source"] + if not unit_src_path.exists(): + objdiff_config["units"].append(unit_config) + return + + cflags = obj.options["cflags"] or lib["cflags"] + src_obj_path = build_path / "src" / f"{base_object}.o" + + reverse_fn_order = False + if type(cflags) is list: + for flag in cflags: + if not flag.startswith("-inline "): + continue + for value in flag.split(" ")[1].split(","): + if value == "deferred": + reverse_fn_order = True + elif value == "nodeferred": + reverse_fn_order = False + + unit_config["base_path"] = unix_str(src_obj_path) + unit_config["reverse_fn_order"] = reverse_fn_order + unit_config["complete"] = obj.completed + objdiff_config["units"].append(unit_config) + + # Add DOL units + for unit in build_config["units"]: + add_unit(unit, build_config["name"]) + + # Add REL units + for module in build_config["modules"]: + for unit in module["units"]: + add_unit(unit, module["name"]) + + # Write objdiff.json + with open("objdiff.json", "w", encoding="utf-8") as w: + json.dump(objdiff_config, w, indent=4) + + +# Calculate, print and write progress to progress.json +def calculate_progress(config): + out_path = config.out_path() + build_config = load_build_config(config, out_path / "config.json") + if not build_config: + return + + class ProgressUnit: + def __init__(self, name): + self.name = name + self.code_total = 0 + self.code_progress = 0 + self.data_total = 0 + self.data_progress = 0 + self.objects_progress = 0 + self.objects_total = 0 + self.objects = set() + + def add(self, build_obj): + self.code_total += build_obj["code_size"] + self.data_total += build_obj["data_size"] + + # Avoid counting the same object in different modules twice + include_object = build_obj["name"] not in self.objects + if include_object: + self.objects.add(build_obj["name"]) + self.objects_total += 1 + + if build_obj["autogenerated"]: + # Skip autogenerated objects + return + + result = config.find_object(build_obj["name"]) + if not result: + return + + _, obj = result + if not obj.completed: + return + + self.code_progress += build_obj["code_size"] + self.data_progress += build_obj["data_size"] + if include_object: + self.objects_progress += 1 + + def code_frac(self): + return self.code_progress / self.code_total + + def data_frac(self): + return self.data_progress / self.data_total + + # Add DOL units + all_progress = ProgressUnit("All") if config.progress_all else None + dol_progress = ProgressUnit("DOL") + for unit in build_config["units"]: + if all_progress: + all_progress.add(unit) + dol_progress.add(unit) + + # Add REL units + rels_progress = ProgressUnit("Modules") if config.progress_modules else None + modules_progress = [] + for module in build_config["modules"]: + progress = ProgressUnit(module["name"]) + modules_progress.append(progress) + for unit in module["units"]: + if all_progress: + all_progress.add(unit) + if rels_progress: + rels_progress.add(unit) + progress.add(unit) + + # Print human-readable progress + print("Progress:") + + def print_category(unit): + code_frac = unit.code_frac() + data_frac = unit.data_frac() + print( + f" {unit.name}: {code_frac:.2%} code, {data_frac:.2%} data ({unit.objects_progress} / {unit.objects_total} files)" + ) + print(f" Code: {unit.code_progress} / {unit.code_total} bytes") + print(f" Data: {unit.data_progress} / {unit.data_total} bytes") + + if all_progress: + print_category(all_progress) + print_category(dol_progress) + module_count = len(build_config["modules"]) + if module_count > 0: + print_category(rels_progress) + if config.progress_each_module: + for progress in modules_progress: + print_category(progress) + + # Generate and write progress.json + progress_json = {} + + def add_category(category, unit): + progress_json[category] = { + "code": unit.code_progress, + "code/total": unit.code_total, + "data": unit.data_progress, + "data/total": unit.data_total, + } + + if all_progress: + add_category("all", all_progress) + add_category("dol", dol_progress) + if len(build_config["modules"]) > 0: + if rels_progress: + add_category("modules", rels_progress) + if config.progress_each_module: + for progress in modules_progress: + add_category(progress.name, progress) + with open(out_path / "progress.json", "w", encoding="utf-8") as w: + json.dump(progress_json, w, indent=4) \ No newline at end of file diff --git a/tools/transform_dep.py b/tools/transform_dep.py new file mode 100644 index 0000000..86bd2ec --- /dev/null +++ b/tools/transform_dep.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +### +# Transforms .d files, converting Windows paths to Unix paths. +# Allows usage of the mwcc -MMD flag on platforms other than Windows. +# +# Usage: +# python3 tools/transform_dep.py build/src/file.d build/src/file.d +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import argparse +import os +from platform import uname + +wineprefix = os.path.join(os.environ["HOME"], ".wine") +if "WINEPREFIX" in os.environ: + wineprefix = os.environ["WINEPREFIX"] +winedevices = os.path.join(wineprefix, "dosdevices") + + +def in_wsl() -> bool: + return "microsoft-standard" in uname().release + + +def import_d_file(in_file) -> str: + out_text = "" + + with open(in_file) as file: + for idx, line in enumerate(file): + if idx == 0: + if line.endswith(" \\\n"): + out_text += line[:-3].replace("\\", "/") + " \\\n" + else: + out_text += line.replace("\\", "/") + else: + suffix = "" + if line.endswith(" \\\n"): + suffix = " \\" + path = line.lstrip()[:-3] + else: + path = line.strip() + # lowercase drive letter + path = path[0].lower() + path[1:] + if path[0] == "z": + # shortcut for z: + path = path[2:].replace("\\", "/") + elif in_wsl(): + path = path[0:1] + path[2:] + path = os.path.join("/mnt", path.replace("\\", "/")) + else: + # use $WINEPREFIX/dosdevices to resolve path + path = os.path.realpath( + os.path.join(winedevices, path.replace("\\", "/")) + ) + out_text += "\t" + path + suffix + "\n" + + return out_text + + +def main(): + parser = argparse.ArgumentParser( + description="""Transform a .d file from Wine paths to normal paths""" + ) + parser.add_argument( + "d_file", + help="""Dependency file in""", + ) + parser.add_argument( + "d_file_out", + help="""Dependency file out""", + ) + args = parser.parse_args() + + output = import_d_file(args.d_file) + + with open(args.d_file_out, "w", encoding="UTF-8") as f: + f.write(output) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/upload_progress.py b/tools/upload_progress.py new file mode 100644 index 0000000..673bb3d --- /dev/null +++ b/tools/upload_progress.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +### +# Uploads progress information to https://github.com/decompals/frogress. +# +# Usage: +# python3 tools/upload_progress.py -b https://progress.decomp.club/ -p [project] -v [version] build/[version]/progress.json +# +# If changes are made, please submit a PR to +# https://github.com/encounter/dtk-template +### + +import argparse +import json +import os +import requests +import subprocess +import sys + + +def get_git_commit_timestamp() -> int: + return int( + subprocess.check_output(["git", "show", "-s", "--format=%ct"]) + .decode("ascii") + .rstrip() + ) + + +def get_git_commit_sha() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip() + + +def generate_url(args: argparse.Namespace) -> str: + url_components = [args.base_url.rstrip("/"), "data"] + + for arg in [args.project, args.version]: + if arg != "": + url_components.append(arg) + + return str.join("/", url_components) + "/" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Upload progress information.") + parser.add_argument("-b", "--base_url", help="API base URL", required=True) + parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)") + parser.add_argument("-p", "--project", help="Project slug", required=True) + parser.add_argument("-v", "--version", help="Version slug", required=True) + parser.add_argument("input", help="Progress JSON input") + + args = parser.parse_args() + api_key = args.api_key or os.environ.get("PROGRESS_API_KEY") + if not api_key: + raise "API key required" + url = generate_url(args) + + entries = [] + with open(args.input, "r") as f: + data = json.load(f) + entries.append( + { + "timestamp": get_git_commit_timestamp(), + "git_hash": get_git_commit_sha(), + "categories": data, + } + ) + + print("Publishing entry to", url) + json.dump(entries[0], sys.stdout, indent=4) + print() + r = requests.post(url, json={ + "api_key": api_key, + "entries": entries, + }) + r.raise_for_status() + print("Done!") \ No newline at end of file