encounter
/
dtk-template
mirror of https://github.com/encounter/dtk-template.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Initial commit

This commit is contained in:
Luke Street 2023-10-04 23:26:02 -04:00
commit 5b8fbc6cd5
29 changed files with 2575 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.gitattributes vendored Normal file
View File

@ -0,0 +1,13 @@
# Auto detect text files and perform LF normalization
* text=auto
# Explicitly declare text files
*.py text
# Enforce platform-specific encodings
*.bat text eol=crlf
*.sh text eol=lf
*.sha1 text eol=lf
# DTK keeps these files with LF
config/**/*.txt text eol=lf

11
.gitignore vendored Normal file
View File

@ -0,0 +1,11 @@
__pycache__
.idea
.vscode
.ninja_*
*.exe
build
build.ninja
objdiff.json
orig/*/*
!orig/*/.gitkeep
/*.txt

22
.vscode.example/c_cpp_properties.json Normal file
View File

@ -0,0 +1,22 @@
{
"configurations": [
{
"name": "Linux",
"includePath": [
"${workspaceFolder}/include/**"
],
"cStandard": "c99",
"cppStandard": "c++98",
"intelliSenseMode": "linux-clang-x86",
"compilerPath": "",
"configurationProvider": "ms-vscode.makefile-tools",
"browse": {
"path": [
"${workspaceFolder}/include"
],
"limitSymbolsToIncludedHeaders": true
}
}
],
"version": 4
}

26
.vscode.example/settings.json Normal file
View File

@ -0,0 +1,26 @@
{
"[c]": {
"files.encoding": "utf8",
"editor.defaultFormatter": "xaver.clang-format"
},
"[cpp]": {
"files.encoding": "utf8",
"editor.defaultFormatter": "xaver.clang-format"
},
"[python]": {
"editor.defaultFormatter": "ms-python.black-formatter"
},
"files.insertFinalNewline": true,
"files.trimFinalNewlines": true,
"files.associations": {
"*.inc": "cpp"
},
"search.useIgnoreFiles": false,
"search.exclude": {
"build/*/config.json": true,
"build/**/*.MAP": true,
"build.ninja": true,
".ninja_*": true,
"objdiff.json": true
}
}

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2023 Luke Street
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

114
README.example.md Normal file
View File

@ -0,0 +1,114 @@
Some Game
[![Build Status]][actions] ![Progress] ![DOL Progress] ![RELs Progress] [![Discord Badge]][discord]
=============
<!--
Replace with your repository's URL.
-->
[Build Status]: https://github.com/zeldaret/tww/actions/workflows/build.yml/badge.svg
[actions]: https://github.com/zeldaret/tww/actions/workflows/build.yml
<!---
Code progress URL:
https://progress.decomp.club/data/[project]/[version]/all/?mode=shield&measure=code
URL encoded then appended to: https://img.shields.io/endpoint?label=Code&url=
-->
[Progress]: https://img.shields.io/endpoint?label=Code&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Ftww%2FGZLE01%2Fall%2F%3Fmode%3Dshield%26measure%3Dcode
<!---
DOL progress URL:
https://progress.decomp.club/data/[project]/[version]/dol/?mode=shield&measure=code
URL encoded then appended to: https://img.shields.io/endpoint?label=DOL&url=
-->
[DOL Progress]: https://img.shields.io/endpoint?label=DOL&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Ftww%2FGZLE01%2Fdol%2F%3Fmode%3Dshield%26measure%3Dcode
<!--
REL progress URL:
https://progress.decomp.club/data/[project]/[version]/modules/?mode=shield&measure=code
URL encoded then appended to: https://img.shields.io/endpoint?label=RELs&url=
-->
[RELs Progress]: https://img.shields.io/endpoint?label=RELs&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Ftww%2FGZLE01%2Fmodules%2F%3Fmode%3Dshield%26measure%3Dcode
<!--
Replace with your Discord server's ID and invite URL.
-->
[Discord Badge]: https://img.shields.io/discord/727908905392275526?color=%237289DA&logo=discord&logoColor=%23FFFFFF
[discord]: https://discord.gg/hKx3FJJgrV
A work-in-progress decompilation of Some Game.
This repository does **not** contain any game assets or assembly whatsoever. An existing copy of the game is required.
Supported versions:
- `GAMEID`: Rev 0 (USA)
Dependencies
============
Windows:
--------
On Windows, it's **highly recommended** to use native tooling. WSL or msys2 are **not** required.
When running under WSL, [objdiff](#diffing) is unable to get filesystem notifications for automatic rebuilds.
- Install [Python](https://www.python.org/downloads/) and add it to `%PATH%`.
- Also available from the [Windows Store](https://apps.microsoft.com/store/detail/python-311/9NRWMJP3717K).
- Download [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`.
- Quick install via pip: `pip install ninja`
macOS:
------
- Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages):
```
brew install ninja
```
- Install [wine-crossover](https://github.com/Gcenx/homebrew-wine):
```
brew install --cask --no-quarantine gcenx/wine/wine-crossover
```
After OS upgrades, if macOS complains about `Wine Crossover.app` being unverified, you can unquarantine it using:
```sh
sudo xattr -rd com.apple.quarantine '/Applications/Wine Crossover.app'
```
Linux:
------
- Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages).
- For non-x86(_64) platforms: Install wine from your package manager.
- For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.
Building
========
- Clone the repository:
```
git clone https://github.com/my/repo.git
```
- Using [Dolphin Emulator](https://dolphin-emu.org/), extract your game to `orig/GAMEID`.
![](assets/dolphin-extract.png)
- To save space, the only necessary files are the following. Any others can be deleted.
- `sys/main.dol`
- `files/rels/*.rel`
- Configure:
```
python configure.py
```
To use a version other than `GAMEID` (USA), specify it with `--version`.
- Build:
```
ninja
```
Visual Studio Code
==================
If desired, use the recommended Visual Studio Code settings by renaming the `.vscode.example` directory to `.vscode`.
Diffing
=======
Once the initial build succeeds, an `objdiff.json` should exist in the project root.
Download the latest release from [encounter/objdiff](https://github.com/encounter/objdiff). Under project settings, set `Project directory`. The configuration should be loaded automatically.
Select an object from the left sidebar to begin diffing. Changes to the project will rebuild automatically: changes to source files, headers, `configure.py`, `splits.txt` or `symbols.txt`.
![](assets/objdiff.png)

65
README.md Normal file
View File

@ -0,0 +1,65 @@
decomp-toolkit Project Template
===============================
If starting a new GameCube / Wii decompilation project, this repository can be used as a scaffold.
See [decomp-toolkit](https://github.com/encounter/decomp-toolkit) for background on the concept and more information on the tooling used.
Documentation
-------------
- [Dependencies](docs/dependencies.md)
- [Getting Started](docs/getting_started.md)
- [`symbols.txt`](docs/symbols.md)
- [`splits.txt`](docs/splits.md)
General:
- [Common BSS](docs/common_bss.md)
- [`.comment` section](docs/comment_section.md)
References
--------
- [Discord: GC/Wii Decompilation](https://discord.gg/hKx3FJJgrV) (Come to `#dtk` for help!)
- [objdiff](https://github.com/encounter/objdiff) (Local diffing tool)
- [decomp.me](https://decomp.me) (Collaborate on matches)
- [frogress](https://github.com/decompals/frogress) (Decompilation progress API)
- [wibo](https://github.com/decompals/wibo) (Minimal Win32 wrapper for Linux)
- [sjiswrap](https://github.com/encounter/sjiswrap) (UTF-8 to Shift JIS wrapper)
Projects using this structure:
- [zeldaret/tww](https://github.com/zeldaret/tww)
- [PrimeDecomp/echoes](https://github.com/PrimeDecomp/echoes)
- [DarkRTA/rb3](https://github.com/DarkRTA/rb3)
- [InputEvelution/wp](https://github.com/InputEvelution/wp)
- [Rainchus/ttyd_dtk](https://github.com/Rainchus/ttyd_dtk)
Features
--------
- Few external dependencies: Just `python` for the generator and `ninja` for the build system. See [Dependencies](docs/dependencies.md).
- Simple configuration: Everything lives in `config.yml`, `symbols.txt`, and `splits.txt`.
- Multi-version support: Separate configurations for each game version, and a `configure.py --version` flag to switch between them.
- Feature-rich analyzer: Many time-consuming tasks are automated, allowing you to focus on the decompilation itself. See [Analyzer features](https://github.com/encounter/decomp-toolkit#analyzer-features).
- REL support: RELs each have their own `symbols.txt` and `splits.txt`, and will automatically be built and linked against the main binary.
- No manual assembly: decomp-toolkit handles splitting the DOL into relocatable objects based on the configuration. No game assets are committed to the repository.
- Progress calculation and upload script for [frogress](https://github.com/decompals/frogress).
- Integration with [objdiff](https://github.com/encounter/objdiff) for a diffing workflow.
- (TODO) CI workflow template for GitHub Actions.
Project structure
-----------------
- `configure.py` - Project configuration and generator script.
- `config/[GAMEID]` - Configuration files for each game version.
- `config/[GAMEID]/build.sha1` - SHA-1 hashes for each built artifact, for final verification.
- `build/` - Build artifacts generated by the the build process. Ignored by `.gitignore`.
- `orig/[GAMEID]` - Original game files, extracted from the disc. Ignored by `.gitignore`.
- `orig/[GAMEID]/.gitkeep` - Empty checked-in file to ensure the directory is created on clone.
- `src/` - C/C++ source files.
- `include/` - C/C++ header files.
- `tools/` - Scripts shared between projects.
Temporary, delete when done:
- `config/GAMEID/config.example.yml` - Example configuration file and documentation.
- `docs/` - Documentation for decomp-toolkit configuration.
- `README.md` - This file, replace with your own. For a template, see [`README.example.md`](README.example.md).

BIN
assets/dolphin-extract.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

BIN
assets/objdiff.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 65 KiB

2
config/GAMEID/build.sha1 Normal file
View File

@ -0,0 +1,2 @@
0123456789abcdef0123456789abcdef01234567 build/GAMEID/main.dol
0123456789abcdef0123456789abcdef01234567 build/GAMEID/module/module.rel

100
config/GAMEID/config.example.yml Normal file
View File

@ -0,0 +1,100 @@
# Path to the main.dol file.
object: orig/GAMEID/sys/main.dol
# (optional) SHA-1 hash of the main.dol file for verification.
hash: 0123456789abcdef0123456789abcdef01234567
# (optional) Name override. Defaults to "main".
name: main
# (optional) Path to the symbols.txt file.
# This file will be created if it does not exist.
# See docs/symbols.md for more information.
symbols: config/GAMEID/symbols.txt
# (optional) Path to the splits.txt file.
# This file will be created if it does not exist.
# See docs/splits.md for more information.
splits: config/GAMEID/splits.txt
# (optional) Path to the DOL's .map file.
# This should only used for initial analysis, and generating the symbols and splits files.
# Once those files are generated, remove this to avoid conflicts.
map: orig/GAMEID/files/main.MAP
# (optional) Start address of common BSS symbols, if any.
# Useful along with `map`, but not required otherwise, since common BSS
# is marked in the splits file.
common_start: 0x80001234
# (optional) Version used to generate `.comment` sections in the split objects.
# If not specified, no `.comment` sections will be generated.
# See docs/comment_section.md for more information.
mw_comment_version: 8
# (optional) Path to `selfile.sel` for Wii games with RSO files.
selfile: orig/GAMEID/files/selfile.sel
# (optional) SHA-1 hash of the `selfile.sel` file for verification.
selfile_hash: 0123456789abcdef0123456789abcdef01234567
# (optional) When enabled, function boundary analysis will be skipped.
# Only valid _after_ initial analysis has been performed and
# the symbols and splits files have been generated.
quick_analysis: false
# (optional) When enabled, the analyzer will attempt to detect sizes
# and data types of objects based on code usage and alignment.
detect_objects: true
# (optional) When enabled, the analyzer will attempt to detect strings,
# wide strings, and string tables.
detect_strings: true
# (optional) Whether to write disassembly to the split output directory.
# While not used in the build process, the disassembly is useful
# for reading and usage with other tools, like decomp.me.
write_asm: true
# (optional) If symbols are _fully_ known (e.g. from a complete map file),
# this can be set to true to skip most analysis steps, and ensure new
# symbols are not created by the analyzer.
# If you're not sure, leave this false.
symbols_known: false
# (optional) Whether to create `gap_` symbols to prevent the linker from
# adjusting the alignment / address of symbols.
# When alignments are fully known (e.g. from a complete map file),
# this can be set to false.
fill_gaps: true
# (optional) Custom template for `ldscript.lcf`. Avoid unless necessary.
# See https://github.com/encounter/decomp-toolkit/blob/main/assets/ldscript.lcf
ldscript_template: config/GAMEID/module/ldscript.tpl
# (optional) Configuration for modules.
modules:
- # Path to the module.
object: orig/GAMEID/files/module.rel
# (optional) SHA-1 hash of the module for verification.
hash: 0123456789abcdef0123456789abcdef01234567
# (optional) Name of the module. Defaults to the module's filename.
name: module
# (optional) Path to the module's symbols.txt file.
# This file will be created if it does not exist.
# See docs/symbols.md for more information.
symbols: config/GAMEID/module/symbols.txt
# (optional) Path to the module's splits.txt file.
# This file will be created if it does not exist.
# See docs/splits.md for more information.
splits: config/GAMEID/module/splits.txt
# (optional) Path to the module's .map file.
# See `map` above for more information.
map: orig/GAMEID/files/module.MAP
# (optional) Mark symbols as "force active" / "exported".
force_active: []
# (optional) Custom template for `ldscript.lcf`, if needed.
# See https://github.com/encounter/decomp-toolkit/blob/main/assets/ldscript_partial.lcf
ldscript_template: config/GAMEID/module/ldscript.tpl

12
config/GAMEID/config.yml Normal file
View File

@ -0,0 +1,12 @@
# See config.example.yml for documentation.
object: orig/GAMEID/sys/main.dol
hash: 0123456789abcdef0123456789abcdef01234567
symbols: config/GAMEID/symbols.txt
splits: config/GAMEID/splits.txt
mw_comment_version: 8
modules:
- object: orig/GAMEID/files/module.rel
hash: 0123456789abcdef0123456789abcdef01234567
symbols: config/GAMEID/module/symbols.txt
splits: config/GAMEID/module/splits.txt

1
config/GAMEID/splits.txt Normal file
View File

@ -0,0 +1 @@
// Intentionally empty. Initial analysis will generate this automatically.

1
config/GAMEID/symbols.txt Normal file
View File

@ -0,0 +1 @@
// Intentionally empty. Initial analysis will generate this automatically.

233
configure.py Normal file
View File

@ -0,0 +1,233 @@
#!/usr/bin/env python3
###
# Generates build files for the project.
# This file also includes the project configuration,
# such as compiler flags and the object matching status.
#
# Usage:
# python3 configure.py
# ninja
#
# Append --help to see available options.
###
import sys
import argparse
from pathlib import Path
from tools.project import (
Object,
ProjectConfig,
calculate_progress,
generate_build,
is_windows,
)
# Game versions
DEFAULT_VERSION = 0
VERSIONS = [
"GAMEID", # 0
]
if len(VERSIONS) > 1:
versions_str = ", ".join(VERSIONS[:-1]) + f" or {VERSIONS[-1]}"
else:
versions_str = VERSIONS[0]
parser = argparse.ArgumentParser()
parser.add_argument(
"mode",
default="configure",
help="configure or progress (default: configure)",
nargs="?",
)
parser.add_argument(
"--version",
dest="version",
default=VERSIONS[DEFAULT_VERSION],
help=f"version to build ({versions_str})",
)
parser.add_argument(
"--build-dir",
dest="build_dir",
type=Path,
default=Path("build"),
help="base build directory (default: build)",
)
parser.add_argument(
"--compilers",
dest="compilers",
type=Path,
help="path to compilers (optional)",
)
parser.add_argument(
"--map",
dest="map",
action="store_true",
help="generate map file(s)",
)
parser.add_argument(
"--debug",
dest="debug",
action="store_true",
help="build with debug info (non-matching)",
)
if not is_windows():
parser.add_argument(
"--wrapper",
dest="wrapper",
type=Path,
help="path to wibo or wine (optional)",
)
parser.add_argument(
"--build-dtk",
dest="build_dtk",
type=Path,
help="path to decomp-toolkit source (optional)",
)
parser.add_argument(
"--sjiswrap",
dest="sjiswrap",
type=Path,
help="path to sjiswrap.exe (optional)",
)
parser.add_argument(
"--verbose",
dest="verbose",
action="store_true",
help="print verbose output",
)
args = parser.parse_args()
config = ProjectConfig()
config.version = args.version.upper()
if config.version not in VERSIONS:
sys.exit(f"Invalid version '{config.version}', expected {versions_str}")
version_num = VERSIONS.index(config.version)
# Apply arguments
config.build_dir = args.build_dir
config.build_dtk_path = args.build_dtk
config.compilers_path = args.compilers
config.debug = args.debug
config.generate_map = args.map
config.sjiswrap_path = args.sjiswrap
if not is_windows():
config.wrapper = args.wrapper
# Tool versions
config.compilers_tag = "1"
config.dtk_tag = "v0.5.5"
config.sjiswrap_tag = "v1.1.1"
config.wibo_tag = "0.6.3"
# Project
config.config_path = Path("config") / config.version / "config.yml"
config.check_sha_path = Path("config") / config.version / "build.sha1"
config.ldflags = [
"-fp hardware",
"-nodefaults",
"-listclosure",
]
# Base flags, common to most GC/Wii games.
# Generally leave untouched, with overrides added below.
cflags_base = [
"-nodefaults",
"-proc gekko",
"-align powerpc",
"-enum int",
"-fp hardware",
"-Cpp_exceptions off",
# "-W all",
"-O4,p",
"-inline auto",
'-pragma "cats off"',
'-pragma "warn_notinlined off"',
"-maxerrors 1",
"-nosyspath",
"-RTTI off",
"-fp_contract on",
"-str reuse",
"-i include",
"-i libc",
"-enc SJIS",
f"-DVERSION={version_num}",
]
# Debug flags
if config.debug:
cflags_base.extend(["-sym on", "-DDEBUG=1"])
else:
cflags_base.append("-DNDEBUG=1")
# Metrowerks library flags
cflags_runtime = [
*cflags_base,
"-use_lmw_stmw on",
"-str reuse,pool,readonly",
"-gccinc",
"-common off",
"-inline auto",
]
# REL flags
cflags_rel = [
*cflags_base,
"-sdata 0",
"-sdata2 0",
]
config.linker_version = "Wii/1.3"
# Helper function for Dolphin libraries
def DolphinLib(lib_name, objects):
return {
"lib": lib_name,
"mw_version": "Wii/1.1",
"cflags": cflags_base,
"host": False,
"objects": objects,
}
# Helper function for REL script objects
def Rel(lib_name, objects):
return {
"lib": lib_name,
"mw_version": "Wii/1.3",
"cflags": cflags_rel,
"host": True,
"objects": objects,
}
Matching = True
NonMatching = False
config.warn_missing_config = True
config.warn_missing_source = False
config.libs = [
{
"lib": "Runtime.PPCEABI.H",
"mw_version": config.linker_version,
"cflags": cflags_runtime,
"host": False,
"objects": [
Object(NonMatching, "Runtime.PPCEABI.H/global_destructor_chain.c"),
Object(NonMatching, "Runtime.PPCEABI.H/__init_cpp_exceptions.cpp"),
],
},
]
if args.mode == "configure":
# Write build.ninja and objdiff.json
generate_build(config)
elif args.mode == "progress":
# Print progress and write progress.json
config.progress_each_module = args.verbose
calculate_progress(config)
else:
sys.exit("Unknown mode: " + args.mode)

106
docs/comment_section.md Normal file
View File

@ -0,0 +1,106 @@
# CodeWarrior `.comment` section
Files built with `mwcc` contain a `.comment` section:
```
$ powerpc-eabi-readelf -We object.o
Section Headers:
[Nr] Name Type Addr Off Size ES Flg Lk Inf Al
[ 0] NULL 00000000 000000 000000 00 0 0 0
[ 1] .text PROGBITS 00000000 000034 000708 00 AX 0 0 4
...
[16] .comment PROGBITS 00000000 00153b 0001b4 01 0 0 1
```
The `.comment` section contains information that `mwld` uses during linking, primarily symbol alignment and a "force active" / export flag.
If missing, `mwld` will **not** adjust the alignment of symbols or remove any unused symbols.
This behavior is quite useful in some cases. When we split our program into objects, we're working from the final post-aligned, post-stripped result, and don't want the linker to make any changes. Most decompilation projects rely on this behavior unintentionally, since their generated objects don't contain a `.comment` section. (For example, objects built with `powerpc-eabi-as`.)
However, we need the `.comment` section for some purposes:
- Reproducing the [common BSS inflation bug](common_bss.md#inflation-bug) requires the `.comment` section present, due to the above. The linker inflates the size of the first common BSS symbol in a TU, but won't actually move any data around unless the `.comment` section is present.
- In newer versions of the linker, using common BSS at all _without_ a valid `.comment` section will cause an internal linker error.
When the `.comment` section is generated, decomp-toolkit will mark all global symbols as "exported" to prevent any deadstripping, since the presence of the `.comment` section itself enables deadstripping.
Generating the `.comment` section and setting the "export" flag is also useful to prevent the linker from removing entire objects. A missing `.comment` section will prevent the removal of unused symbols _inside_ of an object, but the linker will still remove the entire object itself if it thinks it's unused.
## Contents
The contents of this section follow a very simple format:
### Header
`[0x0 size: 0xB]` Magic: `43 6F 64 65 57 61 72 72 69 6F 72` ("CodeWarrior")
`[0xB size: 0x1]` Version(?): `XX`
It's not known whether this field actually affects `mwld` in any way, but it's configurable for completeness sake. (See `mw_comment_version` in [`config.example.yml`](/config/GAMEID/config.example.yml).)
Known values:
- `08` - CodeWarrior for GameCube 1.0+
- `0A` - CodeWarrior for GameCube 1.3.2+
- `0B`, `0C` - CodeWarrior for GameCube 2.7+ (difference unknown)
- `0E`, `0F` - CodeWarrior for GameCube 3.0a3+ (difference unknown)
`[0xC size: 0x4]` Compiler version: `XX XX XX XX`
First 3 bytes are major, minor, and patch version numbers.
4th byte is unknown, but is always `01`.
Example: `Version 2.3.3 build 144` -> `02 03 00 01`
Often the `.exe`'s properties (which `--help` reads from) and the internal version number (here) will differ.
`[0x10 size: 1]` Pool data: `XX`
- `00` - Data pooling disabled
- `01` - Data pooling enabled
`[0x11 size: 1]` Float type: `XX`
- `00` - Floating point disabled
- `01` - Software floating point
- `02` - Hardware floating point
`[0x12 size: 2]` Processor type: `00 16` (Gekko)
`[0x14 size: 1]` Unknown, always `2C`. Possibly the start of symbol entries.
`[0x15 size: 1]` "Quirk" flags: `XX`
Bitfield of miscellaneous flags. Known flags:
- `01` - "Incompatible return small structs"
- `02` - "Incompatible SFPE double params"
- `04` - "Unsafe global reg vars"
`[0x16 size: 22]` Padding until `0x2C`
### Symbol entry
At `0x2C` is the first symbol entry. There is one 8 byte entry per ELF symbol.
This includes the "null" ELF symbol, so the first entry will be all 0's.
`[0x0 size: 4]` Alignment: `XX XX XX XX`
`[0x4 size: 1]` Visibility flags(?): `XX`
Known values:
- `00` - Default
- `0D` - Weak
- `0E` - Unknown, also weak?
`[0x5 size: 1]` Active flags(?): `XX`
Known values:
- `00` - Default
- `08` - Force active / export. Prevents the symbol from being deadstripped.
When applied on a section symbol, the entire section is kept as-is. This is used
by `mwcc` when data pooling is triggered (indicated by a symbol like `...data.0`), likely to prevent the hard-coded section-relative offsets from breaking.
Can also be set using `#pragma force_active on` or `__declspec(export)`.
- `10` - Unknown
- `20` - Unknown
`[0x6 size: 2]` Padding(?): `00 00`

69
docs/common_bss.md Normal file
View File

@ -0,0 +1,69 @@
# Common BSS
When passed the `-common on` flag, `mwcc` will generate global BSS symbols as **common**. The linker deduplicates common symbols with the same name, and allocates an area at the **end** of `.bss` for them.
This is a legacy feature, allowing uninitialized global variables to be defined in headers without linker errors:
```c
// foo.h
int foo;
```
With `-common on`, any TU that includes `foo.h` will define `foo` as a **common** symbol. The linker will deduplicate `foo` across TUs, similar to weak symbols. Common symbols are then generated at the **end** of `.bss`, after all other `.bss` symbols.
With `-common off`, `foo` would be defined as a **global** symbol, and the linker would error out with a duplicate symbol error if `foo.h` was included in multiple TUs.
In `splits.txt`, common BSS can be defined with the `common` attribute:
```
foo.cpp:
.text start:0x80047E5C end:0x8004875C
.ctors start:0x803A54C4 end:0x803A54C8
.data start:0x803B1B40 end:0x803B1B60
.bss start:0x803DF828 end:0x803DFA8C
.bss start:0x8040D4AC end:0x8040D4D8 common
```
As shown above, a file can contain both regular `.bss` and common `.bss`. Marking common `.bss` appropriately is important for determining the final link order.
## Detection
Example from Pikmin 2:
```
00016e60 00000c 805069c0 1 .bss utilityU.a PSMainSide_CreaturePrm.cpp
00016e60 00000c 805069c0 4 @3464 utilityU.a PSMainSide_CreaturePrm.cpp
00016e6c 000048 805069cc 4 saoVVOutput_direction___Q214JStudio_JStage14TAdaptor_light JSystem.a object-light.cpp
00016eb4 0000d0 80506a14 4 saoVVOutput___Q214JStudio_JStage14TAdaptor_actor JSystem.a object-actor.cpp
```
In this example, we see a symbol from `utilityU.a PSMainSide_CreaturePrm.cpp`. We know that this file is very close to the _end_ of the link order. Afterwards, there's a symbol from `JSystem.a object-light.cpp`, which is very close to the _beginning_ of the link order.
A file can't be both at the beginning and end of the link order, so it's a strong indication that `saoVVOutput_direction___Q214JStudio_JStage14TAdaptor_light` marks the beginning of the common BSS section.
One other indication from this example is the lack of a `.bss` section symbol from `JSystem.a object-actor.cpp` and any following files in the link order. Section symbols aren't generated for common BSS.
Without a map, it's harder to tell if there's a common BSS section, but guesses can be made. When looking at XREFs in Ghidra, if a symbol is close to the _end_ of `.bss`, but has XREFs from various addresses close to the _beginning_ of `.text`, it could be an indication of common BSS.
For games built with older versions of the linker, the inflation bug (described below) can also be used to detect common BSS.
## Inflation bug
In older versions of the linker (<= GC 2.6?), when calculating the size of common symbols, the linker will accidentally set the size of the first common symbol in a TU to the size of the _entire_ common section in that TU.
Example from Pikmin 2:
```
# Section Addr | Size | Addr | Alignment | Name | File
00017260 000188 80506dc0 4 mPadList__10JUTGamePad JSystem.a JUTGamePad.cpp
000173e8 000030 80506f48 4 mPadStatus__10JUTGamePad JSystem.a JUTGamePad.cpp
00017418 0000c0 80506f78 4 mPadButton__10JUTGamePad JSystem.a JUTGamePad.cpp
000174d8 000040 80507038 4 mPadMStick__10JUTGamePad JSystem.a JUTGamePad.cpp
00017518 000040 80507078 4 mPadSStick__10JUTGamePad JSystem.a JUTGamePad.cpp
00017558 00000c 805070b8 4 sPatternList__19JUTGamePadLongPress JSystem.a JUTGamePad.cpp
```
In this example, `mPadList__10JUTGamePad` is the first common symbol in the TU, and was inflated to include the size of all other common symbols in the TU. In reality, it's only supposed to be `0xC` bytes, given `0x188 - 0x30 - 0xC0 - 0x40 - 0x40 - 0xC`.
This can be useful to determine if symbols are in the same TU without a map: if a `.bss` symbol is much larger than expected, it could be the first common symbol in a TU. One can subtract the sizes of following symbols to find the true size of the symbol, along with the end of the TU's common symbols.
To reproduce this behavior, the `.comment` section must be present in the object. See [`.comment` section](comment_section.md) for more details.

33
docs/dependencies.md Normal file
View File

@ -0,0 +1,33 @@
# Dependencies
## Windows:
On Windows, it's **highly recommended** to use native tooling. WSL or msys2 are **not** required.
When running under WSL, [objdiff](#diffing) is unable to get filesystem notifications for automatic rebuilds.
- Install [Python](https://www.python.org/downloads/) and add it to `%PATH%`.
- Also available from the [Windows Store](https://apps.microsoft.com/store/detail/python-311/9NRWMJP3717K).
- Download [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`.
- Quick install via pip: `pip install ninja`
## macOS:
- Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages):
```
brew install ninja
```
- Install [wine-crossover](https://github.com/Gcenx/homebrew-wine):
```
brew install --cask --no-quarantine gcenx/wine/wine-crossover
```
After OS upgrades, if macOS complains about `Wine Crossover.app` being unverified, you can unquarantine it using:
```sh
sudo xattr -rd com.apple.quarantine '/Applications/Wine Crossover.app'
```
## Linux:
- Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages).
- For non-x86(_64) platforms: Install wine from your package manager.
- For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.

124
docs/getting_started.md Normal file
View File

@ -0,0 +1,124 @@
# Getting Started
See [Dependencies](dependencies.md) first.
Clone this template repository.
Rename `orig/GAMEID` to the game's ID. (For example, `GLZE01` for _The Legend of Zelda: The Wind Waker_.)
Extract your game to `orig/[GAMEID]`. In Dolphin, use "Extract Entire Disc" for GameCube games, or use "Data Partition" -> "Extract Entire Partition" for Wii games.
Rename `config/GAMEID` to the game's ID and modify `config/[GAMEID]/config.yml` appropriately, using [`config.example.yml`](/config/GAMEID/config.example.yml) as a reference. If the game doesn't use RELs, the `modules` list in `config.yml` can be removed.
Generate a `config/[GAMEID]/build.sha1` file for verification. This file is a list of SHA-1 hashes for each build artifact. One possible way:
```shell
$ dtk shasum orig/[GAMEID]/sys/main.dol orig/[GAMEID]/files/*.rel > config/[GAMEID]/build.sha1
```
Then, modify the paths in `config/[GAMEID]/build.sha1` to point to the `build` directory instead of `orig`. The DOL will be built at `build/[GAMEID]/main.dol`, and modules will be built at `build/[GAMEID]/[module_name]/[module_name].rel`.
Update `VERSIONS` in [`configure.py`](/configure.py) with the game ID.
Run `python configure.py` to generate the initial `build.ninja`.
Run `ninja` to perform initial analysis.
If all goes well, the initial `symbols.txt` and `splits.txt` should be automatically generated. Though it's likely it won't build yet. See [Post-analysis](#post-analysis) for next steps.
## Using a `.map`
If the game has `.map` files matching the DOL (and RELs, if applicable), they can be used to fill out `symbols.txt` and `splits.txt` automatically during the initial analysis.
Add the `map` key to `config.yml`, pointing to the `.map` file from the game disc. (For example, `orig/[GAMEID]/files/main.map`.) For RELs, add a `map` key to each module in `config.yml`.
If the game uses [common BSS](common_bss.md), be sure to set `common_start` as well. (See [`config.example.yml`](/config/GAMEID/config.example.yml).) Otherwise, the final link order may fail to be determined.
Once the initial analysis is completed, `symbols.txt` and `splits.txt` will be generated from the map information. **Remove** the `map` fields from `config.yml` to avoid conflicts.
## Post-analysis
After the initial analysis, `symbols.txt` and `splits.txt` will be generated. These files can be modified to adjust symbols and split points.
If the game uses C++ exceptions, it's required to set up a split for the `__init_cpp_exceptions.cpp` file. This differs between linker versions.
Often indicated by the following error:
```
# runtime sources 'global_destructor_chain.c' and
# '__init_cpp_exceptions.cpp' both need to be updated to latest version.
```
### GC 1.0 - 2.6 linkers:
```yaml
# splits.txt
Runtime.PPCEABI.H/__init_cpp_exceptions.cpp:
.text start:0x803294EC end:0x80329568
.ctors start:0x80338680 end:0x80338684
.dtors start:0x80338820 end:0x80338828
.sdata start:0x803F67F0 end:0x803F67F8
```
`.text`:
Find the following symbols in `symbols.txt`:
```
GetR2__Fv = .text:0x803294EC; // type:function size:0x8 scope:local align:4
__fini_cpp_exceptions = .text:0x803294F4; // type:function size:0x34 scope:global align:4
__init_cpp_exceptions = .text:0x80329528; // type:function size:0x40 scope:global align:4
```
The split end is the address of `__init_cpp_exceptions` + size.
`.ctors`:
Find the address of `__init_cpp_exception_reference` or `_ctors` in symbols.txt.
Always size 4.
`.dtors`:
Look for the address of `__destroy_global_chain_reference` or `_dtors` in symbols.txt.
If `__fini_cpp_exceptions_reference` is present, it's size 8, otherwise size 4
`.sdata`:
Find the following symbol in `symbols.txt`:
```
fragmentID = .sdata:0x803F67F0; // type:object size:0x4 scope:local align:4 data:4byte
```
The split end includes any inter-TU padding, so it's usually size 8.
### GC 2.7+ and Wii linkers:
```yaml
# splits.txt
Runtime.PPCEABI.H/__init_cpp_exceptions.cpp:
.text start:0x80345C34 end:0x80345CA4
.ctors start:0x803A54A0 end:0x803A54A4 rename:.ctors$10
.dtors start:0x803A56A0 end:0x803A56A4 rename:.dtors$10
.dtors start:0x803A56A4 end:0x803A56A8 rename:.dtors$15
.sdata start:0x80418CA8 end:0x80418CB0
```
`.text`:
Find the following symbols in `symbols.txt`:
```
__fini_cpp_exceptions = .text:0x80345C34; // type:function size:0x34 scope:global
__init_cpp_exceptions = .text:0x80345C68; // type:function size:0x3C scope:global
```
The split end is the address of `__init_cpp_exceptions` + size.
`.ctors$10`:
Find the address of `__init_cpp_exception_reference` or `_ctors` in symbols.txt.
Always size 4.
`.dtors$10`:
Look for the address of `__destroy_global_chain_reference` or `_dtors` in symbols.txt.
Always size 4.
`.dtors$15`:
Look for the address of `__fini_cpp_exceptions_reference` in symbols.txt.
Always size 4.
`.sdata`:
Find the following symbol in `symbols.txt`:
```
fragmentID = .sdata:0x80418CA8; // type:object size:0x4 scope:local data:4byte
```
The split end includes any inter-TU padding, so it's usually size 8.

42
docs/splits.md Normal file
View File

@ -0,0 +1,42 @@
# `splits.txt`
This file contains file splits for a module.
Example:
```
path/to/file.cpp:
.text start:0x80047E5C end:0x8004875C
.ctors start:0x803A54C4 end:0x803A54C8
.data start:0x803B1B40 end:0x803B1B60
.bss start:0x803DF828 end:0x803DFA8C
.bss start:0x8040D4AC end:0x8040D4D8 common
```
## Format
```
path/to/file.cpp: [file attributes]
section [section attributes]
...
```
- `path/to/file.cpp` The name of the source file, usually relative to `src`. The file does **not** need to exist to start.
This corresponds to an entry in `configure.py` for specifying compiler flags and other options.
### File attributes
- `comment:` Overrides the `mw_comment_version` setting in [`config.yml`](/config/GAMEID/config.example.yml) for this file. See [Comment section](comment_section.md).
`comment:0` is used to disable `.comment` section generation for a file that wasn't compiled with `mwcc`.
Example: `TRK_MINNOW_DOLPHIN/ppc/Export/targsupp.s: comment:0`
This file was assembled and only contains label symbols. Generating a `.comment` section for it will crash `mwld`.
### Section attributes
- `start:` The start address of the section within the file. For DOLs, this is the absolute address (e.g. `0x80001234`). For RELs, this is the section-relative address (e.g. `0x1234`).
- `end:` The end address of the section within the file.
- `align:` Specifies the alignment of the section. If not specified, the default alignment for the section is used.
- `rename:` Writes this section under a different name when generating the split object. Used for `.ctors$10`, etc.
- `common` Only valid for `.bss`. See [Common BSS](common_bss.md).
- `skip` Skips this data when writing the object file. Used for ignoring data that's linker-generated.

35
docs/symbols.md Normal file
View File

@ -0,0 +1,35 @@
# `symbols.txt`
This file contains all symbols for a module, one per line.
Example line:
```
__dt__13mDoExt_bckAnmFv = .text:0x800DD2EC; // type:function size:0x5C scope:global align:4
```
## Format
Numbers can be written as decimal or hexadecimal. Hexadecimal numbers must be prefixed with `0x`.
Comment lines starting with `//` or `#` are permitted, but are currently **not** preserved when updating the file.
```
symbol_name = section:address; // [attributes]
```
- `symbol_name` - The name of the symbol. (For C++, this is the mangled name, e.g. `__dt__13mDoExt_bckAnmFv`)
- `section` - The section the symbol is in.
- `address` - The address of the symbol. For DOLs, this is the absolute address (e.g. `0x80001234`). For RELs, this is the section-relative address (e.g. `0x1234`).
### Attributes
All attributes are optional, and are separated by spaces.
- `type:` The symbol type. `function`, `object`, or `label`.
- `size:` The size of the symbol.
- `scope:` The symbol's visibility. `global` (default), `local` or `weak`.
- `align:` The symbol's alignment.
- `data:` The data type used when writing disassembly. `byte`, `2byte`, `4byte`, `8byte`, `float`, `double`, `string`, `wstring`, `string_table`, or `wstring_table`.
- `hidden` Marked as "hidden" in the generated object. (Only used for extab)
- `force_active` Marked as ["exported"](comment_section.md) in the generated object, and added to `FORCEACTIVE` in the generated `ldscript.lcf`. Prevents the symbol from being deadstripped.
- `noreloc` Prevents the _contents_ of the symbol from being interpreted as addresses. Used for objects containing data that look like pointers, but aren't.

0
orig/GAMEID/.gitkeep Normal file
View File

0
tools/__init__.py Normal file
View File

87
tools/decompctx.py Normal file
View File

@ -0,0 +1,87 @@
#!/usr/bin/env python3
###
# Generates a ctx.c file, usable for "Context" on https://decomp.me.
#
# Usage:
# python3 tools/decompctx.py src/file.cpp
#
# If changes are made, please submit a PR to
# https://github.com/encounter/dtk-template
###
import argparse
import os
import re
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.abspath(os.path.join(script_dir, ".."))
src_dir = os.path.join(root_dir, "src")
include_dir = os.path.join(root_dir, "include")
include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$')
guard_pattern = re.compile(r'^#ifndef\s+(.*)$')
defines = set()
def import_h_file(in_file: str, r_path: str) -> str:
rel_path = os.path.join(root_dir, r_path, in_file)
inc_path = os.path.join(include_dir, in_file)
if os.path.exists(rel_path):
return import_c_file(rel_path)
elif os.path.exists(inc_path):
return import_c_file(inc_path)
else:
print("Failed to locate", in_file)
exit(1)
def import_c_file(in_file) -> str:
in_file = os.path.relpath(in_file, root_dir)
out_text = ''
try:
with open(in_file, encoding="utf-8") as file:
out_text += process_file(in_file, list(file))
except Exception:
with open(in_file) as file:
out_text += process_file(in_file, list(file))
return out_text
def process_file(in_file: str, lines) -> str:
out_text = ''
for idx, line in enumerate(lines):
guard_match = guard_pattern.match(line.strip())
if idx == 0:
if guard_match:
if guard_match[1] in defines:
break
defines.add(guard_match[1])
print("Processing file", in_file)
include_match = include_pattern.match(line.strip())
if include_match and not include_match[1].endswith(".s"):
out_text += f"/* \"{in_file}\" line {idx} \"{include_match[1]}\" */\n"
out_text += import_h_file(include_match[1], os.path.dirname(in_file))
out_text += f"/* end \"{include_match[1]}\" */\n"
else:
out_text += line
return out_text
def main():
parser = argparse.ArgumentParser(
description="""Create a context file which can be used for decomp.me"""
)
parser.add_argument(
"c_file",
help="""File from which to create context""",
)
args = parser.parse_args()
output = import_c_file(args.c_file)
with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f:
f.write(output)
if __name__ == "__main__":
main()

93
tools/download_tool.py Normal file
View File

@ -0,0 +1,93 @@
#!/usr/bin/env python3
###
# Downloads various tools from GitHub releases.
#
# Usage:
# python3 tools/download_tool.py wibo build/tools/wibo --tag 1.0.0
#
# If changes are made, please submit a PR to
# https://github.com/encounter/dtk-template
###
import argparse
import io
import os
import platform
import shutil
import stat
import sys
import urllib.request
import zipfile
from pathlib import Path
def dtk_url(tag):
uname = platform.uname()
suffix = ""
system = uname.system.lower()
if system == "darwin":
system = "macos"
elif system == "windows":
suffix = ".exe"
arch = uname.machine.lower()
if arch == "amd64":
arch = "x86_64"
repo = "https://github.com/encounter/decomp-toolkit"
return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}"
def sjiswrap_url(tag):
repo = "https://github.com/encounter/sjiswrap"
return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe"
def wibo_url(tag):
repo = "https://github.com/decompals/wibo"
return f"{repo}/releases/download/{tag}/wibo"
def compilers_url(tag):
if tag == "1":
return "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip"
else:
sys.exit("Unknown compilers tag %s" % tag)
TOOLS = {
"dtk": dtk_url,
"sjiswrap": sjiswrap_url,
"wibo": wibo_url,
"compilers": compilers_url,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("tool", help="Tool name")
parser.add_argument("output", type=Path, help="output file path")
parser.add_argument("--tag", help="GitHub tag", required=True)
args = parser.parse_args()
url = TOOLS[args.tool](args.tag)
output = Path(args.output)
print(f"Downloading {url} to {output}")
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req) as response:
if url.endswith(".zip"):
data = io.BytesIO(response.read())
with zipfile.ZipFile(data) as f:
f.extractall(output)
output.touch(mode=0o755)
else:
with open(output, "wb") as f:
shutil.copyfileobj(response, f)
st = os.stat(output)
os.chmod(output, st.st_mode | stat.S_IEXEC)
if __name__ == "__main__":
main()

223
tools/ninja_syntax.py Normal file
View File

@ -0,0 +1,223 @@
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Python module for generating .ninja files.
Note that this is emphatically not a required piece of Ninja; it's
just a helpful utility for build-file-generation systems that already
use Python.
"""
import re
import textwrap
def escape_path(word):
return word.replace("$ ", "$$ ").replace(" ", "$ ").replace(":", "$:")
class Writer(object):
def __init__(self, output, width=78):
self.output = output
self.width = width
def newline(self):
self.output.write("\n")
def comment(self, text):
for line in textwrap.wrap(
text, self.width - 2, break_long_words=False, break_on_hyphens=False
):
self.output.write("# " + line + "\n")
def variable(self, key, value, indent=0):
if value is None:
return
if isinstance(value, list):
value = " ".join(filter(None, value)) # Filter out empty strings.
self._line("%s = %s" % (key, value), indent)
def pool(self, name, depth):
self._line("pool %s" % name)
self.variable("depth", depth, indent=1)
def rule(
self,
name,
command,
description=None,
depfile=None,
generator=False,
pool=None,
restat=False,
rspfile=None,
rspfile_content=None,
deps=None,
):
self._line("rule %s" % name)
self.variable("command", command, indent=1)
if description:
self.variable("description", description, indent=1)
if depfile:
self.variable("depfile", depfile, indent=1)
if generator:
self.variable("generator", "1", indent=1)
if pool:
self.variable("pool", pool, indent=1)
if restat:
self.variable("restat", "1", indent=1)
if rspfile:
self.variable("rspfile", rspfile, indent=1)
if rspfile_content:
self.variable("rspfile_content", rspfile_content, indent=1)
if deps:
self.variable("deps", deps, indent=1)
def build(
self,
outputs,
rule,
inputs=None,
implicit=None,
order_only=None,
variables=None,
implicit_outputs=None,
pool=None,
dyndep=None,
):
outputs = as_list(outputs)
out_outputs = [escape_path(x) for x in outputs]
all_inputs = [escape_path(x) for x in as_list(inputs)]
if implicit:
implicit = [escape_path(x) for x in as_list(implicit)]
all_inputs.append("|")
all_inputs.extend(implicit)
if order_only:
order_only = [escape_path(x) for x in as_list(order_only)]
all_inputs.append("||")
all_inputs.extend(order_only)
if implicit_outputs:
implicit_outputs = [escape_path(x) for x in as_list(implicit_outputs)]
out_outputs.append("|")
out_outputs.extend(implicit_outputs)
self._line(
"build %s: %s" % (" ".join(out_outputs), " ".join([rule] + all_inputs))
)
if pool is not None:
self._line(" pool = %s" % pool)
if dyndep is not None:
self._line(" dyndep = %s" % dyndep)
if variables:
if isinstance(variables, dict):
iterator = iter(variables.items())
else:
iterator = iter(variables)
for key, val in iterator:
self.variable(key, val, indent=1)
return outputs
def include(self, path):
self._line("include %s" % path)
def subninja(self, path):
self._line("subninja %s" % path)
def default(self, paths):
self._line("default %s" % " ".join(as_list(paths)))
def _count_dollars_before_index(self, s, i):
"""Returns the number of '$' characters right in front of s[i]."""
dollar_count = 0
dollar_index = i - 1
while dollar_index > 0 and s[dollar_index] == "$":
dollar_count += 1
dollar_index -= 1
return dollar_count
def _line(self, text, indent=0):
"""Write 'text' word-wrapped at self.width characters."""
leading_space = " " * indent
while len(leading_space) + len(text) > self.width:
# The text is too wide; wrap if possible.
# Find the rightmost space that would obey our width constraint and
# that's not an escaped space.
available_space = self.width - len(leading_space) - len(" $")
space = available_space
while True:
space = text.rfind(" ", 0, space)
if space < 0 or self._count_dollars_before_index(text, space) % 2 == 0:
break
if space < 0:
# No such space; just use the first unescaped space we can find.
space = available_space - 1
while True:
space = text.find(" ", space + 1)
if (
space < 0
or self._count_dollars_before_index(text, space) % 2 == 0
):
break
if space < 0:
# Give up on breaking.
break
self.output.write(leading_space + text[0:space] + " $\n")
text = text[space + 1 :]
# Subsequent lines are continuations, so indent them.
leading_space = " " * (indent + 2)
self.output.write(leading_space + text + "\n")
def close(self):
self.output.close()
def as_list(input):
if input is None:
return []
if isinstance(input, list):
return input
return [input]
def escape(string):
"""Escape a string such that it can be embedded into a Ninja file without
further interpretation."""
assert "\n" not in string, "Ninja syntax does not allow newlines"
# We only have one special metacharacter: '$'.
return string.replace("$", "$$")
def expand(string, vars, local_vars={}):
"""Expand a string containing $vars as Ninja would.
Note: doesn't handle the full Ninja variable syntax, but it's enough
to make configure.py's use of it work.
"""
def exp(m):
var = m.group(1)
if var == "$":
return "$"
return local_vars.get(var, vars.get(var, ""))
return re.sub(r"\$(\$|\w*)", exp, string)

982
tools/project.py Normal file
View File

@ -0,0 +1,982 @@
###
# decomp-toolkit project generator
# Generates build.ninja and objdiff.json.
#
# This generator is intentionally project-agnostic
# and shared between multiple projects. Any configuration
# specific to a project should be added to `configure.py`.
#
# If changes are made, please submit a PR to
# https://github.com/encounter/dtk-template
###
import io
import json
import os
import platform
import sys
from pathlib import Path
from . import ninja_syntax
if sys.platform == "cygwin":
sys.exit(
f"Cygwin/MSYS2 is not supported."
f"\nPlease use native Windows Python instead."
f"\n(Current path: {sys.executable})"
)
class ProjectConfig:
def __init__(self):
# Paths
self.build_dir = Path("build")
self.src_dir = Path("src")
self.tools_dir = Path("tools")
# Tooling
self.dtk_tag = None # Git tag
self.build_dtk_path = None # If None, download
self.compilers_tag = None # 1
self.compilers_path = None # If None, download
self.wibo_tag = None # Git tag
self.wrapper = None # If None, download wibo on Linux
self.sjiswrap_tag = None # Git tag
self.sjiswrap_path = None # If None, download
# Project config
self.build_rels = True # Build REL files
self.check_sha_path = None # Path to version.sha1
self.config_path = None # Path to config.yml
self.debug = False # Build with debug info
self.generate_map = False # Generate map file(s)
self.ldflags = None # Linker flags
self.libs = None # List of libraries
self.linker_version = None # mwld version
self.version = None # Version name
self.warn_missing_config = False # Warn on missing unit configuration
self.warn_missing_source = False # Warn on missing source file
# Progress output and progress.json config
self.progress_all = True # Include combined "all" category
self.progress_modules = True # Include combined "modules" category
self.progress_each_module = (
True # Include individual modules, disable for large numbers of modules
)
def validate(self):
required_attrs = [
"build_dir",
"src_dir",
"tools_dir",
"check_sha_path",
"config_path",
"ldflags",
"linker_version",
"libs",
"version",
]
for attr in required_attrs:
if getattr(self, attr) is None:
sys.exit(f"ProjectConfig.{attr} missing")
def find_object(self, name):
for lib in self.libs:
for obj in lib["objects"]:
if obj.name == name:
return [lib, obj]
return None
def out_path(self):
return self.build_dir / self.version
class Object:
def __init__(self, completed, name, **options):
self.name = name
self.completed = completed
self.options = {
"add_to_all": True,
"cflags": None,
"mw_version": None,
"shiftjis": True,
"source": name,
}
self.options.update(options)
def is_windows():
return os.name == "nt"
# On Windows, we need this to use && in commands
CHAIN = "cmd /c " if is_windows() else ""
# Native executable extension
EXE = ".exe" if is_windows() else ""
# Replace forward slashes with backslashes on Windows
def os_str(value):
return str(value).replace("/", os.sep)
# Replace backslashes with forward slashes on Windows
def unix_str(value):
return str(value).replace(os.sep, "/")
# Stringify paths for ninja_syntax
def path(value):
if value is None:
return None
elif isinstance(value, list):
return list(map(os_str, filter(lambda x: x is not None, value)))
else:
return [os_str(value)]
# Load decomp-toolkit generated config.json
def load_build_config(config, build_config_path):
if not build_config_path.is_file():
return None
def versiontuple(v):
return tuple(map(int, (v.split("."))))
f = open(build_config_path, "r", encoding="utf-8")
build_config = json.load(f)
config_version = build_config.get("version")
if not config_version:
# Invalid config.json
f.close()
os.remove(build_config_path)
return None
dtk_version = config.dtk_tag[1:] # Strip v
if versiontuple(config_version) < versiontuple(dtk_version):
# Outdated config.json
f.close()
os.remove(build_config_path)
return None
f.close()
return build_config
# Generate build.ninja and objdiff.json
def generate_build(config):
build_config = load_build_config(config, config.out_path() / "config.json")
generate_build_ninja(config, build_config)
generate_objdiff_config(config, build_config)
# Generate build.ninja
def generate_build_ninja(config, build_config):
config.validate()
out = io.StringIO()
n = ninja_syntax.Writer(out)
n.variable("ninja_required_version", "1.3")
n.newline()
configure_script = os.path.relpath(os.path.abspath(sys.argv[0]))
python_lib = os.path.relpath(__file__)
python_lib_dir = os.path.dirname(python_lib)
n.comment("The arguments passed to configure.py, for rerunning it.")
n.variable("configure_args", sys.argv[1:])
n.variable("python", f'"{sys.executable}"')
n.newline()
###
# Variables
###
n.comment("Variables")
ldflags = " ".join(config.ldflags)
if config.generate_map:
ldflags += " -mapunused"
if config.debug:
ldflags += " -g"
n.variable("ldflags", ldflags)
n.variable("mw_version", config.linker_version)
n.newline()
###
# Tooling
###
n.comment("Tooling")
build_path = config.out_path()
build_tools_path = config.build_dir / "tools"
download_tool = config.tools_dir / "download_tool.py"
n.rule(
name="download_tool",
command=f"$python {download_tool} $tool $out --tag $tag",
description="TOOL $out",
)
if config.build_dtk_path:
dtk = build_tools_path / "release" / f"dtk{EXE}"
n.rule(
name="cargo",
command="cargo build --release --manifest-path $in --bin $bin --target-dir $target",
description="CARGO $bin",
depfile=path(Path("$target") / "release" / "$bin.d"),
deps="gcc",
)
n.build(
outputs=path(dtk),
rule="cargo",
inputs=path(config.build_dtk_path / "Cargo.toml"),
implicit=path(config.build_dtk_path / "Cargo.lock"),
variables={
"bin": "dtk",
"target": build_tools_path,
},
)
elif config.dtk_tag:
dtk = build_tools_path / f"dtk{EXE}"
n.build(
outputs=path(dtk),
rule="download_tool",
implicit=path(download_tool),
variables={
"tool": "dtk",
"tag": config.dtk_tag,
},
)
else:
sys.exit("ProjectConfig.dtk_tag missing")
if config.sjiswrap_path:
sjiswrap = config.sjiswrap_path
elif config.sjiswrap_tag:
sjiswrap = build_tools_path / "sjiswrap.exe"
n.build(
outputs=path(sjiswrap),
rule="download_tool",
implicit=path(download_tool),
variables={
"tool": "sjiswrap",
"tag": config.sjiswrap_tag,
},
)
else:
sys.exit("ProjectConfig.sjiswrap_tag missing")
# Only add an implicit dependency on wibo if we download it
wrapper = config.wrapper
wrapper_implicit = None
if (
config.wibo_tag is not None
and sys.platform == "linux"
and platform.machine() in ("i386", "x86_64")
and config.wrapper is None
):
wrapper = build_tools_path / "wibo"
wrapper_implicit = wrapper
n.build(
outputs=path(wrapper),
rule="download_tool",
implicit=path(download_tool),
variables={
"tool": "wibo",
"tag": config.wibo_tag,
},
)
if not is_windows() and wrapper is None:
wrapper = "wine"
wrapper_cmd = f"{wrapper} " if wrapper else ""
compilers_implicit = None
if config.compilers_path:
compilers = config.compilers_path
elif config.compilers_tag:
compilers = config.build_dir / "compilers"
compilers_implicit = compilers
n.build(
outputs=path(compilers),
rule="download_tool",
implicit=path(download_tool),
variables={
"tool": "compilers",
"tag": config.compilers_tag,
},
)
n.newline()
###
# Build rules
###
compiler_path = compilers / "$mw_version"
# MWCC
mwcc = compiler_path / "mwcceppc.exe"
mwcc_cmd = f"{wrapper_cmd}{mwcc} $cflags -MMD -c $in -o $basedir"
mwcc_implicit = [compilers_implicit or mwcc, wrapper_implicit]
# MWCC with UTF-8 to Shift JIS wrapper
mwcc_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir"
mwcc_sjis_implicit = [*mwcc_implicit, sjiswrap]
# MWLD
mwld = compiler_path / "mwldeppc.exe"
mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp"
mwld_implicit = [compilers_implicit or mwld, wrapper_implicit]
if os.name != "nt":
transform_dep = config.tools_dir / "transform_dep.py"
mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d"
mwcc_sjis_cmd += f" && $python {transform_dep} $basefile.d $basefile.d"
mwcc_implicit.append(transform_dep)
mwcc_sjis_implicit.append(transform_dep)
n.comment("Link ELF file")
n.rule(
name="link",
command=mwld_cmd,
description="LINK $out",
rspfile="$out.rsp",
rspfile_content="$in_newline",
)
n.newline()
n.comment("Generate DOL")
n.rule(
name="elf2dol",
command=f"{dtk} elf2dol $in $out",
description="DOL $out",
)
n.newline()
n.comment("Generate REL(s)")
makerel_rsp = build_path / "makerel.rsp"
n.rule(
name="makerel",
command=f"{dtk} rel make -w -c $config @{makerel_rsp}",
description="REL",
rspfile=path(makerel_rsp),
rspfile_content="$in_newline",
)
n.newline()
n.comment("MWCC build")
n.rule(
name="mwcc",
command=mwcc_cmd,
description="MWCC $out",
depfile="$basefile.d",
deps="gcc",
)
n.newline()
n.comment("MWCC build (with UTF-8 to Shift JIS wrapper)")
n.rule(
name="mwcc_sjis",
command=mwcc_sjis_cmd,
description="MWCC $out",
depfile="$basefile.d",
deps="gcc",
)
n.newline()
n.comment("Host build")
n.variable("host_cflags", "-I include -Wno-trigraphs")
n.variable(
"host_cppflags",
"-std=c++98 -I include -fno-exceptions -fno-rtti -D_CRT_SECURE_NO_WARNINGS -Wno-trigraphs -Wno-c++11-extensions",
)
n.rule(
name="host_cc",
command="clang $host_cflags -c -o $out $in",
description="CC $out",
)
n.rule(
name="host_cpp",
command="clang++ $host_cppflags -c -o $out $in",
description="CXX $out",
)
n.newline()
###
# Source files
###
n.comment("Source files")
build_src_path = build_path / "src"
build_host_path = build_path / "host"
build_config_path = build_path / "config.json"
def map_path(path):
return path.parent / (path.name + ".MAP")
class LinkStep:
def __init__(self, config):
self.name = config["name"]
self.module_id = config["module_id"]
self.ldscript = config["ldscript"]
self.entry = config["entry"]
self.inputs = []
def add(self, obj):
self.inputs.append(obj)
def output(self):
if self.module_id == 0:
return build_path / f"{self.name}.dol"
else:
return build_path / self.name / f"{self.name}.rel"
def partial_output(self):
if self.module_id == 0:
return build_path / f"{self.name}.elf"
else:
return build_path / self.name / f"{self.name}.plf"
def write(self, n):
n.comment(f"Link {self.name}")
if self.module_id == 0:
elf_path = build_path / f"{self.name}.elf"
dol_path = build_path / f"{self.name}.dol"
elf_ldflags = f"$ldflags -lcf {self.ldscript}"
if config.generate_map:
elf_map = map_path(elf_path)
elf_ldflags += f" -map {elf_map}"
else:
elf_map = None
n.build(
outputs=path(elf_path),
rule="link",
inputs=path(self.inputs),
implicit=path([self.ldscript, *mwld_implicit]),
implicit_outputs=path(elf_map),
variables={"ldflags": elf_ldflags},
)
n.build(
outputs=path(dol_path),
rule="elf2dol",
inputs=path(elf_path),
implicit=path(dtk),
)
else:
preplf_path = build_path / self.name / f"{self.name}.preplf"
plf_path = build_path / self.name / f"{self.name}.plf"
preplf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -r"
plf_ldflags = f"$ldflags -sdata 0 -sdata2 0 -m {self.entry} -r1 -strip_partial -lcf {self.ldscript}"
if config.generate_map:
preplf_map = map_path(preplf_path)
preplf_ldflags += f" -map {preplf_map}"
plf_map = map_path(plf_path)
plf_ldflags += f" -map {plf_map}"
else:
preplf_map = None
plf_map = None
n.build(
outputs=path(preplf_path),
rule="link",
inputs=path(self.inputs),
implicit=path(mwld_implicit),
implicit_outputs=path(preplf_map),
variables={"ldflags": preplf_ldflags},
)
n.build(
outputs=path(plf_path),
rule="link",
inputs=path(self.inputs),
implicit=path([self.ldscript, preplf_path, *mwld_implicit]),
implicit_outputs=path(plf_map),
variables={"ldflags": plf_ldflags},
)
n.newline()
if build_config:
link_steps = []
used_compiler_versions = set()
source_inputs = []
host_source_inputs = []
source_added = set()
def add_unit(build_obj, link_step):
obj_path, obj_name = build_obj["object"], build_obj["name"]
result = config.find_object(obj_name)
if not result:
if config.warn_missing_config and not build_obj["autogenerated"]:
print(f"Missing configuration for {obj_name}")
link_step.add(obj_path)
return
lib, obj = result
lib_name = lib["lib"]
options = obj.options
completed = obj.completed
unit_src_path = config.src_dir / options["source"]
if not unit_src_path.exists():
if config.warn_missing_source:
print(f"Missing source file {unit_src_path}")
link_step.add(obj_path)
return
mw_version = options["mw_version"] or lib["mw_version"]
cflags = options["cflags"] or lib["cflags"]
if type(cflags) is list:
cflags_str = " ".join(cflags)
else:
cflags_str = str(cflags)
used_compiler_versions.add(mw_version)
base_object = Path(obj.name).with_suffix("")
src_obj_path = build_src_path / f"{base_object}.o"
src_base_path = build_src_path / base_object
if src_obj_path not in source_added:
source_added.add(src_obj_path)
n.comment(f"{obj_name}: {lib_name} (linked {completed})")
n.build(
outputs=path(src_obj_path),
rule="mwcc_sjis" if options["shiftjis"] else "mwcc",
inputs=path(unit_src_path),
variables={
"mw_version": path(Path(mw_version)),
"cflags": cflags_str,
"basedir": os.path.dirname(src_base_path),
"basefile": path(src_base_path),
},
implicit=path(
mwcc_sjis_implicit if options["shiftjis"] else mwcc_implicit
),
)
if lib["host"]:
host_obj_path = build_host_path / f"{base_object}.o"
host_base_path = build_host_path / base_object
n.build(
outputs=path(host_obj_path),
rule="host_cc" if unit_src_path.suffix == ".c" else "host_cpp",
inputs=path(unit_src_path),
variables={
"basedir": os.path.dirname(host_base_path),
"basefile": path(host_base_path),
},
)
if options["add_to_all"]:
host_source_inputs.append(host_obj_path)
n.newline()
if options["add_to_all"]:
source_inputs.append(src_obj_path)
if completed:
obj_path = src_obj_path
link_step.add(obj_path)
# Add DOL link step
link_step = LinkStep(build_config)
for unit in build_config["units"]:
add_unit(unit, link_step)
link_steps.append(link_step)
if config.build_rels:
# Add REL link steps
for module in build_config["modules"]:
module_link_step = LinkStep(module)
for unit in module["units"]:
add_unit(unit, module_link_step)
link_steps.append(module_link_step)
n.newline()
# Check if all compiler versions exist
for mw_version in used_compiler_versions:
mw_path = compilers / mw_version / "mwcceppc.exe"
if config.compilers_path and not os.path.exists(mw_path):
sys.exit(f"Compiler {mw_path} does not exist")
# Check if linker exists
mw_path = compilers / config.linker_version / "mwldeppc.exe"
if config.compilers_path and not os.path.exists(mw_path):
sys.exit(f"Linker {mw_path} does not exist")
###
# Link
###
for step in link_steps:
step.write(n)
n.newline()
###
# Generate RELs
###
rel_outputs = list(
map(
lambda step: step.output(),
filter(lambda step: step.module_id != 0, link_steps),
)
)
if len(rel_outputs) > 0:
n.comment("Generate RELs")
n.build(
outputs=path(rel_outputs),
rule="makerel",
inputs=path(list(map(lambda step: step.partial_output(), link_steps))),
implicit=path([dtk, config.config_path]),
variables={"config": path(config.config_path)},
)
n.newline()
###
# Helper rule for building all source files
###
n.comment("Build all source files")
n.build(
outputs="all_source",
rule="phony",
inputs=path(source_inputs),
)
n.newline()
###
# Helper rule for building all source files, with a host compiler
###
n.comment("Build all source files with a host compiler")
n.build(
outputs="all_source_host",
rule="phony",
inputs=path(host_source_inputs),
)
n.newline()
###
# Check hash
###
n.comment("Check hash")
ok_path = build_path / "ok"
quiet = "-q " if len(link_steps) > 3 else ""
n.rule(
name="check",
command=f"{dtk} shasum {quiet} -c $in -o $out",
description="CHECK $in",
)
n.build(
outputs=path(ok_path),
rule="check",
inputs=path(config.check_sha_path),
implicit=path([dtk, *map(lambda step: step.output(), link_steps)]),
)
n.newline()
###
# Calculate progress
###
n.comment("Calculate progress")
progress_path = build_path / "progress.json"
n.rule(
name="progress",
command=f"$python {configure_script} $configure_args progress",
description="PROGRESS",
)
n.build(
outputs=path(progress_path),
rule="progress",
implicit=path([ok_path, configure_script, python_lib, config.config_path]),
)
###
# Helper tools
###
# TODO: make these rules work for RELs too
dol_link_step = link_steps[0]
dol_elf_path = dol_link_step.partial_output()
n.comment("Check for mismatching symbols")
n.rule(
name="dol_diff",
command=f"{dtk} -L error dol diff $in",
description=f"DIFF {dol_elf_path}",
)
n.build(
inputs=path([config.config_path, dol_elf_path]),
outputs="dol_diff",
rule="dol_diff",
)
n.build(
outputs="diff",
rule="phony",
inputs="dol_diff",
)
n.newline()
n.comment("Apply symbols from linked ELF")
n.rule(
name="dol_apply",
command=f"{dtk} dol apply $in",
description=f"APPLY {dol_elf_path}",
)
n.build(
inputs=path([config.config_path, dol_elf_path]),
outputs="dol_apply",
rule="dol_apply",
implicit=path([ok_path]),
)
n.build(
outputs="apply",
rule="phony",
inputs="dol_apply",
)
n.newline()
###
# Split DOL
###
n.comment("Split DOL into relocatable objects")
n.rule(
name="split",
command=f"{dtk} dol split $in $out_dir",
description="SPLIT $in",
depfile="$out_dir/dep",
deps="gcc",
)
n.build(
inputs=path(config.config_path),
outputs=path(build_config_path),
rule="split",
implicit=path(dtk),
variables={"out_dir": path(build_path)},
)
n.newline()
###
# Regenerate on change
###
n.comment("Reconfigure on change")
n.rule(
name="configure",
command=f"$python {configure_script} $configure_args",
generator=True,
description=f"RUN {configure_script}",
)
n.build(
outputs="build.ninja",
rule="configure",
implicit=path(
[
build_config_path,
configure_script,
python_lib,
Path(python_lib_dir) / "ninja_syntax.py",
]
),
)
n.newline()
###
# Default rule
###
n.comment("Default rule")
if build_config:
n.default(path(progress_path))
else:
n.default(path(build_config_path))
# Write build.ninja
with open("build.ninja", "w", encoding="utf-8") as f:
f.write(out.getvalue())
out.close()
# Generate objdiff.json
def generate_objdiff_config(config, build_config):
if not build_config:
return
objdiff_config = {
"min_version": "0.4.3",
"custom_make": "ninja",
"build_target": False,
"watch_patterns": [
"*.c",
"*.cp",
"*.cpp",
"*.h",
"*.hpp",
"*.py",
"*.yml",
"*.txt",
"*.json",
],
"units": [],
}
build_path = config.out_path()
def add_unit(build_obj, module_name):
if build_obj["autogenerated"]:
# Skip autogenerated objects
return
obj_path, obj_name = build_obj["object"], build_obj["name"]
base_object = Path(obj_name).with_suffix("")
unit_config = {
"name": unix_str(Path(module_name) / base_object),
"target_path": unix_str(obj_path),
}
result = config.find_object(obj_name)
if not result:
objdiff_config["units"].append(unit_config)
return
lib, obj = result
unit_src_path = config.src_dir / obj.options["source"]
if not unit_src_path.exists():
objdiff_config["units"].append(unit_config)
return
cflags = obj.options["cflags"] or lib["cflags"]
src_obj_path = build_path / "src" / f"{base_object}.o"
reverse_fn_order = False
if type(cflags) is list:
for flag in cflags:
if not flag.startswith("-inline "):
continue
for value in flag.split(" ")[1].split(","):
if value == "deferred":
reverse_fn_order = True
elif value == "nodeferred":
reverse_fn_order = False
unit_config["base_path"] = unix_str(src_obj_path)
unit_config["reverse_fn_order"] = reverse_fn_order
unit_config["complete"] = obj.completed
objdiff_config["units"].append(unit_config)
# Add DOL units
for unit in build_config["units"]:
add_unit(unit, build_config["name"])
# Add REL units
for module in build_config["modules"]:
for unit in module["units"]:
add_unit(unit, module["name"])
# Write objdiff.json
with open("objdiff.json", "w", encoding="utf-8") as w:
json.dump(objdiff_config, w, indent=4)
# Calculate, print and write progress to progress.json
def calculate_progress(config):
out_path = config.out_path()
build_config = load_build_config(config, out_path / "config.json")
if not build_config:
return
class ProgressUnit:
def __init__(self, name):
self.name = name
self.code_total = 0
self.code_progress = 0
self.data_total = 0
self.data_progress = 0
self.objects_progress = 0
self.objects_total = 0
self.objects = set()
def add(self, build_obj):
self.code_total += build_obj["code_size"]
self.data_total += build_obj["data_size"]
# Avoid counting the same object in different modules twice
include_object = build_obj["name"] not in self.objects
if include_object:
self.objects.add(build_obj["name"])
self.objects_total += 1
if build_obj["autogenerated"]:
# Skip autogenerated objects
return
result = config.find_object(build_obj["name"])
if not result:
return
_, obj = result
if not obj.completed:
return
self.code_progress += build_obj["code_size"]
self.data_progress += build_obj["data_size"]
if include_object:
self.objects_progress += 1
def code_frac(self):
return self.code_progress / self.code_total
def data_frac(self):
return self.data_progress / self.data_total
# Add DOL units
all_progress = ProgressUnit("All") if config.progress_all else None
dol_progress = ProgressUnit("DOL")
for unit in build_config["units"]:
if all_progress:
all_progress.add(unit)
dol_progress.add(unit)
# Add REL units
rels_progress = ProgressUnit("Modules") if config.progress_modules else None
modules_progress = []
for module in build_config["modules"]:
progress = ProgressUnit(module["name"])
modules_progress.append(progress)
for unit in module["units"]:
if all_progress:
all_progress.add(unit)
if rels_progress:
rels_progress.add(unit)
progress.add(unit)
# Print human-readable progress
print("Progress:")
def print_category(unit):
code_frac = unit.code_frac()
data_frac = unit.data_frac()
print(
f" {unit.name}: {code_frac:.2%} code, {data_frac:.2%} data ({unit.objects_progress} / {unit.objects_total} files)"
)
print(f" Code: {unit.code_progress} / {unit.code_total} bytes")
print(f" Data: {unit.data_progress} / {unit.data_total} bytes")
if all_progress:
print_category(all_progress)
print_category(dol_progress)
module_count = len(build_config["modules"])
if module_count > 0:
print_category(rels_progress)
if config.progress_each_module:
for progress in modules_progress:
print_category(progress)
# Generate and write progress.json
progress_json = {}
def add_category(category, unit):
progress_json[category] = {
"code": unit.code_progress,
"code/total": unit.code_total,
"data": unit.data_progress,
"data/total": unit.data_total,
}
if all_progress:
add_category("all", all_progress)
add_category("dol", dol_progress)
if len(build_config["modules"]) > 0:
if rels_progress:
add_category("modules", rels_progress)
if config.progress_each_module:
for progress in modules_progress:
add_category(progress.name, progress)
with open(out_path / "progress.json", "w", encoding="utf-8") as w:
json.dump(progress_json, w, indent=4)

84
tools/transform_dep.py Normal file
View File

@ -0,0 +1,84 @@
#!/usr/bin/env python3
###
# Transforms .d files, converting Windows paths to Unix paths.
# Allows usage of the mwcc -MMD flag on platforms other than Windows.
#
# Usage:
# python3 tools/transform_dep.py build/src/file.d build/src/file.d
#
# If changes are made, please submit a PR to
# https://github.com/encounter/dtk-template
###
import argparse
import os
from platform import uname
wineprefix = os.path.join(os.environ["HOME"], ".wine")
if "WINEPREFIX" in os.environ:
wineprefix = os.environ["WINEPREFIX"]
winedevices = os.path.join(wineprefix, "dosdevices")
def in_wsl() -> bool:
return "microsoft-standard" in uname().release
def import_d_file(in_file) -> str:
out_text = ""
with open(in_file) as file:
for idx, line in enumerate(file):
if idx == 0:
if line.endswith(" \\\n"):
out_text += line[:-3].replace("\\", "/") + " \\\n"
else:
out_text += line.replace("\\", "/")
else:
suffix = ""
if line.endswith(" \\\n"):
suffix = " \\"
path = line.lstrip()[:-3]
else:
path = line.strip()
# lowercase drive letter
path = path[0].lower() + path[1:]
if path[0] == "z":
# shortcut for z:
path = path[2:].replace("\\", "/")
elif in_wsl():
path = path[0:1] + path[2:]
path = os.path.join("/mnt", path.replace("\\", "/"))
else:
# use $WINEPREFIX/dosdevices to resolve path
path = os.path.realpath(
os.path.join(winedevices, path.replace("\\", "/"))
)
out_text += "\t" + path + suffix + "\n"
return out_text
def main():
parser = argparse.ArgumentParser(
description="""Transform a .d file from Wine paths to normal paths"""
)
parser.add_argument(
"d_file",
help="""Dependency file in""",
)
parser.add_argument(
"d_file_out",
help="""Dependency file out""",
)
args = parser.parse_args()
output = import_d_file(args.d_file)
with open(args.d_file_out, "w", encoding="UTF-8") as f:
f.write(output)
if __name__ == "__main__":
main()

76
tools/upload_progress.py Normal file
View File

@ -0,0 +1,76 @@
#!/usr/bin/env python3
###
# Uploads progress information to https://github.com/decompals/frogress.
#
# Usage:
# python3 tools/upload_progress.py -b https://progress.decomp.club/ -p [project] -v [version] build/[version]/progress.json
#
# If changes are made, please submit a PR to
# https://github.com/encounter/dtk-template
###
import argparse
import json
import os
import requests
import subprocess
import sys
def get_git_commit_timestamp() -> int:
return int(
subprocess.check_output(["git", "show", "-s", "--format=%ct"])
.decode("ascii")
.rstrip()
)
def get_git_commit_sha() -> str:
return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip()
def generate_url(args: argparse.Namespace) -> str:
url_components = [args.base_url.rstrip("/"), "data"]
for arg in [args.project, args.version]:
if arg != "":
url_components.append(arg)
return str.join("/", url_components) + "/"
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Upload progress information.")
parser.add_argument("-b", "--base_url", help="API base URL", required=True)
parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)")
parser.add_argument("-p", "--project", help="Project slug", required=True)
parser.add_argument("-v", "--version", help="Version slug", required=True)
parser.add_argument("input", help="Progress JSON input")
args = parser.parse_args()
api_key = args.api_key or os.environ.get("PROGRESS_API_KEY")
if not api_key:
raise "API key required"
url = generate_url(args)
entries = []
with open(args.input, "r") as f:
data = json.load(f)
entries.append(
{
"timestamp": get_git_commit_timestamp(),
"git_hash": get_git_commit_sha(),
"categories": data,
}
)
print("Publishing entry to", url)
json.dump(entries[0], sys.stdout, indent=4)
print()
r = requests.post(url, json={
"api_key": api_key,
"entries": entries,
})
r.raise_for_status()
print("Done!")