diff --git a/.gitattributes b/.gitattributes index 0bfeaccb..9545db5f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,6 @@ *.bat text eol=crlf *.sh text eol=lf *.sha1 text eol=lf + +# DTK keeps these files with LF +config/**/*.txt text eol=lf diff --git a/.github/workflows/build.yml.disable b/.github/workflows/build.yml similarity index 60% rename from .github/workflows/build.yml.disable rename to .github/workflows/build.yml index 44ff42ff..4403ad93 100644 --- a/.github/workflows/build.yml.disable +++ b/.github/workflows/build.yml @@ -11,26 +11,28 @@ jobs: strategy: fail-fast: false matrix: - version: [0, 1, kor] + version: [GM8E01_00] # GM8E01_01, GM8E01_48 steps: - name: Checkout uses: actions/checkout@v3 - name: Git config run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Prepare + run: cp -R /orig . - name: Build run: | - python configure.py --map --version ${{matrix.version}} --compilers /compilers/GC + python configure.py --map --version ${{matrix.version}} --compilers /compilers ninja - name: Upload progress - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && matrix.version == 'GM8E01_00' continue-on-error: true env: PROGRESS_API_KEY: ${{secrets.PROGRESS_API_KEY}} run: | - python tools/upload-progress.py -b https://progress.deco.mp/ -p prime -v ${{matrix.version}} \ - build/mp1.${{matrix.version}}/main.dol.progress + python tools/upload_progress.py -b https://progress.decomp.club/ -p prime -v ${{matrix.version}} \ + build/${{matrix.version}}/progress.json - name: Upload map uses: actions/upload-artifact@v3 with: - name: MetroidPrime-${{matrix.version}}.MAP - path: build/*/MetroidPrime.MAP + name: ${{matrix.version}}_maps + path: build/${{matrix.version}}/**/*.MAP diff --git a/.gitignore b/.gitignore index af8b8a87..51ac6183 100644 --- a/.gitignore +++ b/.gitignore @@ -8,18 +8,13 @@ ctx.c tools/elf2dol tools/elf2rel tools/metroidbuildinfo -tools/mwcc_compiler/* -!tools/mwcc_compiler/.gitkeep +tools/mwcc_compiler *.bat -include/lmgr326b.dll .idea/ versions/ build.ninja .ninja_deps .ninja_log -dwarfD/ objdiff.json orig/*/* !orig/*/.gitkeep -tools/mwcc_compiler/* -!tools/mwcc_compiler/.gitkeep diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b895ab0..b5b4e952 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,13 +18,7 @@ Visual Studio Code is recommended. [objdiff](https://github.com/encounter/objdiff) will be your primary diffing tool. You can fetch a binary from the latest GitHub Actions build, or build from source with `cargo run --release`. -objdiff configuration: -- Project dir: `prime` -- Target build dir: `prime/build/mp1.0/asm` -- Base build dir: `prime/build/mp1.0/src` -- Obj: Whatever .o you're currently working on (can select from asm or src build dirs) -- [x] Build target -- [x] Reverse function order (deferred) +Set the project directory to the repository root, and all settings will be loaded automatically. (Assuming you ran `python configure.py` and `objdiff.json` was generated.) objdiff will automatically rebuild and reload the object on source changes, so you can quickly iterate on functions. diff --git a/README.md b/README.md index 85a7e7d9..e4b8ab89 100644 --- a/README.md +++ b/README.md @@ -3,18 +3,20 @@ Metroid Prime [![Build Status]][actions] ![Code Progress] ![Data Progress] [Build Status]: https://github.com/PrimeDecomp/prime/actions/workflows/build.yml/badge.svg [actions]: https://github.com/PrimeDecomp/prime/actions/workflows/build.yml -[Code Progress]: https://img.shields.io/endpoint?label=Code&url=https%3A%2F%2Fprogress.deco.mp%2Fdata%2Fprime%2F0%2Fdol%2F%3Fmode%3Dshield%26measure%3Dcode -[Data Progress]: https://img.shields.io/endpoint?label=Data&url=https%3A%2F%2Fprogress.deco.mp%2Fdata%2Fprime%2F0%2Fdol%2F%3Fmode%3Dshield%26measure%3Ddata +[Code Progress]: https://img.shields.io/endpoint?label=Code&url=https%3A%2F%2Fprogress.deco.mp%2Fdata%2Fprime%2FGM8E01_00%2Fdol%2F%3Fmode%3Dshield%26measure%3Dcode +[Data Progress]: https://img.shields.io/endpoint?label=Data&url=https%3A%2F%2Fprogress.deco.mp%2Fdata%2Fprime%2FGM8E01_00%2Fdol%2F%3Fmode%3Dshield%26measure%3Ddata -A decompilation of Metroid Prime. +A work-in-progress decompilation of Metroid Prime. -This repository builds the following DOLs: +This repository does **not** contain any game assets or assembly whatsoever. An existing copy of the game is required. -``` -949c5ed7368aef547e0b0db1c3678f466e2afbff build/mp1.0/main.dol (USA 0-00) -860141f9671fc141ce8f55448643f713bc64b349 build/mp1.1/main.dol (USA 0-01) -52316d2a71c0d18c84f054fd6f1e58bdd7bf0ded build/mp1.kor/main.dol (KOR) -``` +The following game versions are supported: + +- `GM8E01_00` (USA v1.088) + If you'd like to contribute, see [CONTRIBUTING.md](CONTRIBUTING.md). @@ -23,14 +25,14 @@ Dependencies Windows: -------- -- Install [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`. -- Install [devkitPro](https://github.com/devkitPro/installer/releases/latest) with GameCube development package. -- Open `C:\devkitPro\msys2\msys2.exe` -- Install GameCube development packages: - ``` - pacman -Sy --noconfirm --needed msys2-keyring - pacman -Su --noconfirm --needed gcc git gamecube-dev - ```` + +On Windows, it's **highly recommended** to use native tooling. WSL or msys2 are **not** required. +When running under WSL, [objdiff](#diffing) is unable to get filesystem notifications for automatic rebuilds. + +- Install [Python](https://www.python.org/downloads/) and add it to `%PATH%`. + - Also available from the [Windows Store](https://apps.microsoft.com/store/detail/python-311/9NRWMJP3717K). +- Download [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`. + - Quick install via pip: `pip install ninja` macOS: ------ @@ -42,39 +44,47 @@ macOS: ``` brew install --cask --no-quarantine gcenx/wine/wine-crossover ``` -- Install [devkitPro](https://github.com/devkitPro/pacman/releases/latest). -- Install GameCube development packages: - ``` - sudo dkp-pacman -Syu --noconfirm --needed gamecube-dev - ``` + +After OS upgrades, if macOS complains about `Wine Crossover.app` being unverified, you can unquarantine it using: +```sh +sudo xattr -rd com.apple.quarantine '/Applications/Wine Crossover.app' +``` Linux: ------ - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages). -- Install wine from your package manager. - - Faster alternative: [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper. - Ensure the binary is in `PATH`. -- Install [devkitPro](https://devkitpro.org/wiki/devkitPro_pacman). -- Install GameCube development packages: - ``` - sudo dkp-pacman -Syu --noconfirm --needed gamecube-dev - ``` +- For non-x86(_64) platforms: Install wine from your package manager. + - For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used. Building ======== -- Checkout the repository: +- Clone the repository: ``` git clone https://github.com/PrimeDecomp/prime.git ``` -- Download [GC_WII_COMPILERS.zip](https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip) -- Extract the _contents_ of the `GC` directory to `tools/mwcc_compiler`. - - Resulting structure should be (for example) `tools/mwcc_compiler/1.3.2/mwcceppc.exe` +- Using [Dolphin Emulator](https://dolphin-emu.org/), extract your game to `orig/GM8E01_00` (or the appropriate version). +![](assets/dolphin-extract.png) + - To save space, the only necessary files are the following. Any others can be deleted. + - `sys/main.dol` + - `files/NESemuP.rel` - Configure: ``` python configure.py ``` + To use a version other than `GM8E01_00` (USA), specify `--version GM8E01_01` or similar. - Build: ``` ninja ``` + +Diffing +======= + +Once the initial build succeeds, an `objdiff.json` should exist in the project root. + +Download the latest release from [encounter/objdiff](https://github.com/encounter/objdiff). Under project settings, set `Project directory`. The configuration should be loaded automatically. + +Select an object from the left sidebar to begin diffing. Changes to the project will rebuild automatically: changes to source files, headers, `configure.py`, `splits.txt` or `symbols.txt`. + +![](assets/objdiff.png) diff --git a/assets/dolphin-extract.png b/assets/dolphin-extract.png new file mode 100644 index 00000000..10d36d5c Binary files /dev/null and b/assets/dolphin-extract.png differ diff --git a/assets/objdiff.png b/assets/objdiff.png new file mode 100644 index 00000000..0fa93bcc Binary files /dev/null and b/assets/objdiff.png differ diff --git a/configure.py b/configure.py index 405733ba..6c9930e2 100755 --- a/configure.py +++ b/configure.py @@ -29,7 +29,7 @@ DEFAULT_VERSION = 0 VERSIONS = [ "GM8E01_00", # mp-v1.088 NTSC-U # "GM8E01_01", # mp-v1.093 NTSC-U - # "GM8E01_30", # mp-v1.097 NTSC-K + # "GM8E01_48", # mp-v1.097 NTSC-K # "GM8P01_00", # mp-v1.110 PAL # "GM8J01_00", # mp-v1.111 NTSC-J # "GM8E01_02", # mp-v1.111 NTSC-U @@ -125,10 +125,10 @@ if not is_windows(): config.wrapper = args.wrapper # Tool versions -config.compilers_tag = "1" -config.dtk_tag = "v0.5.6" +config.compilers_tag = "20230715" +config.dtk_tag = "v0.5.7" config.sjiswrap_tag = "v1.1.1" -config.wibo_tag = "0.6.3" +config.wibo_tag = "0.6.4" # Project config.config_path = Path("config") / config.version / "config.yml" @@ -137,6 +137,7 @@ config.ldflags = [ "-fp hardware", "-nodefaults", ] +config.progress_all = False # Base flags, common to most GC/Wii games. # Generally leave untouched, with overrides added below. @@ -155,7 +156,8 @@ cflags_base = [ "-nosyspath", "-i include", "-i libc", - "-DPRIME1" "-DVERSION={version_num}", + "-DPRIME1", + f"-DVERSION={version_num}", "-DNONMATCHING=0", ] diff --git a/diff_settings.py b/diff_settings.py deleted file mode 100644 index 8b64e653..00000000 --- a/diff_settings.py +++ /dev/null @@ -1,9 +0,0 @@ -def apply(config, args): - config["arch"] = "ppc" - config["objdump_executable"] = "/opt/devkitpro/devkitPPC/bin/powerpc-eabi-objdump" - config["objdump_flags"] = ["-M", "gekko"] - config["source_directories"] = ["."] - config["make_flags"] = [ - "VERBOSE=1", - args.objfile.replace("src/", "asm/") # also build asm obj - ] diff --git a/dtk_version b/dtk_version deleted file mode 100644 index 576b7771..00000000 --- a/dtk_version +++ /dev/null @@ -1 +0,0 @@ -v0.2.3 diff --git a/orig/GM8E01_30/.gitkeep b/orig/GM8E01_48/.gitkeep similarity index 100% rename from orig/GM8E01_30/.gitkeep rename to orig/GM8E01_48/.gitkeep diff --git a/progress.py b/progress.py deleted file mode 100644 index 3d334aa8..00000000 --- a/progress.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python3 - -################################################################################ -# Description # -################################################################################ -# calcprogress: Used to calculate the progress of the Metroid Prime decomp. # -# Prints to stdout for now, but eventually will have some form of storage, # -# i.e. CSV, so that it can be used for a webpage display. # -# # -# Usage: No arguments needed # -################################################################################ - - - - -############################################### -# # -# Imports # -# # -############################################### - -import os -import sys -import struct -import re -import math -import argparse -import json - -############################################### -# # -# Constants # -# # -############################################### - -MEM1_HI = 0x81200000 -MEM1_LO = 0x80004000 - -MW_WII_SYMBOL_REGEX = r"^\s*"\ -r"(?P\w{8})\s+"\ -r"(?P\w{6})\s+"\ -r"(?P\w{8})\s+"\ -r"(?P\w{8})\s+"\ -r"(\w{1,2})\s+"\ -r"(?P[0-9A-Za-z_<>$@.*]*)\s*"\ -r"(?P[\S ]*)" - -MW_GC_SYMBOL_REGEX = r"^\s*"\ -r"(?P\w{8})\s+"\ -r"(?P\w{6})\s+"\ -r"(?P\w{8})\s+"\ -r"(\w{1,2})\s+"\ -r"(?P[0-9A-Za-z_<>$@.*]*)\s*"\ -r"(?P[\S ]*)" - -REGEX_TO_USE = MW_GC_SYMBOL_REGEX - -TEXT_SECTIONS = ["init", "text"] -DATA_SECTIONS = [ -"rodata", "data", "bss", "sdata", "sbss", "sdata2", "sbss2", -"ctors", "_ctors", "dtors", "ctors$99", "_ctors$99", "ctors$00", "dtors$99", -"extab", "extabindex", "extab_", "extabindex_", "_extab", "_exidx" -] - -# DOL info -TEXT_SECTION_COUNT = 7 -DATA_SECTION_COUNT = 11 - -SECTION_TEXT = 0 -SECTION_DATA = 1 - -# Progress flavor -codeFrac = 1499 # total code "item" amount -dataFrac = 250 # total data "item" amount -codeItem = "energy" # code flavor item -dataItem = "missiles" # data flavor item - -############################################### -# # -# Entrypoint # -# # -############################################### - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Calculate progress.") - parser.add_argument("dol", help="Path to DOL") - parser.add_argument("map", help="Path to map") - parser.add_argument("-o", "--output", help="JSON output file") - args = parser.parse_args() - - # Sum up DOL section sizes - dol_handle = open(args.dol, "rb") - - # Seek to virtual addresses - dol_handle.seek(0x48) - - # Read virtual addresses - text_starts = list() - for i in range(TEXT_SECTION_COUNT): - text_starts.append(int.from_bytes(dol_handle.read(4), byteorder='big')) - data_starts = list() - for i in range(DATA_SECTION_COUNT): - data_starts.append(int.from_bytes(dol_handle.read(4), byteorder='big')) - - # Read lengths - text_sizes = list() - for i in range(TEXT_SECTION_COUNT): - text_sizes.append(int.from_bytes(dol_handle.read(4), byteorder='big')) - data_sizes = list() - for i in range(DATA_SECTION_COUNT): - data_sizes.append(int.from_bytes(dol_handle.read(4), byteorder='big')) - - - - # BSS address + length - bss_start = int.from_bytes(dol_handle.read(4), byteorder='big') - bss_size = int.from_bytes(dol_handle.read(4), byteorder='big') - bss_end = bss_start + bss_size - - - dol_code_size = 0 - dol_data_size = 0 - for i in range(DATA_SECTION_COUNT): - # Ignore sections inside BSS - if (data_starts[i] >= bss_start) and (data_starts[i] + data_sizes[i] <= bss_end): continue - dol_data_size += data_sizes[i] - - dol_data_size += bss_size - - for i in text_sizes: - dol_code_size += i - - # Open map file - mapfile = open(args.map, "r") - symbols = mapfile.readlines() - - decomp_code_size = 0 - decomp_data_size = 0 - section_type = None - - # Find first section - first_section = 0 - while (symbols[first_section].startswith(".") == False and "section layout" not in symbols[first_section]): first_section += 1 - assert(first_section < len(symbols)), "Map file contains no sections!!!" - - cur_object = None - cur_size = 0 - j = 0 - for i in range(first_section, len(symbols)): - # New section - if (symbols[i].startswith(".") == True or "section layout" in symbols[i]): - # Grab section name (i.e. ".init section layout" -> "init") - sectionName = re.search(r"\.*(?P\w+)\s", symbols[i]).group("Name") - # Determine type of section - section_type = SECTION_DATA if (sectionName in DATA_SECTIONS) else SECTION_TEXT - # Parse symbols until we hit the next section declaration - else: - if "UNUSED" in symbols[i]: continue - if "entry of" in symbols[i]: - if j == i - 1: - if section_type == SECTION_TEXT: - decomp_code_size -= cur_size - else: - decomp_data_size -= cur_size - cur_size = 0 - #print(f"Line* {j}: {symbols[j]}") - #print(f"Line {i}: {symbols[i]}") - continue - assert(section_type != None), f"Symbol found outside of a section!!!\n{symbols[i]}" - match_obj = re.search(REGEX_TO_USE, symbols[i]) - # Should be a symbol in ASM (so we discard it) - if (match_obj == None): - #print(f"Line {i}: {symbols[i]}") - continue - # Has the object file changed? - last_object = cur_object - cur_object = match_obj.group("Object").strip() - if last_object != cur_object or cur_object.endswith(" (asm)"): continue - # Is the symbol a file-wide section? - symb = match_obj.group("Symbol") - if (symb.startswith("*fill*")) or (symb.startswith(".") and symb[1:] in TEXT_SECTIONS or symb[1:] in DATA_SECTIONS): continue - # For sections that don't start with "." - if (symb in DATA_SECTIONS): continue - # If not, we accumulate the file size - cur_size = int(match_obj.group("Size"), 16) - j = i - if (section_type == SECTION_TEXT): - decomp_code_size += cur_size - else: - decomp_data_size += cur_size - - # Calculate percentages - codeCompletionPcnt = (decomp_code_size / dol_code_size) # code completion percent - dataCompletionPcnt = (decomp_data_size / dol_data_size) # data completion percent - bytesPerCodeItem = dol_code_size / codeFrac # bytes per code item - bytesPerDataItem = dol_data_size / dataFrac # bytes per data item - - codeCount = math.floor(decomp_code_size / bytesPerCodeItem) - dataCount = math.floor(decomp_data_size / bytesPerDataItem) - - print("Progress:") - print(f"\tCode sections: {decomp_code_size} / {dol_code_size}\tbytes in src ({codeCompletionPcnt:%})") - print(f"\tData sections: {decomp_data_size} / {dol_data_size}\tbytes in src ({dataCompletionPcnt:%})") - print("\nYou have {} out of {} {} and collected {} out of {} {}.".format(codeCount, codeFrac, codeItem, dataCount, dataFrac, dataItem)) - - if args.output: - data = { - "dol": { - "code": decomp_code_size, - "code/total": dol_code_size, - "data": decomp_data_size, - "data/total": dol_data_size, - } - } - with open(args.output, "w") as f: - json.dump(data, f) diff --git a/tools/download_tool.py b/tools/download_tool.py index 27099669..fef42d6e 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -16,7 +16,6 @@ import os import platform import shutil import stat -import sys import urllib.request import zipfile @@ -50,10 +49,7 @@ def wibo_url(tag): def compilers_url(tag): - if tag == "1": - return "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip" - else: - sys.exit("Unknown compilers tag %s" % tag) + return f"https://files.decomp.dev/compilers_{tag}.zip" TOOLS = { @@ -90,4 +86,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main()