From 6af151fd3bab46576d93925cfa65d5726061be12 Mon Sep 17 00:00:00 2001 From: Luke Street Date: Fri, 25 Nov 2022 23:36:25 -0500 Subject: [PATCH] ci: Auto-upload progress information Former-commit-id: 37d107694c23720f49e552ae9a4a82eff12c2673 --- .github/workflows/build.yml | 9 +++++- configure.py | 2 +- tools/__init__.py | 0 tools/calcprogress.py | 33 +++++++++++++++----- tools/upload-progress.py | 60 +++++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 tools/__init__.py create mode 100644 tools/upload-progress.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c535fdaa..d523211b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get -y install build-essential ninja-build python3 gcc-multilib g++-multilib libc6:i386 + sudo apt-get -y install build-essential ninja-build python3 python3-requests gcc-multilib g++-multilib libc6:i386 curl -L https://cdn.discordapp.com/attachments/727918646525165659/917185027656286218/GC_WII_COMPILERS.zip \ | bsdtar -xvf- -C tools --exclude Wii mv tools/GC/* tools/mwcc_compiler/ @@ -39,6 +39,13 @@ jobs: run: | python3 configure.py --map --version ${{matrix.version}} --wine ./tools/WiBo/build/wibo ninja + - name: Upload progress + continue-on-error: true + env: + PROGRESS_API_KEY: ${{secrets.PROGRESS_API_KEY}} + run: | + python3 tools/upload-progress.py -b https://progress.deco.mp/ -p prime -v ${{matrix.version}} \ + build/mp1.${{matrix.version}}/main.dol.progress - name: Upload map uses: actions/upload-artifact@v2 with: diff --git a/configure.py b/configure.py index e1379151..cb26dc9a 100755 --- a/configure.py +++ b/configure.py @@ -1308,7 +1308,7 @@ if args.check: # Progress script ### if args.map: - n.rule(name="progress", command=ALLOW_CHAIN+"$python tools/calcprogress.py $in && touch $out", + n.rule(name="progress", command=ALLOW_CHAIN+"$python tools/calcprogress.py $in -o $out", description="PROGRESS $in") n.build("$builddir/main.dol.progress", "progress", ["$builddir/main.dol", "$builddir/MetroidPrime.MAP"]) diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/calcprogress.py b/tools/calcprogress.py index 0625820b..78019d59 100644 --- a/tools/calcprogress.py +++ b/tools/calcprogress.py @@ -24,6 +24,8 @@ import sys import struct import re import math +import argparse +import json ############################################### # # @@ -31,9 +33,6 @@ import math # # ############################################### -DOL_PATH = sys.argv[1] -MAP_PATH = sys.argv[2] - MEM1_HI = 0x81200000 MEM1_LO = 0x80004000 @@ -83,6 +82,12 @@ dataItem = "missiles" # data flavor item ############################################### if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Calculate progress.") + parser.add_argument("dol", help="Path to DOL") + parser.add_argument("map", help="Path to map") + parser.add_argument("-o", "--output", help="JSON output file") + args = parser.parse_args() + # HACK: Check asm or src in obj_file.mk # to avoid counting .comm/.lcomm as decompiled asm_objs = [] @@ -92,11 +97,11 @@ if __name__ == "__main__": asm_objs.append(line.strip().rsplit('/', 1)[-1].rstrip('\\')) # Sum up DOL section sizes - dol_handle = open(DOL_PATH, "rb") + dol_handle = open(args.dol, "rb") # Seek to virtual addresses dol_handle.seek(0x48) - + # Read virtual addresses text_starts = list() for i in range(TEXT_SECTION_COUNT): @@ -134,7 +139,7 @@ if __name__ == "__main__": dol_code_size += i # Open map file - mapfile = open(MAP_PATH, "r") + mapfile = open(args.map, "r") symbols = mapfile.readlines() decomp_code_size = 0 @@ -145,7 +150,7 @@ if __name__ == "__main__": first_section = 0 while (symbols[first_section].startswith(".") == False and "section layout" not in symbols[first_section]): first_section += 1 assert(first_section < len(symbols)), "Map file contains no sections!!!" - + cur_object = None cur_size = 0 j = 0 @@ -197,7 +202,7 @@ if __name__ == "__main__": dataCompletionPcnt = (decomp_data_size / dol_data_size) # data completion percent bytesPerCodeItem = dol_code_size / codeFrac # bytes per code item bytesPerDataItem = dol_data_size / dataFrac # bytes per data item - + codeCount = math.floor(decomp_code_size / bytesPerCodeItem) dataCount = math.floor(decomp_data_size / bytesPerDataItem) @@ -205,3 +210,15 @@ if __name__ == "__main__": print(f"\tCode sections: {decomp_code_size} / {dol_code_size}\tbytes in src ({codeCompletionPcnt:%})") print(f"\tData sections: {decomp_data_size} / {dol_data_size}\tbytes in src ({dataCompletionPcnt:%})") print("\nYou have {} out of {} {} and collected {} out of {} {}.".format(codeCount, codeFrac, codeItem, dataCount, dataFrac, dataItem)) + + if args.output: + data = { + "dol": { + "code": decomp_code_size, + "code/total": dol_code_size, + "data": decomp_data_size, + "data/total": dol_data_size, + } + } + with open(args.output, "w") as f: + json.dump(data, f) diff --git a/tools/upload-progress.py b/tools/upload-progress.py new file mode 100644 index 00000000..6665bbc2 --- /dev/null +++ b/tools/upload-progress.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import subprocess +from pprint import pprint + +import requests + + +def get_git_commit_timestamp() -> int: + return int(subprocess.check_output(['git', 'show', '-s', '--format=%ct']).decode('ascii').rstrip()) + + +def get_git_commit_sha() -> str: + return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip() + + +def generate_url(args: argparse.Namespace) -> str: + url_components = [args.base_url.rstrip('/'), 'data'] + + for arg in [args.project, args.version]: + if arg != "": + url_components.append(arg) + + return str.join('/', url_components) + '/' + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Upload progress information.") + parser.add_argument("-b", "--base_url", help="API base URL", required=True) + parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)") + parser.add_argument("-p", "--project", help="Project slug", required=True) + parser.add_argument("-v", "--version", help="Version slug", required=True) + parser.add_argument("input", help="Progress JSON input") + + args = parser.parse_args() + api_key = args.api_key or os.environ.get("PROGRESS_API_KEY") + if not api_key: + raise "API key required" + url = generate_url(args) + + entries = [] + with open(args.input, "r") as f: + data = json.load(f) + entries.append({ + "timestamp": get_git_commit_timestamp(), + "git_hash": get_git_commit_sha(), + "categories": data, + }) + + print("Publishing entries to", url) + pprint(entries) + data = { + "api_key": args.api_key, + "entries": entries, + } + r = requests.post(url, json=data) + r.raise_for_status() + print("Done!")