ci: Auto-upload progress information

.github/workflows/build.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         sudo dpkg --add-architecture i386
         sudo apt-get update
-        sudo apt-get -y install build-essential ninja-build python3 gcc-multilib g++-multilib libc6:i386
+        sudo apt-get -y install build-essential ninja-build python3 python3-requests gcc-multilib g++-multilib libc6:i386
         curl -L https://cdn.discordapp.com/attachments/727918646525165659/917185027656286218/GC_WII_COMPILERS.zip \
           | bsdtar -xvf- -C tools --exclude Wii
         mv tools/GC/* tools/mwcc_compiler/
@@ -39,6 +39,13 @@ jobs:
       run: |
         python3 configure.py --map --version ${{matrix.version}} --wine ./tools/WiBo/build/wibo
         ninja
+    - name: Upload progress
+      continue-on-error: true
+      env:
+        PROGRESS_API_KEY: ${{secrets.PROGRESS_API_KEY}}
+      run: |
+        python3 tools/upload-progress.py -b https://progress.deco.mp/ -p prime -v ${{matrix.version}} \
+          build/mp1.${{matrix.version}}/main.dol.progress
     - name: Upload map
       uses: actions/upload-artifact@v2
       with:

configure.py

@@ -1308,7 +1308,7 @@ if args.check:
 # Progress script
 ###
 if args.map:
-    n.rule(name="progress", command=ALLOW_CHAIN+"$python tools/calcprogress.py $in && touch $out",
+    n.rule(name="progress", command=ALLOW_CHAIN+"$python tools/calcprogress.py $in -o $out",
            description="PROGRESS $in")
     n.build("$builddir/main.dol.progress", "progress",
             ["$builddir/main.dol", "$builddir/MetroidPrime.MAP"])

tools/calcprogress.py

@@ -24,6 +24,8 @@ import sys
 import struct
 import re
 import math
+import argparse
+import json
 
 ###############################################
 #                                             #
@@ -31,9 +33,6 @@ import math
 #                                             #
 ###############################################
 
-DOL_PATH = sys.argv[1]
-MAP_PATH = sys.argv[2]
-
 MEM1_HI = 0x81200000
 MEM1_LO = 0x80004000
 
@@ -83,6 +82,12 @@ dataItem = "missiles"   # data flavor item
 ###############################################
 
 if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Calculate progress.")
+    parser.add_argument("dol", help="Path to DOL")
+    parser.add_argument("map", help="Path to map")
+    parser.add_argument("-o", "--output", help="JSON output file")
+    args = parser.parse_args()
+
     # HACK: Check asm or src in obj_file.mk
     # to avoid counting .comm/.lcomm as decompiled
     asm_objs = []
@@ -92,11 +97,11 @@ if __name__ == "__main__":
                 asm_objs.append(line.strip().rsplit('/', 1)[-1].rstrip('\\'))
 
     # Sum up DOL section sizes
-    dol_handle = open(DOL_PATH, "rb")
+    dol_handle = open(args.dol, "rb")
 
     # Seek to virtual addresses
     dol_handle.seek(0x48)
-    
+
     # Read virtual addresses
     text_starts = list()
     for i in range(TEXT_SECTION_COUNT):
@@ -134,7 +139,7 @@ if __name__ == "__main__":
         dol_code_size += i
 
     # Open map file
-    mapfile = open(MAP_PATH, "r")
+    mapfile = open(args.map, "r")
     symbols = mapfile.readlines()
 
     decomp_code_size = 0
@@ -145,7 +150,7 @@ if __name__ == "__main__":
     first_section = 0
     while (symbols[first_section].startswith(".") == False and "section layout" not in symbols[first_section]): first_section += 1
     assert(first_section < len(symbols)), "Map file contains no sections!!!"
-    
+
     cur_object = None
     cur_size = 0
     j = 0
@@ -197,7 +202,7 @@ if __name__ == "__main__":
     dataCompletionPcnt = (decomp_data_size / dol_data_size) # data completion percent
     bytesPerCodeItem = dol_code_size / codeFrac # bytes per code item
     bytesPerDataItem = dol_data_size / dataFrac # bytes per data item
-    
+
     codeCount = math.floor(decomp_code_size / bytesPerCodeItem)
     dataCount = math.floor(decomp_data_size / bytesPerDataItem)
 
@@ -205,3 +210,15 @@ if __name__ == "__main__":
     print(f"\tCode sections: {decomp_code_size} / {dol_code_size}\tbytes in src ({codeCompletionPcnt:%})")
     print(f"\tData sections: {decomp_data_size} / {dol_data_size}\tbytes in src ({dataCompletionPcnt:%})")
     print("\nYou have {} out of {} {} and collected {} out of {} {}.".format(codeCount, codeFrac, codeItem, dataCount, dataFrac, dataItem))
+
+    if args.output:
+        data = {
+            "dol": {
+                "code": decomp_code_size,
+                "code/total": dol_code_size,
+                "data": decomp_data_size,
+                "data/total": dol_data_size,
+            }
+        }
+        with open(args.output, "w") as f:
+            json.dump(data, f)

tools/upload-progress.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import subprocess
+from pprint import pprint
+
+import requests
+
+
+def get_git_commit_timestamp() -> int:
+    return int(subprocess.check_output(['git', 'show', '-s', '--format=%ct']).decode('ascii').rstrip())
+
+
+def get_git_commit_sha() -> str:
+    return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip()
+
+
+def generate_url(args: argparse.Namespace) -> str:
+    url_components = [args.base_url.rstrip('/'), 'data']
+
+    for arg in [args.project, args.version]:
+        if arg != "":
+            url_components.append(arg)
+
+    return str.join('/', url_components) + '/'
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="Upload progress information.")
+    parser.add_argument("-b", "--base_url", help="API base URL", required=True)
+    parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)")
+    parser.add_argument("-p", "--project", help="Project slug", required=True)
+    parser.add_argument("-v", "--version", help="Version slug", required=True)
+    parser.add_argument("input", help="Progress JSON input")
+
+    args = parser.parse_args()
+    api_key = args.api_key or os.environ.get("PROGRESS_API_KEY")
+    if not api_key:
+        raise "API key required"
+    url = generate_url(args)
+
+    entries = []
+    with open(args.input, "r") as f:
+        data = json.load(f)
+        entries.append({
+            "timestamp": get_git_commit_timestamp(),
+            "git_hash": get_git_commit_sha(),
+            "categories": data,
+        })
+
+    print("Publishing entries to", url)
+    pprint(entries)
+    data = {
+        "api_key": args.api_key,
+        "entries": entries,
+    }
+    r = requests.post(url, json=data)
+    r.raise_for_status()
+    print("Done!")