Update progress script with new assets categories and update csv output format (#510)

* Reorganize csvs and progress.py

* Put stuff in the correct folders

* Reduce lots of repeated code

* Change csv output format

* Filter out automaticaly named variables in "Matching" progress calculation for assets

* Address Elliptic's review

* Don't count handwritten files in progress and add a way to fix files detected in the wrong section

* Add missing "total"

* More fixing

* Add two missing columns

* Update paths in Jenkinsfile

* Update progress shield in readme

* Update progress link

Jenkinsfile

@@ -52,9 +52,9 @@ pipeline {
             }
             steps {
                 sh 'mkdir reports'
-                sh 'python3 ./tools/progress.py csv >> reports/progress_mm.us.rev1.csv'
-                sh 'python3 ./tools/progress.py csv -m >> reports/progress_matching_mm.us.rev1.csv'
-                sh 'python3 ./tools/progress.py shield-json > reports/progress_shield_mm.us.rev1.json'
+                sh 'python3 ./tools/progress.py csv >> reports/progress-mm-nonmatching.csv'
+                sh 'python3 ./tools/progress.py csv -m >> reports/progress-mm-matching.csv'
+                sh 'python3 ./tools/progress.py shield-json > reports/progress-mm-shield.json'
                 stash includes: 'reports/*', name: 'reports'
             }
         }
@@ -67,9 +67,9 @@ pipeline {
             }
             steps {
                 unstash 'reports'
-                sh 'cat reports/progress_mm.us.rev1.csv >> /var/www/html/reports/progress_mm.us.rev1.csv'
-                sh 'cat reports/progress_matching_mm.us.rev1.csv >> /var/www/html/reports/progress_matching_mm.us.rev1.csv'
-                sh 'cat reports/progress_shield_mm.us.rev1.json > /var/www/html/reports/progress_shield_mm.us.rev1.json'
+                sh 'cat reports/progress-mm-nonmatching.csv >> /var/www/zelda64.dev/assets/csv/progress-mm-nonmatching.csv'
+                sh 'cat reports/progress-mm-matching.csv >> /var/www/zelda64.dev/assets/csv/progress-mm-matching.csv'
+                sh 'cat reports/progress-mm-shield.json > /var/www/zelda64.dev/assets/csv/progress-mm-shield.json'
             }
         }
     }

README.md

@@ -5,8 +5,8 @@
 [jenkins]: https://jenkins.deco.mp/job/MM/job/master
 [jenkins-badge]: https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fjenkins.deco.mp%2Fjob%2FMM%2Fjob%2Fmaster
 
-[progress]: https://zelda64.dev/progress.html
-[progress-badge]: https://img.shields.io/endpoint?url=https://zelda64.dev/reports/progress_shield_mm.us.rev1.json
+[progress]: https://zelda64.dev/games/mm
+[progress-badge]: https://img.shields.io/endpoint?url=https://zelda64.dev/reports/progress-mm-shield.json
 
 [contributors]: https://github.com/zeldaret/mm/graphs/contributors
 [contributors-badge]: https://img.shields.io/github/contributors/zeldaret/mm

spec

@@ -312,7 +312,7 @@ beginseg
     name "icon_item_gameover_static"
     compress
     romalign 0x1000
-    include "build/assets/static/icon_item_gameover_static/icon_item_gameover_static.o"
+    include "build/assets/interface/icon_item_gameover_static/icon_item_gameover_static.o"
     number 12
 endseg
 
@@ -8835,7 +8835,7 @@ beginseg
     name "nintendo_rogo_static"
     compress
     romalign 0x1000
-    include "build/assets/static/nintendo_rogo_static/nintendo_rogo_static.o"
+    include "build/assets/misc/nintendo_rogo_static/nintendo_rogo_static.o"
     number 1
 endseg
 
@@ -8879,7 +8879,7 @@ beginseg
     name "daytelop_static"
     compress
     romalign 0x1000
-    include "build/assets/static/daytelop_static/daytelop_static.o"
+    include "build/assets/misc/daytelop_static/daytelop_static.o"
     number 9
 endseg
 
@@ -8887,7 +8887,7 @@ beginseg
     name "ger_daytelop_static"
     compress
     romalign 0x1000
-    include "build/assets/static/ger_daytelop_static/ger_daytelop_static.o"
+    include "build/assets/misc/ger_daytelop_static/ger_daytelop_static.o"
     number 9
 endseg
 
@@ -8895,7 +8895,7 @@ beginseg
     name "fra_daytelop_static"
     compress
     romalign 0x1000
-    include "build/assets/static/fra_daytelop_static/fra_daytelop_static.o"
+    include "build/assets/misc/fra_daytelop_static/fra_daytelop_static.o"
     number 9
 endseg
 
@@ -8903,7 +8903,7 @@ beginseg
     name "esp_daytelop_static"
     compress
     romalign 0x1000
-    include "build/assets/static/esp_daytelop_static/esp_daytelop_static.o"
+    include "build/assets/misc/esp_daytelop_static/esp_daytelop_static.o"
     number 9
 endseg
 

src/overlays/gamestates/ovl_daytelop/z_daytelop.c

@@ -5,8 +5,8 @@
  */
 
 #include "z_daytelop.h"
-#include "static/daytelop_static/daytelop_static.h"
-#include "static/icon_item_gameover_static/icon_item_gameover_static.h"
+#include "misc/daytelop_static/daytelop_static.h"
+#include "interface/icon_item_gameover_static/icon_item_gameover_static.h"
 
 // unused
 UNK_TYPE D_808158E0[] = {

src/overlays/gamestates/ovl_title/z_title.c

@@ -6,7 +6,7 @@
 
 #include "z_title.h"
 #include "overlays/gamestates/ovl_opening/z_opening.h"
-#include "static/nintendo_rogo_static/nintendo_rogo_static.h"
+#include "misc/nintendo_rogo_static/nintendo_rogo_static.h"
 
 void Title_UpdateCounters(TitleContext* this) {
     if ((this->coverAlpha == 0) && (this->visibleDuration != 0)) {

tools/filelists/mm.us.rev1/all.csv

@@ -1537,3 +1537,16 @@
 1536,KAKUSIANA_room_13
 1537,KAKUSIANA_room_14
 1538,bump_texture_static
+1539,anime_model_1_static
+1540,anime_model_2_static
+1541,anime_model_3_static
+1542,anime_model_4_static
+1543,anime_model_5_static
+1544,anime_model_6_static
+1545,anime_texture_1_static
+1546,anime_texture_2_static
+1547,anime_texture_3_static
+1548,anime_texture_4_static
+1549,anime_texture_5_static
+1550,anime_texture_6_static
+1551,softsprite_matrix_static

tools/filelists/mm.us.rev1/archives.csv

@@ -0,0 +1,7 @@
+15,map_i_static
+16,map_grand_static
+17,item_name_static
+18,map_name_static
+19,icon_item_static_test
+20,icon_item_24_static_test
+22,schedule_dma_static_test

tools/filelists/mm.us.rev1/asm.csv

@@ -2,3 +2,5 @@
 1,boot
 2,dmadata
 31,code
+1135,elf_message_field
+1136,elf_message_ydan

tools/filelists/mm.us.rev1/deleted.csv

@@ -0,0 +1,3 @@
+8,icon_item_static_old
+9,icon_item_24_static_old
+21,schedule_dma_static_old

tools/filelists/mm.us.rev1/interface.csv

@@ -0,0 +1,11 @@
+6,kanji
+10,icon_item_field_static
+11,icon_item_dungeon_static
+12,icon_item_gameover_static
+13,icon_item_jpn_static
+14,icon_item_vtx_static
+23,schedule_static
+25,do_action_static
+26,message_static
+27,message_texture_static
+28,nes_font_static

tools/filelists/mm.us.rev1/misc.csv

@@ -1,6 +1,24 @@
 7,link_animetion
+24,story_static
+1114,scene_texture_01
+1115,scene_texture_02
+1116,scene_texture_03
+1117,scene_texture_04
+1118,scene_texture_05
+1119,scene_texture_06
+1120,scene_texture_07
+1121,scene_texture_08
 1122,nintendo_rogo_static
+1123,title_static
 1124,memerrmsg
 1125,locerrmsg
-1135,elf_message_field
-1136,elf_message_ydan
+1126,parameter_static
+1127,week_static
+1128,daytelop_static
+1129,ger_daytelop_static
+1130,fra_daytelop_static
+1131,esp_daytelop_static
+1132,d2_fine_static
+1133,d2_cloud_static
+1134,d2_fine_pal_static
+1538,bump_texture_static

tools/filelists/mm.us.rev1/segments.csv

@@ -0,0 +1,13 @@
+1539,anime_model_1_static
+1540,anime_model_2_static
+1541,anime_model_3_static
+1542,anime_model_4_static
+1543,anime_model_5_static
+1544,anime_model_6_static
+1545,anime_texture_1_static
+1546,anime_texture_2_static
+1547,anime_texture_3_static
+1548,anime_texture_4_static
+1549,anime_texture_5_static
+1550,anime_texture_6_static
+1551,softsprite_matrix_static

tools/filelists/mm.us.rev1/text.csv

@@ -0,0 +1,2 @@
+29,message_data_static
+30,staff_message_data_static

tools/filelists/mm.us.rev1/texture.csv

@@ -1,43 +0,0 @@
-6,kanji
-8,icon_item_static_old
-9,icon_item_24_static_old
-10,icon_item_field_static
-11,icon_item_dungeon_static
-12,icon_item_gameover_static
-13,icon_item_jpn_static
-14,icon_item_vtx_static
-15,map_i_static
-16,map_grand_static
-17,item_name_static
-18,map_name_static
-19,icon_item_static_test
-20,icon_item_24_static_test
-21,schedule_dma_static_old
-22,schedule_dma_static_test
-23,schedule_static
-24,story_static
-25,do_action_static
-26,message_static
-27,message_texture_static
-28,nes_font_static
-29,message_data_static
-30,staff_message_data_static
-1114,scene_texture_01
-1115,scene_texture_02
-1116,scene_texture_03
-1117,scene_texture_04
-1118,scene_texture_05
-1119,scene_texture_06
-1120,scene_texture_07
-1121,scene_texture_08
-1123,title_static
-1126,parameter_static
-1127,week_static
-1128,daytelop_static
-1129,ger_daytelop_static
-1130,fra_daytelop_static
-1131,esp_daytelop_static
-1132,d2_fine_static
-1133,d2_cloud_static
-1134,d2_fine_pal_static
-1538,bump_texture_static

tools/progress.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-import argparse, csv, git, json, os, re
+import argparse, csv, git, json, os, re, sys
 
 parser = argparse.ArgumentParser()
 
@@ -12,9 +12,16 @@ args = parser.parse_args()
 NON_MATCHING_PATTERN = r'#ifdef\s+NON_MATCHING.*?#pragma\s+GLOBAL_ASM\s*\(\s*"(.*?)"\s*\).*?#endif'
 NOT_ATTEMPTED_PATTERN = r'#pragma\s+GLOBAL_ASM\s*\(\s*"(.*?)"\s*\)'
 
+# This is the format ZAPD uses to autogenerate variable names
+# It should not be used for properly documented variables
+AUTOGENERATED_ASSET_NAME = re.compile(r".+[0-9A-Fa-f]{6}$")
+
 # TODO: consider making this a parameter of this script
 GAME_VERSION = "mm.us.rev1"
 
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
 def GetFunctionsByPattern(pattern, files):
     functions = []
 
@@ -71,6 +78,52 @@ def GetRemovableSize(functions_to_count):
 
     return size
 
+def CalculateMapSizes(mapFileList):
+    for mapFile in mapFileList:
+        accumulatedSize = 0
+
+        if mapFile["section"] != ".data":
+            continue
+        if not mapFile["name"].startswith("build/assets/"):
+            continue
+
+        symbolCount = len(mapFile["symbols"])
+        if symbolCount == 0:
+            continue
+
+        # Calculate size of each symbol
+        for index in range(symbolCount - 1):
+            symbol = mapFile["symbols"][index]
+            nextSymbol = mapFile["symbols"][index+1]
+
+            size = nextSymbol["vram"] - symbol["vram"]
+            accumulatedSize += size
+
+            mapFile["symbols"][index]["size"] = size
+
+        # Calculate size of last symbol of the file
+        symbol = mapFile["symbols"][-1]
+        size = mapFile["size"] - accumulatedSize
+        mapFile["symbols"][-1]["size"] = size
+    return mapFileList
+
+def CalculateNonNamedAssets(mapFileList, assetsTracker):
+    for mapFile in mapFileList:
+        if mapFile["section"] != ".data":
+            continue
+        if not mapFile["name"].startswith("build/assets/"):
+            continue
+
+        assetCat = mapFile["name"].split("/")[2]
+
+        for symbol in mapFile["symbols"]:
+            symbolName = symbol["name"]
+            if AUTOGENERATED_ASSET_NAME.search(symbolName) is not None:
+                if assetCat in assetsTracker:
+                    assetsTracker[assetCat]["removableSize"] += symbol["size"]
+    return assetsTracker
+
+
 map_file = ReadAllLines('build/mm.map')
 
 # Get list of Non-Matchings
@@ -86,12 +139,36 @@ not_attempted_functions = list(set(not_attempted_functions).difference(non_match
 if not args.matching:
     non_matching_functions = []
 
-# Get asset files
-audio_files = GetCsvFilelist(GAME_VERSION, "audio.csv")
-misc_files = GetCsvFilelist(GAME_VERSION, "misc.csv")
-object_files = GetCsvFilelist(GAME_VERSION, "object.csv")
-scene_files = GetCsvFilelist(GAME_VERSION, "scene.csv")
-texture_files = GetCsvFilelist(GAME_VERSION, "texture.csv")
+# The order of this list should not change to prevent breaking the graph of the website
+# New stuff shall be appended at the end of the list
+assetsCategories = [
+    "archives",
+    "audio",
+    "interface",
+    "misc",
+    "objects",
+    "scenes",
+    "text",
+    # "deleted",
+    # "segments",
+]
+assetsTracker = dict()
+
+# Manual fixer for files that would be counted in wrong categories
+# "filename": "correctSection"
+fileSectionFixer = {
+    "osFlash": "code" # Currently in `src/libultra` (would be counted as boot)
+}
+
+for assetCat in assetsCategories:
+    assetsTracker[assetCat] = dict()
+    # Get asset files
+    assetsTracker[assetCat]["files"] = GetCsvFilelist(GAME_VERSION, f"{assetCat}.csv")
+    assetsTracker[assetCat]["currentSize"] = 0
+    assetsTracker[assetCat]["removableSize"] = 0
+    assetsTracker[assetCat]["totalSize"] = 0
+    assetsTracker[assetCat]["percent"] = 0
+
 
 # Initialize all the code values
 src = 0
@@ -104,22 +181,36 @@ asm_code = 0
 asm_boot = 0
 asm_ovl = 0
 asm_libultra = 0
-audio = 0
-misc = 0
-object_ = 0
-scene = 0
-texture = 0
+
+mapFileList = []
 
 for line in map_file:
     line_split =  list(filter(None, line.split(" ")))
 
     if (len(line_split) == 4 and line_split[0].startswith(".")):
         section = line_split[0]
+        obj_vram = int(line_split[1], 16)
         file_size = int(line_split[2], 16)
-        obj_file = line_split[3]
+        obj_file = line_split[3].strip()
+        objFileSplit = obj_file.split("/")
+
+        fileData = {"name": obj_file, "vram": obj_vram, "size": file_size, "section": section, "symbols": []}
+        mapFileList.append(fileData)
 
         if (section == ".text"):
-            if (obj_file.startswith("build/src")):
+            objFileName = objFileSplit[-1].split(".o")[0]
+
+            if objFileName in fileSectionFixer:
+                correctSection = fileSectionFixer[objFileName]
+                if correctSection == "code":
+                    src_code += file_size
+                elif correctSection == "libultra":
+                    src_libultra += file_size
+                elif correctSection == "boot":
+                    src_boot += file_size
+                elif correctSection == "overlays":
+                    src_ovl += file_size
+            elif (obj_file.startswith("build/src")):
                 if (obj_file.startswith("build/src/code")):
                     src_code += file_size
                 elif (obj_file.startswith("build/src/libultra")):
@@ -138,17 +229,29 @@ for line in map_file:
                 elif (obj_file.startswith("build/asm/overlays")):
                     asm_ovl += file_size
 
-        if (section == ".data"):
-            if (obj_file.startswith("build/assets/audio")):
-                audio += file_size
-            elif (obj_file.startswith("build/assets/misc")):
-                misc += file_size
-            elif (obj_file.startswith("build/assets/objects")):
-                object_ += file_size
-            elif (obj_file.startswith("build/assets/scenes")):
-                scene += file_size
-            elif (obj_file.startswith("build/assets/textures")):
-                texture += file_size
+        if section == ".data":
+            if obj_file.startswith("build/assets/"):
+                assetCat = obj_file.split("/")[2]
+                if assetCat in assetsTracker:
+                    assetsTracker[assetCat]["currentSize"] += file_size
+                else:
+                    eprint(f"Found file '{obj_file}' in unknown asset category '{assetCat}'")
+                    eprint("I'll ignore this for now, but please fix it!")
+
+    elif len(line_split) == 2 and line_split[0].startswith("0x00000000"):
+        varVramStr, varName = line_split
+        varVram = int(varVramStr, 16)
+        varName = varName.strip()
+        if varName == "0x0":
+            continue
+        #print(varVram, varName)
+        symbolData = {"name": varName, "vram": varVram, "size": 0}
+        mapFileList[-1]["symbols"].append(symbolData)
+
+mapFileList = CalculateMapSizes(mapFileList)
+
+assetsTracker = CalculateNonNamedAssets(mapFileList, assetsTracker)
+
 
 # Add libultra to boot.
 src_boot += src_libultra
@@ -184,27 +287,22 @@ boot = src_boot - (non_matching_asm_boot + not_attempted_asm_boot)
 ovl = src_ovl - (non_matching_asm_ovl + not_attempted_asm_ovl)
 
 # Total code bucket sizes
-code_size = src_code + asm_code
-boot_size = src_boot + asm_boot
-ovl_size = src_ovl + asm_ovl
-handwritten = 0 # Currently unsure of any handwritten asm in MM
+code_size = src_code # + asm_code
+boot_size = src_boot # + asm_boot
+ovl_size  = src_ovl  # + asm_ovl
+handwritten = asm_code + asm_boot + asm_ovl
+
+# Calculate the total amount of decompilable code
+total = code_size + boot_size + ovl_size
 
 # Calculate size of all assets
-audio_size = 0
-misc_size = 0
-object_size = 0
-scene_size = 0
-texture_size = 0
-for index, f in audio_files:
-    audio_size += os.stat(os.path.join("baserom", f)).st_size
-for index, f in misc_files:
-    misc_size += os.stat(os.path.join("baserom", f)).st_size
-for index, f in object_files:
-    object_size += os.stat(os.path.join("baserom", f)).st_size
-for index, f in scene_files:
-    scene_size += os.stat(os.path.join("baserom", f)).st_size
-for index, f in texture_files:
-    texture_size += os.stat(os.path.join("baserom", f)).st_size
+for assetCat in assetsTracker:
+    for index, f in assetsTracker[assetCat]["files"]:
+        assetsTracker[assetCat]["totalSize"] += os.stat(os.path.join("baserom", f)).st_size
+
+if args.matching:
+    for assetCat in assetsTracker:
+        assetsTracker[assetCat]["currentSize"] -= assetsTracker[assetCat]["removableSize"]
 
 # Calculate asm and src totals
 src = src_code + src_boot + src_ovl
@@ -214,12 +312,9 @@ asm = asm_code + asm_boot + asm_ovl
 src -= non_matching_asm + not_attempted_asm
 asm += non_matching_asm + not_attempted_asm
 
-# Calculate the total amount of decompilable code
-total = src + asm
-
 # Calculate assets totals
-assets = audio + misc + object_ + scene + texture
-assets_total = audio_size + misc_size + object_size + scene_size + texture_size
+assets = sum(x["currentSize"] for x in assetsTracker.values())
+assets_total = sum(x["totalSize"] for x in assetsTracker.values())
 
 # Convert vaules to percentages
 src_percent = 100 * src / total
@@ -227,12 +322,11 @@ asm_percent = 100 * asm / total
 code_percent = 100 * code / code_size
 boot_percent = 100 * boot / boot_size
 ovl_percent = 100 * ovl / ovl_size
+
 assets_percent = 100 * assets / assets_total
-audio_percent = 100 * audio / audio_size
-misc_percent = 100 * misc / misc_size
-object_percent = 100 * object_ / object_size
-scene_percent = 100 * scene / scene_size
-texture_percent = 100 * texture / texture_size
+
+for assetCat in assetsTracker:
+    assetsTracker[assetCat]["percent"] = 100 * assetsTracker[assetCat]["currentSize"] / assetsTracker[assetCat]["totalSize"]
 
 # convert bytes to masks and rupees
 num_masks = 24
@@ -273,16 +367,22 @@ rupees = int((src % bytes_per_mask) / bytes_per_rupee)
 #print("")
 
 if args.format == 'csv':
-    version = 1
+    version = 2
     git_object = git.Repo().head.object
     timestamp = str(git_object.committed_date)
     git_hash = git_object.hexsha
-    csv_list = [str(version), timestamp, git_hash, str(code), str(code_size), str(boot), str(boot_size),
-                str(ovl), str(ovl_size), str(src), str(asm), str(len(non_matching_functions)),
-                str(audio), str(audio_size), str(misc), str(misc_size), str(object_), str(object_size),
-                str(scene), str(scene_size), str(texture), str(texture_size)]
+    csv_list = [
+        version, timestamp, git_hash, src, total,
+        boot, boot_size, code, code_size, ovl, ovl_size,
+        asm, len(non_matching_functions),
+    ]
+    csv_list += [
+        assets, assets_total,
+    ]
+    for assetCat in assetsCategories:
+        csv_list += [assetsTracker[assetCat]["currentSize"], assetsTracker[assetCat]["totalSize"]]
 
-    print(",".join(csv_list))
+    print(",".join(map(str, csv_list)))
 elif args.format == 'shield-json':
     # https://shields.io/endpoint
     print(json.dumps({
@@ -293,18 +393,19 @@ elif args.format == 'shield-json':
     }))
 elif args.format == 'text':
     adjective = "decompiled" if not args.matching else "matched"
+    assetsAdjective = "debinarized" if not args.matching else "identified"
 
     print("src:    {:>9} / {:>8} total bytes {:<13} {:>9.4f}%".format(src, total, adjective, round(src_percent, 4)))
     print("    boot:       {:>9} / {:>8} bytes {:<13} {:>9.4f}%".format(boot, boot_size, adjective, round(boot_percent, 4)))
     print("    code:       {:>9} / {:>8} bytes {:<13} {:>9.4f}%".format(code, code_size, adjective, round(code_percent, 4)))
     print("    overlays:   {:>9} / {:>8} bytes {:<13} {:>9.4f}%".format(ovl, ovl_size, adjective, round(ovl_percent, 4)))
     print()
-    print("assets:     {:>9} / {:>8} bytes reconstructed {:>9.4f}%".format(assets, assets_total, round(assets_percent, 4)))
-    print("    audio:    {:>9} / {:>8} bytes reconstructed {:>9.4f}%".format(audio, audio_size, round(audio_percent, 4)))
-    print("    misc:     {:>9} / {:>8} bytes reconstructed {:>9.4f}%".format(misc, misc_size, round(misc_percent, 4)))
-    print("    objects:  {:>9} / {:>8} bytes reconstructed {:>9.4f}%".format(object_, object_size, round(object_percent, 4)))
-    print("    scenes:   {:>9} / {:>8} bytes reconstructed {:>9.4f}%".format(scene, scene_size, round(scene_percent, 4)))
-    print("    textures: {:>9} / {:>8} bytes reconstructed {:>9.4f}%".format(texture, texture_size, round(texture_percent, 4)))
+
+    print("assets: {:>9} / {:>8} total bytes {:<13} {:>9.4f}%".format(assets, assets_total, assetsAdjective, round(assets_percent, 4)))
+    for assetCat in assetsTracker:
+        data = assetsTracker[assetCat]
+        print("    {:<10}  {:>9} / {:>8} bytes {:<13} {:>9.4f}%".format(f"{assetCat}:", data["currentSize"], data["totalSize"], assetsAdjective, round(data["percent"], 4)))
+
     print()
     print("------------------------------------\n")