Add versioning (#187)

* fix vscode cpp extension messing with files.associations

* move stuff

* it builds!

* symlink papermario.us.z64

* ci: put baserom in right place

* add jp

* fix splat dir

* ignore starrod dump

* .s deps

* update jenkins

* add dsl back

* configure.py versions

* wups

* fine ethan

* fix paths

* configure: default to only the version(s) with existing baseroms

* fix coverage

* fix progress.py

* progress.py verisoning

* remove format.sh from CONTRIBUTING

* update CONTRIBUTING

* fix first_diff

* diff.py: use ver/current/

* update splat.yaml

* trying to fix subrepo

* git subrepo pull tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "06a737f02d"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "06a737f02d"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "2f68596"

* configure fix

* git subrepo pull tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "41786effd3"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "41786effd3"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "2f68596"

Co-authored-by: Ethan Roseman <ethteck@gmail.com>

.gitignore

@@ -11,15 +11,20 @@ ctx.c
 expected/
 .vscode/launch.json
 /tools/star-rod
+/ver/current
 
 # Build artifacts
 build.ninja
+.ninja*
 *.ld
 *.elf
 *.z64
 *.Yay0
 *.msg.h
-/build/
+*.bin
+*.o
+build/
+assets/
 /docs/doxygen/
 /include/ld_addrs.h
 /include/message_ids.h
@@ -27,11 +32,6 @@ build.ninja
 /include/map
 /tools/permuter_settings.toml
 
-# Assets
-/*assets
-*.bin
-*.o
-
 # Star Rod
 /sprite/SpriteTable.xml
 /mod.cfg
@@ -39,6 +39,7 @@ build.ninja
 /editor
 /logs
 /out
+dump
 
 /tools/Yay0compress
 /tools/n64crc

.vscode/c_cpp_properties.json

@@ -12,13 +12,14 @@
             },
             "includePath": [
                 "${workspaceFolder}/include",
-                "${workspaceFolder}/build/include",
+                "${workspaceFolder}/ver/us/build/include",
                 "${workspaceFolder}/src"
             ],
             "defines": [
                 "F3DEX_GBI_2",
                 "_LANGUAGE_C",
-                "SCRIPT(...)={}"
+                "SCRIPT(...)={}",
+                "VERSION=us"
             ],
             "cStandard": "c89",
             "cppStandard": "c++17",

.vscode/settings.json

@@ -17,8 +17,9 @@
     ],
     "git.ignoreLimitWarning": true,
     "search.exclude": {
-        "build": true,
+        "**/build/src": true,
         "docs/doxygen": true,
+        "ctx.c": true,
     },
     "python.autoComplete.extraPaths": [
         "./tools"
@@ -31,16 +32,11 @@
     },
     "files.associations": {
         "*.h": "c",
-        "random": "c",
-        "array": "c",
-        "deque": "c",
-        "string": "c",
-        "unordered_map": "c",
-        "vector": "c",
-        "string_view": "c",
-        "initializer_list": "c",
-        "ranges": "c",
-        "regex": "c",
-        "variant": "c"
     },
+    "C_Cpp.autoAddFileAssociations": false,
+    "files.exclude": {
+        "**/.git": true,
+        "**/.splat_cache": true,
+        ".ninja*": true,
+    }
 }

CONTRIBUTING.md

@@ -18,65 +18,67 @@ If you use Visual Studio Code, you can use _Run Build Task_ (Ctrl+Shift+B) to ru
 
 ### Setup
 
-Once you've created a successful (`OK`) build, copy `build/` to `expected/build/`:
+Once you've created a successful (`OK`) build, copy `ver/us/build/` to `ver/us/expected/build/`:
 
 ```sh
-$ mkdir -p expected
+$ mkdir -p ver/us/expected
-$ cp -r build expected
+$ cp -r ver/us/build ver/us/expected
 ```
 
+(If you're working with other versions of the game, replace `us` in the file paths.)
+
 ### Roughly converting assembly to C
 
-Decide on a function to match. These can be found in the subdirectories of `asm/nonmatchings/`. Currently, functions which use float constants, data sections, or jump tables are unmatchable.
+Decide on a function to match. These can be found in the subdirectories of `ver/us/asm/nonmatchings/`.
 
-Take the relevant `.s` file and pass it to [mips_to_c](https://github.com/matt-kempster/mips_to_c) ([web version](https://simonsoftware.se/other/mips_to_c.py)).
+Take the relevant `.s` file and pass it to [mips_to_c](https://github.com/matt-kempster/mips_to_c) ([online version](https://simonsoftware.se/other/mips_to_c.py)).
 
-You can also use mips_to_c locally installed to a destination of your choice. Then register a function in `~/.bashrc` that calls `path/to/mips_to_c.py (with args)`:
+Open up the `.c` file that uses your function and replace the function's `INCLUDE_ASM` macro with the output from mips_to_c. For example, for a function `asm/nonmatchings/code_FOO/func_DEADBEEF`:
-```
+
-git clone https://github.com/matt-kempster/mips_to_c /path/to/mips_to_c
+```diff
+  // src/code_FOO.c
+- INCLUDE_ASM("code_FOO", func_DEADBEEF);
++ s32 func_DEADBEEF() {
++    // ...
++ }
 ```
 
-Here's a starter function you can use:
+Compile the ROM:
+
 ```sh
-# don't forget to replace /path/to/mips_to_c with your path
+ninja
-function mipstoc() {
-    if [ "$#" -gt 1 ]; then
-        /path/to/mips_to_c/mips_to_c.py $@;
-    else
-        printf "Please call mipstoc using this format and make sure you're at the repo root:";
-        printf "\nmipstoc \033[0;31marg1 - the nonmatching asm file\033[0m \033[0;34marg2 - the target function\033[0m \033[0;33margN - any of the optional mips_to_c.py flags\033[0m";
-        printf "\nmipstoc \033[0;31m./asm/nonmatchings/code_13870_len_6980/func_8003B3D0.s\033[0m \033[0;34mfunc_8003B3D0\033[0m \033[0;33m--flag1 --flag2 --flagN\033[0m\n";
-        /path/to/mips_to_c/mips_to_c.py;
-    fi
-}
-export -f mipstoc
 ```
 
-Open up the relevant `.c` file and replace the function's `INCLUDE_ASM` macro with the output from mips_to_c. Run the following command to attempt to compile, replacing `function_name` with the name of the function you're working with:
+This will probably end up either `FAIL`ing (the resulting ROM does not match the baserom), or the compilation of the C file you just modified did not succeed. mips_to_c loves to use void pointers and weird syntax that won't compile properly. Fixing this will involve typing the function signature correctly, which you may find in [Star Rod's library database](https://github.com/nanaian/star-rod/blob/master/database/common_func_library.lib). For structs, see [common_structs.h](include/common_structs.h).
+
+Once the C file compiles, you can compare the assembly generated by your code versus the original assembly with the following command, replacing `function_name` with the name of the function you're working on:
 
 ```sh
 ./diff.py -mwo function_name
 ```
 
-Fix any errors and rerun `diff.py`. This will involve typing the function signature correctly, which you will probably find in [Star Rod's library database](https://github.com/nanaian/star-rod/blob/master/database/common_func_library.lib). See also [common_structs.h](include/common_structs.h).
+(Sometimes, `-mwo` doesn't work. We don't know why yet; use `-mw` if you encounter issues.)
 
-Once a successful build is made, `diff.py` will show you the difference between the original game's assembly (on the left) and what your C code generated (on the right).
+`diff.py` displays the difference between the original game's assembly (on the left) and what your C code generated (on the right).
 
 ### Matching the function
 
-You're on your own now. Get your C code compiling to match the original assembly! `diff.py`, when running, will automatically recompile your code whenever you save the `.c` file.
+You're on your own now. Get your C code compiling to match the original assembly! `diff.py`, when running with `-m`, will automatically recompile your code whenever you save the `.c` file.
 
-If you use Visual Studio Code, you can use _Run Test Task_ to run `diff.py` and show you errors and warnings from the compiler inline. You might want to attach _Run Test Task_ to a keybinding, as you'll be using it often.
+If you use Visual Studio Code, you can use _Run Test Task_ to run `diff.py` and show you errors and warnings from the compiler inline. (You might want to attach _Run Test Task_ to a keybinding, as you'll be using it often.)
+
+If you have any questions or encounter any issues, we suggest:
+
+- Reaching out [on Discord](https://discord.gg/urUm3VG)
+- Using [decomp permuter](https://github.com/simonlindholm/decomp-permuter) if your code is logically equivalent
+- Wrapping what you have in `#ifdef NON_MATCHING` (see other examples of this in the codebase) and trying a smaller function
 
 ### After matching
 
-Once you've matched a function, run the following scripts:
+Once you've matched a function, run the following:
 
 ```sh
-$ ./coverage.py --delete-matched
+./coverage.py --delete-matched
-$ ./format.sh
 ```
 
-If `format.sh` has any problems with your code, go and fix the issues. If you can't fix a warning without making the function not match anymore, append `// NOLINT` to the offending line.
+Then, go ahead and [create a pull request](https://github.com/pmret/papermario/pulls)!
-
-Then, please [create a pull request](https://github.com/pmret/papermario/pulls)!

Jenkinsfile

@@ -13,7 +13,9 @@ pipeline {
     stages {
         stage('Setup') {
             steps {
-                sh './configure.py --baserom /usr/local/etc/roms/papermario.us.z64'
+                sh 'cp /usr/local/etc/roms/papermario.us.z64 ver/us/baserom.z64'
+                sh 'cp /usr/local/etc/roms/papermario.jp.z64 ver/jp/baserom.z64'
+                sh './configure.py'
             }
         }
         stage('Build') {
@@ -26,8 +28,11 @@ pipeline {
                 branch 'master'
             }
             steps {
-                sh 'python3 progress.py --csv >> /var/www/papermar.io/html/reports/progress.csv'
+                sh 'python3 progress.py us --csv >> /var/www/papermar.io/html/reports/progress_us.csv'
-                sh 'python3 progress.py --shield-json > /var/www/papermar.io/html/reports/progress_shield.json'
+                sh 'python3 progress.py us --shield-json > /var/www/papermar.io/html/reports/progress_us_shield.json'
+
+                sh 'python3 progress.py jp --csv >> /var/www/papermar.io/html/reports/progress_jp.csv'
+                sh 'python3 progress.py jp --shield-json > /var/www/papermar.io/html/reports/progress_jp_shield.json'
             }
         }
     }

README.md

@@ -1,21 +1,28 @@
 # Paper Mario
 
-[![Build Status][jenkins-badge]][jenkins] [![Progress][progress-badge]][progress] [![Discord Channel][discord-badge]][discord]
+[![Build Status][jenkins-badge]][jenkins]
+[![Progress (US)][progress-us-badge]][progress-us]
+[![Progress (JP)][progress-jp-badge]][progress-jp]
+[![Discord Channel][discord-badge]][discord]
 
 [jenkins]: https://jenkins.zelda64.dev/job/papermario/job/master
 [jenkins-badge]: https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fjenkins.zelda64.dev%2Fjob%2Fpapermario%2Fjob%2Fmaster
 
-[progress]: https://papermar.io/progress
+[progress-us]: https://papermar.io/progress-us
-[progress-badge]: https://img.shields.io/endpoint?url=https://papermar.io/reports/progress_shield.json
+[progress-us-badge]: https://img.shields.io/endpoint?url=https://papermar.io/reports/progress_us_shield.json
+
+[progress-jp]: https://papermar.io/progress-us
+[progress-jp-badge]: https://img.shields.io/endpoint?url=https://papermar.io/reports/progress_jp_shield.json
 
 [discord]: https://discord.gg/urUm3VG
 [discord-badge]: https://img.shields.io/discord/279322074412089344?color=%237289DA&logo=discord&logoColor=ffffff
 
-This is a work-in-progress decompilation of Paper Mario (USA).
+This is a work-in-progress decompilation of Paper Mario.
 
-It builds the following ROM:
+It builds the following ROMs:
 
-* papermario.z64 `sha1: 3837f44cda784b466c9a2d99df70d77c322b97a0`
+* papermario.us.z64 `sha1: 3837f44cda784b466c9a2d99df70d77c322b97a0`
+* papermario.jp.z64 `sha1: b9cca3ff260b9ff427d981626b82f96de73586d3`
 
 To set up the repository, see [INSTALL.md](INSTALL.md).
 

checksum.sha1

@@ -1 +0,0 @@
-3837f44cda784b466c9a2d99df70d77c322b97a0  papermario.z64

configure.py

@@ -8,40 +8,22 @@ from argparse import ArgumentParser
 import asyncio
 from subprocess import PIPE
 import subprocess
-import hashlib
 
 sys.path.append(os.path.dirname(__file__) + "/tools/splat")
 import split
 from segtypes.n64.code import Subsegment
 
 INCLUDE_ASM_RE = re.compile(r"___INCLUDE_ASM\([^,]+, ([^,]+), ([^,)]+)") # note _ prefix
-
+SUPPORTED_VERSIONS = ["us", "jp"]
-CFG = {}
+TARGET = "papermario"
-with open("build.cfg", "r") as f:
-    for line in f.readlines():
-        if line.strip() != "":
-            key, value = [part.strip() for part in line.split("=", 1)]
-            CFG[key] = value
-
-TARGET = CFG.get("target", "papermario")
-BUILD_DIR = "build"
-ASSET_DIRS = CFG.get("asset_dirs", "assets").split(" ")
-
-NPC_SPRITES = CFG.get("npc_sprites", "").split(" ")
-MAPS = CFG.get("maps", "").split(" ")
-TEXTURE_ARCHIVES = CFG.get("texture_archives", "").split(" ")
-BACKGROUNDS = CFG.get("backgrounds", "").split(" ")
-PARTY_IMAGES = CFG.get("party_images", "").split(" ")
-
-ASSETS = sum([[f"{map_name}_shape", f"{map_name}_hit"] for map_name in MAPS], []) + TEXTURE_ARCHIVES + BACKGROUNDS + ["title_data"] + PARTY_IMAGES
 
 def obj(path: str):
-    if not path.startswith("$builddir/"):
+    if not path.startswith("ver/"):
-        path = "$builddir/" + path
+        path = f"ver/{version}/build/{path}"
-    path = re.sub(r"/assets/", "/", path)
+    path = re.sub(r"/assets/", "/build/", path) # XXX what about other asset dirs?
     return path + ".o"
 
-def read_splat(splat_config: str):
+def read_splat(splat_config: str, version: str):
     import argparse
     import yaml
     from segtypes.n64.code import N64SegCode
@@ -61,10 +43,14 @@ def read_splat(splat_config: str):
 
     for segment in all_segments:
         for subdir, path, obj_type, start in segment.get_ld_files():
+            # src workaround
+            if subdir.startswith("../../"):
+                subdir = subdir[6:]
             if path.endswith(".c") or path.endswith(".s") or path.endswith(".data") or path.endswith(".rodata"):
                 path = subdir + "/" + path
             else:
                 assert subdir == "assets", subdir + " " + path
+                subdir = "ver/" + version + "/assets"
 
             objects.add(path)
             segments[path] = segment
@@ -88,6 +74,7 @@ def rm_recursive(path):
 
     path = Path(path)
 
+    if path.exists():
         if path.is_dir():
             for f in path.iterdir():
                 rm_recursive(f)
@@ -105,33 +92,15 @@ async def shell(cmd: str):
 
     return stdout.decode("utf-8"), stderr.decode("utf-8")
 
-async def task(coro):
+async def shell_status(cmd: str):
-    global num_tasks, num_tasks_done
+    async with task_sem:
+        proc = await asyncio.create_subprocess_shell(cmd, stdout=PIPE, stderr=PIPE)
+        stdout, stderr = await proc.communicate()
 
-    await coro
+    return proc.returncode
-
-    num_tasks_done += 1
-    print(f"\rConfiguring build... {(num_tasks_done / num_tasks) * 100:.0f}%", end="")
-
-async def build_c_file(c_file: str, generated_headers, ccache, cppflags):
-    # preprocess c_file, but simply put an _ in front of INCLUDE_ASM and SCRIPT
-    stdout, stderr = await shell(f"{cpp} {cppflags} '-DINCLUDE_ASM(...)=___INCLUDE_ASM(__VA_ARGS__)' '-DSCRIPT(...)=___SCRIPT(__VA_ARGS__)' {c_file} -o - | grep -E '___SCRIPT|___INCLUDE_ASM' || true")
-
-    # search for macro usage (note _ prefix)
-    uses_dsl = "___SCRIPT(" in stdout
-
-    s_deps = []
-    for line in stdout.splitlines():
-        if line.startswith("___INCLUDE_ASM"):
-            match = INCLUDE_ASM_RE.match(line)
-            if match:
-                s_deps.append("asm/nonmatchings/" + eval(match[1]) + "/" + match[2] + ".s")
-
-    # add build task to ninja
-    n.build(obj(c_file), "cc_dsl" if uses_dsl else "cc", c_file, implicit=s_deps, order_only=generated_headers)
 
 def build_yay0_file(bin_file: str):
-    yay0_file = f"$builddir/{os.path.splitext(bin_file)[0]}.Yay0"
+    yay0_file = f"ver/{version}/build/{os.path.splitext(bin_file)[0]}.Yay0"
     n.build(yay0_file, "yay0compress", find_asset(bin_file), implicit="tools/Yay0compress")
     build_bin_object(yay0_file)
 
@@ -140,7 +109,7 @@ def build_bin_object(bin_file: str):
 
 def build_image(f: str, segment):
     path, img_type, png = f.rsplit(".", 2)
-    out = "$builddir/" + path + "." + img_type + ".png"
+    out = f"ver/{version}/build/" + path + "." + img_type + ".png"
 
     flags = ""
     if img_type != "palette" and not isinstance(segment, dict):
@@ -160,31 +129,46 @@ def cmd_exists(cmd):
         stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
 
 def find_asset_dir(path):
+    global ASSET_DIRS
     for d in ASSET_DIRS:
         if os.path.exists(d + "/" + path):
             return d
 
     print("Unable to find asset: " + path)
-    print("The asset dump may be incomplete. Run")
+    print("The asset dump may be incomplete. Try:")
-    print("    rm .splat_cache")
+    print("    ./configure.py --clean")
-    print("And then run ./configure.py again.")
     exit(1)
 
 def find_asset(path):
     return find_asset_dir(path) + "/" + path
 
 async def main():
-    global n, cpp, task_sem, num_tasks, num_tasks_done
+    global n, cpp, task_sem, num_tasks, num_tasks_done, ASSET_DIRS, version
 
     task_sem = asyncio.Semaphore(8)
 
     parser = ArgumentParser(description="Paper Mario build.ninja generator")
+    parser.add_argument("version", nargs="*", default=[], help="Version(s) to configure for. Most tools will operate on the first-provided only. Supported versions: " + ','.join(SUPPORTED_VERSIONS))
     parser.add_argument("--cpp", help="GNU C preprocessor command")
-    parser.add_argument("--baserom", default="baserom.z64", help="Path to unmodified Paper Mario (U) z64 ROM")
     parser.add_argument("--cflags", default="", help="Extra cc/cpp flags")
-    parser.add_argument("--no-splat", action="store_true", help="Don't split the baserom")
+    parser.add_argument("--no-splat", action="store_true", help="Don't split assets from the baserom(s)")
     parser.add_argument("--clean", action="store_true", help="Delete assets and previously-built files")
     args = parser.parse_args()
+    versions = args.version
+
+    # default version behaviour is to only do those that exist
+    if len(versions) == 0:
+        for version in SUPPORTED_VERSIONS:
+            rom = f"ver/{version}/baserom.z64"
+            if os.path.exists(rom):
+                versions.append(version)
+
+        if len(versions) == 0:
+            print("error: no baserom.z64 files could be found in the ver/*/ directories.")
+            exit(1)
+
+    print("Configuring for versions: " + ', '.join(versions))
+    print("")
 
     # on macOS, /usr/bin/cpp defaults to clang rather than gcc (but we need gcc's)
     if args.cpp is None and sys.platform == "darwin" and "Free Software Foundation" not in (await shell("cpp --version"))[0]:
@@ -194,63 +178,58 @@ async def main():
         print("    ./configure.py --cpp cpp-10")
         exit(1)
 
-    # verify baserom exists and is clean
-    try:
-        with open(args.baserom, "rb") as f:
-            h = hashlib.sha1()
-            h.update(f.read())
-
-            if h.hexdigest() != "3837f44cda784b466c9a2d99df70d77c322b97a0":
-                print(f"error: baserom '{args.baserom}' is modified, refusing to split it!")
-                print("The baserom must be an unmodified Paper Mario (U) z64 ROM.")
-                exit(1)
-    except IOError:
-        print(f"error: baserom '{args.baserom}' does not exist!")
-        print(f"Please make sure an unmodified Paper Mario (U) z64 ROM exists at '{args.baserom}'.")
-
-        if args.baserom == "baserom.z64": # the default
-            print("Or run this script again with the --baserom option:")
-            print("    ./configure.py --baserom /path/to/papermario.z64")
-            exit(1)
-
     cpp = args.cpp or "cpp"
-    ccache = "ccache" if cmd_exists("ccache") else ""
 
     if args.clean:
         print("Cleaning...")
         await shell("ninja -t clean")
-        rm_recursive("assets")
+        rm_recursive(f".splat_cache")
-        rm_recursive("build")
+
-        rm_recursive(".splat_cache")
+        for version in versions:
+            rm_recursive(f"ver/{version}/assets")
+            rm_recursive(f"ver/{version}/build")
+            rm_recursive(f"ver/{version}/.splat_cache")
 
     if not args.no_splat:
         # compile splat dependencies
         await shell("make -C tools/splat")
 
-        # split assets
+        has_any_rom = False
-        print("Splitting segments from baserom", end="")
+        for version in versions:
+            rom = f"ver/{version}/baserom.z64"
+            has_rom = False
+
+            try:
+                with open(rom, "rb") as f:
+                    has_rom = True
+                    has_any_rom = True
+            except IOError:
+                print(f"error: could not find baserom file '{rom}'!")
+                if len(versions) >= 2:
+                    print(f"You can avoid building version '{version}' by specifying versions on the command-line:")
+                    print(f"    ./configure.py {' '.join(ver for ver in versions if ver != version)}")
+                exit(1)
+
+            if has_rom:
+                print(f"Splitting assets from {rom}", end="")
                 split.main(
-            "tools/splat.yaml",
+                    f"ver/{version}/splat.yaml",
-            ".",
+                    f"ver/{version}",
-            args.baserom,
+                    rom,
                     [ "ld", "bin", "Yay0", "PaperMarioMapFS", "PaperMarioMessages", "img", "PaperMarioNpcSprites" ],
                     False,
                     False,
                 )
-
                 print("")
 
-    print("Configuring build...", end="")
+    print("Configuring build...")
 
-    # generate build.ninja
     n = ninja_syntax.Writer(open("build.ninja", "w"), width=120)
 
-    cppflags = f"-I{BUILD_DIR}/include -Iinclude -Isrc -D _LANGUAGE_C -D _FINALROM -ffreestanding -DF3DEX_GBI_2 -D_MIPS_SZLONG=32 " + args.cflags
+    cflags = " " + args.cflags
-    cflags = "-O2 -quiet -G 0 -mcpu=vr4300 -mfix4300 -mips3 -mgp32 -mfp32 -Wuninitialized -Wshadow " + args.cflags
     iconv = "tools/iconv.py UTF-8 SHIFT-JIS" if sys.platform == "darwin" else "iconv --from UTF-8 --to SHIFT-JIS"
     cross = "mips-linux-gnu-"
 
-    n.variable("builddir", BUILD_DIR)
     n.variable("target", TARGET)
     n.variable("cross", cross)
     n.variable("python", sys.executable)
@@ -266,31 +245,28 @@ async def main():
         print(f"Unsupported platform {sys.platform}")
         sys.exit(1)
 
-    n.variable("os", os_dir)
+    # $version
-    n.variable("iconv", iconv)
-    n.variable("cppflags", f"{cppflags} -Wcomment")
-    n.variable("cflags", cflags)
-    n.newline()
-
     n.rule("cc",
-        command=f"bash -o pipefail -c '{cpp} $cppflags -MD -MF $out.d $in -o - | $iconv | tools/$os/cc1 $cflags -o - | tools/$os/mips-nintendo-nu64-as -EB -G 0 - -o $out'",
+        command=f"bash -o pipefail -c '{cpp} -Iver/$version/build/include -Iinclude -Isrc -D _LANGUAGE_C -D _FINALROM -D VERSION=$version -ffreestanding -DF3DEX_GBI_2 -D_MIPS_SZLONG=32 {args.cflags} -MD -MF $out.d $in -o - | {iconv} | tools/{os_dir}/cc1 -O2 -quiet -G 0 -mcpu=vr4300 -mfix4300 -mips3 -mgp32 -mfp32 -Wuninitialized -Wshadow {args.cflags} -o - | tools/{os_dir}/mips-nintendo-nu64-as -EB -G 0 - -o $out'",
         description="cc $in",
         depfile="$out.d",
         deps="gcc")
     n.rule("cc_dsl",
-        command=f"bash -o pipefail -c '{cpp} $cppflags -MD -MF $out.d $in -o - | $python tools/compile_dsl_macros.py | $iconv | tools/$os/cc1 $cflags -o - | tools/$os/mips-nintendo-nu64-as -EB -G 0 - -o $out'",
+        command=f"bash -o pipefail -c '{cpp} -Iver/$version/build/include -Iinclude -Isrc -D _LANGUAGE_C -D _FINALROM -D VERSION=$version -ffreestanding -DF3DEX_GBI_2 -D_MIPS_SZLONG=32 {args.cflags} -MD -MF $out.d $in -o - | $python tools/compile_dsl_macros.py | {iconv} | tools/{os_dir}/cc1 -O2 -quiet -G 0 -mcpu=vr4300 -mfix4300 -mips3 -mgp32 -mfp32 -Wuninitialized -Wshadow {args.cflags} -o - | tools/{os_dir}/mips-nintendo-nu64-as -EB -G 0 - -o $out'",
-        description="cc (with dsl) $in",
+        description="dsl $in",
         depfile="$out.d",
         deps="gcc")
     n.newline()
 
     with open("tools/permuter_settings.toml", "w") as f:
-        f.write(f"compiler_command = \"{cpp} {cppflags} -D SCRIPT(...)={{}} | {iconv} | tools/{os_dir}/cc1 {cflags} -o - | tools/{os_dir}/mips-nintendo-nu64-as -EB -G 0 -\"\n")
+        version = versions[0]
+        f.write(f"compiler_command = \"{cpp} -Iver/{version}/build/include -Iinclude -Isrc -D _LANGUAGE_C -D _FINALROM -D VERSION={version} -ffreestanding -DF3DEX_GBI_2 -D_MIPS_SZLONG=32 {args.cflags} -D SCRIPT(...)={{}} | {iconv} | tools/{os_dir}/cc1 -O2 -quiet -G 0 -mcpu=vr4300 -mfix4300 -mips3 -mgp32 -mfp32 -Wuninitialized -Wshadow {args.cflags} -o - | tools/{os_dir}/mips-nintendo-nu64-as -EB -G 0 -\"\n")
         f.write(f"assembler_command = \"{cross}as -march=vr4300 -mabi=32\"\n")
 
+    # $version
     n.rule("cpp",
-        command=f"{cpp} -P -DBUILD_DIR=$builddir $in -o $out",
+        command=f"{cpp} -P -DBUILD_DIR=ver/$version/build $in -o $out",
-        description="cc (with dsl) $in",
+        description="cpp $in",
         depfile="$out.d",
         deps="gcc")
     n.newline()
@@ -349,8 +325,9 @@ async def main():
         description="combine assets")
     n.newline()
 
+    # $version
     n.rule("link",
-        command="${cross}ld -T undefined_syms.txt -T undefined_syms_auto.txt -T undefined_funcs_auto.txt -T dead_syms.txt -Map $builddir/$target.map --no-check-sections -T $in -o $out",
+        command="${cross}ld -T ver/$version/undefined_syms.txt -T ver/$version/undefined_syms_auto.txt -T ver/$version/undefined_funcs_auto.txt  -T ver/$version/dead_syms.txt -Map ver/$version/build/$target.map --no-check-sections -T $in -o $out",
         description="link $out")
     n.newline()
 
@@ -359,51 +336,61 @@ async def main():
         description="rom $in")
     n.newline()
 
-    objects, segments = read_splat("tools/splat.yaml") # no .o extensions!
+    # $version
-    c_files = (f for f in objects if f.endswith(".c")) # glob("src/**/*.c", recursive=True)
+    n.rule("checksum",
-
+        command=f"sha1sum -c ver/$version/checksum.sha1 && touch $out",
-    n.rule("checksums",
+        description="compare")
-        command=f"(sha1sum -c checksum.sha1 && bash $out.bash > $out) || sha1sum -c $out --quiet",
-        description="compare",
-        rspfile="$out.bash",
-        rspfile_content=f"sha1sum {' '.join([obj(o) for o in objects])}")
     n.newline()
 
     n.rule("cc_modern_exe", command="cc $in -O3 -o $out")
     n.newline()
 
-    n.comment("target")
+    for version in versions:
-    n.build("$builddir/$target.ld", "cpp", "$target.ld")
+        objects, segments = read_splat(f"ver/{version}/splat.yaml", version) # no .o extensions!
-    n.build("$builddir/$target.elf", "link", "$builddir/$target.ld", implicit=[obj(o) for o in objects], implicit_outputs="$builddir/$target.map")
+        #c_files = (f for f in objects if f.endswith(".c"))
-    n.build("$target.z64", "rom", "$builddir/$target.elf", implicit="tools/n64crc")
-    n.build("$builddir/expected.sha1", "checksums", implicit="$target.z64")
-    n.newline()
 
-    n.default("$builddir/expected.sha1")
+        n.build(f"ver/{version}/build/$target.ld", "cpp", f"ver/{version}/$target.ld", variables={ "version": version })
-    n.newline()
+        n.build(f"ver/{version}/build/$target.elf", "link", f"ver/{version}/build/$target.ld", implicit=[obj(o) for o in objects], implicit_outputs=f"ver/{version}/$target.map", variables={ "version": version })
+        n.build(f"ver/{version}/build/$target.z64", "rom", f"ver/{version}/build/$target.elf", implicit="tools/n64crc")
+        n.build(f"ver/{version}/build/ok", "checksum", implicit=f"ver/{version}/build/$target.z64", variables={ "version": version })
+        n.build(version, "phony", f"ver/{version}/build/ok")
+        n.build(f"$target.{version}.z64", "phony", f"ver/{version}/build/$target.z64")
+
+        CFG = {}
+        with open(f"ver/{version}/build.cfg", "r") as f:
+            for line in f.readlines():
+                if line.strip() != "":
+                    key, value = [part.strip() for part in line.split("=", 1)]
+                    CFG[key] = value
+
+        ASSET_DIRS = CFG.get("asset_dirs", "assets").split(" ")
+
+        NPC_SPRITES = CFG.get("npc_sprites", "").split(" ")
+        MAPS = CFG.get("maps", "").split(" ")
+        TEXTURE_ARCHIVES = CFG.get("texture_archives", "").split(" ")
+        BACKGROUNDS = CFG.get("backgrounds", "").split(" ")
+        PARTY_IMAGES = CFG.get("party_images", "").split(" ")
+
+        ASSETS = sum([[f"{map_name}_shape", f"{map_name}_hit"] for map_name in MAPS], []) + TEXTURE_ARCHIVES + BACKGROUNDS + ["title_data"] + PARTY_IMAGES
 
-    # generated headers
-    n.comment("generated headers")
         generated_headers = []
 
         def add_generated_header(h: str):
             generated_headers.append(h)
 
-        he = re.sub(r"\$builddir", BUILD_DIR, h)
+            if not os.path.exists(h):
-
-        if not os.path.exists(he):
                 # mkdir -p
-            os.makedirs(os.path.dirname(he), exist_ok=True)
+                os.makedirs(os.path.dirname(h), exist_ok=True)
 
                 # touch it so cpp doesn't complain if its #included
-            open(he, "w").close()
+                open(h, "w").close()
 
                 # mark it as really old so ninja builds it
-            os.utime(he, (0, 0))
+                os.utime(h, (0, 0))
 
             return h
 
-    n.build(add_generated_header("$builddir/include/ld_addrs.h"), "ld_addrs_h", "$builddir/$target.ld")
+        n.build(add_generated_header(f"ver/{version}/build/include/ld_addrs.h"), "ld_addrs_h", f"ver/{version}/build/$target.ld")
 
         # messages
         msg_files = set()
@@ -413,24 +400,26 @@ async def main():
         msg_files = list(msg_files)
         for msg_file in msg_files:
             n.build(
-            f"$builddir/{msg_file.split('/', 1)[1]}.bin",
+                f"ver/{version}/build/{msg_file.split('/', 1)[1]}.bin",
                 "msg",
                 msg_file,
                 implicit="tools/msg/parse_compile.py",
             )
-    #msg_headers = [add_generated_header(f"$builddir/include/{msg_file.split('/', 1)[1]}.h") for msg_file in msg_files]
+        msg_bins = [f"ver/{version}/build/{msg_file.split('/', 1)[1]}.bin" for msg_file in msg_files]
-    msg_bins = [f"$builddir/{msg_file.split('/', 1)[1]}.bin" for msg_file in msg_files]
         n.build(
-        ["$builddir/msg.bin", add_generated_header(f"$builddir/include/message_ids.h")],
+            [f"ver/{version}/build/msg.bin", add_generated_header(f"ver/{version}/build/include/message_ids.h")],
             "msg_combine",
             msg_bins,
             implicit="tools/msg/combine.py",
         )
-    n.build("$builddir/msg.o", "bin", "$builddir/msg.bin")
+        n.build(f"ver/{version}/build/msg.o", "bin", f"ver/{version}/build/msg.bin")
 
         # sprites
         npc_sprite_yay0s = []
         for sprite_id, sprite_name in enumerate(NPC_SPRITES, 1):
+            if len(sprite_name) == 0 or sprite_name == "_":
+                continue
+
             asset_dir = find_asset_dir(f"sprite/npc/{sprite_name}")
             sources = glob(f"{asset_dir}/sprite/npc/{sprite_name}/**/*.*", recursive=True)
             variables = {
@@ -441,7 +430,7 @@ async def main():
 
             # generated header
             n.build(
-            add_generated_header(f"$builddir/include/sprite/npc/{sprite_name}.h"),
+                add_generated_header(f"ver/{version}/build/include/sprite/npc/{sprite_name}.h"),
                 "sprite_animations_h",
                 implicit=sources + ["tools/gen_sprite_animations_h.py"],
                 variables=variables,
@@ -449,24 +438,23 @@ async def main():
 
             # sprite bin/yay0
             n.build(
-            f"$builddir/sprite/npc/{sprite_name}",
+                f"ver/{version}/build/sprite/npc/{sprite_name}",
                 "npc_sprite",
                 implicit=sources + ["tools/compile_npc_sprite.py"],
                 variables=variables,
             )
-        yay0 = f"$builddir/sprite/npc/{sprite_name}.Yay0"
+            yay0 = f"ver/{version}/build/sprite/npc/{sprite_name}.Yay0"
             npc_sprite_yay0s.append(yay0)
             n.build(
                 yay0,
                 "yay0compress",
-            f"$builddir/sprite/npc/{sprite_name}",
+                f"ver/{version}/build/sprite/npc/{sprite_name}",
                 implicit=["tools/Yay0compress"],
             )
 
         n.newline()
 
         # fast tasks
-    n.comment("data")
         for f in objects:
             segment = segments[f]
 
@@ -477,15 +465,15 @@ async def main():
             elif f.endswith(".bin"):
                 build_bin_object(find_asset(f))
             elif f.endswith(".data"):
-            n.build(obj(f), "as", "asm/" + f + ".s")
+                n.build(obj(f), "as", f"ver/{version}/asm/" + f + ".s")
             elif f.endswith(".rodata"):
-            n.build(obj(f), "as", "asm/" + f[2:] + ".s")
+                n.build(obj(f), "as", f"ver/{version}/asm/" + f[2:] + ".s")
             elif f.endswith(".s"):
-            n.build(obj(f), "as", f)
+                n.build(obj(f), "as", f"ver/{version}/" + f)
             elif f.endswith(".png"):
                 if isinstance(segment, Subsegment):
                     # image within a code section
-                out = "$builddir/" + f + ".bin"
+                    out = f"ver/{version}/build/{f}.bin"
                     infile = find_asset(re.sub(r"\.pal\.png", ".png", f))
 
                     n.build(out, "img", infile, implicit="tools/img/build.py", variables={
@@ -494,15 +482,15 @@ async def main():
                     })
 
                     if ".pal.png" not in f:
-                    n.build(add_generated_header("$builddir/include/" + f + ".h"), "img_header", infile, implicit="tools/img/header.py")
+                        n.build(add_generated_header(f"ver/{version}/build/include/" + f + ".h"), "img_header", infile, implicit="tools/img/header.py")
 
-                n.build("$builddir/" + f + ".o", "bin", out)
+                    n.build(f"ver/{version}/build/{f}.o", "bin", out)
                 else:
                     build_image(f, segment)
             elif f == "sprite/npc":
                 # combine sprites
-            n.build(f"$builddir/{f}.bin", "npc_sprites", npc_sprite_yay0s, implicit="tools/compile_npc_sprites.py")
+                n.build(f"ver/{version}/build/{f}.bin", "npc_sprites", npc_sprite_yay0s, implicit="tools/compile_npc_sprites.py")
-            n.build(obj(f), "bin", f"$builddir/{f}.bin")
+                n.build(obj(f), "bin", f"ver/{version}/build/{f}.bin")
             elif segment.type == "PaperMarioMessages":
                 continue # done already above
             elif segment.type == "PaperMarioMapFS":
@@ -513,8 +501,8 @@ async def main():
                         asset_files.append(find_asset(f"map/{asset_name}.bin"))
                         asset_files.append(find_asset(f"map/{asset_name}.bin"))
                     elif asset_name.startswith("party_"):
-                    source_file = f"$builddir/{asset_name}.bin"
+                        source_file = f"ver/{version}/build/{asset_name}.bin"
-                    asset_file = f"$builddir/{asset_name}.Yay0"
+                        asset_file = f"ver/{version}/build/{asset_name}.Yay0"
 
                         n.build(source_file, "img", find_asset(f"party/{asset_name}.png"), implicit="tools/img/build.py", variables={
                             "img_type": "party",
@@ -525,8 +513,8 @@ async def main():
                         asset_files.append(asset_file)
                         n.build(asset_file, "yay0compress", source_file, implicit="tools/Yay0compress")
                     elif asset_name.endswith("_bg"):
-                    source_file = f"$builddir/{asset_name}.bin"
+                        source_file = f"ver/{version}/build/{asset_name}.bin"
-                    asset_file = f"$builddir/{asset_name}.Yay0"
+                        asset_file = f"ver/{version}/build/{asset_name}.Yay0"
 
                         n.build(source_file, "img", find_asset(f"map/{asset_name}.png"), implicit="tools/img/build.py", variables={
                             "img_type": "bg",
@@ -538,46 +526,66 @@ async def main():
                         n.build(asset_file, "yay0compress", source_file, implicit="tools/Yay0compress")
                     elif asset_name.endswith("_shape") or asset_name.endswith("_hit"):
                         source_file = find_asset(f"map/{asset_name}.bin")
-                    asset_file = f"$builddir/assets/{asset_name}.Yay0"
+                        asset_file = f"ver/{version}/build/assets/{asset_name}.Yay0"
 
                         asset_files.append(source_file)
                         asset_files.append(asset_file)
                         n.build(asset_file, "yay0compress", source_file, implicit="tools/Yay0compress")
                     else:
                         source_file = find_asset(f"{asset_name}.bin")
-                    asset_file = f"$builddir/assets/{asset_name}.Yay0"
+                        asset_file = f"ver/{version}/build/assets/{asset_name}.Yay0"
 
                         asset_files.append(source_file)
                         asset_files.append(asset_file)
                         n.build(asset_file, "yay0compress", source_file, implicit="tools/Yay0compress")
 
-            n.build("$builddir/assets.bin", "assets", asset_files)
+                n.build(f"ver/{version}/build/assets.bin", "assets", asset_files)
-            n.build(obj(f), "bin", "$builddir/assets.bin")
+                n.build(obj(f), "bin", f"ver/{version}/build/assets.bin")
             else:
                 print("warning: dont know what to do with object " + f)
         n.newline()
 
-    n.build("generated_headers", "phony", generated_headers)
+        n.build("generated_headers_" + version, "phony", generated_headers)
         n.newline()
 
-    # slow tasks generated concurrently
+    for c_file in glob("src/**/*.c", recursive=True):
-    n.comment("c")
+        if c_file.endswith(".inc.c"):
-    tasks = [task(build_c_file(f, "generated_headers", ccache, cppflags)) for f in c_files]
+            continue
-    num_tasks = len(tasks)
+
-    num_tasks_done = 0
+        status = await shell_status(f"grep -q SCRIPT\( {c_file}")
-    await asyncio.gather(*tasks)
+
+        for version in versions:
+            s_glob = "ver/" + version + "/" + re.sub("src/", "asm/nonmatchings/", c_file)[:-2] + "/*.s"
+            n.build(
+                obj(c_file),
+                "cc_dsl" if status == 0 else "cc",
+                c_file,
+                implicit=glob(s_glob),
+                order_only="generated_headers_" + version,
+                variables={ "version": version }
+            )
+
     print("")
-    n.newline()
 
     # c tools that need to be compiled
     n.build("tools/Yay0compress", "cc_modern_exe", "tools/Yay0compress.c")
     n.build("tools/n64crc", "cc_modern_exe", "tools/n64crc.c")
     n.newline()
 
-    print("")
+    n.build("all", "phony", versions)
+    n.default("all")
+
+    # update ver/current to versions[0]
+    try:
+        os.remove("ver/current")
+    except Exception:
+        pass
+    os.symlink(versions[0], "ver/current")
+    n.build("ver/current/build/papermario.z64", "phony", "ver/" + versions[0] + "/build/papermario.z64")
+
     print("Build configuration complete! Now run")
     print("    ninja")
-    print(f"to compile '{TARGET}.z64'.")
+    print("to compile " + ', '.join(f'\'{TARGET}.{version}.z64\'' for version in versions) + ".")
 
 if __name__ == "__main__":
     loop = asyncio.get_event_loop()

coverage.py

@@ -5,12 +5,6 @@ import re
 import sys
 from pathlib import Path
 
-DIR = os.path.dirname(__file__)
-NONMATCHINGS_DIR = Path(os.path.join(DIR, "asm", "nonmatchings"))
-
-C_FILES = Path(os.path.join(DIR, "src")).rglob("*.c")
-ASM_FILES = NONMATCHINGS_DIR.rglob("*.s")
-
 def strip_c_comments(text):
     def replacer(match):
         s = match.group(0)
@@ -38,25 +32,32 @@ asm_func_pattern = re.compile(
 def include_asms_in_c(text):
     return (match.group(1) for match in asm_func_pattern.finditer(text))
 
-matched = []
+def stuff(version):
-asm = []
+    DIR = os.path.dirname(__file__)
-for filename in C_FILES:
+    NONMATCHINGS_DIR = Path(os.path.join(DIR, "ver", version, "asm", "nonmatchings"))
+
+    C_FILES = Path(os.path.join(DIR, "src")).rglob("*.c")
+    ASM_FILES = NONMATCHINGS_DIR.rglob("*.s")
+
+    matched = []
+    asm = []
+    for filename in C_FILES:
         with open(filename, "r") as file:
             text = strip_c_comments(file.read())
             matched.extend((m for m in funcs_in_c(text) if not m in matched))
             asm.extend((m for m in include_asms_in_c(text) if not m in asm))
 
-non_matched = [os.path.splitext(os.path.basename(filename))[0] for filename in ASM_FILES]
+    non_matched = [os.path.splitext(os.path.basename(filename))[0] for filename in ASM_FILES]
 
-partial_matched = [f for f in matched if f in asm]
+    partial_matched = [f for f in matched if f in asm]
-matched = [f for f in matched if not f in partial_matched]
+    matched = [f for f in matched if not f in partial_matched]
-matched_but_undeleted_asm = set([f for f in matched if f in non_matched and not f in partial_matched])
+    matched_but_undeleted_asm = set([f for f in matched if f in non_matched and not f in partial_matched])
-orphan_asm = set(non_matched) - set(asm) - matched_but_undeleted_asm
+    orphan_asm = set(non_matched) - set(asm) - matched_but_undeleted_asm
-missing_asm = set(asm) - set(non_matched)
+    missing_asm = set(asm) - set(non_matched)
 
-to_delete = matched_but_undeleted_asm | orphan_asm
+    to_delete = matched_but_undeleted_asm | orphan_asm
 
-if __name__ == "__main__":
+    if __name__ == "__main__":
         if "--help" in sys.argv:
             print("--fail-undeleted            exit with error code 1 if obsolete .s functions exist")
             print("--delete                    delete obsolete .s functions without asking")
@@ -64,11 +65,13 @@ if __name__ == "__main__":
 
         if len(matched_but_undeleted_asm) > 0:
             print(f"The following functions have been matched but their .s files remain: {matched_but_undeleted_asm}")
+        """
         if len(set(asm)) != len(set(non_matched)):
             if len(set(non_matched)) > len(set(asm)) and len(orphan_asm) > 0:
                 print(f"The following functions are unmatched but are also unINCLUDEd: {orphan_asm}")
             elif len(missing_asm) > 0:
                 print(f"warning: The following .s files are INCLUDEd but don't exist: {missing_asm}")
+        """
 
         if len(to_delete) > 0:
             if "--fail-undeleted" in sys.argv:
@@ -82,3 +85,6 @@ if __name__ == "__main__":
                 for folder in list(os.walk(NONMATCHINGS_DIR)):
                     if not os.listdir(folder[0]):
                         os.removedirs(folder[0])
+
+stuff("us")
+stuff("jp")

diff_settings.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 
 def apply(config, args):
-    config['baseimg'] = 'baserom.z64'
+    ver_dir = 'ver/current/'
-    config['myimg'] = 'papermario.z64'
+    config['baseimg'] = f'{ver_dir}baserom.z64'
-    config['mapfile'] = 'build/papermario.map'
+    config['myimg'] = f'{ver_dir}papermario.z64'
-    config['source_directories'] = ['src', 'asm', 'include', 'assets']
+    config['mapfile'] = f'{ver_dir}build/papermario.map'
+    config['source_directories'] = ['src', f'{ver_dir}asm', 'include', f'{ver_dir}assets']
     config['make_command'] = ['ninja']

first_diff.py

@@ -32,13 +32,13 @@ args = parser.parse_args()
 diff_count = args.count
 
 if args.make:
-    check_call(["ninja", "papermario.z64"])
+    check_call(["ninja", "ver/current/build/papermario.z64"])
 
-baseimg = f"baserom.z64"
+baseimg = f"ver/current/baserom.z64"
-basemap = f"expected/build/papermario.map"
+basemap = f"ver/current/expected/build/papermario.map"
 
-myimg = f"papermario.z64"
+myimg = f"ver/current/build/papermario.z64"
-mymap = f"build/papermario.map"
+mymap = f"ver/current/build/papermario.map"
 
 if not os.path.isfile(baseimg):
     print(f"{baseimg} must exist.")

include/macros.h

@@ -3,10 +3,13 @@
 
 #include "common.h"
 
+#define STRINGIFY_(x) #x
+#define STRINGIFY(x) STRINGIFY_(x)
+
 #ifndef SPLAT
 #ifndef INCLUDE_ASM
 #define INCLUDE_ASM(TYPE, FOLDER, NAME, ARGS...) \
-  TYPE __attribute__((naked)) NAME(ARGS) { __asm__( ".include \"include/macro.inc\"\n.include \"asm/nonmatchings/"FOLDER"/"#NAME".s\"\n.set reorder\n.set at"); }
+  TYPE __attribute__((naked)) NAME(ARGS) { __asm__( ".include \"include/macro.inc\"\n.include \"ver/"STRINGIFY(VERSION)"/asm/nonmatchings/"FOLDER"/"#NAME".s\"\n.set reorder\n.set at"); }
 #endif
 #else
 #define INCLUDE_ASM(TYPE, FOLDER, NAME, ARGS...)

papermario.jp.z64

@@ -0,0 +1 @@
+ver/jp/papermario.z64

papermario.us.z64

@@ -0,0 +1 @@
+ver/us/build/papermario.z64

progress.py

@@ -5,12 +5,15 @@ import git
 import os
 import subprocess
 import sys
+from colour import Color
 
-script_dir = os.path.dirname(os.path.realpath(__file__))
+def set_version(version):
-root_dir = script_dir
+    global script_dir, root_dir, asm_dir, build_dir, elf_path
-asm_dir = os.path.join(root_dir, "asm", "nonmatchings")
+    script_dir = os.path.dirname(os.path.realpath(__file__))
-build_dir = os.path.join(root_dir, "build")
+    root_dir = os.path.join(script_dir, "ver", version)
-elf_path = os.path.join(build_dir, "papermario.elf")
+    asm_dir = os.path.join(root_dir, "asm", "nonmatchings")
+    build_dir = os.path.join(root_dir, "build")
+    elf_path = os.path.join(build_dir, "papermario.elf")
 
 def get_func_sizes():
     try:
@@ -59,7 +62,12 @@ def get_funcs_sizes(sizes, matchings, nonmatchings):
 
     return msize, nmsize
 
+def lerp(a, b, alpha):
+    return a + (b - a) * alpha
+
 def main(args):
+    set_version(args.version)
+
     func_sizes, total_size = get_func_sizes()
     all_funcs = set(func_sizes.keys())
 
@@ -68,6 +76,10 @@ def main(args):
 
     matching_size, nonmatching_size = get_funcs_sizes(func_sizes, matching_funcs, nonmatching_funcs)
 
+    if len(all_funcs) == 0:
+        funcs_matching_ratio = 0.0
+        matching_ratio = 0.0
+    else:
         funcs_matching_ratio = (len(matching_funcs) / len(all_funcs)) * 100
         matching_ratio = (matching_size / total_size) * 100
 
@@ -83,11 +95,12 @@ def main(args):
         import json
 
         # https://shields.io/endpoint
+        color = Color("#50ca22", hue=lerp(0, 105/255, matching_ratio / 100))
         print(json.dumps({
             "schemaVersion": 1,
-            "label": "progress",
+            "label": f"progress ({args.version})",
             "message": f"{matching_ratio:.2f}%",
-            "color": "yellow",
+            "color": color.hex,
         }))
     else:
         if matching_size + nonmatching_size != total_size:
@@ -98,6 +111,7 @@ def main(args):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Reports progress for the project")
+    parser.add_argument("version", default="current", nargs="?")
     parser.add_argument("--csv", action="store_true")
     parser.add_argument("--shield-json", action="store_true")
     args = parser.parse_args()

requirements_extra.txt

@@ -4,3 +4,4 @@ python-Levenshtein
 stringcase
 watchdog
 gitpython
+colour

tools/asm_sizes.py

@@ -36,7 +36,7 @@ def do_dir(root, dir):
 
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-asm_dir = script_dir + "/../asm/nonmatchings"
+asm_dir = script_dir + "/../ver/current/asm/nonmatchings"
 
 for root, dirs, files in os.walk(asm_dir):
     for asm_dir in dirs:

tools/fix_s_filenames.py

@@ -8,7 +8,7 @@ import re
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = script_dir + "/../"
-asm_dir = root_dir + "asm/nonmatchings/"
+asm_dir = root_dir + "ver/current/asm/nonmatchings/"
 
 for root, dirs, files in os.walk(asm_dir):
     for f_name in files:

tools/fixc.py

@@ -11,7 +11,7 @@ from pathlib import Path
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = script_dir + "/../"
 src_dir = root_dir + "src/world/"
-asm_dir = root_dir + "asm/nonmatchings/world/"
+asm_dir = root_dir + "ver/current/asm/nonmatchings/world/"
 
 
 def sub_func(match):

tools/img/header.py

@@ -12,6 +12,9 @@ if __name__ == "__main__":
 
     cname = re.sub(r"[^0-9a-zA-Z_]", "_", infile)
 
+    if cname.startswith("ver_"):
+        cname = "_".join(cname.split("_")[2:])
+
     if cname.startswith("src_"):
         cname = cname[4:]
     elif cname.startswith("assets_"):

tools/m_map_funcs.py

@@ -8,7 +8,7 @@ from pathlib import Path
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.abspath(os.path.join(script_dir, ".."))
 src_dir = root_dir + "/src/"
-asm_dir = root_dir + "/asm/"
+asm_dir = root_dir + "/ver/current/asm/"
 
 common_files = []
 

tools/mapheaders.py

@@ -11,7 +11,7 @@ from pathlib import Path
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = script_dir + "/../"
 src_dir = root_dir + "src/world/"
-asm_dir = root_dir + "asm/nonmatchings/world/"
+asm_dir = root_dir + "ver/current/asm/nonmatchings/world/"
 
 for root, dirs, files in os.walk(src_dir):
     for dir_name in dirs:

tools/rename.py

@@ -7,7 +7,7 @@ import re
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = script_dir + "/../"
 src_dir = root_dir + "src/"
-asm_dir = root_dir + "asm/"
+asm_dir = root_dir + "ver/current/asm/"
 
 with open(os.path.join(script_dir, "duplicate_renames.txt")) as f:
     renames_text = f.readlines()

tools/smallest_with_symbol.py

@@ -7,7 +7,7 @@ import re
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = script_dir + "/../"
 src_dir = root_dir + "src/"
-asm_dir = root_dir + "asm/"
+asm_dir = root_dir + "ver/current/asm/"
 
 symbol = "gMasterGfxPos"
 

tools/sort_symbol_addrs.py

@@ -8,12 +8,13 @@ script_dir = os.path.dirname(os.path.realpath(__file__))
 
 syms = {}
 
-file_path = os.path.join(script_dir, "symbol_addrs.txt")
+for version in ["us", "jp"]:
+    file_path = os.path.join(script_dir, f"ver/{version}/symbol_addrs.txt")
 
-with open(file_path) as f:
+    with open(file_path) as f:
         symbol_lines = f.readlines()
 
-for line in symbol_lines:
+    for line in symbol_lines:
         addr_text = line.split(" = ")[1][:10]
         addr = int(addr_text, 0)
         if addr in syms:
@@ -21,6 +22,6 @@ for line in symbol_lines:
             sys.exit(55)
         syms[addr] = line
 
-with open(file_path, newline="\n", mode="w") as f:
+    with open(file_path, newline="\n", mode="w") as f:
         for addr in sorted(syms):
             f.write(syms[addr])

tools/splat/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/ethteck/splat.git
 	branch = master
-	commit = c46026725ab663685241cab84349984defcb5811
+	commit = 41786effd390f82d31468d1ebdb2d297e5f7ea75
-	parent = 0f7f21e5fc1d4c082a5dbd7c1c571d43725e6b08
+	parent = 2d00806512f19c00e00e20d2bfa55b15353e877a
 	method = merge
 	cmdver = 0.4.3

tools/splat/segtypes/n64/code.py

@@ -46,7 +46,7 @@ class Subsegment():
 
     def get_out_subdir(self, options):
         if self.type in ["c", ".data", ".rodata", ".bss"]:
-            return "src"
+            return options.get("src_path", "src")
         elif self.type in ["asm", "hasm", "header"]:
             return "asm"
         elif self.type == "bin":

tools/splat/segtypes/segment.py

@@ -154,6 +154,10 @@ class Segment:
             else:
                 path = PurePath(subdir) / PurePath(path)
 
+            # Remove leading ..s
+            while path.parts[0] == "..":
+                path = path.relative_to("..")
+
             path = path.with_suffix(".o" if replace_ext else path.suffix + ".o")
 
             s += f"    BUILD_DIR/{path}({obj_type});\n"

tools/substitute.py

@@ -8,7 +8,7 @@ from pathlib import Path
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = script_dir + "/../"
 src_dir = root_dir + "src/"
-asm_dir = root_dir + "asm/"
+asm_dir = root_dir + "ver/current/asm/"
 
 parser = argparse.ArgumentParser(description="Replace many functions with one")
 parser.add_argument("from_list", help="path to line-separated file of functions to be replaced. first line is the string to replace them with")

tools/sym_info.py

@@ -24,9 +24,9 @@ parser.add_argument(
 )
 args = parser.parse_args()
 
-mymap = os.path.join(root_dir, "build", "papermario.map")
+mymap = os.path.join(root_dir, "ver", "current" "build", "papermario.map")
 if args.use_expected:
-    mymap = os.path.join(root_dir, "expected", "build", "papermario.map")
+    mymap = os.path.join(root_dir, "ver", "current", "expected", "build", "papermario.map")
 
 if not os.path.isfile(mymap):
     print(f"{mymap} must exist.")

ver/jp/asm/header.s

@@ -0,0 +1,16 @@
+.section .header, "a"
+
+.word 0x80371240       /* PI BSB Domain 1 register */
+.word 0x0000000F       /* Clockrate setting */
+.word 0x80125C00       /* Entrypoint address */
+.word 0x0000144B       /* Revision */
+.word 0x3BA7CDDC       /* Checksum 1 */
+.word 0x464E52A0       /* Checksum 2 */
+.word 0x00000000       /* Unknown 1 */
+.word 0x00000000       /* Unknown 2 */
+.ascii "MARIO STORY         " /* Internal name */
+.word 0x00000000       /* Unknown 3 */
+.word 0x0000004E       /* Cartridge */
+.ascii "MQ"            /* Cartridge ID */
+.ascii "J"             /* Country code */
+.byte 0x00             /* Version */

ver/jp/build.cfg

@@ -0,0 +1,6 @@
+asset_dirs = ver/jp/assets
+maps =
+npc_sprites =
+backgrounds =
+party_images =
+texture_archives =

ver/jp/checksum.sha1

@@ -0,0 +1 @@
+b9cca3ff260b9ff427d981626b82f96de73586d3  ver/jp/build/papermario.z64

ver/jp/splat.yaml

@@ -0,0 +1,23 @@
+name: Mario Story
+basename: papermario
+options:
+  find_file_boundaries: True
+  compiler: GCC
+  mnemonic_ljust: 10
+  ld_o_replace_extension: False
+  ld_addrs_header: build/include/ld_addrs.h
+  extensions: ../../tools/splat_ext
+  symbol_addrs_path: symbol_addrs.txt
+  platform: n64
+  out_dir: .
+  target_path: baserom.z64
+  assets_dir: assets
+segments:
+  - name: header
+    type: header
+    start: 0x00
+    vram: 0
+    subsections:
+    - [0x0000, header, header]
+  - [0x0040, bin]
+  - [0x2800000]