Update z64compress (#516)

* git subrepo pull (merge) --force tools/z64compress

subrepo:
  subdir:   "tools/z64compress"
  merged:   "5da31326"
upstream:
  origin:   "https://github.com/z64me/z64compress.git"
  branch:   "main"
  commit:   "5da31326"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "2f68596"

* Update wrapper

* If building with NON_MATCHING, do not pass --matching to z64compress

Makefile

@@ -103,6 +103,12 @@ else
   CC_CHECK += -m32
 endif
 
+# rom compression flags
+COMPFLAGS := --threads $(N_THREADS)
+ifneq ($(NON_MATCHING),1)
+	COMPFLAGS += --matching
+endif
+
 #### Files ####
 
 # ROM image
@@ -211,7 +217,7 @@ $(ROM): $(ELF)
 	$(ELF2ROM) -cic 6105 $< $@
 
 $(ROMC): uncompressed
-	python3 tools/z64compress_wrapper.py --mb 32 --matching --threads $(N_THREADS) $(ROM) $@ $(ELF) build/$(SPEC)
+	python3 tools/z64compress_wrapper.py $(COMPFLAGS) $(ROM) $@ $(ELF) build/$(SPEC)
 
 $(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) build/ldscript.txt build/undefined_syms.txt
 	$(LD) -T build/undefined_syms.txt -T build/ldscript.txt --no-check-sections --accept-unknown-input-arch --emit-relocs -Map build/mm.map -o $@

tools/z64compress/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/z64me/z64compress.git
 	branch = main
-	commit = ac5b1a0d0b23346362a68a107da3478227f8e703
-	parent = 33e3aa92181a0543826adc10a5f3304d60b391dd
+	commit = 5da3132606e4fd427a8d72b8428e4f921cd6e56f
+	parent = 837eb1c80687fe9b57a3c7b841547c33feeed6c7
 	method = merge
 	cmdver = 0.4.3

tools/z64compress/Makefile

@@ -1,23 +1,28 @@
 CC := gcc
-CFLAGS := -DNDEBUG -s -Os -flto -Wall -Wextra -march=native -mtune=native
+CFLAGS := -DNDEBUG -s -Os -flto -Wall -Wextra
 
-OBJ_DIR := o
-
-# Target platform, specify with TARGET= on the command line, linux64 is default
+# Target platform, specify with TARGET= on the command line, linux64 is default.
 # Currently supported: linux64, linux32, win32
 TARGET ?= linux64
 
 ifeq ($(TARGET),linux32)
 	TARGET_CFLAGS := -m32
 else ifeq ($(TARGET),win32)
-# If using a cross compiler, specify the compiler executable on the command line
+# If using a cross compiler, specify the compiler executable on the command line.
 # make TARGET=win32 CC=~/c/mxe/usr/bin/i686-w64-mingw32.static-gcc
 	TARGET_LIBS := -mconsole -municode
 else ifneq ($(TARGET),linux64)
 	$(error Supported targets: linux64, linux32, win32)
 endif
 
-OBJ_DIR := $(OBJ_DIR)/$(TARGET)
+# Whether to use native optimizations, specify with NATIVE_OPT=0/1 on the command line, default is 0.
+# This is not supported by all compilers which is particularly an issue on Mac, and may inhibit tests.
+NATIVE_OPT ?= 0
+ifeq ($(NATIVE_OPT),1)
+	TARGET_CFLAGS += -march=native -mtune=native
+endif
+
+OBJ_DIR := o/$(TARGET)
 
 $(OBJ_DIR)/src/enc/%.o: CFLAGS := -DNDEBUG -s -Ofast -flto -Wall
 

tools/z64compress_wrapper.py

@@ -1,15 +1,19 @@
 #!/usr/bin/env python3
 #
 #   z64compress wrapper for decomp projects
-#   Arguments: <rom in> <rom out> <elf> <spec> [cache directory] [num threads]
+#     https://github.com/z64me/z64compress
+#   Arguments: <rom in> <rom out> <elf> <spec> [--cache [cache directory]] [--threads [num threads]] [--mb [target rom size]] [--matching]
+#   Example Makefile usage:
+#     python3 tools/z64compress_wrapper.py --matching --threads $(shell nproc) $< $@ $(ELF) build/$(SPEC)
 #
 
 import argparse, itertools, subprocess, sys
+
 from elftools.elf.elffile import ELFFile
 from elftools.elf.sections import SymbolTableSection
 
 # Args from command line
-parser = argparse.ArgumentParser(description="Compress rom produced by the OoT Decomp project")
+parser = argparse.ArgumentParser(description="Compress rom produced by the OoT and MM Decomp projects")
 
 parser.add_argument("in_rom", help="uncompressed input rom filename")
 parser.add_argument("out_rom", help="compressed output rom filename")
@@ -17,7 +21,7 @@ parser.add_argument("elf", help="path to the uncompressed rom elf file")
 parser.add_argument("spec", help="path to processed spec file")
 parser.add_argument("--cache", help="cache directory")
 parser.add_argument("--threads", help="number of threads to run compression on, 0 disables multithreading")
-parser.add_argument("--mb", help="compressed rom size in MB, default 32")
+parser.add_argument("--mb", help="compressed rom size in MB, default is the smallest multiple of 8mb fitting the whole rom")
 parser.add_argument("--matching", help="matching compression, forfeits some useful optimizations", action="store_true")
 parser.add_argument("--stderr", help="z64compress will write its output messages to stderr instead of stdout", action="store_true")
 
@@ -26,14 +30,13 @@ args = parser.parse_args()
 IN_ROM = args.in_rom
 OUT_ROM = args.out_rom
 
-STDOUT = not args.stderr
-
 elf_path = args.elf
 
 CACHE_DIR = args.cache
 N_THREADS = int(args.threads or 0)
-MB = int(args.mb or 32)
-matching = args.matching
+MB = args.mb
+MATCHING = args.matching
+STDOUT = not args.stderr
 
 # Get segments to compress
 
@@ -83,8 +86,8 @@ def get_dmadata_start_len():
                 if dmadata_start != -1 and dmadata_end != -1:
                     break
 
-        assert dmadata_start != -1
-        assert dmadata_end != -1
+        assert dmadata_start != -1, "_dmadataSegmentRomStart symbol not found in supplied ELF"
+        assert dmadata_end != -1, "_dmadataSegmentRomEnd symbol not found in supplied ELF"
 
     return dmadata_start, (dmadata_end - dmadata_start)//0x10
 
@@ -92,13 +95,21 @@ DMADATA_ADDR, DMADATA_COUNT = get_dmadata_start_len()
 
 # Run
 
-cmd = f"./tools/z64compress/z64compress --in {IN_ROM} --out {OUT_ROM}{' --matching' if matching else ''} \
---mb {MB} --codec yaz{f' --cache {CACHE_DIR}' if CACHE_DIR is not None else ''} --dma 0x{DMADATA_ADDR:X},{DMADATA_COUNT} \
---compress {COMPRESS_INDICES}{f' --threads {N_THREADS}' if N_THREADS > 0 else ''}{f' --only-stdout' if STDOUT else ''}"
+cmd = f"./tools/z64compress/z64compress \
+--in {IN_ROM} \
+--out {OUT_ROM}\
+{' --matching' if MATCHING else ''}\
+{f' --mb {MB}' if MB is not None else ''} \
+--codec yaz\
+{f' --cache {CACHE_DIR}' if CACHE_DIR is not None else ''} \
+--dma 0x{DMADATA_ADDR:X},{DMADATA_COUNT} \
+--compress {COMPRESS_INDICES}\
+{f' --threads {N_THREADS}' if N_THREADS > 0 else ''}\
+{f' --only-stdout' if STDOUT else ''}"
 
 print(cmd)
-
 try:
     subprocess.check_call(cmd, shell=True)
 except subprocess.CalledProcessError as e:
+    # Return the same error code for the wrapper if z64compress fails
     sys.exit(e.returncode)