Ports over OoT's version config (#1750)

* Port over version_config from OoT

* extract_text

* remove need for calling by python modules

* Remove extract audio comment

* Newline

* pyyaml

Makefile

@@ -160,7 +160,6 @@ SCHC        := $(PYTHON) tools/buildtools/schc.py
 SCHC_FLAGS  :=
 
 # Audio tools
-AUDIO_EXTRACT := $(PYTHON) tools/audio_extraction.py
 SAMPLECONV    := tools/audio/sampleconv/sampleconv
 SBC           := tools/audio/sbc
 SFC           := tools/audio/sfc
@@ -474,6 +473,7 @@ clean:
 assetclean:
 	$(RM) -r $(EXTRACTED_DIR)/assets
 	$(RM) -r $(EXTRACTED_DIR)/text
+	$(RM) -r $(EXTRACTED_DIR)/.extracted-assets.json
 	$(RM) -r $(BUILD_DIR)/assets
 
 distclean: assetclean clean
@@ -489,18 +489,14 @@ venv:
 
 setup:
 	$(MAKE) -C tools
-	$(PYTHON) tools/buildtools/decompress_baserom.py -v $(VERSION)
+	$(PYTHON) tools/decompress_baserom.py -v $(VERSION)
-	$(PYTHON) tools/buildtools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 $(EXTRACTED_DIR)/baserom --dmadata-start `cat $(BASEROM_DIR)/dmadata_start.txt` --dmadata-names $(BASEROM_DIR)/dmadata_names.txt
+	$(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 $(EXTRACTED_DIR)/baserom -v $(VERSION)
-	$(PYTHON) tools/buildtools/extract_yars.py $(EXTRACTED_DIR)/baserom -v $(VERSION)
+	$(PYTHON) tools/extract_yars.py $(EXTRACTED_DIR)/baserom -v $(VERSION)
-
-# TODO this is a temporary rule for testing audio, to be removed
-setup-audio:
-	$(AUDIO_EXTRACT) -o $(EXTRACTED_DIR) -v $(VERSION) --read-xml
 
 assets:
 	$(PYTHON) tools/extract_assets.py $(EXTRACTED_DIR)/baserom $(EXTRACTED_DIR)/assets -j$(N_THREADS) -Z Wno-hardcoded-pointer -v $(VERSION)
-	$(PYTHON) tools/text/msgdis.py $(EXTRACTED_DIR)/baserom $(EXTRACTED_DIR)/text -v $(VERSION)
+	$(PYTHON) tools/extract_text.py $(EXTRACTED_DIR)/baserom $(EXTRACTED_DIR)/text -v $(VERSION)
-	$(AUDIO_EXTRACT) -o $(EXTRACTED_DIR) -v $(VERSION) --read-xml
+	$(PYTHON) tools/extract_audio.py -o $(EXTRACTED_DIR) -v $(VERSION) --read-xml
 
 ## Assembly generation
 disasm:

baseroms/n64-us/config.yml

@@ -0,0 +1,18 @@
+dmadata_start: 0x1A500
+variables:
+  sMessageTableNES: 0x801C6B98
+  sMessageTableCredits: 0x801CFB08
+  gSoundFontTable: 0x801E1180
+  gSequenceFontTable: 0x801E1420
+  gSequenceTable:  0x801E1630
+  gSampleBankTable: 0x801E1E40
+archives:
+  - map_i_static
+  - map_grand_static
+  - item_name_static
+  - map_name_static
+  - icon_item_static_yar
+  - icon_item_24_static_yar
+  - schedule_dma_static_yar
+incbins:
+assets:

baseroms/n64-us/dmadata_start.txt

@@ -1 +0,0 @@
-0x1A500

requirements.txt

@@ -2,6 +2,7 @@
 colorama>=0.4.3
 crunch64>=0.3.1,<1.0.0
 ipl3checksum>=1.2.0,<2.0.0
+pyyaml>=6.0.1,<7.0.0
 
 # disasm
 rabbitizer>=1.3.0,<2.0.0

tools/decompress_baserom.py

@@ -16,7 +16,8 @@ import crunch64
 import ipl3checksum
 import zlib
 
-import dmadata
+from buildtools import dmadata
+from version import version_config
 
 
 def decompress_zlib(data: bytes) -> bytes:
@@ -174,7 +175,9 @@ def main():
 
     uncompressed_path = baserom_dir / "baserom-decompressed.z64"
 
-    dmadata_start = int((baserom_dir / "dmadata_start.txt").read_text(), 16)
+    config = version_config.load_version_config(version)
+
+    dmadata_start = config.dmadata_start
     correct_str_hash = (baserom_dir / "checksum.md5").read_text().split()[0]
 
     if check_existing_rom(uncompressed_path, correct_str_hash):

tools/extract_audio.py

@@ -7,6 +7,8 @@
 
 import argparse
 
+from version import version_config
+
 from audio.extraction.audio_extract import extract_audio_for_version, GameVersionInfo
 from audio.extraction.disassemble_sequence import MMLVersion, SequenceTableSpec, SqSection
 
@@ -20,12 +22,13 @@ if __name__ == '__main__':
 
     version = args.version
 
-    # TODO move these to config yaml
+    config = version_config.load_version_config(version)
-    soundfont_table_code_offset, seq_font_table_code_offset, seq_table_code_offset, sample_bank_table_code_offset = {
+
-        "n64-jp-1.0"     : (0xC97130-0xB5F000, 0xC973C0-0xB5F000, 0xC975D0-0xB5F000, 0xC97DE0-0xB5F000),
+    code_vram = config.dmadata_segments["code"].vram
-        "n64-us"         : (0xC776C0-0xB3C000, 0xC77960-0xB3C000, 0xC77B70-0xB3C000, 0xC78380-0xB3C000),
+    soundfont_table_code_offset = config.variables["gSoundFontTable"] - code_vram
-        "n64-eu-1.1-dbg" : (0xE0F7E0-0xC95000, 0xE0FA80-0xC95000, 0xE0FC90-0xC95000, 0xE104A0-0xC95000),
+    seq_font_table_code_offset = config.variables["gSequenceFontTable"] - code_vram
-    }[version]
+    seq_table_code_offset = config.variables["gSequenceTable"] - code_vram
+    sample_bank_table_code_offset = config.variables["gSampleBankTable"] - code_vram
 
     # List any sequences that are "handwritten", we don't extract these by
     # default as we want these checked in for documentation.

tools/extract_baserom.py

@@ -9,7 +9,9 @@ import argparse
 from pathlib import Path
 import sys
 
-import dmadata
+from buildtools import dmadata
+from version import version_config
+
 
 
 def main():
@@ -25,26 +27,28 @@ def main():
         help="Output directory for segments",
     )
     parser.add_argument(
-        "--dmadata-start",
+        "-v",
-        type=lambda s: int(s, 16),
+        "--version",
-        required=True,
+        help="version to process",
-        help=(
+        default="n64-us",
-            "The dmadata location in the rom, as a hexadecimal offset (e.g. 0x12f70)"
-        ),
     )
     parser.add_argument(
-        "--dmadata-names",
+        "--dmadata-start",
-        type=Path,
+        type=lambda s: int(s, 16),
-        required=True,
+        help=(
-        help="Path to file containing segment names",
+            "Override dmadata location for non-matching ROMs, as a hexadecimal offset (e.g. 0x12F70)"
+        ),
     )
 
     args = parser.parse_args()
 
     rom_data = memoryview(args.rom.read_bytes())
 
-    dma_names = args.dmadata_names.read_text().splitlines()
+    config = version_config.load_version_config(args.version)
-    dma_entries = dmadata.read_dmadata(rom_data, args.dmadata_start)
+    dmadata_start = args.dmadata_start or config.dmadata_start
+    dma_names = config.dmadata_segments.keys()
+
+    dma_entries = dmadata.read_dmadata(rom_data, dmadata_start)
     if len(dma_names) != len(dma_entries):
         print(
             f"Error: expected {len(dma_names)} DMA entries but found {len(dma_entries)} in ROM",

tools/extract_text.py

@@ -0,0 +1,38 @@
+import argparse
+from pathlib import Path
+
+from version import version_config
+from text import msgdis
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract text from the baserom into .h files")
+    parser.add_argument(
+        "baserom_segments_dir",
+        type=Path,
+        help="Directory of uncompressed ROM segments",
+    )
+    parser.add_argument(
+        "output_dir",
+        type=Path,
+        help="Output directory to place files in",
+    )
+    parser.add_argument(
+        "-v",
+        "--version",
+        help="version to process",
+        default="n64-us",
+    )
+    args = parser.parse_args()
+
+    baserom_segments_dir : Path = args.baserom_segments_dir
+    version : str = args.version
+    output_dir : Path = args.output_dir
+
+    config = version_config.load_version_config(version)
+
+    version_info = msgdis.GameVersionInfo(config.dmadata_segments["code"].vram, config.variables)
+
+    msgdis.extract(version_info, baserom_segments_dir, output_dir)
+
+if __name__ == '__main__':
+    main()

tools/extract_yars.py

@@ -21,6 +21,7 @@ import dataclasses
 from pathlib import Path
 import struct
 
+from version import version_config
 
 PRINT_XML = False
 
@@ -121,11 +122,9 @@ def main():
     global PRINT_XML
     PRINT_XML = args.xml
 
-    archivesCsvPath = Path(f"tools/filelists/{version}/archives.csv")
+    config = version_config.load_version_config(version)
 
-    with archivesCsvPath.open() as f:
+    for archiveName in config.archives:
-        for line in f:
-            archiveName = line.strip().split(",")[1]
         archivePath = baseromSegmentsDir / archiveName
 
         extractedPath = Path(str(archivePath) + ".unarchive")

tools/text/msgdis.py

@@ -3,10 +3,20 @@
 #   message_data_static disassembler/decompiler
 #
 
-import argparse, re, struct
+import struct
 from pathlib import Path
 from typing import Callable, Dict, List, Optional, Tuple, TypeVar
 
+from dataclasses import dataclass
+
+@dataclass
+class GameVersionInfo:
+    # Vram location of the code segment
+    code_vram : int
+    # Variables dict from config
+    variables : Dict[str, int]
+
+
 T = TypeVar("T")
 
 item_ids = {
@@ -3054,9 +3064,8 @@ class MessageEntry:
 
         return out
 
-# TODO: Use version_config instead
 def collect_messages(message_tables : List[Optional[MessageTableDesc]], baserom_segments_dir : Path, 
-                     config : Dict[str,int], code_vram : int, code_bin : bytes):
+                     version_info : GameVersionInfo, code_vram : int, code_bin : bytes):
 
     messages : Dict[int,MessageEntry] = {}
 
@@ -3067,7 +3076,7 @@ def collect_messages(message_tables : List[Optional[MessageTableDesc]], baserom_
             continue
 
         baserom_seg = (baserom_segments_dir / desc.seg_name).read_bytes()
-        code_offset = config[desc.table_name] - code_vram
+        code_offset = version_info.variables[desc.table_name] - code_vram
 
         if desc.parent is None:
             # Complete table
@@ -3122,33 +3131,10 @@ def collect_messages(message_tables : List[Optional[MessageTableDesc]], baserom_
 
     return messages
 
-def main():
+def extract(version_info: GameVersionInfo, baserom_segments_dir : Path, output_dir: Path):
-    parser = argparse.ArgumentParser(description="Extract text from the baserom into .h files")
+    output_dir.mkdir(parents=True, exist_ok=True)
-    parser.add_argument(
-        "baserom_segments_dir",
-        type=Path,
-        help="Directory of uncompressed ROM segments",
-    )
-    parser.add_argument(
-        "output_dir",
-        type=Path,
-        help="Output directory to place files in",
-    )
-    parser.add_argument(
-        "-v",
-        "--version",
-        help="version to process",
-        default="n64-us",
-    )
-    args = parser.parse_args()
 
-    baserom_segments_dir : Path = args.baserom_segments_dir
+    code_vram = version_info.code_vram
-    output_dir : Path = args.output_dir
-
-    args.output_dir.mkdir(parents=True, exist_ok=True)
-
-    # TODO: use version config instead to get code vram
-    code_vram = 0x800A5AC0
 
     code_bin = (baserom_segments_dir / "code").read_bytes()
 
@@ -3161,14 +3147,8 @@ def main():
     message_tables[0]  = MessageTableDesc("sMessageTableNES", "message_data_static", nes_decoder, None)
     message_table_staff = MessageTableDesc("sMessageTableCredits", "staff_message_data_static", credits_decoder, None)
 
-    # TODO: use version config instead
+    messages = collect_messages(message_tables, baserom_segments_dir, version_info, code_vram, code_bin)
-    config = {
+    staff_messages = collect_messages([message_table_staff], baserom_segments_dir, version_info, code_vram, code_bin)
-        "sMessageTableNES": 0x801C6B98,
-        "sMessageTableCredits": 0x801CFB08,
-    }
-
-    messages = collect_messages(message_tables, baserom_segments_dir, config, code_vram, code_bin)
-    staff_messages = collect_messages([message_table_staff], baserom_segments_dir, config, code_vram, code_bin)
 
     message_data = []
 
@@ -3184,5 +3164,3 @@ def main():
     (output_dir / "message_data.h").write_text(message_data)
     (output_dir / "message_data_staff.h").write_text(message_data_staff)
 
-if __name__ == "__main__":
-    main()

tools/version/version_config.py

@@ -0,0 +1,100 @@
+# Version-specific configuration for setup and assets extraction
+
+# SPDX-FileCopyrightText: © 2024 ZeldaRET
+# SPDX-License-Identifier: CC0-1.0
+
+from __future__ import annotations
+
+from collections import OrderedDict
+import csv
+import dataclasses
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+
+
+@dataclasses.dataclass
+class VersionConfig:
+    # Version name
+    version: str
+    # ROM offset to start of DMA table
+    dmadata_start: int
+    # DMA segment information, in ROM order
+    dmadata_segments: OrderedDict[str, SegmentInfo]
+    # ROM pieces that are copied directly into the build with .incbin
+    incbins: list[IncbinConfig]
+    # Addresses of important variables needed for asset extraction
+    variables: dict[str, int]
+    # Assets to extract
+    assets: list[AssetConfig]
+    # Archive names to extract
+    archives: list[str]
+
+
+@dataclasses.dataclass
+class SegmentInfo:
+    name: str
+    vram: Optional[int]
+
+
+@dataclasses.dataclass
+class IncbinConfig:
+    name: str
+    segment: str
+    vram: int
+    size: int
+
+
+@dataclasses.dataclass
+class AssetConfig:
+    name: str
+    xml_path: Path
+    start_offset: Optional[int]
+    end_offset: Optional[int]
+
+
+def load_dmadata_segments(version: str) -> OrderedDict[str, SegmentInfo]:
+    segments = OrderedDict()
+    with open(PROJECT_ROOT / f"baseroms/{version}/segments.csv", "r") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            name = row["Name"]
+            vram = int(row["VRAM start"], 16) if row["VRAM start"] else None
+            segments[name] = SegmentInfo(name, vram)
+    return segments
+
+
+def load_version_config(version: str) -> VersionConfig:
+    with open(PROJECT_ROOT / f"baseroms/{version}/config.yml", "r") as f:
+        config = yaml.load(f, Loader=yaml.Loader)
+
+    incbins = []
+    if config["incbins"] is not None:
+        for incbin in config["incbins"]:
+            incbins.append(
+                IncbinConfig(
+                    incbin["name"], incbin["segment"], incbin["vram"], incbin["size"]
+                )
+            )
+
+    assets = []
+    if config["assets"] is not None:
+        for asset in config["assets"]:
+            name = asset["name"]
+            xml_path = Path(asset["xml_path"])
+            start_offset = asset.get("start_offset", None)
+            end_offset = asset.get("end_offset", None)
+            assets.append(AssetConfig(name, xml_path, start_offset, end_offset))
+
+    return VersionConfig(
+        version=version,
+        dmadata_start=config["dmadata_start"],
+        dmadata_segments=load_dmadata_segments(version),
+        incbins=incbins,
+        variables=config["variables"],
+        assets=assets,
+        archives=config["archives"]
+    )