Configure incbins in version config instead of using the Debug ROM (#2005)

* Run CPP for data/*.s

* Configure incbins in version config instead of using the Debug ROM

Makefile

@@ -170,7 +170,7 @@ endif
 
 CFLAGS += $(GBI_DEFINES)
 
-ASFLAGS := -march=vr4300 -32 -no-pad-sections -Iinclude
+ASFLAGS := -march=vr4300 -32 -no-pad-sections -Iinclude -I$(EXTRACTED_DIR)
 
 ifeq ($(COMPILER),gcc)
   CFLAGS += -G 0 -nostdinc $(INC) -march=vr4300 -mfix4300 -mabi=32 -mno-abicalls -mdivide-breaks -fno-PIC -fno-common -ffreestanding -fbuiltin -fno-builtin-sinf -fno-builtin-cosf $(CHECK_WARNINGS) -funsigned-char
@@ -394,6 +394,7 @@ setup: venv
 	$(MAKE) -C tools
 	$(PYTHON) tools/decompress_baserom.py $(VERSION)
 	$(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 --oot-version $(VERSION) -o $(EXTRACTED_DIR)/baserom
+	$(PYTHON) tools/extract_incbins.py $(EXTRACTED_DIR)/baserom --oot-version $(VERSION) -o $(EXTRACTED_DIR)/incbin
 	$(PYTHON) tools/msgdis.py $(VERSION)
 	$(PYTHON) extract_assets.py -v $(VERSION) -j$(N_THREADS)
 
@@ -448,7 +449,7 @@ $(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/%
 	$(OBJCOPY) -I binary -O elf32-big $< $@
 
 $(BUILD_DIR)/data/%.o: data/%.s
-	$(AS) $(ASFLAGS) $< -o $@
+	$(CPP) $(CPPFLAGS) -Iinclude $< | $(AS) $(ASFLAGS) -o $@
 
 $(BUILD_DIR)/assets/text/%.enc.jpn.h: assets/text/%.h $(EXTRACTED_DIR)/text/%.h assets/text/charmap.txt
 	$(CPP) $(CPPFLAGS) -I$(EXTRACTED_DIR) $< | $(PYTHON) tools/msgenc.py --encoding jpn --charmap assets/text/charmap.txt - $@

baseroms/gc-eu-mq-dbg/config.yml

@@ -1,5 +1,70 @@
 dmadata_start: 0x12F70
 text_lang_pal: true
+incbins:
+  - name: ipl3
+    segment: makerom
+    vram: 0x80000040
+    size: 0xFC0
+  - name: rspbootText
+    segment: boot
+    vram: 0x80009320
+    size: 0xD0
+  - name: aspMainText
+    segment: code
+    vram: 0x801120C0
+    size: 0xFB0
+  - name: gspS2DEX2d_fifoText
+    segment: code
+    vram: 0x80113070
+    size: 0x18C0
+  - name: njpgdspMainText
+    segment: code
+    vram: 0x80114930
+    size: 0xAF0
+  - name: gMojiFontTLUTs
+    segment: code
+    vram: 0x8012A740
+    size: 0x80
+  - name: gMojiFontTex
+    segment: code
+    vram: 0x8012A7C0
+    size: 0x400
+  - name: gSoundFontTable
+    segment: code
+    vram: 0x801550D0
+    size: 0x270
+  - name: gSequenceFontTable
+    segment: code
+    vram: 0x80155340
+    size: 0x1C0
+  - name: gSequenceTable
+    segment: code
+    vram: 0x80155500
+    size: 0x6F0
+  - name: gSampleBankTable
+    segment: code
+    vram: 0x80155BF0
+    size: 0x80
+  - name: aspMainData
+    segment: code
+    vram: 0x80155C70
+    size: 0x2E0
+  - name: gspF3DZEX2_NoN_PosLight_fifoText
+    segment: code
+    vram: 0x80155F50
+    size: 0x1630
+  - name: gspF3DZEX2_NoN_PosLight_fifoData
+    segment: code
+    vram: 0x80157580
+    size: 0x420
+  - name: gspS2DEX2d_fifoData
+    segment: code
+    vram: 0x801579A0
+    size: 0x390
+  - name: njpgdspMainData
+    segment: code
+    vram: 0x80157D30
+    size: 0x60
 variables:
   gMtxClear: 0x8012DB20
   sNesMessageEntryTable: 0x8014B320

baseroms/gc-eu-mq/config.yml

@@ -1,5 +1,62 @@
 dmadata_start: 0x7170
 text_lang_pal: true
+incbins:
+  - name: ipl3
+    segment: makerom
+    vram: 0x80000040
+    size: 0xFC0
+  - name: rspbootText
+    segment: boot
+    vram: 0x80005FC0
+    size: 0xD0
+  - name: aspMainText
+    segment: code
+    vram: 0x800E11A0
+    size: 0xFB0
+  - name: gspS2DEX2d_fifoText
+    segment: code
+    vram: 0x800E2150
+    size: 0x18C0
+  - name: njpgdspMainText
+    segment: code
+    vram: 0x800E3A10
+    size: 0xAF0
+  - name: gSoundFontTable
+    segment: code
+    vram: 0x80110470
+    size: 0x270
+  - name: gSequenceFontTable
+    segment: code
+    vram: 0x801106E0
+    size: 0x1C0
+  - name: gSequenceTable
+    segment: code
+    vram: 0x801108A0
+    size: 0x6F0
+  - name: gSampleBankTable
+    segment: code
+    vram: 0x80110F90
+    size: 0x80
+  - name: aspMainData
+    segment: code
+    vram: 0x80111010
+    size: 0x2E0
+  - name: gspF3DZEX2_NoN_PosLight_fifoText
+    segment: code
+    vram: 0x801112F0
+    size: 0x1630
+  - name: gspF3DZEX2_NoN_PosLight_fifoData
+    segment: code
+    vram: 0x80112920
+    size: 0x420
+  - name: gspS2DEX2d_fifoData
+    segment: code
+    vram: 0x80112D40
+    size: 0x390
+  - name: njpgdspMainData
+    segment: code
+    vram: 0x801130D0
+    size: 0x60
 variables:
   gMtxClear: 0x800FBC00
   sNesMessageEntryTable: 0x801077F0

baseroms/gc-eu/config.yml

@@ -1,5 +1,62 @@
 dmadata_start: 0x7170
 text_lang_pal: true
+incbins:
+  - name: ipl3
+    segment: makerom
+    vram: 0x80000040
+    size: 0xFC0
+  - name: rspbootText
+    segment: boot
+    vram: 0x80005FC0
+    size: 0xD0
+  - name: aspMainText
+    segment: code
+    vram: 0x800E11C0
+    size: 0xFB0
+  - name: gspS2DEX2d_fifoText
+    segment: code
+    vram: 0x800E2170
+    size: 0x18C0
+  - name: njpgdspMainText
+    segment: code
+    vram: 0x800E3A30
+    size: 0xAF0
+  - name: gSoundFontTable
+    segment: code
+    vram: 0x80110490
+    size: 0x270
+  - name: gSequenceFontTable
+    segment: code
+    vram: 0x80110700
+    size: 0x1C0
+  - name: gSequenceTable
+    segment: code
+    vram: 0x801108C0
+    size: 0x6F0
+  - name: gSampleBankTable
+    segment: code
+    vram: 0x80110FB0
+    size: 0x80
+  - name: aspMainData
+    segment: code
+    vram: 0x80111030
+    size: 0x2E0
+  - name: gspF3DZEX2_NoN_PosLight_fifoText
+    segment: code
+    vram: 0x80111310
+    size: 0x1630
+  - name: gspF3DZEX2_NoN_PosLight_fifoData
+    segment: code
+    vram: 0x80112940
+    size: 0x420
+  - name: gspS2DEX2d_fifoData
+    segment: code
+    vram: 0x80112D60
+    size: 0x390
+  - name: njpgdspMainData
+    segment: code
+    vram: 0x801130F0
+    size: 0x60
 variables:
   gMtxClear: 0x800FBC20
   sNesMessageEntryTable: 0x80107810

baseroms/gc-us/config.yml

@@ -1,5 +1,62 @@
 dmadata_start: 0x7170
 text_lang_pal: false
+incbins:
+  - name: ipl3
+    segment: makerom
+    vram: 0x80000040
+    size: 0xFC0
+  - name: rspbootText
+    segment: boot
+    vram: 0x80005FC0
+    size: 0xD0
+  - name: aspMainText
+    segment: code
+    vram: 0x800E3840
+    size: 0xFB0
+  - name: gspS2DEX2d_fifoText
+    segment: code
+    vram: 0x800E47F0
+    size: 0x18C0
+  - name: njpgdspMainText
+    segment: code
+    vram: 0x800E60B0
+    size: 0xAF0
+  - name: gSoundFontTable
+    segment: code
+    vram: 0x80112C80
+    size: 0x270
+  - name: gSequenceFontTable
+    segment: code
+    vram: 0x80112EF0
+    size: 0x1C0
+  - name: gSequenceTable
+    segment: code
+    vram: 0x801130B0
+    size: 0x6F0
+  - name: gSampleBankTable
+    segment: code
+    vram: 0x801137A0
+    size: 0x80
+  - name: aspMainData
+    segment: code
+    vram: 0x80113820
+    size: 0x2E0
+  - name: gspF3DZEX2_NoN_PosLight_fifoText
+    segment: code
+    vram: 0x80113B00
+    size: 0x1630
+  - name: gspF3DZEX2_NoN_PosLight_fifoData
+    segment: code
+    vram: 0x80115130
+    size: 0x420
+  - name: gspS2DEX2d_fifoData
+    segment: code
+    vram: 0x80115550
+    size: 0x390
+  - name: njpgdspMainData
+    segment: code
+    vram: 0x801158E0
+    size: 0x60
 variables:
   gMtxClear: 0x800FE2A0
   sJpnMessageEntryTable: 0x80109E8C

data/audio_tables.rodata.s

@@ -1,22 +1,22 @@
 .include "macro.inc"
 
-# assembler directives
+/* assembler directives */
-.set noat      # allow manual use of $at
+.set noat      /* allow manual use of $at */
-.set noreorder # don't insert nops after branches
+.set noreorder /* don't insert nops after branches */
-.set gp=64     # allow use of 64-bit general purpose registers
+.set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .rodata
 
 .balign 16
 
 glabel gSoundFontTable
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC270, 0x270
+    .incbin "incbin/gSoundFontTable"
 
 glabel gSequenceFontTable
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC4E0, 0x1C0
+    .incbin "incbin/gSequenceFontTable"
 
 glabel gSequenceTable
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC6A0, 0x6F0
+    .incbin "incbin/gSequenceTable"
 
 glabel gSampleBankTable
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCCD90, 0x80
+    .incbin "incbin/gSampleBankTable"

data/rsp.rodata.s

@@ -1,30 +1,30 @@
 .include "macro.inc"
 
-# assembler directives
+/* assembler directives */
-.set noat      # allow manual use of $at
+.set noat      /* allow manual use of $at */
-.set noreorder # don't insert nops after branches
+.set noreorder /* don't insert nops after branches */
-.set gp=64     # allow use of 64-bit general purpose registers
+.set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .rodata
 
 .balign 16
 
 glabel aspMainDataStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCCE10, 0x2E0
+    .incbin "incbin/aspMainData"
 glabel aspMainDataEnd
 
 glabel gspF3DZEX2_NoN_PosLight_fifoTextStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCD0F0, 0x1630
+    .incbin "incbin/gspF3DZEX2_NoN_PosLight_fifoText"
 glabel gspF3DZEX2_NoN_PosLight_fifoTextEnd
 
 glabel gspF3DZEX2_NoN_PosLight_fifoDataStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCE720, 0x420
+    .incbin "incbin/gspF3DZEX2_NoN_PosLight_fifoData"
 glabel gspF3DZEX2_NoN_PosLight_fifoDataEnd
 
 glabel gspS2DEX2d_fifoDataStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCEB40, 0x390
+    .incbin "incbin/gspS2DEX2d_fifoData"
 glabel gspS2DEX2d_fifoDataEnd
 
 glabel njpgdspMainDataStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCEED0, 0x60
+    .incbin "incbin/njpgdspMainData"
 glabel njpgdspMainDataEnd

data/rsp.text.s

@@ -1,22 +1,22 @@
 .include "macro.inc"
 
-# assembler directives
+/* assembler directives */
-.set noat      # allow manual use of $at
+.set noat      /* allow manual use of $at */
-.set noreorder # don't insert nops after branches
+.set noreorder /* don't insert nops after branches */
-.set gp=64     # allow use of 64-bit general purpose registers
+.set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .text
 
 .balign 16
 
 glabel aspMainTextStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB89260, 0xFB0
+    .incbin "incbin/aspMainText"
 glabel aspMainTextEnd
 
 glabel gspS2DEX2d_fifoTextStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB8A210, 0x18C0
+    .incbin "incbin/gspS2DEX2d_fifoText"
 glabel gspS2DEX2d_fifoTextEnd
 
 glabel njpgdspMainTextStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB8BAD0, 0xAF0
+    .incbin "incbin/njpgdspMainText"
 glabel njpgdspMainTextEnd

data/rsp_boot.text.s

@@ -1,14 +1,14 @@
 .include "macro.inc"
 
-# assembler directives
+/* assembler directives */
-.set noat      # allow manual use of $at
+.set noat      /* allow manual use of $at */
-.set noreorder # don't insert nops after branches
+.set noreorder /* don't insert nops after branches */
-.set gp=64     # allow use of 64-bit general purpose registers
+.set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .text
 
 .balign 16
 
 glabel rspbootTextStart
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0x9F20, 0xD0
+    .incbin "incbin/rspbootText"
 glabel rspbootTextEnd

data/unk_800093F0.data.s

@@ -1,17 +1,18 @@
 .include "macro.inc"
 
-# assembler directives
+/* assembler directives */
-.set noat      # allow manual use of $at
+.set noat      /* allow manual use of $at */
-.set noreorder # don't insert nops after branches
+.set noreorder /* don't insert nops after branches */
-.set gp=64     # allow use of 64-bit general purpose registers
+.set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .data
 
 .balign 16
 
-# Unused.
+/* Unused.
-# This appears to be RSP code, however it is not part of
+ * This appears to be RSP code, however it is not part of
-# rspboot as rspbootTextEnd is at this symbol
+ * rspboot as rspbootTextEnd is at this symbol.
+ */
 glabel D_800093F0
     .word 0xE80C2001 # sqv     $v12[0], 0x10($zero)
     .word 0x34014000 # li      $1, 0x4000

data/unk_80009410.data.s

@@ -1,15 +1,15 @@
 .include "macro.inc"
 
-# assembler directives
+/* assembler directives */
-.set noat      # allow manual use of $at
+.set noat      /* allow manual use of $at */
-.set noreorder # don't insert nops after branches
+.set noreorder /* don't insert nops after branches */
-.set gp=64     # allow use of 64-bit general purpose registers
+.set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .data
 
 .balign 16
 
-# Unused
+/* Unused */
 glabel D_80009410
     .word osStopThread
     .word __osSetHWIntrRoutine

data/unk_8012ABC0.data.s

@@ -1,15 +1,15 @@
 .include "macro.inc"
 
-# assembler directives
+/* assembler directives */
-.set noat      # allow manual use of $at
+.set noat      /* allow manual use of $at */
-.set noreorder # don't insert nops after branches
+.set noreorder /* don't insert nops after branches */
-.set gp=64     # allow use of 64-bit general purpose registers
+.set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .data
 
 .balign 16
 
-# Unused
+/* Unused */
 glabel D_8012ABC0
     .word fmodf
     .word guScale

data/z_text.data.s

@@ -1,18 +1,20 @@
 .include "macro.inc"
 
-# assembler directives
+/* assembler directives */
-.set noat      # allow manual use of $at
+.set noat      /* allow manual use of $at */
-.set noreorder # don't insert nops after branches
+.set noreorder /* don't insert nops after branches */
-.set gp=64     # allow use of 64-bit general purpose registers
+.set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .data
 
 .balign 16
 
-# temporary file name, rename to something more appropriate when decompiled
+/* temporary file name, rename to something more appropriate when decompiled */
 
+#if OOT_DEBUG
 glabel gMojiFontTLUTs
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBA18E0, 0x80
+    .incbin "incbin/gMojiFontTLUTs"
 
 glabel gMojiFontTex
-    .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBA1960, 0x400
+    .incbin "incbin/gMojiFontTex"
+#endif

docs/tutorial/data.md

@@ -37,10 +37,10 @@ Large code block, click to show
 ```
 .include "macro.inc"
 
- # assembler directives
+ /* assembler directives */
- .set noat      # allow manual use of $at
+ .set noat      /* allow manual use of $at */
- .set noreorder # don't insert nops after branches
+ .set noreorder /* don't insert nops after branches */
- .set gp=64     # allow use of 64-bit general purpose registers
+ .set gp=64     /* allow use of 64-bit general purpose registers */
 
 .section .data
 

src/makerom/ipl3.s

@@ -1,4 +1,4 @@
 
 .section .text
 
-.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0x40, 0xFC0
+.incbin "incbin/ipl3"

tools/extract_incbins.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: © 2024 ZeldaRET
+# SPDX-License-Identifier: CC0-1.0
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+import dmadata
+import version_config
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract incbin pieces from an uncompressed ROM."
+    )
+    parser.add_argument(
+        "baserom_dir", metavar="BASEROM_DIR", type=Path, help="Directory of uncompressed ROM segments"
+    )
+    parser.add_argument(
+        "-v",
+        "--oot-version",
+        required=True,
+        help="OOT version",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        type=Path,
+        required=True,
+        help="Output directory for incbin pieces",
+    )
+
+    args = parser.parse_args()
+
+    config = version_config.load_version_config(args.oot_version)
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    for incbin in config.incbins:
+        incbin_path = args.output_dir / incbin.name
+        with open(args.baserom_dir / incbin.segment, "rb") as f:
+            offset = incbin.vram - config.dmadata_segments[incbin.segment].vram
+            f.seek(offset)
+            incbin_data = f.read(incbin.size)
+            incbin_path.write_bytes(incbin_data)
+
+    print(f"Extracted {len(config.incbins)} incbin pieces to {args.output_dir}")
+
+
+if __name__ == "__main__":
+    main()

tools/version_config.py

@@ -26,6 +26,8 @@ class VersionConfig:
     text_lang_pal: bool
     # DMA segment information, in ROM order
     dmadata_segments: OrderedDict[str, SegmentInfo]
+    # ROM pieces that are copied directly into the build with .incbin
+    incbins: list[IncbinConfig]
     # Addresses of important variables needed for asset extraction
     variables: dict[str, int]
     # Assets to extract
@@ -38,6 +40,14 @@ class SegmentInfo:
     vram: Optional[int]
 
 
+@dataclasses.dataclass
+class IncbinConfig:
+    name: str
+    segment: str
+    vram: int
+    size: int
+
+
 @dataclasses.dataclass
 class AssetConfig:
     name: str
@@ -61,6 +71,14 @@ def load_version_config(version: str) -> VersionConfig:
     with open(PROJECT_ROOT / f"baseroms/{version}/config.yml", "r") as f:
         config = yaml.load(f, Loader=yaml.Loader)
 
+    incbins = []
+    for incbin in config["incbins"]:
+        incbins.append(
+            IncbinConfig(
+                incbin["name"], incbin["segment"], incbin["vram"], incbin["size"]
+            )
+        )
+
     assets = []
     for asset in config["assets"]:
         name = asset["name"]
@@ -74,6 +92,7 @@ def load_version_config(version: str) -> VersionConfig:
         dmadata_start=config["dmadata_start"],
         text_lang_pal=config["text_lang_pal"],
         dmadata_segments=load_dmadata_segments(version),
+        incbins=incbins,
         variables=config["variables"],
         assets=assets,
     )