Partial linking of all spec segments

Makefile

@@ -306,6 +306,8 @@ else
   ICONV := iconv
 endif
 
+LD_OFORMAT := $(shell $(LD) --print-output-format)
+
 INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I. -I$(EXTRACTED_DIR)
 
 # Check code syntax with host compiler
@@ -314,14 +316,14 @@ CHECK_WARNINGS += -Werror=implicit-int -Werror=implicit-function-declaration -We
 
 # The `cpp` command behaves differently on macOS (it behaves as if
 # `-traditional-cpp` was passed) so we use `gcc -E` instead.
-CPP        := gcc -E
+CPP         := gcc -E
-MKLDSCRIPT := tools/mkldscript
+MKLDSCRIPT  := tools/mkldscript
-MKOVLRULES := tools/mkovlrules
+MKSPECRULES := tools/mkspecrules
-MKDMADATA  := tools/mkdmadata
+MKDMADATA   := tools/mkdmadata
-BIN2C      := tools/bin2c
+BIN2C       := tools/bin2c
-N64TEXCONV := tools/assets/n64texconv/n64texconv
+N64TEXCONV  := tools/assets/n64texconv/n64texconv
-FADO       := tools/fado/fado.elf
+FADO        := tools/fado/fado.elf
-PYTHON     ?= $(VENV)/bin/python3
+PYTHON      ?= $(VENV)/bin/python3
 
 # Command to replace $(BUILD_DIR) in some files with the build path.
 # We can't use the C preprocessor for this because it won't substitute inside string literals.
@@ -503,13 +505,13 @@ TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG_EXTRACTED:.png=.inc.c),$(f:
                      $(foreach f,$(TEXTURE_FILES_JPG_EXTRACTED:.jpg=.jpg.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \
                      $(foreach f,$(TEXTURE_FILES_JPG_COMMITTED:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f)
 
-OVL_SEGMENTS_DIR := $(BUILD_DIR)/segments
+SEGMENTS_DIR := $(BUILD_DIR)/segments
 
 # create build directories
 $(shell mkdir -p $(BUILD_DIR)/baserom \
                  $(BUILD_DIR)/assets/text \
                  $(BUILD_DIR)/linker_scripts \
-				 $(OVL_SEGMENTS_DIR))
+				 $(SEGMENTS_DIR))
 $(shell mkdir -p $(foreach dir, \
                       $(SRC_DIRS) \
                       $(UNDECOMPILED_DATA_DIRS) \
@@ -529,16 +531,18 @@ $(shell mkdir -p $(foreach dir, \
                     $(dir:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)))
 endif
 
-# Generate and include segment makefile rules for combining overlay .o files into single .plf files, from which
+# Generate and include segment makefile rules for combining .o files into single .plf files for an entire spec segment.
-# overlay relocations will be generated.
+# Overlay relocations will be generated from these if the spec segment has the OVERLAY flag.
 # If this makefile doesn't exist or if the spec has been modified since make was last ran it will use the rule
 # later on in the file to regenerate this file before including it. The test against MAKECMDGOALS ensures this
 # doesn't happen if we're not running a task that needs these partially linked files; this is especially important
 # for setup since the rule to generate the segment makefile rules requires setup to have ran first.
-OVLDFLAGS = -r -T linker_scripts/segment.ld -Map $(@:.plf=.map)
+SEG_LDFLAGS = -r -T linker_scripts/segment.ld -Map $(@:.plf=.map)
+SEG_VERBOSE = @
 ifeq ($(MAKECMDGOALS),$(filter-out clean assetclean distclean setup,$(MAKECMDGOALS)))
-include $(OVL_SEGMENTS_DIR)/Makefile
+include $(SEGMENTS_DIR)/Makefile
 else
+SEGMENT_FILES :=
 OVL_SEGMENT_FILES :=
 endif
 OVL_RELOC_FILES := $(OVL_SEGMENT_FILES:.plf=.reloc.o)
@@ -851,7 +855,7 @@ ifeq ($(PLATFORM),IQUE)
   endif
 endif
 
-$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_SEGMENT_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) \
+$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(SEGMENT_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) \
         $(BUILD_DIR)/linker_scripts/makerom.ld $(BUILD_DIR)/undefined_syms.txt \
         $(SAMPLEBANK_O_FILES) $(SOUNDFONT_O_FILES) $(SEQUENCE_O_FILES) \
         $(BUILD_DIR)/assets/audio/sequence_font_table.o $(BUILD_DIR)/assets/audio/audiobank_padding.o
@@ -873,17 +877,17 @@ $(BUILD_DIR)/spec: $(SPEC) $(SPEC_INCLUDES)
 	$(CPP) $(CPPFLAGS) -MD -MP -MF $@.d -MT $@ -I. $< | $(BUILD_DIR_REPLACE) > $@
 
 $(LDSCRIPT): $(BUILD_DIR)/$(SPEC)
-	$(MKLDSCRIPT) $< $@ $(OVL_SEGMENTS_DIR)
+	$(MKLDSCRIPT) $< $@ $(SEGMENTS_DIR)
 
 # Generates a makefile containing rules for building .plf files
 # from overlay .o files for every overlay defined in the spec.
-$(OVL_SEGMENTS_DIR)/Makefile: $(BUILD_DIR)/$(SPEC)
+$(SEGMENTS_DIR)/Makefile: $(BUILD_DIR)/$(SPEC)
-	$(MKOVLRULES) $< $(OVL_SEGMENTS_DIR) $@
+	$(MKSPECRULES) $< $(SEGMENTS_DIR) $@
 
 # Generates relocations for each overlay after partial linking so that the final
 # link step cannot later insert padding between individual overlay files after
 # relocations have already been calculated.
-$(OVL_SEGMENTS_DIR)/%.reloc.o: $(OVL_SEGMENTS_DIR)/%.plf
+$(SEGMENTS_DIR)/%.reloc.o: $(SEGMENTS_DIR)/%.plf
 	$(FADO) $< -n $(notdir $*) -o $(@:.o=.s)
 	$(AS) $(ASFLAGS) $(@:.o=.s) -o $@
 
@@ -891,7 +895,7 @@ $(BUILD_DIR)/undefined_syms.txt: undefined_syms.txt
 	$(CPP) $(CPPFLAGS) $< > $@
 
 $(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/%
-	$(OBJCOPY) -I binary -O elf32-big $< $@
+	$(OBJCOPY) -I binary -O $(LD_OFORMAT) $< $@
 
 $(BUILD_DIR)/data/%.o: data/%.s
 	$(CPP) $(CPPFLAGS) -MD -MP -MF $(@:.o=.d) -MT $@ -Iinclude $< | $(AS) $(ASFLAGS) -o $@
@@ -1022,6 +1026,10 @@ $(BUILD_DIR)/assets/%.bin.inc.c: $(EXTRACTED_DIR)/assets/%.bin
 $(BUILD_DIR)/assets/%.jpg.inc.c: $(EXTRACTED_DIR)/assets/%.jpg
 	$(N64TEXCONV) JFIF "" $< $@
 
+# .text unaccounted linker padding
+$(BUILD_DIR)/__pad_text.o:
+	echo ".text; .fill 0x10" | $(AS) $(ASFLAGS) -o $@
+
 # Audio
 
 AUDIO_BUILD_DEBUG ?= 0

tools/.gitignore

@@ -4,7 +4,7 @@ bin2c
 makeromfs
 mkdmadata
 mkldscript
-mkovlrules
+mkspecrules
 preprocess_pragmas
 reloc_prereq
 vtxdis

tools/Makefile

@@ -1,5 +1,5 @@
 CFLAGS := -Wall -Wextra -pedantic -std=gnu99 -g -O2
-PROGRAMS := bin2c makeromfs mkdmadata mkldscript mkovlrules preprocess_pragmas reloc_prereq vtxdis
+PROGRAMS := bin2c makeromfs mkdmadata mkldscript mkspecrules preprocess_pragmas reloc_prereq vtxdis
 
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Linux)
@@ -58,7 +58,7 @@ bin2c_SOURCES              := bin2c.c
 makeromfs_SOURCES          := makeromfs.c n64chksum.c util.c
 mkdmadata_SOURCES          := mkdmadata.c spec.c util.c
 mkldscript_SOURCES         := mkldscript.c spec.c util.c
-mkovlrules_SOURCES         := mkovlrules.c spec.c util.c
+mkspecrules_SOURCES        := mkspecrules.c spec.c util.c
 preprocess_pragmas_SOURCES := preprocess_pragmas.c
 reloc_prereq_SOURCES       := reloc_prereq.c spec.c util.c
 vtxdis_SOURCES             := vtxdis.c

tools/fix_bss.py

@@ -283,10 +283,6 @@ def compare_pointers(version: str) -> dict[Path, BssSection]:
                 continue
 
             object_file = file.filepath.relative_to(f"build/{version}")
-            # Hack to handle the combined z_message_z_game_over.o file.
-            # Fortunately z_game_over has no BSS so we can just analyze z_message instead.
-            if str(object_file) == "src/code/z_message_z_game_over.o":
-                object_file = Path("src/code/z_message.o")
 
             # c_file = object_file.with_suffix(".c")
 
@@ -294,7 +290,7 @@ def compare_pointers(version: str) -> dict[Path, BssSection]:
             # not be true if the first BSS variable is not referenced so account for that specifically.
 
             if object_file.suffix == ".plf":
-                # For partially linked overlays, read the map file for the plf to get the
+                # For partially linked files, read the map file for the plf to get the
                 # object file corresponding to a single source file
                 plf_map = mapfile_parser.mapfile.MapFile()
                 plf_map.readMapFile(file.filepath.with_suffix(".map"))
@@ -302,7 +298,10 @@ def compare_pointers(version: str) -> dict[Path, BssSection]:
                     for plf_file in plf_seg:
                         if not plf_file.sectionType == ".bss":
                             continue
-                        c_file = plf_file.filepath.relative_to(f"build/{version}").with_suffix(".c")
+                        object_file = plf_file.filepath.relative_to(f"build/{version}")
+                        if str(object_file) == "src/code/z_message_z_game_over.o":
+                            object_file = Path("src/code/z_message.o")
+                        c_file = object_file.with_suffix(".c")
 
                         pointers_in_section = [
                             p
@@ -323,6 +322,11 @@ def compare_pointers(version: str) -> dict[Path, BssSection]:
 
                         bss_sections[c_file] = BssSection(base_start_address, file.vram + plf_file.vram, pointers_in_section)
             else:
+                # Hack to handle the combined z_message_z_game_over.o file.
+                # Fortunately z_game_over has no BSS so we can just analyze z_message instead.
+                if str(object_file) == "src/code/z_message_z_game_over.o":
+                    object_file = Path("src/code/z_message.o")
+
                 c_file = object_file.with_suffix(".c")
 
                 pointers_in_section = [

tools/mkldscript.c

@@ -12,23 +12,11 @@
 struct Segment *g_segments;
 int g_segmentsCount;
 
-static void write_includes(const struct Segment *seg, FILE *fout, const char *segments_dir, const char *section,
+static void write_includes(const struct Segment *seg, FILE *fout, const char *segments_dir, const char *section)
-                           bool linker_pad)
 {
     // Note sections contain a suffix wildcard as compilers other than IDO such as GCC may emit sections titled
     // e.g. .rodata.cstN, .rodata.strN.M, .text.FUNCNAME depending on their settings.
-    if (seg->flags & FLAG_OVL) {
+    fprintf(fout, "            %s/%s.plf (%s*)\n", segments_dir, seg->name, section);
-        // For overlays they are already partially linked.
-        fprintf(fout, "            %s/%s.plf (%s*)\n", segments_dir, seg->name, section);
-    } else {
-        // For non-overlays, list each include separately.
-        int i;
-        for (i = 0; i < seg->includesCount; i++) {
-            fprintf(fout, "            %s (%s*)\n", seg->includes[i].fpath, section);
-            if (linker_pad && seg->includes[i].linkerPadding != 0)
-                fprintf(fout, "            . += 0x%X;\n", seg->includes[i].linkerPadding);
-        }
-    }
 }
 
 static void write_ld_script(FILE *fout, const char *segments_dir)
@@ -83,7 +71,7 @@ static void write_ld_script(FILE *fout, const char *segments_dir)
 
         // Write .text
         fprintf(fout, "        _%sSegmentTextStart = .;\n", seg->name);
-        write_includes(seg, fout, segments_dir, ".text", true);
+        write_includes(seg, fout, segments_dir, ".text");
         fprintf(fout, "        . = ALIGN(0x10);\n"
                       "        _%sSegmentTextEnd = .;\n"
                       "        _%sSegmentTextSize = ABSOLUTE( _%sSegmentTextEnd - _%sSegmentTextStart );\n"
@@ -91,7 +79,7 @@ static void write_ld_script(FILE *fout, const char *segments_dir)
 
         // Write .data
         fprintf(fout, "        _%sSegmentDataStart = .;\n", seg->name);
-        write_includes(seg, fout, segments_dir, ".data", false);
+        write_includes(seg, fout, segments_dir, ".data");
         fprintf(fout, "        . = ALIGN(0x10);\n"
                       "        _%sSegmentDataEnd = .;\n"
                       "        _%sSegmentDataSize = ABSOLUTE( _%sSegmentDataEnd - _%sSegmentDataStart );\n"
@@ -99,7 +87,7 @@ static void write_ld_script(FILE *fout, const char *segments_dir)
 
         // Write .rodata
         fprintf(fout, "        _%sSegmentRoDataStart = .;\n", seg->name);
-        write_includes(seg, fout, segments_dir, ".rodata", false);
+        write_includes(seg, fout, segments_dir, ".rodata");
         fprintf(fout, "        . = ALIGN(0x10);\n"
                       "        _%sSegmentRoDataEnd = .;\n"
                       "        _%sSegmentRoDataSize = ABSOLUTE( _%sSegmentRoDataEnd - _%sSegmentRoDataStart );\n"
@@ -145,8 +133,8 @@ static void write_ld_script(FILE *fout, const char *segments_dir)
                       seg->name, seg->name);
 
         // Write .bss and COMMON
-        write_includes(seg, fout, segments_dir, ".bss", false);
+        write_includes(seg, fout, segments_dir, ".bss");
-        write_includes(seg, fout, segments_dir, "COMMON", false);
+        write_includes(seg, fout, segments_dir, "COMMON");
 
         // End uninitialized data
         fprintf(fout, "        . = ALIGN(8);\n"

tools/mkspecrules.c

@@ -13,24 +13,29 @@ static void write_overlay_rules(FILE *fout, const char *ovls_dir)
     int i, j;
 
     for (i = 0; i < g_segmentsCount; i++) {
-        if (!(g_segments[i].flags & FLAG_OVL))
+        Segment *seg = &g_segments[i];
-            continue;
 
         /* Write rule for partial linkage of this segment */
-        fprintf(fout, "%s/%s.plf:", ovls_dir, g_segments[i].name);
+        fprintf(fout, "%s/%s.plf:", ovls_dir, seg->name);
-        for (j = 0; j < g_segments[i].includesCount; j++)
+        for (j = 0; j < seg->includesCount; j++)
-            fprintf(fout, " \\\n\t\t%s", g_segments[i].includes[j].fpath);
+            fprintf(fout, " \\\n\t\t%s", seg->includes[j].fpath);
         fprintf(fout, "\n"
-                      "\t$(LD) $(OVLDFLAGS) $^ -o $@\n"
+                      "\t@echo Linking \"%s\"\n"
-                      "\n");
+                      "\t$(SEG_VERBOSE)$(LD) $(SEG_LDFLAGS) $^ -o $@\n"
+                      "\n", seg->name);
     }
 
     /* List every expected plf in a variable */
-    fprintf(fout, "OVL_SEGMENT_FILES :=");
+    fprintf(fout, "SEGMENT_FILES :=");
+    for (i = 0; i < g_segmentsCount; i++) {
+        fprintf(fout, " \\\n\t\t%s/%s.plf", ovls_dir, g_segments[i].name);
+    }
+
+    /* List overlay plfs in a variable */
+    fprintf(fout, "\n\nOVL_SEGMENT_FILES :=");
     for (i = 0; i < g_segmentsCount; i++) {
         if (!(g_segments[i].flags & FLAG_OVL))
             continue;
-        
         fprintf(fout, " \\\n\t\t%s/%s.plf", ovls_dir, g_segments[i].name);
     }
     fprintf(fout, "\n");

tools/spec.c

@@ -143,7 +143,6 @@ static const char *const stmtNames[] =
     [STMT_romalign]  = "romalign",
     [STMT_stack]     = "stack",
     [STMT_increment] = "increment",
-    [STMT_pad_text]  = "pad_text",
 };
 
 STMTId get_stmt_id_by_stmt_name(const char *stmtName, int lineNum) {
@@ -158,8 +157,8 @@ STMTId get_stmt_id_by_stmt_name(const char *stmtName, int lineNum) {
 }
 
 bool parse_segment_statement(struct Segment *currSeg, STMTId stmt, char* args, int lineNum) {
-    // ensure no duplicates (except for 'include' or 'pad_text')
+    // ensure no duplicates (except for 'include')
-    if (stmt != STMT_include && stmt != STMT_pad_text &&
+    if (stmt != STMT_include &&
         (currSeg->fields & (1 << stmt)))
         util_fatal_error("line %i: duplicate '%s' statement", lineNum, stmtNames[stmt]);
 
@@ -217,8 +216,6 @@ bool parse_segment_statement(struct Segment *currSeg, STMTId stmt, char* args, i
 
         if (!parse_quoted_string(args, &currSeg->includes[currSeg->includesCount - 1].fpath))
             util_fatal_error("line %i: invalid filename", lineNum);
-
-        currSeg->includes[currSeg->includesCount - 1].linkerPadding = 0;
         break;
     case STMT_increment:
         if (!parse_number(args, &currSeg->increment))
@@ -227,9 +224,6 @@ bool parse_segment_statement(struct Segment *currSeg, STMTId stmt, char* args, i
     case STMT_compress:
         currSeg->compress = true;
         break;
-    case STMT_pad_text:
-        currSeg->includes[currSeg->includesCount - 1].linkerPadding += 0x10;
-        break;
     default:
         fprintf(stderr, "warning: '%s' is not implemented\n", stmtNames[stmt]);
         break;

tools/spec.h

@@ -19,7 +19,6 @@ typedef enum {
     STMT_romalign,
     STMT_stack,
     STMT_increment,
-    STMT_pad_text,
 } STMTId;
 
 enum {
@@ -33,7 +32,6 @@ enum {
 
 struct Include {
     char* fpath;
-    int linkerPadding;
 };
 
 typedef struct Segment {