From a1ef3d254c204cfd15cea398a64b04af20c533cd Mon Sep 17 00:00:00 2001 From: Thar0 <17233964+Thar0@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:46:59 +0000 Subject: [PATCH] Partial linking of all spec segments --- Makefile | 48 ++++++++++++++++----------- tools/.gitignore | 2 +- tools/Makefile | 4 +-- tools/fix_bss.py | 16 +++++---- tools/mkldscript.c | 26 ++++----------- tools/{mkovlrules.c => mkspecrules.c} | 23 ++++++++----- tools/spec.c | 10 ++---- tools/spec.h | 2 -- 8 files changed, 64 insertions(+), 67 deletions(-) rename tools/{mkovlrules.c => mkspecrules.c} (72%) diff --git a/Makefile b/Makefile index 92a7ddabbd..b50811b0ee 100644 --- a/Makefile +++ b/Makefile @@ -306,6 +306,8 @@ else ICONV := iconv endif +LD_OFORMAT := $(shell $(LD) --print-output-format) + INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I. -I$(EXTRACTED_DIR) # Check code syntax with host compiler @@ -314,14 +316,14 @@ CHECK_WARNINGS += -Werror=implicit-int -Werror=implicit-function-declaration -We # The `cpp` command behaves differently on macOS (it behaves as if # `-traditional-cpp` was passed) so we use `gcc -E` instead. -CPP := gcc -E -MKLDSCRIPT := tools/mkldscript -MKOVLRULES := tools/mkovlrules -MKDMADATA := tools/mkdmadata -BIN2C := tools/bin2c -N64TEXCONV := tools/assets/n64texconv/n64texconv -FADO := tools/fado/fado.elf -PYTHON ?= $(VENV)/bin/python3 +CPP := gcc -E +MKLDSCRIPT := tools/mkldscript +MKSPECRULES := tools/mkspecrules +MKDMADATA := tools/mkdmadata +BIN2C := tools/bin2c +N64TEXCONV := tools/assets/n64texconv/n64texconv +FADO := tools/fado/fado.elf +PYTHON ?= $(VENV)/bin/python3 # Command to replace $(BUILD_DIR) in some files with the build path. # We can't use the C preprocessor for this because it won't substitute inside string literals. @@ -503,13 +505,13 @@ TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG_EXTRACTED:.png=.inc.c),$(f: $(foreach f,$(TEXTURE_FILES_JPG_EXTRACTED:.jpg=.jpg.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \ $(foreach f,$(TEXTURE_FILES_JPG_COMMITTED:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f) -OVL_SEGMENTS_DIR := $(BUILD_DIR)/segments +SEGMENTS_DIR := $(BUILD_DIR)/segments # create build directories $(shell mkdir -p $(BUILD_DIR)/baserom \ $(BUILD_DIR)/assets/text \ $(BUILD_DIR)/linker_scripts \ - $(OVL_SEGMENTS_DIR)) + $(SEGMENTS_DIR)) $(shell mkdir -p $(foreach dir, \ $(SRC_DIRS) \ $(UNDECOMPILED_DATA_DIRS) \ @@ -529,16 +531,18 @@ $(shell mkdir -p $(foreach dir, \ $(dir:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%))) endif -# Generate and include segment makefile rules for combining overlay .o files into single .plf files, from which -# overlay relocations will be generated. +# Generate and include segment makefile rules for combining .o files into single .plf files for an entire spec segment. +# Overlay relocations will be generated from these if the spec segment has the OVERLAY flag. # If this makefile doesn't exist or if the spec has been modified since make was last ran it will use the rule # later on in the file to regenerate this file before including it. The test against MAKECMDGOALS ensures this # doesn't happen if we're not running a task that needs these partially linked files; this is especially important # for setup since the rule to generate the segment makefile rules requires setup to have ran first. -OVLDFLAGS = -r -T linker_scripts/segment.ld -Map $(@:.plf=.map) +SEG_LDFLAGS = -r -T linker_scripts/segment.ld -Map $(@:.plf=.map) +SEG_VERBOSE = @ ifeq ($(MAKECMDGOALS),$(filter-out clean assetclean distclean setup,$(MAKECMDGOALS))) -include $(OVL_SEGMENTS_DIR)/Makefile +include $(SEGMENTS_DIR)/Makefile else +SEGMENT_FILES := OVL_SEGMENT_FILES := endif OVL_RELOC_FILES := $(OVL_SEGMENT_FILES:.plf=.reloc.o) @@ -851,7 +855,7 @@ ifeq ($(PLATFORM),IQUE) endif endif -$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_SEGMENT_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) \ +$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(SEGMENT_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) \ $(BUILD_DIR)/linker_scripts/makerom.ld $(BUILD_DIR)/undefined_syms.txt \ $(SAMPLEBANK_O_FILES) $(SOUNDFONT_O_FILES) $(SEQUENCE_O_FILES) \ $(BUILD_DIR)/assets/audio/sequence_font_table.o $(BUILD_DIR)/assets/audio/audiobank_padding.o @@ -873,17 +877,17 @@ $(BUILD_DIR)/spec: $(SPEC) $(SPEC_INCLUDES) $(CPP) $(CPPFLAGS) -MD -MP -MF $@.d -MT $@ -I. $< | $(BUILD_DIR_REPLACE) > $@ $(LDSCRIPT): $(BUILD_DIR)/$(SPEC) - $(MKLDSCRIPT) $< $@ $(OVL_SEGMENTS_DIR) + $(MKLDSCRIPT) $< $@ $(SEGMENTS_DIR) # Generates a makefile containing rules for building .plf files # from overlay .o files for every overlay defined in the spec. -$(OVL_SEGMENTS_DIR)/Makefile: $(BUILD_DIR)/$(SPEC) - $(MKOVLRULES) $< $(OVL_SEGMENTS_DIR) $@ +$(SEGMENTS_DIR)/Makefile: $(BUILD_DIR)/$(SPEC) + $(MKSPECRULES) $< $(SEGMENTS_DIR) $@ # Generates relocations for each overlay after partial linking so that the final # link step cannot later insert padding between individual overlay files after # relocations have already been calculated. -$(OVL_SEGMENTS_DIR)/%.reloc.o: $(OVL_SEGMENTS_DIR)/%.plf +$(SEGMENTS_DIR)/%.reloc.o: $(SEGMENTS_DIR)/%.plf $(FADO) $< -n $(notdir $*) -o $(@:.o=.s) $(AS) $(ASFLAGS) $(@:.o=.s) -o $@ @@ -891,7 +895,7 @@ $(BUILD_DIR)/undefined_syms.txt: undefined_syms.txt $(CPP) $(CPPFLAGS) $< > $@ $(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/% - $(OBJCOPY) -I binary -O elf32-big $< $@ + $(OBJCOPY) -I binary -O $(LD_OFORMAT) $< $@ $(BUILD_DIR)/data/%.o: data/%.s $(CPP) $(CPPFLAGS) -MD -MP -MF $(@:.o=.d) -MT $@ -Iinclude $< | $(AS) $(ASFLAGS) -o $@ @@ -1022,6 +1026,10 @@ $(BUILD_DIR)/assets/%.bin.inc.c: $(EXTRACTED_DIR)/assets/%.bin $(BUILD_DIR)/assets/%.jpg.inc.c: $(EXTRACTED_DIR)/assets/%.jpg $(N64TEXCONV) JFIF "" $< $@ +# .text unaccounted linker padding +$(BUILD_DIR)/__pad_text.o: + echo ".text; .fill 0x10" | $(AS) $(ASFLAGS) -o $@ + # Audio AUDIO_BUILD_DEBUG ?= 0 diff --git a/tools/.gitignore b/tools/.gitignore index 3d63f3312c..bd4364866d 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -4,7 +4,7 @@ bin2c makeromfs mkdmadata mkldscript -mkovlrules +mkspecrules preprocess_pragmas reloc_prereq vtxdis diff --git a/tools/Makefile b/tools/Makefile index 30e5066b02..5e50fa0613 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,5 +1,5 @@ CFLAGS := -Wall -Wextra -pedantic -std=gnu99 -g -O2 -PROGRAMS := bin2c makeromfs mkdmadata mkldscript mkovlrules preprocess_pragmas reloc_prereq vtxdis +PROGRAMS := bin2c makeromfs mkdmadata mkldscript mkspecrules preprocess_pragmas reloc_prereq vtxdis UNAME_S := $(shell uname -s) ifeq ($(UNAME_S),Linux) @@ -58,7 +58,7 @@ bin2c_SOURCES := bin2c.c makeromfs_SOURCES := makeromfs.c n64chksum.c util.c mkdmadata_SOURCES := mkdmadata.c spec.c util.c mkldscript_SOURCES := mkldscript.c spec.c util.c -mkovlrules_SOURCES := mkovlrules.c spec.c util.c +mkspecrules_SOURCES := mkspecrules.c spec.c util.c preprocess_pragmas_SOURCES := preprocess_pragmas.c reloc_prereq_SOURCES := reloc_prereq.c spec.c util.c vtxdis_SOURCES := vtxdis.c diff --git a/tools/fix_bss.py b/tools/fix_bss.py index 897a1f70cd..201ee1e94d 100755 --- a/tools/fix_bss.py +++ b/tools/fix_bss.py @@ -283,10 +283,6 @@ def compare_pointers(version: str) -> dict[Path, BssSection]: continue object_file = file.filepath.relative_to(f"build/{version}") - # Hack to handle the combined z_message_z_game_over.o file. - # Fortunately z_game_over has no BSS so we can just analyze z_message instead. - if str(object_file) == "src/code/z_message_z_game_over.o": - object_file = Path("src/code/z_message.o") # c_file = object_file.with_suffix(".c") @@ -294,7 +290,7 @@ def compare_pointers(version: str) -> dict[Path, BssSection]: # not be true if the first BSS variable is not referenced so account for that specifically. if object_file.suffix == ".plf": - # For partially linked overlays, read the map file for the plf to get the + # For partially linked files, read the map file for the plf to get the # object file corresponding to a single source file plf_map = mapfile_parser.mapfile.MapFile() plf_map.readMapFile(file.filepath.with_suffix(".map")) @@ -302,7 +298,10 @@ def compare_pointers(version: str) -> dict[Path, BssSection]: for plf_file in plf_seg: if not plf_file.sectionType == ".bss": continue - c_file = plf_file.filepath.relative_to(f"build/{version}").with_suffix(".c") + object_file = plf_file.filepath.relative_to(f"build/{version}") + if str(object_file) == "src/code/z_message_z_game_over.o": + object_file = Path("src/code/z_message.o") + c_file = object_file.with_suffix(".c") pointers_in_section = [ p @@ -323,6 +322,11 @@ def compare_pointers(version: str) -> dict[Path, BssSection]: bss_sections[c_file] = BssSection(base_start_address, file.vram + plf_file.vram, pointers_in_section) else: + # Hack to handle the combined z_message_z_game_over.o file. + # Fortunately z_game_over has no BSS so we can just analyze z_message instead. + if str(object_file) == "src/code/z_message_z_game_over.o": + object_file = Path("src/code/z_message.o") + c_file = object_file.with_suffix(".c") pointers_in_section = [ diff --git a/tools/mkldscript.c b/tools/mkldscript.c index aecb8ff904..efba2c94ec 100644 --- a/tools/mkldscript.c +++ b/tools/mkldscript.c @@ -12,23 +12,11 @@ struct Segment *g_segments; int g_segmentsCount; -static void write_includes(const struct Segment *seg, FILE *fout, const char *segments_dir, const char *section, - bool linker_pad) +static void write_includes(const struct Segment *seg, FILE *fout, const char *segments_dir, const char *section) { // Note sections contain a suffix wildcard as compilers other than IDO such as GCC may emit sections titled // e.g. .rodata.cstN, .rodata.strN.M, .text.FUNCNAME depending on their settings. - if (seg->flags & FLAG_OVL) { - // For overlays they are already partially linked. - fprintf(fout, " %s/%s.plf (%s*)\n", segments_dir, seg->name, section); - } else { - // For non-overlays, list each include separately. - int i; - for (i = 0; i < seg->includesCount; i++) { - fprintf(fout, " %s (%s*)\n", seg->includes[i].fpath, section); - if (linker_pad && seg->includes[i].linkerPadding != 0) - fprintf(fout, " . += 0x%X;\n", seg->includes[i].linkerPadding); - } - } + fprintf(fout, " %s/%s.plf (%s*)\n", segments_dir, seg->name, section); } static void write_ld_script(FILE *fout, const char *segments_dir) @@ -83,7 +71,7 @@ static void write_ld_script(FILE *fout, const char *segments_dir) // Write .text fprintf(fout, " _%sSegmentTextStart = .;\n", seg->name); - write_includes(seg, fout, segments_dir, ".text", true); + write_includes(seg, fout, segments_dir, ".text"); fprintf(fout, " . = ALIGN(0x10);\n" " _%sSegmentTextEnd = .;\n" " _%sSegmentTextSize = ABSOLUTE( _%sSegmentTextEnd - _%sSegmentTextStart );\n" @@ -91,7 +79,7 @@ static void write_ld_script(FILE *fout, const char *segments_dir) // Write .data fprintf(fout, " _%sSegmentDataStart = .;\n", seg->name); - write_includes(seg, fout, segments_dir, ".data", false); + write_includes(seg, fout, segments_dir, ".data"); fprintf(fout, " . = ALIGN(0x10);\n" " _%sSegmentDataEnd = .;\n" " _%sSegmentDataSize = ABSOLUTE( _%sSegmentDataEnd - _%sSegmentDataStart );\n" @@ -99,7 +87,7 @@ static void write_ld_script(FILE *fout, const char *segments_dir) // Write .rodata fprintf(fout, " _%sSegmentRoDataStart = .;\n", seg->name); - write_includes(seg, fout, segments_dir, ".rodata", false); + write_includes(seg, fout, segments_dir, ".rodata"); fprintf(fout, " . = ALIGN(0x10);\n" " _%sSegmentRoDataEnd = .;\n" " _%sSegmentRoDataSize = ABSOLUTE( _%sSegmentRoDataEnd - _%sSegmentRoDataStart );\n" @@ -145,8 +133,8 @@ static void write_ld_script(FILE *fout, const char *segments_dir) seg->name, seg->name); // Write .bss and COMMON - write_includes(seg, fout, segments_dir, ".bss", false); - write_includes(seg, fout, segments_dir, "COMMON", false); + write_includes(seg, fout, segments_dir, ".bss"); + write_includes(seg, fout, segments_dir, "COMMON"); // End uninitialized data fprintf(fout, " . = ALIGN(8);\n" diff --git a/tools/mkovlrules.c b/tools/mkspecrules.c similarity index 72% rename from tools/mkovlrules.c rename to tools/mkspecrules.c index ec9d3e3544..1238a94128 100644 --- a/tools/mkovlrules.c +++ b/tools/mkspecrules.c @@ -13,24 +13,29 @@ static void write_overlay_rules(FILE *fout, const char *ovls_dir) int i, j; for (i = 0; i < g_segmentsCount; i++) { - if (!(g_segments[i].flags & FLAG_OVL)) - continue; + Segment *seg = &g_segments[i]; /* Write rule for partial linkage of this segment */ - fprintf(fout, "%s/%s.plf:", ovls_dir, g_segments[i].name); - for (j = 0; j < g_segments[i].includesCount; j++) - fprintf(fout, " \\\n\t\t%s", g_segments[i].includes[j].fpath); + fprintf(fout, "%s/%s.plf:", ovls_dir, seg->name); + for (j = 0; j < seg->includesCount; j++) + fprintf(fout, " \\\n\t\t%s", seg->includes[j].fpath); fprintf(fout, "\n" - "\t$(LD) $(OVLDFLAGS) $^ -o $@\n" - "\n"); + "\t@echo Linking \"%s\"\n" + "\t$(SEG_VERBOSE)$(LD) $(SEG_LDFLAGS) $^ -o $@\n" + "\n", seg->name); } /* List every expected plf in a variable */ - fprintf(fout, "OVL_SEGMENT_FILES :="); + fprintf(fout, "SEGMENT_FILES :="); + for (i = 0; i < g_segmentsCount; i++) { + fprintf(fout, " \\\n\t\t%s/%s.plf", ovls_dir, g_segments[i].name); + } + + /* List overlay plfs in a variable */ + fprintf(fout, "\n\nOVL_SEGMENT_FILES :="); for (i = 0; i < g_segmentsCount; i++) { if (!(g_segments[i].flags & FLAG_OVL)) continue; - fprintf(fout, " \\\n\t\t%s/%s.plf", ovls_dir, g_segments[i].name); } fprintf(fout, "\n"); diff --git a/tools/spec.c b/tools/spec.c index 4c079834f3..de85a4d834 100644 --- a/tools/spec.c +++ b/tools/spec.c @@ -143,7 +143,6 @@ static const char *const stmtNames[] = [STMT_romalign] = "romalign", [STMT_stack] = "stack", [STMT_increment] = "increment", - [STMT_pad_text] = "pad_text", }; STMTId get_stmt_id_by_stmt_name(const char *stmtName, int lineNum) { @@ -158,8 +157,8 @@ STMTId get_stmt_id_by_stmt_name(const char *stmtName, int lineNum) { } bool parse_segment_statement(struct Segment *currSeg, STMTId stmt, char* args, int lineNum) { - // ensure no duplicates (except for 'include' or 'pad_text') - if (stmt != STMT_include && stmt != STMT_pad_text && + // ensure no duplicates (except for 'include') + if (stmt != STMT_include && (currSeg->fields & (1 << stmt))) util_fatal_error("line %i: duplicate '%s' statement", lineNum, stmtNames[stmt]); @@ -217,8 +216,6 @@ bool parse_segment_statement(struct Segment *currSeg, STMTId stmt, char* args, i if (!parse_quoted_string(args, &currSeg->includes[currSeg->includesCount - 1].fpath)) util_fatal_error("line %i: invalid filename", lineNum); - - currSeg->includes[currSeg->includesCount - 1].linkerPadding = 0; break; case STMT_increment: if (!parse_number(args, &currSeg->increment)) @@ -227,9 +224,6 @@ bool parse_segment_statement(struct Segment *currSeg, STMTId stmt, char* args, i case STMT_compress: currSeg->compress = true; break; - case STMT_pad_text: - currSeg->includes[currSeg->includesCount - 1].linkerPadding += 0x10; - break; default: fprintf(stderr, "warning: '%s' is not implemented\n", stmtNames[stmt]); break; diff --git a/tools/spec.h b/tools/spec.h index 8c48910430..22d1fc1dc8 100644 --- a/tools/spec.h +++ b/tools/spec.h @@ -19,7 +19,6 @@ typedef enum { STMT_romalign, STMT_stack, STMT_increment, - STMT_pad_text, } STMTId; enum { @@ -33,7 +32,6 @@ enum { struct Include { char* fpath; - int linkerPadding; }; typedef struct Segment {