diff --git a/.gitignore b/.gitignore index a0beceb6ef..9f929d2529 100644 --- a/.gitignore +++ b/.gitignore @@ -31,14 +31,9 @@ tools/ido_recomp/* binary ctx.c # Assets -*.rgba32.png -*.rgb5a1.png -*.i4.png -*.i8.png -*.ia4.png -*.ia8.png -*.ci4.png -*.ci8.png +*.png +*.jpg +!*_custom* # Per-user configuration .python-version diff --git a/Makefile b/Makefile index 85d21759be..1a46449ad2 100644 --- a/Makefile +++ b/Makefile @@ -77,10 +77,7 @@ SRC_DIRS := $(shell find src -type d) BASEROM_DIRS := $(shell find baserom -type d 2>/dev/null) COMP_DIRS := $(BASEROM_DIRS:baserom%=comp%) BINARY_DIRS := $(BASEROM_DIRS:baserom%=binary%) -ASSET_XML_DIRS := $(shell find assets/xml* -type d) -ASSET_SRC_DIRS := $(patsubst assets/xml%,assets/src%,$(ASSET_XML_DIRS)) -ASSET_FILES_XML := $(foreach dir,$(ASSET_XML_DIRS),$(wildcard $(dir)/*.xml)) -ASSET_FILES_OUT := $(patsubst assets/xml%,assets/src%,$(patsubst %.xml,%.c,$(ASSET_FILES_XML))) +ASSET_C_FILES := $(shell find assets/ -type f -name "*.c") # Because we may not have disassembled the code files yet, there might not be any assembly files. # Instead, generate a list of assembly files based on what's listed in the linker script. @@ -88,11 +85,18 @@ S_FILES := $(shell grep build/asm ./linker_scripts/code_script.txt | sed 's/\s*b C_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.c)) C_O_FILES := $(C_FILES:%.c=build/%.o) S_O_FILES := $(S_FILES:asm/%.asm=build/asm/%.o) -ASSET_O_FILES := $(ASSET_FILES_OUT:%.c=build/%.o) +ASSET_O_FILES := $(ASSET_C_FILES:%.c=build/%.o) O_FILES := $(C_O_FILES) $(S_O_FILES) $(ASSET_O_FILES) +ASSET_BIN_DIRS := $(shell find assets/* -type d -not -path "assets/xml*") + +TEXTURE_FILES_PNG := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.png)) +TEXTURE_FILES_JPG := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.jpg)) +TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG:.png=.inc.c),build/$f) \ + $(foreach f,$(TEXTURE_FILES_JPG:.jpg=.jpg.inc.c),build/$f) \ + # create build directories -$(shell mkdir -p build/linker_scripts build/asm build/asm/boot build/asm/code build/asm/overlays $(foreach dir,$(BASEROM_DIRS) $(COMP_DIRS) $(BINARY_DIRS) $(SRC_DIRS) $(ASSET_SRC_DIRS),$(shell mkdir -p build/$(dir)))) +$(shell mkdir -p build/linker_scripts build/asm build/asm/boot build/asm/code build/asm/overlays $(foreach dir,$(BASEROM_DIRS) $(COMP_DIRS) $(BINARY_DIRS) $(SRC_DIRS) $(ASSET_BIN_DIRS),$(shell mkdir -p build/$(dir)))) # This file defines `ROM_FILES`, `UNCOMPRESSED_ROM_FILES`, and rules for generating `.yaz0` files ifneq ($(MAKECMDGOALS), clean) @@ -124,12 +128,12 @@ build/src/libultra/voice/%: CC := ./tools/preprocess.py $(CC_OLD) -- $(AS) $(ASF CC := ./tools/preprocess.py $(CC) -- $(AS) $(ASFLAGS) -- -.PHONY: all clean setup diff-init init +.PHONY: all clean setup diff-init init assetclean distclean # make will delete any generated assembly files that are not a prerequisite for anything, so keep it from doing so -.PRECIOUS: asm/%.asm $(ASSET_FILES_OUT) +.PRECIOUS: asm/%.asm .DEFAULT_GOAL := $(UNCOMPRESSED_ROM) -$(UNCOMPRESSED_ROM): $(UNCOMPRESSED_ROM_FILES) +$(UNCOMPRESSED_ROM): $(TEXTURE_FILES_OUT) $(UNCOMPRESSED_ROM_FILES) ./tools/makerom.py ./tables/dmadata_table.txt $@ ifeq ($(COMPARE),1) @md5sum $(UNCOMPRESSED_ROM) @@ -186,16 +190,21 @@ asm/disasm.dep: tables/files.txt tables/functions.txt tables/objects.txt tables/ ./tools/disasm.py -d ./asm -l ./tables/files.txt -f ./tables/functions.txt -o ./tables/objects.txt -v ./tables/variables.txt -v ./tables/vrom_variables.txt || rm $@ clean: - rm -rf $(ROM) $(UNCOMPRESSED_ROM) build asm + $(RM) -rf $(ROM) $(UNCOMPRESSED_ROM) build asm -distclean: clean - rm -rf assets/src baserom/ +assetclean: + $(RM) -r $(ASSET_BIN_DIRS) + $(RM) -r build/assets + +distclean: assetclean clean + $(RM) -r baserom/ setup: git submodule update --init --recursive python3 -m pip install -r requirements.txt $(MAKE) -C tools ./tools/extract_rom.py $(MM_BASEROM) + python3 extract_assets.py diff-init: all rm -rf expected/ @@ -228,11 +237,8 @@ build/src/overlays/%.o: src/overlays/%.c build/%.o: %.c $(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $< -build/assets/src/scenes/%.o: assets/src/scenes/%.c - $(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $< - find assets/src/scenes/ -path "assets/src/scenes/$*_room*.c" -not -path "*.inc.c" | \ - sed 's/\(.*\)\.c/\1/' | \ - xargs -n 1 -r -I'{}' $(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o build/'{}'.o '{}'.c +build/assets/%.o: assets/%.c + $(CC) -I build/ -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $< build/src/libultra/libc/ll.o: src/libultra/libc/ll.c $(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $< @@ -270,11 +276,13 @@ build/linker_scripts/%.ld: linker_scripts/%.txt build/assets/%.d: assets/%.c @$(GCC) $< -Iinclude -I./ -MM -MT 'build/assets/$*.o' > $@ -assets/src/scenes/%.c: assets/xml/scenes/%.xml - $(ZAPD) e -b baserom/assets/scenes -i $< -o $(dir assets/src/scenes/$*) - find $(dir assets/src/scenes/$*) -path "assets/src/scenes/$**.png" | \ - sed 's/\([^\.]*\)\.\([^\.]*\)\.png/$(subst /,\/,$(ZAPD)) btex -tt \2 -i \1.\2.png -o \1.\2.inc.c/' | \ - bash +# Build C files from assets + +build/%.inc.c: %.png + $(ZAPD) btex -eh -tt $(lastword ,$(subst ., ,$(basename $<))) -i $< -o $@ + +build/assets/%.jpg.inc.c: assets/%.jpg + $(ZAPD) bren -eh -i $< -o $@ ifneq ($(MAKECMDGOALS), clean) ifneq ($(MAKECMDGOALS), distclean) diff --git a/assets/.gitignore b/assets/.gitignore index aa850f42a4..7e53529fcc 100644 --- a/assets/.gitignore +++ b/assets/.gitignore @@ -1 +1,6 @@ -src/* \ No newline at end of file +*.bin +*.c +*.h +*.cfg +*.vtx.inc +*.dlist.inc diff --git a/extract_assets.py b/extract_assets.py new file mode 100755 index 0000000000..e8b952253e --- /dev/null +++ b/extract_assets.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +import argparse +from multiprocessing import Pool, cpu_count, Event +import os + +# Returns True if outFile doesn't exists +# or if inFile has been modified after the last modification of outFile +def checkTouchedFile(inFile: str, outFile: str) -> bool: + if not os.path.exists(outFile): + return True + return os.path.getmtime(inFile) > os.path.getmtime(outFile) + +def ExtractFile(xmlPath, basromPath, outputPath, outputSourcePath): + if globalAbort.is_set(): + # Don't extract if another file wasn't extracted properly. + return + + execStr = "tools/ZAPD/ZAPD.out e -eh -i %s -b %s -o %s -osf %s -gsf 1 -rconf tools/ZAPDConfigs/MM/Config.xml" % (xmlPath, basromPath, outputPath, outputSourcePath) + if globalUnaccounted: + execStr += " -wu" + + print(execStr) + exitValue = os.system(execStr) + if exitValue != 0: + globalAbort.set() + print("\n") + print("Error when extracting from file " + xmlPath, file=os.sys.stderr) + print("Aborting...", file=os.sys.stderr) + print("\n") + +def ExtractFunc(fullPath): + *pathList, xmlName = fullPath.split(os.sep) + objectName = os.path.splitext(xmlName)[0] + + outPath = os.path.join("assets", *pathList[2:-1], objectName) + basromPath = os.path.join("baserom", "assets", *pathList[2:-1]) + outSourcePath = outPath + + isScene = fullPath.startswith("assets/xml/scenes/") + if isScene: + objectName += "_scene" + + if not globalForce: + cFile = os.path.join(outPath, objectName + ".c") + hFile = os.path.join(outPath, objectName + ".h") + if not checkTouchedFile(fullPath, cFile) and not checkTouchedFile(fullPath, hFile): + return + + ExtractFile(fullPath, basromPath, outPath, outSourcePath) + +def initializeWorker(force: bool, abort, unaccounted: bool): + global globalForce + global globalAbort + global globalUnaccounted + globalForce = force + globalAbort = abort + globalUnaccounted = unaccounted + +def main(): + parser = argparse.ArgumentParser(description="baserom asset extractor") + parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep") + parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true") + parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true") + args = parser.parse_args() + + abort = Event() + + asset_path = args.single + if asset_path is not None: + # Always force if -s is used. + initializeWorker(True, abort, args.unaccounted) + fullPath = os.path.join("assets", "xml", asset_path + ".xml") + ExtractFunc(fullPath) + else: + xmlFiles = [] + for currentPath, folders, files in os.walk(os.path.join("assets", "xml")): + for file in files: + fullPath = os.path.join(currentPath, file) + if file.endswith(".xml"): + xmlFiles.append(fullPath) + + numCores = cpu_count() + print("Extracting assets with " + str(numCores) + " CPU cores.") + with Pool(numCores, initializer=initializeWorker, initargs=(args.force, abort, args.unaccounted)) as p: + p.map(ExtractFunc, xmlFiles) + + if abort.is_set(): + exit(1) + +if __name__ == "__main__": + main() diff --git a/linker_scripts/code_script.txt b/linker_scripts/code_script.txt index 0a24259854..e08eed699d 100644 --- a/linker_scripts/code_script.txt +++ b/linker_scripts/code_script.txt @@ -7,8 +7,8 @@ RomLocation += . - _##_Name##SegmentStart; \ _##_Name##SegmentEnd = .; -#define DECL_SEG_SCENE(_SceneName) DECL_SEG(_SceneName, build/assets/src/scenes/_SceneName, 0x02000000) -#define DECL_SEG_ROOM(_SceneName, _RoomNum) DECL_SEG(_SceneName##_room_##_RoomNum, build/assets/src/scenes/_SceneName, 0x03000000) +#define DECL_SEG_SCENE(_SceneName) DECL_SEG(_SceneName, build/assets/scenes/_SceneName, 0x02000000) +#define DECL_SEG_ROOM(_SceneName, _RoomNum) DECL_SEG(_SceneName##_room_##_RoomNum, build/assets/scenes/_SceneName, 0x03000000) #define DECL_ACTOR(_ActorName, _File) \ _ovl_##_ActorName##SegmentStart = .; \