diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 97a5208789..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "tools/n64splat"] - path = tools/n64splat - url = https://github.com/ethteck/n64splat.git -[submodule "tools/star-rod"] - path = tools/star-rod - url = https://github.com/nanaian/star-rod-for-decomp.git diff --git a/Makefile b/Makefile index f3f45b92f7..f712579516 100644 --- a/Makefile +++ b/Makefile @@ -129,14 +129,9 @@ clean-code: tools: -setup: clean-all submodules tools +setup: clean-all tools @make split -# tools/star-rod submodule intentionally omitted -submodules: - git submodule init tools/n64splat - git submodule update --recursive - split: make $(LD_SCRIPT) -W $(SPLAT_YAML) @@ -296,10 +291,6 @@ test: $(ROM) STAR_ROD := cd tools/star-rod && $(JAVA) -jar StarRod.jar -# lazily initialise the submodule -tools/star-rod: - git submodule init tools/star-rod - sprite/SpriteTable.xml: tools/star-rod sources.mk $(PYTHON) tools/star-rod/spritetable.xml.py $(NPC_SPRITES) > $@ @@ -309,7 +300,7 @@ editor: tools/star-rod sprite/SpriteTable.xml ### Make Settings ### -.PHONY: clean tools test setup submodules split editor $(ROM) +.PHONY: clean tools test setup split editor $(ROM) .DELETE_ON_ERROR: .SECONDARY: .PRECIOUS: $(ROM) %.Yay0 diff --git a/asm/nonmatchings/code_1060_len_310/func_80025C60.s b/asm/nonmatchings/code_1060_len_310/func_80025C60.s deleted file mode 100644 index f57ce2f17f..0000000000 --- a/asm/nonmatchings/code_1060_len_310/func_80025C60.s +++ /dev/null @@ -1,28 +0,0 @@ -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches - -glabel func_80025C60 -/* 1060 80025C60 27BDFFE8 */ addiu $sp, $sp, -0x18 -/* 1064 80025C64 3C05B3FF */ lui $a1, 0xb3ff -/* 1068 80025C68 34A50014 */ ori $a1, $a1, 0x14 -/* 106C 80025C6C AFB00010 */ sw $s0, 0x10($sp) -/* 1070 80025C70 3C10800A */ lui $s0, %hi(carthandle) -/* 1074 80025C74 2610A638 */ addiu $s0, $s0, %lo(carthandle) -/* 1078 80025C78 AFBF0014 */ sw $ra, 0x14($sp) -/* 107C 80025C7C 8E040000 */ lw $a0, ($s0) -/* 1080 80025C80 0C018358 */ jal osEPiWriteIo -/* 1084 80025C84 0000302D */ daddu $a2, $zero, $zero -/* 1088 80025C88 3C05B3FF */ lui $a1, 0xb3ff -/* 108C 80025C8C 34A50004 */ ori $a1, $a1, 4 -/* 1090 80025C90 8E040000 */ lw $a0, ($s0) -/* 1094 80025C94 0C018358 */ jal osEPiWriteIo -/* 1098 80025C98 0000302D */ daddu $a2, $zero, $zero -/* 109C 80025C9C 3C05B3FF */ lui $a1, 0xb3ff -/* 10A0 80025CA0 3C064953 */ lui $a2, 0x4953 -/* 10A4 80025CA4 8E040000 */ lw $a0, ($s0) -/* 10A8 80025CA8 0C018358 */ jal osEPiWriteIo -/* 10AC 80025CAC 34C63634 */ ori $a2, $a2, 0x3634 -/* 10B0 80025CB0 8FBF0014 */ lw $ra, 0x14($sp) -/* 10B4 80025CB4 8FB00010 */ lw $s0, 0x10($sp) -/* 10B8 80025CB8 03E00008 */ jr $ra -/* 10BC 80025CBC 27BD0018 */ addiu $sp, $sp, 0x18 diff --git a/asm/nonmatchings/code_7345A0/func_802A1000_7345A0.s b/asm/nonmatchings/code_7345A0/func_802A1000_7345A0.s deleted file mode 100644 index e0a60022e0..0000000000 --- a/asm/nonmatchings/code_7345A0/func_802A1000_7345A0.s +++ /dev/null @@ -1,25 +0,0 @@ -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches - -glabel func_802A1000_7345A0 -/* 7345A0 802A1000 27BDFFE8 */ addiu $sp, $sp, -0x18 -/* 7345A4 802A1004 AFB00010 */ sw $s0, 0x10($sp) -/* 7345A8 802A1008 0080802D */ daddu $s0, $a0, $zero -/* 7345AC 802A100C 2402000F */ addiu $v0, $zero, 0xf -/* 7345B0 802A1010 3C03800E */ lui $v1, %hi(gBattleStatus+0x83) -/* 7345B4 802A1014 8063C0F3 */ lb $v1, %lo(gBattleStatus+0x83)($v1) -/* 7345B8 802A1018 24040015 */ addiu $a0, $zero, 0x15 -/* 7345BC 802A101C AFBF0014 */ sw $ra, 0x14($sp) -/* 7345C0 802A1020 AE020088 */ sw $v0, 0x88($s0) -/* 7345C4 802A1024 0C03A752 */ jal is_ability_active -/* 7345C8 802A1028 AE030084 */ sw $v1, 0x84($s0) -/* 7345CC 802A102C 10400003 */ beqz $v0, .L802A103C -/* 7345D0 802A1030 24020028 */ addiu $v0, $zero, 0x28 -/* 7345D4 802A1034 AE000084 */ sw $zero, 0x84($s0) -/* 7345D8 802A1038 AE020088 */ sw $v0, 0x88($s0) -.L802A103C: -/* 7345DC 802A103C 8FBF0014 */ lw $ra, 0x14($sp) -/* 7345E0 802A1040 8FB00010 */ lw $s0, 0x10($sp) -/* 7345E4 802A1044 24020002 */ addiu $v0, $zero, 2 -/* 7345E8 802A1048 03E00008 */ jr $ra -/* 7345EC 802A104C 27BD0018 */ addiu $sp, $sp, 0x18 diff --git a/asm/nonmatchings/code_7345A0/func_802A1050_7345F0.s b/asm/nonmatchings/code_7345A0/func_802A1050_7345F0.s deleted file mode 100644 index 00b00787b5..0000000000 --- a/asm/nonmatchings/code_7345A0/func_802A1050_7345F0.s +++ /dev/null @@ -1,26 +0,0 @@ -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches - -glabel func_802A1050_7345F0 -/* 7345F0 802A1050 27BDFFE8 */ addiu $sp, $sp, -0x18 -/* 7345F4 802A1054 AFB00010 */ sw $s0, 0x10($sp) -/* 7345F8 802A1058 0080802D */ daddu $s0, $a0, $zero -/* 7345FC 802A105C 24040016 */ addiu $a0, $zero, 0x16 -/* 734600 802A1060 AFBF0014 */ sw $ra, 0x14($sp) -/* 734604 802A1064 0C03A752 */ jal is_ability_active -/* 734608 802A1068 AE000084 */ sw $zero, 0x84($s0) -/* 73460C 802A106C 10400002 */ beqz $v0, .L802A1078 -/* 734610 802A1070 24020001 */ addiu $v0, $zero, 1 -/* 734614 802A1074 AE020084 */ sw $v0, 0x84($s0) -.L802A1078: -/* 734618 802A1078 3C02800E */ lui $v0, %hi(gBattleStatus) -/* 73461C 802A107C 8C42C070 */ lw $v0, %lo(gBattleStatus)($v0) -/* 734620 802A1080 30421000 */ andi $v0, $v0, 0x1000 -/* 734624 802A1084 10400002 */ beqz $v0, .L802A1090 -/* 734628 802A1088 24020001 */ addiu $v0, $zero, 1 -/* 73462C 802A108C AE020084 */ sw $v0, 0x84($s0) -.L802A1090: -/* 734630 802A1090 8FBF0014 */ lw $ra, 0x14($sp) -/* 734634 802A1094 8FB00010 */ lw $s0, 0x10($sp) -/* 734638 802A1098 24020002 */ addiu $v0, $zero, 2 -/* 73463C 802A109C 03E00008 */ jr $ra diff --git a/include/macros.h b/include/macros.h index 1031703c84..c09358012a 100644 --- a/include/macros.h +++ b/include/macros.h @@ -79,4 +79,6 @@ #define _NS(x, y) x ## _ ## y #define NS(x, y) _NS(x, y) +#define ASCII_TO_U32(a, b, c, d) ((u32)((a << 24) | (b << 16) | (c << 8) | (d << 0))) + #endif diff --git a/src/code_1060_len_310.c b/src/code_1060_len_310.c index 57f3c89846..44a8397aee 100644 --- a/src/code_1060_len_310.c +++ b/src/code_1060_len_310.c @@ -1,6 +1,12 @@ #include "common.h" -INCLUDE_ASM(s32, "code_1060_len_310", func_80025C60); +void func_80025C60(void) { + OSPiHandle** handle = &carthandle; + + osEPiWriteIo(*handle, 0xB3FF0014, 0); + osEPiWriteIo(*handle, 0xB3FF0004, 0); + osEPiWriteIo(*handle, 0xB3FF0000, ASCII_TO_U32('I', 'S', '6', '4')); +} INCLUDE_ASM(s32, "code_1060_len_310", func_80025CC0); diff --git a/src/code_7345A0.c b/src/code_7345A0.c index 6d2b8d9a87..a17ef87937 100644 --- a/src/code_7345A0.c +++ b/src/code_7345A0.c @@ -1,5 +1,27 @@ #include "common.h" -INCLUDE_ASM(s32, "code_7345A0", func_802A1000_7345A0); +ApiStatus func_802A1000_7345A0(ScriptInstance* script, s32 isInitialCall) { + script->varTable[0] = BATTLE_STATUS->unk_83; + script->varTable[1] = 15; -INCLUDE_ASM(s32, "code_7345A0", func_802A1050_7345F0); + if (is_ability_active(Ability_BERSERKER)) { + script->varTable[0] = 0; + script->varTable[1] = 40; + } + + return ApiStatus_DONE2; +} + +ApiStatus func_802A1050_7345F0(ScriptInstance* script, s32 isInitialCall) { + script->varTable[0] = 0; + + if (is_ability_active(Ability_RIGHT_ON)) { + script->varTable[0] = 1; + } + + if (gBattleStatus.flags1 & 0x1000) { + script->varTable[0] = 1; + } + + return ApiStatus_DONE2; +} diff --git a/tools/n64splat b/tools/n64splat deleted file mode 160000 index 41146bdb8f..0000000000 --- a/tools/n64splat +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 41146bdb8f07bf82c7004f141126d6186ce3d43e diff --git a/tools/n64splat/.gitignore b/tools/n64splat/.gitignore new file mode 100644 index 0000000000..921e677191 --- /dev/null +++ b/tools/n64splat/.gitignore @@ -0,0 +1,5 @@ +.idea/ +venv/ +.vscode/ +__pycache__/ +util/Yay0decompress diff --git a/tools/n64splat/.gitrepo b/tools/n64splat/.gitrepo new file mode 100644 index 0000000000..a36d64ec01 --- /dev/null +++ b/tools/n64splat/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme +; +[subrepo] + remote = https://github.com/ethteck/n64splat.git + branch = master + commit = 7574db712ef19ca420904c82d3559e9ac4b8c5f5 + parent = 86760369a5ab977c037c21aebf6f10484570642f + method = merge + cmdver = 0.4.3 diff --git a/tools/n64splat/Makefile b/tools/n64splat/Makefile new file mode 100644 index 0000000000..cca29fc676 --- /dev/null +++ b/tools/n64splat/Makefile @@ -0,0 +1,11 @@ +UTIL_DIR := util + +default: all + +all: Yay0decompress + +Yay0decompress: + gcc $(UTIL_DIR)/Yay0decompress.c -fPIC -shared -O3 -o $(UTIL_DIR)/Yay0decompress + +clean: + rm -f $(UTIL_DIR)/Yay0decompress diff --git a/tools/n64splat/README.md b/tools/n64splat/README.md new file mode 100644 index 0000000000..c76c3d775a --- /dev/null +++ b/tools/n64splat/README.md @@ -0,0 +1,8 @@ +# n64splat +A n64 rom splitting tool to assist with decompilation and modding projects + +For example usage, see https://github.com/ethteck/papermario +The Makefile `setup` target calls n64splat with a config file that you can use for reference. More documentation coming soon. + +### Requirements +Python package requirements can be installed via `pip3 install -r requirements.txt` diff --git a/tools/n64splat/create_config.py b/tools/n64splat/create_config.py new file mode 100755 index 0000000000..cb7bfbc25d --- /dev/null +++ b/tools/n64splat/create_config.py @@ -0,0 +1,65 @@ +#! /usr/bin/env python3 + +from capstone import * +from capstone.mips import * + +import argparse +from util import rominfo +from segtypes.code import N64SegCode + +parser = argparse.ArgumentParser(description="Create a splat config from a rom") +parser.add_argument("rom", help="path to a .z64 rom") + + +def main(rom_path): + rom = rominfo.get_info(rom_path) + basename = rom.name.replace(" ", "").lower() + + header = \ +"""name: {0} ({1}) +basename: {2} +options: + find_file_boundaries: True + compiler: "IDO" +""".format(rom.name.title(), rom.get_country_name(), basename) + + with open(rom_path, "rb") as f: + fbytes = f.read() + + rom_addr = 0x1000 + + md = Cs(CS_ARCH_MIPS, CS_MODE_MIPS64 + CS_MODE_BIG_ENDIAN) + for insn in md.disasm(fbytes[rom_addr:], rom.entry_point): + rom_addr += 4 + + segments = \ +"""segments: + - name: header + type: header + start: 0x0 + vram: 0 + files: + - [0x0, header, header] + - name: boot + type: bin + start: 0x40 + - name: main + type: code + start: 0x1000 + vram: 0x{:X} + files: + - [0x1000, asm] + - type: bin + start: 0x{:X} + - [0x{:X}] +""".format(rom.entry_point, rom_addr, rom.size) + + outstr = header + segments + + outname = rom.name.replace(" ", "").lower() + with open(outname + ".yaml", "w", newline="\n") as f: + f.write(outstr) + +if __name__ == "__main__": + args = parser.parse_args() + main(args.rom) diff --git a/tools/n64splat/list_objects.py b/tools/n64splat/list_objects.py new file mode 100644 index 0000000000..ea37af8ca5 --- /dev/null +++ b/tools/n64splat/list_objects.py @@ -0,0 +1,32 @@ +#! /usr/bin/python3 + +import argparse +import yaml +from pathlib import PurePath + +from split import initialize_segments + +parser = argparse.ArgumentParser(description="List output objects for linker script") +parser.add_argument("config", help="path to a compatible config .yaml file") + +def main(config_path): + # Load config + with open(config_path) as f: + config = yaml.safe_load(f.read()) + + options = config.get("options") + replace_ext = options.get("ld_o_replace_extension", True) + + # Initialize segments + all_segments = initialize_segments(options, config_path, config["segments"]) + + for segment in all_segments: + for subdir, path, obj_type, start in segment.get_ld_files(): + path = PurePath(subdir) / PurePath(path) + path = path.with_suffix(".o" if replace_ext else path.suffix + ".o") + + print(path) + +if __name__ == "__main__": + args = parser.parse_args() + main(args.config) diff --git a/tools/n64splat/requirements.txt b/tools/n64splat/requirements.txt new file mode 100644 index 0000000000..20331e4a2a --- /dev/null +++ b/tools/n64splat/requirements.txt @@ -0,0 +1,5 @@ +PyYAML>=5.3.1,<6 +pypng==0.0.20 +colorama>=0.4.4,<0.5 +python-ranges>=0.1.3,<0.2 +capstone>=4.0.2,<5 \ No newline at end of file diff --git a/tools/n64splat/segtypes/Yay0.py b/tools/n64splat/segtypes/Yay0.py new file mode 100644 index 0000000000..7bc376fec4 --- /dev/null +++ b/tools/n64splat/segtypes/Yay0.py @@ -0,0 +1,25 @@ +import os +from segtypes.segment import N64Segment +from pathlib import Path +from util import Yay0decompress + +class N64SegYay0(N64Segment): + def split(self, rom_bytes, base_path): + out_dir = self.create_parent_dir(base_path + "/bin", self.name) + + path = os.path.join(out_dir, os.path.basename(self.name) + ".bin") + with open(path, "wb") as f: + self.log(f"Decompressing {self.name}...") + compressed_bytes = rom_bytes[self.rom_start : self.rom_end] + decompressed_bytes = Yay0decompress.decompress_yay0(compressed_bytes) + f.write(decompressed_bytes) + self.log(f"Wrote {self.name} to {path}") + + + def get_ld_files(self): + return [("bin", f"{self.name}.Yay0", ".data", self.rom_start)] + + + @staticmethod + def get_default_name(addr): + return "Yay0/{:X}".format(addr) diff --git a/tools/n64splat/segtypes/__init__.py b/tools/n64splat/segtypes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/n64splat/segtypes/bin.py b/tools/n64splat/segtypes/bin.py new file mode 100644 index 0000000000..92ca4eb41c --- /dev/null +++ b/tools/n64splat/segtypes/bin.py @@ -0,0 +1,21 @@ +import os +from segtypes.segment import N64Segment +from pathlib import Path + + +class N64SegBin(N64Segment): + def split(self, rom_bytes, base_path): + out_dir = self.create_split_dir(base_path, "bin") + + bin_path = os.path.join(out_dir, self.name + ".bin") + Path(bin_path).parent.mkdir(parents=True, exist_ok=True) + with open(bin_path, "wb") as f: + f.write(rom_bytes[self.rom_start: self.rom_end]) + self.log(f"Wrote {self.name} to {bin_path}") + + def get_ld_files(self): + return [("bin", f"{self.name}.bin", ".data", self.rom_start)] + + @staticmethod + def get_default_name(addr): + return "bin_{:X}".format(addr) diff --git a/tools/n64splat/segtypes/ci4.py b/tools/n64splat/segtypes/ci4.py new file mode 100644 index 0000000000..92bb5ef69d --- /dev/null +++ b/tools/n64splat/segtypes/ci4.py @@ -0,0 +1,15 @@ +from segtypes.ci8 import N64SegCi8 + +class N64SegCi4(N64SegCi8): + def parse_image(self, data): + img_data = bytearray() + + for i in range(self.width * self.height // 2): + img_data.append(data[i] >> 4) + img_data.append(data[i] & 0xF) + + return img_data + + def max_length(self): + if self.compressed: return None + return self.width * self.height // 2 diff --git a/tools/n64splat/segtypes/ci8.py b/tools/n64splat/segtypes/ci8.py new file mode 100644 index 0000000000..5bf2115d99 --- /dev/null +++ b/tools/n64splat/segtypes/ci8.py @@ -0,0 +1,62 @@ +from segtypes.segment import N64Segment +from segtypes.rgba16 import N64SegRgba16 +import png +import os +from util import Yay0decompress + + +class N64SegCi8(N64SegRgba16): + def __init__(self, segment, next_segment, options): + super().__init__(segment, next_segment, options) + + self.path = None + + def split(self, rom_bytes, base_path): + out_dir = self.create_parent_dir(base_path + "/img", self.name) + self.path = os.path.join(out_dir, os.path.basename(self.name) + ".png") + + data = rom_bytes[self.rom_start: self.rom_end] + if self.compressed: + data = Yay0decompress.decompress_yay0(data) + + self.image = self.parse_image(data) + + def postsplit(self, segments): + palettes = [seg for seg in segments if seg.type == + "palette" and seg.image_name == self.name] + + if len(palettes) == 0: + self.error(f"no palette sibling segment exists\n(hint: add a segment with type 'palette' and name '{self.name}')") + return + + seen_paths = [] + + for pal_seg in palettes: + if pal_seg.path in seen_paths: + self.error(f"palette name '{pal_seg.name}' is not unique") + return + seen_paths.append(pal_seg.path) + + w = png.Writer(self.width, self.height, palette=pal_seg.palette) + + with open(pal_seg.path, "wb") as f: + w.write_array(f, self.image) + self.log(f"Wrote {pal_seg.name} to {pal_seg.path}") + + # canonical version of image (not palette!) data + if self.path not in seen_paths: + w = png.Writer(self.width, self.height, + palette=palettes[0].palette) + + with open(self.path, "wb") as f: + w.write_array(f, self.image) + self.log( + f"No unnamed palette for {self.name}; wrote image data to {self.path}") + + def parse_image(self, data): + return data + + def max_length(self): + if self.compressed: + return None + return self.width * self.height diff --git a/tools/n64splat/segtypes/code.py b/tools/n64splat/segtypes/code.py new file mode 100644 index 0000000000..be4262a443 --- /dev/null +++ b/tools/n64splat/segtypes/code.py @@ -0,0 +1,780 @@ +from re import split +from capstone import * +from capstone.mips import * + +from collections import OrderedDict +from segtypes.segment import N64Segment, parse_segment_name +import os +from pathlib import Path, PurePath +from ranges import Range, RangeDict +import re +import sys +from util import floats + + +STRIP_C_COMMENTS_RE = re.compile( + r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', + re.DOTALL | re.MULTILINE +) + +C_FUNC_RE = re.compile( + r"^(static\s+)?[^\s]+\s+([^\s(]+)\(([^;)]*)\)[^;]+?{", + re.MULTILINE +) + +def strip_c_comments(text): + def replacer(match): + s = match.group(0) + if s.startswith("/"): + return " " + else: + return s + return re.sub(STRIP_C_COMMENTS_RE, replacer, text) + + +def get_funcs_defined_in_c(c_file): + with open(c_file, "r") as f: + text = strip_c_comments(f.read()) + + return set(m.group(2) for m in C_FUNC_RE.finditer(text)) + + +def parse_segment_files(segment, segment_class, seg_start, seg_end, seg_name, seg_vram): + prefix = seg_name if seg_name.endswith("/") else f"{seg_name}_" + + ret = [] + prev_start = -1 + + if "files" in segment: + for i, split_file in enumerate(segment["files"]): + if type(split_file) is dict: + start = split_file["start"] + end = split_file["end"] + name = None if "name" not in split_file else split_file["name"] + subtype = split_file["type"] + else: + start = split_file[0] + end = seg_end if i == len(segment["files"]) - 1 else segment["files"][i + 1][0] + name = None if len(split_file) < 3 else split_file[2] + subtype = split_file[1] + + if start < prev_start: + print(f"Error: Code segment {seg_name} has files out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})") + sys.exit(1) + + if not name: + name = N64SegCode.get_default_name(start) if seg_name == N64SegCode.get_default_name(seg_start) else f"{prefix}{start:X}" + + vram = seg_vram + (start - seg_start) + + fl = {"start": start, "end": end, "name": name, "vram": vram, "subtype": subtype} + + ret.append(fl) + prev_start = start + else: + fl = {"start": seg_start, "end": seg_end, + "name": seg_name, "vram": seg_vram, "subtype": "asm"} + ret.append(fl) + + return ret + + +class N64SegCode(N64Segment): + def __init__(self, segment, next_segment, options): + super().__init__(segment, next_segment, options) + self.files = parse_segment_files(segment, self.__class__, self.rom_start, self.rom_end, self.name, self.vram_addr) + self.is_overlay = segment.get("overlay", False) + self.labels_to_add = set() + self.jtbl_glabels = set() + self.glabels_to_add = set() + self.special_labels = {} + self.undefined_syms_to_add = set() + self.glabels_added = {} + self.all_functions = {} + self.provided_symbols = {} + self.c_labels_to_add = set() + self.ld_section_name = "." + segment.get("ld_name", f"text_{self.rom_start:X}") + self.symbol_ranges = RangeDict() + self.detected_syms = {} + self.reported_file_split = False + self.jtbl_jumps = {} + self.jumptables = {} + + @staticmethod + def get_default_name(addr): + return f"code_{addr:X}" + + def get_func_name(self, addr): + return self.provided_symbols.get(addr, f"func_{addr:X}") + + def get_unique_func_name(self, func_addr, rom_addr): + func_name = self.get_func_name(func_addr) + + if self.is_overlay and (func_addr >= self.vram_addr) and (func_addr <= self.vram_addr + self.rom_end - self.rom_start): + return func_name + "_{:X}".format(rom_addr) + return func_name + + def add_glabel(self, ram_addr, rom_addr): + func = self.get_unique_func_name(ram_addr, rom_addr) + self.glabels_to_add.discard(func) + self.glabels_added[ram_addr] = func + if not self.is_overlay: + self.all_functions[ram_addr] = func + return "glabel " + func + + def get_asm_header(self): + ret = [] + + ret.append(".include \"macro.inc\"") + ret.append("") + ret.append("# assembler directives") + ret.append(".set noat # allow manual use of $at") + ret.append(".set noreorder # don't insert nops after branches") + ret.append(".set gp=64 # allow use of 64-bit general purpose registers") + ret.append("") + ret.append(".section .text, \"ax\"") + ret.append("") + + return ret + + def get_gcc_inc_header(self): + ret = [] + ret.append(".set noat # allow manual use of $at") + ret.append(".set noreorder # don't insert nops after branches") + ret.append("") + + return ret + + @staticmethod + def is_nops(insns): + for insn in insns: + if insn.mnemonic != "nop": + return False + return True + + @staticmethod + def is_branch_insn(mnemonic): + return (mnemonic.startswith("b") and not mnemonic.startswith("binsl") and not mnemonic == "break") or mnemonic == "j" + + def process_insns(self, insns, rom_addr): + ret = OrderedDict() + + func = [] + end_func = False + labels = [] + + # Collect labels + for insn in insns: + if self.is_branch_insn(insn.mnemonic): + op_str_split = insn.op_str.split(" ") + branch_target = op_str_split[-1] + branch_addr = int(branch_target, 0) + labels.append((insn.address, branch_addr)) + + # Main loop + for i, insn in enumerate(insns): + mnemonic = insn.mnemonic + op_str = insn.op_str + func_addr = insn.address if len(func) == 0 else func[0][0].address + + if mnemonic == "move": + # Let's get the actual instruction out + opcode = insn.bytes[3] & 0b00111111 + op_str += ", $zero" + + if opcode == 37: + mnemonic = "or" + elif opcode == 45: + mnemonic = "daddu" + elif opcode == 33: + mnemonic = "addu" + else: + print("INVALID INSTRUCTION " + insn) + elif mnemonic == "jal": + jal_addr = int(op_str, 0) + jump_func = self.get_func_name(jal_addr) + if ( + jump_func.startswith("func_") + and self.is_overlay + and jal_addr >= self.vram_addr + and jal_addr <= (self.vram_addr + self.rom_end - self.rom_start) + ): + func_loc = self.rom_start + jal_addr - self.vram_addr + jump_func += "_{:X}".format(func_loc) + + if jump_func not in self.provided_symbols.values(): + self.glabels_to_add.add(jump_func) + op_str = jump_func + elif self.is_branch_insn(insn.mnemonic): + op_str_split = op_str.split(" ") + branch_target = op_str_split[-1] + branch_target_int = int(branch_target, 0) + label = "" + + if branch_target_int in self.special_labels: + label = self.special_labels[branch_target_int] + else: + self.labels_to_add.add(branch_target_int) + label = ".L" + branch_target[2:].upper() + + op_str = " ".join(op_str_split[:-1] + [label]) + elif mnemonic == "mtc0" or mnemonic == "mfc0": + rd = (insn.bytes[2] & 0xF8) >> 3 + op_str = op_str.split(" ")[0] + " $" + str(rd) + + func.append((insn, mnemonic, op_str, rom_addr)) + rom_addr += 4 + + if mnemonic == "jr": + # Record potential jtbl jumps + if op_str != "$ra": + self.jtbl_jumps[insn.address] = op_str + + keep_going = False + for label in labels: + if (label[0] > insn.address and label[1] <= insn.address) or (label[0] <= insn.address and label[1] > insn.address): + keep_going = True + break + if not keep_going: + end_func = True + continue + + if i < len(insns) - 1 and self.get_func_name(insns[i + 1].address) in self.c_labels_to_add: + end_func = True + + if end_func: + if self.is_nops(insns[i:]) or i < len(insns) - 1 and insns[i + 1].mnemonic != "nop": + end_func = False + ret[func_addr] = func + func = [] + + # Add the last function (or append nops to the previous one) + if not self.is_nops([i[0] for i in func]): + ret[func_addr] = func + else: + next(reversed(ret.values())).extend(func) + + return ret + + def get_file_for_addr(self, addr): + for fl in self.files: + if addr >= fl["vram"] and addr < fl["vram"] + fl["end"] - fl["start"]: + return fl + return None + + def store_symbol_access(self, addr, mnemonic): + # Don't overwrite useful info with addiu + if addr in self.detected_syms and self.detected_syms[addr] != "addiu": + return + + self.detected_syms[addr] = mnemonic + + def get_symbol_name(self, addr, rom_addr, funcs=None): + if funcs and addr in funcs: + return self.get_unique_func_name(addr, rom_addr) + if addr in self.all_functions: + return self.all_functions[addr] # todo clean up funcs vs all_functions + if addr in self.provided_symbols: + return self.provided_symbols[addr] + if addr in self.jumptables: + return f"jtbl_{addr:X}_{rom_addr:X}" + if addr in self.symbol_ranges: + ret = self.symbol_ranges.get(addr) + offset = addr - self.symbol_ranges.getrange(addr).start + if offset != 0: + ret += f"+0x{offset:X}" + return ret + + return f"D_{addr:X}" + + # Determine symbols + def determine_symbols(self, funcs, rom_addr): + ret = {} + + for func_addr in funcs: + func = funcs[func_addr] + func_end_addr = func[-1][0].address + 4 + + possible_jtbl_jumps = [(k, v) for k, v in self.jtbl_jumps.items() if k >= func_addr and k < func_end_addr] + possible_jtbl_jumps.sort(key=lambda x:x[0]) + + for i in range(len(func)): + insn = func[i][0] + + # Ensure the first item in the list is always ahead of where we're looking + while len(possible_jtbl_jumps) > 0 and possible_jtbl_jumps[0][0] < insn.address: + del possible_jtbl_jumps[0] + + if insn.mnemonic == "lui": + op_split = insn.op_str.split(", ") + reg = op_split[0] + + if not op_split[1].startswith("0x"): + continue + + lui_val = int(op_split[1], 0) + if lui_val >= 0x8000: + for j in range(i + 1, min(i + 6, len(func))): + s_insn = func[j][0] + + s_op_split = s_insn.op_str.split(", ") + + if s_insn.mnemonic == "lui" and reg == s_op_split[0]: + break + + if s_insn.mnemonic in ["addiu", "ori"]: + s_reg = s_op_split[-2] + else: + s_reg = s_op_split[-1][s_op_split[-1].rfind("(") + 1: -1] + + if reg == s_reg: + if s_insn.mnemonic not in ["addiu", "lw", "sw", "lh", "sh", "lhu", "lb", "sb", "lbu", "lwc1", "swc1", "ldc1", "sdc1"]: + break + + # Match! + reg_ext = "" + + junk_search = re.search( + r"[\(]", s_op_split[-1]) + if junk_search is not None: + if junk_search.start() == 0: + break + s_str = s_op_split[-1][:junk_search.start()] + reg_ext = s_op_split[-1][junk_search.start():] + else: + s_str = s_op_split[-1] + + symbol_addr = (lui_val * 0x10000) + int(s_str, 0) + symbol_name = self.get_symbol_name(symbol_addr, symbol_addr - next(iter(funcs)) + rom_addr, funcs) + symbol_tag = s_insn.mnemonic + + vram_end = self.vram_addr + self.rom_end - self.rom_start + if symbol_addr > func_addr and symbol_addr < vram_end and len(possible_jtbl_jumps) > 0 and func_end_addr - s_insn.address >= 0x30: + for jump in possible_jtbl_jumps: + if jump[1] == s_op_split[0]: + dist_to_jump = possible_jtbl_jumps[0][0] - s_insn.address + if dist_to_jump <= 16: + symbol_name = f"jtbl_{symbol_addr:X}_{self.ram_to_rom(symbol_addr):X}" + symbol_tag = "jtbl" + self.jumptables[symbol_addr] = (func_addr, func_end_addr) + break + + self.store_symbol_access(symbol_addr, symbol_tag) + symbol_file = self.get_file_for_addr(symbol_addr) + + if not symbol_file or symbol_file["subtype"] == "bin": + if "+" not in symbol_name: + self.undefined_syms_to_add.add((symbol_name, symbol_addr)) + + func[i] += ("%hi({})".format(symbol_name),) + func[j] += ("%lo({}){}".format(symbol_name, reg_ext),) + break + ret[func_addr] = func + return ret + + def add_labels(self, funcs): + ret = {} + + for func in funcs: + func_text = [] + + # Add function glabel + rom_addr = funcs[func][0][3] + func_text.append(self.add_glabel(func, rom_addr)) + + indent_next = False + + mnemonic_ljust = self.options.get("mnemonic_ljust", 11) + rom_addr_padding = self.options.get("rom_address_padding", None) + + for insn in funcs[func]: + insn_addr = insn[0].address + # Add a label if we need one + if insn_addr in self.labels_to_add: + self.labels_to_add.remove(insn_addr) + func_text.append(".L{:X}:".format(insn_addr)) + if insn_addr in self.jtbl_glabels: + func_text.append(f"glabel L{insn_addr:X}_{insn[3]:X}") + + if rom_addr_padding: + rom_str = "{0:0{1}X}".format(insn[3], rom_addr_padding) + else: + rom_str = "{:X}".format(insn[3]) + + asm_comment = "/* {} {:X} {} */".format(rom_str, insn_addr, insn[0].bytes.hex().upper()) + + if len(insn) > 4: + op_str = ", ".join(insn[2].split(", ")[:-1] + [insn[4]]) + else: + op_str = insn[2] + + insn_text = insn[1] + if indent_next: + indent_next = False + insn_text = " " + insn_text + + asm_insn_text = " {}{}".format(insn_text.ljust(mnemonic_ljust), op_str).rstrip() + + func_text.append(asm_comment + asm_insn_text) + + if insn[0].mnemonic != "branch" and insn[0].mnemonic.startswith("b") or insn[0].mnemonic.startswith("j"): + indent_next = True + + ret[func] = (func_text, rom_addr) + + if self.options.get("find_file_boundaries"): + # If this is not the last function in the file + if func != list(funcs.keys())[-1]: + + # Find where the function returns + jr_pos = None + for i, insn in enumerate(reversed(funcs[func])): + if insn[0].mnemonic == "jr" and insn[0].op_str == "$ra": + jr_pos = i + break + + # If there is more than 1 nop after the return + if jr_pos and jr_pos > 1 and self.is_nops([i[0] for i in funcs[func][-jr_pos + 1:]]): + new_file_addr = funcs[func][-1][3] + 4 + if (new_file_addr % 16) == 0: + if not self.reported_file_split: + self.reported_file_split = True + print(f"Segment {self.name}, function at vram {func:X} ends with extra nops, indicating a likely file split.") + print("File split suggestions for this segment will follow in config yaml format:") + print(f" - [0x{new_file_addr:X}, asm]") + + return ret + + def should_run(self): + possible_subtypes = ["c", "asm", "hasm", "bin", "data", "rodata"] + subtypes = set(f["subtype"] for f in self.files) + + return super().should_run() or (st in self.options["modes"] and st in subtypes for st in possible_subtypes) + + def is_valid_ascii(self, bytes): + if len(bytes) < 8: + return False + + num_empty_bytes = 0 + for b in bytes: + if b == 0: + num_empty_bytes += 1 + + empty_ratio = num_empty_bytes / len(bytes) + if empty_ratio > 0.2: + return False + + return True + + def get_symbols_for_file(self, split_file): + vram_start = split_file["vram"] + vram_end = split_file["vram"] + split_file["end"] - split_file["start"] + + return [(s, self.detected_syms[s]) for s in self.detected_syms if s >= vram_start and s <= vram_end] + + def disassemble_symbol(self, sym_bytes, sym_type): + if sym_type == "jtbl": + sym_str = ".word " + else: + sym_str = f".{sym_type} " + + if sym_type == "double": + slen = 8 + elif sym_type in ["float", "word", "jtbl"]: + slen = 4 + elif sym_type == "short": + slen = 2 + else: + slen = 1 + + i = 0 + while i < len(sym_bytes): + adv_amt = min(slen, len(sym_bytes) - i) + bits = int.from_bytes(sym_bytes[i : i + adv_amt], "big") + + if sym_type == "jtbl": + if bits == 0: + byte_str = "0" + else: + rom_addr = self.ram_to_rom(bits) + + if rom_addr: + byte_str = f"L{bits:X}_{rom_addr:X}" + else: + byte_str = f"0x{bits:X}" + else: + byte_str = self.provided_symbols.get(bits, '0x{0:0{1}X}'.format(bits, 2 * slen)) + + if sym_type in ["float", "double"]: + if sym_type == "float": + float_str = floats.format_f32_imm(bits) + elif sym_type == "double": + float_str = floats.format_f64_imm(bits) + + # Fall back to .word if we see weird float values + # todo cut the symbol in half maybe where we see the first nan or something + if "e-" in float_str or "nan" in float_str: + return self.disassemble_symbol(sym_bytes, "word") + else: + byte_str = float_str + + sym_str += byte_str + + i += adv_amt + + if i < len(sym_bytes): + sym_str += ", " + + return sym_str + + def disassemble_data(self, split_file, rom_bytes): + rodata_encountered = split_file["subtype"] == "rodata" + ret = ".include \"macro.inc\"\n\n" + ret += f'.section .{split_file["subtype"]}' + + syms = self.get_symbols_for_file(split_file) + syms.sort(key=lambda x:x[0]) + + if len(syms) == 0: + self.warn("No symbol accesses detected for " + split_file["name"] + "; the output will most likely be an ugly blob") + + # check beginning + if syms[0][0] != split_file["vram"]: + syms.insert(0, (split_file["vram"], None)) + + # add end + vram_end = split_file["vram"] + split_file["end"] - split_file["start"] + if syms[-1][0] != vram_end: + syms.append((vram_end, None)) + + for i in range(len(syms) - 1): + mnemonic = syms[i][1] + start = syms[i][0] + end = syms[i + 1][0] + sym_rom_start = start - split_file["vram"] + split_file["start"] + sym_rom_end = end - split_file["vram"] + split_file["start"] + sym_name = self.get_symbol_name(start, sym_rom_start) + sym_str = f"\n\nglabel {sym_name}\n" + sym_bytes = rom_bytes[sym_rom_start : sym_rom_end] + + # .ascii + if self.is_valid_ascii(sym_bytes) and mnemonic == "addiu": + # mnemonic thing may be too picky, we'll see + try: + ascii_str = sym_bytes.decode("EUC-JP") + ascii_str = ascii_str.replace("\\", "\\\\") + ascii_str = ascii_str.replace("\x00", "\\0") + sym_str += f'.ascii "{ascii_str}"' + ret += sym_str + continue + except: + pass + + # Fallback to raw data + if mnemonic == "jtbl": + stype = "jtbl" + elif len(sym_bytes) % 8 == 0 and mnemonic in ["ldc1", "sdc1"]: + stype = "double" + elif len(sym_bytes) % 4 == 0 and mnemonic in ["addiu", "sw", "lw", "jtbl"]: + stype = "word" + elif len(sym_bytes) % 4 == 0 and mnemonic in ["lwc1", "swc1"]: + stype = "float" + elif len(sym_bytes) % 2 == 0 and mnemonic in ["addiu", "lh", "sh", "lhu"]: + stype = "short" + else: + stype = "byte" + + if not rodata_encountered and mnemonic == "jtbl": + rodata_encountered = True + ret += "\n\n\n.section .rodata" + + sym_str += self.disassemble_symbol(sym_bytes, stype) + ret += sym_str + + ret += "\n" + + return ret + + def get_c_preamble(self): + ret = [] + + preamble = self.options.get("generated_c_preamble", "#include \"common.h\"") + ret.append(preamble) + ret.append("") + + return ret + + def gather_jumptable_labels(self, section_vram, section_rom, rom_bytes): + for jumptable in self.jumptables: + start, end = self.jumptables[jumptable] + rom_offset = section_rom + jumptable - section_vram + + if rom_offset <= 0: + return + + while (rom_offset): + word = rom_bytes[rom_offset : rom_offset + 4] + word_int = int.from_bytes(word, "big") + if word_int >= start and word_int <= end: + self.jtbl_glabels.add(word_int) + else: + break + + rom_offset += 4 + + + def split(self, rom_bytes, base_path): + md = Cs(CS_ARCH_MIPS, CS_MODE_MIPS64 + CS_MODE_BIG_ENDIAN) + md.detail = True + md.skipdata = True + + for split_file in self.files: + file_type = split_file["subtype"] + + if file_type in ["asm", "hasm", "c"]: + if self.type not in self.options["modes"] and "all" not in self.options["modes"]: + continue + + if split_file["start"] == split_file["end"]: + continue + + out_dir = self.create_split_dir(base_path, "asm") + + rom_addr = split_file["start"] + + insns = [insn for insn in md.disasm(rom_bytes[split_file["start"]: split_file["end"]], split_file["vram"])] + + funcs = self.process_insns(insns, rom_addr) + funcs = self.determine_symbols(funcs, rom_addr) + self.gather_jumptable_labels(self.vram_addr, self.rom_start, rom_bytes) + funcs_text = self.add_labels(funcs) + + if file_type == "c": + c_path = os.path.join( + base_path, "src", split_file["name"] + "." + self.get_ext(split_file["subtype"])) + + if os.path.exists(c_path): + defined_funcs = get_funcs_defined_in_c(c_path) + else: + defined_funcs = set() + + out_dir = self.create_split_dir( + base_path, os.path.join("asm", "nonmatchings")) + + for func in funcs_text: + func_name = self.get_unique_func_name( + func, funcs_text[func][1]) + + if func_name not in defined_funcs: + if self.options.get("compiler", "IDO") == "GCC": + out_lines = self.get_gcc_inc_header() + else: + out_lines = [] + out_lines.extend(funcs_text[func][0]) + out_lines.append("") + + outpath = Path(os.path.join( + out_dir, split_file["name"], func_name + ".s")) + outpath.parent.mkdir(parents=True, exist_ok=True) + + with open(outpath, "w", newline="\n") as f: + f.write("\n".join(out_lines)) + self.log(f"Disassembled {func_name} to {outpath}") + + # Creation of c files + if not os.path.exists(c_path): # and some option is enabled + c_lines = self.get_c_preamble() + + for func in funcs_text: + func_name = self.get_unique_func_name(func, funcs_text[func][1]) + if self.options.get("compiler", "IDO") == "GCC": + c_lines.append("INCLUDE_ASM(s32, \"{}\", {});".format(split_file["name"], func_name)) + else: + outpath = Path(os.path.join(out_dir, split_file["name"], func_name + ".s")) + rel_outpath = os.path.relpath(outpath, base_path) + c_lines.append(f"#pragma GLOBAL_ASM(\"{rel_outpath}\")") + c_lines.append("") + + Path(c_path).parent.mkdir(parents=True, exist_ok=True) + with open(c_path, "w") as f: + f.write("\n".join(c_lines)) + print(f"Wrote {split_file['name']} to {c_path}") + + else: + out_lines = self.get_asm_header() + for func in funcs_text: + out_lines.extend(funcs_text[func][0]) + out_lines.append("") + + outpath = Path(os.path.join(out_dir, split_file["name"] + ".s")) + outpath.parent.mkdir(parents=True, exist_ok=True) + + with open(outpath, "w", newline="\n") as f: + f.write("\n".join(out_lines)) + + elif file_type in ["data", "rodata"] and (file_type in self.options["modes"] or "all" in self.options["modes"]): + out_dir = self.create_split_dir(base_path, os.path.join("asm", "data")) + + outpath = Path(os.path.join(out_dir, split_file["name"] + f".{file_type}.s")) + outpath.parent.mkdir(parents=True, exist_ok=True) + + file_text = self.disassemble_data(split_file, rom_bytes) + if file_text: + with open(outpath, "w", newline="\n") as f: + f.write(file_text) + + elif file_type == "bin" and ("bin" in self.options["modes"] or "all" in self.options["modes"]): + out_dir = self.create_split_dir(base_path, "bin") + + bin_path = os.path.join( + out_dir, split_file["name"] + "." + self.get_ext(split_file["subtype"])) + Path(bin_path).parent.mkdir(parents=True, exist_ok=True) + with open(bin_path, "wb") as f: + f.write(rom_bytes[split_file["start"]: split_file["end"]]) + + @staticmethod + def get_subdir(subtype): + if subtype in ["c", ".data", ".rodata", ".bss"]: + return "src" + elif subtype in ["asm", "hasm", "header"]: + return "asm" + return subtype + + @staticmethod + def get_ext(subtype): + if subtype in ["c", ".data", ".rodata", ".bss"]: + return "c" + elif subtype in ["asm", "hasm", "header"]: + return "s" + elif subtype == "bin": + return "bin" + return subtype + + @staticmethod + def get_ld_obj_type(subtype, section_name): + if subtype in "c": + return ".text" + elif subtype in ["bin", ".data", "data"]: + return ".data" + elif subtype in [".rodata", "rodata"]: + return ".rodata" + elif subtype == ".bss": + return ".bss" + return section_name + + def get_ld_files(self): + def transform(split_file): + subdir = self.get_subdir(split_file["subtype"]) + obj_type = self.get_ld_obj_type(split_file["subtype"], ".text") + ext = self.get_ext(split_file['subtype']) + start = split_file["start"] + + return subdir, f"{split_file['name']}.{ext}", obj_type, start + + return [transform(file) for file in self.files] + + def get_ld_section_name(self): + path = PurePath(self.name) + name = path.name if path.name != "" else path.parent + + return f"code_{name}" diff --git a/tools/n64splat/segtypes/header.py b/tools/n64splat/segtypes/header.py new file mode 100644 index 0000000000..9a09a06962 --- /dev/null +++ b/tools/n64splat/segtypes/header.py @@ -0,0 +1,61 @@ +import os +from segtypes.segment import N64Segment +from pathlib import Path +from util import rominfo + +class N64SegHeader(N64Segment): + def should_run(self): + return N64Segment.should_run(self) or "asm" in self.options["modes"] + + @staticmethod + def get_line(typ, data, comment): + if typ == "ascii": + dstr = "\"" + data.decode("ASCII").strip() + "\"" + else: # .word, .byte + dstr = "0x" + data.hex().upper() + + dstr = dstr.ljust(20 - len(typ)) + + return f".{typ} {dstr} /* {comment} */" + + def split(self, rom_bytes, base_path): + out_dir = self.create_split_dir(base_path, "asm") + + encoding = self.options.get("header_encoding", "ASCII") + + header_lines = [] + header_lines.append(f".section .{self.name}, \"a\"\n") + header_lines.append(self.get_line("word", rom_bytes[0x00:0x04], "PI BSB Domain 1 register")) + header_lines.append(self.get_line("word", rom_bytes[0x04:0x08], "Clockrate setting")) + header_lines.append(self.get_line("word", rom_bytes[0x08:0x0C], "Entrypoint address")) + header_lines.append(self.get_line("word", rom_bytes[0x0C:0x10], "Revision")) + header_lines.append(self.get_line("word", rom_bytes[0x10:0x14], "Checksum 1")) + header_lines.append(self.get_line("word", rom_bytes[0x14:0x18], "Checksum 2")) + header_lines.append(self.get_line("word", rom_bytes[0x18:0x1C], "Unknown 1")) + header_lines.append(self.get_line("word", rom_bytes[0x1C:0x20], "Unknown 2")) + header_lines.append(".ascii \"" + rom_bytes[0x20:0x34].decode(encoding).strip().ljust(20) + "\" /* Internal name */") + header_lines.append(self.get_line("word", rom_bytes[0x34:0x38], "Unknown 3")) + header_lines.append(self.get_line("word", rom_bytes[0x38:0x3C], "Cartridge")) + header_lines.append(self.get_line("ascii", rom_bytes[0x3C:0x3E], "Cartridge ID")) + header_lines.append(self.get_line("ascii", rom_bytes[0x3E:0x3F], "Country code")) + header_lines.append(self.get_line("byte", rom_bytes[0x3F:0x40], "Version")) + header_lines.append("") + + s_path = os.path.join(out_dir, self.name + ".s") + Path(s_path).parent.mkdir(parents=True, exist_ok=True) + with open(s_path, "w", newline="\n") as f: + f.write("\n".join(header_lines)) + self.log(f"Wrote {self.name} to {s_path}") + + + def get_ld_section_name(self): + return self.name + + + def get_ld_files(self): + return [("asm", f"{self.name}.s", ".data", self.rom_start)] + + + @staticmethod + def get_default_name(addr): + return "header" diff --git a/tools/n64splat/segtypes/i4.py b/tools/n64splat/segtypes/i4.py new file mode 100644 index 0000000000..0524d83e0a --- /dev/null +++ b/tools/n64splat/segtypes/i4.py @@ -0,0 +1,27 @@ +from segtypes.rgba16 import N64SegRgba16 +import png +from math import ceil + +class N64SegI4(N64SegRgba16): + def png_writer(self): + return png.Writer(self.width, self.height, greyscale = True) + + def parse_image(self, data): + img = bytearray() + + for x, y, i in self.iter_image_indexes(0.5, 1): + b = data[i] + + i1 = (b >> 4) & 0xF + i2 = b & 0xF + + i1 = ceil(0xFF * (i1 / 15)) + i2 = ceil(0xFF * (i2 / 15)) + + img += bytes((i1, i2)) + + return img + + def max_length(self): + if self.compressed: return None + return self.width * self.height // 2 diff --git a/tools/n64splat/segtypes/i8.py b/tools/n64splat/segtypes/i8.py new file mode 100644 index 0000000000..a82741be58 --- /dev/null +++ b/tools/n64splat/segtypes/i8.py @@ -0,0 +1,10 @@ +from segtypes.i4 import N64SegI4 +from math import ceil + +class N64SegI8(N64SegI4): + def parse_image(self, data): + return data + + def max_length(self): + if self.compressed: return None + return self.width * self.height diff --git a/tools/n64splat/segtypes/ia16.py b/tools/n64splat/segtypes/ia16.py new file mode 100644 index 0000000000..7d2b10ad8f --- /dev/null +++ b/tools/n64splat/segtypes/ia16.py @@ -0,0 +1,9 @@ +from segtypes.ia4 import N64SegIa4 + +class N64SegIa8(N64SegIa4): + def parse_image(self, data): + return data + + def max_length(self): + if self.compressed: return None + return self.width * self.height * 2 diff --git a/tools/n64splat/segtypes/ia4.py b/tools/n64splat/segtypes/ia4.py new file mode 100644 index 0000000000..c2eec32bfd --- /dev/null +++ b/tools/n64splat/segtypes/ia4.py @@ -0,0 +1,33 @@ +import os +from segtypes.rgba16 import N64SegRgba16 +import png +from math import ceil + +class N64SegIa4(N64SegRgba16): + def png_writer(self): + return png.Writer(self.width, self.height, greyscale = True, alpha = True) + + def parse_image(self, data): + img = bytearray() + + for x, y, i in self.iter_image_indexes(0.5, 1): + b = data[i] + + h = (b >> 4) & 0xF + l = b & 0xF + + i1 = (h >> 1) & 0xF + a1 = (h & 1) * 0xFF + i1 = ceil(0xFF * (i1 / 7)) + + i2 = (l >> 1) & 0xF + a2 = (l & 1) * 0xFF + i2 = ceil(0xFF * (i2 / 7)) + + img += bytes((i1, a1, i2, a2)) + + return img + + def max_length(self): + if self.compressed: return None + return self.width * self.height // 2 diff --git a/tools/n64splat/segtypes/ia8.py b/tools/n64splat/segtypes/ia8.py new file mode 100644 index 0000000000..b4e3cbec04 --- /dev/null +++ b/tools/n64splat/segtypes/ia8.py @@ -0,0 +1,24 @@ +from segtypes.ia4 import N64SegIa4 +import png +from math import ceil + +class N64SegIa8(N64SegIa4): + def parse_image(self, data): + img = bytearray() + + for x, y, i in self.iter_image_indexes(): + b = data[i] + + i = (b >> 4) & 0xF + a = b & 0xF + + i = ceil(0xFF * (i / 15)) + a = ceil(0xFF * (a / 15)) + + img += bytes((i, a)) + + return img + + def max_length(self): + if self.compressed: return None + return self.width * self.height diff --git a/tools/n64splat/segtypes/palette.py b/tools/n64splat/segtypes/palette.py new file mode 100644 index 0000000000..f20e45c199 --- /dev/null +++ b/tools/n64splat/segtypes/palette.py @@ -0,0 +1,64 @@ +import os +from segtypes.segment import N64Segment +from util.color import unpack_color +from util.iter import iter_in_groups + + +class N64SegPalette(N64Segment): + require_unique_name = False + + def __init__(self, segment, next_segment, options): + super().__init__(segment, next_segment, options) + + # palette segments must be named as one of the following: + # 1) same as the relevant ci4/ci8 segment name (max. 1 palette) + # 2) relevant ci4/ci8 segment name + "." + unique palette name + # 3) unique, referencing the relevant ci4/ci8 segment using `image_name` + self.image_name = segment.get("image_name", self.name.split( + ".")[0]) if type(segment) is dict else self.name.split(".")[0] + + self.compressed = segment.get("compressed", False) if type( + segment) is dict else False + + def should_run(self): + return super().should_run() or ( + "img" in self.options["modes"] or + "ci4" in self.options["modes"] or + "ci8" in self.options["modes"] or + "i4" in self.options["modes"] or + "i8" in self.options["modes"] or + "ia4" in self.options["modes"] or + "ia8" in self.options["modes"] or + "ia16" in self.options["modes"] + ) + + def split(self, rom_bytes, base_path): + out_dir = self.create_parent_dir(base_path + "/img", self.name) + self.path = os.path.join( + out_dir, os.path.basename(self.name) + ".png") + + data = rom_bytes[self.rom_start: self.rom_end] + if self.compressed: + data = Yay0decompress.decompress_yay0(data) + + self.palette = self.parse_palette(data) + + def parse_palette(self, data): + palette = [] + + for a, b in iter_in_groups(data, 2): + palette.append(unpack_color([a, b])) + + return palette + + def max_length(self): + if self.compressed: + return None + return 256 * 2 + + def get_ld_files(self): + ext = f".{self.type}.png" + if self.compressed: + ext += ".Yay0" + + return [("img", f"{self.name}{ext}", ".data", self.rom_start)] diff --git a/tools/n64splat/segtypes/rgba16.py b/tools/n64splat/segtypes/rgba16.py new file mode 100644 index 0000000000..d6dbb097eb --- /dev/null +++ b/tools/n64splat/segtypes/rgba16.py @@ -0,0 +1,86 @@ +import os +from segtypes.segment import N64Segment +from pathlib import Path +from util import Yay0decompress +import png +from math import ceil +from util.color import unpack_color + + +class N64SegRgba16(N64Segment): + def __init__(self, segment, next_segment, options): + super().__init__(segment, next_segment, options) + + if type(segment) is dict: + self.compressed = segment.get("compressed", False) + self.width = segment["width"] + self.height = segment["height"] + self.flip = segment.get("flip", "noflip") + elif len(segment) < 5: + self.error("missing parameters") + else: + self.compressed = False + self.width = segment[3] + self.height = segment[4] + self.flip = "noflip" + + @property + def flip_vertical(self): + return self.flip == "both" or self.flip.startswith("v") or self.flip == "y" + + @property + def flip_horizontal(self): + return self.flip == "both" or self.flip.startswith("h") or self.flip == "x" + + def iter_image_indexes(self, bytes_per_x=1, bytes_per_y=1): + w = int(self.width * bytes_per_x) + h = int(self.height * bytes_per_y) + + xrange = range(w - ceil(bytes_per_x), -1, -ceil(bytes_per_x) + ) if self.flip_horizontal else range(0, w, ceil(bytes_per_x)) + yrange = range(h - ceil(bytes_per_y), -1, -ceil(bytes_per_y) + ) if self.flip_vertical else range(0, h, ceil(bytes_per_y)) + + for y in yrange: + for x in xrange: + yield x, y, (y * w) + x + + def should_run(self): + return super().should_run() or "img" in self.options["modes"] + + def split(self, rom_bytes, base_path): + out_dir = self.create_parent_dir(base_path + "/img", self.name) + path = os.path.join(out_dir, os.path.basename(self.name) + ".png") + + data = rom_bytes[self.rom_start: self.rom_end] + if self.compressed: + data = Yay0decompress.decompress_yay0(data) + + w = self.png_writer() + with open(path, "wb") as f: + w.write_array(f, self.parse_image(data)) + + self.log(f"Wrote {self.name} to {path}") + + def png_writer(self): + return png.Writer(self.width, self.height, greyscale=False, alpha=True) + + def parse_image(self, data): + img = bytearray() + + for x, y, i in self.iter_image_indexes(2, 1): + img += bytes(unpack_color(data[i:])) + + return img + + def max_length(self): + if self.compressed: + return None + return self.width * self.height * 2 + + def get_ld_files(self): + ext = f".{self.type}.png" + if self.compressed: + ext += ".Yay0" + + return [("img", f"{self.name}{ext}", ".data", self.rom_start)] diff --git a/tools/n64splat/segtypes/rgba32.py b/tools/n64splat/segtypes/rgba32.py new file mode 100644 index 0000000000..009feb614e --- /dev/null +++ b/tools/n64splat/segtypes/rgba32.py @@ -0,0 +1,9 @@ +from segtypes.rgba16 import N64SegRgba16 + +class N64SegRgba32(N64SegRgba16): + def parse_image(self, data): + return data + + def max_length(self): + if self.compressed: return None + return self.width * self.height * 4 diff --git a/tools/n64splat/segtypes/segment.py b/tools/n64splat/segtypes/segment.py new file mode 100644 index 0000000000..f7cc819035 --- /dev/null +++ b/tools/n64splat/segtypes/segment.py @@ -0,0 +1,188 @@ +import os +from pathlib import Path, PurePath +import re +import json +from util import log + +default_subalign = 16 + + +def parse_segment_start(segment): + return segment[0] if "start" not in segment else segment["start"] + + +def parse_segment_type(segment): + if type(segment) is dict: + return segment["type"] + else: + return segment[1] + + +def parse_segment_name(segment, segment_class): + if type(segment) is dict and "name" in segment: + return segment["name"] + elif type(segment) is list and len(segment) >= 3 and type(segment[2]) is str: + return segment[2] + else: + return segment_class.get_default_name(parse_segment_start(segment)) + + +def parse_segment_vram(segment): + if type(segment) is dict: + return segment.get("vram", 0) + else: + if len(segment) >= 3 and type(segment[-1]) is int: + return segment[-1] + else: + return 0 + + +def parse_segment_subalign(segment): + if type(segment) is dict: + return segment.get("subalign", default_subalign) + return default_subalign + + +class N64Segment: + require_unique_name = True + + def __init__(self, segment, next_segment, options): + self.rom_start = parse_segment_start(segment) + self.rom_end = parse_segment_start(next_segment) + self.type = parse_segment_type(segment) + self.name = parse_segment_name(segment, self.__class__) + self.vram_addr = parse_segment_vram(segment) + self.ld_name_override = segment.get( + "ld_name", None) if type(segment) is dict else None + self.options = options + self.config = segment + self.subalign = parse_segment_subalign(segment) + + self.errors = [] + self.warnings = [] + self.did_run = False + + def check(self): + if self.rom_start > self.rom_end: + self.warn(f"out-of-order (starts at 0x{self.rom_start:X}, but next segment starts at 0x{self.rom_end:X})") + elif self.max_length(): + expected_len = int(self.max_length()) + actual_len = self.rom_end - self.rom_start + if actual_len > expected_len: + print(f"should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after {self.name})") + + @property + def size(self): + return self.rom_end - self.rom_start + + @property + def vram_end(self): + return self.vram_addr + self.size + + def rom_to_ram(self, rom_addr): + if rom_addr < self.rom_start or rom_addr > self.rom_end: + return None + + return self.vram_addr + rom_addr - self.rom_start + + def ram_to_rom(self, ram_addr): + if ram_addr < self.vram_addr or ram_addr > self.vram_end: + return None + + return self.rom_start + ram_addr - self.vram_addr + + def create_split_dir(self, base_path, subdir): + out_dir = Path(base_path, subdir) + out_dir.mkdir(parents=True, exist_ok=True) + return out_dir + + def create_parent_dir(self, base_path, filename): + out_dir = Path(base_path, filename).parent + out_dir.mkdir(parents=True, exist_ok=True) + return out_dir + + def should_run(self): + return self.type in self.options["modes"] or "all" in self.options["modes"] + + def split(self, rom_bytes, base_path): + pass + + def postsplit(self, segments): + pass + + def cache(self): + return (self.config, self.rom_end) + + def get_ld_section(self): + replace_ext = self.options.get("ld_o_replace_extension", True) + sect_name = self.ld_name_override if self.ld_name_override else self.get_ld_section_name() + vram_or_rom = self.rom_start if self.vram_addr == 0 else self.vram_addr + subalign_str = "" if self.subalign == default_subalign else f"SUBALIGN({self.subalign})" + + s = ( + f"SPLAT_BEGIN_SEG({sect_name}, 0x{self.rom_start:X}, 0x{vram_or_rom:X}, {subalign_str})\n" + ) + + i = 0 + for subdir, path, obj_type, start in self.get_ld_files(): + # Hack for non-0x10 alignment + if start % 0x10 != 0 and i != 0: + tmp_sect_name = path.replace(".", "_") + tmp_sect_name = tmp_sect_name.replace("/", "_") + tmp_vram = start - self.rom_start + self.vram_addr + s += ( + "}\n" + f"SPLAT_BEGIN_SEG({tmp_sect_name}, 0x{start:X}, 0x{tmp_vram:X}, {subalign_str})\n" + ) + + path = PurePath(subdir) / PurePath(path) + path = path.with_suffix(".o" if replace_ext else path.suffix + ".o") + + s += f" BUILD_DIR/{path}({obj_type});\n" + i += 1 + + s += ( + f"SPLAT_END_SEG({sect_name}, 0x{self.rom_end:X})\n" + ) + + return s + + def get_ld_section_name(self): + return f"data_{self.rom_start:X}" + + # returns list of (basedir, filename, obj_type) + def get_ld_files(self): + return [] + + def log(self, msg): + if self.options.get("verbose", False): + log.write(f"{self.type} {self.name}: {msg}") + + def warn(self, msg): + self.warnings.append(msg) + + def error(self, msg): + self.errors.append(msg) + + def max_length(self): + return None + + def is_name_default(self): + return self.name == self.get_default_name(self.rom_start) + + def unique_id(self): + return self.type + "_" + self.name + + def status(self): + if len(self.errors) > 0: + return "error" + elif len(self.warnings) > 0: + return "warn" + elif self.did_run: + return "ok" + else: + return "skip" + + @staticmethod + def get_default_name(addr): + return "{:X}".format(addr) diff --git a/tools/n64splat/split.py b/tools/n64splat/split.py new file mode 100755 index 0000000000..3c2b626fa6 --- /dev/null +++ b/tools/n64splat/split.py @@ -0,0 +1,408 @@ +#! /usr/bin/python3 + +import argparse +import importlib +import importlib.util +import os +from ranges import Range, RangeDict +import re +from pathlib import Path +import segtypes +import sys +import yaml +import pickle +from colorama import Style, Fore +from collections import OrderedDict +from segtypes.segment import N64Segment, parse_segment_type +from segtypes.code import N64SegCode +from util import log + +parser = argparse.ArgumentParser( + description="Split a rom given a rom, a config, and output directory") +parser.add_argument("rom", help="path to a .z64 rom") +parser.add_argument("config", help="path to a compatible config .yaml file") +parser.add_argument("outdir", help="a directory in which to extract the rom") +parser.add_argument("--modes", nargs="+", default="all") +parser.add_argument("--verbose", action="store_true", + help="Enable debug logging") +parser.add_argument("--new", action="store_true", + help="Only split changed segments in config") + + +def write_ldscript(rom_name, repo_path, sections, options): + with open(os.path.join(repo_path, rom_name + ".ld"), "w", newline="\n") as f: + f.write( + "#ifndef SPLAT_BEGIN_SEG\n" + "#ifndef SHIFT\n" + "#define SPLAT_BEGIN_SEG(name, start, vram, subalign) \\\n" + " . = start;\\\n" + " name##_ROM_START = .;\\\n" + " name##_VRAM = ADDR(.name);\\\n" + " .name vram : AT(name##_ROM_START) subalign {\n" + "#else\n" + "#define SPLAT_BEGIN_SEG(name, start, vram, subalign) \\\n" + " name##_ROM_START = .;\\\n" + " name##_VRAM = ADDR(.name);\\\n" + " .name vram : AT(name##_ROM_START) subalign {\n" + "#endif\n" + "#endif\n" + "\n" + "#ifndef SPLAT_END_SEG\n" + "#ifndef SHIFT\n" + "#define SPLAT_END_SEG(name, end) \\\n" + " } \\\n" + " . = end;\\\n" + " name##_ROM_END = .;\n" + "#else\n" + "#define SPLAT_END_SEG(name, end) \\\n" + " } \\\n" + " name##_ROM_END = .;\n" + "#endif\n" + "#endif\n" + "\n" + ) + + if options.get("ld_bare", False): + f.write("\n".join(sections)) + else: + f.write( + "SECTIONS\n" + "{\n" + " " + ) + f.write("\n ".join(s.replace("\n", "\n ") for s in sections)[:-4]) + f.write( + "}\n" + ) + + +def parse_file_start(split_file): + return split_file[0] if "start" not in split_file else split_file["start"] + + +def get_symbol_addrs_path(repo_path, options): + return os.path.join(repo_path, options.get("symbol_addrs", "symbol_addrs.txt")) + + +def get_undefined_syms_path(repo_path, options): + return os.path.join(repo_path, options.get("undefined_syms", "undefined_syms.txt")) + + +def gather_symbols(symbol_addrs_path, undefined_syms_path): + symbols = {} + special_labels = {} + labels_to_add = set() + ranges = RangeDict() + + # Manual list of func name / addrs + if os.path.exists(symbol_addrs_path): + with open(symbol_addrs_path) as f: + func_addrs_lines = f.readlines() + + for line in func_addrs_lines: + line = line.strip() + if not line == "" and not line.startswith("//"): + comment_loc = line.find("//") + line_ext = "" + + if comment_loc != -1: + line_ext = line[comment_loc + 2:].strip() + line = line[:comment_loc].strip() + + line_split = line.split("=") + name = line_split[0].strip() + addr = int(line_split[1].strip()[:-1], 0) + symbols[addr] = name + + if line_ext: + for info in line_ext.split(" "): + if info == "!": + labels_to_add.add(name) + special_labels[addr] = name + if info.startswith("size:"): + size = int(info.split(":")[1], 0) + ranges.add(Range(addr, addr + size), name) + + if os.path.exists(undefined_syms_path): + with open(undefined_syms_path) as f: + us_lines = f.readlines() + + for line in us_lines: + line = line.strip() + if not line == "" and not line.startswith("//"): + line_split = line.split("=") + name = line_split[0].strip() + addr = int(line_split[1].strip()[:-1], 0) + symbols[addr] = name + + return symbols, labels_to_add, special_labels, ranges + + +def gather_c_variables(undefined_syms_path): + vars = {} + + if os.path.exists(undefined_syms_path): + with open(undefined_syms_path) as f: + us_lines = f.readlines() + + for line in us_lines: + line = line.strip() + if not line == "" and not line.startswith("//"): + line_split = line.split("=") + name = line_split[0].strip() + addr = int(line_split[1].strip()[:-1], 0) + vars[addr] = name + + return vars + + +def get_base_segment_class(seg_type): + try: + segmodule = importlib.import_module("segtypes." + seg_type) + except ModuleNotFoundError: + return None + + return getattr(segmodule, "N64Seg" + seg_type[0].upper() + seg_type[1:]) + + +def get_extension_dir(options, config_path): + if "extensions" not in options: + return None + return os.path.join(Path(config_path).parent, options["extensions"]) + + +def get_extension_class(options, config_path, seg_type): + ext_dir = get_extension_dir(options, config_path) + if ext_dir == None: + return None + + try: + ext_spec = importlib.util.spec_from_file_location(f"segtypes.{seg_type}", os.path.join(ext_dir, f"{seg_type}.py")) + ext_mod = importlib.util.module_from_spec(ext_spec) + ext_spec.loader.exec_module(ext_mod) + except Exception as err: + log.write(err, status="error") + return None + + return getattr(ext_mod, "N64Seg" + seg_type[0].upper() + seg_type[1:]) + + +def fmt_size(size): + if size > 1000000: + return str(size // 1000000) + " MB" + elif size > 1000: + return str(size // 1000) + " KB" + else: + return str(size) + " B" + + +def initialize_segments(options, config_path, config_segments): + seen_segment_names = set() + ret = [] + + for i, segment in enumerate(config_segments): + if len(segment) == 1: + # We're at the end + continue + + seg_type = parse_segment_type(segment) + + segment_class = get_base_segment_class(seg_type) + if segment_class == None: + # Look in extensions + segment_class = get_extension_class(options, config_path, seg_type) + + if segment_class == None: + log.write(f"fatal error: could not load segment type '{seg_type}'\n(hint: confirm your extension directory is configured correctly)", status="error") + return 2 + + try: + segment = segment_class(segment, config_segments[i + 1], options) + except (IndexError, KeyError) as e: + try: + segment = N64Segment(segment, config_segments[i + 1], options) + segment.error(e) + except Exception as e: + log.write(f"fatal error (segment type = {seg_type}): " + str(e), status="error") + return 2 + + if segment_class.require_unique_name: + if segment.name in seen_segment_names: + segment.error("segment name is not unique") + seen_segment_names.add(segment.name) + + ret.append(segment) + + return ret + + +def main(rom_path, config_path, repo_path, modes, verbose, ignore_cache=False): + with open(rom_path, "rb") as f: + rom_bytes = f.read() + + # Create main output dir + Path(repo_path).mkdir(parents=True, exist_ok=True) + + # Load config + with open(config_path) as f: + config = yaml.safe_load(f.read()) + + options = config.get("options") + options["modes"] = modes + options["verbose"] = verbose + + symbol_addrs_path = get_symbol_addrs_path(repo_path, options) + undefined_syms_path = get_undefined_syms_path(repo_path, options) + provided_symbols, c_func_labels_to_add, special_labels, ranges = gather_symbols(symbol_addrs_path, undefined_syms_path) + + processed_segments = [] + ld_sections = [] + + defined_funcs = {} + undefined_funcs = set() + undefined_syms = set() + + seg_sizes = {} + seg_split = {} + seg_cached = {} + + # Load cache + cache_path = Path(repo_path) / ".splat_cache" + try: + with open(cache_path, "rb") as f: + cache = pickle.load(f) + except Exception: + cache = {} + + # Initialize segments + all_segments = initialize_segments(options, config_path, config["segments"]) + + for segment in all_segments: + if type(segment) == N64SegCode: + segment.all_functions = defined_funcs + segment.provided_symbols = provided_symbols + segment.special_labels = special_labels + segment.c_labels_to_add = c_func_labels_to_add + segment.symbol_ranges = ranges + + segment.check() + + tp = segment.type + if segment.type == "bin" and segment.is_name_default(): + tp = "unk" + + if tp not in seg_sizes: + seg_sizes[tp] = 0 + seg_split[tp] = 0 + seg_cached[tp] = 0 + seg_sizes[tp] += segment.size + + if len(segment.errors) == 0: + if segment.should_run(): + # Check cache + cached = segment.cache() + if not ignore_cache and cached == cache.get(segment.unique_id()): + # Cache hit + seg_cached[tp] += 1 + else: + # Cache miss; split + cache[segment.unique_id()] = cached + + segment.did_run = True + segment.split(rom_bytes, repo_path) + + if len(segment.errors) == 0: + processed_segments.append(segment) + + if type(segment) == N64SegCode: + undefined_funcs |= segment.glabels_to_add + defined_funcs = {**defined_funcs, **segment.glabels_added} + undefined_syms |= segment.undefined_syms_to_add + + seg_split[tp] += 1 + + log.dot(status=segment.status()) + ld_sections.append(segment.get_ld_section()) + + for segment in processed_segments: + segment.postsplit(processed_segments) + log.dot(status=segment.status()) + + # Write ldscript + if "ld" in options["modes"] or "all" in options["modes"]: + if verbose: + log.write(f"saving {config['basename']}.ld") + write_ldscript(config['basename'], repo_path, ld_sections, options) + + # Write undefined_funcs_auto.txt + if verbose: + log.write(f"saving undefined_funcs_auto.txt") + c_predefined_funcs = set(provided_symbols.keys()) + to_write = sorted(undefined_funcs - set(defined_funcs.values()) - c_predefined_funcs) + if len(to_write) > 0: + with open(os.path.join(repo_path, "undefined_funcs_auto.txt"), "w", newline="\n") as f: + for line in to_write: + f.write(line + " = 0x" + line.split("_")[1][:8].upper() + ";\n") + + # write undefined_syms_auto.txt + if verbose: + log.write(f"saving undefined_syms_auto.txt") + to_write = sorted(undefined_syms, key=lambda x:x[0]) + if len(to_write) > 0: + with open(os.path.join(repo_path, "undefined_syms_auto.txt"), "w", newline="\n") as f: + for sym in to_write: + f.write(f"{sym[0]} = 0x{sym[1]:X};\n") + + # print warnings and errors during split/postsplit + had_error = False + for segment in all_segments: + if len(segment.warnings) > 0 or len(segment.errors) > 0: + log.write(f"{Style.DIM}0x{segment.rom_start:06X}{Style.RESET_ALL} {segment.type} {Style.BRIGHT}{segment.name}{Style.RESET_ALL}:") + + for warn in segment.warnings: + log.write("warning: " + warn, status="warn") + + for error in segment.errors: + log.write("error: " + error, status="error") + had_error = True + + log.write("") # empty line + + if had_error: + return 1 + + # Statistics + unk_size = seg_sizes.get("unk", 0) + rest_size = 0 + total_size = len(rom_bytes) + + for tp in seg_sizes: + if tp != "unk": + rest_size += seg_sizes[tp] + + assert(unk_size + rest_size == total_size) + + known_ratio = rest_size / total_size + unk_ratio = unk_size / total_size + + log.write(f"Split {fmt_size(rest_size)} ({known_ratio:.2%}) in defined segments") + for tp in seg_sizes: + if tp != "unk": + tmp_size = seg_sizes[tp] + tmp_ratio = tmp_size / total_size + log.write(f"{tp:>20}: {fmt_size(tmp_size):>8} ({tmp_ratio:.2%}) {Fore.GREEN}{seg_split[tp]} split{Style.RESET_ALL}, {Style.DIM}{seg_cached[tp]} cached") + log.write(f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files") + + # Save cache + if cache != {}: + if verbose: + print("Writing cache") + with open(cache_path, "wb") as f: + pickle.dump(cache, f) + + return 0 # no error + +if __name__ == "__main__": + args = parser.parse_args() + error_code = main(args.rom, args.config, args.outdir, args.modes, args.verbose, not args.new) + exit(error_code) diff --git a/tools/n64splat/util/Yay0decompress.c b/tools/n64splat/util/Yay0decompress.c new file mode 100644 index 0000000000..ec20f142bd --- /dev/null +++ b/tools/n64splat/util/Yay0decompress.c @@ -0,0 +1,43 @@ +#include +#include +#include +#include + +typedef struct { + uint32_t magic; + uint32_t uncompressedLength; + uint32_t opPtr; + uint32_t dataPtr; +} Yay0Header; + +void decompress(Yay0Header* hdr, uint8_t* srcPtr, uint8_t* dstPtr, bool isBigEndian) { + uint8_t byte = 0, mask = 0; + uint8_t* ctrl, * ops, * data; + uint16_t copy, op; + uint32_t written = 0; + + ctrl = srcPtr + sizeof(Yay0Header); + ops = srcPtr + hdr->opPtr; + data = srcPtr + hdr->dataPtr; + + while (written < hdr->uncompressedLength) { + if ((mask >>= 1) == 0) { + byte = *ctrl++; + mask = 0x80; + } + + if (byte & mask) { + *dstPtr++ = *data++; + written++; + } else { + op = isBigEndian ? (ops[0] << 8) | ops[1] : (ops[1] << 8) | ops[0]; + ops += 2; + + written += copy = (op >> 12) ? (2 + (op >> 12)) : (18 + *data++); + + while (copy--) { + *dstPtr++ = dstPtr[-(op & 0xfff) - 1]; + } + } + } +} diff --git a/tools/n64splat/util/Yay0decompress.py b/tools/n64splat/util/Yay0decompress.py new file mode 100644 index 0000000000..9d4b82b7f2 --- /dev/null +++ b/tools/n64splat/util/Yay0decompress.py @@ -0,0 +1,132 @@ +import argparse +import sys +import os +from ctypes import * +from struct import pack, unpack_from + +tried_loading = False +lib = None + +def setup_lib(): + global lib + global tried_loading + if lib: + return True + if tried_loading: + return False + try: + tried_loading = True + lib = cdll.LoadLibrary(os.path.dirname(os.path.realpath(__file__)) + "/Yay0decompress") + return True + except Exception: + print(f"Failed to load Yay0decompress, falling back to python method") + tried_loading = True + return False + +def decompress_yay0(in_bytes, byte_order="big"): + # attempt to load the library only once per execution + global lib + if not setup_lib(): + return decompress_yay0_python(in_bytes, byte_order) + + class Yay0(Structure): + _fields_ = [ + ("magic", c_uint32), + ("uncompressedLength", c_uint32), + ("opPtr", c_uint32), + ("dataPtr", c_uint32), + ] + + # read the file header + bigEndian = byte_order == "big" + if bigEndian: + # the struct is only a view, so when passed to C it will keep + # its BigEndian values and crash. Explicitly convert them here to little + hdr = Yay0.from_buffer_copy(pack("IIII", in_bytes, 0))) + else: + hdr = Yay0.from_buffer_copy(in_bytes, 0) + + # create the input/output buffers, copying data to in + src = (c_uint8 * len(in_bytes)).from_buffer_copy(in_bytes, 0) + dst = (c_uint8 * hdr.uncompressedLength)() + + # call decompress, equivilant to, in C: + # decompress(&hdr, &src, &dst, bigEndian) + lib.decompress(byref(hdr), byref(src), byref(dst), c_bool(bigEndian)) + + # other functions want the results back as a non-ctypes type + return bytearray(dst) + +def decompress_yay0_python(in_bytes, byte_order="big"): + if in_bytes[:4] != b"Yay0": + sys.exit("Input file is not yay0") + + decompressed_size = int.from_bytes(in_bytes[4:8], byteorder=byte_order) + link_table_offset = int.from_bytes(in_bytes[8:12], byteorder=byte_order) + chunk_offset = int.from_bytes(in_bytes[12:16], byteorder=byte_order) + + link_table_idx = link_table_offset + chunk_idx = chunk_offset + other_idx = 16 + + mask_bit_counter = 0 + current_mask = 0 + + # preallocate result and index into it + idx = 0 + ret = bytearray(decompressed_size); + + while idx < decompressed_size: + # If we're out of bits, get the next mask + if mask_bit_counter == 0: + current_mask = int.from_bytes(in_bytes[other_idx : other_idx + 4], byteorder=byte_order) + other_idx += 4 + mask_bit_counter = 32 + + if (current_mask & 0x80000000): + ret[idx] = in_bytes[chunk_idx] + idx += 1 + chunk_idx += 1 + else: + link = int.from_bytes(in_bytes[link_table_idx : link_table_idx + 2], byteorder=byte_order) + link_table_idx += 2 + + offset = idx - (link & 0xfff) + + count = link >> 12 + + if count == 0: + count_modifier = in_bytes[chunk_idx] + chunk_idx += 1 + count = count_modifier + 18 + else: + count += 2 + + # Copy the block + for i in range(count): + ret[idx] = ret[offset + i - 1] + idx += 1 + + current_mask <<= 1 + mask_bit_counter -= 1 + + return ret + + +def main(args): + with open(args.infile, "rb") as f: + raw_bytes = f.read() + + decompressed = decompress_yay0(raw_bytes, args.byte_order) + + with open(args.outfile, "wb") as f: + f.write(decompressed) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("infile") + parser.add_argument("outfile") + parser.add_argument("--byte-order", default="big", choices=["big", "little"]) + + args = parser.parse_args() + main(args) diff --git a/tools/n64splat/util/__init__.py b/tools/n64splat/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/n64splat/util/color.py b/tools/n64splat/util/color.py new file mode 100644 index 0000000000..93383a197b --- /dev/null +++ b/tools/n64splat/util/color.py @@ -0,0 +1,16 @@ +from math import ceil + +# RRRRRGGG GGBBBBBA +def unpack_color(data): + s = int.from_bytes(data[0:2], byteorder="big") + + r = (s >> 11) & 0x1F + g = (s >> 6) & 0x1F + b = (s >> 1) & 0x1F + a = (s & 1) * 0xFF + + r = ceil(0xFF * (r / 31)) + g = ceil(0xFF * (g / 31)) + b = ceil(0xFF * (b / 31)) + + return r, g, b, a diff --git a/tools/n64splat/util/find_code_length.py b/tools/n64splat/util/find_code_length.py new file mode 100755 index 0000000000..ebffe2d859 --- /dev/null +++ b/tools/n64splat/util/find_code_length.py @@ -0,0 +1,48 @@ +#! /usr/bin/python3 + +from capstone import * +from capstone.mips import * + +import argparse +import hashlib +import rominfo +import zlib + +md = Cs(CS_ARCH_MIPS, CS_MODE_MIPS64 + CS_MODE_BIG_ENDIAN) + +parser = argparse.ArgumentParser(description="Given a rom and start offset, find where the code ends") +parser.add_argument("rom", help="path to a .z64 rom") +parser.add_argument("start", help="start offset") +parser.add_argument("--end", help="end offset", default=None) +parser.add_argument("--vram", help="vram address to start disassembly at", default="0x80000000") + +def run(rom_bytes, start_offset, vram, end_offset=None): + rom_addr = start_offset + last_return = rom_addr + + for insn in md.disasm(rom_bytes[start_offset:], vram): + if insn.mnemonic == "jr" and insn.op_str == "$ra": + last_return = rom_addr + rom_addr += 4 + if end_offset and rom_addr >= end_offset: + break + + return last_return + (0x10 - (last_return % 0x10)) + + +def main(): + args = parser.parse_args() + + rom_bytes = rominfo.read_rom(args.rom) + start = int(args.start, 0) + end = None + vram = int(args.vram, 0) + + if args.end: + end = int(args.end, 0) + + print(f"{run(rom_bytes, start, vram, end):X}") + + +if __name__ == "__main__": + main() diff --git a/tools/n64splat/util/floats.py b/tools/n64splat/util/floats.py new file mode 100644 index 0000000000..1d9fd001ad --- /dev/null +++ b/tools/n64splat/util/floats.py @@ -0,0 +1,62 @@ + +import math +import struct + +# From mips_to_c: https://github.com/matt-kempster/mips_to_c/blob/d208400cca045113dada3e16c0d59c50cdac4529/src/translate.py#L2085 +def format_f32_imm(num: int) -> str: + packed = struct.pack(">I", num & (2 ** 32 - 1)) + value = struct.unpack(">f", packed)[0] + + if not value or value == 4294967296.0: + # Zero, negative zero, nan, or INT_MAX. + return str(value) + + # Write values smaller than 1e-7 / greater than 1e7 using scientific notation, + # and values in between using fixed point. + if abs(math.log10(abs(value))) > 6.9: + fmt_char = "e" + elif abs(value) < 1: + fmt_char = "f" + else: + fmt_char = "g" + + def fmt(prec: int) -> str: + """Format 'value' with 'prec' significant digits/decimals, in either scientific + or regular notation depending on 'fmt_char'.""" + ret = ("{:." + str(prec) + fmt_char + "}").format(value) + if fmt_char == "e": + return ret.replace("e+", "e").replace("e0", "e").replace("e-0", "e-") + if "e" in ret: + # The "g" format character can sometimes introduce scientific notation if + # formatting with too few decimals. If this happens, return an incorrect + # value to prevent the result from being used. + # + # Since the value we are formatting is within (1e-7, 1e7) in absolute + # value, it will at least be possible to format with 7 decimals, which is + # less than float precision. Thus, this annoying Python limitation won't + # lead to us outputting numbers with more precision than we really have. + return "0" + return ret + + # 20 decimals is more than enough for a float. Start there, then try to shrink it. + prec = 20 + while prec > 0: + prec -= 1 + value2 = float(fmt(prec)) + if struct.pack(">f", value2) != packed: + prec += 1 + break + + if prec == 20: + # Uh oh, even the original value didn't format correctly. Fall back to str(), + # which ought to work. + return str(value) + + ret = fmt(prec) + if "." not in ret: + ret += ".0" + return ret + +def format_f64_imm(num: int) -> str: + (value,) = struct.unpack(">d", struct.pack(">Q", num & (2 ** 64 - 1))) + return str(value) \ No newline at end of file diff --git a/tools/n64splat/util/iter.py b/tools/n64splat/util/iter.py new file mode 100644 index 0000000000..eeef3aa771 --- /dev/null +++ b/tools/n64splat/util/iter.py @@ -0,0 +1,5 @@ +from itertools import zip_longest + +def iter_in_groups(iterable, n, fillvalue=None): + args = [iter(iterable)] * n + return zip_longest(*args, fillvalue=fillvalue) diff --git a/tools/n64splat/util/log.py b/tools/n64splat/util/log.py new file mode 100644 index 0000000000..03523c46f0 --- /dev/null +++ b/tools/n64splat/util/log.py @@ -0,0 +1,32 @@ +from colorama import init, Fore, Style + +init(autoreset=True) + +newline = True + +def write(*args, status=None, **kwargs): + global newline + + if not newline: + print("") + newline = True + + print(status_to_ansi(status) + str(args[0]), *args[1:], **kwargs) + +def dot(status=None): + global newline + + print(status_to_ansi(status) + ".", end="") + newline = False + +def status_to_ansi(status): + if status == "ok": + return Fore.GREEN + elif status == "warn": + return Fore.YELLOW + Style.BRIGHT + elif status == "error": + return Fore.RED + Style.BRIGHT + elif status == "skip": + return Style.DIM + else: + return "" diff --git a/tools/n64splat/util/rominfo.py b/tools/n64splat/util/rominfo.py new file mode 100755 index 0000000000..526fcb15d3 --- /dev/null +++ b/tools/n64splat/util/rominfo.py @@ -0,0 +1,119 @@ +#! /usr/bin/python3 + +import argparse +import hashlib +import zlib + +parser = argparse.ArgumentParser(description='Gives information on n64 roms') +parser.add_argument('rom', help='path to a .z64 rom') +parser.add_argument('--encoding', help='Text encoding the game header is using, defaults to ASCII, see docs.python.org/2.4/lib/standard-encodings.html for valid encodings', default='ASCII') + +country_codes = { + 0x37: "Beta", + 0x41: "Asian (NTSC)", + 0x42: "Brazillian", + 0x43: "Chiniese", + 0x44: "German", + 0x45: "North America", + 0x46: "French", + 0x47: "Gateway 64 (NTSC)", + 0x48: "Dutch", + 0x49: "Italian", + 0x4A: "Japanese", + 0x4B: "Korean", + 0x4C: "Gateway 64 (PAL)", + 0x4E: "Canadian", + 0x50: "European (basic spec.)", + 0x53: "Spanish", + 0x55: "Australian", + 0x57: "Scandanavian", + 0x58: "European", + 0x59: "European", +} + +crc_to_cic = { + 0x6170A4A1: {"ntsc-name": "6101", "pal-name": "7102", "offset": 0x000000}, + 0x90BB6CB5: {"ntsc-name": "6102", "pal-name": "7101", "offset": 0x000000}, + 0x0B050EE0: {"ntsc-name": "6103", "pal-name": "7103", "offset": 0x100000}, + 0x98BC2C86: {"ntsc-name": "6105", "pal-name": "7105", "offset": 0x000000}, + 0xACC8580A: {"ntsc-name": "6106", "pal-name": "7106", "offset": 0x200000}, + 0x00000000: {"ntsc-name": "unknown", "pal-name": "unknown", "offset": 0x0000000} +} + + +def read_rom(rom): + with open(rom, "rb") as f: + return f.read() + + +def get_cic(rom_bytes): + crc = zlib.crc32(rom_bytes[0x40:0x1000]) + if crc in crc_to_cic: + return crc_to_cic[crc] + else: + return crc_to_cic[0] + + +def get_entry_point(program_counter, cic): + return program_counter - cic["offset"] + + +def get_info(rom_path, encoding="ASCII"): + return get_info_bytes(read_rom(rom_path), encoding) + + +def get_info_bytes(rom_bytes, encoding): + program_counter = int(rom_bytes[0x8:0xC].hex(), 16) + libultra_version = chr(rom_bytes[0xF]) + crc1 = rom_bytes[0x10:0x14].hex().upper() + crc2 = rom_bytes[0x14:0x18].hex().upper() + + try: + name = rom_bytes[0x20:0x34].decode(encoding).strip() + except: + print("n64splat could not decode the game name, try using a different encoding by passing the --encoding argument (see docs.python.org/2.4/lib/standard-encodings.html for valid encodings)") + exit(1) + + country_code = rom_bytes[0x3E] + + cic = get_cic(rom_bytes) + entry_point = get_entry_point(program_counter, cic) + + # todo add support for + # compression_formats = [] + # for format in ["Yay0", "vpk0"]: + # if rom_bytes.find(bytes(format, "ASCII")) != -1: + # compression_formats.append(format) + + return N64Rom(name, country_code, libultra_version, crc1, crc2, cic, entry_point, len(rom_bytes)) + + +class N64Rom: + def __init__(self, name, country_code, libultra_version, crc1, crc2, cic, entry_point, size): + self.name = name + self.country_code = country_code + self.libultra_version = libultra_version + self.crc1 = crc1 + self.crc2 = crc2 + self.cic = cic + self.entry_point = entry_point + self.size = size + + def get_country_name(self): + return country_codes[self.country_code] + + +def main(): + args = parser.parse_args() + rom = get_info(args.rom, args.encoding) + print("Image name: " + rom.name) + print("Country code: " + chr(rom.country_code) + " - " + rom.get_country_name()) + print("Libultra version: " + rom.libultra_version) + print("CRC1: " + rom.crc1) + print("CRC2: " + rom.crc2) + print("CIC: " + rom.cic["ntsc-name"] + " / " + rom.cic["pal-name"]) + print("RAM entry point: " + hex(rom.entry_point)) + + +if __name__ == "__main__": + main() diff --git a/tools/splat.yaml b/tools/splat.yaml index b79732682a..af5f7d74ff 100644 --- a/tools/splat.yaml +++ b/tools/splat.yaml @@ -2285,7 +2285,7 @@ segments: overlay: True files: - [0x7345A0, c] - - [0x734640, bin] + - [0x734650, bin] - type: code start: 0x737890 vram: 0x802A1000 diff --git a/tools/star-rod b/tools/star-rod deleted file mode 160000 index aec5d4c037..0000000000 --- a/tools/star-rod +++ /dev/null @@ -1 +0,0 @@ -Subproject commit aec5d4c037e95227fb5f118075564031636697fe