From 14f8d62e3ec0e8eaa4daadbbab1fdc013acc445e Mon Sep 17 00:00:00 2001 From: Ryan Dwyer Date: Sat, 7 Dec 2019 18:03:53 +1000 Subject: [PATCH] Build ROM from C source * ROM is mismatching but functionally equivalent. * ROM does not shift, so any edits must use the same amount of bytecode. * Asset files such as stage setup and lang are not included yet (they are copied from the base ROM). --- Makefile | 31 +++++- README.md | 10 +- tools/buildrom | 108 +++++++++++++++++++++ tools/checksum | 76 +++++++++++++++ tools/inject | 239 ----------------------------------------------- tools/mkgamezips | 69 ++++++++++++++ tools/rarezip | 20 +--- 7 files changed, 287 insertions(+), 266 deletions(-) create mode 100755 tools/buildrom create mode 100755 tools/checksum delete mode 100755 tools/inject create mode 100755 tools/mkgamezips diff --git a/Makefile b/Makefile index 780c0806d..b2c86670e 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,14 @@ tiles: $(TILE_BIN_FILES) LANG_C_FILES := $(wildcard src/files/lang/*.c) LANG_BIN_FILES := $(patsubst src/files/lang/%.c, $(B_DIR)/files/lang/L%.bin, $(LANG_C_FILES)) -LANG_BINZ_FILES := $(patsubst src/files/lang/%.c, $(B_DIR)/files/L%, $(LANG_C_FILES)) +LANG_BINZ_FILES := \ + $(patsubst src/files/lang/%E.c, $(B_DIR)/files/L%E, $(wildcard src/files/lang/*E.c)) \ + $(patsubst src/files/lang/%J.c, $(B_DIR)/files/L%J, $(wildcard src/files/lang/*J.c)) \ + $(patsubst src/files/lang/%P.c, $(B_DIR)/files/L%P, $(wildcard src/files/lang/*P.c)) \ + $(patsubst src/files/lang/%_str_f.c, $(B_DIR)/files/L%_str_fZ, $(wildcard src/files/lang/*_str_f.c)) \ + $(patsubst src/files/lang/%_str_g.c, $(B_DIR)/files/L%_str_gZ, $(wildcard src/files/lang/*_str_g.c)) \ + $(patsubst src/files/lang/%_str_i.c, $(B_DIR)/files/L%_str_iZ, $(wildcard src/files/lang/*_str_i.c)) \ + $(patsubst src/files/lang/%_str_s.c, $(B_DIR)/files/L%_str_sZ, $(wildcard src/files/lang/*_str_s.c)) $(B_DIR)/files/lang/%.elf: src/files/lang/%.o mkdir -p $(B_DIR)/files/lang @@ -177,6 +184,12 @@ $(B_DIR)/ucode/gvars.bin: $(B_DIR)/stage1.bin gvars: $(B_DIR)/ucode/gvars.bin +################################################################################ +# Build related + +$(B_DIR)/ucode/gamezips.bin: $(B_DIR)/ucode/game.bin + tools/mkgamezips + ################################################################################ # Test related @@ -204,8 +217,20 @@ $(B_DIR)/stage1.bin: $(B_DIR)/stage1.elf all: stagesetup lang boot library setup tiles rarezip game gvars -rom: all - tools/inject pd.$(ROMID).z64 +UCODE_BIN_FILES := \ + $(B_DIR)/ucode/boot.bin \ + $(B_DIR)/ucode/game.bin \ + $(B_DIR)/ucode/gamezips.bin \ + $(B_DIR)/ucode/gvars.bin \ + $(B_DIR)/ucode/library.bin \ + $(B_DIR)/ucode/rarezip.bin \ + $(B_DIR)/ucode/setup.bin + +FINAL_ASSET_FILES := $(SETUP_BINZ_FILES) $(LANG_BINZ_FILES) $(TILES_BINZ_FILES) + +rom: $(UCODE_BIN_FILES) $(FINAL_ASSET_FILES) + tools/buildrom + tools/checksum build/ntsc-final/pd.z64 --write clean: rm -rf build/* diff --git a/README.md b/README.md index e1b54f68f..799d568d8 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This repository contains a work-in-progress decompilation of Perfect Dark for th | 0x3050 library | 27/950 functions done (2.84%) | | 0x39850 setup | About 50% identified | | 0x4e850 rarezip | 2/8 functions done (25.00%) | -| 0x5032e game | 478/4196 functions done (11.39%) | +| 0x4fc40 game | 478/4196 functions done (11.39%) | | Lang files | Done | | Setup files | Done | | Prop files | Not started | @@ -68,12 +68,10 @@ Before you do anything you need an existing ROM to extract assets from. The project can do the following: * Build individual ucode binaries (boot, library, setup, rarezip and game) which match the ones extracted from the base ROM. -* Build a functioning ROM by splicing your stage setup and lang files into an existing ROM. +* Build a functioning ROM by splicing the C source into an existing ROM. Files in the "files" folder (eg. stage setup and lang) are not included yet. Additionally, the built ROM is not byte perfect yet, but is is functionally equivalent. -The project does NOT build a full ROM using the C code yet. - -* Run `make` to build the individual ucode binaries. These files will be written to `build/ntsc-final`. -* Run `make rom` to build a ROM from the stage setup and lang files. The ROM will be written to `build/ntsc-final/pd.z64`. +* Run `make` to build the assets that will be included in the ROM. These files will be written to `build/ntsc-final` and are matching what's in the `extracted/ntsc-final` folder. +* Run `make rom` to build a ROM from the C source. The ROM will be written to `build/ntsc-final/pd.z64`. ## How do I know the built files are matching? diff --git a/tools/buildrom b/tools/buildrom new file mode 100755 index 000000000..29015114f --- /dev/null +++ b/tools/buildrom @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 + +import re +import subprocess + +def main(): + fd = open('build/ntsc-final/pd.z64', 'wb+') + write_binary(fd, 0, get_header()) + write_binary(fd, 0x40, get_rspboot()) + write_binary(fd, 0x1000, get_boot()) + write_binary(fd, 0x3050, get_library()) + write_binary(fd, 0x39850, get_setup()) + write_binary(fd, 0x4e850, get_rarezip()) + write_binary(fd, 0x4fc40, get_gamezips()) + write_binary(fd, 0x157810, get_unknown()) + write_binary(fd, 0x7f2388, get_fonts()) + write_binary(fd, 0x80a250, get_sfxctl()) + write_binary(fd, 0x839dd0, get_sfxtbl()) + write_binary(fd, 0xcfbf30, get_seqctl()) + write_binary(fd, 0xd05f90, get_seqtbl()) + write_binary(fd, 0xe82000, get_midi()) + write_binary(fd, 0xed83a0, get_files()) + write_binary(fd, 0x1d65f40, get_textures()) + fd.close() + +def write_binary(fd, address, binary): + fd.seek(address) + fd.write(binary) + +def get_header(): + binary = bytearray() + binary.extend(b'\x80\x37\x12\x40') # Identifier + binary.extend(b'\x00\x00\x00\x0f') # Clock rate + binary.extend(b'\x80\x00\x10\x00') # Program counter + binary.extend(b'\x00\x00\x14\x49') # Release address + binary.extend(b'\x00\x00\x00\x00') # CRC 1 + binary.extend(b'\x00\x00\x00\x00') # CRC 2 + binary.extend(b'\x00\x00\x00\x00') + binary.extend(b'\x00\x00\x00\x00') + binary.extend(b'Perfect Dark ') + binary.extend(b'\x00\x00\x00\x00') + binary.extend(b'\x00\x00\x00') + binary.extend(b'NPDE') + binary.extend(b'\x01') + return binary + +def get_rspboot(): + return getfilecontents('extracted/ntsc-final/ucode/rspboot.bin') + +def get_boot(): + return getfilecontents('build/ntsc-final/ucode/boot.bin') + +def get_library(): + return zip('build/ntsc-final/ucode/library.bin') + +def get_setup(): + return zip('build/ntsc-final/ucode/setup.bin') + +def get_rarezip(): + return getfilecontents('build/ntsc-final/ucode/rarezip.bin') + +def get_gamezips(): + return getfilecontents('build/ntsc-final/ucode/gamezips.bin') + +def get_unknown(): + return getfrombaserom(0x157810, 0x69ab78) + +def get_fonts(): + return getfrombaserom(0x7f2388, 0x17ec8) + +def get_sfxctl(): + return getfilecontents('extracted/ntsc-final/audio/sfx.ctl') + +def get_sfxtbl(): + return getfilecontents('extracted/ntsc-final/audio/sfx.tbl') + +def get_seqctl(): + return getfilecontents('extracted/ntsc-final/audio/music.ctl') + +def get_seqtbl(): + return getfilecontents('extracted/ntsc-final/audio/music.tbl') + +def get_midi(): + return getfilecontents('extracted/ntsc-final/audio/midi.bin') + +def get_files(): + return getfrombaserom(0xed83a0, 0xe8d3a0) + +def get_textures(): + return getfrombaserom(0x01d65f40, 0x29a0c0) + +def getfilecontents(filename): + fd = open(filename, 'rb') + binary = fd.read() + fd.close() + return binary + +def getfrombaserom(offset, len): + fd = open('pd.ntsc-final.z64', 'rb') + fd.seek(offset) + binary = fd.read(len) + fd.close() + return binary + +def zip(filename): + return subprocess.check_output(['tools/rarezip', filename]) + +main() diff --git a/tools/checksum b/tools/checksum new file mode 100755 index 000000000..a1e949255 --- /dev/null +++ b/tools/checksum @@ -0,0 +1,76 @@ +#!/usr/bin/python + +import sys; + +class Tool: + + def ROL(self, i, b): + return ((i << b) | (i >> (32 - b))) & 0xffffffff + + def R4(self, b): + return b[0]*0x1000000 + b[1]*0x10000 + b[2]*0x100 + b[3] + + def crc(self, f): + seed = 0xdf26f436 + t1 = t2 = t3 = t4 = t5 = t6 = seed + + f.seek(0x0710 + 0x40) + lookup = f.read(0x100) + + f.seek(0x1000) + for i in range(0x1000, 0x101000, 4): + d = self.R4(f.read(4)) + + if ((t6 + d) & 0xffffffff) < t6: + t4 += 1 + t4 &= 0xffffffff + + t6 += d + t6 &= 0xffffffff + + t3 ^= d + + r = self.ROL(d, d & 0x1F) + + t5 += r + t5 &= 0xffffffff + + if t2 > d: + t2 ^= r + else: + t2 ^= t6 ^ d + + o = i & 0xFF + temp = self.R4(lookup[o:o + 4]) + t1 += temp ^ d + t1 &= 0xffffffff + + crc1 = t6 ^ t4 ^ t3 + crc2 = t5 ^ t2 ^ t1 + + return crc1 & 0xffffffff, crc2 & 0xffffffff + +fd = open(sys.argv[1], 'rb') + +# Read existing CRC +fd.seek(0x10) +old = [ + int.from_bytes(fd.read(4), 'big'), + int.from_bytes(fd.read(4), 'big'), +] + +# Calculate new CRC +tool = Tool() +new = tool.crc(fd) +fd.close() + +if '--verbose' in sys.argv: + print('Old CRCs: %08x %08x' % (old[0], old[1])) + print('New CRCs: %08x %08x' % (new[0], new[1])) + +if new != old and '--write' in sys.argv: + fd = open(sys.argv[1], 'r+b') + fd.seek(0x10) + fd.write(new[0].to_bytes(4, 'big')) + fd.write(new[1].to_bytes(4, 'big')) + fd.close() diff --git a/tools/inject b/tools/inject deleted file mode 100755 index 06522a66e..000000000 --- a/tools/inject +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/python - -import sys, zlib - -class Injector: - - vacancies = [ - (0x00ed83a0, 0x01d5ca00), # Normal files spot - (0x0002ea70, 0x00039850), # Unused space 1 - (0x00157810, 0x001a15c0), # Unused space 2 - ] - - def rarezip(self, buffer): - length = len(buffer) - - header = bytes([0x11, 0x73]) - lendata = bytes([length >> 16]) - lendata += bytes([(length >> 8) & 0xff]) - lendata += bytes([length & 0xff]) - - obj = zlib.compressobj(level=9, wbits=-15) - return header + lendata + obj.compress(buffer) + obj.flush() - - def rareunzip(self, compressed): - return zlib.decompress(compressed[5:], wbits=-15) - - def load(self, romfile): - fp = open(romfile, 'rb') - rombuffer = fp.read() - fp.close() - - self.rompart1 = rombuffer[0:0x2ea70] - self.rompart2 = rombuffer[0x0004e850:0x00157810] - self.rompart3 = rombuffer[0x001a15c0:0x00ed83a0] - self.rompart4 = rombuffer[0x01d5ca00:] - - fp = open('build/ntsc-final/ucode/setup.bin', 'rb') - setup = fp.read() - fp.close() - - self.globaltop = setup[0:0x28080] - self.globalbot = setup[0x2a000:] - - self.files = [] - i = 0 - - while i <= 0x7dd: - romnameaddr = int.from_bytes(rombuffer[0x01d5ca00 + i * 4:0x01d5ca00 + i * 4 + 4], 'big') + 0x01d5ca00 - romdataaddr = int.from_bytes(setup[0x28080 + i * 4:0x28080 + i * 4 + 4], 'big') - - name = '' - while rombuffer[romnameaddr] != 0: - name = name + chr(rombuffer[romnameaddr]) - romnameaddr += 1 - - self.files.append({ - 'name': name, - 'romaddr': romdataaddr, - 'data': None, - }) - i += 1 - - self.files.sort(key=lambda file: file['romaddr']) - - for index, file in enumerate(self.files): - start = file['romaddr'] - end = self.files[index + 1]['romaddr'] if index < 0x7dd else 0x01d5ca00 - file['data'] = rombuffer[start:end] - - def transform(self): - # Replace file contents - for file in self.files: - if self.isFilePointless(file['name']): - file['data'] = None - continue - - # Zipped file - try: - fp = open('build/ntsc-final/files/%s' % file['name'], 'rb') - contents = fp.read() - fp.close() - except: - continue - - file['data'] = self.align(contents) - - # Calculate new ROM addresses - vacancyid = 0 - romaddr = self.vacancies[0][0] - vacend = self.vacancies[0][1] - - for file in self.files: - if file['name'] == '': - continue - - filelen = len(file['data']) - available = vacend - romaddr - - if filelen > available: - vacancyid += 1 - romaddr = self.vacancies[vacancyid][0] - vacend = self.vacancies[vacancyid][1] - available = vacend - romaddr - - file['romaddr'] = romaddr - romaddr += filelen - - def isFilePointless(self, name): - return False - - def align(self, buffer): - length = len(buffer) - - if length % 0x10 == 0: - return buffer - - over = length % 0x10 - pad = 0x10 - over - - buffer += (0).to_bytes(1, 'big') * pad - - return buffer - - def compile(self): - buffer = self.rompart1 - assert(len(buffer) == 0x2ea70) - buffer += self.compileVacancy(1) - assert(len(buffer) == 0x39850) - buffer += self.compileGlobals() - assert(len(buffer) == 0x4e850) - buffer += self.rompart2 - assert(len(buffer) == 0x157810) - buffer += self.compileVacancy(2) - assert(len(buffer) == 0x1a15c0) - buffer += self.rompart3 - assert(len(buffer) == 0xed83a0) - buffer += self.compileVacancy(0) - assert(len(buffer) == 0x01d5ca00) - buffer += self.rompart4 - assert(len(buffer) == 0x2000000) - return buffer - - def compileGlobals(self): - buffer = self.globaltop - - for file in self.files: - buffer += file['romaddr'].to_bytes(4, 'big') - - buffer += (0x01d5ca00).to_bytes(4, 'big') - buffer += (0).to_bytes(4, 'big') - buffer += self.globalbot - - buffer = self.rarezip(buffer) - - available = 0x15000 - len(buffer) - buffer += (0).to_bytes(1, 'big') * available - return buffer - - def compileVacancy(self, vacid): - buffer = bytes() - vacstart = self.vacancies[vacid][0] - vacend = self.vacancies[vacid][1] - - for file in self.files: - if file['romaddr'] >= vacstart and file['romaddr'] < vacend: - buffer += file['data'] - - available = vacend - vacstart - len(buffer) - buffer += (0).to_bytes(1, 'big') * available - - return buffer - - def ROL(self, i, b): - return ((i << b) | (i >> (32 - b))) & 0xffffffff - - def R4(self, b): - return b[0]*0x1000000 + b[1]*0x10000 + b[2]*0x100 + b[3] - - def crc(self, f): - seed = 0xdf26f436 - t1 = t2 = t3 = t4 = t5 = t6 = seed - - f.seek(0x0710 + 0x40) - lookup = f.read(0x100) - - f.seek(0x1000) - for i in range(0x1000, 0x101000, 4): - d = self.R4(f.read(4)) - - if ((t6 + d) & 0xffffffff) < t6: - t4 += 1 - t4 &= 0xffffffff - - t6 += d - t6 &= 0xffffffff - - t3 ^= d - - r = self.ROL(d, d & 0x1F) - - t5 += r - t5 &= 0xffffffff - - if t2 > d: - t2 ^= r - else: - t2 ^= t6 ^ d - - o = i & 0xFF - temp = self.R4(lookup[o:o + 4]) - t1 += temp ^ d - t1 &= 0xffffffff - - crc1 = t6 ^ t4 ^ t3 - crc2 = t5 ^ t2 ^ t1 - - return crc1 & 0xffffffff, crc2 & 0xffffffff - -injector = Injector() -injector.load(sys.argv[1]) -injector.transform() - -filename = 'build/ntsc-final/pd.z64' - -fp = open(filename, 'wb') -fp.write(injector.compile()) -fp.close() - -fp = open(filename, 'rb') -crcs = injector.crc(fp) -fp.seek(0) -buffer = fp.read() -fp.close() - -fp = open(filename, 'r+b') -fp.seek(0x10) -fp.write(crcs[0].to_bytes(4, 'big')) -fp.write(crcs[1].to_bytes(4, 'big')) -fp.close() diff --git a/tools/mkgamezips b/tools/mkgamezips new file mode 100755 index 000000000..27ca171da --- /dev/null +++ b/tools/mkgamezips @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import os +import subprocess + +""" +mkgamezips - Creates the ucode/gamezips.bin from ucode/game.bin + +game.bin is the compiled game code from ld. This game code is split into +4KB chunks. Each chunk is individually zipped. + +The format of the gamezips binary is: +* Array of offsets to each zip, where each offset is 4 bytes and relative to the + start of the gamezips segment. +* After the array of offsets comes the data it points to. Each data consists of: + * 2 bytes of probable garbage data (set to 0x0000 by this script) + * Zip data (starting with 0x1173001000) + * Optional single byte to align it to the next 2 byte boundary. The added + byte is probable garbage data (set to 0x00 by this script). +""" +def main(): + zips = get_zips() + + fd = open('build/ntsc-final/ucode/gamezips.bin', 'wb') + pos = len(zips) * 4 + 4 + + # Write pointer array + for zip in zips: + fd.write(pos.to_bytes(4, byteorder='big')) + pos += 2 + len(zip) + if pos % 2 == 1: + pos += 1 + + # Last pointer points to end + fd.write(pos.to_bytes(4, byteorder='big')) + + # Write data + for index, zip in enumerate(zips): + if pos % 2 == 1: + fd.write(b'\x00') + pos += 1 + + fd.write(b'\x00\x00') + fd.write(zip) + pos += len(zip) + + fd.close() + +def get_filecontents(filename): + fd = open(filename, 'rb') + binary = fd.read() + fd.close() + return binary + +def get_zips(): + binary = get_filecontents('build/ntsc-final/ucode/game.bin') + parts = [binary[i:i+0x1000] for i in range(0, len(binary), 0x1000)] + return [zip(part) for part in parts] + +def zip(binary): + fd = open('build/part.bin', 'wb') + fd.write(binary) + fd.close() + + zipped = subprocess.check_output(['tools/rarezip', 'build/part.bin']) + os.remove('build/part.bin') + return zipped + +main() diff --git a/tools/rarezip b/tools/rarezip index 913ee50dd..33dc9f19f 100755 --- a/tools/rarezip +++ b/tools/rarezip @@ -2,22 +2,6 @@ size=$(stat --format="%s" $1) -printf "0: 1173 %.6x" $size | xxd -r -g0 > $1.tmp - -cat $1 | tools/gzip --no-name --best | head --bytes=-8 | tail --bytes=+11 >> $1.tmp - -# Pad to 0x10 boundary -compsize=$(stat --format="%s" $1.tmp) -over=$((compsize % 0x10)) - -if [ "$over" -gt 0 ]; then - while [ "$over" -lt 16 ]; do - echo -ne \\x00 >> $1.tmp - over=$((over + 1)) - done -fi - -cat $1.tmp - -rm -f $1.tmp +printf "0: 1173 %.6x" $size | xxd -r -g0 +cat $1 | tools/gzip --no-name --best | head --bytes=-8 | tail --bytes=+11