mirror of https://github.com/zeldaret/mm.git
Build tools rework (#1542)
* yeet libyaz0 * replace elf2rom * makeyar.py * some cleanup * Yeet unused tools * format scripts * Review * review * Rework checksummer.py to modify input file instead
This commit is contained in:
Anghelo Carvajal
GitHub
parent
6fb4a07325
commit
2c600ddd3d
41
Makefile
41
Makefile
|
|
@ -27,8 +27,11 @@ OBJDUMP_BUILD ?= 0
|
|||
ASM_PROC_FORCE ?= 0
|
||||
# Number of threads to disassmble, extract, and compress with
|
||||
N_THREADS ?= $(shell nproc)
|
||||
#MIPS toolchain
|
||||
# MIPS toolchain prefix
|
||||
MIPS_BINUTILS_PREFIX ?= mips-linux-gnu-
|
||||
# Python interpreter
|
||||
PYTHON ?= python3
|
||||
|
||||
#### Setup ####
|
||||
|
||||
# Ensure the map file being created using English localization
|
||||
|
|
@ -88,7 +91,7 @@ AS := $(MIPS_BINUTILS_PREFIX)as
|
|||
LD := $(MIPS_BINUTILS_PREFIX)ld
|
||||
OBJCOPY := $(MIPS_BINUTILS_PREFIX)objcopy
|
||||
OBJDUMP := $(MIPS_BINUTILS_PREFIX)objdump
|
||||
ASM_PROC := python3 tools/asm-processor/build.py
|
||||
ASM_PROC := $(PYTHON) tools/asm-processor/build.py
|
||||
|
||||
ASM_PROC_FLAGS := --input-enc=utf-8 --output-enc=euc-jp --convert-statics=global-with-filename
|
||||
|
||||
|
|
@ -115,14 +118,13 @@ else
|
|||
CC_CHECK := @:
|
||||
endif
|
||||
|
||||
CPP := cpp
|
||||
ELF2ROM := tools/buildtools/elf2rom
|
||||
MKLDSCRIPT := tools/buildtools/mkldscript
|
||||
MKDMADATA := tools/buildtools/mkdmadata
|
||||
YAZ0 := tools/buildtools/yaz0
|
||||
ZAPD := tools/ZAPD/ZAPD.out
|
||||
FADO := tools/fado/fado.elf
|
||||
MAKEYAR := tools/buildtools/makeyar
|
||||
CPP := cpp
|
||||
MKLDSCRIPT := tools/buildtools/mkldscript
|
||||
MKDMADATA := tools/buildtools/mkdmadata
|
||||
ZAPD := tools/ZAPD/ZAPD.out
|
||||
FADO := tools/fado/fado.elf
|
||||
MAKEYAR := $(PYTHON) tools/buildtools/makeyar.py
|
||||
CHECKSUMMER := $(PYTHON) tools/buildtools/checksummer.py
|
||||
|
||||
OPTFLAGS := -O2 -g3
|
||||
ASFLAGS := -march=vr4300 -32 -Iinclude
|
||||
|
|
@ -274,10 +276,11 @@ endif
|
|||
all: uncompressed compressed
|
||||
|
||||
$(ROM): $(ELF)
|
||||
$(ELF2ROM) -cic 6105 $< $@
|
||||
$(OBJCOPY) --gap-fill=0x00 -O binary $< $@
|
||||
$(CHECKSUMMER) $@
|
||||
|
||||
$(ROMC): $(ROM)
|
||||
python3 tools/z64compress_wrapper.py $(COMPFLAGS) $(ROM) $@ $(ELF) build/$(SPEC)
|
||||
$(PYTHON) tools/z64compress_wrapper.py $(COMPFLAGS) $(ROM) $@ $(ELF) build/$(SPEC)
|
||||
|
||||
$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) build/ldscript.txt build/undefined_syms.txt
|
||||
$(LD) -T build/undefined_syms.txt -T build/ldscript.txt --no-check-sections --accept-unknown-input-arch --emit-relocs -Map build/mm.map -o $@
|
||||
|
|
@ -313,17 +316,17 @@ distclean: assetclean clean
|
|||
## Extraction step
|
||||
setup:
|
||||
$(MAKE) -C tools
|
||||
python3 tools/fixbaserom.py
|
||||
python3 tools/extract_baserom.py
|
||||
python3 tools/decompress_yars.py
|
||||
$(PYTHON) tools/fixbaserom.py
|
||||
$(PYTHON) tools/extract_baserom.py
|
||||
$(PYTHON) tools/decompress_yars.py
|
||||
|
||||
assets:
|
||||
python3 extract_assets.py -j $(N_THREADS) -Z Wno-hardcoded-pointer
|
||||
$(PYTHON) extract_assets.py -j $(N_THREADS) -Z Wno-hardcoded-pointer
|
||||
|
||||
## Assembly generation
|
||||
disasm:
|
||||
$(RM) -rf asm data
|
||||
python3 tools/disasm/disasm.py -j $(N_THREADS) $(DISASM_FLAGS)
|
||||
$(PYTHON) tools/disasm/disasm.py -j $(N_THREADS) $(DISASM_FLAGS)
|
||||
|
||||
diff-init: uncompressed
|
||||
$(RM) -rf expected/
|
||||
|
|
@ -403,14 +406,14 @@ build/src/%.o: src/%.c
|
|||
build/src/libultra/libc/ll.o: src/libultra/libc/ll.c
|
||||
$(CC_CHECK) $<
|
||||
$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
|
||||
python3 tools/set_o32abi_bit.py $@
|
||||
$(PYTHON) tools/set_o32abi_bit.py $@
|
||||
$(OBJDUMP_CMD)
|
||||
$(RM_MDEBUG)
|
||||
|
||||
build/src/libultra/libc/llcvt.o: src/libultra/libc/llcvt.c
|
||||
$(CC_CHECK) $<
|
||||
$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
|
||||
python3 tools/set_o32abi_bit.py $@
|
||||
$(PYTHON) tools/set_o32abi_bit.py $@
|
||||
$(OBJDUMP_CMD)
|
||||
$(RM_MDEBUG)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# Setup
|
||||
libyaz0>=0.5
|
||||
colorama>=0.4.3
|
||||
crunch64>=0.3.1,<1.0.0
|
||||
ipl3checksum>=1.2.0,<2.0.0
|
||||
|
||||
# disasm
|
||||
rabbitizer>=1.0.0,<2.0.0
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
CFLAGS := -Wall -Wextra -Wpedantic -std=c99 -g -Os
|
||||
PROGRAMS := elf2rom makeromfs mkdmadata mkldscript reloc_prereq yaz0 makeyar
|
||||
PROGRAMS := mkdmadata mkldscript reloc_prereq
|
||||
|
||||
ifeq ($(shell command -v clang >/dev/null 2>&1; echo $$?),0)
|
||||
CC := clang
|
||||
|
|
@ -12,13 +12,9 @@ all: $(PROGRAMS)
|
|||
clean:
|
||||
$(RM) $(PROGRAMS)
|
||||
|
||||
elf2rom_SOURCES := elf2rom.c elf32.c n64chksum.c util.c
|
||||
makeromfs_SOURCES := makeromfs.c n64chksum.c util.c
|
||||
mkdmadata_SOURCES := mkdmadata.c spec.c util.c
|
||||
mkldscript_SOURCES := mkldscript.c spec.c util.c
|
||||
reloc_prereq_SOURCES := reloc_prereq.c spec.c util.c
|
||||
yaz0_SOURCES := yaz0tool.c yaz0.c util.c
|
||||
makeyar_SOURCES := makeyar.c elf32.c yaz0.c util.c
|
||||
|
||||
define COMPILE =
|
||||
$(1): $($1_SOURCES)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-FileCopyrightText: © 2024 ZeldaRET
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ipl3checksum
|
||||
from pathlib import Path
|
||||
import struct
|
||||
from typing import BinaryIO
|
||||
|
||||
|
||||
def round_up(n: int, shift: int) -> int:
|
||||
mod = 1 << shift
|
||||
return (n + mod - 1) >> shift << shift
|
||||
|
||||
|
||||
def pad_rom(f: BinaryIO, rom_len: int):
|
||||
fill_00 = round_up(rom_len, 12)
|
||||
fill_FF = round_up(fill_00, 17)
|
||||
|
||||
f.seek(rom_len)
|
||||
|
||||
if fill_00 > rom_len:
|
||||
f.write(b"\0" * (fill_00 - rom_len))
|
||||
if fill_FF > fill_00:
|
||||
f.write(b"\xFF" * (fill_FF - fill_00))
|
||||
|
||||
|
||||
def update_checksum(f: BinaryIO, detect: bool):
|
||||
rom_bytes = f.read(0x101000)
|
||||
assert len(rom_bytes) == 0x101000, "Small ROM?"
|
||||
|
||||
# Detect CIC
|
||||
if detect:
|
||||
cicKind = ipl3checksum.detectCIC(rom_bytes)
|
||||
if cicKind is None:
|
||||
print("Not able to detect CIC, defaulting to 6105")
|
||||
cicKind = ipl3checksum.CICKind.CIC_X105
|
||||
else:
|
||||
cicKind = ipl3checksum.CICKind.CIC_X105
|
||||
|
||||
# Calculate checksum
|
||||
calculatedChecksum = cicKind.calculateChecksum(rom_bytes)
|
||||
assert calculatedChecksum is not None, "Not able to calculate checksum"
|
||||
|
||||
# Write checksum
|
||||
checksum_bytes = struct.pack(f">II", calculatedChecksum[0], calculatedChecksum[1])
|
||||
f.seek(0x10)
|
||||
f.write(checksum_bytes)
|
||||
|
||||
|
||||
def checksummer_main():
|
||||
description = "Pads a rom in-place and updates its header checksum"
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
|
||||
parser.add_argument("rom", help="input rom filename")
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--detect",
|
||||
action="store_true",
|
||||
help="Try to detect the IPL3 binary on the ROM instead of assuming the 6105 is being used. If not able to detect the IPL3 binary then this program will default to 6105",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
rom_path = Path(args.rom)
|
||||
detect: bool = args.detect
|
||||
|
||||
rom_len = rom_path.stat().st_size
|
||||
|
||||
with rom_path.open("rb+") as f:
|
||||
update_checksum(f, detect)
|
||||
pad_rom(f, rom_len)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
checksummer_main()
|
||||
|
|
@ -1,242 +0,0 @@
|
|||
#include <stdarg.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "elf32.h"
|
||||
#include "n64chksum.h"
|
||||
#include "util.h"
|
||||
|
||||
#define ROM_SEG_START_SUFFIX ".rom_start"
|
||||
#define ROM_SEG_END_SUFFIX ".rom_end"
|
||||
|
||||
struct RomSegment {
|
||||
const char* name;
|
||||
const void* data;
|
||||
int size;
|
||||
int romStart;
|
||||
int romEnd;
|
||||
};
|
||||
|
||||
static struct RomSegment* g_romSegments = NULL;
|
||||
static int g_romSegmentsCount = 0;
|
||||
static int g_romSize;
|
||||
|
||||
static bool parse_number(const char* str, int* num) {
|
||||
char* endptr;
|
||||
long int n = strtol(str, &endptr, 0);
|
||||
*num = n;
|
||||
return endptr > str;
|
||||
}
|
||||
|
||||
static unsigned int round_up(unsigned int num, unsigned int multiple) {
|
||||
num += multiple - 1;
|
||||
return num / multiple * multiple;
|
||||
}
|
||||
|
||||
static char* sprintf_alloc(const char* fmt, ...) {
|
||||
va_list args;
|
||||
int size;
|
||||
char* buffer;
|
||||
|
||||
va_start(args, fmt);
|
||||
size = vsnprintf(NULL, 0, fmt, args) + 1;
|
||||
va_end(args);
|
||||
|
||||
buffer = malloc(size);
|
||||
|
||||
va_start(args, fmt);
|
||||
vsprintf(buffer, fmt, args);
|
||||
va_end(args);
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
static struct RomSegment* add_rom_segment(const char* name) {
|
||||
int index = g_romSegmentsCount;
|
||||
|
||||
g_romSegmentsCount++;
|
||||
g_romSegments = realloc(g_romSegments, g_romSegmentsCount * sizeof(*g_romSegments));
|
||||
|
||||
g_romSegments[index].name = name;
|
||||
g_romSegments[index].romStart = -1;
|
||||
g_romSegments[index].romEnd = -1;
|
||||
return &g_romSegments[index];
|
||||
}
|
||||
|
||||
static void find_segment_info(struct Elf32* elf, struct RomSegment* segment) {
|
||||
int i;
|
||||
char* romStartSymName = sprintf_alloc("_%sSegmentRomStart", segment->name);
|
||||
char* romEndSymName = sprintf_alloc("_%sSegmentRomEnd", segment->name);
|
||||
|
||||
segment->romStart = -1;
|
||||
segment->romEnd = -1;
|
||||
|
||||
// TODO: use a hashmap for this instead of an O(n) loop
|
||||
for (i = 0; i < elf->numsymbols; i++) {
|
||||
struct Elf32_Symbol sym;
|
||||
|
||||
if (!elf32_get_symbol(elf, &sym, i))
|
||||
util_fatal_error("invalid or corrupt ELF file");
|
||||
|
||||
if (strcmp(sym.name, romStartSymName) == 0) {
|
||||
segment->romStart = sym.value;
|
||||
if (segment->romEnd != -1)
|
||||
break;
|
||||
} else if (strcmp(sym.name, romEndSymName) == 0) {
|
||||
segment->romEnd = sym.value;
|
||||
if (segment->romStart != -1)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (segment->romStart == -1)
|
||||
util_fatal_error("ROM start address of %s is not defined\n", segment->name);
|
||||
if (segment->romEnd == -1)
|
||||
util_fatal_error("ROM end address of %s is not defined\n", segment->name);
|
||||
|
||||
free(romStartSymName);
|
||||
free(romEndSymName);
|
||||
}
|
||||
|
||||
static void parse_input_file(const char* filename) {
|
||||
struct Elf32 elf;
|
||||
const void* data;
|
||||
size_t size;
|
||||
int i;
|
||||
|
||||
data = util_read_whole_file(filename, &size);
|
||||
|
||||
if (!elf32_init(&elf, data, size) || elf.machine != ELF_MACHINE_MIPS)
|
||||
util_fatal_error("%s is not a valid 32-bit MIPS ELF file", filename);
|
||||
|
||||
// get ROM segments
|
||||
// sections of type SHT_PROGBITS and whose name is ..secname are considered ROM segments
|
||||
for (i = 0; i < elf.shnum; i++) {
|
||||
struct Elf32_Section sec;
|
||||
struct RomSegment* segment;
|
||||
|
||||
if (!elf32_get_section(&elf, &sec, i))
|
||||
util_fatal_error("invalid or corrupt ELF file");
|
||||
if (sec.type == SHT_PROGBITS && sec.name[0] == '.' &&
|
||||
sec.name[1] == '.'
|
||||
// HACK! ld sometimes marks NOLOAD sections as SHT_PROGBITS for no apparent reason,
|
||||
// so we must ignore the ..secname.bss sections explicitly
|
||||
&& strchr(sec.name + 2, '.') == NULL) {
|
||||
|
||||
segment = add_rom_segment(sec.name + 2);
|
||||
find_segment_info(&elf, segment);
|
||||
segment->data = elf.data + sec.offset;
|
||||
}
|
||||
}
|
||||
|
||||
// find ROM size
|
||||
for (i = 0; i < elf.numsymbols; i++) {
|
||||
struct Elf32_Symbol sym;
|
||||
|
||||
if (!elf32_get_symbol(&elf, &sym, i))
|
||||
util_fatal_error("invalid or corrupt ELF file");
|
||||
if (strcmp(sym.name, "_RomSize") == 0) {
|
||||
g_romSize = sym.value;
|
||||
goto got_rom_size;
|
||||
}
|
||||
}
|
||||
util_fatal_error("could not find symbol _RomSize");
|
||||
got_rom_size:
|
||||
|
||||
// verify segment info
|
||||
for (i = 0; i < g_romSegmentsCount; i++) {
|
||||
if (g_romSegments[i].romStart == -1)
|
||||
util_fatal_error("segment %s has no ROM start address defined.", g_romSegments[i].name);
|
||||
if (g_romSegments[i].romEnd == -1)
|
||||
util_fatal_error("segment %s has no ROM end address defined.", g_romSegments[i].name);
|
||||
}
|
||||
}
|
||||
|
||||
// Writes the N64 ROM, padding the file size to a multiple of 1 MiB
|
||||
static void write_rom_file(const char* filename, int cicType) {
|
||||
size_t fileSize = round_up(g_romSize, 0x100000);
|
||||
uint8_t* buffer = calloc(fileSize, 1);
|
||||
int i;
|
||||
uint32_t chksum[2];
|
||||
|
||||
// write segments
|
||||
for (i = 0; i < g_romSegmentsCount; i++) {
|
||||
int size = g_romSegments[i].romEnd - g_romSegments[i].romStart;
|
||||
|
||||
memcpy(buffer + g_romSegments[i].romStart, g_romSegments[i].data, size);
|
||||
}
|
||||
|
||||
// pad the remaining space with 0xFF
|
||||
for (i = g_romSize; i < (int)fileSize; i++)
|
||||
buffer[i] = 0xFF;
|
||||
|
||||
// write checksum
|
||||
if (!n64chksum_calculate(buffer, cicType, chksum))
|
||||
util_fatal_error("invalid cic type %i", cicType);
|
||||
util_write_uint32_be(buffer + 0x10, chksum[0]);
|
||||
util_write_uint32_be(buffer + 0x14, chksum[1]);
|
||||
|
||||
util_write_whole_file(filename, buffer, fileSize);
|
||||
free(buffer);
|
||||
}
|
||||
|
||||
static void usage(const char* execname) {
|
||||
printf("usage: %s\n", execname);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
int i;
|
||||
const char* inputFileName = NULL;
|
||||
const char* outputFileName = NULL;
|
||||
int cicType = -1;
|
||||
|
||||
for (i = 1; i < argc; i++) {
|
||||
if (argv[i][0] == '-') {
|
||||
if (strcmp(argv[i], "-cic") == 0) {
|
||||
i++;
|
||||
if (i >= argc || !parse_number(argv[i], &cicType)) {
|
||||
fputs("error: expected number after -cic\n", stderr);
|
||||
goto bad_args;
|
||||
}
|
||||
} else if (strcmp(argv[i], "-help") == 0) {
|
||||
usage(argv[0]);
|
||||
return 0;
|
||||
} else {
|
||||
fprintf(stderr, "unknown option %s\n", argv[i]);
|
||||
goto bad_args;
|
||||
}
|
||||
} else {
|
||||
if (inputFileName == NULL)
|
||||
inputFileName = argv[i];
|
||||
else if (outputFileName == NULL)
|
||||
outputFileName = argv[i];
|
||||
else {
|
||||
fputs("error: too many parameters specified\n", stderr);
|
||||
goto bad_args;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inputFileName == NULL) {
|
||||
fputs("error: no input file specified\n", stderr);
|
||||
goto bad_args;
|
||||
}
|
||||
if (outputFileName == NULL) {
|
||||
fputs("error: no output file specified\n", stderr);
|
||||
goto bad_args;
|
||||
}
|
||||
if (cicType == -1) {
|
||||
fputs("error: no CIC type specified\n", stderr);
|
||||
goto bad_args;
|
||||
}
|
||||
|
||||
parse_input_file(inputFileName);
|
||||
write_rom_file(outputFileName, cicType);
|
||||
return 0;
|
||||
|
||||
bad_args:
|
||||
usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -1,170 +0,0 @@
|
|||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "elf32.h"
|
||||
|
||||
static uint16_t read16_le(const uint8_t* data) {
|
||||
return data[0] << 0 | data[1] << 8;
|
||||
}
|
||||
|
||||
static uint32_t read32_le(const uint8_t* data) {
|
||||
return data[0] << 0 | data[1] << 8 | data[2] << 16 | data[3] << 24;
|
||||
}
|
||||
|
||||
static uint16_t read16_be(const uint8_t* data) {
|
||||
return data[0] << 8 | data[1] << 0;
|
||||
}
|
||||
|
||||
static uint32_t read32_be(const uint8_t* data) {
|
||||
return data[0] << 24 | data[1] << 16 | data[2] << 8 | data[3] << 0;
|
||||
}
|
||||
|
||||
static const void* get_section_header(struct Elf32* e, int secnum) {
|
||||
size_t secoffset = e->shoff + secnum * 0x28;
|
||||
|
||||
if (secnum >= e->shnum || secoffset >= e->dataSize)
|
||||
return NULL;
|
||||
return e->data + secoffset;
|
||||
}
|
||||
|
||||
static const void* get_section_contents(struct Elf32* e, int secnum) {
|
||||
size_t secoffset = e->shoff + secnum * 0x28;
|
||||
size_t dataoffset;
|
||||
|
||||
if (secnum >= e->shnum || secoffset >= e->dataSize)
|
||||
return NULL;
|
||||
dataoffset = e->read32(e->data + secoffset + 0x10);
|
||||
return e->data + dataoffset;
|
||||
}
|
||||
|
||||
static bool verify_magic(const uint8_t* data) {
|
||||
return (data[0] == 0x7F && data[1] == 'E' && data[2] == 'L' && data[3] == 'F');
|
||||
}
|
||||
|
||||
bool elf32_init(struct Elf32* e, const void* data, size_t size) {
|
||||
unsigned int i;
|
||||
|
||||
e->data = data;
|
||||
e->dataSize = size;
|
||||
|
||||
if (size < 0x34)
|
||||
return false; // not big enough for header
|
||||
|
||||
if (!verify_magic(e->data))
|
||||
return false;
|
||||
|
||||
if (e->data[4] != 1)
|
||||
return false; // must be 32-bit
|
||||
|
||||
e->endian = e->data[5];
|
||||
|
||||
switch (e->endian) {
|
||||
case 1:
|
||||
e->read16 = read16_le;
|
||||
e->read32 = read32_le;
|
||||
break;
|
||||
case 2:
|
||||
e->read16 = read16_be;
|
||||
e->read32 = read32_be;
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
e->type = e->read16(e->data + 0x10);
|
||||
e->machine = e->read16(e->data + 0x12);
|
||||
e->version = e->data[6];
|
||||
e->entry = e->read32(e->data + 0x18);
|
||||
e->phoff = e->read32(e->data + 0x1C);
|
||||
e->shoff = e->read32(e->data + 0x20);
|
||||
e->ehsize = e->read16(e->data + 0x28);
|
||||
e->phentsize = e->read16(e->data + 0x2A);
|
||||
e->phnum = e->read16(e->data + 0x2C);
|
||||
e->shentsize = e->read16(e->data + 0x2E);
|
||||
e->shnum = e->read16(e->data + 0x30);
|
||||
e->shstrndx = e->read16(e->data + 0x32);
|
||||
|
||||
// find symbol table section
|
||||
e->symtabndx = -1;
|
||||
for (i = 0; i < e->shnum; i++) {
|
||||
const uint8_t* sechdr = get_section_header(e, i);
|
||||
uint32_t type = e->read32(sechdr + 0x04);
|
||||
|
||||
if (type == SHT_SYMTAB) {
|
||||
e->symtabndx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// find .strtab section
|
||||
e->strtabndx = -1;
|
||||
for (i = 0; i < e->shnum; i++) {
|
||||
const uint8_t* sechdr = get_section_header(e, i);
|
||||
uint32_t type = e->read32(sechdr + 0x04);
|
||||
|
||||
if (type == SHT_STRTAB) {
|
||||
const char* strings = get_section_contents(e, e->shstrndx);
|
||||
const char* secname = strings + e->read32(sechdr + 0);
|
||||
|
||||
if (strcmp(secname, ".strtab") == 0) {
|
||||
e->strtabndx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
e->numsymbols = 0;
|
||||
if (e->symtabndx != -1) {
|
||||
const uint8_t* sechdr = get_section_header(e, e->symtabndx);
|
||||
// const uint8_t *symtab = get_section_contents(e, e->symtabndx);
|
||||
|
||||
e->numsymbols = e->read32(sechdr + 0x14) / e->read32(sechdr + 0x24);
|
||||
}
|
||||
|
||||
if (e->shoff + e->shstrndx * 0x28 >= e->dataSize)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool elf32_get_section(struct Elf32* e, struct Elf32_Section* sec, int secnum) {
|
||||
const uint8_t* sechdr = get_section_header(e, secnum);
|
||||
const char* strings = get_section_contents(e, e->shstrndx);
|
||||
|
||||
sec->name = strings + e->read32(sechdr + 0);
|
||||
sec->type = e->read32(sechdr + 0x04);
|
||||
sec->flags = e->read32(sechdr + 0x08);
|
||||
sec->addr = e->read32(sechdr + 0x0C);
|
||||
sec->offset = e->read32(sechdr + 0x10);
|
||||
sec->size = e->read32(sechdr + 0x14);
|
||||
sec->addralign = e->read32(sechdr + 0x20);
|
||||
sec->entsize = e->read32(sechdr + 0x24);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool elf32_get_symbol(struct Elf32* e, struct Elf32_Symbol* sym, int symnum) {
|
||||
const uint8_t* sechdr;
|
||||
const uint8_t* symtab;
|
||||
const char* strings;
|
||||
int symcount;
|
||||
|
||||
if (e->symtabndx == -1)
|
||||
return false;
|
||||
|
||||
sechdr = get_section_header(e, e->symtabndx);
|
||||
symtab = get_section_contents(e, e->symtabndx);
|
||||
strings = get_section_contents(e, e->strtabndx);
|
||||
|
||||
symcount = e->read32(sechdr + 0x14) / e->read32(sechdr + 0x24);
|
||||
if (symnum >= symcount)
|
||||
return false;
|
||||
|
||||
sym->name = strings + e->read32(symtab + symnum * 0x10);
|
||||
sym->value = e->read32(symtab + symnum * 0x10 + 4);
|
||||
sym->size = e->read32(symtab + symnum * 0x10 + 8);
|
||||
sym->st_type = symtab[symnum * 0x10 + 0xC] & 0xF;
|
||||
sym->shndx = e->read16(symtab + symnum * 0x10 + 0xE);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
#ifndef ELF32_H
|
||||
#define ELF32_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
enum {
|
||||
ELF_MACHINE_NONE = 0,
|
||||
ELF_MACHINE_MIPS = 8,
|
||||
};
|
||||
|
||||
enum {
|
||||
ELF_TYPE_RELOC = 1,
|
||||
ELF_TYPE_EXEC,
|
||||
ELF_TYPE_SHARED,
|
||||
ELF_TYPE_CORE,
|
||||
};
|
||||
|
||||
struct Elf32 {
|
||||
uint8_t endian;
|
||||
uint16_t type;
|
||||
uint16_t machine;
|
||||
uint32_t version;
|
||||
uint32_t entry;
|
||||
uint32_t phoff;
|
||||
uint32_t shoff;
|
||||
uint16_t ehsize;
|
||||
uint16_t phentsize;
|
||||
uint16_t phnum;
|
||||
uint16_t shentsize;
|
||||
uint16_t shnum;
|
||||
uint16_t shstrndx;
|
||||
int symtabndx;
|
||||
int strtabndx;
|
||||
int numsymbols;
|
||||
|
||||
const uint8_t* data;
|
||||
size_t dataSize;
|
||||
uint16_t (*read16)(const uint8_t*);
|
||||
uint32_t (*read32)(const uint8_t*);
|
||||
};
|
||||
|
||||
enum {
|
||||
SHT_NULL = 0,
|
||||
SHT_PROGBITS,
|
||||
SHT_SYMTAB,
|
||||
SHT_STRTAB,
|
||||
};
|
||||
|
||||
struct Elf32_Section {
|
||||
const char* name;
|
||||
uint32_t type;
|
||||
uint32_t flags;
|
||||
uint32_t addr;
|
||||
uint32_t size;
|
||||
uint32_t offset;
|
||||
uint32_t addralign;
|
||||
uint32_t entsize;
|
||||
};
|
||||
|
||||
#define SHN_UNDEF 0
|
||||
#define SHN_ABS 0xFFF1
|
||||
#define SHN_COMMON 0xFFF2
|
||||
|
||||
#define STT_NOTYPE 0
|
||||
#define STT_OBJECT 1
|
||||
#define STT_FUNC 2
|
||||
#define STT_SECTION 3
|
||||
#define STT_FILE 4
|
||||
#define STT_COMMON 5
|
||||
#define STT_TLS 6
|
||||
#define STT_NUM 7
|
||||
|
||||
struct Elf32_Symbol {
|
||||
const char* name;
|
||||
uint32_t value;
|
||||
uint32_t size;
|
||||
uint8_t st_type;
|
||||
uint16_t shndx;
|
||||
};
|
||||
|
||||
bool elf32_init(struct Elf32* e, const void* data, size_t size);
|
||||
bool elf32_get_section(struct Elf32* e, struct Elf32_Section* sec, int secnum);
|
||||
bool elf32_get_symbol(struct Elf32* e, struct Elf32_Symbol* sym, int symnum);
|
||||
|
||||
#endif
|
||||
|
|
@ -1,304 +0,0 @@
|
|||
#include <assert.h>
|
||||
#include <ctype.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "n64chksum.h"
|
||||
#include "util.h"
|
||||
|
||||
#define ROM_SIZE 0x02000000
|
||||
|
||||
enum InputObjType {
|
||||
OBJ_NULL,
|
||||
OBJ_FILE,
|
||||
OBJ_TABLE,
|
||||
};
|
||||
|
||||
struct InputFile {
|
||||
enum InputObjType type;
|
||||
const char* name;
|
||||
uint8_t* data;
|
||||
size_t size;
|
||||
unsigned int valign;
|
||||
|
||||
uint32_t virtStart;
|
||||
uint32_t virtEnd;
|
||||
uint32_t physStart;
|
||||
uint32_t physEnd;
|
||||
};
|
||||
|
||||
static struct InputFile* g_inputFiles = NULL;
|
||||
static int g_inputFilesCount = 0;
|
||||
|
||||
static unsigned int round_up(unsigned int num, unsigned int multiple) {
|
||||
num += multiple - 1;
|
||||
return num / multiple * multiple;
|
||||
}
|
||||
|
||||
static bool is_yaz0_header(const uint8_t* data) {
|
||||
return data[0] == 'Y' && data[1] == 'a' && data[2] == 'z' && data[3] == '0';
|
||||
}
|
||||
|
||||
static void compute_offsets(void) {
|
||||
size_t physOffset = 0;
|
||||
size_t virtOffset = 0;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < g_inputFilesCount; i++) {
|
||||
bool compressed = false;
|
||||
|
||||
if (g_inputFiles[i].type == OBJ_FILE) {
|
||||
if (is_yaz0_header(g_inputFiles[i].data))
|
||||
compressed = true;
|
||||
} else if (g_inputFiles[i].type == OBJ_TABLE) {
|
||||
g_inputFiles[i].size = g_inputFilesCount * 16;
|
||||
}
|
||||
|
||||
virtOffset = round_up(virtOffset, g_inputFiles[i].valign);
|
||||
|
||||
if (g_inputFiles[i].type == OBJ_NULL) {
|
||||
g_inputFiles[i].virtStart = 0;
|
||||
g_inputFiles[i].virtEnd = 0;
|
||||
g_inputFiles[i].physStart = 0;
|
||||
g_inputFiles[i].physEnd = 0;
|
||||
} else if (compressed) {
|
||||
size_t compSize = round_up(g_inputFiles[i].size, 16);
|
||||
size_t uncompSize = util_read_uint32_be(g_inputFiles[i].data + 4);
|
||||
|
||||
g_inputFiles[i].virtStart = virtOffset;
|
||||
g_inputFiles[i].virtEnd = virtOffset + uncompSize;
|
||||
g_inputFiles[i].physStart = physOffset;
|
||||
g_inputFiles[i].physEnd = physOffset + compSize;
|
||||
|
||||
physOffset += compSize;
|
||||
virtOffset += uncompSize;
|
||||
} else {
|
||||
size_t size = g_inputFiles[i].size;
|
||||
|
||||
g_inputFiles[i].virtStart = virtOffset;
|
||||
g_inputFiles[i].virtEnd = virtOffset + size;
|
||||
g_inputFiles[i].physStart = physOffset;
|
||||
g_inputFiles[i].physEnd = 0;
|
||||
|
||||
physOffset += size;
|
||||
virtOffset += size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void build_rom(const char* filename) {
|
||||
uint8_t* romData = calloc(ROM_SIZE, 1);
|
||||
size_t pos = 0;
|
||||
int i;
|
||||
int j;
|
||||
uint32_t chksum[2];
|
||||
FILE* outFile;
|
||||
|
||||
for (i = 0; i < g_inputFilesCount; i++) {
|
||||
size_t size = g_inputFiles[i].size;
|
||||
|
||||
if (pos + round_up(size, 16) > ROM_SIZE)
|
||||
util_fatal_error("size exceeds max ROM size of 32 KiB");
|
||||
|
||||
assert(pos % 16 == 0);
|
||||
|
||||
switch (g_inputFiles[i].type) {
|
||||
case OBJ_FILE:
|
||||
// write file data
|
||||
memcpy(romData + pos, g_inputFiles[i].data, size);
|
||||
pos += round_up(size, 16);
|
||||
|
||||
free(g_inputFiles[i].data);
|
||||
break;
|
||||
case OBJ_TABLE:
|
||||
for (j = 0; j < g_inputFilesCount; j++) {
|
||||
util_write_uint32_be(romData + pos + 0, g_inputFiles[j].virtStart);
|
||||
util_write_uint32_be(romData + pos + 4, g_inputFiles[j].virtEnd);
|
||||
util_write_uint32_be(romData + pos + 8, g_inputFiles[j].physStart);
|
||||
util_write_uint32_be(romData + pos + 12, g_inputFiles[j].physEnd);
|
||||
|
||||
pos += 16;
|
||||
}
|
||||
break;
|
||||
case OBJ_NULL:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Pad the rest of the ROM
|
||||
while (pos < ROM_SIZE) {
|
||||
// This is such a weird thing to pad with. Whatever, Nintendo.
|
||||
romData[pos] = pos & 0xFF;
|
||||
pos++;
|
||||
}
|
||||
|
||||
// calculate checksum
|
||||
n64chksum_calculate(romData, 6105, chksum);
|
||||
util_write_uint32_be(romData + 0x10, chksum[0]);
|
||||
util_write_uint32_be(romData + 0x14, chksum[1]);
|
||||
|
||||
// write file
|
||||
outFile = fopen(filename, "wb");
|
||||
if (outFile == NULL)
|
||||
util_fatal_error("failed to open file '%s' for writing", filename);
|
||||
fwrite(romData, ROM_SIZE, 1, outFile);
|
||||
fclose(outFile);
|
||||
|
||||
free(romData);
|
||||
}
|
||||
|
||||
static struct InputFile* new_file(void) {
|
||||
int index = g_inputFilesCount;
|
||||
|
||||
g_inputFilesCount++;
|
||||
g_inputFiles = realloc(g_inputFiles, g_inputFilesCount * sizeof(*g_inputFiles));
|
||||
|
||||
g_inputFiles[index].valign = 1;
|
||||
|
||||
return &g_inputFiles[index];
|
||||
}
|
||||
|
||||
// null terminates the current token and returns a pointer to the next token
|
||||
static char* token_split(char* str) {
|
||||
while (!isspace(*str)) {
|
||||
if (*str == 0)
|
||||
return str; // end of string
|
||||
str++;
|
||||
}
|
||||
*str = 0; // terminate token
|
||||
str++;
|
||||
|
||||
// skip remaining whitespace
|
||||
while (isspace(*str))
|
||||
str++;
|
||||
return str;
|
||||
}
|
||||
|
||||
// null terminates the current line and returns a pointer to the next line
|
||||
static char* line_split(char* str) {
|
||||
while (*str != '\n') {
|
||||
if (*str == 0)
|
||||
return str; // end of string
|
||||
str++;
|
||||
}
|
||||
*str = 0; // terminate line
|
||||
return str + 1;
|
||||
}
|
||||
|
||||
static void parse_line(char* line, int lineNum) {
|
||||
char* token = line;
|
||||
int i = 0;
|
||||
|
||||
char* filename = NULL;
|
||||
enum InputObjType type = -1;
|
||||
int valign = 1;
|
||||
struct InputFile* file;
|
||||
|
||||
// iterate through each token
|
||||
while (token[0] != 0) {
|
||||
char* nextToken = token_split(token);
|
||||
|
||||
if (token[0] == '#') // comment - ignore rest of line
|
||||
return;
|
||||
|
||||
switch (i) {
|
||||
case 0:
|
||||
if (strcmp(token, "file") == 0)
|
||||
type = OBJ_FILE;
|
||||
else if (strcmp(token, "filetable") == 0)
|
||||
type = OBJ_TABLE;
|
||||
else if (strcmp(token, "null") == 0)
|
||||
type = OBJ_NULL;
|
||||
else
|
||||
util_fatal_error("unknown object type '%s' on line %i", token, lineNum);
|
||||
break;
|
||||
case 1:
|
||||
filename = token;
|
||||
break;
|
||||
case 2: {
|
||||
int n;
|
||||
|
||||
if (sscanf(token, "align(%i)", &n) == 1)
|
||||
valign = n;
|
||||
else
|
||||
goto junk;
|
||||
} break;
|
||||
default:
|
||||
junk:
|
||||
util_fatal_error("junk '%s' on line %i", token, lineNum);
|
||||
break;
|
||||
}
|
||||
|
||||
token = nextToken;
|
||||
i++;
|
||||
}
|
||||
|
||||
if (i == 0) // empty line
|
||||
return;
|
||||
|
||||
file = new_file();
|
||||
file->valign = valign;
|
||||
|
||||
switch (type) {
|
||||
case OBJ_FILE:
|
||||
if (filename == NULL)
|
||||
util_fatal_error("no filename specified on line %i", lineNum);
|
||||
file->type = OBJ_FILE;
|
||||
file->data = util_read_whole_file(filename, &file->size);
|
||||
break;
|
||||
case OBJ_TABLE:
|
||||
file->type = OBJ_TABLE;
|
||||
break;
|
||||
case OBJ_NULL:
|
||||
file->type = OBJ_NULL;
|
||||
file->size = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static void parse_list(char* list) {
|
||||
char* line = list;
|
||||
int lineNum = 1;
|
||||
|
||||
// iterate through each line
|
||||
while (line[0] != 0) {
|
||||
char* nextLine = line_split(line);
|
||||
|
||||
parse_line(line, lineNum);
|
||||
|
||||
line = nextLine;
|
||||
lineNum++;
|
||||
}
|
||||
}
|
||||
|
||||
static void usage(const char* execName) {
|
||||
printf("usage: %s FILE_LIST OUTPUT_FILE\n"
|
||||
"where FILE_LIST is a list of files to include\n"
|
||||
"and OUTPUT_FILE is the name of the output ROM\n"
|
||||
"note that 'dmadata' refers to the file list itself and not an external file\n",
|
||||
execName);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
char* list;
|
||||
|
||||
if (argc != 3) {
|
||||
puts("invalid args");
|
||||
usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
list = util_read_whole_file(argv[1], NULL);
|
||||
|
||||
parse_list(list);
|
||||
compute_offsets();
|
||||
build_rom(argv[2]);
|
||||
|
||||
free(list);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1,274 +0,0 @@
|
|||
/* SPDX-FileCopyrightText: © 2023 ZeldaRET */
|
||||
/* SPDX-License-Identifier: MIT */
|
||||
|
||||
/**
|
||||
* Program to generate compressed yar (Yaz0 ARchive) files.
|
||||
*
|
||||
* The program expects an .o elf file and outputs a raw yar binary file and a
|
||||
* "symbols" elf.
|
||||
*
|
||||
* A yar file consists of multiple Yaz0 files compressed individually. The
|
||||
* archive begins with a header of non-fixed size, which describes the
|
||||
* location of each individual Yaz0 block within the archive itself. This
|
||||
* header is followed by each Yaz0 file.
|
||||
*
|
||||
* The first word (a 4 byte group) of the header indicates the size in bytes of
|
||||
* the header itself (also describes the offset of the first Yaz0 block). The
|
||||
* rest of the header consists of words describing the offsets of each Yaz0
|
||||
* block relative to the end of the header, because the first Yaz0
|
||||
* block is omitted from the offsets in the header.
|
||||
*
|
||||
* Each Yaz0 block is 0xFF-padded to a multiple of 0x10 in size.
|
||||
*
|
||||
* The entire archive is 0-padded to a multiple of 0x10 in size.
|
||||
*
|
||||
* The program works by compressing each .data symbol in the input elf file as
|
||||
* its own Yaz0 compressed file, appending them in order for the generated
|
||||
* archive. Other elf sections are ignored for the resulting yar file.
|
||||
*
|
||||
* The program also outputs an elf file that's identical to the elf input,
|
||||
* but with its .data section zero'ed out completely. This "symbols" elf can be
|
||||
* used for referencing each symbol as the whole file were completely
|
||||
* uncompressed.
|
||||
*/
|
||||
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "elf32.h"
|
||||
#include "yaz0.h"
|
||||
#include "util.h"
|
||||
|
||||
|
||||
typedef struct Bytearray {
|
||||
uint8_t *bytes;
|
||||
size_t size;
|
||||
} Bytearray;
|
||||
|
||||
typedef struct SymbolList {
|
||||
struct Elf32_Symbol *symbols;
|
||||
size_t size; // allocated size
|
||||
size_t len; // elements in the list
|
||||
} SymbolList;
|
||||
|
||||
typedef struct DataSection {
|
||||
Bytearray data;
|
||||
uint32_t dataOffset;
|
||||
SymbolList symbols;
|
||||
} DataSection;
|
||||
|
||||
|
||||
void Bytearray_Init(Bytearray *bytearr, const uint8_t *bytes, size_t size) {
|
||||
bytearr->bytes = malloc(size);
|
||||
if (bytearr->bytes == NULL) {
|
||||
util_fatal_error("memory error");
|
||||
}
|
||||
|
||||
memcpy(bytearr->bytes, bytes, size);
|
||||
bytearr->size = size;
|
||||
}
|
||||
|
||||
void Bytearray_InitValue(Bytearray *bytearr, uint8_t val, size_t count) {
|
||||
bytearr->bytes = malloc(count * sizeof(uint8_t));
|
||||
if (bytearr->bytes == NULL) {
|
||||
util_fatal_error("memory error");
|
||||
}
|
||||
|
||||
memset(bytearr->bytes, val, count);
|
||||
bytearr->size = count;
|
||||
}
|
||||
|
||||
void Bytearray_ExtendValue(Bytearray *bytearr, uint8_t val, size_t count) {
|
||||
size_t newSize = bytearr->size + count;
|
||||
|
||||
bytearr->bytes = realloc(bytearr->bytes, newSize);
|
||||
if (bytearr->bytes == NULL) {
|
||||
util_fatal_error("memory error");
|
||||
}
|
||||
|
||||
memset(&bytearr->bytes[bytearr->size], val, count);
|
||||
bytearr->size = newSize;
|
||||
}
|
||||
|
||||
void Bytearray_Extend(Bytearray *bytearr, const uint8_t *bytes, size_t size) {
|
||||
size_t newSize = bytearr->size + size;
|
||||
|
||||
bytearr->bytes = realloc(bytearr->bytes, newSize);
|
||||
if (bytearr->bytes == NULL) {
|
||||
util_fatal_error("memory error");
|
||||
}
|
||||
|
||||
memcpy(&bytearr->bytes[bytearr->size], bytes, size);
|
||||
bytearr->size = newSize;
|
||||
}
|
||||
|
||||
void Bytearray_Destroy(Bytearray *bytearr) {
|
||||
free(bytearr->bytes);
|
||||
}
|
||||
|
||||
void SymbolList_Init(SymbolList *list, size_t initialAmount) {
|
||||
list->symbols = malloc(initialAmount * sizeof(struct Elf32_Symbol));
|
||||
if (list->symbols == NULL) {
|
||||
util_fatal_error("memory error");
|
||||
}
|
||||
list->size = initialAmount;
|
||||
list->len = 0;
|
||||
}
|
||||
|
||||
void SymbolList_Destroy(SymbolList *list) {
|
||||
free(list->symbols);
|
||||
}
|
||||
|
||||
void DataSection_FromElf(DataSection *dst, const Bytearray *elfBytes){
|
||||
struct Elf32 elf;
|
||||
size_t i;
|
||||
int symIndex;
|
||||
size_t dataShndx = 0;
|
||||
|
||||
if (!elf32_init(&elf, elfBytes->bytes, elfBytes->size) || (elf.machine != ELF_MACHINE_MIPS)) {
|
||||
util_fatal_error("not a valid 32-bit MIPS ELF file");
|
||||
}
|
||||
|
||||
for (i = 0; i < elf.shnum; i++) {
|
||||
struct Elf32_Section sec;
|
||||
|
||||
if (!elf32_get_section(&elf, &sec, i)) {
|
||||
util_fatal_error("invalid or corrupt ELF file");
|
||||
}
|
||||
|
||||
if (strcmp(sec.name, ".data") == 0) {
|
||||
dst->dataOffset = sec.offset;
|
||||
Bytearray_Init(&dst->data, &elfBytes->bytes[sec.offset], sec.size);
|
||||
dataShndx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
SymbolList_Init(&dst->symbols, elf.numsymbols);
|
||||
for (symIndex = 0; symIndex < elf.numsymbols; symIndex++) {
|
||||
struct Elf32_Symbol sym;
|
||||
|
||||
if (!elf32_get_symbol(&elf, &sym, symIndex)) {
|
||||
util_fatal_error("invalid or corrupt ELF file");
|
||||
}
|
||||
|
||||
if (sym.shndx != dataShndx) {
|
||||
continue;
|
||||
}
|
||||
if (sym.st_type != STT_OBJECT) {
|
||||
continue;
|
||||
}
|
||||
|
||||
dst->symbols.symbols[dst->symbols.len++] = sym;
|
||||
}
|
||||
}
|
||||
|
||||
void DataSection_Destroy(DataSection *dataSect) {
|
||||
Bytearray_Destroy(&dataSect->data);
|
||||
SymbolList_Destroy(&dataSect->symbols);
|
||||
}
|
||||
|
||||
#define ALIGN16(val) (((val) + 0xF) & ~0xF)
|
||||
|
||||
void createArchive(Bytearray *archive, const DataSection *dataSect) {
|
||||
uint32_t firstEntryOffset = (dataSect->symbols.len + 1) * sizeof(uint32_t);
|
||||
size_t i;
|
||||
size_t offset;
|
||||
|
||||
// Fill with zeroes until the compressed data start
|
||||
Bytearray_InitValue(archive, 0, firstEntryOffset);
|
||||
|
||||
util_write_uint32_be(&archive->bytes[0], firstEntryOffset);
|
||||
|
||||
offset = firstEntryOffset;
|
||||
for (i = 0; i < dataSect->symbols.len; i++) {
|
||||
const struct Elf32_Symbol *sym = &dataSect->symbols.symbols[i];
|
||||
size_t realUncompressedSize = sym->size;
|
||||
size_t alignedUncompressedSize = ALIGN16(realUncompressedSize);
|
||||
uint8_t *inputBuf = malloc(alignedUncompressedSize* sizeof(uint8_t));
|
||||
uint8_t *output = malloc(alignedUncompressedSize * sizeof(uint8_t)); // assume compressed shouldn't be bigger than uncompressed
|
||||
size_t compressedSize;
|
||||
|
||||
// Make sure to pad each entry to a 0x10 boundary
|
||||
memcpy(inputBuf, &dataSect->data.bytes[sym->value], realUncompressedSize);
|
||||
if (realUncompressedSize < alignedUncompressedSize) {
|
||||
memset(&inputBuf[realUncompressedSize], 0, alignedUncompressedSize - realUncompressedSize);
|
||||
}
|
||||
|
||||
output[0] = 'Y';
|
||||
output[1] = 'a';
|
||||
output[2] = 'z';
|
||||
output[3] = '0';
|
||||
util_write_uint32_be(&output[4], alignedUncompressedSize);
|
||||
memset(&output[8], 0, 8);
|
||||
compressedSize = 0x10;
|
||||
|
||||
compressedSize += yaz0_encode(inputBuf, &output[0x10], alignedUncompressedSize);
|
||||
|
||||
// Pad to 0x10
|
||||
while (compressedSize % 0x10 != 0) {
|
||||
output[compressedSize++] = 0xFF;
|
||||
}
|
||||
|
||||
Bytearray_Extend(archive, output, compressedSize);
|
||||
|
||||
if (i > 0) {
|
||||
util_write_uint32_be(&archive->bytes[i * sizeof(uint32_t)], offset - firstEntryOffset);
|
||||
}
|
||||
|
||||
offset += compressedSize;
|
||||
free(output);
|
||||
}
|
||||
|
||||
util_write_uint32_be(&archive->bytes[i * sizeof(uint32_t)], offset - firstEntryOffset);
|
||||
|
||||
if (archive->size % 16 != 0) {
|
||||
size_t extraPad = ALIGN16(archive->size) - archive->size;
|
||||
|
||||
Bytearray_ExtendValue(archive, 0, extraPad);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
const char *inPath;
|
||||
const char *binPath;
|
||||
const char *symPath;
|
||||
Bytearray elfBytes;
|
||||
DataSection dataSect;
|
||||
Bytearray archive;
|
||||
|
||||
if (argc != 4) {
|
||||
fprintf(stderr, "%s in_file out_bin out_sym\n", argv[0]);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
inPath = argv[1];
|
||||
binPath = argv[2];
|
||||
symPath = argv[3];
|
||||
|
||||
elfBytes.bytes = util_read_whole_file(inPath, &elfBytes.size);
|
||||
|
||||
DataSection_FromElf(&dataSect, &elfBytes);
|
||||
|
||||
createArchive(&archive, &dataSect);
|
||||
|
||||
// Write the compressed archive file as a raw binary
|
||||
util_write_whole_file(binPath, archive.bytes, archive.size);
|
||||
|
||||
// Zero out data
|
||||
memset(&elfBytes.bytes[dataSect.dataOffset], 0, dataSect.data.size);
|
||||
|
||||
util_write_whole_file(symPath, elfBytes.bytes, elfBytes.size);
|
||||
|
||||
Bytearray_Destroy(&archive);
|
||||
DataSection_Destroy(&dataSect);
|
||||
Bytearray_Destroy(&elfBytes);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-FileCopyrightText: © 2023-2024 ZeldaRET
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
# Program to generate compressed yar (Yaz0 ARchive) files.
|
||||
#
|
||||
# The program expects an .o elf file and outputs a raw yar binary file and a
|
||||
# "symbols" elf.
|
||||
#
|
||||
# A yar file consists of multiple Yaz0 files compressed individually. The
|
||||
# archive begins with a header of non-fixed size, which describes the
|
||||
# location of each individual Yaz0 block within the archive itself. This
|
||||
# header is followed by each Yaz0 file.
|
||||
#
|
||||
# The first word (a 4 byte group) of the header indicates the size in bytes of
|
||||
# the header itself (also describes the offset of the first Yaz0 block). The
|
||||
# rest of the header consists of words describing the offsets of each Yaz0
|
||||
# block relative to the end of the header, because the first Yaz0
|
||||
# block is omitted from the offsets in the header.
|
||||
#
|
||||
# Each Yaz0 block is 0xFF-padded to a multiple of 0x10 in size.
|
||||
#
|
||||
# The entire archive is 0-padded to a multiple of 0x10 in size.
|
||||
#
|
||||
# The program works by compressing each .data symbol in the input elf file as
|
||||
# its own Yaz0 compressed file, appending them in order for the generated
|
||||
# archive. Other elf sections are ignored for the resulting yar file.
|
||||
#
|
||||
# The program also outputs an elf file that's identical to the elf input,
|
||||
# but with its .data section zero'ed out completely. This "symbols" elf can be
|
||||
# used for referencing each symbol as the whole file were completely
|
||||
# uncompressed.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import crunch64
|
||||
|
||||
from elftools.elf.elffile import ELFFile
|
||||
from elftools.elf.sections import SymbolTableSection
|
||||
|
||||
|
||||
def write_word_as_bytes(buff: bytearray, offset: int, word: int):
|
||||
struct.pack_into(f">I", buff, offset, word)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Symbol:
|
||||
name: str
|
||||
offset: int
|
||||
size: int
|
||||
|
||||
|
||||
def get_data_from_elf(elf_path: Path) -> tuple[bytearray, list[Symbol], int]:
|
||||
uncompressed_data = bytearray()
|
||||
symbol_list: list[Symbol] = []
|
||||
data_offset = -1
|
||||
|
||||
with elf_path.open("rb") as elfFile:
|
||||
elf = ELFFile(elfFile)
|
||||
for section in elf.iter_sections():
|
||||
if section.name == ".data":
|
||||
assert len(uncompressed_data) == 0
|
||||
uncompressed_data.extend(section.data())
|
||||
assert len(uncompressed_data) == section["sh_size"]
|
||||
data_offset = section["sh_offset"]
|
||||
elif section.name == ".symtab":
|
||||
assert isinstance(section, SymbolTableSection)
|
||||
for sym in section.iter_symbols():
|
||||
if sym["st_shndx"] == "SHN_UNDEF":
|
||||
continue
|
||||
if sym["st_info"]["type"] != "STT_OBJECT":
|
||||
continue
|
||||
symbol_list.append(
|
||||
Symbol(sym.name, sym["st_value"], sym["st_size"])
|
||||
)
|
||||
return uncompressed_data, symbol_list, data_offset
|
||||
|
||||
|
||||
def align_16(val: int) -> int:
|
||||
return (val + 0xF) & ~0xF
|
||||
|
||||
|
||||
def create_archive(
|
||||
uncompressed_data: bytearray, symbol_list: list[Symbol]
|
||||
) -> bytearray:
|
||||
archive = bytearray()
|
||||
|
||||
first_entry_offset = (len(symbol_list) + 1) * 4
|
||||
|
||||
# Fill with zeroes until the compressed data start
|
||||
archive.extend([0] * first_entry_offset)
|
||||
|
||||
write_word_as_bytes(archive, 0, first_entry_offset)
|
||||
|
||||
offset = first_entry_offset
|
||||
|
||||
i = 0
|
||||
for sym in symbol_list:
|
||||
uncompressed_size = sym.size
|
||||
uncompressed_size_aligned = align_16(uncompressed_size)
|
||||
|
||||
input_buf = uncompressed_data[sym.offset : sym.offset + uncompressed_size]
|
||||
# Make sure to pad each entry to a 0x10 boundary
|
||||
if uncompressed_size_aligned > uncompressed_size:
|
||||
input_buf.extend([0x00] * (uncompressed_size_aligned - uncompressed_size))
|
||||
|
||||
compressed = bytearray(crunch64.yaz0.compress(input_buf))
|
||||
compressed_size = len(compressed)
|
||||
|
||||
# Pad to 0x10
|
||||
compressed_size_aligned = align_16(compressed_size)
|
||||
if compressed_size_aligned > compressed_size:
|
||||
compressed.extend([0xFF] * (compressed_size_aligned - compressed_size))
|
||||
|
||||
archive.extend(compressed)
|
||||
|
||||
if i > 0:
|
||||
write_word_as_bytes(archive, i * 4, offset - first_entry_offset)
|
||||
|
||||
i += 1
|
||||
offset += len(compressed)
|
||||
|
||||
write_word_as_bytes(archive, i * 4, offset - first_entry_offset)
|
||||
|
||||
archive_len = len(archive)
|
||||
archive_len_aligned = align_16(archive_len)
|
||||
if archive_len_aligned > archive_len:
|
||||
archive.extend([0x00] * (archive_len_aligned - archive_len))
|
||||
|
||||
return archive
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Program to generate compressed yar (Yaz0 ARchive) files from a built C file. Said file must only contain data symbols that do not reference other symbols (i.e. textures)."
|
||||
)
|
||||
parser.add_argument("in_file", help="Path to built .o file")
|
||||
parser.add_argument(
|
||||
"out_bin", help="Output path for the generated compressed yar binary"
|
||||
)
|
||||
parser.add_argument("out_sym", help="Output path for the generated syms elf file")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
in_path = Path(args.in_file)
|
||||
out_bin_path = Path(args.out_bin)
|
||||
out_sym_path = Path(args.out_sym)
|
||||
|
||||
# Delete output files if they already exist
|
||||
out_bin_path.unlink(missing_ok=True)
|
||||
out_sym_path.unlink(missing_ok=True)
|
||||
|
||||
elf_bytes = bytearray(in_path.read_bytes())
|
||||
|
||||
uncompressed_data, symbol_list, data_offset = get_data_from_elf(in_path)
|
||||
assert len(uncompressed_data) > 0
|
||||
assert len(symbol_list) > 0
|
||||
assert data_offset > 0
|
||||
|
||||
archive = create_archive(uncompressed_data, symbol_list)
|
||||
|
||||
# Write the compressed archive file as a raw binary
|
||||
out_bin_path.write_bytes(archive)
|
||||
|
||||
# Zero out data
|
||||
for i in range(data_offset, data_offset + len(uncompressed_data)):
|
||||
elf_bytes[i] = 0
|
||||
|
||||
out_sym_path.write_bytes(elf_bytes)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
#include <assert.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "n64chksum.h"
|
||||
#include "util.h"
|
||||
|
||||
// Based on uCON64's N64 checksum algorithm by Andreas Sterbenz
|
||||
|
||||
#define ROL(i, b) (((i) << (b)) | ((i) >> (32 - (b))))
|
||||
|
||||
bool n64chksum_calculate(const uint8_t* romData, int cicType, uint32_t* chksum) {
|
||||
unsigned int seed;
|
||||
unsigned int t1, t2, t3, t4, t5, t6;
|
||||
size_t pos;
|
||||
|
||||
const size_t START = 0x1000;
|
||||
const size_t END = START + 0x100000;
|
||||
|
||||
// determine initial seed
|
||||
switch (cicType) {
|
||||
case 6101:
|
||||
case 6102:
|
||||
seed = 0xF8CA4DDC;
|
||||
break;
|
||||
case 6103:
|
||||
seed = 0xA3886759;
|
||||
break;
|
||||
case 6105:
|
||||
seed = 0xDF26F436;
|
||||
break;
|
||||
case 6106:
|
||||
seed = 0x1FEA617A;
|
||||
break;
|
||||
default:
|
||||
return false; // unknown CIC type
|
||||
}
|
||||
|
||||
t1 = t2 = t3 = t4 = t5 = t6 = seed;
|
||||
|
||||
for (pos = START; pos < END; pos += 4) {
|
||||
unsigned int d = util_read_uint32_be(romData + pos);
|
||||
unsigned int r = ROL(d, (d & 0x1F));
|
||||
|
||||
// increment t4 if t6 overflows
|
||||
if ((t6 + d) < t6)
|
||||
t4++;
|
||||
|
||||
t6 += d;
|
||||
t3 ^= d;
|
||||
t5 += r;
|
||||
|
||||
if (t2 > d)
|
||||
t2 ^= r;
|
||||
else
|
||||
t2 ^= t6 ^ d;
|
||||
|
||||
if (cicType == 6105)
|
||||
t1 += util_read_uint32_be(&romData[0x0750 + (pos & 0xFF)]) ^ d;
|
||||
else
|
||||
t1 += t5 ^ d;
|
||||
}
|
||||
|
||||
if (cicType == 6103) {
|
||||
chksum[0] = (t6 ^ t4) + t3;
|
||||
chksum[1] = (t5 ^ t2) + t1;
|
||||
} else if (cicType == 6106) {
|
||||
chksum[0] = (t6 * t4) + t3;
|
||||
chksum[1] = (t5 * t2) + t1;
|
||||
} else {
|
||||
chksum[0] = t6 ^ t4 ^ t3;
|
||||
chksum[1] = t5 ^ t2 ^ t1;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
#ifndef N64CHKSUM_H
|
||||
#define N64CHKSUM_H
|
||||
|
||||
bool n64chksum_calculate(const uint8_t* romData, int cicType, uint32_t* chksum);
|
||||
|
||||
#endif
|
||||
|
|
@ -1,212 +0,0 @@
|
|||
#include <assert.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "yaz0.h"
|
||||
|
||||
// decoder implementation by thakis of http://www.amnoid.de
|
||||
|
||||
// src points to the yaz0 source data (to the "real" source data, not at the header!)
|
||||
// dst points to a buffer uncompressedSize bytes large (you get uncompressedSize from
|
||||
// the second 4 bytes in the Yaz0 header).
|
||||
void yaz0_decode(uint8_t* src, uint8_t* dst, int uncompressedSize) {
|
||||
int srcPlace = 0, dstPlace = 0; // current read/write positions
|
||||
|
||||
unsigned int validBitCount = 0; // number of valid bits left in "code" byte
|
||||
uint8_t currCodeByte;
|
||||
while (dstPlace < uncompressedSize) {
|
||||
// read new "code" byte if the current one is used up
|
||||
if (validBitCount == 0) {
|
||||
currCodeByte = src[srcPlace];
|
||||
++srcPlace;
|
||||
validBitCount = 8;
|
||||
}
|
||||
|
||||
if ((currCodeByte & 0x80) != 0) {
|
||||
// straight copy
|
||||
dst[dstPlace] = src[srcPlace];
|
||||
dstPlace++;
|
||||
srcPlace++;
|
||||
} else {
|
||||
// RLE part
|
||||
uint8_t byte1 = src[srcPlace];
|
||||
uint8_t byte2 = src[srcPlace + 1];
|
||||
srcPlace += 2;
|
||||
|
||||
unsigned int dist = ((byte1 & 0xF) << 8) | byte2;
|
||||
unsigned int copySource = dstPlace - (dist + 1);
|
||||
|
||||
unsigned int numBytes = byte1 >> 4;
|
||||
if (numBytes == 0) {
|
||||
numBytes = src[srcPlace] + 0x12;
|
||||
srcPlace++;
|
||||
} else {
|
||||
numBytes += 2;
|
||||
}
|
||||
|
||||
// copy run
|
||||
for (unsigned int i = 0; i < numBytes; ++i) {
|
||||
dst[dstPlace] = dst[copySource];
|
||||
copySource++;
|
||||
dstPlace++;
|
||||
}
|
||||
}
|
||||
|
||||
// use next bit from "code" byte
|
||||
currCodeByte <<= 1;
|
||||
validBitCount -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
// encoder implementation by shevious, with bug fixes by notwa
|
||||
|
||||
typedef uint32_t uint32_t;
|
||||
typedef uint8_t uint8_t;
|
||||
|
||||
#define MAX_RUNLEN (0xFF + 0x12)
|
||||
|
||||
// simple and straight encoding scheme for Yaz0
|
||||
static uint32_t simpleEnc(uint8_t* src, int size, int pos, uint32_t* pMatchPos) {
|
||||
int numBytes = 1;
|
||||
int matchPos = 0;
|
||||
|
||||
int startPos = pos - 0x1000;
|
||||
int end = size - pos;
|
||||
|
||||
if (startPos < 0)
|
||||
startPos = 0;
|
||||
|
||||
// maximum runlength for 3 byte encoding
|
||||
if (end > MAX_RUNLEN)
|
||||
end = MAX_RUNLEN;
|
||||
|
||||
for (int i = startPos; i < pos; i++) {
|
||||
int j;
|
||||
|
||||
for (j = 0; j < end; j++) {
|
||||
if (src[i + j] != src[j + pos])
|
||||
break;
|
||||
}
|
||||
if (j > numBytes) {
|
||||
numBytes = j;
|
||||
matchPos = i;
|
||||
}
|
||||
}
|
||||
|
||||
*pMatchPos = matchPos;
|
||||
|
||||
if (numBytes == 2)
|
||||
numBytes = 1;
|
||||
|
||||
return numBytes;
|
||||
}
|
||||
|
||||
// a lookahead encoding scheme for ngc Yaz0
|
||||
static uint32_t nintendoEnc(uint8_t* src, int size, int pos, uint32_t* pMatchPos) {
|
||||
uint32_t numBytes = 1;
|
||||
static uint32_t numBytes1;
|
||||
static uint32_t matchPos;
|
||||
static int prevFlag = 0;
|
||||
|
||||
// if prevFlag is set, it means that the previous position
|
||||
// was determined by look-ahead try.
|
||||
// so just use it. this is not the best optimization,
|
||||
// but nintendo's choice for speed.
|
||||
if (prevFlag == 1) {
|
||||
*pMatchPos = matchPos;
|
||||
prevFlag = 0;
|
||||
return numBytes1;
|
||||
}
|
||||
|
||||
prevFlag = 0;
|
||||
numBytes = simpleEnc(src, size, pos, &matchPos);
|
||||
*pMatchPos = matchPos;
|
||||
|
||||
// if this position is RLE encoded, then compare to copying 1 byte and next position(pos+1) encoding
|
||||
if (numBytes >= 3) {
|
||||
numBytes1 = simpleEnc(src, size, pos + 1, &matchPos);
|
||||
// if the next position encoding is +2 longer than current position, choose it.
|
||||
// this does not guarantee the best optimization, but fairly good optimization with speed.
|
||||
if (numBytes1 >= numBytes + 2) {
|
||||
numBytes = 1;
|
||||
prevFlag = 1;
|
||||
}
|
||||
}
|
||||
return numBytes;
|
||||
}
|
||||
|
||||
int yaz0_encode(uint8_t* src, uint8_t* dst, int srcSize) {
|
||||
int srcPos = 0;
|
||||
int dstPos = 0;
|
||||
int bufPos = 0;
|
||||
|
||||
uint8_t buf[24]; // 8 codes * 3 bytes maximum
|
||||
|
||||
uint32_t validBitCount = 0; // number of valid bits left in "code" byte
|
||||
uint8_t currCodeByte = 0; // a bitfield, set bits meaning copy, unset meaning RLE
|
||||
|
||||
while (srcPos < srcSize) {
|
||||
uint32_t numBytes;
|
||||
uint32_t matchPos;
|
||||
|
||||
numBytes = nintendoEnc(src, srcSize, srcPos, &matchPos);
|
||||
if (numBytes < 3) {
|
||||
// straight copy
|
||||
buf[bufPos] = src[srcPos];
|
||||
bufPos++;
|
||||
srcPos++;
|
||||
// set flag for straight copy
|
||||
currCodeByte |= (0x80 >> validBitCount);
|
||||
} else {
|
||||
// RLE part
|
||||
uint32_t dist = srcPos - matchPos - 1;
|
||||
uint8_t byte1, byte2, byte3;
|
||||
|
||||
if (numBytes >= 0x12) // 3 byte encoding
|
||||
{
|
||||
byte1 = 0 | (dist >> 8);
|
||||
byte2 = dist & 0xFF;
|
||||
buf[bufPos++] = byte1;
|
||||
buf[bufPos++] = byte2;
|
||||
// maximum runlength for 3 byte encoding
|
||||
if (numBytes > MAX_RUNLEN)
|
||||
numBytes = MAX_RUNLEN;
|
||||
byte3 = numBytes - 0x12;
|
||||
buf[bufPos++] = byte3;
|
||||
} else // 2 byte encoding
|
||||
{
|
||||
byte1 = ((numBytes - 2) << 4) | (dist >> 8);
|
||||
byte2 = dist & 0xFF;
|
||||
buf[bufPos++] = byte1;
|
||||
buf[bufPos++] = byte2;
|
||||
}
|
||||
srcPos += numBytes;
|
||||
}
|
||||
|
||||
validBitCount++;
|
||||
|
||||
// write eight codes
|
||||
if (validBitCount == 8) {
|
||||
dst[dstPos++] = currCodeByte;
|
||||
for (int j = 0; j < bufPos; j++)
|
||||
dst[dstPos++] = buf[j];
|
||||
|
||||
currCodeByte = 0;
|
||||
validBitCount = 0;
|
||||
bufPos = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (validBitCount > 0) {
|
||||
dst[dstPos++] = currCodeByte;
|
||||
for (int j = 0; j < bufPos; j++)
|
||||
dst[dstPos++] = buf[j];
|
||||
|
||||
currCodeByte = 0;
|
||||
validBitCount = 0;
|
||||
bufPos = 0;
|
||||
}
|
||||
|
||||
return dstPos;
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
#ifndef YAZ0_H
|
||||
#define YAZ0_H
|
||||
|
||||
void yaz0_decode(uint8_t* src, uint8_t* dst, int uncompressedSize);
|
||||
|
||||
int yaz0_encode(uint8_t* src, uint8_t* dest, int srcSize);
|
||||
|
||||
#endif // YAZ0_H
|
||||
|
|
@ -1,201 +0,0 @@
|
|||
#ifdef __linux__
|
||||
#define _POSIX_C_SOURCE 199309L
|
||||
#endif
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "yaz0.h"
|
||||
#include "util.h"
|
||||
|
||||
// TODO: Windows support
|
||||
static unsigned long int get_time_milliseconds(void)
|
||||
{
|
||||
#ifdef __linux__
|
||||
struct timespec tspec;
|
||||
|
||||
clock_gettime(CLOCK_MONOTONIC, &tspec);
|
||||
return (tspec.tv_sec * 1000) + tspec.tv_nsec / 1000000;
|
||||
#else
|
||||
// dummy
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static void print_report(unsigned long int time, size_t compSize, size_t uncompSize)
|
||||
{
|
||||
unsigned int minutes = time / (1000 * 60);
|
||||
float seconds = (float)(time % (1000 * 60)) / 1000;
|
||||
|
||||
printf("compression ratio: %.2fKiB / %.2fKiB (%.2f%%)\n"
|
||||
"time: %um %.3fs\n",
|
||||
(float)compSize / 1024, (float)uncompSize / 1024,
|
||||
(float)compSize * 100 / (float)uncompSize,
|
||||
minutes, seconds);
|
||||
}
|
||||
|
||||
static void compress_file(const char *inputFileName, const char *outputFileName, bool verbose)
|
||||
{
|
||||
size_t uncompSize;
|
||||
uint8_t *input = util_read_whole_file(inputFileName, &uncompSize);
|
||||
uint8_t *output = malloc(uncompSize * 2); // TODO: figure out how much space we need
|
||||
unsigned long int time;
|
||||
|
||||
if (verbose)
|
||||
{
|
||||
printf("decompressing %s\n", inputFileName);
|
||||
time = get_time_milliseconds();
|
||||
}
|
||||
|
||||
// compress data
|
||||
size_t compSize = yaz0_encode(input, output, uncompSize);
|
||||
|
||||
if (verbose)
|
||||
time = get_time_milliseconds() - time;
|
||||
|
||||
// make Yaz0 header
|
||||
uint8_t header[16] = {0};
|
||||
header[0] = 'Y';
|
||||
header[1] = 'a';
|
||||
header[2] = 'z';
|
||||
header[3] = '0';
|
||||
util_write_uint32_be(header + 4, uncompSize);
|
||||
|
||||
// write output file
|
||||
FILE *outFile = fopen(outputFileName, "wb");
|
||||
if (outFile == NULL)
|
||||
util_fatal_error("failed to open file '%s' for writing", outputFileName);
|
||||
fwrite(header, sizeof(header), 1, outFile);
|
||||
fwrite(output, compSize, 1, outFile);
|
||||
fclose(outFile);
|
||||
|
||||
free(input);
|
||||
free(output);
|
||||
|
||||
if (verbose)
|
||||
print_report(time, compSize, uncompSize);
|
||||
}
|
||||
|
||||
static void decompress_file(const char *inputFileName, const char *outputFileName, bool verbose)
|
||||
{
|
||||
size_t compSize;
|
||||
uint8_t *input = util_read_whole_file(inputFileName, &compSize);
|
||||
size_t uncompSize;
|
||||
uint8_t *output;
|
||||
unsigned long int time = 0;
|
||||
|
||||
// read header
|
||||
if (input[0] != 'Y' || input[1] != 'a' || input[2] != 'z' || input[3] != '0')
|
||||
util_fatal_error("file '%s' does not have a valid Yaz0 header", inputFileName);
|
||||
uncompSize = util_read_uint32_be(input + 4);
|
||||
|
||||
// decompress data
|
||||
output = malloc(uncompSize);
|
||||
|
||||
if (verbose)
|
||||
{
|
||||
printf("decompressing %s\n", inputFileName);
|
||||
time = get_time_milliseconds();
|
||||
}
|
||||
|
||||
yaz0_decode(input + 16, output, uncompSize);
|
||||
|
||||
if (verbose)
|
||||
time = get_time_milliseconds() - time;
|
||||
|
||||
// write output file
|
||||
FILE *outFile = fopen(outputFileName, "wb");
|
||||
fwrite(output, uncompSize, 1, outFile);
|
||||
fclose(outFile);
|
||||
|
||||
free(input);
|
||||
free(output);
|
||||
|
||||
if (verbose)
|
||||
print_report(time, compSize, uncompSize);
|
||||
}
|
||||
|
||||
static void usage(const char *execName)
|
||||
{
|
||||
printf("Yaz0 compressor/decompressor\n"
|
||||
"usage: %s [-d] [-h] [-v] INPUT_FILE OUTPUT_FILE\n"
|
||||
"compresses INPUT_FILE using Yaz0 encoding and writes output to OUTPUT_FILE\n"
|
||||
"Available options:\n"
|
||||
"-d: decompresses INPUT_FILE, a Yaz0 compressed file, and writes decompressed\n"
|
||||
" output to OUTPUT_FILE\n"
|
||||
"-v: prints verbose output (compression ratio and time)\n"
|
||||
"-h: shows this help message\n",
|
||||
execName);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int i;
|
||||
const char *inputFileName = NULL;
|
||||
const char *outputFileName = NULL;
|
||||
bool decompress = false;
|
||||
bool verbose = false;
|
||||
|
||||
// parse arguments
|
||||
for (i = 1; i < argc; i++)
|
||||
{
|
||||
char *arg = argv[i];
|
||||
|
||||
if (arg[0] == '-')
|
||||
{
|
||||
if (strcmp(arg, "-d") == 0)
|
||||
decompress = true;
|
||||
else if (strcmp(arg, "-v") == 0)
|
||||
verbose = true;
|
||||
else if (strcmp(arg, "-h") == 0)
|
||||
{
|
||||
usage(argv[0]);
|
||||
return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("unknown option %s\n", arg);
|
||||
usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (inputFileName == NULL)
|
||||
inputFileName = arg;
|
||||
else if (outputFileName == NULL)
|
||||
outputFileName = arg;
|
||||
else
|
||||
{
|
||||
puts("too many files specified");
|
||||
usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (inputFileName == NULL)
|
||||
{
|
||||
puts("no input file specified");
|
||||
usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
if (outputFileName == NULL)
|
||||
{
|
||||
puts("no output file specified");
|
||||
usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (decompress)
|
||||
decompress_file(inputFileName, outputFileName, verbose);
|
||||
else
|
||||
compress_file(inputFileName, outputFileName, verbose);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -16,8 +16,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import crunch64
|
||||
import dataclasses
|
||||
import libyaz0
|
||||
from pathlib import Path
|
||||
import struct
|
||||
|
||||
|
|
@ -77,7 +77,7 @@ def extractArchive(archivePath: Path, outPath: Path):
|
|||
with outPath.open("wb") as out:
|
||||
currentOffset = 0
|
||||
for meta in archivesOffsets:
|
||||
decompressedBytes = libyaz0.decompress(archiveBytes[meta.start:meta.end])
|
||||
decompressedBytes = crunch64.yaz0.decompress(archiveBytes[meta.start:meta.end])
|
||||
decompressedSize = len(decompressedBytes)
|
||||
out.write(decompressedBytes)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,71 +3,14 @@
|
|||
import hashlib, io, struct, sys
|
||||
from os import path
|
||||
|
||||
from libyaz0 import decompress
|
||||
import crunch64
|
||||
import ipl3checksum
|
||||
|
||||
UNCOMPRESSED_SIZE = 0x2F00000
|
||||
|
||||
def as_word(b, off=0):
|
||||
return struct.unpack(">I", b[off:off+4])[0]
|
||||
|
||||
def as_word_list(b):
|
||||
return [i[0] for i in struct.iter_unpack(">I", b)]
|
||||
|
||||
def calc_crc(rom_data, cic_type):
|
||||
start = 0x1000
|
||||
end = 0x101000
|
||||
|
||||
unsigned_long = lambda i: i & 0xFFFFFFFF
|
||||
rol = lambda i, b: unsigned_long(i << b) | (i >> (-b & 0x1F))
|
||||
|
||||
if cic_type == 6101 or cic_type == 6102:
|
||||
seed = 0xF8CA4DDC
|
||||
elif cic_type == 6103:
|
||||
seed = 0xA3886759
|
||||
elif cic_type == 6105:
|
||||
seed = 0xDF26F436
|
||||
elif cic_type == 6106:
|
||||
seed = 0x1FEA617A
|
||||
else:
|
||||
assert False , f"Unknown cic type: {cic_type}"
|
||||
|
||||
t1 = t2 = t3 = t4 = t5 = t6 = seed
|
||||
|
||||
for pos in range(start, end, 4):
|
||||
d = as_word(rom_data, pos)
|
||||
r = rol(d, d & 0x1F)
|
||||
|
||||
t6d = unsigned_long(t6 + d)
|
||||
if t6d < t6:
|
||||
t4 = unsigned_long(t4 + 1)
|
||||
t6 = t6d
|
||||
t3 ^= d
|
||||
t5 = unsigned_long(t5 + r)
|
||||
|
||||
if t2 > d:
|
||||
t2 ^= r
|
||||
else:
|
||||
t2 ^= t6 ^ d
|
||||
|
||||
if cic_type == 6105:
|
||||
t1 = unsigned_long(t1 + (as_word(rom_data, 0x0750 + (pos & 0xFF)) ^ d))
|
||||
else:
|
||||
t1 = unsigned_long(t1 + (t5 ^ d))
|
||||
|
||||
chksum = [0,0]
|
||||
|
||||
if cic_type == 6103:
|
||||
chksum[0] = unsigned_long((t6 ^ t4) + t3)
|
||||
chksum[1] = unsigned_long((t5 ^ t2) + t1)
|
||||
elif cic_type == 6106:
|
||||
chksum[0] = unsigned_long((t6 * t4) + t3)
|
||||
chksum[1] = unsigned_long((t5 * t2) + t1)
|
||||
else:
|
||||
chksum[0] = t6 ^ t4 ^ t3
|
||||
chksum[1] = t5 ^ t2 ^ t1
|
||||
|
||||
return struct.pack(">II", chksum[0], chksum[1])
|
||||
|
||||
def read_dmadata_entry(addr):
|
||||
return as_word_list(fileContent[addr:addr+0x10])
|
||||
|
||||
|
|
@ -87,7 +30,8 @@ def read_dmadata(start):
|
|||
|
||||
def update_crc(decompressed):
|
||||
print("Recalculating crc...")
|
||||
new_crc = calc_crc(decompressed.getbuffer(), 6105)
|
||||
calculated_checksum = ipl3checksum.CICKind.CIC_X105.calculateChecksum(bytes(decompressed.getbuffer()))
|
||||
new_crc = struct.pack(f">II", calculated_checksum[0], calculated_checksum[1])
|
||||
|
||||
decompressed.seek(0x10)
|
||||
decompressed.write(new_crc)
|
||||
|
|
@ -106,7 +50,7 @@ def decompress_rom(dmadata_addr, dmadata):
|
|||
if p_end == 0: # uncompressed
|
||||
rom_segments.update({v_start : fileContent[p_start:p_start + v_end - v_start]})
|
||||
else: # compressed
|
||||
rom_segments.update({v_start : decompress(fileContent[p_start:p_end])})
|
||||
rom_segments.update({v_start : crunch64.yaz0.decompress(fileContent[p_start:p_end])})
|
||||
new_dmadata.extend(struct.pack(">IIII", v_start, v_end, v_start, 0))
|
||||
|
||||
# write rom segments to vaddrs
|
||||
|
|
|
|||
Loading…
Reference in New Issue