From c40f15c9a736a90c33adde98cce9b90ba217a163 Mon Sep 17 00:00:00 2001 From: jdflyer Date: Tue, 24 Jan 2023 21:41:20 -0700 Subject: [PATCH] New yaz0 system --- Makefile | 4 +- tools/extract_game_assets.py | 4 +- tools/librel/rellib.py | 4 +- tools/libyaz0/__init__.py | 1 + tools/libyaz0/yaz0.py | 33 +++++ tools/package_game_assets.py | 17 +-- tools/requirements.txt | 2 - tools/yaz0/Makefile | 7 + tools/yaz0/yaz0.c | 240 +++++++++++++++++++++++++++++++++++ 9 files changed, 294 insertions(+), 18 deletions(-) create mode 100644 tools/libyaz0/__init__.py create mode 100644 tools/libyaz0/yaz0.py create mode 100644 tools/yaz0/Makefile create mode 100644 tools/yaz0/yaz0.c diff --git a/Makefile b/Makefile index 7d874ea2ee0..d169add8392 100644 --- a/Makefile +++ b/Makefile @@ -75,6 +75,7 @@ DOLPHIN_LIB_CC := $(WINE) tools/mwcc_compiler/1.2.5/mwcceppc.exe FRANK_CC := $(WINE) tools/mwcc_compiler/1.2.5e/mwcceppc.exe LD := $(WINE_LD) tools/mwcc_compiler/$(MWCC_VERSION)/mwldeppc.exe ELF2DOL := $(BUILD_PATH)/elf2dol +YAZ0 := $(BUILD_PATH)/yaz0.so PYTHON := python3 ICONV := iconv DOXYGEN := doxygen @@ -151,7 +152,7 @@ clean_rels: rm -f -d -r $(BUILD_DIR)/rel rm -f $(BUILD_PATH)/*.rel -tools: $(ELF2DOL) +tools: $(ELF2DOL) $(YAZ0) assets: @mkdir -p game @@ -232,6 +233,7 @@ $(BUILD_DIR)/rel/%.o: rel/%.cpp # tools include tools/elf2dol/Makefile +include tools/yaz0/Makefile ### Debug Print ### print-% : ; $(info $* is a $(flavor $*) variable set to [$($*)]) @true diff --git a/tools/extract_game_assets.py b/tools/extract_game_assets.py index 106b95ef893..da2ddfab123 100644 --- a/tools/extract_game_assets.py +++ b/tools/extract_game_assets.py @@ -2,7 +2,7 @@ import os import sys import libarc from pathlib import Path -from oead import yaz0 +import libyaz0 """ Extracts the game assets and stores them in the game folder @@ -143,7 +143,7 @@ def writeFile(name, data): if data[0:4] == bytes("Yaz0", "ascii"): splitName = os.path.splitext(name) name = splitName[0] + ".c" + splitName[1] - data = yaz0.decompress(data) + data = libyaz0.decompress(data) extractDef = None splitName = os.path.splitext(name) diff --git a/tools/librel/rellib.py b/tools/librel/rellib.py index 1c08e71c2e7..a9a6539d2f1 100644 --- a/tools/librel/rellib.py +++ b/tools/librel/rellib.py @@ -1,6 +1,6 @@ import struct import os -import yaz0 +import libyaz0 import io from dataclasses import dataclass, field @@ -149,7 +149,7 @@ def read_section(index, buffer): def read(buffer): # check if the rel is compressed if struct.unpack('>I', buffer[:4])[0] == 0x59617A30: - buffer = yaz0.decompress(io.BytesIO(buffer)) + buffer = libyaz0.decompress(io.BytesIO(buffer)) header_size = 0x40 header = struct.unpack('>IIIIIIIIIIIIBBBBIII', buffer[:0x40]) diff --git a/tools/libyaz0/__init__.py b/tools/libyaz0/__init__.py new file mode 100644 index 00000000000..dd448697778 --- /dev/null +++ b/tools/libyaz0/__init__.py @@ -0,0 +1 @@ +from .yaz0 import * \ No newline at end of file diff --git a/tools/libyaz0/yaz0.py b/tools/libyaz0/yaz0.py new file mode 100644 index 00000000000..aedb3e025ec --- /dev/null +++ b/tools/libyaz0/yaz0.py @@ -0,0 +1,33 @@ +import ctypes +import struct + +_yaz0lib = ctypes.cdll.LoadLibrary("build/yaz0.so") + +if _yaz0lib == None: + print("Error: build/yaz0.so failed to load!") + +def decompress(data): + header = data[0:4] + if header != bytearray("Yaz0","ascii"): + return None + decompressedSize = struct.unpack(">I",data[4:8])[0] + compressedDataBuffer = ctypes.c_buffer(bytes(data[16:])) + decompressedDataBuffer = ctypes.c_buffer(bytes(decompressedSize)) + decode = _yaz0lib.yaz0_decode + decode.argtypes = [ctypes.c_char_p,ctypes.c_char_p,ctypes.c_int] + decode.restype = ctypes.c_int + decode(compressedDataBuffer,decompressedDataBuffer,decompressedSize) + return bytearray(decompressedDataBuffer)[:-1] + +def compress(data): + decompresseddDataBuffer = ctypes.c_buffer(data) + compressedDataBuffer = ctypes.c_buffer(bytes(len(data)*2)) + encode = _yaz0lib.yaz0_encode + encode.argtypes = [ctypes.c_char_p,ctypes.c_char_p,ctypes.c_int] + encode.restype = ctypes.c_int + size = encode(decompresseddDataBuffer,compressedDataBuffer,len(data)) + + header_padding = bytearray(8) + ident = bytearray("Yaz0","ascii") + sizeInt = struct.pack(">I",len(data)) + return ident + sizeInt + header_padding + bytearray(compressedDataBuffer)[:size] diff --git a/tools/package_game_assets.py b/tools/package_game_assets.py index be662831c2e..d8f186c5f03 100644 --- a/tools/package_game_assets.py +++ b/tools/package_game_assets.py @@ -3,11 +3,9 @@ import sys import shutil import extract_game_assets from pathlib import Path -import hashlib -import struct -import ctypes -import oead +import libyaz0 import libarc +import threading def getMaxDateFromDir(path): @@ -71,7 +69,7 @@ def convertEntry(file, path, destPath, returnData): if mustBeCompressed == True: if data == None: data = open(path / file, "rb").read() - data = oead.yaz0.compress(data) + data = libyaz0.compress(data) if returnData == True: if data == None and returnData == True: data = open(path / file, "rb").read() @@ -84,7 +82,6 @@ def convertEntry(file, path, destPath, returnData): shutil.copy(path / file, destPath / destFileName) return destFileName - def copy(path, destPath): for file in os.listdir(path): split = os.path.splitext(file) @@ -98,8 +95,6 @@ def copy(path, destPath): convertEntry(file, path, destPath, False) -# copy(Path("srcArc"),Path("arcDest")) - aMemRels = """d_a_alldie.rel d_a_andsw2.rel d_a_bd.rel @@ -259,7 +254,7 @@ def copyRelFiles(gamePath, buildPath, aMemList, mMemList): relSource = open(fullPath, "rb") data = relSource.read() relSource.close() - data = oead.yaz0.compress(data) + data = libyaz0.compress(data) relNew = open( buildPath / "dolzel2/game/files/rel/Final/Release" / file, "wb" ) @@ -290,14 +285,14 @@ def copyRelFiles(gamePath, buildPath, aMemList, mMemList): if str(rel).find(rel2) != -1: sourceRel = open(rel, "rb").read() open(buildPath / "RELS.arc/rels/amem/" / rel2, "wb").write( - oead.yaz0.compress(sourceRel) + libyaz0.compress(sourceRel) ) break for rel2 in mMemRels.splitlines(): if str(rel).find(rel2) != -1: sourceRel = open(rel, "rb").read() open(buildPath / "RELS.arc/rels/mmem/" / rel2, "wb").write( - oead.yaz0.compress(sourceRel) + libyaz0.compress(sourceRel) ) break diff --git a/tools/requirements.txt b/tools/requirements.txt index 3bd1bb7a620..0fa5c69d632 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,6 +1,5 @@ rich click -yaz0 GitPython hexdump colorama @@ -8,5 +7,4 @@ ansiwrap watchdog python-Levenshtein cxxfilt -oead pyelftools \ No newline at end of file diff --git a/tools/yaz0/Makefile b/tools/yaz0/Makefile new file mode 100644 index 00000000000..f0011fb51f7 --- /dev/null +++ b/tools/yaz0/Makefile @@ -0,0 +1,7 @@ +YAZ0_CC := cc +YAZ0_CFLAGS := -fPIC -shared -O3 -Wall -s + +$(YAZ0): include tools/yaz0/Makefile + @echo [tools] building yaz0.so + @$(YAZ0_CC) $(YAZ0_CFLAGS) -o $(YAZ0) tools/yaz0/yaz0.c + diff --git a/tools/yaz0/yaz0.c b/tools/yaz0/yaz0.c new file mode 100644 index 00000000000..2eb6d100d1f --- /dev/null +++ b/tools/yaz0/yaz0.c @@ -0,0 +1,240 @@ +#include +#include +#include +#include + +//yaz0 implementation by BluRose +//patched by Prakxo + +// decoder implementation by thakis of http://www.amnoid.de + +// src points to the yaz0 source data (to the "real" source data, not at the header!) +// dst points to a buffer uncompressedSize bytes large (you get uncompressedSize from +// the second 4 bytes in the Yaz0 header). +int yaz0_decode(uint8_t* src, uint8_t* dst, int uncompressedSize) +{ + int srcPlace = 0, dstPlace = 0; // current read/write positions + + unsigned int validBitCount = 0; // number of valid bits left in "code" byte + uint8_t currCodeByte; + while (dstPlace < uncompressedSize) + { + // read new "code" byte if the current one is used up + if (validBitCount == 0) + { + currCodeByte = src[srcPlace]; + ++srcPlace; + validBitCount = 8; + } + + if ((currCodeByte & 0x80) != 0) + { + // straight copy + dst[dstPlace] = src[srcPlace]; + dstPlace++; + srcPlace++; + } + else + { + // RLE part + uint8_t byte1 = src[srcPlace]; + uint8_t byte2 = src[srcPlace + 1]; + srcPlace += 2; + + unsigned int dist = ((byte1 & 0xF) << 8) | byte2; + unsigned int copySource = dstPlace - (dist + 1); + + unsigned int numBytes = byte1 >> 4; + if (numBytes == 0) + { + numBytes = src[srcPlace] + 0x12; + srcPlace++; + } + else + { + numBytes += 2; + } + + // copy run + for (unsigned int i = 0; i < numBytes; ++i) + { + dst[dstPlace] = dst[copySource]; + copySource++; + dstPlace++; + } + } + + // use next bit from "code" byte + currCodeByte <<= 1; + validBitCount -= 1; + } + return 0; +} + +// encoder implementation by shevious, with bug fixes by notwa + +typedef uint32_t uint32_t; +typedef uint8_t uint8_t; + +#define MAX_RUNLEN (0xFF + 0x12) + +// simple and straight encoding scheme for Yaz0 +static uint32_t simpleEnc(uint8_t *src, int size, int pos, uint32_t *pMatchPos) +{ + int numBytes = 1; + int matchPos = 0; + + int startPos = pos - 0x1000; + int end = size - pos; + + if (startPos < 0) + startPos = 0; + + // maximum runlength for 3 byte encoding + if (end > MAX_RUNLEN) + end = MAX_RUNLEN; + + for (int i = startPos; i < pos; i++) + { + int j; + + for (j = 0; j < end; j++) + { + if (src[i + j] != src[j + pos]) + break; + } + if (j > numBytes) + { + numBytes = j; + matchPos = i; + } + } + + *pMatchPos = matchPos; + + if (numBytes == 2) + numBytes = 1; + + return numBytes; +} + +// a lookahead encoding scheme for ngc Yaz0 +static uint32_t nintendoEnc(uint8_t *src, int size, int pos, uint32_t *pMatchPos) +{ + uint32_t numBytes = 1; + static uint32_t numBytes1; + static uint32_t matchPos; + static int prevFlag = 0; + + // if prevFlag is set, it means that the previous position + // was determined by look-ahead try. + // so just use it. this is not the best optimization, + // but nintendo's choice for speed. + if (prevFlag == 1) + { + *pMatchPos = matchPos; + prevFlag = 0; + return numBytes1; + } + + prevFlag = 0; + numBytes = simpleEnc(src, size, pos, &matchPos); + *pMatchPos = matchPos; + + // if this position is RLE encoded, then compare to copying 1 byte and next position(pos+1) encoding + if (numBytes >= 3) + { + numBytes1 = simpleEnc(src, size, pos + 1, &matchPos); + // if the next position encoding is +2 longer than current position, choose it. + // this does not guarantee the best optimization, but fairly good optimization with speed. + if (numBytes1 >= numBytes + 2) + { + numBytes = 1; + prevFlag = 1; + } + } + return numBytes; +} + +int yaz0_encode(uint8_t *src, uint8_t *dst, int srcSize) +{ + int srcPos = 0; + int dstPos = 0; + int bufPos = 0; + + uint8_t buf[24]; // 8 codes * 3 bytes maximum + + uint32_t validBitCount = 0; // number of valid bits left in "code" byte + uint8_t currCodeByte = 0; // a bitfield, set bits meaning copy, unset meaning RLE + + while (srcPos < srcSize) + { + uint32_t numBytes; + uint32_t matchPos; + + numBytes = nintendoEnc(src, srcSize, srcPos, &matchPos); + if (numBytes < 3) + { + // straight copy + buf[bufPos] = src[srcPos]; + bufPos++; + srcPos++; + //set flag for straight copy + currCodeByte |= (0x80 >> validBitCount); + } + else + { + //RLE part + uint32_t dist = srcPos - matchPos - 1; + uint8_t byte1, byte2, byte3; + + if (numBytes >= 0x12) // 3 byte encoding + { + byte1 = 0 | (dist >> 8); + byte2 = dist & 0xFF; + buf[bufPos++] = byte1; + buf[bufPos++] = byte2; + // maximum runlength for 3 byte encoding + if (numBytes > MAX_RUNLEN) + numBytes = MAX_RUNLEN; + byte3 = numBytes - 0x12; + buf[bufPos++] = byte3; + } + else // 2 byte encoding + { + byte1 = ((numBytes - 2) << 4) | (dist >> 8); + byte2 = dist & 0xFF; + buf[bufPos++] = byte1; + buf[bufPos++] = byte2; + } + srcPos += numBytes; + } + + validBitCount++; + + // write eight codes + if (validBitCount == 8) + { + dst[dstPos++] = currCodeByte; + for (int j = 0; j < bufPos; j++) + dst[dstPos++] = buf[j]; + + currCodeByte = 0; + validBitCount = 0; + bufPos = 0; + } + } + + if (validBitCount > 0) + { + dst[dstPos++] = currCodeByte; + for (int j = 0; j < bufPos; j++) + dst[dstPos++] = buf[j]; + + currCodeByte = 0; + validBitCount = 0; + bufPos = 0; + } + + return dstPos; +} \ No newline at end of file