New yaz0 system

Makefile

@@ -75,6 +75,7 @@ DOLPHIN_LIB_CC := $(WINE) tools/mwcc_compiler/1.2.5/mwcceppc.exe
 FRANK_CC  := $(WINE) tools/mwcc_compiler/1.2.5e/mwcceppc.exe
 LD        := $(WINE_LD) tools/mwcc_compiler/$(MWCC_VERSION)/mwldeppc.exe
 ELF2DOL   := $(BUILD_PATH)/elf2dol
+YAZ0   := $(BUILD_PATH)/yaz0.so
 PYTHON    := python3
 ICONV     := iconv
 DOXYGEN   := doxygen
@@ -151,7 +152,7 @@ clean_rels:
 	rm -f -d -r $(BUILD_DIR)/rel
 	rm -f $(BUILD_PATH)/*.rel
 
-tools: $(ELF2DOL)
+tools: $(ELF2DOL) $(YAZ0)
 
 assets:
 	@mkdir -p game
@@ -232,6 +233,7 @@ $(BUILD_DIR)/rel/%.o: rel/%.cpp
 
 # tools
 include tools/elf2dol/Makefile
+include tools/yaz0/Makefile
 
 ### Debug Print ###
 print-% : ; $(info $* is a $(flavor $*) variable set to [$($*)]) @true

tools/extract_game_assets.py

@@ -2,7 +2,7 @@ import os
 import sys
 import libarc
 from pathlib import Path
-from oead import yaz0
+import libyaz0
 
 """
 Extracts the game assets and stores them in the game folder
@@ -143,7 +143,7 @@ def writeFile(name, data):
     if data[0:4] == bytes("Yaz0", "ascii"):
         splitName = os.path.splitext(name)
         name = splitName[0] + ".c" + splitName[1]
-        data = yaz0.decompress(data)
+        data = libyaz0.decompress(data)
 
     extractDef = None
     splitName = os.path.splitext(name)

tools/librel/rellib.py

@@ -1,6 +1,6 @@
 import struct
 import os
-import yaz0
+import libyaz0
 import io
 
 from dataclasses import dataclass, field
@@ -149,7 +149,7 @@ def read_section(index, buffer):
 def read(buffer):
     # check if the rel is compressed
     if struct.unpack('>I', buffer[:4])[0] == 0x59617A30:
-        buffer = yaz0.decompress(io.BytesIO(buffer))
+        buffer = libyaz0.decompress(io.BytesIO(buffer))
 
     header_size = 0x40
     header = struct.unpack('>IIIIIIIIIIIIBBBBIII', buffer[:0x40])

tools/libyaz0/__init__.py

@@ -0,0 +1 @@
+from .yaz0 import *

tools/libyaz0/yaz0.py

@@ -0,0 +1,33 @@
+import ctypes
+import struct
+
+_yaz0lib = ctypes.cdll.LoadLibrary("build/yaz0.so")
+
+if _yaz0lib == None:
+    print("Error: build/yaz0.so failed to load!")
+
+def decompress(data):
+    header = data[0:4]
+    if header != bytearray("Yaz0","ascii"):
+        return None
+    decompressedSize = struct.unpack(">I",data[4:8])[0]
+    compressedDataBuffer = ctypes.c_buffer(bytes(data[16:]))
+    decompressedDataBuffer = ctypes.c_buffer(bytes(decompressedSize))
+    decode = _yaz0lib.yaz0_decode
+    decode.argtypes = [ctypes.c_char_p,ctypes.c_char_p,ctypes.c_int]
+    decode.restype = ctypes.c_int
+    decode(compressedDataBuffer,decompressedDataBuffer,decompressedSize)
+    return bytearray(decompressedDataBuffer)[:-1]
+
+def compress(data):
+    decompresseddDataBuffer = ctypes.c_buffer(data)
+    compressedDataBuffer = ctypes.c_buffer(bytes(len(data)*2))
+    encode = _yaz0lib.yaz0_encode
+    encode.argtypes = [ctypes.c_char_p,ctypes.c_char_p,ctypes.c_int]
+    encode.restype = ctypes.c_int
+    size = encode(decompresseddDataBuffer,compressedDataBuffer,len(data))
+
+    header_padding = bytearray(8)
+    ident = bytearray("Yaz0","ascii")
+    sizeInt = struct.pack(">I",len(data))
+    return ident + sizeInt + header_padding + bytearray(compressedDataBuffer)[:size]

tools/package_game_assets.py

@@ -3,11 +3,9 @@ import sys
 import shutil
 import extract_game_assets
 from pathlib import Path
-import hashlib
-import struct
-import ctypes
-import oead
+import libyaz0
 import libarc
+import threading
 
 
 def getMaxDateFromDir(path):
@@ -71,7 +69,7 @@ def convertEntry(file, path, destPath, returnData):
     if mustBeCompressed == True:
         if data == None:
             data = open(path / file, "rb").read()
-        data = oead.yaz0.compress(data)
+        data = libyaz0.compress(data)
     if returnData == True:
         if data == None and returnData == True:
             data = open(path / file, "rb").read()
@@ -84,7 +82,6 @@ def convertEntry(file, path, destPath, returnData):
             shutil.copy(path / file, destPath / destFileName)
         return destFileName
 
-
 def copy(path, destPath):
     for file in os.listdir(path):
         split = os.path.splitext(file)
@@ -98,8 +95,6 @@ def copy(path, destPath):
             convertEntry(file, path, destPath, False)
 
 
-# copy(Path("srcArc"),Path("arcDest"))
-
 aMemRels = """d_a_alldie.rel
 d_a_andsw2.rel
 d_a_bd.rel
@@ -259,7 +254,7 @@ def copyRelFiles(gamePath, buildPath, aMemList, mMemList):
                     relSource = open(fullPath, "rb")
                     data = relSource.read()
                     relSource.close()
-                    data = oead.yaz0.compress(data)
+                    data = libyaz0.compress(data)
                     relNew = open(
                         buildPath / "dolzel2/game/files/rel/Final/Release" / file, "wb"
                     )
@@ -290,14 +285,14 @@ def copyRelFiles(gamePath, buildPath, aMemList, mMemList):
             if str(rel).find(rel2) != -1:
                 sourceRel = open(rel, "rb").read()
                 open(buildPath / "RELS.arc/rels/amem/" / rel2, "wb").write(
-                    oead.yaz0.compress(sourceRel)
+                    libyaz0.compress(sourceRel)
                 )
                 break
         for rel2 in mMemRels.splitlines():
             if str(rel).find(rel2) != -1:
                 sourceRel = open(rel, "rb").read()
                 open(buildPath / "RELS.arc/rels/mmem/" / rel2, "wb").write(
-                    oead.yaz0.compress(sourceRel)
+                    libyaz0.compress(sourceRel)
                 )
                 break
 

tools/requirements.txt

@@ -1,6 +1,5 @@
 rich
 click
-yaz0
 GitPython
 hexdump
 colorama
@@ -8,5 +7,4 @@ ansiwrap
 watchdog
 python-Levenshtein
 cxxfilt
-oead
 pyelftools

tools/yaz0/Makefile

@@ -0,0 +1,7 @@
+YAZ0_CC := cc
+YAZ0_CFLAGS := -fPIC -shared -O3 -Wall -s
+
+$(YAZ0): include tools/yaz0/Makefile
+	@echo [tools] building yaz0.so 
+	@$(YAZ0_CC) $(YAZ0_CFLAGS) -o $(YAZ0) tools/yaz0/yaz0.c
+

tools/yaz0/yaz0.c

@@ -0,0 +1,240 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+//yaz0 implementation by BluRose
+//patched by Prakxo
+
+// decoder implementation by thakis of http://www.amnoid.de
+
+// src points to the yaz0 source data (to the "real" source data, not at the header!)
+// dst points to a buffer uncompressedSize bytes large (you get uncompressedSize from
+// the second 4 bytes in the Yaz0 header).
+int yaz0_decode(uint8_t* src, uint8_t* dst, int uncompressedSize)
+{
+    int srcPlace = 0, dstPlace = 0;  // current read/write positions
+
+    unsigned int validBitCount = 0;  // number of valid bits left in "code" byte
+    uint8_t currCodeByte;
+    while (dstPlace < uncompressedSize)
+    {
+        // read new "code" byte if the current one is used up
+        if (validBitCount == 0)
+        {
+            currCodeByte = src[srcPlace];
+            ++srcPlace;
+            validBitCount = 8;
+        }
+
+        if ((currCodeByte & 0x80) != 0)
+        {
+            // straight copy
+            dst[dstPlace] = src[srcPlace];
+            dstPlace++;
+            srcPlace++;
+        }
+        else
+        {
+            // RLE part
+            uint8_t byte1 = src[srcPlace];
+            uint8_t byte2 = src[srcPlace + 1];
+            srcPlace += 2;
+
+            unsigned int dist = ((byte1 & 0xF) << 8) | byte2;
+            unsigned int copySource = dstPlace - (dist + 1);
+
+            unsigned int numBytes = byte1 >> 4;
+            if (numBytes == 0)
+            {
+                numBytes = src[srcPlace] + 0x12;
+                srcPlace++;
+            }
+            else
+            {
+                numBytes += 2;
+            }
+
+            // copy run
+            for (unsigned int i = 0; i < numBytes; ++i)
+            {
+                dst[dstPlace] = dst[copySource];
+                copySource++;
+                dstPlace++;
+            }
+        }
+
+        // use next bit from "code" byte
+        currCodeByte <<= 1;
+        validBitCount -= 1;
+    }
+    return 0;
+}
+
+// encoder implementation by shevious, with bug fixes by notwa
+
+typedef uint32_t uint32_t;
+typedef uint8_t uint8_t;
+
+#define MAX_RUNLEN (0xFF + 0x12)
+
+// simple and straight encoding scheme for Yaz0
+static uint32_t simpleEnc(uint8_t *src, int size, int pos, uint32_t *pMatchPos)
+{
+    int numBytes = 1;
+    int matchPos = 0;
+
+    int startPos = pos - 0x1000;
+    int end = size - pos;
+
+    if (startPos < 0)
+        startPos = 0;
+
+    // maximum runlength for 3 byte encoding
+    if (end > MAX_RUNLEN)
+        end = MAX_RUNLEN;
+
+    for (int i = startPos; i < pos; i++)
+    {
+        int j;
+
+        for (j = 0; j < end; j++)
+        {
+            if (src[i + j] != src[j + pos])
+                break;
+        }
+        if (j > numBytes)
+        {
+            numBytes = j;
+            matchPos = i;
+        }
+    }
+
+    *pMatchPos = matchPos;
+
+    if (numBytes == 2)
+        numBytes = 1;
+
+    return numBytes;
+}
+
+// a lookahead encoding scheme for ngc Yaz0
+static uint32_t nintendoEnc(uint8_t *src, int size, int pos, uint32_t *pMatchPos)
+{
+    uint32_t numBytes = 1;
+    static uint32_t numBytes1;
+    static uint32_t matchPos;
+    static int prevFlag = 0;
+
+    // if prevFlag is set, it means that the previous position
+    // was determined by look-ahead try.
+    // so just use it. this is not the best optimization,
+    // but nintendo's choice for speed.
+    if (prevFlag == 1)
+    {
+        *pMatchPos = matchPos;
+        prevFlag = 0;
+        return numBytes1;
+    }
+
+    prevFlag = 0;
+    numBytes = simpleEnc(src, size, pos, &matchPos);
+    *pMatchPos = matchPos;
+
+    // if this position is RLE encoded, then compare to copying 1 byte and next position(pos+1) encoding
+    if (numBytes >= 3)
+    {
+        numBytes1 = simpleEnc(src, size, pos + 1, &matchPos);
+        // if the next position encoding is +2 longer than current position, choose it.
+        // this does not guarantee the best optimization, but fairly good optimization with speed.
+        if (numBytes1 >= numBytes + 2)
+        {
+            numBytes = 1;
+            prevFlag = 1;
+        }
+    }
+    return numBytes;
+}
+
+int yaz0_encode(uint8_t *src, uint8_t *dst, int srcSize)
+{
+    int srcPos = 0;
+    int dstPos = 0;
+    int bufPos = 0;
+
+    uint8_t buf[24]; // 8 codes * 3 bytes maximum
+
+    uint32_t validBitCount = 0; // number of valid bits left in "code" byte
+    uint8_t currCodeByte = 0; // a bitfield, set bits meaning copy, unset meaning RLE
+
+    while (srcPos < srcSize)
+    {
+        uint32_t numBytes;
+        uint32_t matchPos;
+
+        numBytes = nintendoEnc(src, srcSize, srcPos, &matchPos);
+        if (numBytes < 3)
+        {
+            // straight copy
+            buf[bufPos] = src[srcPos];
+            bufPos++;
+            srcPos++;
+            //set flag for straight copy
+            currCodeByte |= (0x80 >> validBitCount);
+        }
+        else
+        {
+            //RLE part
+            uint32_t dist = srcPos - matchPos - 1;
+            uint8_t byte1, byte2, byte3;
+
+            if (numBytes >= 0x12)  // 3 byte encoding
+            {
+                byte1 = 0 | (dist >> 8);
+                byte2 = dist & 0xFF;
+                buf[bufPos++] = byte1;
+                buf[bufPos++] = byte2;
+                // maximum runlength for 3 byte encoding
+                if (numBytes > MAX_RUNLEN)
+                    numBytes = MAX_RUNLEN;
+                byte3 = numBytes - 0x12;
+                buf[bufPos++] = byte3;
+            }
+            else  // 2 byte encoding
+            {
+                byte1 = ((numBytes - 2) << 4) | (dist >> 8);
+                byte2 = dist & 0xFF;
+                buf[bufPos++] = byte1;
+                buf[bufPos++] = byte2;
+            }
+            srcPos += numBytes;
+        }
+
+        validBitCount++;
+
+        // write eight codes
+        if (validBitCount == 8)
+        {
+            dst[dstPos++] = currCodeByte;
+            for (int j = 0; j < bufPos; j++)
+                dst[dstPos++] = buf[j];
+
+            currCodeByte = 0;
+            validBitCount = 0;
+            bufPos = 0;
+        }
+    }
+
+    if (validBitCount > 0)
+    {
+        dst[dstPos++] = currCodeByte;
+        for (int j = 0; j < bufPos; j++)
+            dst[dstPos++] = buf[j];
+
+        currCodeByte = 0;
+        validBitCount = 0;
+        bufPos = 0;
+    }
+
+    return dstPos;
+}