From a7cc87394ed4afb5ebe1f2318526545b6bbe205d Mon Sep 17 00:00:00 2001
From: angie <angheloalf95@gmail.com>
Date: Sat, 15 Jan 2022 14:01:18 -0300
Subject: [PATCH] git subrepo pull tools/z64compress --force

subrepo:
  subdir:   "tools/z64compress"
  merged:   "9e7a6dbfa"
upstream:
  origin:   "https://github.com/z64me/z64compress.git"
  branch:   "main"
  commit:   "9e7a6dbfa"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"
---
 tools/z64compress/.gitmodules        |   3 +
 tools/z64compress/.gitrepo           |   4 +-
 tools/z64compress/Makefile           |   2 +-
 tools/z64compress/README.md          |   3 +-
 tools/z64compress/release-linux.sh   |   2 +-
 tools/z64compress/release-linux32.sh |   2 +-
 tools/z64compress/release-win32.sh   |   4 +-
 tools/z64compress/src/enc/aplib.c    |   8 +--
 tools/z64compress/src/enc/enc.h      |   9 +++
 tools/z64compress/src/enc/libdeflate |   1 +
 tools/z64compress/src/enc/lzo.c      |   8 +--
 tools/z64compress/src/enc/ucl.c      |   8 +--
 tools/z64compress/src/enc/yaz.c      |  18 ++---
 tools/z64compress/src/enc/zlib.c     | 103 +++++++++++++++++++++++++++
 tools/z64compress/src/enc/zx7.c      |   8 +--
 tools/z64compress/src/main.c         |  14 ++++
 tools/z64compress/src/rom.c          |  10 ++-
 17 files changed, 174 insertions(+), 33 deletions(-)
 create mode 100644 tools/z64compress/.gitmodules
 create mode 160000 tools/z64compress/src/enc/libdeflate
 create mode 100644 tools/z64compress/src/enc/zlib.c

diff --git a/tools/z64compress/.gitmodules b/tools/z64compress/.gitmodules
new file mode 100644
index 0000000000..089f977876
--- /dev/null
+++ b/tools/z64compress/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "src/enc/libdeflate"]
+	path = src/enc/libdeflate
+	url = https://github.com/ebiggers/libdeflate
diff --git a/tools/z64compress/.gitrepo b/tools/z64compress/.gitrepo
index 183bfc3267..7f44fc8265 100644
--- a/tools/z64compress/.gitrepo
+++ b/tools/z64compress/.gitrepo
@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/z64me/z64compress.git
 	branch = main
-	commit = 5da3132606e4fd427a8d72b8428e4f921cd6e56f
-	parent = 837eb1c80687fe9b57a3c7b841547c33feeed6c7
+	commit = 9e7a6dbfade2970a3412a1f9752fec43303ba5b6
+	parent = 28fadecd8d074ab985b310021641e200458dc70e
 	method = merge
 	cmdver = 0.4.3
diff --git a/tools/z64compress/Makefile b/tools/z64compress/Makefile
index f16167ccd2..8c94cf4213 100644
--- a/tools/z64compress/Makefile
+++ b/tools/z64compress/Makefile
@@ -38,7 +38,7 @@ $(shell mkdir -p $(foreach dir,$(SRC_DIRS),$(OBJ_DIR)/$(dir)))
 all: z64compress
 
 z64compress: $(O_FILES)
-	$(CC) $(TARGET_CFLAGS) $(CFLAGS) $(O_FILES) -lm -lpthread $(TARGET_LIBS) -o z64compress
+	$(CC) $(TARGET_CFLAGS) $(CFLAGS) $(O_FILES) -lm -lpthread -lz $(TARGET_LIBS) -o z64compress
 
 $(OBJ_DIR)/%.o: %.c
 	$(CC) -c $(TARGET_CFLAGS) $(CFLAGS) $< -o $@
diff --git a/tools/z64compress/README.md b/tools/z64compress/README.md
index 1731a8ab03..3f19e18a0f 100644
--- a/tools/z64compress/README.md
+++ b/tools/z64compress/README.md
@@ -2,7 +2,7 @@
 
 `z64compress` is a program for compressing Zelda 64 roms: be they retail, hacked traditionally, or custom-built from the [`Ocarina of Time`](https://github.com/zeldaret/oot) or [`Majora's Mask`](https://github.com/zeldaret/mm) reverse engineering projects. It is written in highly efficient C and leverages the power of multithreading to make compression as fast as possible. To reduce overhead on subsequent compressions, an optional cache directory can be specified.
 
-In addition to the default `yaz`, it supports some faster and more compact algorithms such as `lzo`, `ucl`, and `aplib`. In order to use these, grab patches or code from my [`z64enc` repository](https://github.com/z64me/z64enc).
+In addition to the default `yaz`, it supports some faster and more compact algorithms such as `DEFLATE`, `lzo`, `ucl`, and `aplib`. In order to use these, grab patches or code from my [`z64enc` repository](https://github.com/z64me/z64enc).
 
 If you add an algorithm, please make sure `valgrind` reports no memory leaks or other errors before making a pull request. Thank you!
 
@@ -59,6 +59,7 @@ This is a command line application. Learn from these common examples and adapt t
                       yaz
                       ucl
                       lzo
+                      zlib
                       aplib
                  * to use non-yaz codecs, find patches
                    and code on my z64enc repo
diff --git a/tools/z64compress/release-linux.sh b/tools/z64compress/release-linux.sh
index 24f4cb2d5a..bdac70dcc7 100755
--- a/tools/z64compress/release-linux.sh
+++ b/tools/z64compress/release-linux.sh
@@ -4,7 +4,7 @@ mkdir -p o
 mv *.o o
 
 # build everything else
-gcc -o z64compress -DNDEBUG src/*.c o/*.o -Wall -Wextra -s -Os -flto -lpthread -march=native -mtune=native
+gcc -o z64compress -DNDEBUG src/*.c o/*.o src/enc/libdeflate/lib/deflate_compress.c src/enc/libdeflate/lib/utils.c -Isrc/enc/libdeflate -Wall -Wextra -s -Os -flto -lpthread -lz -march=native -mtune=native
 
 # move to bin directory
 mkdir -p bin/linux64
diff --git a/tools/z64compress/release-linux32.sh b/tools/z64compress/release-linux32.sh
index 9f413e2965..06d829a7d6 100755
--- a/tools/z64compress/release-linux32.sh
+++ b/tools/z64compress/release-linux32.sh
@@ -4,7 +4,7 @@ mkdir -p o
 mv *.o o
 
 # build everything else
-gcc -m32 -o z64compress -DNDEBUG src/*.c o/*.o -Wall -Wextra -s -Os -flto -lpthread -march=native -mtune=native
+gcc -m32 -o z64compress -DNDEBUG src/*.c o/*.o src/enc/libdeflate/lib/deflate_compress.c src/enc/libdeflate/lib/utils.c -Isrc/enc/libdeflate -Wall -Wextra -s -Os -flto -lpthread -lz -march=native -mtune=native
 
 # move to bin directory
 mkdir -p bin/linux32
diff --git a/tools/z64compress/release-win32.sh b/tools/z64compress/release-win32.sh
index 5b115816d8..94fc245c91 100755
--- a/tools/z64compress/release-win32.sh
+++ b/tools/z64compress/release-win32.sh
@@ -1,10 +1,10 @@
 # build compression functions (slow)
-~/c/mxe/usr/bin/i686-w64-mingw32.static-gcc -DNDEBUG -s -Ofast -flto -lm -c -Wall src/enc/*.c src/enc/lzo/*.c src/enc/ucl/comp/*.c src/enc/apultra/*.c
+i686-w64-mingw32.static-gcc -DNDEBUG -s -Ofast -flto -lm -c -Wall src/enc/*.c src/enc/lzo/*.c src/enc/ucl/comp/*.c src/enc/apultra/*.c
 mkdir -p o
 mv *.o o
 
 # build everything else
-~/c/mxe/usr/bin/i686-w64-mingw32.static-gcc -o z64compress.exe -DNDEBUG src/*.c o/*.o -Wall -Wextra -s -Os -flto -lpthread -mconsole -municode
+i686-w64-mingw32.static-gcc -o z64compress.exe -DNDEBUG src/*.c o/*.o src/enc/libdeflate/lib/deflate_compress.c src/enc/libdeflate/lib/utils.c -Isrc/enc/libdeflate -Wall -Wextra -s -Os -flto -lpthread -lz -mconsole -municode
 
 # move to bin directory
 mkdir -p bin/win32
diff --git a/tools/z64compress/src/enc/aplib.c b/tools/z64compress/src/enc/aplib.c
index e4aa95f812..c2e720a7b7 100644
--- a/tools/z64compress/src/enc/aplib.c
+++ b/tools/z64compress/src/enc/aplib.c
@@ -21,8 +21,8 @@ aplenc(
 	int nMaxCompressedSize = apultra_get_max_compressed_size(src_sz);
 	apultra_stats stats;
 	
-	int hlen = 8; /* header length; required due to MM's archives */
-	memset(dst, 0, hlen);
+	extern int g_hlen; /* header length */
+	memset(dst, 0, g_hlen);
 	memcpy(dst, "APL0", 4);
 	dst[4] = (src_sz >> 24);
 	dst[5] = (src_sz >> 16);
@@ -31,7 +31,7 @@ aplenc(
 	
 	*dst_sz = apultra_compress(
 		src
-		, dst + hlen
+		, dst + g_hlen
 		, src_sz
 		, nMaxCompressedSize
 		, 0 /* flags */
@@ -41,7 +41,7 @@ aplenc(
 		, &stats
 	);
 	
-	*dst_sz = *dst_sz + hlen;
+	*dst_sz = *dst_sz + g_hlen;
 	
 	return 0;
 }
diff --git a/tools/z64compress/src/enc/enc.h b/tools/z64compress/src/enc/enc.h
index 48a24757b9..60bae3b96e 100644
--- a/tools/z64compress/src/enc/enc.h
+++ b/tools/z64compress/src/enc/enc.h
@@ -38,6 +38,15 @@ int zx7enc(
 	, void *_ctx
 );
 
+int
+zlibenc(
+	void *_src
+	, unsigned src_sz
+	, void *_dst
+	, unsigned *dst_sz
+	, void *_ctx
+);
+
 int aplenc(
 	void *_src
 	, unsigned src_sz
diff --git a/tools/z64compress/src/enc/libdeflate b/tools/z64compress/src/enc/libdeflate
new file mode 160000
index 0000000000..6c095314d0
--- /dev/null
+++ b/tools/z64compress/src/enc/libdeflate
@@ -0,0 +1 @@
+Subproject commit 6c095314d0c49061f41e1e40be2625dfc2253afa
diff --git a/tools/z64compress/src/enc/lzo.c b/tools/z64compress/src/enc/lzo.c
index 64f79b722b..a6f9b3b267 100644
--- a/tools/z64compress/src/enc/lzo.c
+++ b/tools/z64compress/src/enc/lzo.c
@@ -33,8 +33,8 @@ lzoenc(
 	unsigned char *wrkmem = _ctx;
 	lzo_uint result_sz = 0;
 	
-	int hlen = 8; /* header length; required due to MM's archives */
-	memset(dst, 0, hlen);
+	extern int g_hlen; /* header length */
+	memset(dst, 0, g_hlen);
 	memcpy(dst, "LZO0", 4);
 	dst[4] = (src_sz >> 24);
 	dst[5] = (src_sz >> 16);
@@ -46,9 +46,9 @@ lzoenc(
 	
 	memset(wrkmem, 0, LZO1X_999_MEM_COMPRESS);
 	
-	lzo1x_999_compress(src, src_sz, dst + hlen, &result_sz, wrkmem);
+	lzo1x_999_compress(src, src_sz, dst + g_hlen, &result_sz, wrkmem);
 	
-	*dst_sz = result_sz + hlen;
+	*dst_sz = result_sz + g_hlen;
 	
 	return 0;
 }
diff --git a/tools/z64compress/src/enc/ucl.c b/tools/z64compress/src/enc/ucl.c
index 59b6936add..49474af250 100644
--- a/tools/z64compress/src/enc/ucl.c
+++ b/tools/z64compress/src/enc/ucl.c
@@ -18,8 +18,8 @@ uclenc(
 	int level = 10;
 	ucl_uint result_sz;
 	
-	int hlen = 8; /* header length; required due to MM's archives */
-	memset(dst, 0, hlen);
+	extern int g_hlen; /* header length */
+	memset(dst, 0, g_hlen);
 	memcpy(dst, "UCL0", 4);
 	dst[4] = (src_sz >> 24);
 	dst[5] = (src_sz >> 16);
@@ -29,7 +29,7 @@ uclenc(
 	r = ucl_nrv2b_99_compress(
 		src          /* in */
 		, src_sz     /* in size */
-		, dst + hlen /* out */
+		, dst + g_hlen /* out */
 		, &result_sz /* out size */
 		, NULL       /* callback */
 		, level      /* level */
@@ -43,7 +43,7 @@ uclenc(
 		exit(EXIT_FAILURE);
 	}
 	
-	*dst_sz = result_sz + hlen;
+	*dst_sz = result_sz + g_hlen;
 	
 	return 0;
 }
diff --git a/tools/z64compress/src/enc/yaz.c b/tools/z64compress/src/enc/yaz.c
index a6897a7fd3..e69130d3d4 100644
--- a/tools/z64compress/src/enc/yaz.c
+++ b/tools/z64compress/src/enc/yaz.c
@@ -272,16 +272,20 @@ uint32_t encode(struct yazCtx *ctx, uint8_t *data, uint32_t data_size, uint8_t *
 	// block and stream differentiation
 	// Yay is block, Yaz is stream
 	int mode_block=1, mode_stream=1; // temporary, for testing
+	extern int g_hlen;
 	mode_block=!strcmp(mode,"Yay0");
-	if (mode_block ) {
-		uint32_t l = (sb_count(ctx->cmds) << 2) + 16;
-		uint32_t o = (sb_count(ctx->ctrl) << 1) + l;
+	if (g_hlen) {
 		memcpy(output, mode, 4);
 		U32wr(output+4, sz);
+	} else
+		output -= 8; /* headerless */
+	if (mode_block) {
+		uint32_t l = (sb_count(ctx->cmds) << 2) + 16;
+		uint32_t o = (sb_count(ctx->ctrl) << 1) + l;
 		U32wr(output+8, l);
 		U32wr(output+12, o);
 		
-		uint32_t output_position = 16;
+		uint32_t output_position = g_hlen + 8;
 		uint32_t x;
 		for (x=0; x<sb_count(ctx->cmds); x++) {
 			U32wr(output+output_position, ctx->cmds[x]);
@@ -296,8 +300,6 @@ uint32_t encode(struct yazCtx *ctx, uint8_t *data, uint32_t data_size, uint8_t *
 		}
 		return output_position;
 	} else if(mode_stream) {
-		memcpy(output, mode, 4);
-		U32wr(output+4, sz);
 		U32wr(output+8, 0);
 		U32wr(output+12, 0);
 		
@@ -315,8 +317,8 @@ uint32_t encode(struct yazCtx *ctx, uint8_t *data, uint32_t data_size, uint8_t *
 			sb_push(ctx->back, (ctx->ctrl[x]>>8)&0xFF);
 			sb_push(ctx->back, (ctx->ctrl[x])&0xFF);
 		}
-		output_position = _enc_z_from_tables(ctx, ctx->ctl, ctx->back, ctx->raws, output+16, data_size, mode);
-		return 16 + output_position;
+		output_position = _enc_z_from_tables(ctx, ctx->ctl, ctx->back, ctx->raws, output+g_hlen+8, data_size, mode);
+		return output_position + g_hlen + 8;
 	}
 	return 0;
 }
diff --git a/tools/z64compress/src/enc/zlib.c b/tools/z64compress/src/enc/zlib.c
new file mode 100644
index 0000000000..28b7fd7565
--- /dev/null
+++ b/tools/z64compress/src/enc/zlib.c
@@ -0,0 +1,103 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <zlib.h>
+#include "libdeflate/libdeflate.h"
+
+#define  CAPACITY  (1024 * 1024 * 4)  /* output buffer max (4 mb) */
+
+int
+zlibenc(
+	void *_src
+	, unsigned src_sz
+	, void *_dst
+	, unsigned *dst_sz
+	, void *_ctx
+)
+{
+	unsigned char *src = _src;
+	unsigned char *dst = _dst;
+	unsigned result_sz;
+	
+	extern int g_hlen; /* header length */
+	memset(dst, 0, g_hlen);
+	memcpy(dst, "ZLIB", 4);
+	dst[4] = (src_sz >> 24);
+	dst[5] = (src_sz >> 16);
+	dst[6] = (src_sz >>  8);
+	dst[7] = (src_sz >>  0);
+	
+	/* zlib and gzip have different header lengths
+	 * https://stackoverflow.com/a/68538037
+	 */
+#if 1
+	
+	#if 0 /* zlib */
+	z_stream stream = {0};
+	int r;
+	stream.avail_in = src_sz;
+	stream.next_in = src;
+	stream.avail_out = CAPACITY;
+	stream.next_out = dst + g_hlen;
+	#define HEADER_LEN 2
+	if ((r = deflateInit(&stream, Z_BEST_COMPRESSION)) != Z_OK)
+	{
+		fprintf(stderr, "[!] fatal compression error %d\n", r);
+		exit(EXIT_FAILURE);
+	}
+	if ((r = deflate(&stream, Z_FINISH)) == Z_STREAM_ERROR)
+	{
+		fprintf(stderr, "[!] Z_STREAM_ERROR\n");
+		exit(EXIT_FAILURE);
+	}
+	deflateEnd(&stream);
+	
+	result_sz = CAPACITY - stream.avail_out;
+	#else /* libdeflate */
+	#define HEADER_LEN 0
+	int level = 12;
+	struct libdeflate_compressor *compressor;
+	compressor = libdeflate_alloc_compressor(level);
+	result_sz = libdeflate_deflate_compress(
+		compressor
+		, src, src_sz
+		, dst + g_hlen
+		, CAPACITY
+	);
+	libdeflate_free_compressor(compressor);
+	#endif
+#else
+	/* this gzip code was left in for testing purposes; it may
+	 * be useful if matching ique recompression is ever revisited;
+	 * ique matches (except for one byte...) when compressed using
+	 * gzip 1.2.4 or 1.2.4a (they produce identical results),
+	 * available here: https://ftp.gnu.org/gnu/gzip/
+	 * this is not a compression error, because decompressing the
+	 * recompressed rom produces a rom identical to the original
+	 * decompressed ique rom;
+	 * TODO: find out why that byte doesn't match on recompression;
+	 * TODO: once that's working, add --codec ique for those wanting
+	 * matching ique recompression; otherwise, modern zlib works great!
+	 */
+	#define HEADER_LEN 10
+	FILE *fp = fopen("tmp.bin", "wb");
+	fwrite(src, 1, src_sz, fp);
+	fclose(fp);
+	system("./gzip -c -9 -n tmp.bin > tmp.bin.gzip");
+	fp = fopen("tmp.bin.gzip", "rb");
+	fseek(fp, 0, SEEK_END);
+	result_sz = ftell(fp);
+	fseek(fp, 0, SEEK_SET);
+	fread(dst, 1, result_sz, fp);
+	fclose(fp);
+#endif
+	*dst_sz = result_sz + g_hlen;
+	
+	/* trim zlib/gzip header */
+	memmove(dst + g_hlen, dst + g_hlen + HEADER_LEN, result_sz);
+	*dst_sz -= HEADER_LEN;
+	
+	return 0;
+	(void)_ctx; /* -Wunused-parameter */
+}
+
diff --git a/tools/z64compress/src/enc/zx7.c b/tools/z64compress/src/enc/zx7.c
index 4b4fdb0f9a..a2e8982334 100644
--- a/tools/z64compress/src/enc/zx7.c
+++ b/tools/z64compress/src/enc/zx7.c
@@ -16,20 +16,20 @@ zx7enc(
 	unsigned char *src = _src;
 	unsigned char *dst = _dst;
 	
-	int hlen = 8; /* header length; required due to MM's archives */
-	memset(dst, 0, hlen);
+	extern int g_hlen; /* header length */
+	memset(dst, 0, g_hlen);
 	memcpy(dst, "ZX70", 4);
 	dst[4] = (src_sz >> 24);
 	dst[5] = (src_sz >> 16);
 	dst[6] = (src_sz >>  8);
 	dst[7] = (src_sz >>  0);
 	
-	*dst_sz = ZX7Compress(src, src_sz, dst + hlen);
+	*dst_sz = ZX7Compress(src, src_sz, dst + g_hlen);
 	
 	if (!*dst_sz)
 		return 1;
 	
-	*dst_sz += hlen;
+	*dst_sz += g_hlen;
 	
 	return 0;
 }
diff --git a/tools/z64compress/src/main.c b/tools/z64compress/src/main.c
index bd044059d1..104be22621 100644
--- a/tools/z64compress/src/main.c
+++ b/tools/z64compress/src/main.c
@@ -8,6 +8,7 @@
 #include "rom.h"
 
 FILE* printer;
+int g_hlen = 8;
 
 static void compress(struct rom *rom, int start, int end)
 {
@@ -167,6 +168,7 @@ static void usage(void)
 	fprintf(printer, "                      yaz\n");
 	fprintf(printer, "                      ucl\n");
 	fprintf(printer, "                      lzo\n");
+	fprintf(printer, "                      zlib\n");
 	fprintf(printer, "                      aplib\n");
 	fprintf(printer, "                 * to use non-yaz codecs, find patches\n");
 	fprintf(printer, "                   and code on my z64enc repo\n");
@@ -184,6 +186,8 @@ static void usage(void)
 	fprintf(printer, "\n");
 	fprintf(printer, "    --skip         disable compression on specified files\n");
 	fprintf(printer, "\n");
+	fprintf(printer, "    --headerless   don't write file headers (for iQue)\n");
+	fprintf(printer, "\n");
 	fprintf(printer, "    --repack       handles Majora's Mask archives\n");
 	fprintf(printer, "\n");
 	fprintf(printer, "    --threads      optional multithreading;\n");
@@ -209,6 +213,7 @@ wow_main
 	int Athreads = 0;
 	bool Amatching = false;
 	bool Aonly_stdout = false;
+	bool Aheaderless = false;
 	wow_main_argv;
 
 	printer = stderr;
@@ -253,6 +258,15 @@ wow_main
 			i--;
 			continue;
 		}
+		else if (!strcmp(arg, "--headerless"))
+		{
+			if (Aheaderless)
+				die("--headerless arg provided more than once");
+			Aheaderless = true;
+			g_hlen = 0;
+			i--;
+			continue;
+		}
 		
 		/* arguments with additional parameters */
 
diff --git a/tools/z64compress/src/rom.c b/tools/z64compress/src/rom.c
index 008597b524..41dd2846cf 100644
--- a/tools/z64compress/src/rom.c
+++ b/tools/z64compress/src/rom.c
@@ -462,6 +462,14 @@ static const struct encoder *encoder(const char *name)
 		
 		return &zx7;
 	}*/
+	else if (!strcmp(name, "zlib"))
+	{
+		static const struct encoder zlib = {
+			.encfunc = zlibenc
+		};
+		
+		return &zlib;
+	}
 	else if (!strcmp(name, "aplib"))
 	{
 		static const struct encoder aplib = {
@@ -1112,7 +1120,7 @@ void rom_compress(struct rom *rom, int mb, int numThreads, bool matching)
 		comp_total += sz16;
 		
 		if (mb != 0 && dma->Pend > compsz)
-			die("ran out of compressed rom space");
+			die("ran out of compressed rom space (try increasing --mb)");
 	}
 
 	/* adaptive final size */