falso
/
mm
mirror of https://github.com/zeldaret/mm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

git subrepo pull tools/z64compress --force 

subrepo:
  subdir:   "tools/z64compress"
  merged:   "9e7a6dbfa"
upstream:
  origin:   "https://github.com/z64me/z64compress.git"
  branch:   "main"
  commit:   "9e7a6dbfa"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"
This commit is contained in:
angie 2022-01-15 14:01:18 -03:00
parent 28fadecd8d
commit a7cc87394e
17 changed files with 174 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tools/z64compress/.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "src/enc/libdeflate"]
path = src/enc/libdeflate
url = https://github.com/ebiggers/libdeflate

4
tools/z64compress/.gitrepo
View File

@ -6,7 +6,7 @@
[subrepo]
remote = https://github.com/z64me/z64compress.git
branch = main
commit = 5da3132606e4fd427a8d72b8428e4f921cd6e56f
parent = 837eb1c80687fe9b57a3c7b841547c33feeed6c7
commit = 9e7a6dbfade2970a3412a1f9752fec43303ba5b6
parent = 28fadecd8d074ab985b310021641e200458dc70e
method = merge
cmdver = 0.4.3

2
tools/z64compress/Makefile
View File

@ -38,7 +38,7 @@ $(shell mkdir -p $(foreach dir,$(SRC_DIRS),$(OBJ_DIR)/$(dir)))
all: z64compress
z64compress: $(O_FILES)
$(CC) $(TARGET_CFLAGS) $(CFLAGS) $(O_FILES) -lm -lpthread $(TARGET_LIBS) -o z64compress
$(CC) $(TARGET_CFLAGS) $(CFLAGS) $(O_FILES) -lm -lpthread -lz $(TARGET_LIBS) -o z64compress
$(OBJ_DIR)/%.o: %.c
$(CC) -c $(TARGET_CFLAGS) $(CFLAGS) $< -o $@

3
tools/z64compress/README.md
View File

@ -2,7 +2,7 @@
`z64compress` is a program for compressing Zelda 64 roms: be they retail, hacked traditionally, or custom-built from the [`Ocarina of Time`](https://github.com/zeldaret/oot) or [`Majora's Mask`](https://github.com/zeldaret/mm) reverse engineering projects. It is written in highly efficient C and leverages the power of multithreading to make compression as fast as possible. To reduce overhead on subsequent compressions, an optional cache directory can be specified.
In addition to the default `yaz`, it supports some faster and more compact algorithms such as `lzo`, `ucl`, and `aplib`. In order to use these, grab patches or code from my [`z64enc` repository](https://github.com/z64me/z64enc).
In addition to the default `yaz`, it supports some faster and more compact algorithms such as `DEFLATE`, `lzo`, `ucl`, and `aplib`. In order to use these, grab patches or code from my [`z64enc` repository](https://github.com/z64me/z64enc).
If you add an algorithm, please make sure `valgrind` reports no memory leaks or other errors before making a pull request. Thank you!
@ -59,6 +59,7 @@ This is a command line application. Learn from these common examples and adapt t
yaz
ucl
lzo
zlib
aplib
* to use non-yaz codecs, find patches
and code on my z64enc repo

2
tools/z64compress/release-linux.sh
View File

@ -4,7 +4,7 @@ mkdir -p o
mv *.o o
# build everything else
gcc -o z64compress -DNDEBUG src/*.c o/*.o -Wall -Wextra -s -Os -flto -lpthread -march=native -mtune=native
gcc -o z64compress -DNDEBUG src/*.c o/*.o src/enc/libdeflate/lib/deflate_compress.c src/enc/libdeflate/lib/utils.c -Isrc/enc/libdeflate -Wall -Wextra -s -Os -flto -lpthread -lz -march=native -mtune=native
# move to bin directory
mkdir -p bin/linux64

2
tools/z64compress/release-linux32.sh
View File

@ -4,7 +4,7 @@ mkdir -p o
mv *.o o
# build everything else
gcc -m32 -o z64compress -DNDEBUG src/*.c o/*.o -Wall -Wextra -s -Os -flto -lpthread -march=native -mtune=native
gcc -m32 -o z64compress -DNDEBUG src/*.c o/*.o src/enc/libdeflate/lib/deflate_compress.c src/enc/libdeflate/lib/utils.c -Isrc/enc/libdeflate -Wall -Wextra -s -Os -flto -lpthread -lz -march=native -mtune=native
# move to bin directory
mkdir -p bin/linux32

4
tools/z64compress/release-win32.sh
View File

@ -1,10 +1,10 @@
# build compression functions (slow)
~/c/mxe/usr/bin/i686-w64-mingw32.static-gcc -DNDEBUG -s -Ofast -flto -lm -c -Wall src/enc/*.c src/enc/lzo/*.c src/enc/ucl/comp/*.c src/enc/apultra/*.c
i686-w64-mingw32.static-gcc -DNDEBUG -s -Ofast -flto -lm -c -Wall src/enc/*.c src/enc/lzo/*.c src/enc/ucl/comp/*.c src/enc/apultra/*.c
mkdir -p o
mv *.o o
# build everything else
~/c/mxe/usr/bin/i686-w64-mingw32.static-gcc -o z64compress.exe -DNDEBUG src/*.c o/*.o -Wall -Wextra -s -Os -flto -lpthread -mconsole -municode
i686-w64-mingw32.static-gcc -o z64compress.exe -DNDEBUG src/*.c o/*.o src/enc/libdeflate/lib/deflate_compress.c src/enc/libdeflate/lib/utils.c -Isrc/enc/libdeflate -Wall -Wextra -s -Os -flto -lpthread -lz -mconsole -municode
# move to bin directory
mkdir -p bin/win32

8
tools/z64compress/src/enc/aplib.c
View File

@ -21,8 +21,8 @@ aplenc(
int nMaxCompressedSize = apultra_get_max_compressed_size(src_sz);
apultra_stats stats;
int hlen = 8; /* header length; required due to MM's archives */
memset(dst, 0, hlen);
extern int g_hlen; /* header length */
memset(dst, 0, g_hlen);
memcpy(dst, "APL0", 4);
dst[4] = (src_sz >> 24);
dst[5] = (src_sz >> 16);
@ -31,7 +31,7 @@ aplenc(
*dst_sz = apultra_compress(
src
, dst + hlen
, dst + g_hlen
, src_sz
, nMaxCompressedSize
, 0 /* flags */
@ -41,7 +41,7 @@ aplenc(
, &stats
);
*dst_sz = *dst_sz + hlen;
*dst_sz = *dst_sz + g_hlen;
return 0;
}

9
tools/z64compress/src/enc/enc.h
View File

@ -38,6 +38,15 @@ int zx7enc(
, void *_ctx
);
int
zlibenc(
void *_src
, unsigned src_sz
, void *_dst
, unsigned *dst_sz
, void *_ctx
);
int aplenc(
void *_src
, unsigned src_sz

1
tools/z64compress/src/enc/libdeflate Submodule

@ -0,0 +1 @@
Subproject commit 6c095314d0c49061f41e1e40be2625dfc2253afa

8
tools/z64compress/src/enc/lzo.c
View File

@ -33,8 +33,8 @@ lzoenc(
unsigned char *wrkmem = _ctx;
lzo_uint result_sz = 0;
int hlen = 8; /* header length; required due to MM's archives */
memset(dst, 0, hlen);
extern int g_hlen; /* header length */
memset(dst, 0, g_hlen);
memcpy(dst, "LZO0", 4);
dst[4] = (src_sz >> 24);
dst[5] = (src_sz >> 16);
@ -46,9 +46,9 @@ lzoenc(
memset(wrkmem, 0, LZO1X_999_MEM_COMPRESS);
lzo1x_999_compress(src, src_sz, dst + hlen, &result_sz, wrkmem);
lzo1x_999_compress(src, src_sz, dst + g_hlen, &result_sz, wrkmem);
*dst_sz = result_sz + hlen;
*dst_sz = result_sz + g_hlen;
return 0;
}

8
tools/z64compress/src/enc/ucl.c
View File

@ -18,8 +18,8 @@ uclenc(
int level = 10;
ucl_uint result_sz;
int hlen = 8; /* header length; required due to MM's archives */
memset(dst, 0, hlen);
extern int g_hlen; /* header length */
memset(dst, 0, g_hlen);
memcpy(dst, "UCL0", 4);
dst[4] = (src_sz >> 24);
dst[5] = (src_sz >> 16);
@ -29,7 +29,7 @@ uclenc(
r = ucl_nrv2b_99_compress(
src /* in */
, src_sz /* in size */
, dst + hlen /* out */
, dst + g_hlen /* out */
, &result_sz /* out size */
, NULL /* callback */
, level /* level */
@ -43,7 +43,7 @@ uclenc(
exit(EXIT_FAILURE);
}
*dst_sz = result_sz + hlen;
*dst_sz = result_sz + g_hlen;
return 0;
}

18
tools/z64compress/src/enc/yaz.c
View File

@ -272,16 +272,20 @@ uint32_t encode(struct yazCtx *ctx, uint8_t *data, uint32_t data_size, uint8_t *
// block and stream differentiation
// Yay is block, Yaz is stream
int mode_block=1, mode_stream=1; // temporary, for testing
extern int g_hlen;
mode_block=!strcmp(mode,"Yay0");
if (mode_block ) {
uint32_t l = (sb_count(ctx->cmds) << 2) + 16;
uint32_t o = (sb_count(ctx->ctrl) << 1) + l;
if (g_hlen) {
memcpy(output, mode, 4);
U32wr(output+4, sz);
} else
output -= 8; /* headerless */
if (mode_block) {
uint32_t l = (sb_count(ctx->cmds) << 2) + 16;
uint32_t o = (sb_count(ctx->ctrl) << 1) + l;
U32wr(output+8, l);
U32wr(output+12, o);
uint32_t output_position = 16;
uint32_t output_position = g_hlen + 8;
uint32_t x;
for (x=0; x<sb_count(ctx->cmds); x++) {
U32wr(output+output_position, ctx->cmds[x]);
@ -296,8 +300,6 @@ uint32_t encode(struct yazCtx *ctx, uint8_t *data, uint32_t data_size, uint8_t *
}
return output_position;
} else if(mode_stream) {
memcpy(output, mode, 4);
U32wr(output+4, sz);
U32wr(output+8, 0);
U32wr(output+12, 0);
@ -315,8 +317,8 @@ uint32_t encode(struct yazCtx *ctx, uint8_t *data, uint32_t data_size, uint8_t *
sb_push(ctx->back, (ctx->ctrl[x]>>8)&0xFF);
sb_push(ctx->back, (ctx->ctrl[x])&0xFF);
}
output_position = _enc_z_from_tables(ctx, ctx->ctl, ctx->back, ctx->raws, output+16, data_size, mode);
return 16 + output_position;
output_position = _enc_z_from_tables(ctx, ctx->ctl, ctx->back, ctx->raws, output+g_hlen+8, data_size, mode);
return output_position + g_hlen + 8;
}
return 0;
}

103
tools/z64compress/src/enc/zlib.c Normal file
View File

@ -0,0 +1,103 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <zlib.h>
#include "libdeflate/libdeflate.h"
#define CAPACITY (1024 * 1024 * 4) /* output buffer max (4 mb) */
int
zlibenc(
void *_src
, unsigned src_sz
, void *_dst
, unsigned *dst_sz
, void *_ctx
)
{
unsigned char *src = _src;
unsigned char *dst = _dst;
unsigned result_sz;
extern int g_hlen; /* header length */
memset(dst, 0, g_hlen);
memcpy(dst, "ZLIB", 4);
dst[4] = (src_sz >> 24);
dst[5] = (src_sz >> 16);
dst[6] = (src_sz >> 8);
dst[7] = (src_sz >> 0);
/* zlib and gzip have different header lengths
* https://stackoverflow.com/a/68538037
*/
#if 1
#if 0 /* zlib */
z_stream stream = {0};
int r;
stream.avail_in = src_sz;
stream.next_in = src;
stream.avail_out = CAPACITY;
stream.next_out = dst + g_hlen;
#define HEADER_LEN 2
if ((r = deflateInit(&stream, Z_BEST_COMPRESSION)) != Z_OK)
{
fprintf(stderr, "[!] fatal compression error %d\n", r);
exit(EXIT_FAILURE);
}
if ((r = deflate(&stream, Z_FINISH)) == Z_STREAM_ERROR)
{
fprintf(stderr, "[!] Z_STREAM_ERROR\n");
exit(EXIT_FAILURE);
}
deflateEnd(&stream);
result_sz = CAPACITY - stream.avail_out;
#else /* libdeflate */
#define HEADER_LEN 0
int level = 12;
struct libdeflate_compressor *compressor;
compressor = libdeflate_alloc_compressor(level);
result_sz = libdeflate_deflate_compress(
compressor
, src, src_sz
, dst + g_hlen
, CAPACITY
);
libdeflate_free_compressor(compressor);
#endif
#else
/* this gzip code was left in for testing purposes; it may
* be useful if matching ique recompression is ever revisited;
* ique matches (except for one byte...) when compressed using
* gzip 1.2.4 or 1.2.4a (they produce identical results),
* available here: https://ftp.gnu.org/gnu/gzip/
* this is not a compression error, because decompressing the
* recompressed rom produces a rom identical to the original
* decompressed ique rom;
* TODO: find out why that byte doesn't match on recompression;
* TODO: once that's working, add --codec ique for those wanting
* matching ique recompression; otherwise, modern zlib works great!
*/
#define HEADER_LEN 10
FILE *fp = fopen("tmp.bin", "wb");
fwrite(src, 1, src_sz, fp);
fclose(fp);
system("./gzip -c -9 -n tmp.bin > tmp.bin.gzip");
fp = fopen("tmp.bin.gzip", "rb");
fseek(fp, 0, SEEK_END);
result_sz = ftell(fp);
fseek(fp, 0, SEEK_SET);
fread(dst, 1, result_sz, fp);
fclose(fp);
#endif
*dst_sz = result_sz + g_hlen;
/* trim zlib/gzip header */
memmove(dst + g_hlen, dst + g_hlen + HEADER_LEN, result_sz);
*dst_sz -= HEADER_LEN;
return 0;
(void)_ctx; /* -Wunused-parameter */
}

8
tools/z64compress/src/enc/zx7.c
View File

@ -16,20 +16,20 @@ zx7enc(
unsigned char *src = _src;
unsigned char *dst = _dst;
int hlen = 8; /* header length; required due to MM's archives */
memset(dst, 0, hlen);
extern int g_hlen; /* header length */
memset(dst, 0, g_hlen);
memcpy(dst, "ZX70", 4);
dst[4] = (src_sz >> 24);
dst[5] = (src_sz >> 16);
dst[6] = (src_sz >> 8);
dst[7] = (src_sz >> 0);
*dst_sz = ZX7Compress(src, src_sz, dst + hlen);
*dst_sz = ZX7Compress(src, src_sz, dst + g_hlen);
if (!*dst_sz)
return 1;
*dst_sz += hlen;
*dst_sz += g_hlen;
return 0;
}

14
tools/z64compress/src/main.c
View File

@ -8,6 +8,7 @@
#include "rom.h"
FILE* printer;
int g_hlen = 8;
static void compress(struct rom *rom, int start, int end)
{
@ -167,6 +168,7 @@ static void usage(void)
fprintf(printer, " yaz\n");
fprintf(printer, " ucl\n");
fprintf(printer, " lzo\n");
fprintf(printer, " zlib\n");
fprintf(printer, " aplib\n");
fprintf(printer, " * to use non-yaz codecs, find patches\n");
fprintf(printer, " and code on my z64enc repo\n");
@ -184,6 +186,8 @@ static void usage(void)
fprintf(printer, "\n");
fprintf(printer, " --skip disable compression on specified files\n");
fprintf(printer, "\n");
fprintf(printer, " --headerless don't write file headers (for iQue)\n");
fprintf(printer, "\n");
fprintf(printer, " --repack handles Majora's Mask archives\n");
fprintf(printer, "\n");
fprintf(printer, " --threads optional multithreading;\n");
@ -209,6 +213,7 @@ wow_main
int Athreads = 0;
bool Amatching = false;
bool Aonly_stdout = false;
bool Aheaderless = false;
wow_main_argv;
printer = stderr;
@ -253,6 +258,15 @@ wow_main
i--;
continue;
}
else if (!strcmp(arg, "--headerless"))
{
if (Aheaderless)
die("--headerless arg provided more than once");
Aheaderless = true;
g_hlen = 0;
i--;
continue;
}
/* arguments with additional parameters */

10
tools/z64compress/src/rom.c
View File

@ -462,6 +462,14 @@ static const struct encoder *encoder(const char *name)
return &zx7;
}*/
else if (!strcmp(name, "zlib"))
{
static const struct encoder zlib = {
.encfunc = zlibenc
};
return &zlib;
}
else if (!strcmp(name, "aplib"))
{
static const struct encoder aplib = {
@ -1112,7 +1120,7 @@ void rom_compress(struct rom *rom, int mb, int numThreads, bool matching)
comp_total += sz16;
if (mb != 0 && dma->Pend > compsz)
die("ran out of compressed rom space");
die("ran out of compressed rom space (try increasing --mb)");
}
/* adaptive final size */