add real compression to PSP iso in zip file

This commit is contained in:
Martins Mozeiko
2017-11-06 23:50:22 -08:00
parent fdf3978df6
commit 27315027c3
11 changed files with 1652 additions and 44 deletions
+1 -1
View File
@@ -10,4 +10,4 @@ set CL=%CL% /Ox
rem set CL=%CL% /Od /Zi
rem set LINK=%LINK% /DEBUG
cl.exe pkg2zip*.c puff.c /Fepkg2zip.exe
cl.exe pkg2zip*.c miniz_tdef.c puff.c /Fepkg2zip.exe
+1 -1
View File
@@ -6,7 +6,7 @@ else
endif
BIN=pkg2zip${EXE}
SRC=${wildcard pkg2zip*.c} puff.c
SRC=${wildcard pkg2zip*.c} miniz_tdef.c puff.c
OBJ=${SRC:.c=.o}
DEP=${SRC:.c=.d}
+1367
View File
File diff suppressed because it is too large Load Diff
+180
View File
@@ -0,0 +1,180 @@
#pragma once
// public domain code from https://github.com/richgel999/miniz/tree/42c29103c304a6e9201d000c96c31cd3031efc4f
#include <string.h>
#include <stdint.h>
// ------------------- Types and macros
typedef uint8_t mz_uint8;
typedef int16_t mz_int16;
typedef uint16_t mz_uint16;
typedef uint32_t mz_uint32;
typedef uint32_t mz_uint;
typedef int64_t mz_int64;
typedef uint64_t mz_uint64;
typedef int mz_bool;
#define MZ_FALSE (0)
#define MZ_TRUE (1)
// Works around MSVC's spammy "warning C4127: conditional expression is constant" message.
#ifdef _MSC_VER
#define MZ_MACRO_END while (0, 0)
#else
#define MZ_MACRO_END while (0)
#endif
#define MZ_ASSERT(x)
#define MZ_MAX(a, b) (((a) > (b)) ? (a) : (b))
#define MZ_MIN(a, b) (((a) < (b)) ? (a) : (b))
#define MZ_CLEAR_OBJ(obj) memset(&(obj), 0, sizeof(obj))
#ifdef _MSC_VER
#define MZ_FORCEINLINE __forceinline
#elif defined(__GNUC__)
#define MZ_FORCEINLINE inline __attribute__((__always_inline__))
#else
#define MZ_FORCEINLINE inline
#endif
// ------------------- Low-level Compression API Definitions
// tdefl_init() compression flags logically OR'd together (low 12 bits contain the max. number of probes per dictionary search):
// TDEFL_DEFAULT_MAX_PROBES: The compressor defaults to 128 dictionary probes per dictionary search. 0=Huffman only, 1=Huffman+LZ (fastest/crap compression), 4095=Huffman+LZ (slowest/best compression).
enum
{
TDEFL_HUFFMAN_ONLY = 0,
TDEFL_DEFAULT_MAX_PROBES = 128,
TDEFL_MAX_PROBES_MASK = 0xFFF
};
// TDEFL_WRITE_ZLIB_HEADER: If set, the compressor outputs a zlib header before the deflate data, and the Adler-32 of the source data at the end. Otherwise, you'll get raw deflate data.
// TDEFL_COMPUTE_ADLER32: Always compute the adler-32 of the input data (even when not writing zlib headers).
// TDEFL_GREEDY_PARSING_FLAG: Set to use faster greedy parsing, instead of more efficient lazy parsing.
// TDEFL_NONDETERMINISTIC_PARSING_FLAG: Enable to decrease the compressor's initialization time to the minimum, but the output may vary from run to run given the same input (depending on the contents of memory).
// TDEFL_RLE_MATCHES: Only look for RLE matches (matches with a distance of 1)
// TDEFL_FILTER_MATCHES: Discards matches <= 5 chars if enabled.
// TDEFL_FORCE_ALL_STATIC_BLOCKS: Disable usage of optimized Huffman tables.
// TDEFL_FORCE_ALL_RAW_BLOCKS: Only use raw (uncompressed) deflate blocks.
// The low 12 bits are reserved to control the max # of hash probes per dictionary lookup (see TDEFL_MAX_PROBES_MASK).
enum
{
TDEFL_WRITE_ZLIB_HEADER = 0x01000,
TDEFL_COMPUTE_ADLER32 = 0x02000,
TDEFL_GREEDY_PARSING_FLAG = 0x04000,
TDEFL_NONDETERMINISTIC_PARSING_FLAG = 0x08000,
TDEFL_RLE_MATCHES = 0x10000,
TDEFL_FILTER_MATCHES = 0x20000,
TDEFL_FORCE_ALL_STATIC_BLOCKS = 0x40000,
TDEFL_FORCE_ALL_RAW_BLOCKS = 0x80000
};
enum
{
TDEFL_MAX_HUFF_TABLES = 3,
TDEFL_MAX_HUFF_SYMBOLS_0 = 288,
TDEFL_MAX_HUFF_SYMBOLS_1 = 32,
TDEFL_MAX_HUFF_SYMBOLS_2 = 19,
TDEFL_LZ_DICT_SIZE = 32768,
TDEFL_LZ_DICT_SIZE_MASK = TDEFL_LZ_DICT_SIZE - 1,
TDEFL_MIN_MATCH_LEN = 3,
TDEFL_MAX_MATCH_LEN = 258
};
enum
{
TDEFL_LZ_CODE_BUF_SIZE = 64 * 1024,
TDEFL_OUT_BUF_SIZE = (TDEFL_LZ_CODE_BUF_SIZE * 13) / 10,
TDEFL_MAX_HUFF_SYMBOLS = 288,
TDEFL_LZ_HASH_BITS = 15,
TDEFL_LEVEL1_HASH_SIZE_MASK = 4095,
TDEFL_LZ_HASH_SHIFT = (TDEFL_LZ_HASH_BITS + 2) / 3,
TDEFL_LZ_HASH_SIZE = 1 << TDEFL_LZ_HASH_BITS
};
// The low-level tdefl functions below may be used directly if the above helper functions aren't flexible enough. The low-level functions don't make any heap allocations, unlike the above helper functions.
typedef enum
{
TDEFL_STATUS_BAD_PARAM = -2,
TDEFL_STATUS_PUT_BUF_FAILED = -1,
TDEFL_STATUS_OKAY = 0,
TDEFL_STATUS_DONE = 1,
} tdefl_status;
// Must map to MZ_NO_FLUSH, MZ_SYNC_FLUSH, etc. enums
typedef enum
{
TDEFL_NO_FLUSH = 0,
TDEFL_SYNC_FLUSH = 2,
TDEFL_FULL_FLUSH = 3,
TDEFL_FINISH = 4
} tdefl_flush;
// tdefl's compression state structure.
typedef struct
{
mz_uint m_flags, m_max_probes[2];
int m_greedy_parsing;
mz_uint m_adler32, m_lookahead_pos, m_lookahead_size, m_dict_size;
mz_uint8 *m_pLZ_code_buf, *m_pLZ_flags, *m_pOutput_buf, *m_pOutput_buf_end;
mz_uint m_num_flags_left, m_total_lz_bytes, m_lz_code_buf_dict_pos, m_bits_in, m_bit_buffer;
mz_uint m_saved_match_dist, m_saved_match_len, m_saved_lit, m_output_flush_ofs, m_output_flush_remaining, m_finished, m_block_index, m_wants_to_finish;
tdefl_status m_prev_return_status;
const void *m_pIn_buf;
void *m_pOut_buf;
size_t *m_pIn_buf_size, *m_pOut_buf_size;
tdefl_flush m_flush;
const mz_uint8 *m_pSrc;
size_t m_src_buf_left, m_out_buf_ofs;
mz_uint8 m_dict[TDEFL_LZ_DICT_SIZE + TDEFL_MAX_MATCH_LEN - 1];
mz_uint16 m_huff_count[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];
mz_uint16 m_huff_codes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];
mz_uint8 m_huff_code_sizes[TDEFL_MAX_HUFF_TABLES][TDEFL_MAX_HUFF_SYMBOLS];
mz_uint8 m_lz_code_buf[TDEFL_LZ_CODE_BUF_SIZE];
mz_uint16 m_next[TDEFL_LZ_DICT_SIZE];
mz_uint16 m_hash[TDEFL_LZ_HASH_SIZE];
mz_uint8 m_output_buf[TDEFL_OUT_BUF_SIZE];
} tdefl_compressor;
// Initializes the compressor.
// There is no corresponding deinit() function because the tdefl API's do not dynamically allocate memory.
// flags: See the above enums (TDEFL_HUFFMAN_ONLY, TDEFL_WRITE_ZLIB_HEADER, etc.)
tdefl_status tdefl_init(tdefl_compressor *d, int flags);
// Compresses a block of data, consuming as much of the specified input buffer as possible, and writing as much compressed data to the specified output buffer as possible.
tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush);
#define MZ_ADLER32_INIT (1)
// mz_adler32() returns the initial adler-32 value to use when called with ptr==NULL.
mz_uint32 mz_adler32(mz_uint32 adler, const unsigned char *ptr, size_t buf_len);
// Compression strategies.
enum
{
MZ_DEFAULT_STRATEGY = 0,
MZ_FILTERED = 1,
MZ_HUFFMAN_ONLY = 2,
MZ_RLE = 3,
MZ_FIXED = 4
};
// Compression levels: 0-9 are the standard zlib-style levels, 10 is best possible compression (not zlib compatible, and may be very slow), MZ_DEFAULT_COMPRESSION=MZ_DEFAULT_LEVEL.
enum
{
MZ_NO_COMPRESSION = 0,
MZ_BEST_SPEED = 1,
MZ_BEST_COMPRESSION = 9,
MZ_UBER_COMPRESSION = 10,
MZ_DEFAULT_LEVEL = 6,
MZ_DEFAULT_COMPRESSION = -1
};
#define MZ_DEFAULT_WINDOW_BITS 15
// Create tdefl_compress() flags given zlib-style compression parameters.
// level may range from [0,10] (where 10 is absolute max compression, but may be much slower on some files)
// window_bits may be -15 (raw deflate) or 15 (zlib)
// strategy may be either MZ_DEFAULT_STRATEGY, MZ_FILTERED, MZ_HUFFMAN_ONLY, MZ_RLE, or MZ_FIXED
mz_uint tdefl_create_comp_flags_from_zip_params(int level, int window_bits, int strategy);
+7 -7
View File
@@ -737,7 +737,7 @@ int main(int argc, char* argv[])
uint64_t offset = data_offset;
out_begin_file(path);
out_begin_file(path, 0);
while (data_size != 0)
{
uint8_t buffer[1 << 16];
@@ -769,7 +769,7 @@ int main(int argc, char* argv[])
printf("[*] creating sce_sys/package/head.bin\n");
snprintf(path, sizeof(path), "%s/sce_sys/package/head.bin", root);
out_begin_file(path);
out_begin_file(path, 0);
uint64_t head_size = enc_offset + items_size;
uint64_t head_offset = 0;
while (head_size != 0)
@@ -786,7 +786,7 @@ int main(int argc, char* argv[])
printf("[*] creating sce_sys/package/tail.bin\n");
snprintf(path, sizeof(path), "%s/sce_sys/package/tail.bin", root);
out_begin_file(path);
out_begin_file(path, 0);
uint64_t tail_offset = enc_offset + enc_size;
while (tail_offset != pkg_size)
{
@@ -802,7 +802,7 @@ int main(int argc, char* argv[])
snprintf(path, sizeof(path), "%s/sce_sys/package/stat.bin", root);
uint8_t stat[768] = { 0 };
out_begin_file(path);
out_begin_file(path, 0);
out_write(stat, sizeof(stat));
out_end_file();
}
@@ -824,7 +824,7 @@ int main(int argc, char* argv[])
snprintf(path, sizeof(path), "%s/sce_sys/package/work.bin", root);
}
out_begin_file(path);
out_begin_file(path, 0);
out_write(rif, rif_size);
out_end_file();
}
@@ -849,7 +849,7 @@ int main(int argc, char* argv[])
printf("[*] creating RW/System/content_id\n");
snprintf(path, sizeof(path), "%s/RW/System/content_id", root);
out_begin_file(path);
out_begin_file(path, 0);
out_write(pkg_header + 0x30, 0x30);
out_end_file();
@@ -857,7 +857,7 @@ int main(int argc, char* argv[])
snprintf(path, sizeof(path), "%s/RW/System/pm.dat", root);
uint8_t pm[1 << 16] = { 0 };
out_begin_file(path);
out_begin_file(path, 0);
out_write(pm, sizeof(pm));
out_end_file();
}
+2 -2
View File
@@ -36,11 +36,11 @@ void out_add_folder(const char* path)
}
}
void out_begin_file(const char* name)
void out_begin_file(const char* name, int compress)
{
if (out_zipped)
{
zip_begin_file(&out_zip, name);
zip_begin_file(&out_zip, name, compress);
}
else
{
+1 -1
View File
@@ -5,6 +5,6 @@
void out_begin(const char* name, int zipped);
void out_end(void);
void out_add_folder(const char* path);
void out_begin_file(const char* name);
void out_begin_file(const char* name, int compress);
void out_end_file(void);
void out_write(const void* buffer, uint32_t size);
+2 -2
View File
@@ -366,7 +366,7 @@ void unpack_psp_eboot(const char* path, const aes128_key* pkg_key, const uint8_t
fatal("ERROR: offset table in data.psar file is too large!");
}
out_begin_file(path);
out_begin_file(path, 1);
for (uint32_t i = 0; i < block_count; i++)
{
uint64_t table_offset = item_offset + psar_offset + iso_table + 32 * i;
@@ -463,7 +463,7 @@ void unpack_psp_key(const char* path, const aes128_key* pkg_key, const uint8_t*
init_psp_decrypt(&psp_key, psp_iv, 0, mac, key_header, 0x70, 0x30);
aes128_psp_decrypt(&psp_key, psp_iv, 0, key_header + 0x90, 0x10);
out_begin_file(path);
out_begin_file(path, 0);
out_write(key_header + 0x90, 0x10);
out_end_file();
}
+86 -14
View File
@@ -12,8 +12,11 @@
#define ZIP_MEMORY_BLOCK (1024 * 1024)
// https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
#define ZIP_VERSION 45
#define ZIP_METHOD_STORE 0
#define ZIP_METHOD_DEFLATE 8
#define ZIP_UTF8_FLAG (1 << 11)
#define ZIP_DOS_ATTRIBUTE_DIRECTORY 0x10
@@ -32,7 +35,9 @@ struct zip_file
{
uint64_t offset;
uint64_t size;
uint64_t compressed;
uint32_t crc32;
int compress;
};
static zip_file* zip_new_file(zip* z)
@@ -74,7 +79,9 @@ void zip_add_folder(zip* z, const char* name)
zip_file* f = zip_new_file(z);
f->offset = z->total;
f->size = 0;
f->compressed = 0;
f->crc32 = 0;
f->compress = 0;
uint8_t header[ZIP_LOCAL_HEADER_SIZE] = { 0x50, 0x4b, 0x03, 0x04 };
// version needed to extract
@@ -101,7 +108,7 @@ void zip_add_folder(zip* z, const char* name)
z->total += 1;
}
void zip_begin_file(zip* z, const char* name)
void zip_begin_file(zip* z, const char* name, int compress)
{
size_t name_length = strlen(name);
if (name_length > ZIP_MAX_FILENAME)
@@ -112,6 +119,8 @@ void zip_begin_file(zip* z, const char* name)
zip_file* f = zip_new_file(z);
f->offset = z->total;
f->size = 0;
f->compressed = 0;
f->compress = compress;
z->current = f;
crc32_init(&z->crc32);
@@ -122,7 +131,7 @@ void zip_begin_file(zip* z, const char* name)
// general purpose bit flag
set16le(header + 6, ZIP_UTF8_FLAG);
// compression method
set16le(header + 8, ZIP_METHOD_STORE);
set16le(header + 8, compress ? ZIP_METHOD_DEFLATE : ZIP_METHOD_STORE);
// last mod file time
set16le(header + 10, z->time);
// last mod file date
@@ -135,29 +144,84 @@ void zip_begin_file(zip* z, const char* name)
sys_write(z->file, z->total, name, (uint16_t)name_length);
z->total += name_length;
if (compress)
{
int flags = tdefl_create_comp_flags_from_zip_params(MZ_BEST_SPEED, -MZ_DEFAULT_WINDOW_BITS, MZ_DEFAULT_STRATEGY);
tdefl_init(&z->tdefl, flags);
}
}
void zip_write_file(zip* z, const void* data, uint32_t size)
{
sys_write(z->file, z->total, data, size);
z->total += size;
z->current->size += size;
crc32_update(&z->crc32, data, size);
if (z->current->compress)
{
const uint8_t* data8 = data;
while (size != 0)
{
uint8_t buffer[4096];
size_t isize = size;
size_t osize = sizeof(buffer);
tdefl_compress(&z->tdefl, data8, &isize, buffer, &osize, TDEFL_NO_FLUSH);
if (osize != 0)
{
sys_write(z->file, z->total, buffer, (uint32_t)osize);
z->current->compressed += osize;
z->total += osize;
}
data8 += isize;
size -= (uint32_t)isize;
}
}
else
{
sys_write(z->file, z->total, data, size);
z->current->compressed += size;
z->total += size;
}
}
void zip_end_file(zip* z)
{
if (z->current->compress)
{
for (;;)
{
uint8_t buffer[4096];
size_t isize = 0;
size_t osize = sizeof(buffer);
tdefl_status st = tdefl_compress(&z->tdefl, NULL, &isize, buffer, &osize, TDEFL_FINISH);
if (osize != 0)
{
sys_write(z->file, z->total, buffer, (uint32_t)osize);
z->current->compressed += osize;
z->total += osize;
}
if (st == TDEFL_STATUS_DONE)
{
break;
}
}
}
z->current->crc32 = crc32_done(&z->crc32);
uint64_t size = z->current->size;
if (size > 0)
if (z->current->size != 0)
{
uint8_t update[3 * sizeof(uint32_t)];
// crc-32
set32le(update + 0, z->current->crc32);
// compressed size
set32le(update + 4, (uint32_t)min64(size, 0xffffffff));
set32le(update + 4, (uint32_t)min64(z->current->compressed, 0xffffffff));
// uncompressed size
set32le(update + 8, (uint32_t)min64(size, 0xffffffff));
set32le(update + 8, (uint32_t)min64(z->current->size, 0xffffffff));
sys_write(z->file, z->current->offset + ZIP_LOCAL_HEADER_CRC32_OFFSET, update, sizeof(update));
}
@@ -186,6 +250,7 @@ void zip_close(zip* z)
uint8_t extra[28];
uint16_t extra_size = 0;
uint64_t size = f->size;
uint64_t compressed = f->compressed;
uint64_t offset = f->offset;
uint32_t attributes = ZIP_DOS_ATTRIBUTE_ARCHIVE;
if (is_folder)
@@ -200,7 +265,11 @@ void zip_close(zip* z)
{
if (size > 0xffffffff)
{
extra_size += 2 * sizeof(uint64_t);
extra_size += sizeof(uint64_t);
}
if (compressed > 0xffffffff)
{
extra_size += sizeof(uint64_t);
}
if (offset > 0xffffffff)
{
@@ -220,7 +289,7 @@ void zip_close(zip* z)
// general purpose bit flag
set16le(global + 8, ZIP_UTF8_FLAG);
// compression method
set16le(global + 10, ZIP_METHOD_STORE);
set16le(global + 10, f->compress ? ZIP_METHOD_DEFLATE : ZIP_METHOD_STORE);
// last mod file time
set16le(global + 12, z->time);
// last mod file date
@@ -228,7 +297,7 @@ void zip_close(zip* z)
// crc-32
set32le(global + 16, f->crc32);
// compressed size
set32le(global + 20, (uint32_t)min64(size, 0xffffffff));
set32le(global + 20, (uint32_t)min64(compressed, 0xffffffff));
// uncompressed size
set32le(global + 24, (uint32_t)min64(size, 0xffffffff));
// file name length
@@ -248,14 +317,17 @@ void zip_close(zip* z)
// size of this "extra" block
uint32_t extra_offset = 2 * sizeof(uint16_t);
set16le(extra + 2, (uint16_t)(extra_size - extra_offset));
if (compressed > 0xffffffff)
{
// size of compressed data
set64le(extra + extra_offset, compressed);
extra_offset += sizeof(uint64_t);
}
if (size > 0xffffffff)
{
// original uncompressed file size
set64le(extra + extra_offset, size);
extra_offset += sizeof(uint64_t);
// size of compressed data
set64le(extra + extra_offset, size);
extra_offset += sizeof(uint64_t);
}
if (offset > 0xffffffff)
{
+3 -1
View File
@@ -2,6 +2,7 @@
#include "pkg2zip_sys.h"
#include "pkg2zip_crc32.h"
#include "miniz_tdef.h"
#include <stddef.h>
#include <stdint.h>
@@ -17,6 +18,7 @@ typedef struct {
uint32_t max;
uint16_t time;
uint16_t date;
tdefl_compressor tdefl;
crc32_ctx crc32;
uint32_t allocated; // bytes
zip_file* files;
@@ -25,7 +27,7 @@ typedef struct {
void zip_create(zip* z, const char* name);
void zip_add_folder(zip* z, const char* name);
void zip_begin_file(zip* z, const char* name);
void zip_begin_file(zip* z, const char* name, int compress);
void zip_write_file(zip* z, const void* data, uint32_t size);
void zip_end_file(zip* z);
void zip_close(zip* z);
+2 -15
View File
@@ -1,5 +1,6 @@
#include "pkg2zip_zrif.h"
#include "pkg2zip_utils.h"
#include "miniz_tdef.h"
#include "puff.h"
#include <assert.h>
@@ -61,20 +62,6 @@ static const uint8_t b64d[] =
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
};
static uint32_t adler32(const uint8_t* data, size_t size)
{
uint32_t a = 1;
uint32_t b = 0;
for (size_t i = 0; i < size; i++)
{
a = (a + data[i]) % ADLER32_MOD;
b = (b + a) % ADLER32_MOD;
}
return (b << 16) | a;
}
static uint32_t base64_decode(const char* in, uint8_t* out)
{
const uint8_t* out0 = out;
@@ -163,7 +150,7 @@ static uint32_t zlib_inflate(const uint8_t* in, uint32_t inlen, uint8_t* out, ui
}
memmove(out, out + dictlen, dlen);
if (adler32(out, dlen) != get32be(in + slen))
if (mz_adler32(MZ_ADLER32_INIT, out, dlen) != get32be(in + slen))
{
fatal("ERROR: zRIF is corrupted, wrong checksum\n");
}