Skip to content

Commit

Permalink
Add zopfli gzip encoder for better compression
Browse files Browse the repository at this point in the history
  • Loading branch information
@chaosmaster @topjohnwu
chaosmaster authored and topjohnwu committed Aug 28, 2021
1 parent f41575d commit 92a8a3e
Show file tree
Hide file tree
Showing 5 changed files with 158 additions and 1 deletion.
3 changes: 3 additions & 0 deletions .gitmodules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,6 @@
[submodule "termux-elf-cleaner"]
path = tools/termux-elf-cleaner
url = https://github.com/termux/termux-elf-cleaner.git
[submodule "zopfli"]
path = native/jni/external/zopfli
url = https://android.googlesource.com/platform/external/zopfli
2 changes: 1 addition & 1 deletion native/jni/Android.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ ifdef B_BOOT

include $(CLEAR_VARS)
LOCAL_MODULE := magiskboot
LOCAL_STATIC_LIBRARIES := libmincrypt liblzma liblz4 libbz2 libfdt libutils libz
LOCAL_STATIC_LIBRARIES := libmincrypt liblzma liblz4 libbz2 libfdt libutils libz libzopfli
LOCAL_C_INCLUDES := jni/include

LOCAL_SRC_FILES := \
Expand Down
21 changes: 21 additions & 0 deletions native/jni/external/Android.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,27 @@ LOCAL_SRC_FILES := \
zlib/zutil.c
include $(BUILD_STATIC_LIBRARY)

# libzopfli.a
include $(CLEAR_VARS)
LOCAL_MODULE:= libzopfli
LOCAL_C_INCLUDES := $(LOCAL_PATH)/zopfli/src
LOCAL_EXPORT_C_INCLUDES := $(LOCAL_C_INCLUDES)
LOCAL_CFLAGS := -O2 -Wall -Werror -Wno-unused -Wno-unused-parameter
LOCAL_SRC_FILES := \
zopfli/src/zopfli/blocksplitter.c \
zopfli/src/zopfli/cache.c \
zopfli/src/zopfli/deflate.c \
zopfli/src/zopfli/gzip_container.c \
zopfli/src/zopfli/hash.c \
zopfli/src/zopfli/katajainen.c \
zopfli/src/zopfli/lz77.c \
zopfli/src/zopfli/squeeze.c \
zopfli/src/zopfli/tree.c \
zopfli/src/zopfli/util.c \
zopfli/src/zopfli/zlib_container.c \
zopfli/src/zopfli/zopfli_lib.c
include $(BUILD_STATIC_LIBRARY)

CWD := $(LOCAL_PATH)
include $(CWD)/systemproperties/Android.mk
include $(CWD)/mincrypt/Android.mk
Expand Down
1 change: 1 addition & 0 deletions native/jni/external/zopfli
Submodule zopfli added at 7809db
132 changes: 132 additions & 0 deletions native/jni/magiskboot/compress.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#include <lz4.h>
#include <lz4frame.h>
#include <lz4hc.h>
#include <zopfli/util.h>
#include <zopfli/deflate.h>

#include <utils.hpp>

Expand All @@ -21,6 +23,12 @@ constexpr size_t CHUNK = 0x40000;
constexpr size_t LZ4_UNCOMPRESSED = 0x800000;
constexpr size_t LZ4_COMPRESSED = LZ4_COMPRESSBOUND(LZ4_UNCOMPRESSED);

#if defined(ZOPFLI_MASTER_BLOCK_SIZE) && ZOPFLI_MASTER_BLOCK_SIZE > 0
constexpr size_t ZOPFLI_CHUNK = ZOPFLI_MASTER_BLOCK_SIZE;
#else
constexpr size_t ZOPFLI_CHUNK = CHUNK;
#endif

class cpr_stream : public filter_stream {
public:
using filter_stream::filter_stream;
Expand Down Expand Up @@ -106,6 +114,129 @@ class gz_encoder : public gz_strm {
explicit gz_encoder(stream_ptr &&base) : gz_strm(ENCODE, std::move(base)) {};
};

class zopfli_gz_strm : public cpr_stream {
public:
int write(const void *buf, size_t len) override {
return len ? write(buf, len, 0) : 0;
}

~zopfli_gz_strm() override {
switch(mode) {
case ENCODE:
write(nullptr, 0, 1);
break;
}
}

protected:
enum mode_t {
ENCODE
} mode;

ZopfliOptions zo;

zopfli_gz_strm(mode_t mode, stream_ptr &&base) :
cpr_stream(std::move(base)), mode(mode), out(nullptr), outsize(0), bp(0), crcvalue(0xffffffffu), in_read(0) {
switch(mode) {
case ENCODE:
out = 0;
outsize = 0;
bp = 0;
crcvalue = crc32_z(0L, Z_NULL, 0);

ZopfliInitOptions(&zo);

// Speed things up a bit, this still leads to better compression than zlib
zo.numiterations = 1;
zo.blocksplitting = 0;

ZOPFLI_APPEND_DATA(31, &out, &outsize); /* ID1 */
ZOPFLI_APPEND_DATA(139, &out, &outsize); /* ID2 */
ZOPFLI_APPEND_DATA(8, &out, &outsize); /* CM */
ZOPFLI_APPEND_DATA(0, &out, &outsize); /* FLG */
/* MTIME */
ZOPFLI_APPEND_DATA(0, &out, &outsize);
ZOPFLI_APPEND_DATA(0, &out, &outsize);
ZOPFLI_APPEND_DATA(0, &out, &outsize);
ZOPFLI_APPEND_DATA(0, &out, &outsize);

ZOPFLI_APPEND_DATA(2, &out, &outsize); /* XFL, 2 indicates best compression. */
ZOPFLI_APPEND_DATA(3, &out, &outsize); /* OS follows Unix conventions. */
break;
}
}

private:
unsigned char* out = nullptr;
size_t outsize = 0;
unsigned char bp = 0;
unsigned long crcvalue = 0xffffffffu;
uint32_t in_read = 0;

int write(const void *buf, size_t len, int flush) {
int ret = 0;
switch(mode) {
case ENCODE:
in_read += len;
if (len)
crcvalue = crc32_z(crcvalue, (Bytef *)buf, len);
if (flush) {
ZopfliDeflate(&zo, 2, 1, (const unsigned char *)buf, len, &bp, &out, &outsize);

/* CRC */
ZOPFLI_APPEND_DATA(crcvalue % 256, &out, &outsize);
ZOPFLI_APPEND_DATA((crcvalue >> 8) % 256, &out, &outsize);
ZOPFLI_APPEND_DATA((crcvalue >> 16) % 256, &out, &outsize);
ZOPFLI_APPEND_DATA((crcvalue >> 24) % 256, &out, &outsize);

/* ISIZE */
ZOPFLI_APPEND_DATA(in_read % 256, &out, &outsize);
ZOPFLI_APPEND_DATA((in_read >> 8) % 256, &out, &outsize);
ZOPFLI_APPEND_DATA((in_read >> 16) % 256, &out, &outsize);
ZOPFLI_APPEND_DATA((in_read >> 24) % 256, &out, &outsize);
ret += bwrite(out, outsize);
free(out);
out = nullptr;
bp = 0;
outsize = 0;
}
else {
for(size_t offset = 0; offset < len; offset += ZOPFLI_CHUNK) {
ZopfliDeflatePart(&zo, 2, 0, (const unsigned char *)buf, offset, offset + ((len - offset) < ZOPFLI_CHUNK ? len - offset : ZOPFLI_CHUNK), &bp, &out, &outsize);
bp &= 7;
if (bp & 1) {
if (bp == 7)
ZOPFLI_APPEND_DATA(0, &out, &outsize);
ZOPFLI_APPEND_DATA(0, &out, &outsize);
ZOPFLI_APPEND_DATA(0, &out, &outsize);
ZOPFLI_APPEND_DATA(0xff, &out, &outsize);
ZOPFLI_APPEND_DATA(0xff, &out, &outsize);
}
else if (bp) {
do {
out[outsize - 1] += 2 << bp;
ZOPFLI_APPEND_DATA(0, &out, &outsize);
bp += 2;
} while (bp < 8);
}

ret += bwrite(out, outsize);
free(out);
out = nullptr;
bp = 0;
outsize = 0;
}
}
return ret;
}
}
};

class zopfli_gz_encoder : public zopfli_gz_strm {
public:
explicit zopfli_gz_encoder(stream_ptr &&base) : zopfli_gz_strm(ENCODE, std::move(base)) {};
};

class bz_strm : public cpr_stream {
public:
ssize_t write(const void *buf, size_t len) override {
Expand Down Expand Up @@ -535,6 +666,7 @@ stream_ptr get_encoder(format_t type, stream_ptr &&base) {
case LZ4_LG:
return make_unique<LZ4_encoder>(std::move(base), true);
case GZIP:
return make_unique<zopfli_gz_encoder>(std::move(base));
default:
return make_unique<gz_encoder>(std::move(base));
}
Expand Down

0 comments on commit 92a8a3e

Please sign in to comment.