From 5a776d149f41a19728efb24d48b5eff13dc9c11c Mon Sep 17 00:00:00 2001 From: ketmar Date: Wed, 3 Nov 2021 06:24:34 +0000 Subject: [PATCH] backend: added LZMA packer/unpacker, with iv.dlzma (unused now, because LZMA doesn't shine on small buffers) FossilOrigin-Name: 4520496323cf253c379bf5744f6eb664b8ac60ae7251af8b4594a99ee0bd97ea --- chibackend/sqbase.d | 83 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 9 deletions(-) diff --git a/chibackend/sqbase.d b/chibackend/sqbase.d index 40d48b4..ef2364c 100644 --- a/chibackend/sqbase.d +++ b/chibackend/sqbase.d @@ -108,6 +108,9 @@ private __gshared string ExpandedMailDBPath = null; public __gshared int ChiroCompressionLevel = ChiroDefaultPackLevel; public __gshared bool ChiroSQLiteSilent = false; +// if `true`, will try both ZLib and LZMA +public __gshared bool ChiroPackTryHard = false; + public __gshared bool ChiroTimerEnabled = false; private __gshared Timer chiTimer = Timer(false); private __gshared char[] chiTimerMsg = null; @@ -997,7 +1000,7 @@ private void sq3Fn_ChiroPackLZMA (sqlite3_context *ctx, int argc, sqlite3_value if (xszlen) { xszlen += 5; immutable uint bsz = cast(uint)sz; - char* cbuf = cast(char*)malloc(bsz+xszlen+LZMA_PROPS_SIZE+1); + char* cbuf = cast(char*)malloc(bsz+xszlen+LZMA_PROPS_SIZE+1+16); if (cbuf is null) { if (isBadPrefix(vs[0..cast(uint)sz])) { sqlite3_result_error_nomem(ctx); return; } } else { @@ -1026,7 +1029,7 @@ private void sq3Fn_ChiroPackLZMA (sqlite3_context *ctx, int argc, sqlite3_value } if (isBadPrefix(vs[0..cast(uint)sz])) { - char *res = cast(char *)malloc(sz+4); + char *res = cast(char *)malloc(sz+5); if (res is null) { sqlite3_result_error_nomem(ctx); return; } res[0..5] = "\x1bRAWB"; res[5..sz+5] = vs[0..sz]; @@ -1099,9 +1102,9 @@ private void sq3Fn_ChiroPackCommon (sqlite3_context *ctx, sqlite3_value *val, in } else { xsz[0..5] = "\x1bZLIB"; } - uint xszlen = encodeUInt(xsz[5..$], cast(uint)sz); - if (xszlen) { - xszlen += 5; + immutable uint xszlenNum = encodeUInt(xsz[5..$], cast(uint)sz); + if (xszlenNum) { + immutable uint xszlen = xszlenNum+5; //xsz[xszlen++] = ':'; version(use_libbrieflz) { immutable usize bsz = blz_max_packed_size(cast(usize)sz); @@ -1112,7 +1115,7 @@ private void sq3Fn_ChiroPackCommon (sqlite3_context *ctx, sqlite3_value *val, in } else { immutable uint bsz = cast(uint)sz; } - char* cbuf = cast(char*)malloc(bsz+xszlen); + char* cbuf = cast(char*)malloc(bsz+xszlen+64); if (cbuf is null) { if (isBadPrefix(vs[0..cast(uint)sz])) { sqlite3_result_error_nomem(ctx); return; } } else { @@ -1262,19 +1265,61 @@ private void sq3Fn_ChiroPackCommon (sqlite3_context *ctx, sqlite3_value *val, in //uint bsz = cast(uint)compressBound(cast(uint)sz); if (packlevel > 9) packlevel = 9; usize dsize = bsz; - int zres = compress2(cast(ubyte *)(cbuf+xszlen), &dsize, cast(const(ubyte) *)vs, sz, packlevel); + immutable int zres = compress2(cast(ubyte *)(cbuf+xszlen), &dsize, cast(const(ubyte) *)vs, sz, packlevel); if (zres == Z_OK && dsize+xszlen < cast(usize)sz) { + if (!ChiroPackTryHard) { + sqlite3_result_blob(ctx, cbuf, dsize+xszlen, &free); + return; + } + } else { + free(cbuf); + cbuf = null; + } + // try LZMA? + if (ChiroPackTryHard) { + char* lzmabuf = cast(char*)malloc(bsz+xszlen+LZMA_PROPS_SIZE+1+64); + if (lzmabuf !is null) { + lzmabuf[0..xszlen] = xsz[0..xszlen]; + lzmabuf[1..5] = "LZMA"; + usize destLen = (cbuf is null ? bsz : dsize); // do not take more than zlib + if (destLen > bsz) destLen = bsz; // just in case + ubyte[LZMA_PROPS_SIZE+8] hdr = void; + uint hdrSize = cast(uint)hdr.sizeof; + + CLzmaEncProps props; + props.level = packlevel; + props.dictSize = 1<<22; //4MB + props.reduceSize = bsz; + + immutable SRes nres = LzmaEncode(cast(ubyte*)(lzmabuf+xszlen+LZMA_PROPS_SIZE+1), &destLen, cast(const(ubyte)*)vs, bsz, &props, hdr.ptr, &hdrSize, 0/*writeEndMark*/, null, &lzmaDefAllocator, &lzmaDefAllocator); + assert(hdrSize == LZMA_PROPS_SIZE); + if (nres == SZ_OK && destLen+xszlen+LZMA_PROPS_SIZE+1 < cast(usize)sz) { + if (cbuf is null || destLen+xszlen+LZMA_PROPS_SIZE+1 < dsize+xszlen) { + if (cbuf !is null) free(cbuf); // free zlib result + import core.stdc.string : memcpy; + lzmabuf[xszlen] = LZMA_PROPS_SIZE; + memcpy(lzmabuf+xszlen+1, hdr.ptr, LZMA_PROPS_SIZE); + sqlite3_result_blob(ctx, lzmabuf, destLen+xszlen+LZMA_PROPS_SIZE+1, &free); + return; + } + } + free(lzmabuf); + } + } + // return zlib result? + if (cbuf !is null) { + assert(dsize < bsz); sqlite3_result_blob(ctx, cbuf, dsize+xszlen, &free); return; } } - free(cbuf); + if (cbuf !is null) free(cbuf); } } } if (isBadPrefix(vs[0..cast(uint)sz])) { - char *res = cast(char *)malloc(sz+4); + char *res = cast(char *)malloc(sz+5); if (res is null) { sqlite3_result_error_nomem(ctx); return; } res[0..5] = "\x1bRAWB"; res[5..sz+5] = vs[0..sz]; @@ -1319,6 +1364,24 @@ private void sq3Fn_ChiroPackDPArg (sqlite3_context *ctx, int argc, sqlite3_value /* +** ChiroGetPackType(content) +*/ +private void sq3Fn_ChiroGetPackType (sqlite3_context *ctx, int argc, sqlite3_value **argv) { + if (argc != 1) { sqlite3_result_error(ctx, "invalid number of arguments to `ChiroGetPackType()`", -1); return; } + + int sz = sqlite3_value_bytes(argv[0]); + if (sz < 5 || sz > 0x3fffffff-4) { sqlite3_result_text(ctx, "RAWB", 4, SQLITE_STATIC); return; } + + const(char)* vs = cast(const(char) *)sqlite3_value_blob(argv[0]); + if (!vs) { sqlite3_result_error(ctx, "cannot get blob data in `ChiroUnpack()`", -1); return; } + + if (!isBadPrefix(vs[0..cast(uint)sz])) { sqlite3_result_text(ctx, "RAWB", 4, SQLITE_STATIC); return; } + + sqlite3_result_text(ctx, vs+1, 4, SQLITE_TRANSIENT); +} + + +/* ** ChiroUnpack(content) ** ** it is (almost) safe to pass non-packed content here @@ -2274,6 +2337,8 @@ public void chiroRegisterSQLite3Functions (ref Database db) { db.createFunction("ChiroPackLZMA", 1, &sq3Fn_ChiroPackLZMA, moreflags:/*SQLITE_DIRECTONLY*/SQLITE_INNOCUOUS); db.createFunction("ChiroPackLZMA", 2, &sq3Fn_ChiroPackLZMA, moreflags:/*SQLITE_DIRECTONLY*/SQLITE_INNOCUOUS); + db.createFunction("ChiroGetPackType", 1, &sq3Fn_ChiroGetPackType, moreflags:/*SQLITE_DIRECTONLY*/SQLITE_INNOCUOUS); + db.createFunction("ChiroNormCRLF", 1, &sq3Fn_ChiroNormCRLF, moreflags:/*SQLITE_DIRECTONLY*/SQLITE_INNOCUOUS); db.createFunction("ChiroNormHeaders", 1, &sq3Fn_ChiroNormHeaders, moreflags:/*SQLITE_DIRECTONLY*/SQLITE_INNOCUOUS); db.createFunction("ChiroExtractHeaders", 1, &sq3Fn_ChiroExtractHeaders, moreflags:/*SQLITE_DIRECTONLY*/SQLITE_INNOCUOUS); -- 2.11.4.GIT