From f59328bb2d1f5575c0514123abc8e3e2be28a954 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 1 Nov 2023 14:55:17 +0000 Subject: [PATCH] HPCC-29917 Refactor compressToBuffer to support different compression methods Also add some tests, and fix some issues that the tests showed up with small input sizes. Ensure back compatibility for LZW LE compression Signed-off-by: Richard Chapman --- common/dllserver/thorplugin.cpp | 5 +- system/jlib/jfcmp.hpp | 4 - system/jlib/jlz4.cpp | 15 +- system/jlib/jlzw.cpp | 238 +++++++++++++++++++------------- system/jlib/jlzw.hpp | 14 +- testing/unittests/jlibtests.cpp | 32 +++-- testing/unittests/unittests.cpp | 82 +++++++++++ 7 files changed, 263 insertions(+), 127 deletions(-) diff --git a/common/dllserver/thorplugin.cpp b/common/dllserver/thorplugin.cpp index 5db7bfce4e7..0fbf4079305 100644 --- a/common/dllserver/thorplugin.cpp +++ b/common/dllserver/thorplugin.cpp @@ -679,10 +679,7 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Str extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress) { mb.append((byte)0x80).append(resourceHeaderVersion); - if (compress) - compressToBuffer(mb, len, data); - else - appendToBuffer(mb, len, data); + compressToBuffer(mb, len, data, compress ? COMPRESS_METHOD_LZW : COMPRESS_METHOD_NONE); } extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data) diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp index f3d407452f3..49f4dbeb69c 100644 --- a/system/jlib/jfcmp.hpp +++ b/system/jlib/jfcmp.hpp @@ -72,8 +72,6 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf virtual void open(void *buf,size32_t max) override { - if (max<1024) - throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", max); wrmax = max; originalMax = max; if (buf) @@ -103,8 +101,6 @@ class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf { if (!initialSize) initialSize = FCMP_BUFFER_SIZE; // 1MB - if (initialSize<1024) - throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", initialSize); wrmax = initialSize; if (bufalloc) { diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp index 3b684a3bd60..59ec73bdf12 100644 --- a/system/jlib/jlz4.cpp +++ b/system/jlib/jlz4.cpp @@ -34,18 +34,17 @@ class CLZ4Compressor final : public CFcmpCompressor protected: virtual void setinmax() override { - inmax = blksz-outlen-sizeof(size32_t); - if (inmax<256) + if (blksz <= outlen+sizeof(size32_t)) trailing = true; // too small to bother compressing else { trailing = false; + inmax = blksz-outlen-sizeof(size32_t); size32_t slack = LZ4_COMPRESSBOUND(inmax) - inmax; - int inmax2 = inmax - (slack + sizeof(size32_t)); - if (inmax2<256) + if (inmax <= (slack + sizeof(size32_t))) trailing = true; else - inmax = inmax2; + inmax = inmax - (slack + sizeof(size32_t)); } } @@ -73,12 +72,6 @@ class CLZ4Compressor final : public CFcmpCompressor if (toflush == 0) return; - if (toflush < 256) - { - trailing = true; - return; - } - size32_t outSzRequired = outlen+sizeof(size32_t)*2+LZ4_COMPRESSBOUND(toflush); if (!dynamicOutSz) assertex(outSzRequired<=blksz); diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 2dbf48aca62..7c269594f02 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -303,7 +303,7 @@ void CLZWCompressor::open(void *buf,size32_t max) outbuf = malloc(bufalloc); } outBufMb = NULL; - ASSERT(max>SAFETY_MARGIN+sizeof(size32_t)); // minimum required + assertex(max>SAFETY_MARGIN+sizeof(size32_t)); // minimum required maxlen=max-SAFETY_MARGIN; initCommon(); } @@ -758,82 +758,76 @@ size32_t RLEExpand(void *dst,const void *src,size32_t expsize) return (size32_t)(in-(const byte *)src); } -void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src) +void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method, const char *options) { - out.append(false); - out.append(len); - out.append(len, src); -} - -void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src) -{ - unsigned originalLength = out.length(); - out.append(true); - out.append((size32_t)0); - - if (len >= 32) + if (method != COMPRESS_METHOD_NONE && len >= 32) { + ICompressHandler *handler = queryCompressHandler(method); + if (!handler) + { + VStringBuffer s("Unknown compression method %x requested in compressToBuffer", (byte) method); + throw makeStringException(0, s.str()); + } + unsigned originalLength = out.length(); + // For back-compatibility, we always store COMPRESS_METHOD_LZW_LITTLE_ENDIAN as 1 as earlier versions stored a boolean here + // rather than an enum + // This means that compressToBuffer/decompressToBuffer cannot bs used for rowdiff compression - this is not likely to be an issue + // Alternative would be a separate enum for compressToBuffer formats, but that seems more likely to cause confusion + out.append((byte) (method == COMPRESS_METHOD_LZW_LITTLE_ENDIAN ? COMPRESS_METHOD_LZWLEGACY : method)); + out.append((size32_t)0); size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ... - Owned compressor = createLZWCompressor(); + Owned compressor = handler->getCompressor(options); void *newData = out.reserve(newSize); - compressor->open(newData, newSize); - if (compressor->write(src, len)==len) + try { - compressor->close(); - size32_t compressedLen = compressor->buflen(); - out.setWritePos(originalLength + sizeof(bool)); - out.append(compressedLen); - out.setWritePos(originalLength + sizeof(bool) + sizeof(size32_t) + compressedLen); - return; + compressor->open(newData, newSize); + if (compressor->write(src, len)==len) + { + compressor->close(); + size32_t compressedLen = compressor->buflen(); + out.setWritePos(originalLength + sizeof(byte)); + out.append(compressedLen); + out.setWritePos(originalLength + sizeof(byte) + sizeof(size32_t) + compressedLen); + return; + } } + catch (IException *E) + { + E->Release(); + } + // failed to compress... + out.setWritePos(originalLength); } - - // all or don't compress - out.setWritePos(originalLength); - appendToBuffer(out, len, src); -} - -void decompressToBuffer(MemoryBuffer & out, const void * src) -{ - Owned expander = createLZWExpander(); - unsigned outSize = expander->init(src); - void * buff = out.reserve(outSize); - expander->expand(buff); + out.append((byte) COMPRESS_METHOD_NONE); + out.append(len); + out.append(len, src); } - -void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in) +void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options) { - bool compressed; size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) - decompressToBuffer(out, in.readDirect(srcLen)); - else + unsigned char _method; + in.read(_method).read(srcLen); + CompressionMethod method = (CompressionMethod) _method; + if (method==COMPRESS_METHOD_NONE) out.append(srcLen, in.readDirect(srcLen)); -} - -void decompressToAttr(MemoryAttr & out, const void * src) -{ - Owned expander = createLZWExpander(); - unsigned outSize = expander->init(src); - void * buff = out.allocate(outSize); - expander->expand(buff); -} - -void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in) -{ - bool compressed; - size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) - decompressToAttr(out, in.readDirect(srcLen)); else - out.set(srcLen, in.readDirect(srcLen)); + { + if (method==COMPRESS_METHOD_LZWLEGACY) + method = COMPRESS_METHOD_LZW_LITTLE_ENDIAN; // Back compatibilty + ICompressHandler *handler = queryCompressHandler(method); + if (!handler) + { + VStringBuffer s("Unknown decompression method %x required in decompressToBuffer", (byte) method); + throw makeStringException(0, s.str()); + } + Owned expander = handler->getExpander(options); + unsigned outSize = expander->init(in.readDirect(srcLen)); + void * buff = out.reserve(outSize); + expander->expand(buff); + } } - - /* Simple Diff compression format is @@ -1378,7 +1372,7 @@ class jlib_decl CRDiffCompressor : public ICompressor, public CInterface outbuf = malloc(bufalloc); } outBufMb = NULL; - ASSERT(max>2+sizeof(size32_t)*2); // minimum required (actually will need enough for recsize so only a guess) + assertex(max>2+sizeof(size32_t)*2); // minimum required (actually will need enough for recsize so only a guess) initCommon(); remaining = max-outlen; } @@ -1661,7 +1655,7 @@ class jlib_decl CRandRDiffCompressor : public ICompressor, public CInterface outbuf = malloc(bufalloc); } outBufMb = NULL; - ASSERT(max>MIN_RRDHEADER_SIZE+sizeof(unsigned short)+3); // hopefully a lot bigger! + assertex(max>MIN_RRDHEADER_SIZE+sizeof(unsigned short)+3); // hopefully a lot bigger! initCommon(); } @@ -2672,6 +2666,8 @@ class CAESCompressor : implements ICompressor, public CInterface else outbuf = outattr.allocate(blksize); outBufMb = NULL; + if (blksize <= AES_PADDING_SIZE+sizeof(size32_t)) + throw makeStringException(0, "CAESCompressor: target buffer too small"); size32_t subsz = blksize-AES_PADDING_SIZE-sizeof(size32_t); comp->open(compattr.reserveTruncate(subsz),subsz); } @@ -2862,44 +2858,88 @@ IPropertyTree *getBlockedFileDetails(IFile *file) return NULL; } -class CCompressHandlerArray : public IArrayOf +class CCompressHandlerArray { + IArrayOf registered; // Owns the relevant handler objects + ICompressHandler *byMethod[COMPRESS_METHOD_LAST] = { nullptr }; + ICompressHandler *AESbyMethod[COMPRESS_METHOD_LAST] = { nullptr }; + public: ICompressHandler *lookup(const char *type) const { - ForEachItemIn(h, *this) + ForEachItemIn(h, registered) { - ICompressHandler &handler = item(h); + ICompressHandler &handler = registered.item(h); if (0 == stricmp(type, handler.queryType())) return &handler; } return NULL; } + ICompressHandler *lookup(CompressionMethod method) const + { + if ((method & ~COMPRESS_METHOD_AES) >= COMPRESS_METHOD_LAST) + return nullptr; + else if (method & COMPRESS_METHOD_AES) + return AESbyMethod[method & ~COMPRESS_METHOD_AES]; + else + return byMethod[method]; + } + ICompressHandlerIterator *getIterator() + { + return new ArrayIIteratorOf, ICompressHandler, ICompressHandlerIterator>(registered); + } + bool addCompressor(ICompressHandler *handler) + { + CompressionMethod method = handler->queryMethod(); + if (lookup(method)) + { + handler->Release(); + return false; // already registered + } + registered.append(* handler); + if ((method & ~COMPRESS_METHOD_AES) < COMPRESS_METHOD_LAST) + { + if (method & COMPRESS_METHOD_AES) + AESbyMethod[method & ~COMPRESS_METHOD_AES] = handler; + else + byMethod[method] = handler; + } + return true; + } + bool removeCompressor(ICompressHandler *handler) + { + CompressionMethod method = handler->queryMethod(); + if (registered.zap(* handler)) + { + if ((method & ~COMPRESS_METHOD_AES) < COMPRESS_METHOD_LAST) + { + if (method & COMPRESS_METHOD_AES) + AESbyMethod[method & ~COMPRESS_METHOD_AES] = nullptr; + else + byMethod[method] = nullptr; + } + return true; + } + else + return false; + } } compressors; typedef IIteratorOf ICompressHandlerIterator; ICompressHandlerIterator *getCompressHandlerIterator() { - return new ArrayIIteratorOf, ICompressHandler, ICompressHandlerIterator>(compressors); + return compressors.getIterator(); } - - bool addCompressorHandler(ICompressHandler *handler) { - if (compressors.lookup(handler->queryType())) - { - handler->Release(); - return false; // already registered - } - compressors.append(* handler); - return true; + return compressors.addCompressor(handler); } bool removeCompressorHandler(ICompressHandler *handler) { - return compressors.zap(* handler); + return compressors.removeCompressor(handler); } Linked defaultCompressor; @@ -2908,38 +2948,38 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) { class CCompressHandlerBase : implements ICompressHandler, public CInterface { - StringAttr type; public: IMPLEMENT_IINTERFACE; - CCompressHandlerBase(const char *_type) : type(_type) { } - // ICompressHandler - virtual const char *queryType() const { return type; } }; class CFLZCompressHandler : public CCompressHandlerBase { public: - CFLZCompressHandler() : CCompressHandlerBase("FLZ") { } + virtual const char *queryType() const { return "FLZ"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_FASTLZ; } virtual ICompressor *getCompressor(const char *options) { return createFastLZCompressor(); } virtual IExpander *getExpander(const char *options) { return createFastLZExpander(); } }; class CLZ4CompressHandler : public CCompressHandlerBase { public: - CLZ4CompressHandler() : CCompressHandlerBase("LZ4") { } + virtual const char *queryType() const { return "LZ4"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4; } virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(options, false); } virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); } }; class CLZ4HCCompressHandler : public CCompressHandlerBase { public: - CLZ4HCCompressHandler() : CCompressHandlerBase("LZ4HC") { } + virtual const char *queryType() const { return "LZ4HC"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZ4HC; } virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(options, true); } virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); } }; class CAESCompressHandler : public CCompressHandlerBase { public: - CAESCompressHandler() : CCompressHandlerBase("AES") { } + virtual const char *queryType() const { return "AES"; } + virtual CompressionMethod queryMethod() const { return (CompressionMethod) (COMPRESS_METHOD_AES|COMPRESS_METHOD_LZW); } virtual ICompressor *getCompressor(const char *options) { assertex(options); @@ -2954,34 +2994,47 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) class CDiffCompressHandler : public CCompressHandlerBase { public: - CDiffCompressHandler() : CCompressHandlerBase("DIFF") { } + virtual const char *queryType() const { return "DIFF"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; } virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); } }; class CRDiffCompressHandler : public CCompressHandlerBase { public: - CRDiffCompressHandler() : CCompressHandlerBase("RDIFF") { } + virtual const char *queryType() const { return "RDIFF"; } // Synonym for DIFF + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_ROWDIF; } virtual ICompressor *getCompressor(const char *options) { return createRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { return createRDiffExpander(); } }; class CRandRDiffCompressHandler : public CCompressHandlerBase { public: - CRandRDiffCompressHandler() : CCompressHandlerBase("RANDROW") { } + virtual const char *queryType() const { return "RANDROW"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_RANDROW; } virtual ICompressor *getCompressor(const char *options) { return createRandRDiffCompressor(); } virtual IExpander *getExpander(const char *options) { UNIMPLEMENTED; } // Expander has a different interface }; class CLZWCompressHandler : public CCompressHandlerBase { public: - CLZWCompressHandler() : CCompressHandlerBase("LZW") { } + virtual const char *queryType() const { return "LZW"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZW; } virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(true); } virtual IExpander *getExpander(const char *options) { return createLZWExpander(true); } }; + class CLZWLECompressHandler : public CCompressHandlerBase + { + public: + virtual const char *queryType() const { return "LZWLE"; } + virtual CompressionMethod queryMethod() const { return COMPRESS_METHOD_LZW_LITTLE_ENDIAN; } + virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(false); } + virtual IExpander *getExpander(const char *options) { return createLZWExpander(false); } + }; + addCompressorHandler(new CLZWLECompressHandler()); + addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CAESCompressHandler()); addCompressorHandler(new CDiffCompressHandler()); - addCompressorHandler(new CLZWCompressHandler()); addCompressorHandler(new CRDiffCompressHandler()); addCompressorHandler(new CRandRDiffCompressHandler()); addCompressorHandler(new CFLZCompressHandler()); @@ -2999,8 +3052,7 @@ ICompressHandler *queryCompressHandler(const char *type) ICompressHandler *queryCompressHandler(CompressionMethod method) { - //Could be more efficient, but doesn't matter - return compressors.lookup(translateFromCompMethod(method)); + return compressors.lookup(method); } void setDefaultCompressor(const char *type) diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp index 1d7c7196390..daa3f1614b0 100644 --- a/system/jlib/jlzw.hpp +++ b/system/jlib/jlzw.hpp @@ -35,9 +35,12 @@ enum CompressionMethod COMPRESS_METHOD_LZ4, COMPRESS_METHOD_LZ4HC, COMPRESS_METHOD_RANDROW, + COMPRESS_METHOD_LZW_LITTLE_ENDIAN, + COMPRESS_METHOD_LAST, COMPRESS_METHOD_AES = 0x80, + COMPRESS_METHOD_LZWLEGACY = 1, // Matches value of boolean 'true' used to indicate LZW little endian compression by legacy compressToBuffer }; @@ -117,13 +120,9 @@ extern jlib_decl ICompressor *createRandRDiffCompressor(); // similar to RDiffCo extern jlib_decl IRandRowExpander *createRandRDiffExpander(); // NB only supports fixed row size -//Some helper functions to make it easy to compress/decompress to memorybuffers. -extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src); -extern jlib_decl void decompressToBuffer(MemoryBuffer & out, const void * src); -extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in); -extern jlib_decl void decompressToAttr(MemoryAttr & out, const void * src); -extern jlib_decl void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in); -extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src); //format as failed compression +// Helper functions to make it easy to compress/decompress to memorybuffers. +extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, CompressionMethod method=COMPRESS_METHOD_LZW_LITTLE_ENDIAN, const char *options=nullptr); +extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in, const char *options=nullptr); interface ICompressedFileIO: extends IFileIO @@ -158,6 +157,7 @@ extern jlib_decl IPropertyTree *getBlockedFileDetails(IFile *file); interface ICompressHandler : extends IInterface { virtual const char *queryType() const = 0; + virtual CompressionMethod queryMethod() const = 0; virtual ICompressor *getCompressor(const char *options=NULL) = 0; virtual IExpander *getExpander(const char *options=NULL) = 0; }; diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 9918b77c5ef..e7178994109 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3155,7 +3155,7 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_END(); static constexpr size32_t sz = 100*0x100000; // 100MB - enum CompressOpt { RowCompress, AllRowCompress, BlockCompress }; + enum CompressOpt { RowCompress, AllRowCompress, BlockCompress, CompressToBuffer }; public: void test() { @@ -3218,6 +3218,7 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture testCompressor(handler, "hclevel=10", rowSz, src.length(), src.bytes(), RowCompress); } testCompressor(handler, options, rowSz, src.length(), src.bytes(), RowCompress); + testCompressor(handler, options, rowSz, src.length(), src.bytes(), CompressToBuffer); if (streq(type, "LZ4")) { testCompressor(handler, "allrow", rowSz, src.length(), src.bytes(), AllRowCompress); // block doesn't affect the compressor, just tracing @@ -3276,23 +3277,38 @@ class JlibCompressionTestsStress : public CppUnit::TestFixture compressed.setLength(written); break; } + case CompressToBuffer: + { + compressToBuffer(compressed, srcLen, ptr, handler.queryMethod(), options); + break; + } } cycle_t compressCycles = timer.elapsedCycles(); Owned expander = handler.getExpander(options); - + MemoryBuffer tgt; timer.reset(); - size32_t required = expander->init(compressed.bytes()); - MemoryBuffer tgt(required); - expander->expand(tgt.bufferBase()); - tgt.setWritePos(required); + if (opt==CompressToBuffer) + { + decompressToBuffer(tgt, compressed, options); + } + else + { + size32_t required = expander->init(compressed.bytes()); + tgt.reserveTruncate(required); + expander->expand(tgt.bufferBase()); + tgt.setWritePos(required); + } cycle_t decompressCycles = timer.elapsedCycles(); float ratio = (float)(srcLen) / compressed.length(); StringBuffer name(handler.queryType()); - if (options) - name.append("(").append(options).append(")"); + if (opt == CompressToBuffer) + name.append("-c2b"); + if (options && *options) + name.append("-").append(options); + if (name.length() > 19) name.setLength(19); diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp index a4c7e7e8ca6..6a857133192 100644 --- a/testing/unittests/unittests.cpp +++ b/testing/unittests/unittests.cpp @@ -1052,5 +1052,87 @@ class RelaxedAtomicTimingTest : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_REGISTRATION( RelaxedAtomicTimingTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( RelaxedAtomicTimingTest, "RelaxedAtomicTimingTest" ); +#include "jlzw.hpp" +class compressToBufferTest : public CppUnit::TestFixture +{ + CPPUNIT_TEST_SUITE( compressToBufferTest ); + CPPUNIT_TEST(testCompressors); + CPPUNIT_TEST_SUITE_END(); + + bool testOne(unsigned len, CompressionMethod method, bool prevResult, const char *options=nullptr) + { + constexpr const char *in = + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello" + "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello"; + assertex(len <= strlen(in)); + MemoryBuffer compressed; + compressToBuffer(compressed, len, in, method, options); + bool ret; + if (compressed.length() == len+5) + { + if (prevResult) + DBGLOG("compressToBuffer %x size %u did not compress", (byte) method, len); + ret = false; + } + else + { + if (!prevResult) + DBGLOG("compressToBuffer %x size %u compressed to %u", (byte) method, len, compressed.length()); + ret = true; + } + CPPUNIT_ASSERT(compressed.length() <= len+5); + MemoryBuffer out; + decompressToBuffer(out, compressed, options); + CPPUNIT_ASSERT(out.length() == len); + if (len) + CPPUNIT_ASSERT(memcmp(out.bytes(), in, len) == 0); + return ret; + } + + void testCompressor(CompressionMethod method, const char *options=nullptr) + { + bool result = true; + for (unsigned i = 0; i < 256; i++) + result = testOne(i, method, result, options); + testOne(1000, method, false, options); + + } + void testCompressors() + { + testCompressor(COMPRESS_METHOD_NONE); + testCompressor(COMPRESS_METHOD_LZW); + testCompressor(COMPRESS_METHOD_LZ4); + testCompressor((CompressionMethod) (COMPRESS_METHOD_LZW|COMPRESS_METHOD_AES), "0123456789abcdef"); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( compressToBufferTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( compressToBufferTest, "CompressToBufferTest" ); + #endif // _USE_CPPUNIT