Merge pull request #18148 from ghalliday/issue30928

HPCC-30928 Improve compressToBuffer using LZ4 Reviewed-By: Richard Chapman <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Jan 3, 2024 · 5d5418b · 5d5418b
2 parents 1be74a3 + 5c43cb2
commit 5d5418b
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 8 deletions.
diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp
@@ -778,22 +778,36 @@ void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src, Compre
         size32_t newSize = len * 4 / 5; // Copy if compresses less than 80% ...
         Owned<ICompressor> compressor = handler->getCompressor(options);
         void *newData = out.reserve(newSize);
-        try
+        if (compressor->supportsBlockCompression())
         {
-            compressor->open(newData, newSize);
-            if (compressor->write(src, len)==len)
+            size32_t compressedLen = compressor->compressBlock(newSize, newData, len, src);
+            if (compressedLen != 0)
             {
-                compressor->close();
-                size32_t compressedLen = compressor->buflen();
                 out.setWritePos(originalLength + sizeof(byte));
                 out.append(compressedLen);
                 out.setWritePos(originalLength + sizeof(byte) + sizeof(size32_t) + compressedLen);
                 return;
             }
         }
-        catch (IException *E)
+        else
         {
-            E->Release();
+            try
+            {
+                compressor->open(newData, newSize);
+                if (compressor->write(src, len)==len)
+                {
+                    compressor->close();
+                    size32_t compressedLen = compressor->buflen();
+                    out.setWritePos(originalLength + sizeof(byte));
+                    out.append(compressedLen);
+                    out.setWritePos(originalLength + sizeof(byte) + sizeof(size32_t) + compressedLen);
+                    return;
+                }
+            }
+            catch (IException *E)
+            {
+                E->Release();
+            }
         }
         // failed to compress...
         out.setWritePos(originalLength);

diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp
@@ -1091,6 +1091,7 @@ class compressToBufferTest : public CppUnit::TestFixture
           "HelloHelloHelloHelloHelloHelloHelloHelloHelloHello";
         assertex(len <= strlen(in));
         MemoryBuffer compressed;
+        CCycleTimer start;
         compressToBuffer(compressed, len, in, method, options);
         bool ret;
         if (compressed.length() == len+5)
@@ -1102,7 +1103,7 @@ class compressToBufferTest : public CppUnit::TestFixture
         else 
         {
             if (!prevResult)
-                DBGLOG("compressToBuffer %x size %u compressed to %u", (byte) method, len, compressed.length());
+                DBGLOG("compressToBuffer %x size %u compressed to %u in %lluns", (byte) method, len, compressed.length(), start.elapsedNs());
             ret = true;
         }
         CPPUNIT_ASSERT(compressed.length() <= len+5);