Merge remote-tracking branch 'origin/candidate-9.6.x' into candidate-…

…9.6.0 Signed-off-by: Gavin Halliday <[email protected]>
hpcc-systems · Apr 2, 2024 · 803fa28 · 803fa28
2 parents 57c188d + 3e2079f
commit 803fa28
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 13 deletions.
diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml
@@ -439,12 +439,12 @@ jobs:
           - os: "windows-2022"
             mono: ""
             sudo: ""
-            cmake_config_options: '-T host=x64 -A x64 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
+            cmake_config_options: '-T host=x64 -A x64 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_PARQUET=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
             cmake_build_options: "--config Release --parallel $NUMBER_OF_PROCESSORS"
           - os: "macos-12"
             mono: "mono"
             sudo: "sudo"
-            cmake_config_options: "-DCMAKE_BUILD_TYPE=Release -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF"
+            cmake_config_options: "-DCMAKE_BUILD_TYPE=Release -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_PARQUET=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF"
             cmake_build_options: "--parallel $(nproc)"
       fail-fast: false
 

diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml
@@ -173,15 +173,15 @@ jobs:
     uses: ./.github/workflows/build-gh_runner.yml
     with:
       os: windows-2022
-      cmake-configuration-ex: '-T host=x64 -A x64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DINCLUDE_PLUGINS=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
+      cmake-configuration-ex: '-T host=x64 -A x64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DINCLUDE_PLUGINS=OFF -DUSE_CASSANDRA=OFF -DUSE_PARQUET=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
     secrets: inherit
 
   build-gh_runner-windows-2019:
     if: ${{ contains('schedule,push', github.event_name) }}
     uses: ./.github/workflows/build-gh_runner.yml
     with:
       os: windows-2019
-      cmake-configuration-ex: '-T host=x64 -A x64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
+      cmake-configuration-ex: '-T host=x64 -A x64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_PARQUET=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
     secrets: inherit
 
   build-gh_runner-macos-13:
@@ -190,7 +190,7 @@ jobs:
     with:
       os: macos-13
       build-type: 'Release'
-      cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
+      cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_PARQUET=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
     secrets: inherit
 
   build-gh_runner-macos-12:
@@ -199,6 +199,6 @@ jobs:
     with:
       os: macos-12
       build-type: 'Release'
-      cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
+      cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_PARQUET=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF'
     secrets: inherit
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -167,6 +167,7 @@ if ( PLUGIN )
     HPCC_ADD_SUBDIRECTORY (system/security/securesocket)
     HPCC_ADD_SUBDIRECTORY (system/security/zcrypt)
     HPCC_ADD_SUBDIRECTORY (dali/base)
+    HPCC_ADD_SUBDIRECTORY (plugins/cassandra "CASSANDRAEMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/eclblas "ECLBLAS")
     HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED")

diff --git a/plugins/parquet/CMakeLists.txt b/plugins/parquet/CMakeLists.txt
@@ -59,7 +59,8 @@ if(PARQUETEMBED OR USE_PARQUET)
 
         install(
             TARGETS parquetembed
-            DESTINATION ${LIB_DIR}
+            RUNTIME DESTINATION ${EXEC_DIR}
+            LIBRARY DESTINATION ${LIB_DIR}
             COMPONENT Runtime CALC_DEPS
         )
 

diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp
@@ -668,7 +668,7 @@ class CNodeMapping : public HTMapping<CNodeCacheEntry, CKeyIdAndPos>
 };
 
 typedef OwningSimpleHashTableOf<CNodeMapping, CKeyIdAndPos> CNodeTable;
-class CNodeMRUCache : public CMRUCacheOf<CKeyIdAndPos, CNodeCacheEntry, CNodeMapping, CNodeTable>
+class CNodeMRUCache final : public CMRUCacheOf<CKeyIdAndPos, CNodeCacheEntry, CNodeMapping, CNodeTable>
 {
     std::atomic<size32_t> sizeInMem{0};
     size32_t memLimit = 0;
@@ -684,15 +684,31 @@ class CNodeMRUCache : public CMRUCacheOf<CKeyIdAndPos, CNodeCacheEntry, CNodeMap
     virtual void makeSpace()
     {
         // remove LRU until !full
+        // This code could walk the list, rather than restarting at the end each time - but there are unlikely to be
+        // many entries that have no associated node, and nodes could have been associated in the meantime.
         do
         {
-            //Never evict an entry that hasn't yet loaded - otherwise the sizeInMem can become inconsistent
             CNodeMapping *tail = mruList.tail();
-            assertex(tail);
-            if (!tail->queryElement().isReady() )
-                break;
+            if (unlikely(!tail))
+                throw makeStringExceptionV(9999, "Index cache appears full but contains no entries size=%x limit=%x", sizeInMem.load(), memLimit);
+
+            //Never evict an entry that hasn't yet loaded - otherwise the sizeInMem can become inconsistent
+            //When running with slow remote storage this can take a long time to be ready - so we need
+            //to walk on to the next entry in the lrulist, otherwise we can run out of memory since nothing
+            //would be removed.
+            while (!tail->queryElement().isReady())
+            {
+                tail = tail->prev;
+                if (!tail)
+                {
+                     // no pages in the cache are ready - this could possibly happen in a tiny race-window where
+                     // sizes in the cache have been updated, but no nodes have yet been associated with the entries.
+                    return;
+                }
+            }
 
-            clear(1);
+            mruList.remove(tail);
+            table.removeExact(tail);
         }
         while (full());
     }