Merge pull request #19032 from jackdelv/HPCC-32507

HPCC-32507 Parquet Plugin not reading Arrow Partitioned files Reviewed-By: Dan S. Camper <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Aug 23, 2024 · 193bec1 · 193bec1
2 parents 5d8b6c5 + 8be1a41
commit 193bec1
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/plugins/parquet/parquet.ecllib b/plugins/parquet/parquet.ecllib
@@ -43,7 +43,7 @@ EXPORT ParquetIO := MODULE
         ENDMACRO;
 
         EXPORT Read(resultLayout, basePath, partitionFieldList) := FUNCTIONMACRO
-            LOCAL STREAMED DATASET(resultLayout) _DoParquetReadPartition() := EMBED(parquet: activity, option('readdirectorypartition'), location(basePath)), partitionFields(partitionFieldList)
+            LOCAL STREAMED DATASET(resultLayout) _DoParquetReadPartition() := EMBED(parquet: activity, option('readdirectorypartition'), location(basePath), partitionFields(partitionFieldList))
             ENDEMBED;
             RETURN _DoParquetReadPartition();
         ENDMACRO;

diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp
@@ -462,6 +462,8 @@ __int64 ParquetReader::next(TableColumns *&nextTable)
         if (endsWithIgnoreCase(partOption.c_str(), "partition"))
         {
             PARQUET_ASSIGN_OR_THROW(table, queryRows()); // Sets rowsProcessed to current row in table corresponding to startRow
+            rowsCount = table->num_rows();
+            splitTable(table);
         }
         else
         {