Merge pull request #17879 from jackdelv/HPCC-30454

HPCC-30454 ParquetIO.Write() can only be used as a standalone action Reviewed-By: Dan S. Camper <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Oct 13, 2023 · 6fc46ef · 6fc46ef
2 parents 9872e80 + c9204a9
commit 6fc46ef
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 13 deletions.
diff --git a/plugins/parquet/examples/nested_io.ecl b/plugins/parquet/examples/nested_io.ecl
@@ -16,15 +16,16 @@ parentRec := RECORD
     UTF8_de firstname;
 	UTF8_de lastname;
     childRec details;
-END; 
-nested_dataset := DATASET([{U'J\353ck', U'\353ackson', { {22, 2, ['James', 'Jonathon']}, 5.9, 600}}, {'John', 'Johnson', { {17, 0, []}, 6.3, 18}}, 
+END;
+nested_dataset := DATASET([{U'J\353ck', U'\353ackson', { {22, 2, ['James', 'Jonathon']}, 5.9, 600}}, {'John', 'Johnson', { {17, 0, []}, 6.3, 18}},
                                 {'Amy', U'Amy\353on', { {59, 1, ['Andy']}, 3.9, 59}}, {'Grace', U'Graceso\353', { {11, 3, ['Grayson', 'Gina', 'George']}, 7.9, 100}}], parentRec);
 
-#IF(1)
-ParquetIO.Write(nested_dataset, '/datadrive/dev/test_data/nested.parquet');
-#END
+writeStepA := ParquetIO.Write(nested_dataset, '/datadrive/dev/test_data/nested_A.parquet');
+writeStepB := ParquetIO.Write(nested_dataset, '/datadrive/dev/test_data/nested_B.parquet');
+
+read_in_a := ParquetIO.Read(parentRec, '/datadrive/dev/test_data/nested_A.parquet');
+readStepA := OUTPUT(read_in_a, NAMED('NESTED_PARQUET_IO_A'));
+read_in_b := ParquetIO.Read(parentRec, '/datadrive/dev/test_data/nested_B.parquet');
+readStepB := OUTPUT(read_in_b, NAMED('NESTED_PARQUET_IO_B'));
 
-#IF(1)
-read_in := ParquetIO.Read(parentRec, '/datadrive/dev/test_data/nested.parquet');
-OUTPUT(read_in, NAMED('NESTED_PARQUET_IO'));
-#END
+SEQUENTIAL(writeStepA, writeStepB, readStepA, readStepB);
diff --git a/plugins/parquet/parquet.ecllib b/plugins/parquet/parquet.ecllib
@@ -33,16 +33,16 @@ EXPORT ParquetIO := MODULE
         RETURN _DoParquetReadPartition();
     ENDMACRO;
 
-    EXPORT Write(outDS, filePath) := MACRO
+    EXPORT Write(outDS, filePath) := FUNCTIONMACRO
         LOCAL _DoParquetWrite(STREAMED DATASET(RECORDOF(outDS)) _ds) := EMBED(parquet : activity, option('write'), destination(filePath))
         ENDEMBED;
-        _doParquetWrite(outDS);
+        RETURN _doParquetWrite(outDS);
     ENDMACRO;
 
-    EXPORT WritePartition(outDS, outRows = 100000, basePath) := MACRO
+    EXPORT WritePartition(outDS, outRows = 100000, basePath) := FUNCTIONMACRO
         LOCAL _DoParquetWritePartition(STREAMED DATASET(RECORDOF(outDS)) _ds) := EMBED(parquet : activity, option('writepartition'), destination(basePath), MaxRowSize(outRows))
         ENDEMBED;
-        _DoParquetWritePartition(outDS)
+        RETURN _DoParquetWritePartition(outDS);
     ENDMACRO;
 END;