diff --git a/plugins/parquet/examples/nested_io.ecl b/plugins/parquet/examples/nested_io.ecl index fe0a0785cc6..a7a03a4e1bc 100644 --- a/plugins/parquet/examples/nested_io.ecl +++ b/plugins/parquet/examples/nested_io.ecl @@ -16,15 +16,16 @@ parentRec := RECORD UTF8_de firstname; UTF8_de lastname; childRec details; -END; -nested_dataset := DATASET([{U'J\353ck', U'\353ackson', { {22, 2, ['James', 'Jonathon']}, 5.9, 600}}, {'John', 'Johnson', { {17, 0, []}, 6.3, 18}}, +END; +nested_dataset := DATASET([{U'J\353ck', U'\353ackson', { {22, 2, ['James', 'Jonathon']}, 5.9, 600}}, {'John', 'Johnson', { {17, 0, []}, 6.3, 18}}, {'Amy', U'Amy\353on', { {59, 1, ['Andy']}, 3.9, 59}}, {'Grace', U'Graceso\353', { {11, 3, ['Grayson', 'Gina', 'George']}, 7.9, 100}}], parentRec); -#IF(1) -ParquetIO.Write(nested_dataset, '/datadrive/dev/test_data/nested.parquet'); -#END +writeStepA := ParquetIO.Write(nested_dataset, '/datadrive/dev/test_data/nested_A.parquet'); +writeStepB := ParquetIO.Write(nested_dataset, '/datadrive/dev/test_data/nested_B.parquet'); + +read_in_a := ParquetIO.Read(parentRec, '/datadrive/dev/test_data/nested_A.parquet'); +readStepA := OUTPUT(read_in_a, NAMED('NESTED_PARQUET_IO_A')); +read_in_b := ParquetIO.Read(parentRec, '/datadrive/dev/test_data/nested_B.parquet'); +readStepB := OUTPUT(read_in_b, NAMED('NESTED_PARQUET_IO_B')); -#IF(1) -read_in := ParquetIO.Read(parentRec, '/datadrive/dev/test_data/nested.parquet'); -OUTPUT(read_in, NAMED('NESTED_PARQUET_IO')); -#END \ No newline at end of file +SEQUENTIAL(writeStepA, writeStepB, readStepA, readStepB); \ No newline at end of file diff --git a/plugins/parquet/parquet.ecllib b/plugins/parquet/parquet.ecllib index 07529024153..ff15be29f66 100644 --- a/plugins/parquet/parquet.ecllib +++ b/plugins/parquet/parquet.ecllib @@ -33,16 +33,16 @@ EXPORT ParquetIO := MODULE RETURN _DoParquetReadPartition(); ENDMACRO; - EXPORT Write(outDS, filePath) := MACRO + EXPORT Write(outDS, filePath) := FUNCTIONMACRO LOCAL _DoParquetWrite(STREAMED DATASET(RECORDOF(outDS)) _ds) := EMBED(parquet : activity, option('write'), destination(filePath)) ENDEMBED; - _doParquetWrite(outDS); + RETURN _doParquetWrite(outDS); ENDMACRO; - EXPORT WritePartition(outDS, outRows = 100000, basePath) := MACRO + EXPORT WritePartition(outDS, outRows = 100000, basePath) := FUNCTIONMACRO LOCAL _DoParquetWritePartition(STREAMED DATASET(RECORDOF(outDS)) _ds) := EMBED(parquet : activity, option('writepartition'), destination(basePath), MaxRowSize(outRows)) ENDEMBED; - _DoParquetWritePartition(outDS) + RETURN _DoParquetWritePartition(outDS); ENDMACRO; END;