HPCC-31860 Test suite for the Parquet plugin

testing/regress/ecl/key/parqueCorrupt.xml

@@ -0,0 +1 @@
+<Exception><Source>eclagent</Source><Message>Error: 0: parquet: Invalid: Parquet magic bytes not found in footer. Either the file is corrupted or this is not a parquet file..</Message></Exception>

testing/regress/ecl/key/parquetEmpty.xml

@@ -0,0 +1,2 @@
+<Dataset name='Result 1'>
+</Dataset>

testing/regress/ecl/key/parquetOverwrite.xml

@@ -0,0 +1,2 @@
+<Exception><Source>eclagent</Source><Message>Error: 0: parquet: The target file /var/lib/HPCCSystems/mydropzone/SingleRowTest.parquet already exists. To delete the file set the overwrite option to true.</Message></Exception>
+

testing/regress/ecl/key/parquetPartition.xml

@@ -0,0 +1,10 @@
+<Dataset name='HivePartitionedSampleData'>
+ <Row><id>1</id><name>Alice                    </name><age>30</age><city>New York                                          </city></Row>
+ <Row><id>2</id><name>Bob                      </name><age>25</age><city>Los Angeles                                       </city></Row>
+ <Row><id>3</id><name>Charlie                  </name><age>40</age><city>Chicago                                           </city></Row>
+</Dataset>
+<Dataset name='DirPartitionedSampleData'>
+ <Row><id>1</id><name>Alice                    </name><age>30</age><city>New York                                          </city></Row>
+ <Row><id>2</id><name>Bob                      </name><age>25</age><city>Los Angeles                                       </city></Row>
+ <Row><id>3</id><name>Charlie                  </name><age>40</age><city>Chicago                                           </city></Row>
+</Dataset>

testing/regress/ecl/key/parquetSchema.xml

@@ -0,0 +1,4 @@
+<Dataset name='ReadData'>
+ <Row><name>Alice</name><salary>50000.5</salary><id>1</id></Row>
+ <Row><name>Bob</name><salary>60000.75</salary><id>2</id></Row>
+</Dataset>

testing/regress/ecl/key/parquetSize.xml

@@ -0,0 +1,13 @@
+<Dataset name='singleDataset'>
+ <Row><id>1</id><name>Alice</name><price>10.5</price><isactive>yes</isactive></Row>
+ <Row><id>2</id><name>Bob</name><price>20.75</price><isactive>no</isactive></Row>
+ <Row><id>3</id><name>Charlie</name><price>15.25</price><isactive>yes</isactive></Row>
+ <Row><id>1</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
+</Dataset>
+<Dataset name='multiDataset'>
+ <Row><id>1</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
+ <Row><id>3</id><name>Charlie</name><price>15.25</price><isactive>yes</isactive></Row>
+ <Row><id>2</id><name>Bob</name><price>20.75</price><isactive>no</isactive></Row>
+ <Row><id>1</id><name>A</name><price>10.0</price><isactive>1</isactive></Row>
+ <Row><id>1</id><name>Alice</name><price>10.5</price><isactive>yes</isactive></Row>
+</Dataset>

testing/regress/ecl/key/parquetString.xml

@@ -0,0 +1,8 @@
+<Dataset name='ComparisonResult'>
+ <Row><s1>Hello     </s1><s2>World               </s2><s3>Test Data 1                   </s3></Row>
+ <Row><s1>HPCC      </s1><s2>Systems             </s2><s3>Test Data 2                   </s3></Row>
+ <Row><s1>Parquet   </s1><s2>I/O                 </s2><s3>Test Data 3                   </s3></Row>
+</Dataset>
+<Dataset name='TestResult'>
+ <Row><TestResult>All records match</TestResult></Row>
+</Dataset>

testing/regress/ecl/key/parquetTypes.xml

@@ -0,0 +1,33 @@
+<Dataset name='BooleanTest'>
+ <Row><BooleanTest>Pass</BooleanTest></Row>
+</Dataset>
+<Dataset name='IntegerTest'>
+ <Row><IntegerTest>Pass</IntegerTest></Row>
+</Dataset>
+<Dataset name='UnsignedTest'>
+ <Row><UnsignedTest>Pass</UnsignedTest></Row>
+</Dataset>
+<Dataset name='RealTest'>
+ <Row><RealTest>Pass</RealTest></Row>
+</Dataset>
+<Dataset name='DecimalTest'>
+ <Row><DecimalTest>Pass</DecimalTest></Row>
+</Dataset>
+<Dataset name='StringTest'>
+ <Row><StringTest>Pass</StringTest></Row>
+</Dataset>
+<Dataset name='DataAsStringTest'>
+ <Row><DataAsStringTest>Pass</DataAsStringTest></Row>
+</Dataset>
+<Dataset name='VarStringTest'>
+ <Row><VarStringTest>Pass</VarStringTest></Row>
+</Dataset>
+<Dataset name='QStringTest'>
+ <Row><QStringTest>Pass</QStringTest></Row>
+</Dataset>
+<Dataset name='UTF8Test'>
+ <Row><UTF8Test>Pass</UTF8Test></Row>
+</Dataset>
+<Dataset name='UnicodeTest'>
+ <Row><UnicodeTest>Pass</UnicodeTest></Row>
+</Dataset>

testing/regress/ecl/parquetCorrupt.ecl

@@ -0,0 +1,39 @@
+/*##############################################################################
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=parquet
+//fail
+//nothor
+//noroxe
+
+//This ECL code reads a potentially corrupt Parquet file, handling the case where it might be corrupt
+//by outputting either the file contents or a single informative record if the file can't be read.
+
+IMPORT Std;
+IMPORT Parquet;
+
+RECORDDEF := RECORD
+    UNSIGNED4 index;
+    STRING name;
+    STRING director;
+END;
+
+filePath2 := Std.File.GetDefaultDropZone() + '/corrupt.parquet';
+
+CORRUPT_PARQUET := ParquetIO.Read(RECORDDEF, filePath2);
+
+CORRUPT_RESULT := IF(COUNT(CORRUPT_PARQUET) = 0,
+                     DATASET([{0, 'Corrupt Parquet File', ''}], RECORDDEF),
+                     CORRUPT_PARQUET);
+
+OUTPUT(CORRUPT_RESULT);

testing/regress/ecl/parquetEmpty.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=parquet
+//nothor
+//noroxie
+
+IMPORT Std;
+IMPORT Parquet;
+
+RECORDDEF := RECORD
+    UNSIGNED4 index;
+    STRING director;
+END;
+
+basePath := Std.File.GetDefaultDropZone() + '/regress/parquet/';
+filePath := basePath + 'empty_test.parquet';
+
+EMPTY_PARQUET := DATASET([], RECORDDEF);
+
+ParquetIO.Write(EMPTY_PARQUET, filePath, TRUE);
+
+read_data := ParquetIO.Read(RECORDDEF, filePath);
+
+OUTPUT(read_data);
+

testing/regress/ecl/parquetOverwrite.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=parquet
+//fail
+//nothor
+//noroxie
+
+IMPORT Std;
+IMPORT Parquet;
+
+SingleRowDataset := DATASET([{1, 'SingleRow', TRUE}], {UNSIGNED id, STRING name, BOOLEAN flag});
+
+basePath := Std.File.GetDefaultDropZone() + '/regress/parquet/';
+FilePath := basePath + 'SingleRowTest.parquet';
+
+writeParquetFile := ParquetIO.write(SingleRowDataset, FilePath);
+
+SEQUENTIAL(writeParquetFile, writeParquetFile);

testing/regress/ecl/parquetPartition.ecl

@@ -0,0 +1,74 @@
+/*##############################################################################
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+//class=parquet
+//nothor
+//noroxie
+
+IMPORT Std;
+IMPORT Parquet;
+
+// Define the record layout with explicit field lengths
+datasetRecordLayout := RECORD
+    UNSIGNED4 id;
+    STRING25 name;
+    UNSIGNED4 age;
+    STRING50 city;
+END;
+
+// Create a small dataset - ensure all records have valid data
+smallData := DATASET([
+    {1, 'Alice', 30, 'New York'},
+    {2, 'Bob', 25, 'Los Angeles'},
+    {3, 'Charlie', 40, 'Chicago'}
+], datasetRecordLayout);
+
+// Set options
+overwriteOption := TRUE;
+rowSize := 1024;  // Increased buffer size
+
+// Define base path (without 'file://' prefix)
+basePath := Std.File.GetDefaultDropZone() + '/regress/parquet/';
+
+// Define partition keys as a semicolon-separated string with all keys
+partitionKeys := 'id';
+
+// Write out the dataset with Hive partitioning on all keys
+ParquetIO.HivePartition.Write(
+    smallData,
+    rowSize,
+    basePath + 'hive_partitioned/',
+    overwriteOption,
+    partitionKeys
+);
+
+// Write out the dataset with Directory partitioning on all keys
+ParquetIO.DirectoryPartition.Write(
+    smallData,
+    rowSize,
+    basePath + 'dir_partitioned/',
+    overwriteOption,
+    partitionKeys
+);
+
+// Define file paths for partitioned datasets
+hiveFilePath := basePath + 'hive_partitioned/';
+dirFilePath := basePath + 'dir_partitioned/';
+
+// Read back the partitioned data
+readBackHiveData := ParquetIO.HivePartition.Read(datasetRecordLayout, hiveFilePath);
+readBackDirData := ParquetIO.DirectoryPartition.Read(datasetRecordLayout, dirFilePath, partitionKeys);
+
+// Output the entire dataset for verification
+OUTPUT(readBackHiveData, NAMED('HivePartitionedSampleData'));
+OUTPUT(readBackDirData, NAMED('DirPartitionedSampleData'));
+

testing/regress/ecl/parquetSchema.ecl

@@ -0,0 +1,46 @@
+/*##############################################################################
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=parquet
+//nothor
+//noroxie
+//fail
+
+IMPORT Std;
+IMPORT Parquet;
+
+Layout1 := RECORD
+    INTEGER id;
+    STRING name;
+    REAL salary;
+END;
+
+Layout2 := RECORD
+    STRING name;
+    REAL salary;
+    INTEGER id;
+END;
+
+testData := DATASET([
+    { 1, 'Alice', 50000.50 },
+    { 2, 'Bob', 60000.75 }
+], Layout1);
+
+basePath := Std.File.GetDefaultDropZone() + '/regress/parquet/';
+FilePath := basePath + 'reorder_test.parquet';
+
+ParquetIO.Write(testData, filePath, TRUE);
+
+readData := ParquetIO.Read(Layout2, filePath);
+
+OUTPUT(readData, NAMED('ReadData'));

testing/regress/ecl/parquetSize.ecl

@@ -0,0 +1,45 @@
+/*##############################################################################
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+##############################################################################*/
+
+//class=parquet
+//nothor
+//noroxie
+//nohthor
+
+IMPORT Std;
+IMPORT Parquet;
+
+recordLayout := RECORD
+    UNSIGNED4 id;
+    STRING name;
+    REAL8 price;
+    STRING isactive;
+END;
+
+basePath := Std.File.GetDefaultDropZone() + '/regress/parquet/';
+
+singleFilePath := basePath + 'single.parquet';
+multiFilePath := basePath + 'multi.parquet';
+
+// Reading the single and multi-part files
+singleDataset := ParquetIO.Read(recordLayout, singleFilePath);
+multiDataset := ParquetIO.Read(recordLayout, multiFilePath);
+
+// Output the datasets
+SEQUENTIAL(
+    OUTPUT(singleDataset, NAMED('singleDataset')),  // Output for the single file
+    OUTPUT(multiDataset, NAMED('multiDataset'))     // Output for the combined multi-part files
+);