HPCC4J-555 Provide Test jar (#661)

- Moved BaseRemoteTests into testing dir - Added TestResultNotifier to record test failures - Updated POMs to use and generate test-jar Signed-off-by: James McMullan [email protected] Signed-off-by: James McMullan [email protected]
hpcc-systems · Dec 5, 2023 · d78e7fe · d78e7fe
1 parent d8e42d1
commit d78e7fe
Show file tree

Hide file tree

Showing 30 changed files with 143 additions and 100 deletions.
diff --git a/.github/workflows/javadocTest.yml b/.github/workflows/javadocTest.yml
@@ -28,4 +28,4 @@ jobs:
         restore-keys: ${{ runner.os }}-m2
 
     - name: Build with Maven
-      run: mvn -B -Dmaven.test.skip=True clean install javadoc:javadoc
+      run: mvn -B clean install javadoc:javadoc
diff --git a/dfsclient/pom.xml b/dfsclient/pom.xml
@@ -27,12 +27,20 @@
       <plugin>
         <artifactId>maven-surefire-plugin</artifactId>
         <version>${maven.surefire.version}</version>
+          <configuration>
+            <properties>
+              <property>
+                <name>listener</name>
+                <value>org.hpccsystems.ws.client.TestResultNotifier</value>
+              </property>
+            </properties>
+          </configuration>
       </plugin>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>templating-maven-plugin</artifactId>
         <version>${codehaus.template.version}</version>
-      </plugin>    
+      </plugin>
     </plugins>
   </build>
 
@@ -83,6 +91,13 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.hpccsystems</groupId>
+      <artifactId>wsclient</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>commons-cli</groupId>
       <artifactId>commons-cli</artifactId>

diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSBenchmarkTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSBenchmarkTest.java
@@ -25,12 +25,11 @@
 import org.hpccsystems.dfs.client.HPCCRecord;
 import org.hpccsystems.dfs.client.HPCCRecordBuilder;
 import org.hpccsystems.dfs.client.HpccRemoteFileReader;
+import org.hpccsystems.ws.client.BaseRemoteTest;
 import org.hpccsystems.dfs.client.DataPartition;
 
 import org.hpccsystems.commons.ecl.FieldDef;
 import org.hpccsystems.commons.errors.HpccFileException;
-import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
-
 import org.json.JSONObject;
 import org.json.JSONArray;
 
@@ -86,7 +85,7 @@ public void readBenchmarks() throws Exception
     {
         System.out.println("Starting Raw Read Tests");
         System.out.println("-------------------------------------------------------------");
-        
+
         MetricSumTransformer sumTransformer = new MetricSumTransformer();
         MetricAverageTransformer aggregateTransformer = new MetricAverageTransformer();
         ArrayList<BenchmarkResult> rawReadTests = new ArrayList<BenchmarkResult>();
@@ -128,7 +127,7 @@ public void readBenchmarks() throws Exception
             }
             avgdMetrics = aggregateTransformer.transform(avgdMetrics);
             result.addMetrics(avgdMetrics);
-            
+
             // Calculate and add bandwidth
             IMetric readTimeMetric = result.getMetric(READ_TIME_METRIC);
             double avgReadTime = readTimeMetric.getValue() * Units.calculateScaleConversion(readTimeMetric.getUnits().scale,Units.Scale.UNIT);
@@ -174,7 +173,7 @@ public void readBenchmarks() throws Exception
 
                     metrics = sumTransformer.transform(metrics);
                     metrics.add(new SimpleMetric((double)readTimeNS, READ_TIME_METRIC, new Units(Units.Type.SECONDS, Units.Scale.NANO)));
-                    
+
                     avgdMetrics.addAll(metrics);
                 }
                 catch(Exception e)
@@ -203,7 +202,7 @@ public void readBenchmarks() throws Exception
 
         // Output plugin results
         JSONArray testGroups = new JSONArray();
-        
+
         JSONObject rawReadGroup = new JSONObject();
         rawReadGroup.put("name","Raw Read Tests");
 
@@ -230,7 +229,7 @@ public void readBenchmarks() throws Exception
 
         JSONObject output = new JSONObject();
         output.put("groups",testGroups);
-        
+
         String outputPath = "results.json";
         FileWriter fileWriter = new FileWriter(outputPath);
         fileWriter.write(output.toString());
@@ -302,7 +301,7 @@ public void readRawFileData(HPCCFile file, List<IMetric> metrics) throws Excepti
                 {
                     bytesToRead = buffer.length;
                 }
-                inputStream.read(buffer,0,bytesToRead); 
+                inputStream.read(buffer,0,bytesToRead);
 
                 try
                 {
@@ -322,7 +321,7 @@ public void readRawFileData(HPCCFile file, List<IMetric> metrics) throws Excepti
                     hasMoreData = nextByte >= 0;
                 }
             }
-            
+
             metrics.addAll(inputStream.getMetrics());
             inputStream.close();
         }
@@ -434,7 +433,7 @@ public int readFileInParallel(HPCCFile file, int numThreads) throws Exception
                     Integer filePartRecordCount = 0;
                     FieldDef recordDefinition = originalRD;
                     public void run()
-                    { 
+                    {
                         HpccRemoteFileReader<HPCCRecord> fileReader = null;
                         try
                         {
@@ -459,14 +458,14 @@ public void run()
                             {
                                 Assert.fail("Received null record during read");
                             }
-                            
+
                             filePartRecordCount++;
                         }
-                        
+
                         recordCounts[filePartIndex] = filePartRecordCount;
                         try
                         {
-                            fileReader.close();  
+                            fileReader.close();
                         }
                         catch (Exception e){}
                     }

diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSHPCCFile.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSHPCCFile.java
@@ -7,7 +7,7 @@
 import org.hpccsystems.commons.ecl.FileFilter;
 import org.hpccsystems.commons.errors.HpccFileException;
 import org.hpccsystems.dfs.cluster.RemapInfo;
-import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
+import org.hpccsystems.ws.client.BaseRemoteTest;
 import org.hpccsystems.ws.client.utils.Connection;
 import org.junit.AfterClass;
 import org.junit.Assert;

diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSIndexTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSIndexTest.java
@@ -27,9 +27,10 @@
 import java.lang.StringBuilder;
 import java.lang.Math;
 
+import org.hpccsystems.ws.client.BaseRemoteTest;
 import org.hpccsystems.ws.client.HPCCWsDFUClient;
 import org.hpccsystems.ws.client.HPCCWsWorkUnitsClient;
-import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
+import org.hpccsystems.ws.client.utils.Connection;
 import org.hpccsystems.ws.client.wrappers.wsworkunits.WorkunitWrapper;
 import org.hpccsystems.ws.client.wrappers.wsdfu.DFUCreateFileWrapper;
 import org.hpccsystems.ws.client.wrappers.wsdfu.DFUFileDetailWrapper;
@@ -88,7 +89,7 @@ public void setup() throws Exception
             String datasetName = datasetNames[i];
             FieldDef recordDef = datasetRecordDefinitions[i];
             createIndexableFile(datasetName, recordDef, partitionRangeStart, partitionRangeEnd);
-            
+
             //------------------------------------------------------------------------------
             // Create index
             //------------------------------------------------------------------------------
@@ -104,7 +105,7 @@ public void setup() throws Exception
             }
         }
     }
-    
+
     @Test
     public void hpccTLKFilterTest() throws Exception
     {
@@ -165,7 +166,7 @@ public void hpccTLKFilterTest() throws Exception
                     {
                         System.out.println("Partition: " + j + " Filter: " + filterStr);
                         System.out.println("Partition range: " + file.getPartitionProcessor().getPartitionRangeAsString(j));
-                        Assert.fail("PartitionProcessor: " + j 
+                        Assert.fail("PartitionProcessor: " + j
                                 + " filtering result did not contain partition"
                                 + partitionListToString(matchedPartitions));
                     }
@@ -181,21 +182,21 @@ public void tlkFilterExample() throws Exception
         //------------------------------------------------------------------------------
         // Read index and check TLK against known partition ranges
         //------------------------------------------------------------------------------
-        
+
         HPCCFile file = new HPCCFile("~test::index::integer::key", connString , hpccUser, hpccPass);
-        
+
         // Find partitions that match the provided filter
         Long searchValue = 3L;
         FileFilter filter = new FileFilter("key = " + searchValue);
         List<DataPartition> filteredPartitions = file.findMatchingPartitions(filter);
-        
+
         assertTrue("Unexpected number of partitions", filteredPartitions.size() == 1);
 
         DataPartition matchedPart = filteredPartitions.get(0);
         HPCCRecordBuilder recordBuilder = new HPCCRecordBuilder(file.getProjectedRecordDefinition());
         HpccRemoteFileReader<HPCCRecord> fileReader = new HpccRemoteFileReader<HPCCRecord>(matchedPart, file.getRecordDefinition(), recordBuilder);
-        
-        boolean foundRecord = false; 
+
+        boolean foundRecord = false;
         while (fileReader.hasNext())
         {
             HPCCRecord record = fileReader.next();
@@ -273,7 +274,7 @@ private void createIndexableFile(String fileName, FieldDef recordDef, List<HPCCR
             //------------------------------------------------------------------------------
             //  Write partitions to file parts and keep track of record ranges
             //------------------------------------------------------------------------------
-            
+
             partitionRangeStart.clear();
             partitionRangeEnd.clear();
 
@@ -404,7 +405,7 @@ List<HPCCRecord> createRecordRange(int partitionIndex, int numPartitions, FieldD
             boolean isKeyField = (i == 0);
             boolean isStart = true;
             rangeStartFields[i] = createFieldValue(partitionIndex, numPartitions, recordDef.getDef(i), isKeyField, isStart);
-            
+
             isStart = false;
             rangeEndFields[i] = createFieldValue(partitionIndex, numPartitions, recordDef.getDef(i),isKeyField, isStart);
         }
@@ -454,13 +455,13 @@ Object createFieldValue(int partitionIndex, int numPartitions, FieldDef fd, bool
                     {
                         rangeNum = partitionIndex * 4 + 3;
                     }
-                    
+
                     StringBuilder builder = new StringBuilder("                ");
                     int charIndex = (int) Math.ceil(Math.log(numPartitions) / Math.log(26));
                     while (rangeNum > 0)
                     {
                         char currentLetter = (char) ('A' + (rangeNum % 26));
-                        builder.setCharAt(charIndex,currentLetter); 
+                        builder.setCharAt(charIndex,currentLetter);
 
                         rangeNum /= 26;
                         charIndex--;

diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java
@@ -39,11 +39,11 @@
 import org.hpccsystems.commons.ecl.RecordDefinitionTranslator;
 import org.hpccsystems.commons.errors.HpccFileException;
 import org.hpccsystems.commons.utils.Utils;
+import org.hpccsystems.ws.client.BaseRemoteTest;
 import org.hpccsystems.ws.client.HPCCWsDFUClient;
 import org.hpccsystems.ws.client.HPCCWsWorkUnitsClient;
 import org.hpccsystems.ws.client.platform.Version;
 import org.hpccsystems.ws.client.wrappers.wsworkunits.WorkunitWrapper;
-import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
 import org.hpccsystems.ws.client.wrappers.wsdfu.DFUCreateFileWrapper;
 import org.hpccsystems.ws.client.wrappers.wsdfu.DFUFileDetailWrapper;
 import org.hpccsystems.ws.client.wrappers.wsdfu.DFUFilePartWrapper;

diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/FileUtilityTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/FileUtilityTest.java
@@ -21,7 +21,7 @@
 import java.io.File;
 
 import org.hpccsystems.dfs.client.FileUtility;
-import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
+import org.hpccsystems.ws.client.BaseRemoteTest;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.Assert;

diff --git a/pom.xml b/pom.xml
@@ -227,6 +227,13 @@
               <exclude>**/WSDLs</exclude>
             </excludes>
           </configuration>
+          <executions>
+            <execution>
+              <goals>
+                <goal>test-jar</goal>
+              </goals>
+            </execution>
+          </executions>
         </plugin>
         <plugin>
           <artifactId>maven-install-plugin</artifactId>

diff --git a/wsclient/pom.xml b/wsclient/pom.xml
@@ -78,9 +78,17 @@
           </excludes>
         </configuration>
       </plugin>
-            <plugin>
+      <plugin>
         <artifactId>maven-surefire-plugin</artifactId>
         <version>${maven.surefire.version}</version>
+          <configuration>
+            <properties>
+              <property>
+                <name>listener</name>
+                <value>org.hpccsystems.ws.client.TestResultNotifier</value>
+              </property>
+            </properties>
+          </configuration>
       </plugin>
    </plugins>
  </build>

diff --git a/.../client/platform/test/BaseRemoteTest.java → ...hpccsystems/ws/client/BaseRemoteTest.java b/.../client/platform/test/BaseRemoteTest.java → ...hpccsystems/ws/client/BaseRemoteTest.java
@@ -15,11 +15,13 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
     limitations under the License.
 ############################################################################## */
 
-package org.hpccsystems.ws.client.platform.test;
+package org.hpccsystems.ws.client;
 
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
 import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.UnknownHostException;
@@ -69,49 +71,6 @@ public abstract class BaseRemoteTest
     protected final static String sockTO = System.getProperty("sockettimeoutmillis");
 
     protected final static int  testThreadCount = Integer.parseInt(System.getProperty("testthreadcount", "10"));
-
-    /*
-      Code used to generate HPCC file
-      unique_keys :=  100000;  // Should be less than number of records
-      unique_values := 10212; // Should be less than number of records
-      dataset_name := '~benchmark::all_types::200KB';
-      totalrecs := 779449/500;
-
-      childRec := {STRING8 childField1, INTEGER8 childField2, REAL8 childField3};
-
-      rec := { INTEGER8 int8, UNSIGNED8 uint8, INTEGER4 int4, UNSIGNED4 uint4,
-               INTEGER2 int2, UNSIGNED2 uint2, REAL8 r8, REAL4 r4,
-               DECIMAL16_8 dec16, UDECIMAL16_8 udec16, QSTRING qStr,
-               STRING8 fixStr8, STRING str, VARSTRING varStr, VARSTRING varStr8,
-               UTF8 utfStr, UNICODE8 uni8, UNICODE uni, VARUNICODE varUni,
-               DATASET(childRec) childDataset,  SET OF INTEGER1 int1Set
-             };
-
-             ds := DATASET(totalrecs, transform(rec,
-                                  self.int8 := (INTEGER)(random() % unique_keys);
-                                  self.uint8 := (INTEGER)(random() % unique_values);
-                                  self.int4 := (INTEGER)(random() % unique_values);
-                                  self.uint4 := (INTEGER)(random() % unique_values);
-                                  self.int2 := (INTEGER)(random() % unique_values);
-                                  self.uint2 := (INTEGER)(random() % unique_values);
-                                  self.r8 := (REAL)(random() % unique_values);
-                                  self.r4 := (REAL)(random() % unique_values);
-                                  self.dec16 := (REAL)(random() % unique_values);
-                                  self.udec16 := (REAL)(random() % unique_values);
-                                  self.qStr := (STRING)(random() % unique_values);
-                                  self.fixStr8 := (STRING)(random() % unique_values);
-                                  self.str := (STRING)(random() % unique_values);
-                                  self.varStr := (STRING)(random() % unique_values);
-                                  self.varStr8 := (STRING)(random() % unique_values);
-                                  self.utfStr := (STRING)(random() % unique_values);
-                                  self.uni8 := (STRING)(random() % unique_values);
-                                  self.uni := (STRING)(random() % unique_values);
-                                  self.varUni := (STRING)(random() % unique_values);
-                                  self.childDataset := DATASET([{'field1',2,3},{'field1',2,3}],childRec);
-                                  self.int1Set := [1,2,3];
-                           ), DISTRIBUTED);
-              OUTPUT(ds,,dataset_name,overwrite);
-     */
     public static final String DEFAULTHPCCFILENAME      = "benchmark::all_types::200kb";
 
     /*
@@ -249,10 +208,17 @@ public boolean verify(String hostname,javax.net.ssl.SSLSession sslSession)
 
     public String executeECLScript(String eclFile) throws Exception
     {
-        URL eclFileURL = getClass().getClassLoader().getResource(eclFile);
-        Path eclFilePath = Paths.get(eclFileURL.toURI());
+        InputStream resourceStream = BaseRemoteTest.class.getClassLoader().getResourceAsStream(eclFile);
+        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+
+        byte[] buffer = new byte[4096];
+        int bytesRead = resourceStream.read(buffer);
+        while (bytesRead > -1) {
+            byteArrayOutputStream.write(buffer, 0, bytesRead);
+            bytesRead = resourceStream.read(buffer);
+        }
 
-        byte[] eclData = Files.readAllBytes(eclFilePath);
+        byte[] eclData = byteArrayOutputStream.toByteArray();
         String ecl = new String(eclData, "UTF-8");
 
         WorkunitWrapper wu = new WorkunitWrapper();