Skip to content

Commit

Permalink
HPCC4J-555 Provide Test Jar
Browse files Browse the repository at this point in the history 
- Moved BaseRemoteTests into testing dir
- Added TestResultNotifier to record test failures
- Updated POMs to use and generate test-jar

Signed-off-by: James McMullan [email protected]
  • Loading branch information
@jpmcmu
jpmcmu committed Nov 29, 2023
1 parent b63ae1d commit 35b3485
Show file tree
Hide file tree
Showing 29 changed files with 112 additions and 93 deletions.
17 changes: 16 additions & 1 deletion dfsclient/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,20 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven.surefire.version}</version>
<configuration>
<properties>
<property>
<name>listener</name>
<value>org.hpccsystems.ws.client.TestResultNotifier</value>
</property>
</properties>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>templating-maven-plugin</artifactId>
<version>${codehaus.template.version}</version>
</plugin>
</plugin>
</plugins>
</build>

Expand Down Expand Up @@ -83,6 +91,13 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.hpccsystems</groupId>
<artifactId>wsclient</artifactId>
<type>test-jar</type>
<scope>test</scope>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
Expand Down
25 changes: 12 additions & 13 deletions dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSBenchmarkTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,11 @@
import org.hpccsystems.dfs.client.HPCCRecord;
import org.hpccsystems.dfs.client.HPCCRecordBuilder;
import org.hpccsystems.dfs.client.HpccRemoteFileReader;
import org.hpccsystems.ws.client.BaseRemoteTest;
import org.hpccsystems.dfs.client.DataPartition;

import org.hpccsystems.commons.ecl.FieldDef;
import org.hpccsystems.commons.errors.HpccFileException;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;

import org.json.JSONObject;
import org.json.JSONArray;

Expand Down Expand Up @@ -86,7 +85,7 @@ public void readBenchmarks() throws Exception
{
System.out.println("Starting Raw Read Tests");
System.out.println("-------------------------------------------------------------");

MetricSumTransformer sumTransformer = new MetricSumTransformer();
MetricAverageTransformer aggregateTransformer = new MetricAverageTransformer();
ArrayList<BenchmarkResult> rawReadTests = new ArrayList<BenchmarkResult>();
Expand Down Expand Up @@ -128,7 +127,7 @@ public void readBenchmarks() throws Exception
}
avgdMetrics = aggregateTransformer.transform(avgdMetrics);
result.addMetrics(avgdMetrics);

// Calculate and add bandwidth
IMetric readTimeMetric = result.getMetric(READ_TIME_METRIC);
double avgReadTime = readTimeMetric.getValue() * Units.calculateScaleConversion(readTimeMetric.getUnits().scale,Units.Scale.UNIT);
Expand Down Expand Up @@ -174,7 +173,7 @@ public void readBenchmarks() throws Exception

metrics = sumTransformer.transform(metrics);
metrics.add(new SimpleMetric((double)readTimeNS, READ_TIME_METRIC, new Units(Units.Type.SECONDS, Units.Scale.NANO)));

avgdMetrics.addAll(metrics);
}
catch(Exception e)
Expand Down Expand Up @@ -203,7 +202,7 @@ public void readBenchmarks() throws Exception

// Output plugin results
JSONArray testGroups = new JSONArray();

JSONObject rawReadGroup = new JSONObject();
rawReadGroup.put("name","Raw Read Tests");

Expand All @@ -230,7 +229,7 @@ public void readBenchmarks() throws Exception

JSONObject output = new JSONObject();
output.put("groups",testGroups);

String outputPath = "results.json";
FileWriter fileWriter = new FileWriter(outputPath);
fileWriter.write(output.toString());
Expand Down Expand Up @@ -302,7 +301,7 @@ public void readRawFileData(HPCCFile file, List<IMetric> metrics) throws Excepti
{
bytesToRead = buffer.length;
}
inputStream.read(buffer,0,bytesToRead);
inputStream.read(buffer,0,bytesToRead);

try
{
Expand All @@ -322,7 +321,7 @@ public void readRawFileData(HPCCFile file, List<IMetric> metrics) throws Excepti
hasMoreData = nextByte >= 0;
}
}

metrics.addAll(inputStream.getMetrics());
inputStream.close();
}
Expand Down Expand Up @@ -434,7 +433,7 @@ public int readFileInParallel(HPCCFile file, int numThreads) throws Exception
Integer filePartRecordCount = 0;
FieldDef recordDefinition = originalRD;
public void run()
{
{
HpccRemoteFileReader<HPCCRecord> fileReader = null;
try
{
Expand All @@ -459,14 +458,14 @@ public void run()
{
Assert.fail("Received null record during read");
}

filePartRecordCount++;
}

recordCounts[filePartIndex] = filePartRecordCount;
try
{
fileReader.close();
fileReader.close();
}
catch (Exception e){}
}
Expand Down
2 changes: 1 addition & 1 deletion dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSHPCCFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import org.hpccsystems.commons.ecl.FileFilter;
import org.hpccsystems.commons.errors.HpccFileException;
import org.hpccsystems.dfs.cluster.RemapInfo;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.hpccsystems.ws.client.BaseRemoteTest;
import org.hpccsystems.ws.client.utils.Connection;
import org.junit.AfterClass;
import org.junit.Assert;
Expand Down
2 changes: 1 addition & 1 deletion dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSIndexTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
import java.lang.StringBuilder;
import java.lang.Math;

import org.hpccsystems.ws.client.BaseRemoteTest;
import org.hpccsystems.ws.client.HPCCWsDFUClient;
import org.hpccsystems.ws.client.HPCCWsWorkUnitsClient;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.hpccsystems.ws.client.utils.Connection;
import org.hpccsystems.ws.client.wrappers.wsworkunits.WorkunitWrapper;
import org.hpccsystems.ws.client.wrappers.wsdfu.DFUCreateFileWrapper;
Expand Down
2 changes: 1 addition & 1 deletion dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@
import org.hpccsystems.commons.ecl.RecordDefinitionTranslator;
import org.hpccsystems.commons.errors.HpccFileException;
import org.hpccsystems.commons.utils.Utils;
import org.hpccsystems.ws.client.BaseRemoteTest;
import org.hpccsystems.ws.client.HPCCWsDFUClient;
import org.hpccsystems.ws.client.HPCCWsWorkUnitsClient;
import org.hpccsystems.ws.client.platform.Version;
import org.hpccsystems.ws.client.wrappers.wsworkunits.WorkunitWrapper;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.hpccsystems.ws.client.wrappers.wsdfu.DFUCreateFileWrapper;
import org.hpccsystems.ws.client.wrappers.wsdfu.DFUFileDetailWrapper;
import org.hpccsystems.ws.client.wrappers.wsdfu.DFUFilePartWrapper;
Expand Down
2 changes: 1 addition & 1 deletion dfsclient/src/test/java/org/hpccsystems/dfs/client/FileUtilityTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import java.io.File;

import org.hpccsystems.dfs.client.FileUtility;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.hpccsystems.ws.client.BaseRemoteTest;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Assert;
Expand Down
7 changes: 7 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,13 @@
<exclude>**/WSDLs</exclude>
</excludes>
</configuration>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
Expand Down
10 changes: 9 additions & 1 deletion wsclient/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,17 @@
</excludes>
</configuration>
</plugin>
<plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven.surefire.version}</version>
<configuration>
<properties>
<property>
<name>listener</name>
<value>org.hpccsystems.ws.client.TestResultNotifier</value>
</property>
</properties>
</configuration>
</plugin>
</plugins>
</build>
Expand Down
63 changes: 12 additions & 51 deletions .../client/platform/test/BaseRemoteTest.java → ...hpccsystems/ws/client/BaseRemoteTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
limitations under the License.
############################################################################## */

package org.hpccsystems.ws.client.platform.test;
package org.hpccsystems.ws.client;

import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.UnknownHostException;
Expand Down Expand Up @@ -69,49 +71,6 @@ public abstract class BaseRemoteTest
protected final static String sockTO = System.getProperty("sockettimeoutmillis");

protected final static int testThreadCount = Integer.parseInt(System.getProperty("testthreadcount", "10"));

/*
Code used to generate HPCC file
unique_keys := 100000; // Should be less than number of records
unique_values := 10212; // Should be less than number of records
dataset_name := '~benchmark::all_types::200KB';
totalrecs := 779449/500;
childRec := {STRING8 childField1, INTEGER8 childField2, REAL8 childField3};
rec := { INTEGER8 int8, UNSIGNED8 uint8, INTEGER4 int4, UNSIGNED4 uint4,
INTEGER2 int2, UNSIGNED2 uint2, REAL8 r8, REAL4 r4,
DECIMAL16_8 dec16, UDECIMAL16_8 udec16, QSTRING qStr,
STRING8 fixStr8, STRING str, VARSTRING varStr, VARSTRING varStr8,
UTF8 utfStr, UNICODE8 uni8, UNICODE uni, VARUNICODE varUni,
DATASET(childRec) childDataset, SET OF INTEGER1 int1Set
};
ds := DATASET(totalrecs, transform(rec,
self.int8 := (INTEGER)(random() % unique_keys);
self.uint8 := (INTEGER)(random() % unique_values);
self.int4 := (INTEGER)(random() % unique_values);
self.uint4 := (INTEGER)(random() % unique_values);
self.int2 := (INTEGER)(random() % unique_values);
self.uint2 := (INTEGER)(random() % unique_values);
self.r8 := (REAL)(random() % unique_values);
self.r4 := (REAL)(random() % unique_values);
self.dec16 := (REAL)(random() % unique_values);
self.udec16 := (REAL)(random() % unique_values);
self.qStr := (STRING)(random() % unique_values);
self.fixStr8 := (STRING)(random() % unique_values);
self.str := (STRING)(random() % unique_values);
self.varStr := (STRING)(random() % unique_values);
self.varStr8 := (STRING)(random() % unique_values);
self.utfStr := (STRING)(random() % unique_values);
self.uni8 := (STRING)(random() % unique_values);
self.uni := (STRING)(random() % unique_values);
self.varUni := (STRING)(random() % unique_values);
self.childDataset := DATASET([{'field1',2,3},{'field1',2,3}],childRec);
self.int1Set := [1,2,3];
), DISTRIBUTED);
OUTPUT(ds,,dataset_name,overwrite);
*/
public static final String DEFAULTHPCCFILENAME = "benchmark::all_types::200kb";

/*
Expand Down Expand Up @@ -259,15 +218,17 @@ public boolean verify(String hostname,javax.net.ssl.SSLSession sslSession)

public static String executeECLScript(String eclFile) throws Exception
{
URL eclFileURL = BaseRemoteTest.class.getClassLoader().getResource(eclFile);
if (eclFileURL == null)
{
return null;
InputStream resourceStream = BaseRemoteTest.class.getClassLoader().getResourceAsStream(eclFile);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

byte[] buffer = new byte[4096];
int bytesRead = resourceStream.read(buffer);
while (bytesRead > -1) {
byteArrayOutputStream.write(buffer, 0, bytesRead);
bytesRead = resourceStream.read(buffer);
}

Path eclFilePath = Paths.get(eclFileURL.toURI());

byte[] eclData = Files.readAllBytes(eclFilePath);
byte[] eclData = byteArrayOutputStream.toByteArray();
String ecl = new String(eclData, "UTF-8");

WorkunitWrapper wu = new WorkunitWrapper();
Expand Down
1 change: 0 additions & 1 deletion wsclient/src/test/java/org/hpccsystems/ws/client/BaseWsAttributesClientIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import org.apache.commons.io.FilenameUtils;
import org.hpccsystems.ws.client.extended.HPCCWsAttributesClient;
import org.hpccsystems.ws.client.platform.Cluster;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper;
import org.hpccsystems.ws.client.wrappers.ECLAttributeWrapper;
import org.junit.Before;
Expand Down
1 change: 0 additions & 1 deletion wsclient/src/test/java/org/hpccsystems/ws/client/BaseWsWorkunitsClientIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import org.hpccsystems.ws.client.platform.Cluster;
import org.hpccsystems.ws.client.platform.QuerySetFilterType;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.hpccsystems.ws.client.wrappers.ApplicationValueWrapper;
import org.hpccsystems.ws.client.wrappers.ArrayOfECLExceptionWrapper;
import org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper;
Expand Down
1 change: 0 additions & 1 deletion wsclient/src/test/java/org/hpccsystems/ws/client/FileSprayClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import java.util.List;

import org.apache.axis2.AxisFault;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.hpccsystems.ws.client.utils.Connection;
import org.hpccsystems.ws.client.utils.Utils;
import org.hpccsystems.ws.client.wrappers.ArrayOfBaseExceptionWrapper;
Expand Down
48 changes: 48 additions & 0 deletions wsclient/src/test/java/org/hpccsystems/ws/client/TestResultNotifier.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package org.hpccsystems.ws.client;

import org.junit.runner.Description;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunListener;

import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;

public class TestResultNotifier extends RunListener
{
PrintWriter failedTestsFile = null;

public void testFailure(Failure failure) throws Exception
{
synchronized (this)
{
if (failedTestsFile == null)
{
try
{
failedTestsFile = new PrintWriter( new FileWriter("./FailedTests.csv"));
}
catch (IOException e)
{
System.out.println("Failed to open FailedTests file with error: " + e.getMessage() + " redirecting output to System.out");
failedTestsFile = new PrintWriter(System.out);
}
}

Description description = failure.getDescription();
failedTestsFile.println(description.getClassName() + "," + description.getMethodName() + "," + failure.getMessage());
failedTestsFile.flush();
}
}

protected void finalize() throws Throwable
{
synchronized (this)
{
if (failedTestsFile != null)
{
failedTestsFile.close();
}
}
}
}
1 change: 0 additions & 1 deletion wsclient/src/test/java/org/hpccsystems/ws/client/WSCloudClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ HPCC SYSTEMS software Copyright (C) 2022 HPCC Systems®.

import static org.junit.Assume.assumeTrue;

import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Assert;
Expand Down
1 change: 0 additions & 1 deletion wsclient/src/test/java/org/hpccsystems/ws/client/WSCodeSignClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
import java.util.List;

import org.apache.axis2.AxisFault;
import org.hpccsystems.ws.client.platform.test.BaseRemoteTest;
import org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper;
import org.hpccsystems.ws.client.wrappers.gen.wscodesign.ListUserIDsRequestWrapper;
import org.hpccsystems.ws.client.wrappers.gen.wscodesign.ListUserIDsResponseWrapper;
Expand Down
Loading

0 comments on commit 35b3485

Please sign in to comment.