Skip to content

Commit

Permalink
HPCC4J-567 Adapt WsFileIOClient to latest ESP behavior
Browse files Browse the repository at this point in the history 
- Adds new methods with exposed LZ Address
- Provides LZ in non-containerized junits
- Changes default LZ name from localhost to mylandingzone

Signed-off-by: Rodrigo Pastrana <[email protected]>
  • Loading branch information
@rpastrana
rpastrana committed Apr 2, 2024
1 parent 5e0b6d6 commit 8dbcfb6
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 17 deletions.
101 changes: 97 additions & 4 deletions wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsFileIOClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,14 @@ public boolean ping() throws Exception
}

/**
* Creates the HPCC file.
* @deprecated Due to change in server behavior
* Use boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile, String lzAddress) instead
* Creates an HPCC file.
*
* @param fileName
* - The target HPCC file name
* @param targetLandingZone
* - The "netaddress" of the target landing, can be localhost, should be fetched from landingzones in filesprayclient
* - The LZ name, no longer the netaddress of the LZ. should be fetched from landingzones in filesprayclient
* @param overwritefile
* - If the file exists, should it be overwritten?
* @return true, if successful
Expand All @@ -247,17 +249,47 @@ public boolean ping() throws Exception
* the array of esp exception wrapper
*/
public boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile) throws Exception, ArrayOfEspExceptionWrapper
{
return createHPCCFile(fileName, targetLandingZone, overwritefile, null);
}

/**
* Creates an HPCC file.
*
* @param fileName
* - The target HPCC file name
* @param targetLandingZone
* - The LZ name, no longer the netaddress of the LZ. should be fetched from landingzones in filesprayclient
* @param overwritefile
* - If the file exists, should it be overwritten?
* @param lzAddress
* - The Landing zone address
* @return true, if successful
* @throws java.lang.Exception
* - Caller should handle exception in case of errors
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile, String lzAddress) throws Exception, ArrayOfEspExceptionWrapper
{
boolean success = false;
log.debug("Attempting to create HPCC File: " + fileName);

if (targetLandingZone == null || targetLandingZone.isEmpty())
throw new Exception("HPCCWsFileIOClient::createHPCCFile: targetLandingZone required!");

if (fileName == null || fileName.isEmpty())
throw new Exception("HPCCWsFileIOClient::createHPCCFile: fileName required!");

verifyStub(); // Throws exception if stub failed

CreateFileRequest request = new CreateFileRequest();

request.setDestDropZone(targetLandingZone);
request.setDestRelativePath(fileName);
request.setOverwrite(overwritefile);
if (lzAddress != null && !lzAddress.isEmpty())
request.setDestNetAddress(lzAddress);

CreateFileResponse resp = null;
try
Expand Down Expand Up @@ -298,13 +330,15 @@ public boolean createHPCCFile(String fileName, String targetLandingZone, boolean
* - At what offset should this be written - Specify 0 if necessary
* @param uploadchunksize
* - Chunksize to upload the data
* @param lzAddress
* - The Landing zone address
* @return true, if successful
* @throws java.lang.Exception
* the exception
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize)
public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize, String lzAddress)
throws Exception, ArrayOfEspExceptionWrapper
{
boolean success = true;
Expand All @@ -318,6 +352,8 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
request.setDestDropZone(targetLandingZone);
request.setDestRelativePath(fileName);
request.setOffset(offset);
if (lzAddress != null && !lzAddress.isEmpty())
request.setDestNetAddress(lzAddress);

int dataindex = 0;
int limit = uploadchunksize <= 0 ? defaultUploadChunkSize : uploadchunksize;
Expand Down Expand Up @@ -368,6 +404,35 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
return success;
}

/**
* @deprecated Due to change in server behavior
* Use boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize, String lzAddress) instead
* Write HPCC file data.
*
* @param data
* - The data to write
* @param fileName
* - The target HPCC file to write to
* @param targetLandingZone
* - The "netaddress" of the target landing, can be localhost, should be fetched from landingzones in filesprayclient
* @param append
* - Should this data be appended?
* @param offset
* - At what offset should this be written - Specify 0 if necessary
* @param uploadchunksize
* - Chunksize to upload the data
* @return true, if successful
* @throws java.lang.Exception
* the exception
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize)
throws Exception, ArrayOfEspExceptionWrapper
{
return writeHPCCFileData(data, fileName, targetLandingZone, append, offset, uploadchunksize, null);
}

/**
* Read file data.
*
Expand All @@ -379,19 +444,23 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
* the datasize
* @param offset
* the offset
* @param dropzoneAddress
* the dropzone address (not needed in containerized mode)
* @return the string
* @throws java.lang.Exception
* the exception
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public String readFileData(String dropzone, String fileName, long datasize, long offset) throws Exception, ArrayOfEspExceptionWrapper
public String readFileData(String dropzone, String fileName, long datasize, long offset, String dropzoneAddress) throws Exception, ArrayOfEspExceptionWrapper
{
ReadFileDataRequest readFileDataRequest = new ReadFileDataRequest();
readFileDataRequest.setDestDropZone(dropzone);
readFileDataRequest.setDestRelativePath(fileName);
readFileDataRequest.setDataSize(datasize);
readFileDataRequest.setOffset(offset);
if (dropzoneAddress != null && !dropzoneAddress.isEmpty())
readFileDataRequest.setDestNetAddress(dropzoneAddress);

ReadFileDataResponse resp = null;
try
Expand Down Expand Up @@ -426,4 +495,28 @@ public String readFileData(String dropzone, String fileName, long datasize, long

return data;
}

/**
* @deprecated Due to change in server behavior
* Use String readFileData(String dropzone, String fileName, long datasize, long offset, String dropzoneAddress) instead
* Read file data.
*
* @param dropzone
* the dropzone
* @param fileName
* the file name
* @param datasize
* the datasize
* @param offset
* the offset
* @return the string
* @throws java.lang.Exception
* the exception
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public String readFileData(String dropzone, String fileName, long datasize, long offset) throws Exception, ArrayOfEspExceptionWrapper
{
return readFileData(dropzone, fileName, datasize, offset, null);
}
}
60 changes: 47 additions & 13 deletions wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,24 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
public class WSFileIOClientTest extends BaseRemoteTest
{
private final static HPCCWsFileIOClient client = wsclient.getWsFileIOClient();

private static boolean isContainerized = false;
private final static String testfilename = System.getProperty("lztestfile", "myfilename.txt");
private final static String targetLZ = System.getProperty("lzname", "localhost");
private final static String targetLZ = System.getProperty("lzname", "mydropzone"); //targetLZ accepted the address "localhost" once upon a time.
private final static String targetLZPath = System.getProperty("lzpath", "/var/lib/HPCCSystems/mydropzone");
private final static String HPCC_30117 = System.getProperty("HPCC30117", "fixed");
private final static String targetLZAddress = System.getProperty("lzaddress", ".");

static
{
try
{
if (client.isTargetHPCCContainerized())
isContainerized = true;
}
catch (Exception e)
{
System.out.println("Could not determine if target service is containerized, default: 'false'");
}

if (System.getProperty("lztestfile") == null)
System.out.println("lztestfile not provided - defaulting to myfilename.txt");

Expand All @@ -60,20 +70,18 @@ public class WSFileIOClientTest extends BaseRemoteTest
if (System.getProperty("lzpath") == null)
System.out.println("lzpath not provided - defaulting to /var/lib/HPCCSystems/mydropzone");

if (System.getProperty("HPCC30117") == null)
System.out.println("HPCC30117 status not provided - defaulting to fixed");
else
System.out.println("HPCC30117 status: '" + HPCC_30117 + "'");
if (System.getProperty("lzaddress") == null)
System.out.println("lzaddress not provided - defaulting to '.'");
}

@Test
public void copyFile() throws Exception
{
String lzfile=System.currentTimeMillis() + "_csvtest.csv";
String hpccfilename="temp::" + lzfile;
client.createHPCCFile(lzfile, targetLZ, true);
client.createHPCCFile(lzfile, targetLZ, true, isContainerized ? null : targetLZAddress);
byte[] data = "Product,SKU,Color\r\nBike,1234,Blue\r\nCar,2345,Red\r\n".getBytes();
client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20);
client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20, isContainerized ? null : targetLZAddress);
try
{
System.out.println("Starting file spray.");
Expand Down Expand Up @@ -152,23 +160,49 @@ public void copyFile() throws Exception
@Test
public void AcreateHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
{
System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true));
if (isContainerized)
{
System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
System.out.println("Target HPCC is containerized, not providing targetLZAddress");
Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true, null));
}
else
{
System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' targetLZaddress: '" + targetLZAddress + "' on HPCC: '" + super.connString +"'");
System.out.println("Target HPCC is NOT containerized, providing targetLZAddress");
Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true, targetLZAddress));
}
}

@Test
public void BwriteHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
{
System.out.println("Writing data to file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
byte[] data = "HELLO MY DARLING, HELLO MY DEAR!1234567890ABCDEFGHIJKLMNOPQRSTUVXYZ".getBytes();
Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20));
if (isContainerized)
{
Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20, null));
}
else
{
Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20, targetLZAddress));
}
}

@Test
public void CreadHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
{
System.out.println("reading data from file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
byte[] data = "HELLO MY DARLING, HELLO MY DEAR!1234567890ABCDEFGHIJKLMNOPQRSTUVXYZ".getBytes();
String response = client.readFileData(targetLZ, testfilename, data.length, 0);
String response = null;
if (isContainerized)
{
response = client.readFileData(targetLZ, testfilename, data.length, 0, null);
}
else
{
response = client.readFileData(targetLZ, testfilename, data.length, 0, targetLZAddress);
}
Assert.assertNotNull(response);
Assert.assertArrayEquals(data, response.getBytes());
}
Expand Down

0 comments on commit 8dbcfb6

Please sign in to comment.