From cbbe7b4a96077c360a514ae33dddb193fde9fa15 Mon Sep 17 00:00:00 2001 From: Rodrigo Date: Thu, 4 Apr 2024 09:48:39 -0400 Subject: [PATCH] HPCC4J-567 Adapt WsFileIOClient to latest ESP behavior (#703) - Adds new methods with exposed LZ Address - Provides LZ in non-containerized junits - Changes default LZ name from localhost to mylandingzone Signed-off-by: Rodrigo Pastrana --- .../ws/client/HPCCWsFileIOClient.java | 101 +++++++++++++++++- .../ws/client/WSFileIOClientTest.java | 60 ++++++++--- 2 files changed, 144 insertions(+), 17 deletions(-) diff --git a/wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsFileIOClient.java b/wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsFileIOClient.java index fc99737d6..62282ae58 100644 --- a/wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsFileIOClient.java +++ b/wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsFileIOClient.java @@ -232,12 +232,14 @@ public boolean ping() throws Exception } /** - * Creates the HPCC file. + * @deprecated Due to change in server behavior + * Use boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile, String lzAddress) instead + * Creates an HPCC file. * * @param fileName * - The target HPCC file name * @param targetLandingZone - * - The "netaddress" of the target landing, can be localhost, should be fetched from landingzones in filesprayclient + * - The LZ name, no longer the netaddress of the LZ. should be fetched from landingzones in filesprayclient * @param overwritefile * - If the file exists, should it be overwritten? * @return true, if successful @@ -247,10 +249,38 @@ public boolean ping() throws Exception * the array of esp exception wrapper */ public boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile) throws Exception, ArrayOfEspExceptionWrapper + { + return createHPCCFile(fileName, targetLandingZone, overwritefile, null); + } + + /** + * Creates an HPCC file. + * + * @param fileName + * - The target HPCC file name + * @param targetLandingZone + * - The LZ name, no longer the netaddress of the LZ. should be fetched from landingzones in filesprayclient + * @param overwritefile + * - If the file exists, should it be overwritten? + * @param lzAddress + * - The Landing zone address + * @return true, if successful + * @throws java.lang.Exception + * - Caller should handle exception in case of errors + * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper + * the array of esp exception wrapper + */ + public boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile, String lzAddress) throws Exception, ArrayOfEspExceptionWrapper { boolean success = false; log.debug("Attempting to create HPCC File: " + fileName); + if (targetLandingZone == null || targetLandingZone.isEmpty()) + throw new Exception("HPCCWsFileIOClient::createHPCCFile: targetLandingZone required!"); + + if (fileName == null || fileName.isEmpty()) + throw new Exception("HPCCWsFileIOClient::createHPCCFile: fileName required!"); + verifyStub(); // Throws exception if stub failed CreateFileRequest request = new CreateFileRequest(); @@ -258,6 +288,8 @@ public boolean createHPCCFile(String fileName, String targetLandingZone, boolean request.setDestDropZone(targetLandingZone); request.setDestRelativePath(fileName); request.setOverwrite(overwritefile); + if (lzAddress != null && !lzAddress.isEmpty()) + request.setDestNetAddress(lzAddress); CreateFileResponse resp = null; try @@ -298,13 +330,15 @@ public boolean createHPCCFile(String fileName, String targetLandingZone, boolean * - At what offset should this be written - Specify 0 if necessary * @param uploadchunksize * - Chunksize to upload the data + * @param lzAddress + * - The Landing zone address * @return true, if successful * @throws java.lang.Exception * the exception * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper * the array of esp exception wrapper */ - public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize) + public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize, String lzAddress) throws Exception, ArrayOfEspExceptionWrapper { boolean success = true; @@ -318,6 +352,8 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand request.setDestDropZone(targetLandingZone); request.setDestRelativePath(fileName); request.setOffset(offset); + if (lzAddress != null && !lzAddress.isEmpty()) + request.setDestNetAddress(lzAddress); int dataindex = 0; int limit = uploadchunksize <= 0 ? defaultUploadChunkSize : uploadchunksize; @@ -368,6 +404,35 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand return success; } + /** + * @deprecated Due to change in server behavior + * Use boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize, String lzAddress) instead + * Write HPCC file data. + * + * @param data + * - The data to write + * @param fileName + * - The target HPCC file to write to + * @param targetLandingZone + * - The "netaddress" of the target landing, can be localhost, should be fetched from landingzones in filesprayclient + * @param append + * - Should this data be appended? + * @param offset + * - At what offset should this be written - Specify 0 if necessary + * @param uploadchunksize + * - Chunksize to upload the data + * @return true, if successful + * @throws java.lang.Exception + * the exception + * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper + * the array of esp exception wrapper + */ + public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize) + throws Exception, ArrayOfEspExceptionWrapper + { + return writeHPCCFileData(data, fileName, targetLandingZone, append, offset, uploadchunksize, null); + } + /** * Read file data. * @@ -379,19 +444,23 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand * the datasize * @param offset * the offset + * @param dropzoneAddress + * the dropzone address (not needed in containerized mode) * @return the string * @throws java.lang.Exception * the exception * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper * the array of esp exception wrapper */ - public String readFileData(String dropzone, String fileName, long datasize, long offset) throws Exception, ArrayOfEspExceptionWrapper + public String readFileData(String dropzone, String fileName, long datasize, long offset, String dropzoneAddress) throws Exception, ArrayOfEspExceptionWrapper { ReadFileDataRequest readFileDataRequest = new ReadFileDataRequest(); readFileDataRequest.setDestDropZone(dropzone); readFileDataRequest.setDestRelativePath(fileName); readFileDataRequest.setDataSize(datasize); readFileDataRequest.setOffset(offset); + if (dropzoneAddress != null && !dropzoneAddress.isEmpty()) + readFileDataRequest.setDestNetAddress(dropzoneAddress); ReadFileDataResponse resp = null; try @@ -426,4 +495,28 @@ public String readFileData(String dropzone, String fileName, long datasize, long return data; } + + /** + * @deprecated Due to change in server behavior + * Use String readFileData(String dropzone, String fileName, long datasize, long offset, String dropzoneAddress) instead + * Read file data. + * + * @param dropzone + * the dropzone + * @param fileName + * the file name + * @param datasize + * the datasize + * @param offset + * the offset + * @return the string + * @throws java.lang.Exception + * the exception + * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper + * the array of esp exception wrapper + */ + public String readFileData(String dropzone, String fileName, long datasize, long offset) throws Exception, ArrayOfEspExceptionWrapper + { + return readFileData(dropzone, fileName, datasize, offset, null); + } } diff --git a/wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java b/wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java index e27714e24..3d5fb5a57 100644 --- a/wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java +++ b/wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java @@ -43,14 +43,24 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®. public class WSFileIOClientTest extends BaseRemoteTest { private final static HPCCWsFileIOClient client = wsclient.getWsFileIOClient(); - + private static boolean isContainerized = false; private final static String testfilename = System.getProperty("lztestfile", "myfilename.txt"); - private final static String targetLZ = System.getProperty("lzname", "localhost"); + private final static String targetLZ = System.getProperty("lzname", "mydropzone"); //targetLZ accepted the address "localhost" once upon a time. private final static String targetLZPath = System.getProperty("lzpath", "/var/lib/HPCCSystems/mydropzone"); - private final static String HPCC_30117 = System.getProperty("HPCC30117", "fixed"); + private final static String targetLZAddress = System.getProperty("lzaddress", "."); static { + try + { + if (client.isTargetHPCCContainerized()) + isContainerized = true; + } + catch (Exception e) + { + System.out.println("Could not determine if target service is containerized, default: 'false'"); + } + if (System.getProperty("lztestfile") == null) System.out.println("lztestfile not provided - defaulting to myfilename.txt"); @@ -60,10 +70,8 @@ public class WSFileIOClientTest extends BaseRemoteTest if (System.getProperty("lzpath") == null) System.out.println("lzpath not provided - defaulting to /var/lib/HPCCSystems/mydropzone"); - if (System.getProperty("HPCC30117") == null) - System.out.println("HPCC30117 status not provided - defaulting to fixed"); - else - System.out.println("HPCC30117 status: '" + HPCC_30117 + "'"); + if (System.getProperty("lzaddress") == null) + System.out.println("lzaddress not provided - defaulting to '.'"); } @Test @@ -71,9 +79,9 @@ public void copyFile() throws Exception { String lzfile=System.currentTimeMillis() + "_csvtest.csv"; String hpccfilename="temp::" + lzfile; - client.createHPCCFile(lzfile, targetLZ, true); + client.createHPCCFile(lzfile, targetLZ, true, isContainerized ? null : targetLZAddress); byte[] data = "Product,SKU,Color\r\nBike,1234,Blue\r\nCar,2345,Red\r\n".getBytes(); - client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20); + client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20, isContainerized ? null : targetLZAddress); try { System.out.println("Starting file spray."); @@ -152,15 +160,33 @@ public void copyFile() throws Exception @Test public void AcreateHPCCFile() throws Exception, ArrayOfEspExceptionWrapper { - System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'"); - Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true)); + if (isContainerized) + { + System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'"); + System.out.println("Target HPCC is containerized, not providing targetLZAddress"); + Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true, null)); + } + else + { + System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' targetLZaddress: '" + targetLZAddress + "' on HPCC: '" + super.connString +"'"); + System.out.println("Target HPCC is NOT containerized, providing targetLZAddress"); + Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true, targetLZAddress)); + } } @Test public void BwriteHPCCFile() throws Exception, ArrayOfEspExceptionWrapper { + System.out.println("Writing data to file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'"); byte[] data = "HELLO MY DARLING, HELLO MY DEAR!1234567890ABCDEFGHIJKLMNOPQRSTUVXYZ".getBytes(); - Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20)); + if (isContainerized) + { + Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20, null)); + } + else + { + Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20, targetLZAddress)); + } } @Test @@ -168,7 +194,15 @@ public void CreadHPCCFile() throws Exception, ArrayOfEspExceptionWrapper { System.out.println("reading data from file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'"); byte[] data = "HELLO MY DARLING, HELLO MY DEAR!1234567890ABCDEFGHIJKLMNOPQRSTUVXYZ".getBytes(); - String response = client.readFileData(targetLZ, testfilename, data.length, 0); + String response = null; + if (isContainerized) + { + response = client.readFileData(targetLZ, testfilename, data.length, 0, null); + } + else + { + response = client.readFileData(targetLZ, testfilename, data.length, 0, targetLZAddress); + } Assert.assertNotNull(response); Assert.assertArrayEquals(data, response.getBytes()); }