From e9ef06a5025176a5d19745c48fb907ab537ba5ca Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Tue, 31 Oct 2023 17:28:16 +0100 Subject: [PATCH] add function S3Download --- build.number | 4 +- source/fld/function.fld | 87 +++++++- .../resource/s3/function/S3Download.java | 198 ++++++++++++++++++ .../resource/s3/function/S3Write.java | 18 +- tests/functions/S3Download.cfc | 109 ++++++++++ 5 files changed, 405 insertions(+), 11 deletions(-) create mode 100644 source/java/src/org/lucee/extension/resource/s3/function/S3Download.java create mode 100644 tests/functions/S3Download.cfc diff --git a/build.number b/build.number index 12ebe05..948df68 100644 --- a/build.number +++ b/build.number @@ -1,3 +1,3 @@ #Build Number for ANT. Do not edit! -#Thu Oct 26 10:22:47 CEST 2023 -build.number=3 +#Tue Oct 31 17:27:17 CET 2023 +build.number=4 diff --git a/source/fld/function.fld b/source/fld/function.fld index 547660d..f1d3984 100755 --- a/source/fld/function.fld +++ b/source/fld/function.fld @@ -665,7 +665,6 @@ force - bucket boolean No true @@ -709,7 +708,6 @@ S3Delete org.lucee.extension.resource.s3.function.S3Delete deletes a bucket or an object within a bucket. - deprecated bucketName bucket @@ -726,7 +724,6 @@ force - bucket boolean No true @@ -764,6 +761,90 @@ query + + + + + S3Download + org.lucee.extension.resource.s3.function.S3Download + Downloads an object from an S3 bucket. + It can save the object to a specified path, return its content directly, or process the content in parts through a provided closure or UDF (User-Defined Function). + + + bucketName + bucket + string + Yes + Name of the bucket to download. + + + objectName + object + string + Yes + Name of the object to download. + + + target + closure,udf,function,path,file + any + No + true + Optional. Defines the target where the downloaded data will be directed. + If a file path is provided, the data is saved to that path, the file path must be provided with help of the function "fileOpen" like this [fileOpen(path,"write")]. + If a closure or function is given, it will be invoked with parts of the downloaded data as its argument. + The function should accept a single argument named 'line' for line-by-line processing, + 'string{Number}' for string blocks of a specified size, + or 'binary{Number}' for binary blocks of a specified size. + + The function should return a boolean value: returning false will stop further reading from S3, + while true will continue the process. + + If this argument is omitted, the function returns the downloaded data directly. + + + + charset + string + No + + charset to use to store the content. + + + accessKeyId + accessKey + string + No + S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. + + + secretAccessKey + secretkey + string + No + S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. + + + host + provider,server + string + No + the provider to connect, if not set Amazon AWS is used. + + + timeout + number + No + 10000 + timeout for this execution + + + any + + + + + S3ListBucket diff --git a/source/java/src/org/lucee/extension/resource/s3/function/S3Download.java b/source/java/src/org/lucee/extension/resource/s3/function/S3Download.java new file mode 100644 index 0000000..f7cc089 --- /dev/null +++ b/source/java/src/org/lucee/extension/resource/s3/function/S3Download.java @@ -0,0 +1,198 @@ +package org.lucee.extension.resource.s3.function; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.Arrays; + +import org.lucee.extension.resource.s3.S3; +import org.lucee.extension.resource.s3.util.print; + +import com.amazonaws.services.s3.model.S3Object; + +import lucee.commons.io.res.Resource; +import lucee.loader.engine.CFMLEngine; +import lucee.loader.engine.CFMLEngineFactory; +import lucee.runtime.PageContext; +import lucee.runtime.exp.PageException; +import lucee.runtime.type.FunctionArgument; +import lucee.runtime.type.UDF; +import lucee.runtime.util.Cast; + +public class S3Download extends S3Function { + + private static final long serialVersionUID = 8926919958105910628L; + + public static final short TYPE_ANY = 0; + public static final short TYPE_BOOLEAN = 2; + public static final short TYPE_STRING = 7; + + public static final short MODE_LINE = 1; + public static final short MODE_BINARY = 2; + public static final short MODE_STRING = 4; + + @Override + public Object invoke(PageContext pc, Object[] args) throws PageException { + CFMLEngine eng = CFMLEngineFactory.getInstance(); + Cast cast = eng.getCastUtil(); + if (args.length > 8 || args.length < 2) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 2, 8, args.length); + + // required + String bucketName = cast.toString(args[0]); + String objectName = cast.toString(args[1]); + + // optional + Object target = args.length > 2 && args[2] != null ? args[2] : null; + Charset charset = args.length > 3 && args[3] != null ? cast.toCharset(cast.toString(args[3])) : null; + String accessKeyId = args.length > 4 && args[4] != null ? cast.toString(args[4]) : null; + String secretAccessKey = args.length > 5 && args[5] != null ? cast.toString(args[5]) : null; + String host = args.length > 6 && args[6] != null ? cast.toString(args[6]) : null; + double timeout = args.length > 7 && !isEmpty(args[7]) ? cast.toDoubleValue(args[7]) : 0; + + // validate + UDF targetUDF = null; + Resource targetRes = null; + int mode = 0; + int blockSize = 0; + if (target != null) { + if (target instanceof UDF) { + targetUDF = (UDF) target; + + // function return type + if (!(targetUDF.getReturnType() == TYPE_ANY || targetUDF.getReturnType() == TYPE_BOOLEAN)) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 3, + "target", "the function invoke of the component listener must have the return type boolean.", ""); + + // function invoke arguments + FunctionArgument[] udfArgs = targetUDF.getFunctionArguments(); + if (udfArgs.length < 1 || udfArgs.length > 1) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 3, "target", + "you need to define an argument for the closure/function passed in following this pattern (string line|binary{number}|string{number})", ""); + + FunctionArgument arg = udfArgs[0]; + if (!(arg.getType() == TYPE_ANY || arg.getType() == TYPE_STRING)) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 3, "target", + "the first argument of the closuere/function need to be defined as a string or no defintion at all", ""); + print.e("name:" + arg.getName()); + + String name = (arg.getName().getString() + "").toLowerCase().trim(); + if ("line".equals(name)) { + mode = MODE_LINE; + } + else if (name.startsWith("binary")) { + mode = MODE_BINARY; + blockSize = eng.getCastUtil().toIntValue(name.substring(6)); + if (blockSize <= 0) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 3, "target", + "invalid block size defintion with the argument [binary{Number}], blocksize need to be a positive number, so the argument name should for example look like this [binary1000] to get block size 1000", + ""); + } + else if (name.startsWith("string")) { + mode = MODE_STRING; + blockSize = eng.getCastUtil().toIntValue(name.substring(6)); + if (blockSize <= 0) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 3, "target", + "invalid block size defintion with the argument [string{Number}], blocksize need to be a positive number, so the argument name should for example look like this [string1000] to get block size 1000", + ""); + } + else { + throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 3, "target", + "the first argument of the closuere/function need to be define an argument where the name does follow one of this patterns [line, binary(Number), string(Number) ]", + ""); + } + + } + else if ((targetRes = S3Write.toResource(pc, target, false, null)) == null) { + // can also be a charset defintion + Charset tmp = charset == null ? eng.getCastUtil().toCharset(eng.getCastUtil().toString(target, null), null) : null; + if (tmp == null) throw eng.getExceptionUtil().createFunctionException(pc, "S3Download", 3, "target", + "the value of the argument needs to be a closure/function, a file name or not defined at all", ""); + charset = tmp; + } + } + + // create S3 Instance + try { + S3 s3 = S3.getInstance(toS3Properties(pc, accessKeyId, secretAccessKey, host), toTimeout(timeout)); + S3Object obj = s3.getData(bucketName, objectName); + Cast caster = eng.getCastUtil(); + // stream to UDF + if (targetUDF != null) { + // LINE + if (MODE_LINE == mode) { + BufferedReader reader = null; + try { + if (charset == null) charset = pc.getConfig().getResourceCharset(); + reader = new BufferedReader(new InputStreamReader(obj.getObjectContent(), charset)); + String line; + while ((line = reader.readLine()) != null) { + if (!caster.toBooleanValue(targetUDF.call(pc, new Object[] { line }, true))) return null; + } + return null; + } + finally { + eng.getIOUtil().closeSilent(reader); + } + } + // STRING + else if (MODE_STRING == mode) { + BufferedReader reader = null; + char[] buffer = new char[blockSize]; + try { + if (charset == null) charset = pc.getConfig().getResourceCharset(); + reader = new BufferedReader(new InputStreamReader(obj.getObjectContent(), charset)); + int numCharsRead; + while ((numCharsRead = reader.read(buffer, 0, buffer.length)) != -1) { + String block = new String(buffer, 0, numCharsRead); + if (!caster.toBooleanValue(targetUDF.call(pc, new Object[] { block }, true))) return null; + } + return null; + } + finally { + eng.getIOUtil().closeSilent(reader); + } + } + // BINARY + else { + InputStream input = null; + byte[] buffer = new byte[blockSize]; + try { + input = new BufferedInputStream(obj.getObjectContent()); + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + if (bytesRead == buffer.length) { + if (!caster.toBooleanValue(targetUDF.call(pc, new Object[] { buffer }, true))) return null; + } + else { + // create a smaller array for the last block, which might not be a full 1000 bytes + byte[] lastBlock = Arrays.copyOf(buffer, bytesRead); + targetUDF.call(pc, new Object[] { lastBlock }, true); + } + } + return null; + } + finally { + eng.getIOUtil().closeSilent(input); + } + } + + } + // store to file + else if (targetRes != null) { + eng.getIOUtil().copy(obj.getObjectContent(), targetRes, true); + return null; + } + // return the value + else { + if (charset == null) { + // TODO get mimetype info from S3 and decide based on this if we provide a byte array or a string + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + eng.getIOUtil().copy(obj.getObjectContent(), baos, true, true); + return baos.toByteArray(); + } + return eng.getIOUtil().toString(obj.getObjectContent(), charset); + } + } + catch (Exception e) { + throw eng.getCastUtil().toPageException(e); + } + } +} \ No newline at end of file diff --git a/source/java/src/org/lucee/extension/resource/s3/function/S3Write.java b/source/java/src/org/lucee/extension/resource/s3/function/S3Write.java index b17fd4e..6c6ada3 100644 --- a/source/java/src/org/lucee/extension/resource/s3/function/S3Write.java +++ b/source/java/src/org/lucee/extension/resource/s3/function/S3Write.java @@ -7,7 +7,6 @@ import org.lucee.extension.resource.s3.AccessControlListUtil; import org.lucee.extension.resource.s3.S3; import org.lucee.extension.resource.s3.S3Exception; -import org.lucee.extension.resource.s3.S3ResourceProvider; import lucee.commons.io.res.Resource; import lucee.loader.engine.CFMLEngine; @@ -69,9 +68,15 @@ else if (value instanceof Resource) { } public static Object toResource(PageContext pc, Object value) { + Resource res = toResource(pc, value, true, null); + if (res != null) return res; + return value; + } + + public static Resource toResource(PageContext pc, Object value, boolean needToExist, Resource defaultValue) { if (value instanceof CharSequence) { String str = value.toString(); - if (str.length() <= 10240) { + if (str.length() <= 10240 && needToExist) { try { return CFMLEngineFactory.getInstance().getResourceUtil().toResourceExisting(pc, str); } @@ -79,14 +84,15 @@ public static Object toResource(PageContext pc, Object value) { } } } - if (value instanceof Resource || value instanceof File) return value; + else if (value instanceof Resource) return (Resource) value; + else if (value instanceof File) return CFMLEngineFactory.getInstance().getCastUtil().toResource(value, defaultValue); // getResource try { Method m = value.getClass().getMethod("getResource", new Class[0]); if (m != null) { Object obj = m.invoke(value, new Object[0]); - if (obj instanceof Resource) return obj; + if (obj instanceof Resource) return (Resource) obj; } } catch (Exception e) { @@ -97,13 +103,13 @@ public static Object toResource(PageContext pc, Object value) { Method m = value.getClass().getMethod("getFile", new Class[0]); if (m != null) { Object obj = m.invoke(value, new Object[0]); - if (obj instanceof File) return obj; + if (obj instanceof File) return CFMLEngineFactory.getInstance().getCastUtil().toResource(obj, defaultValue); } } catch (Exception e) { } - return value; + return defaultValue; } @Override diff --git a/tests/functions/S3Download.cfc b/tests/functions/S3Download.cfc new file mode 100644 index 0000000..eb13e4f --- /dev/null +++ b/tests/functions/S3Download.cfc @@ -0,0 +1,109 @@ +component extends="org.lucee.cfml.test.LuceeTestCase" labels="s3" { + function run( testResults , testBox ) { + describe( title="Test suite for S3Download()",skip=Util::isAWSNotSupported(), body=function() { + + var cred=Util::getAWSCredentials(); + var bucketName=cred.PREFIX&"-download"&listFirst(replace(server.lucee.version,".","","all"),"-"); + var objectName="sub/test.txt"; + var content="Susi +Sorglos"; + if(!S3Exists( + bucketName:bucketName, objectName:objectName, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST))) { + S3Write( + value:content, + bucketName:bucketName, objectName:objectName, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)); + } + + it(title="download as binary", body = function( currentSpec ) { + var data=s3Download(bucket:bucketName,object:objectName,accessKeyId:cred.accessKeyId,secretAccessKey:cred.awsSecretKey); + assertTrue(isBinary(data)); + assertEquals(len(data),13); + assertEquals(toString(data), content); + }); + + it(title="download as string", body = function( currentSpec ) { + var data=s3Download(bucket:bucketName,object:objectName,charset:"UTF-8",accessKeyId:cred.accessKeyId,secretAccessKey:cred.awsSecretKey); + assertTrue(isSimpleValue(data)); + assertEquals(len(data),13); + assertEquals(data, content); + }); + + it(title="download to file", body = function( currentSpec ) { + var target=getDirectoryFromPath(getCurrentTemplatePath())&"temp.txt"; + try { + s3Download(bucket:bucketName,object:objectName,target:fileOpen(target,"write"),accessKeyId:cred.accessKeyId,secretAccessKey:cred.awsSecretKey); + var data=fileRead(target); + assertTrue(isSimpleValue(data)); + assertEquals(len(data),13); + assertEquals(data, content); + } + finally { + if(fileExists(target)) fileDelete(target); + } + }); + + it(title="download to UDF:line", body = function( currentSpec ) { + var data=""; + s3Download(bucket:bucketName,object:objectName,target:function(line){ + data&=line; + return false; + },accessKeyId:cred.accessKeyId,secretAccessKey:cred.awsSecretKey); + + assertTrue(isSimpleValue(data)); + assertEquals(len(data),4); + assertEquals(data, "Susi"); + }); + + it(title="download to UDF:line with charset", body = function( currentSpec ) { + var data=""; + s3Download(bucket:bucketName,object:objectName,charset:"UTF-8",target:function(line){ + data&=line; + return false; + },accessKeyId:cred.accessKeyId,secretAccessKey:cred.awsSecretKey); + + assertTrue(isSimpleValue(data)); + assertEquals(len(data),4); + assertEquals(data, "Susi"); + }); + + it(title="download to UDF:line with charset", body = function( currentSpec ) { + var data=""; + s3Download(bucket:bucketName,object:objectName,charset:"UTF-8",target:function(string4){ + data&=string4&":"&len(string4)&";"; + return true; + },accessKeyId:cred.accessKeyId,secretAccessKey:cred.awsSecretKey); + + assertTrue(isSimpleValue(data)); + assertEquals(len(data),4); + assertEquals(data, "Susi"); + }); + + it(title="download to UDF:string with charset", body = function( currentSpec ) { + var data=""; + s3Download(bucket:bucketName,object:objectName,charset:"UTF-8",target:function(string4){ + data&=string4&":"&len(string4)&";"; + return true; + },accessKeyId:cred.accessKeyId,secretAccessKey:cred.awsSecretKey); + + assertTrue(isSimpleValue(data)); + assertEquals(len(data),4); + assertEquals(data, "Susi"); + }); + + it(title="download to UDF:binary", body = function( currentSpec ) { + var data=""; + s3Download(bucket:bucketName,object:objectName,charset:"UTF-8",target:function(binary4){ + data&=len(binary4)&";"; + return true; + },accessKeyId:cred.accessKeyId,secretAccessKey:cred.awsSecretKey); + + assertTrue(isSimpleValue(data)); + assertEquals(len(data),4); + assertEquals(data, "Susi"); + }); + + }); + } +} \ No newline at end of file