From c8377a906669cb118763090a0e0bbe3d1436ea04 Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Wed, 8 Nov 2023 17:43:17 +0100 Subject: [PATCH] LDEV-4750 - add the function S3Upload --- build.number | 4 +- source/fld/function.fld | 150 +++++++++++++----- .../resource/s3/function/S3Upload.java | 50 ++++++ tests/functions/S3Upload.cfc | 112 +++++++++++++ 4 files changed, 275 insertions(+), 41 deletions(-) create mode 100644 source/java/src/org/lucee/extension/resource/s3/function/S3Upload.java create mode 100644 tests/functions/S3Upload.cfc diff --git a/build.number b/build.number index d02e280..3a689b0 100644 --- a/build.number +++ b/build.number @@ -1,3 +1,3 @@ #Build Number for ANT. Do not edit! -#Wed Nov 01 18:24:40 CET 2023 -build.number=6 +#Wed Nov 08 17:40:32 CET 2023 +build.number=7 diff --git a/source/fld/function.fld b/source/fld/function.fld index f1d3984..2ff592e 100755 --- a/source/fld/function.fld +++ b/source/fld/function.fld @@ -64,14 +64,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -133,14 +133,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -216,14 +216,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -288,14 +288,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -371,14 +371,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -423,14 +423,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -482,14 +482,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -533,14 +533,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -563,6 +563,78 @@ binary + + + + S3Upload + org.lucee.extension.resource.s3.function.S3Upload + Uploads a file to S3. + + bucketName + bucket + string + Yes + Name of the bucket of your object to write + + + objectName + object + string + Yes + Name of the object (path) within the bucket of your object to write + + + source + object + yes + Source file to write to S3. A string path to a file or a object created with the function "fileOpen". + + + acl + aclObject + object + No + an array of struct where each struct represents an ACL grant + + + location + region + string + No + location of the S3 storage. + + + accessKeyId + accessKey,awsAccessKeyId,awsAccessKey + string + No + S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. + + + secretAccessKey + secretkey,awsSecretkey,awsSecretAccessKey + string + No + S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. + + + host + provider,server + string + No + the provider to connect, if not set Amazon AWS is used. + + + timeout + number + No + 10000 + timeout for this execution + + + void + + S3Write @@ -619,14 +691,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -646,7 +718,7 @@ timeout for this execution - binary + void @@ -672,14 +744,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -731,14 +803,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -812,14 +884,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -859,14 +931,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -898,14 +970,14 @@ List all buckets accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -950,14 +1022,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -1032,14 +1104,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -1114,14 +1186,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -1243,14 +1315,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. @@ -1321,14 +1393,14 @@ accessKeyId - accessKey + accessKey,awsAccessKeyId,awsAccessKey string No S3 accessKeyId, if not defined it checks the system property/environment variable for [lucee.s3.accesskeyid]. secretAccessKey - secretkey + secretkey,awsSecretkey,awsSecretAccessKey string No S3 secretAccessKey, if not defined it checks the system property/environment variable for [lucee.s3.secretaccesskey]. diff --git a/source/java/src/org/lucee/extension/resource/s3/function/S3Upload.java b/source/java/src/org/lucee/extension/resource/s3/function/S3Upload.java new file mode 100644 index 0000000..fc579a1 --- /dev/null +++ b/source/java/src/org/lucee/extension/resource/s3/function/S3Upload.java @@ -0,0 +1,50 @@ +package org.lucee.extension.resource.s3.function; + +import org.lucee.extension.resource.s3.AccessControlListUtil; +import org.lucee.extension.resource.s3.S3; + +import lucee.commons.io.res.Resource; +import lucee.loader.engine.CFMLEngine; +import lucee.loader.engine.CFMLEngineFactory; +import lucee.runtime.PageContext; +import lucee.runtime.exp.PageException; +import lucee.runtime.util.Cast; + +public class S3Upload extends S3Function { + + private static final long serialVersionUID = 4408651588846605652L; + + @Override + public Object invoke(PageContext pc, Object[] args) throws PageException { + CFMLEngine eng = CFMLEngineFactory.getInstance(); + Cast cast = eng.getCastUtil(); + if (args.length > 9 || args.length < 3) throw eng.getExceptionUtil().createFunctionException(pc, "S3Upload", 3, 9, args.length); + + // required + String bucketName = cast.toString(args[0]); + String objectName = cast.toString(args[1]); + Resource source = S3Write.toResource(pc, args[2], true, null); + if (source == null) throw eng.getExceptionUtil().createFunctionException(pc, "S3Upload", 3, "source", "could not convert the given source to a file", ""); + else if (!source.isFile()) throw eng.getExceptionUtil().createFunctionException(pc, "S3Upload", 3, "source", "the file [" + source + "] does not exist", ""); + + // optional + Object objACL = args.length > 3 && args[3] != null ? args[3] : null; + String location = args.length > 4 && !isEmpty(args[4]) ? cast.toString(args[4]) : null; + String accessKeyId = args.length > 5 && args[5] != null ? cast.toString(args[5]) : null; + String secretAccessKey = args.length > 6 && args[6] != null ? cast.toString(args[6]) : null; + String host = args.length > 7 && args[7] != null ? cast.toString(args[7]) : null; + double timeout = args.length > 8 && !isEmpty(args[8]) ? cast.toDoubleValue(args[8]) : 0; + + // validate + Object acl = null; + try { + acl = objACL != null ? AccessControlListUtil.toAccessControlList(objACL) : null; + S3 s3 = S3.getInstance(toS3Properties(pc, accessKeyId, secretAccessKey, host), toTimeout(timeout)); + s3.write(bucketName, objectName, source, acl, location); + } + catch (Exception e) { + throw eng.getCastUtil().toPageException(e); + } + return null; + } +} \ No newline at end of file diff --git a/tests/functions/S3Upload.cfc b/tests/functions/S3Upload.cfc new file mode 100644 index 0000000..8122223 --- /dev/null +++ b/tests/functions/S3Upload.cfc @@ -0,0 +1,112 @@ +component extends="org.lucee.cfml.test.LuceeTestCase" labels="s3" { + function run( testResults , testBox ) { + describe( title="Test suite for S3Write()", body=function() { + it(title="check with blackbaze",skip=Util::isBackBlazeNotSupported(), body = function( currentSpec ) { + testit(Util::getBackBlazeCredentials()); + }); + + it(title="check with amazon",skip=Util::isAWSNotSupported(), body = function( currentSpec ) { + testit(Util::getAWSCredentials()); + }); + + it(title="check with wasabi",skip=Util::isWasabiNotSupported(), body = function( currentSpec ) { + testit(Util::getWasabiCredentials()); + }); + + it(title="check with google",skip=Util::isGoogleNotSupported(), body = function( currentSpec ) { + testit(Util::getGoogleCredentials()); + }); + + }); + } + + + private function testit(cred) { + + var file1=expandPath("{temp-directory}/test-s3upload1.txt"); + fileWrite(file1, "Susi Sorglos 1"); + var file2=expandPath("{temp-directory}/test-s3upload2.txt"); + fileWrite(file2, "Susi Sorglos 2"); + + + try { + // create variables + var bucketName=cred.PREFIX&"-upload"&listFirst(replace(server.lucee.version,".","","all"),"-"); + var objectNameFile1="sub/file1.txt"; + var objectNameFile2="sub/file2.txt"; + + Util::deleteIfExists(cred,bucketName,objectNameFile1); + Util::deleteIfExists(cred,bucketName,objectNameFile2); + + + // create string file source + if(!S3Exists( + bucketName:bucketName, objectName:objectNameFile1, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST))) { + S3Upload( + source:file1, + bucketName:bucketName, objectName:objectNameFile1, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)); + } + + // create file source + if(!S3Exists( + bucketName:bucketName, objectName:objectNameFile2, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST))) { + S3Upload( + source:fileOpen(file2), + bucketName:bucketName, objectName:objectNameFile2, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)); + } + + // existing bucket + assertTrue( + S3Exists( + bucketName:bucketName, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)) + ); + + + // existing bucket/object + assertTrue( + S3Exists( + bucketName:bucketName, objectName:objectNameFile1, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)) + ); + + assertEquals( + "Susi Sorglos 1", + S3Read( + bucketName:bucketName, objectName:objectNameFile1, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)) + ); + + assertTrue( + S3Exists( + bucketName:bucketName, objectName:objectNameFile2, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)) + ); + + assertEquals( + "Susi Sorglos 2", + S3Read( + bucketName:bucketName, objectName:objectNameFile2, + accessKeyId:cred.ACCESS_KEY_ID, secretAccessKey:cred.SECRET_KEY, host:(isNull(cred.HOST)?nullvalue():cred.HOST)) + ); + + + } + catch(e) { + if(!findNoCase("Transaction cap exceeded", e.message) ) throw e; + } + finally { + Util::deleteBucketEL(cred,bucketName); + } + } + + + private function doFind(value){ + return value EQ "world"; + } + +} \ No newline at end of file