fix custom s3 host handling LDEV-3594

.github/workflows/main.yml

@@ -3,7 +3,9 @@
 
 name: Java CI
 
-on: [push, pull_request,workflow_dispatch]
+on: [push, pull_request, workflow_dispatch]
+
+concurrency: s3_extension
 
 jobs:
  build:
@@ -61,7 +63,10 @@ jobs:
  testLabels: s3
  testAdditional: ${{ github.workspace }}/tests
  S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_ID_TEST }}
- S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY_TEST }}
+ SS3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY_TEST }}
  S3_CUSTOM_ACCESS_KEY_ID: "minioadmin"
  S3_CUSTOM_SECRET_KEY: "minioadmin"
- S3_CUSTOM_HOST: "localhost:9000"
+ S3_CUSTOM_HOST: "http://localhost:9000"
+ S3_GOOGLE_ACCESS_KEY_ID: ${{ secrets.S3_GOOGLE_SECRET_KEY }}
+ SS3_GOOGLE_SECRET_KEY: ${{ secrets.S3_GOOGLE_ACCESS_KEY_ID }}
+ S3_GOOGLE_HOST: "storage.googleapis.com"

build.xml

@@ -92,7 +92,7 @@ resource: "[{'class':'${class}','bundleName':'${bundlename}','bundleVersion':'${
  <target name="compile" depends="copy"
  description="compile the source " >
  <!-- Compile ACF-Infinspan source -->
- <javac srcdir="${temp}" target="1.7" destdir="${build}" debug="true" debuglevel="lines,vars,source">
+ <javac srcdir="${temp}" target="1.8" source="1.8" destdir="${build}" debug="true" debuglevel="lines,vars,source">
  <classpath refid="classpath" />
  </javac>
  </target>

source/java/src/org/lucee/extension/resource/s3/S3.java

@@ -136,7 +136,7 @@ public S3Bucket createDirectory(String bucketName, AccessControlList acl, String
  }
  catch (ServiceException se) {
  throw toS3Exception(se, "could not create the bucket [" + bucketName
- + "], please consult the following website to learn about Bucket Restrictions and limitations: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html");
+ + "] at [" + host + "], please consult the following website to learn about Bucket Restrictions and limitations: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html");
  }
  catch (FactoryConfigurationError fce) {
  XMLUtil.validateDocumentBuilderFactory();
@@ -314,8 +314,11 @@ public List<S3Info> list(boolean recursive, boolean listPseudoFolder) throws S3E
  * list all allements in a specific bucket
  * 
  * @param bucketName name of the bucket
+ * @param objectName name of the object
  * @param recursive show all objects (recursive==true) or direct kids
  * @param listPseudoFolder if recursive false also list the "folder" of objects with sub folders
+ * @param onlyChildren 
+ * @param noCache 
  * @return
  * @throws S3Exception
  * 
@@ -423,8 +426,8 @@ private ValidUntilMap<S3Info> _list(String bucketName, String objectName, boolea
  // not cached
  ValidUntilMap<S3Info> _list = timeout <= 0 || noCache ? null : objects.get(key);
  if (_list == null || _list.validUntil < System.currentTimeMillis()) {
+ /*
  S3Object[] kids = hasObjName ? getS3Service().listObjects(bucketName, nameFile, ",") : getS3Service().listObjects(bucketName);
-
  long validUntil = System.currentTimeMillis() + timeout;
  _list = new ValidUntilMap<S3Info>(validUntil);
  objects.put(key, _list);
@@ -450,13 +453,71 @@ private ValidUntilMap<S3Info> _list(String bucketName, String objectName, boolea
  exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil));
  }
  }
+ */
+
+ /* ------------------------ chunked, which includes prefixes ------------------------------ */
+ StorageObjectsChunk chunk = hasObjName ? 
+ listObjectsChunkedAll(bucketName, nameFile, ",", -1)
+ : listObjectsChunkedAll(bucketName, null, null, -1);
+ StorageObject[] _objects = chunk == null ? null : chunk.getObjects();
+ S3Info info;
+
+ long validUntil = System.currentTimeMillis() + timeout;
+ int index;
+ _list = new ValidUntilMap<S3Info>(validUntil);
+ objects.put(key, _list);
+
+ // add bucket
+ if (!hasObjName && !onlyChildren) {
+ S3Bucket b = getS3Service().getBucket(bucketName);
+ _list.put("", new S3BucketWrapper(b, validUntil));
+ }
+
+ ArrayList<String> commonPrefixes = new ArrayList();
+ if (chunk != null && chunk.getCommonPrefixes().length > 0){
+ commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
+ for (String cp: commonPrefixes){
+ info = new ParentObject(bucketName, cp, null, validUntil);
+ if (!hasObjName || cp.equals(nameFile) || cp.startsWith(nameDir)) _list.put(cp, info);
+ exists.put(toKey(bucketName, cp), info);
+ // add parent pseudo folders
+ while ((index = cp.lastIndexOf('/')) != -1) {
+ cp = cp.substring(0, index);
+ exists.put(toKey(bucketName, cp), new ParentObject(bucketName, cp, null, validUntil));
+ }
+ }
+ }
+ if ( _objects != null ){
+ String name;
+ StorageObjectWrapper tmp;
+ StorageObject stoObj = null;
+ // direct match
+ for (StorageObject kids: _objects) {
+ name = kids.getName();
+ tmp = new StorageObjectWrapper(this, stoObj = kids, bucketName, validUntil);
+
+ if (!hasObjName || name.equals(nameFile) || name.startsWith(nameDir)) _list.put(kids.getKey(), tmp);
+ exists.put(toKey(kids.getBucketName(), name), tmp);
+
+ // add parent pseudo folders
+ while ((index = name.lastIndexOf('/')) != -1) {
+ name = name.substring(0, index);
+ exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil));
+ }
+ }
+ }
 
  }
  return _list;
  }
- catch (ServiceException se) {
- throw toS3Exception(se);
+ catch (FactoryConfigurationError fce) {
+ XMLUtil.validateDocumentBuilderFactory();
+ throw fce;
+ }
+ catch (Exception e) {
+ e.printStackTrace();
  }
+ return null;
  }
 
  private String toKey(String bucketName, String objectName) {
@@ -560,41 +621,56 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception
  long validUntil = System.currentTimeMillis() + timeout;
  StorageObject[] objects = chunk == null ? null : chunk.getObjects();
 
- if (objects == null || objects.length == 0) {
- exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it
- // does
- return null;
+ if (chunk !=null && chunk.getCommonPrefixes().length > 0){
+ ArrayList<String> commonPrefixes = new ArrayList();
+ int index;
+ commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
+ for (String cp: commonPrefixes){
+ while ((index = cp.lastIndexOf('/')) != -1) {
+ cp = cp.substring(0, index);
+ if (cp.equals(nameFile) || cp.equals(nameDir))
+ exists.put(toKey(bucketName, cp), info = new ParentObject(bucketName, cp, null, validUntil));
+ else
+ exists.put(toKey(bucketName, cp), new ParentObject(bucketName, cp, null, validUntil));
+ }
+ }
  }
 
- String targetName;
- StorageObject stoObj = null;
- // direct match
- for (StorageObject so: objects) {
- targetName = so.getName();
- if (nameFile.equals(targetName) || nameDir.equals(targetName)) {
- exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = so, bucketName, validUntil));
- }
+ if (info == null && (objects == null || objects.length == 0)) {
+ exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it 
  }
 
- // pseudo directory?
- if (info == null) {
+ if (objects != null){
+ String targetName;
+ StorageObject stoObj = null;
+ // direct match
  for (StorageObject so: objects) {
  targetName = so.getName();
- if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) {
- exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil));
+ if (nameFile.equals(targetName) || nameDir.equals(targetName)) {
+ exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = so, bucketName, validUntil));
  }
  }
- }
 
- for (StorageObject obj: objects) {
- if (stoObj != null && stoObj.equals(obj)) continue;
- exists.put(toKey(obj.getBucketName(), obj.getName()), new StorageObjectWrapper(this, obj, bucketName, validUntil));
- }
+ // pseudo directory?
+ if (info == null) {
+ for (StorageObject so: objects) {
+ targetName = so.getName();
+ if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) {
+ exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil));
+ }
+ }
+ }
+
+ for (StorageObject obj: objects) {
+ if (stoObj != null && stoObj.equals(obj)) continue;
+ exists.put(toKey(obj.getBucketName(), obj.getName()), new StorageObjectWrapper(this, obj, bucketName, validUntil));
+ }
 
- if (info == null) {
- exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil) // we do not return this, we just store it to cache that it does
- // not exis
- );
+ if (info == null) {
+ exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil) // we do not return this, we just store it to cache that it does
+ // not exis
+ );
+ }
  }
  return info;
  }
@@ -621,6 +697,19 @@ public StorageObjectsChunk listObjectsChunkedSilent(String bucketName, String ob
  return null;
  }
 
+ public StorageObjectsChunk listObjectsChunkedAll(String bucketName, String objectName, String delim, int max) {
+ try {
+ return getS3Service().listObjectsChunked(bucketName, objectName, delim, max, "", true);
+ }
+ catch (FactoryConfigurationError fce) {
+ XMLUtil.validateDocumentBuilderFactory();
+ throw fce;
+ }
+ catch (Exception e) {
+ }
+ return null;
+ }
+
  public S3BucketInfo get(String bucketName) throws S3Exception {
  bucketName = improveBucketName(bucketName);
 
@@ -691,9 +780,14 @@ public void delete(String bucketName, String objectName, boolean force) throws S
  }
 
  ObjectKeyAndVersion[] keys = toObjectKeyAndVersions(list, null);
- if (!force
- && (keys.length > 1 || (keys.length == 1 && keys[0].getKey().length() > nameDir.length() && keys[0].getKey().substring(nameDir.length()).indexOf('/') != -1))) {
- throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty");
+ if (!force){
+ // TODO not sure of the logic here, why was it checking only for a sub directory?
+ if (keys.length > 1 || (keys.length == 1 
+ && keys[0].getKey().length() > nameDir.length() 
+ && keys[0].getKey().substring(nameDir.length()).length() > 0) //indexOf('/') != -1)
+ ) {
+ throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty");
+ }
  }
 
  // clear cache
@@ -1278,9 +1372,27 @@ private S3Service getS3Service() {
  synchronized (getToken(accessKeyId + ":" + secretAccessKey)) {
  if (service == null) {
  final Jets3tProperties props = Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME);
- if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) {
- props.setProperty("s3service.s3-endpoint", host);
- }
+ props.clearAllProperties();
+ //if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) {
+ int hasPort = host.indexOf(':');
+ if (hasPort == -1) {
+ props.setProperty("s3service.s3-endpoint", host); // no port, no protocol
+ } else {
+ int hasHttp = host.toLowerCase().indexOf("http", 0); // http(s)://localhost:9000 prefix?
+ if (hasHttp == -1){ // no http prefix
+ props.setProperty("s3service.s3-endpoint", host.substring(0, hasPort));
+ props.setProperty("s3service.s3-endpoint-http-port", host.substring(hasPort + 1));
+ } else { // with http(s) prefix
+ String protocol = host.substring(0,host.indexOf(":"));
+ props.setProperty("s3service.https-only", protocol.equalsIgnoreCase("http") ? "false" : "true" );
+ String _host = host.substring(host.lastIndexOf("/") + 1);
+ hasPort = _host.indexOf(':');
+ props.setProperty("s3service.s3-endpoint", _host.substring(0, hasPort));
+ props.setProperty("s3service.s3-endpoint-http-port", _host.substring(hasPort + 1));
+ props.setProperty("s3service.disable-dns-buckets", "true");
+ }
+ }
+ //}
  service = new RestS3Service(new AWSCredentials(accessKeyId, secretAccessKey), null, null, props);
  }
  }

source/java/src/org/lucee/extension/resource/s3/S3Resource.java

@@ -182,7 +182,11 @@ else if (!Util.isEmpty(s3.getMappingName(), true)) {
  sb.append(s3.getMappingName()).append("@");
  }
 
- if (doHost) sb.append(s3.getHost());
+ if (doHost && s3.getCustomCredentials()){ 
+ if (sb.substring(sb.length()-1) != "@")
+ sb.append("@");
+ sb.append(s3.getHost());
+ }
 
  return sb.toString();
  }

source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java

@@ -167,9 +167,9 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties,
  storage.setValue(defaultLocation);
 
  int atIndex = path.indexOf('@');
- int slashIndex = path.indexOf('/');
+ int slashIndex = path.indexOf('/', atIndex); // secret keys can contain /
  if (slashIndex == -1) {
- slashIndex = path.length();
+ //slashIndex = path.length(); // never used!
  path += "/";
  }
  int index;
@@ -215,23 +215,35 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties,
  //
  }
  }
- path = prettifyPath(path.substring(atIndex + 1));
- index = path.indexOf('/');
- properties.setHost(host);
- if (index == -1) {
- if (path.equalsIgnoreCase(S3.DEFAULT_HOST) || path.equalsIgnoreCase(host)) {
- properties.setHost(path);
- path = "/";
- }
- }
- else {
- String _host = path.substring(0, index);
- if (_host.equalsIgnoreCase(S3.DEFAULT_HOST) || _host.equalsIgnoreCase(host)) {
- properties.setHost(_host);
- path = path.substring(index);
+ String srcPath = path;
+ if (atIndex == -1){
+ // no host information, skip parsing, ie. s3://bucketname/ etc
+ path = prettifyPath(path);
+ } else if ( (slashIndex - atIndex) == 1 ){ 
+ // key/secret@/bucket 
+ path = prettifyPath(path.substring(atIndex + 1));
+ } else {
+ int doubleIndex = path.indexOf("://", atIndex); // do we have a http(s)://
+ if (doubleIndex > 0) {
+ // key/secret@http://localhost:9000/bucket 
+ index = path.indexOf('/', doubleIndex+3 );
+ host = path.substring(atIndex+1, index);
+ path = prettifyPath(path.substring(index));
+ } else {
+ path = prettifyPath(path.substring(atIndex + 1));
+ index = path.indexOf('/');
+ if (index == -1) {
+ path = "/";
+ } else if ( (slashIndex - atIndex) != 1 ) { 
+ // key/[email protected]/bucket 
+ host = path.substring(0, index); 
+ path = path.substring(index);
+ }
  }
  }
-
+ if (host.endsWith("/")) 
+ host = host.substring(0, host.length() - 1); // strip trailing / after hostname with http://localhost:9000/
+ properties.setHost(host); 
  // get from system.properties/env.var
  if (Util.isEmpty(accessKeyId, true)) {
  if (Util.isEmpty(mappingName)) {
@@ -350,4 +362,4 @@ public Map getArguments() {
  public char getSeparator() {
  return '/';
  }
-}
+}