From 0010427abf2e682d9503cf210fcb37a5a0f1cfd8 Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Tue, 10 Oct 2023 13:36:21 +0200 Subject: [PATCH] LDEV-4721 - add Harakiri controller thread --- build.number | 4 +- build.properties | 2 +- .../org/lucee/extension/resource/s3/S3.java | 55 ++++++++----------- 3 files changed, 25 insertions(+), 36 deletions(-) diff --git a/build.number b/build.number index dca3ce2..00f8a15 100644 --- a/build.number +++ b/build.number @@ -1,3 +1,3 @@ #Build Number for ANT. Do not edit! -#Fri Oct 06 12:02:49 CEST 2023 -build.number=26 +#Tue Oct 10 13:30:49 CEST 2023 +build.number=1 diff --git a/build.properties b/build.properties index 15bd28d..d337837 100644 --- a/build.properties +++ b/build.properties @@ -1,6 +1,6 @@ bundlename: s3.extension filename: s3-extension -bundleversion: 2.0.1. +bundleversion: 2.0.2. id: 17AB52DE-B300-A94B-E058BD978511E39E luceeCoreVersion: 5.0.0.157 releaseType: server diff --git a/source/java/src/org/lucee/extension/resource/s3/S3.java b/source/java/src/org/lucee/extension/resource/s3/S3.java index 85f8972..fce3c54 100755 --- a/source/java/src/org/lucee/extension/resource/s3/S3.java +++ b/source/java/src/org/lucee/extension/resource/s3/S3.java @@ -66,6 +66,7 @@ import lucee.loader.engine.CFMLEngine; import lucee.loader.engine.CFMLEngineFactory; import lucee.loader.util.Util; +import lucee.runtime.config.Config; import lucee.runtime.exp.PageException; import lucee.runtime.net.s3.Properties; import lucee.runtime.type.Array; @@ -102,7 +103,6 @@ public class S3 { public static final String[] PROVIDERS = new String[] { AWS, WASABI, BACKBLAZE, ".digitaloceanspaces.com", DREAM_IO, GOOGLE }; private static final ConcurrentHashMap tokens = new ConcurrentHashMap(); - private static final int CHECK_INTERVALL = 1000; private static Map instances = new ConcurrentHashMap(); private static Map caches = new ConcurrentHashMap(); @@ -115,25 +115,26 @@ public class S3 { private final long cacheTimeout; private final long liveTimeout; - private int existCheckIntervall = 0; private final Cache cache; private Log log; /////////////////////// CACHE //////////////// - private static class Cache { - - public Cache() { - regions.put("US", RegionFactory.US_EAST_1); - } + static class Cache { + private final Harakiri harakiri; private ValidUntilMap buckets; private Map existBuckets; private final Map> objects = new ConcurrentHashMap>(); private Map> accessControlLists = new ConcurrentHashMap>(); private Map regions = new ConcurrentHashMap(); private final Map bucketRegions = new ConcurrentHashMap(); - private Map exists = new ConcurrentHashMap(); + Map exists = new ConcurrentHashMap(); + + public Cache(Log log) { + regions.put("US", RegionFactory.US_EAST_1); + harakiri = new Harakiri(this, log); + } } public static S3 getInstance(S3Properties props, long cache) { @@ -144,15 +145,17 @@ public static S3 getInstance(S3Properties props, long cache) { synchronized (instances) { s3 = instances.get(keyS3); if (s3 == null) { - // print.ds("s3:" + key); - // same cache for all locations + String keyCache = props.getAccessKeyId() + ":" + props.getSecretAccessKey() + ":" + props.getHostWithoutRegion() + ":" + cache; Cache c = caches.get(keyCache); if (c == null) { synchronized (caches) { + Log log = null; + Config config = CFMLEngineFactory.getInstance().getThreadConfig(); + if (config != null) log = config.getLog("application"); c = caches.get(keyCache); if (c == null) { - caches.put(keyCache, c = new Cache()); + caches.put(keyCache, c = new Cache(log)); } } } @@ -176,7 +179,6 @@ public static S3 getInstance(S3Properties props, long cache) { */ private S3(Cache cache, String accessKeyId, String secretAccessKey, String host, String defaultLocation, long cacheTimeout, long liveTimeout, boolean cacheRegions, Log log) { this.cache = cache; - this.accessKeyId = accessKeyId; this.secretAccessKey = secretAccessKey; this.host = host; @@ -946,12 +948,12 @@ private ValidUntilMap _list(String bucketName, String objectName, boolea if (!hasObjName || name.equals(nameFile) || name.startsWith(nameDir)) _list.put(kid.getKey(), tmp); cache.exists.put(toKey(bucketName, name), tmp); - _flush(cache.exists); + cache.harakiri.touch(); int index; while ((index = name.lastIndexOf('/')) != -1) { name = name.substring(0, index); cache.exists.put(toKey(bucketName, name), new ParentObject(this, bucketName, name, validUntil, log)); - _flush(cache.exists); + cache.harakiri.touch(); } } @@ -1144,7 +1146,7 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception targetName = summary.getKey(); if (nameFile.equals(targetName) || nameDir.equals(targetName)) { cache.exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = summary, validUntil, log)); - _flush(cache.exists); + cache.harakiri.touch(); } // pseudo directory? @@ -1152,13 +1154,13 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception targetName = summary.getKey(); if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) { cache.exists.put(toKey(bucketName, nameFile), info = new ParentObject(this, bucketName, nameDir, validUntil, log)); - _flush(cache.exists); + cache.harakiri.touch(); } // set the value to exist when not a match if (!(stoObj != null && stoObj.equals(summary))) { cache.exists.put(toKey(bucketName, summary.getKey()), new StorageObjectWrapper(this, summary, validUntil, log)); - _flush(cache.exists); + cache.harakiri.touch(); } // set all the parents when not exist // TODO handle that also a file with that name can exist at the same time @@ -1167,7 +1169,7 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception while ((index = parent.lastIndexOf('/')) != -1) { parent = parent.substring(0, index); cache.exists.put(toKey(bucketName, parent), new ParentObject(this, bucketName, parent, validUntil, log)); - _flush(cache.exists); + cache.harakiri.touch(); } } @@ -1181,7 +1183,7 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception // does // not exis ); - _flush(cache.exists); + cache.harakiri.touch(); } return info; } @@ -1547,20 +1549,6 @@ else if (isS3Info.booleanValue() && ((S3Info) e.getValue()).validUntil() < now) } } - private void _flush(Map map) { - if (++existCheckIntervall < CHECK_INTERVALL) return; - existCheckIntervall = 0; - Iterator> it = map.entrySet().iterator(); - Entry e; - long now = System.currentTimeMillis(); - while (it.hasNext()) { - e = it.next(); - if (e.getValue().validUntil() < now) { - map.remove(e.getKey()); - } - } - } - /** * * @param bucketName target bucket to create if not already exists @@ -2717,4 +2705,5 @@ public void shutdown() throws S3Exception { public long getLiveTimeout() { return liveTimeout; } + } \ No newline at end of file