#110 - Additional Sync Strategy Using Last Modified Date and Size (#111)

* Initial implementation of comparison strategy code.

* Added tests and fixed small bug exposed by test.

* Upped version number and introduced mockito for testing.

* Updated documentation.

README.md

@@ -8,7 +8,10 @@ Designed to be lightning-fast and highly concurrent, with modest CPU and memory
 An object will be copied if and only if at least one of the following holds true:
 
 * The object does not exist in the destination bucket.
-* The size or ETag of the object in the destination bucket are different from the size/ETag in the source bucket.
+* The "sync strategy" triggers (by default uses the Etag sync strategy)
+    * Etag Strategy (Default): If the size or Etags don't match between the source and destination bucket.
+    * Size Strategy: If the sizes don't match between the source and destination bucket.
+    * Size and Last Modified Strategy: If the source and destination objects have a different size, or the source bucket object has a more recent last modified date. 
 
 When copying, the source metadata and ACL lists are also copied to the destination object.
 
@@ -89,6 +92,8 @@ I encourage you to port them to the 2.x branch, if you have the ability.
     -s (--ssl)                    : Use SSL for all S3 api operations (default false)
     -E (--server-side-encryption) : Enable AWS managed server-side encryption (default false)
     -l (--storage-class)		  : S3 storage class "Standard" or "ReducedRedundancy" (default Standard)
+    -S (--size-only)              : Only takes size of objects in consideration when determining if a copy is required.
+    -L (--size-and-last-modified) : Uses size and last modified to determine if files have change like the AWS CLI and ignores etags. If -S (--size-only) is also specified that strategy is selected over this strategy.
 
 
 ### Examples

pom.xml

@@ -11,7 +11,7 @@
 
     <groupId>org.cobbzilla</groupId>
     <artifactId>s3s3mirror</artifactId>
-    <version>1.2.7-SNAPSHOT</version>
+    <version>1.2.8-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <licenses>
@@ -81,6 +81,13 @@
             <version>3.11</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>3.7.7</version>
+            <scope>test</scope>
+        </dependency>
+
 
         <!-- command line argument handling -->
         <dependency>

s3s3mirror.bat

@@ -1,2 +1,2 @@
 @echo off
-java -Dlog4j.configuration=file:target/classes/log4j.xml -Ds3s3mirror.version=1.2.7 -jar target/s3s3mirror-1.2.7-SNAPSHOT.jar %*
+java -Dlog4j.configuration=file:target/classes/log4j.xml -Ds3s3mirror.version=1.2.8 -jar target/s3s3mirror-1.2.7-SNAPSHOT.jar %*

s3s3mirror.sh

@@ -2,7 +2,7 @@
 
 THISDIR=$(cd "$(dirname $0)" && pwd)
 
-VERSION=1.2.7
+VERSION=1.2.8
 JARFILE="${THISDIR}/target/s3s3mirror-${VERSION}-SNAPSHOT.jar"
 VERSION_ARG="-Ds3s3mirror.version=${VERSION}"
 

src/main/java/org/cobbzilla/s3s3mirror/CopyMaster.java

@@ -2,23 +2,28 @@
 
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.cobbzilla.s3s3mirror.comparisonstrategies.ComparisonStrategy;
+import org.cobbzilla.s3s3mirror.comparisonstrategies.ComparisonStrategyFactory;
+import org.cobbzilla.s3s3mirror.comparisonstrategies.SizeOnlyComparisonStrategy;
 
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 
 public class CopyMaster extends KeyMaster {
+    private final ComparisonStrategy comparisonStrategy;
 
     public CopyMaster(AmazonS3Client client, MirrorContext context, BlockingQueue<Runnable> workQueue, ThreadPoolExecutor executorService) {
         super(client, context, workQueue, executorService);
+        comparisonStrategy = ComparisonStrategyFactory.getStrategy(context.getOptions());
     }
 
     protected String getPrefix(MirrorOptions options) { return options.getPrefix(); }
     protected String getBucket(MirrorOptions options) { return options.getSourceBucket(); }
 
     protected KeyCopyJob getTask(S3ObjectSummary summary) {
         if (summary.getSize() > MirrorOptions.MAX_SINGLE_REQUEST_UPLOAD_FILE_SIZE) {
-            return new MultipartKeyCopyJob(client, context, summary, notifyLock);
+            return new MultipartKeyCopyJob(client, context, summary, notifyLock, new SizeOnlyComparisonStrategy());
         }
-        return new KeyCopyJob(client, context, summary, notifyLock);
+        return new KeyCopyJob(client, context, summary, notifyLock, comparisonStrategy);
     }
 }

src/main/java/org/cobbzilla/s3s3mirror/KeyCopyJob.java

@@ -3,6 +3,7 @@
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.*;
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.s3s3mirror.comparisonstrategies.ComparisonStrategy;
 import org.slf4j.Logger;
 
 import java.util.Date;
@@ -14,8 +15,9 @@
 public class KeyCopyJob extends KeyJob {
 
     protected String keydest;
+    protected ComparisonStrategy comparisonStrategy;
 
-    public KeyCopyJob(AmazonS3Client client, MirrorContext context, S3ObjectSummary summary, Object notifyLock) {
+    public KeyCopyJob(AmazonS3Client client, MirrorContext context, S3ObjectSummary summary, Object notifyLock, ComparisonStrategy comparisonStrategy) {
         super(client, context, summary, notifyLock);
 
         keydest = summary.getKey();
@@ -24,6 +26,7 @@ public KeyCopyJob(AmazonS3Client client, MirrorContext context, S3ObjectSummary
             keydest = keydest.substring(options.getPrefixLength());
             keydest = options.getDestPrefix() + keydest;
         }
+        this.comparisonStrategy = comparisonStrategy;
     }
 
     @Override public Logger getLog() { return log; }
@@ -136,25 +139,9 @@ private boolean shouldTransfer() {
         if (summary.getSize() > MirrorOptions.MAX_SINGLE_REQUEST_UPLOAD_FILE_SIZE) {
             return metadata.getContentLength() != summary.getSize();
         }
-        final boolean objectChanged = objectChanged(metadata);
+        final boolean objectChanged = comparisonStrategy.sourceDifferent(summary, metadata);
         if (verbose && !objectChanged) log.info("Destination file is same as source, not copying: "+ key);
 
         return objectChanged;
     }
-
-    boolean objectChanged(ObjectMetadata metadata) {
-        final MirrorOptions options = context.getOptions();
-        final KeyFingerprint sourceFingerprint;
-        final KeyFingerprint destFingerprint;
-
-        if (options.isSizeOnly()) {
-            sourceFingerprint = new KeyFingerprint(summary.getSize());
-            destFingerprint = new KeyFingerprint(metadata.getContentLength());
-        } else {
-            sourceFingerprint = new KeyFingerprint(summary.getSize(), summary.getETag());
-            destFingerprint = new KeyFingerprint(metadata.getContentLength(), metadata.getETag());
-        }
-
-        return !sourceFingerprint.equals(destFingerprint);
-    }
 }

src/main/java/org/cobbzilla/s3s3mirror/MirrorOptions.java

@@ -117,6 +117,12 @@ public class MirrorOptions implements AWSCredentials {
     @Option(name=OPT_SIZE_ONLY, aliases=LONGOPT_SIZE_ONLY, usage=USAGE_SIZE_ONLY)
     @Getter @Setter private boolean sizeOnly = false;
 
+    public static final String USAGE_SIZE_LAST_MODIFIED = "Uses size and last modified to determine if files have change like the AWS CLI and ignores etags. If size only is also specified that strategy is selected.";
+    public static final String OPT_SIZE_LAST_MODIFIED = "-L";
+    public static final String LONGOPT_SIZE_LAST_MODIFIED = "--size-and-last-modified";
+    @Option(name=OPT_SIZE_LAST_MODIFIED, aliases=LONGOPT_SIZE_LAST_MODIFIED, usage=USAGE_SIZE_LAST_MODIFIED)
+    @Getter @Setter private boolean sizeAndLastModified = false;
+
     public static final String USAGE_CTIME = "Only copy objects whose Last-Modified date is younger than this many days. " +
             "For other time units, use these suffixes: y (years), M (months), d (days), w (weeks), h (hours), m (minutes), s (seconds)";
     public static final String OPT_CTIME = "-c";

src/main/java/org/cobbzilla/s3s3mirror/MultipartKeyCopyJob.java

@@ -3,15 +3,16 @@
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.*;
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.s3s3mirror.comparisonstrategies.ComparisonStrategy;
 
 import java.util.ArrayList;
 import java.util.List;
 
 @Slf4j
 public class MultipartKeyCopyJob extends KeyCopyJob {
 
-    public MultipartKeyCopyJob(AmazonS3Client client, MirrorContext context, S3ObjectSummary summary, Object notifyLock) {
-        super(client, context, summary, notifyLock);
+    public MultipartKeyCopyJob(AmazonS3Client client, MirrorContext context, S3ObjectSummary summary, Object notifyLock, ComparisonStrategy comparisonStrategy) {
+        super(client, context, summary, notifyLock, comparisonStrategy);
     }
 
     @Override
@@ -91,9 +92,4 @@ private List<PartETag> getETags(List<CopyPartResult> copyResponses) {
         }
         return eTags;
     }
-
-    @Override
-    boolean objectChanged(ObjectMetadata metadata) {
-        return summary.getSize() != metadata.getContentLength();
-    }
 }

src/main/java/org/cobbzilla/s3s3mirror/comparisonstrategies/ComparisonStrategy.java

@@ -0,0 +1,8 @@
+package org.cobbzilla.s3s3mirror.comparisonstrategies;
+
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+public interface ComparisonStrategy {
+    boolean sourceDifferent(S3ObjectSummary source, ObjectMetadata destination);
+}

src/main/java/org/cobbzilla/s3s3mirror/comparisonstrategies/ComparisonStrategyFactory.java

@@ -0,0 +1,18 @@
+package org.cobbzilla.s3s3mirror.comparisonstrategies;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import org.cobbzilla.s3s3mirror.MirrorOptions;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class ComparisonStrategyFactory {
+    public static ComparisonStrategy getStrategy(MirrorOptions mirrorOptions) {
+        if (mirrorOptions.isSizeOnly()) {
+            return new SizeOnlyComparisonStrategy();
+        } else if (mirrorOptions.isSizeAndLastModified()) {
+            return new SizeAndLastModifiedComparisonStrategy();
+        } else {
+            return new EtagComparisonStrategy();
+        }
+    }
+}

src/main/java/org/cobbzilla/s3s3mirror/comparisonstrategies/EtagComparisonStrategy.java

@@ -0,0 +1,12 @@
+package org.cobbzilla.s3s3mirror.comparisonstrategies;
+
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+public class EtagComparisonStrategy extends SizeOnlyComparisonStrategy {
+    @Override
+    public boolean sourceDifferent(S3ObjectSummary source, ObjectMetadata destination) {
+
+        return super.sourceDifferent(source, destination) || !source.getETag().equals(destination.getETag());
+    }
+}

src/main/java/org/cobbzilla/s3s3mirror/comparisonstrategies/SizeAndLastModifiedComparisonStrategy.java

@@ -0,0 +1,11 @@
+package org.cobbzilla.s3s3mirror.comparisonstrategies;
+
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+public class SizeAndLastModifiedComparisonStrategy extends SizeOnlyComparisonStrategy {
+    @Override
+    public boolean sourceDifferent(S3ObjectSummary source, ObjectMetadata destination) {
+        return super.sourceDifferent(source, destination) || source.getLastModified().after(destination.getLastModified());
+    }
+}

src/main/java/org/cobbzilla/s3s3mirror/comparisonstrategies/SizeOnlyComparisonStrategy.java

@@ -0,0 +1,11 @@
+package org.cobbzilla.s3s3mirror.comparisonstrategies;
+
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+public class SizeOnlyComparisonStrategy implements ComparisonStrategy {
+    @Override
+    public boolean sourceDifferent(S3ObjectSummary source, ObjectMetadata destination) {
+        return source.getSize() != destination.getContentLength();
+    }
+}