filterFolderNames = List.of("folder1", "folder3");
+
+ public S3DirectoriesDownloader() {
+ setUp();
+ }
+
+ public static void main(String[] args) {
+ S3DirectoriesDownloader downloader = new S3DirectoriesDownloader();
+ Integer numFilesFailedToDownload = null;
+ try {
+ numFilesFailedToDownload = downloader.downloadS3Directories(S3ClientFactory.transferManager,
+ downloader.destinationPathURI, downloader.bucketName);
+ logger.info("Number of files that failed to download [{}].", numFilesFailedToDownload);
+ } catch (Exception e) {
+ logger.error("Exception [{}]", e.getMessage(), e);
+ } finally {
+ downloader.cleanUp();
+ }
+ }
+
+ // snippet-start:[s3.tm.java2.download-s3-directories.main]
+ /**
+ * For standard buckets, S3 provides the illusion of a directory structure through the use of keys. When you upload
+ * an object to an S3 bucket, you specify a key, which is essentially the "path" to the object. The key can contain
+ * forward slashes ("/") to make it appear as if the object is stored in a directory structure, but this is just a
+ * logical representation, not an actual directory.
+ *
+ * In this example, our S3 bucket contains the following objects:
+ *
+ * folder1/file1.txt
+ * folder1/file2.txt
+ * folder1/file3.txt
+ * folder2/file1.txt
+ * folder2/file2.txt
+ * folder2/file3.txt
+ * folder3/file1.txt
+ * folder3/file2.txt
+ * folder3/file3.txt
+ *
+ * When method `downloadS3Directories` is invoked with
+ * `destinationPathURI` set to `/test`, the downloaded
+ * directory looks like:
+ *
+ * |- test
+ * |- folder1
+ * |- file1.txt
+ * |- file2.txt
+ * |- file3.txt
+ * |- folder3
+ * |- file1.txt
+ * |- file2.txt
+ * |- file3.txt
+ *
+ *
+ * @param transferManager An S3TransferManager instance.
+ * @param destinationPathURI local directory to hold the downloaded S3 'directories' and files.
+ * @param bucketName The S3 bucket that contains the 'directories' to download.
+ * @return The number of objects (files, in this case) that were downloaded.
+ */
+ public Integer downloadS3Directories(S3TransferManager transferManager,
+ URI destinationPathURI, String bucketName) {
+
+ // Define the filters for which 'directories' we want to download.
+ DownloadFilter folder1Filter = (S3Object s3Object) -> s3Object.key().startsWith("folder1/");
+ DownloadFilter folder3Filter = (S3Object s3Object) -> s3Object.key().startsWith("folder3/");
+ DownloadFilter folderFilter = s3Object -> folder1Filter.or(folder3Filter).test(s3Object);
+
+ DirectoryDownload directoryDownload = transferManager.downloadDirectory(DownloadDirectoryRequest.builder()
+ .destination(Paths.get(destinationPathURI))
+ .bucket(bucketName)
+ .filter(folderFilter)
+ .build());
+ CompletedDirectoryDownload completedDirectoryDownload = directoryDownload.completionFuture().join();
+
+ Integer numFilesInFolder1 = Paths.get(destinationPathURI).resolve("folder1").toFile().list().length;
+ Integer numFilesInFolder3 = Paths.get(destinationPathURI).resolve("folder3").toFile().list().length;
+
+ try {
+ assert numFilesInFolder1 == 3;
+ assert numFilesInFolder3 == 3;
+ assert !Paths.get(destinationPathURI).resolve("folder2").toFile().exists(); // `folder2` was not downloaded.
+ } catch (AssertionError e) {
+ logger.error("An assertion failed.");
+ }
+
+ completedDirectoryDownload.failedTransfers()
+ .forEach(fail -> logger.warn("Object failed to transfer [{}]", fail.exception().getMessage()));
+ return numFilesInFolder1 + numFilesInFolder3;
+ }
+ // snippet-end:[s3.tm.java2.download-s3-directories.main]
+
+ private void setUp() {
+ S3ClientFactory.s3Client.createBucket(b -> b.bucket(bucketName));
+ S3ClientFactory.s3Waiter.waitUntilBucketExists(r -> r.bucket(bucketName));
+
+ RequestBody requestBody = RequestBody.fromString("Hello World.");
+
+ folderNames.forEach(folderName ->
+ IntStream.rangeClosed(1, 3).forEach(i -> {
+ String fileName = folderName + "/" + "file" + i + ".txt";
+ downloadedFileNameSet.add(fileName);
+ S3ClientFactory.s3Client.putObject(b -> b
+ .bucket(bucketName)
+ .key(fileName),
+ requestBody);
+ }));
+ try {
+ destinationPathURI = S3DirectoriesDownloader.class.getClassLoader().getResource(destinationDirName).toURI();
+ } catch (URISyntaxException | NullPointerException e) {
+ logger.error("Exception creating URI [{}]", e.getMessage());
+ System.exit(1);
+ }
+ }
+
+ public void cleanUp() {
+ // Delete items uploaded to bucket for download.
+ Set items = downloadedFileNameSet
+ .stream()
+ .map(name -> ObjectIdentifier.builder().key(name).build())
+ .collect(Collectors.toSet());
+
+ S3ClientFactory.s3Client.deleteObjects(b -> b
+ .bucket(bucketName)
+ .delete(b1 -> b1.objects(items)));
+ // Delete bucket.
+ S3ClientFactory.s3Client.deleteBucket(b -> b.bucket(bucketName));
+
+ // Delete files downloaded.
+ Predicate filterFolder1 = key -> key.startsWith("folder1");
+ Predicate filterFolder3 = key -> key.startsWith("folder3");
+ Predicate filterForFolders = filterFolder1.or(filterFolder3);
+
+ downloadedFileNameSet.stream()
+ .filter(filterForFolders)
+ .forEach(fileName -> {
+ try {
+ Path basePath = Paths.get(destinationPathURI);
+ Path fullPath = basePath.resolve(fileName);
+ Files.delete(fullPath);
+ } catch (IOException e) {
+ logger.error("Exception deleting file [{}]", fileName);
+ }
+ });
+ filterFolderNames.forEach(folderName -> {
+ try {
+ Path basePath = Paths.get(destinationPathURI);
+ Path fullPath = basePath.resolve(folderName);
+ Files.delete(fullPath);
+ } catch (IOException e) {
+ logger.error("Exception deleting folder [{}]", folderName);
+ }
+ });
+ }
+}
diff --git a/javav2/example_code/s3/src/main/java/com/example/s3/util/MemoryLog4jAppender.java b/javav2/example_code/s3/src/main/java/com/example/s3/util/MemoryLog4jAppender.java
index 63fc69097bd..b3885e601a8 100644
--- a/javav2/example_code/s3/src/main/java/com/example/s3/util/MemoryLog4jAppender.java
+++ b/javav2/example_code/s3/src/main/java/com/example/s3/util/MemoryLog4jAppender.java
@@ -44,7 +44,11 @@ public void append(LogEvent event) {
} else {
eventMap.put (eventWithParameters.toString(), null);
}
- stringBuilder.append(eventWithParameters.getFormattedMessage() + "\n");
+ if (eventWithParameters.getFormat() != null) {
+ stringBuilder.append(eventWithParameters.getFormattedMessage() + "\n");
+ } else {
+ stringBuilder.append(eventWithParameters.getMessage() + "\n");
+ }
}
public Map getEventMap(){
diff --git a/javav2/example_code/s3/src/main/resources/log4j2.xml b/javav2/example_code/s3/src/main/resources/log4j2.xml
index ade99171a77..2329c9d3615 100644
--- a/javav2/example_code/s3/src/main/resources/log4j2.xml
+++ b/javav2/example_code/s3/src/main/resources/log4j2.xml
@@ -1,9 +1,12 @@
+
+
+
+
-
diff --git a/javav2/example_code/s3/src/test/java/TransferManagerTest.java b/javav2/example_code/s3/src/test/java/TransferManagerTest.java
index d00b519815a..f23da9f20ca 100644
--- a/javav2/example_code/s3/src/test/java/TransferManagerTest.java
+++ b/javav2/example_code/s3/src/test/java/TransferManagerTest.java
@@ -5,6 +5,7 @@
import com.example.s3.transfermanager.DownloadToDirectory;
import com.example.s3.transfermanager.ObjectCopy;
import com.example.s3.transfermanager.S3ClientFactory;
+import com.example.s3.transfermanager.S3DirectoriesDownloader;
import com.example.s3.transfermanager.UploadADirectory;
import com.example.s3.transfermanager.UploadFile;
import com.example.s3.transfermanager.UploadStream;
@@ -50,6 +51,7 @@ public static void afterAll() {
logger.info("... S3TransferManager tests finished");
}
+ @Test
@Tag("IntegrationTest")
public void uploadSingleFileWorks() {
UploadFile upload = new UploadFile();
@@ -143,6 +145,15 @@ public void uploadStreamWorks() {
}
}
+ @Test
+ @Tag("IntegrationTest")
+ public void s3DirectoriesDownloadWorks() {
+ S3DirectoriesDownloader downloader = new S3DirectoriesDownloader();
+ Integer numFilesDownloaded = downloader.downloadS3Directories(S3ClientFactory.transferManager, downloader.destinationPathURI, downloader.bucketName);
+ Assertions.assertEquals(6, numFilesDownloaded);
+ downloader.cleanUp();
+ }
+
private String getLoggedMessages() {
final LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false);
final Configuration configuration = context.getConfiguration();