Adds new handle to object stores that support S3 API

CMakeLists.txt

@@ -103,6 +103,13 @@ ecbuild_add_option( FEATURE RADOS
                     DESCRIPTION "Ceph/Rados storage support"
                     REQUIRED_PACKAGES Rados )
 
+### S3 Support
+
+ecbuild_add_option( FEATURE AWS_S3
+                    DEFAULT OFF
+                    REQUIRED_PACKAGES "AWSSDK REQUIRED COMPONENTS s3"
+                    DESCRIPTION "Enables AWS S3 support" )
+
 ### Armadillo
 
 ecbuild_add_option( FEATURE ARMADILLO

src/eckit/CMakeLists.txt

@@ -286,6 +286,34 @@ io/rados/RadosAttributes.cc
 )
 endif()
 
+if(HAVE_AWS_S3)
+list( APPEND eckit_io_srcs
+io/s3/aws/S3ClientAWS.cc
+io/s3/aws/S3ClientAWS.h
+io/s3/aws/S3ContextAWS.cc
+io/s3/aws/S3ContextAWS.h
+io/s3/S3BucketName.cc
+io/s3/S3BucketName.h
+io/s3/S3Client.cc
+io/s3/S3Client.h
+io/s3/S3Config.cc
+io/s3/S3Config.h
+io/s3/S3Credential.h
+io/s3/S3Exception.cc
+io/s3/S3Exception.h
+io/s3/S3Handle.cc
+io/s3/S3Handle.h
+io/s3/S3Name.cc
+io/s3/S3Name.h
+io/s3/S3ObjectName.cc
+io/s3/S3ObjectName.h
+io/s3/S3Session.cc
+io/s3/S3Session.h
+io/s3/S3URIManager.cc
+io/s3/S3URIManager.h
+)
+endif(HAVE_AWS_S3)
+
 list( APPEND eckit_filesystem_srcs
 filesystem/BasePathName.cc
 filesystem/BasePathName.h
@@ -934,12 +962,14 @@ ecbuild_add_library(
               $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
 
           PRIVATE_INCLUDES
+              $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src/eckit/io/s3/aws>
               "${CURL_INCLUDE_DIRS}"
               "${SNAPPY_INCLUDE_DIRS}"
               "${LZ4_INCLUDE_DIRS}"
               "${BZIP2_INCLUDE_DIRS}"
               "${AEC_INCLUDE_DIRS}"
               "${RADOS_INCLUDE_DIRS}"
+              "${AWSSDK_INCLUDE_DIRS}"
               "${OPENSSL_INCLUDE_DIR}"
               "${AIO_INCLUDE_DIRS}"
 
@@ -953,6 +983,7 @@ ecbuild_add_library(
               "${CURL_LIBRARIES}"
               "${AIO_LIBRARIES}"
               "${RADOS_LIBRARIES}"
+              "${AWSSDK_LINK_LIBRARIES}"
 
           PUBLIC_LIBS
               ${CMATH_LIBRARIES}
@@ -984,4 +1015,3 @@ add_subdirectory( web )
 if( HAVE_ECKIT_CODEC )
     add_subdirectory( codec )
 endif()
-

src/eckit/io/s3/S3BucketName.cc

@@ -0,0 +1,95 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project IO-SEA
+ * (Project ID: 955811) iosea-project.eu
+ */
+
+#include "eckit/io/s3/S3BucketName.h"
+
+#include "eckit/config/LibEcKit.h"
+#include "eckit/filesystem/URI.h"
+#include "eckit/io/s3/S3Client.h"
+#include "eckit/io/s3/S3Exception.h"
+#include "eckit/io/s3/S3ObjectName.h"
+
+namespace eckit {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+S3BucketName::S3BucketName(const URI& uri): S3Name(uri) {
+    parse();
+}
+
+S3BucketName::S3BucketName(const net::Endpoint& endpoint, const std::string& bucket):
+    S3Name(endpoint, "/"), bucket_(bucket) { }
+
+//----------------------------------------------------------------------------------------------------------------------
+
+auto S3BucketName::asString() const -> std::string {
+    return S3Name::asString() + "/" + bucket_;
+}
+
+void S3BucketName::print(std::ostream& out) const {
+    out << "S3BucketName[bucket=" << bucket_;
+    S3Name::print(out);
+}
+
+void S3BucketName::parse() {
+    const auto pairs = parseName();
+    if (pairs.empty()) { throw S3SeriousBug("Could not parse bucket name!", Here()); }
+    bucket_ = pairs[0];
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+auto S3BucketName::makeObject(const std::string& object) const -> std::unique_ptr<S3ObjectName> {
+    return std::make_unique<S3ObjectName>(endpoint(), bucket_, object);
+}
+
+auto S3BucketName::exists() const -> bool {
+    return client()->bucketExists(bucket_);
+}
+
+void S3BucketName::create() {
+    client()->createBucket(bucket_);
+}
+
+void S3BucketName::destroy() {
+    client()->deleteBucket(bucket_);
+}
+
+void S3BucketName::ensureCreated() {
+    try {
+        create();
+    }
+    catch (S3EntityAlreadyExists& e) {
+        LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl;
+    }
+}
+
+void S3BucketName::ensureDestroyed() {
+    try {
+        client()->emptyBucket(bucket_);
+        client()->deleteBucket(bucket_);
+    }
+    catch (S3EntityNotFound& e) {
+        LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl;
+    }
+}
+
+auto S3BucketName::listObjects() const -> std::vector<std::string> {
+    return client()->listObjects(bucket_);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit

src/eckit/io/s3/S3BucketName.h

@@ -0,0 +1,65 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project IO-SEA
+ * (Project ID: 955811) iosea-project.eu
+ */
+
+/// @file   S3BucketName.h
+/// @author Metin Cakircali
+/// @author Nicolau Manubens
+/// @date   Feb 2024
+
+#pragma once
+
+#include "eckit/io/s3/S3Name.h"
+
+namespace eckit {
+
+class S3ObjectName;
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class S3BucketName: public S3Name {
+public:  // methods
+    explicit S3BucketName(const URI& uri);
+
+    S3BucketName(const net::Endpoint& endpoint, const std::string& bucket);
+
+    auto makeObject(const std::string& object) const -> std::unique_ptr<S3ObjectName>;
+
+    auto exists() const -> bool override;
+
+    void create();
+
+    void destroy();
+
+    void ensureCreated();
+
+    void ensureDestroyed();
+
+    /// @todo: return S3 object iterator but first add prefix
+    auto listObjects() const -> std::vector<std::string>;
+
+    auto asString() const -> std::string override;
+
+private:  // methods
+    void print(std::ostream& out) const override;
+
+    void parse();
+
+private:  // members
+    std::string bucket_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit

src/eckit/io/s3/S3Client.cc

@@ -0,0 +1,57 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project IO-SEA
+ * (Project ID: 955811) iosea-project.eu
+ */
+
+#include "eckit/io/s3/S3Client.h"
+
+#include "S3Client.h"
+#include "eckit/io/s3/S3Exception.h"
+#include "eckit/io/s3/S3Session.h"
+#include "eckit/io/s3/aws/S3ClientAWS.h"
+
+namespace eckit {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+S3Client::S3Client(const S3Config& config): config_(config) { }
+
+S3Client::~S3Client() = default;
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void S3Client::print(std::ostream& out) const {
+    out << "S3Client[config=" << config_ << "]";
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+auto S3Client::makeShared(const S3Config& config) -> std::shared_ptr<S3Client> {
+    if (config.type == S3Types::AWS) { return std::make_shared<S3ClientAWS>(config); }
+    throw S3SeriousBug("Unkown S3 client type!", Here());
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr<S3Client> {
+    if (config.type == S3Types::AWS) { return std::make_unique<S3ClientAWS>(config); }
+    throw S3SeriousBug("Unkown S3 client type!", Here());
+}
+
+auto S3Client::endpoint() const -> const net::Endpoint& {
+    return config_.endpoint;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit

src/eckit/io/s3/S3Client.h

@@ -0,0 +1,86 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project IO-SEA
+ * (Project ID: 955811) iosea-project.eu
+ */
+
+/// @file   S3Client.h
+/// @author Metin Cakircali
+/// @date   Jan 2024
+
+#pragma once
+
+#include "eckit/io/s3/S3Config.h"
+#include "eckit/memory/NonCopyable.h"
+
+#include <memory>
+#include <vector>
+
+namespace eckit {
+
+class S3Context;
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class S3Client: private NonCopyable {
+public:  // methods
+    virtual ~S3Client();
+
+    static auto makeShared(const S3Config& config) -> std::shared_ptr<S3Client>;
+
+    static auto makeUnique(const S3Config& config) -> std::unique_ptr<S3Client>;
+
+    virtual auto endpoint() const -> const net::Endpoint&;
+
+    virtual void createBucket(const std::string& bucket) const = 0;
+
+    virtual void emptyBucket(const std::string& bucket) const = 0;
+
+    virtual void deleteBucket(const std::string& bucket) const = 0;
+
+    virtual auto bucketExists(const std::string& bucket) const -> bool = 0;
+
+    virtual auto listBuckets() const -> std::vector<std::string> = 0;
+
+    virtual auto putObject(const std::string& bucket, const std::string& object, const void* buffer,
+                           uint64_t length) const -> long long = 0;
+
+    virtual auto getObject(const std::string& bucket, const std::string& object, void* buffer, uint64_t offset,
+                           uint64_t length) const -> long long = 0;
+
+    virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0;
+
+    virtual void deleteObjects(const std::string& bucket, const std::vector<std::string>& objects) const = 0;
+
+    virtual auto listObjects(const std::string& bucket) const -> std::vector<std::string> = 0;
+
+    virtual auto objectExists(const std::string& bucket, const std::string& object) const -> bool = 0;
+
+    virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0;
+
+    friend std::ostream& operator<<(std::ostream& out, const S3Client& client) {
+        client.print(out);
+        return out;
+    }
+
+protected:  // methods
+    explicit S3Client(const S3Config& config);
+
+    virtual void print(std::ostream& out) const;
+
+private:  // members
+    S3Config config_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit