From 23d7a7278c9ad6714635131c1b3e63e14cbe4508 Mon Sep 17 00:00:00 2001 From: Yura Sorokin Date: Tue, 19 Mar 2024 19:57:53 +0100 Subject: [PATCH] Added s3_storage backend stub and AWS SDK C++ integration to GitHub Workflows Reworked how code is checked out in GitHub Workflows. Inside '${{github.workspace}}' we now have: - 'src' (for the main project source tree) - 'aws-sdk-cpp' (for AWS SDK C++ source tree with submodules) All Github Workflows 'checkout' actions are now instructed to fetch tags as well ('fetch-tags: true'). For our own code we also fetch full commit history ('fetch-depth: 0'). Reworked the way how we form labels for matrix jobs: '-', where '' is either 'Debug', 'RelWithDebInfo', or 'ASan' and '' is either 'clang17' or 'gcc13'. These labels are used in directory names and in cache keys. Changed the way how Boost libraries are cached: instead of caching the binary tarball ('.tar.bz2'), the unpacked directory is cached. This helps to eliminate compressing already compressed data. Caching key now also includes the library version (currently, 'boost-libraries-1-83-0'). Added a new step that shows '${{github.workspace}}' and '${{runner.temp}}' for diagnostic purposes. Added a new step that shows the content of the following directories: - '${{github.workspace}}' - '${{runner.temp}}' - '${{runner.temp}}/deps' for diagnostic purposes. GitHub Workflows now builds AWS SDK C++ libraries. * Currently the selected version is 1.11.286 but it is configurable via ${{env.AWS_SDK_CPP_MAJOR}}, ${{env.AWS_SDK_CPP_MINOR}}, and ${{env.AWS_SDK_CPP_PATCH}} variables. * The code is checked out from the 'aws/aws-sdk-cpp' GitHub repo (recursively with submodules) by '1.11.286' tag. * CMake is instructed to build only static versions of the libraries with all the dependencies ('-DBUILD_SHARED_LIBS=OFF' and '-DFORCE_SHARED_CRT=OFF'). * The same compiler and build configuration as for the main project are used to build the libraries ('-DCMAKE_BUILD_TYPE=...', '-DCMAKE_C_COMPILER=...', and '-DCMAKE_CXX_COMPILER=...'). * C++ standard is set to 'c++20' as in the main project ('-DCPP_STANDARD=20'). * In order to instruct AWS SDK C++ to use 'libc++' standard library implementation (for 'clang' compiler), '-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++' option is passed manually for such configurations. * For Address Sanitizer builds '-DENABLE_ADDRESS_SANITIZER=ON' option is added. * Unity build is enabled ('-DENABLE_UNITY_BUILD=ON'). * Building/running tests is disabled ('-DENABLE_TESTING=OFF' and '-DAUTORUN_UNIT_TESTS=OFF'). * Currently only 's3-crt' component is build ('-DBUILD_ONLY=s3-crt'). * The full specification of these options is available at https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/cmake-params.html * 'conan' package manager recipe https://github.com/conan-io/conan-center-index/blob/master/recipes/aws-sdk-cpp/all/conanfile.py and 'vcpkg' port https://github.com/microsoft/vcpkg/blob/master/ports/aws-sdk-cpp/portfile.cmake were taken as examples. The libraries are built in the '${{github.workspace}}/aws-sdk-cpp-build-${{matrix.config.label}}' directory. The libraries are installed into the '${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}}' directory. The build results (the content of the installation directory) are cached under the 'ws-cpp-sdk-libraries---' key (e,g, 'aws-cpp-sdk-libraries-1-11-286-RelWithDebInfo-clang17'). This allows to skip building AWS SDK C++ libraries every time a new PR is created. The rebuild is expected to happen only when the SDK version is updated or a new version of the compiler is added. '-DCPP_STANDARD=20' CMake configuration option is now also added for the main project to synchronize with AWS SDK C++ libraries. Main 'CMakeLists.txt' file now tries to find AWS SDK C++ libraries via 'find_package()'. Developers are now expected to pass '-DCMAKE_PREFIX_PATH=' CMake option when they configure the project, where '' should point to the AWS SDK C++ installation directory. Main application target now depends on 'aws-cpp-sdk-s3-crt'. Added a stub for the second implementation of the 'binsrv::basic_storage_backend' interface, called 'binsrv::s3_storage_backend', which in future will provide basic operations for working with AWS S3 storage. 'binsrv::storage_backend_factory' factory class extended with a new concrete implemetation. Now: - for 'fs' we return an instance of 'binsrv::filesystem_storage_backend' - for 's3' we return an instance of 'binsrv::s3_storage_backend'. 'binsrv::basic_storage_backend' interface extended with one more abstract method 'get_description()': - 'binsrv::filesystem_storage_backend' ('fs') implementation returns "local filesystem" - 'binsrv::s3_storage_backend' ('s3') implementation returns "AWS S3 (SDK )" ('' is populated from the 'Aws::SDKOptions' struct initialized via 'Aws::InitAPI()' call). Main application extended with writing 'get_description()' output into the log. --- .github/workflows/cmake.yml | 138 +++++++++++++++++----- CMakeLists.txt | 7 ++ src/app.cpp | 3 + src/binsrv/basic_storage_backend.cpp | 4 + src/binsrv/basic_storage_backend.hpp | 4 + src/binsrv/filesystem_storage_backend.cpp | 5 + src/binsrv/filesystem_storage_backend.hpp | 2 + src/binsrv/s3_storage_backend.cpp | 70 +++++++++++ src/binsrv/s3_storage_backend.hpp | 42 +++++++ src/binsrv/storage_backend_factory.cpp | 12 +- 10 files changed, 254 insertions(+), 33 deletions(-) create mode 100644 src/binsrv/s3_storage_backend.cpp create mode 100644 src/binsrv/s3_storage_backend.hpp diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index a97157c..ebd63a9 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -7,6 +7,9 @@ on: branches: [ "main" ] env: + AWS_SDK_CPP_MAJOR: 1 + AWS_SDK_CPP_MINOR: 11 + AWS_SDK_CPP_PATCH: 286 BOOST_MAJOR: 1 BOOST_MINOR: 83 BOOST_PATCH: 0 @@ -17,11 +20,15 @@ jobs: runs-on: ubuntu-22.04 name: Formatting checks steps: - - uses: actions/checkout@v4 + - name: Cheking out source tree + uses: actions/checkout@v4 with: - fetch-depth: 2 + path: src + fetch-depth: 0 + fetch-tags: true - name: Check formatting with git diff --check + working-directory: ${{github.workspace}}/src run: git diff --check --color HEAD~ - name: Install dependencies on ubuntu @@ -35,6 +42,7 @@ jobs: run: clang-format-17 --version - name: Check formatting with git clang-format-17 + working-directory: ${{github.workspace}}/src run: git clang-format-17 --diff --binary=clang-format-17 HEAD~ build: @@ -52,7 +60,7 @@ jobs: build_type: "Debug", cc: "gcc-13", cxx: "g++-13", - label: "debug_gcc13", + label: "Debug-gcc13", run_mtr: true } - { @@ -60,7 +68,7 @@ jobs: build_type: "RelWithDebInfo", cc: "gcc-13", cxx: "g++-13", - label: "relwithdebinfo_gcc13", + label: "RelWithDebInfo-gcc13", run_mtr: true } - { @@ -69,7 +77,8 @@ jobs: cc: "gcc-13", cxx: "g++-13", sanitizer_cmake_flags: "-DWITH_ASAN=ON", - label: "asan_gcc13", + aws_sanitizer_cmake_flags: "-DENABLE_ADDRESS_SANITIZER=ON", + label: "ASan-gcc13", run_mtr: true, mtr_options: "--sanitize" } @@ -79,7 +88,8 @@ jobs: cc: "clang-17", cxx: "clang++-17", libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON", - label: "debug_clang17", + aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++", + label: "Debug-clang17", run_clang_tidy: true, } - { @@ -88,7 +98,8 @@ jobs: cc: "clang-17", cxx: "clang++-17", libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON", - label: "relwithdebinfo_clang17", + aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++", + label: "RelWithDebInfo-clang17", run_clang_tidy: true, } - { @@ -97,8 +108,10 @@ jobs: cc: "clang-17", cxx: "clang++-17", libcxx_cmake_flags: "-DWITH_STDLIB_LIBCXX=ON", + aws_libcxx_cmake_flags: "-DCMAKE_CXX_FLAGS_INIT=-stdlib=libc++", sanitizer_cmake_flags: "-DWITH_ASAN=ON", - label: "asan_clang17", + aws_sanitizer_cmake_flags: "-DENABLE_ADDRESS_SANITIZER=ON", + label: "ASan-clang17", run_mtr: true, mtr_options: "--sanitize" } @@ -117,10 +130,15 @@ jobs: - name: Info df run: df -h - - name: Install MySQL client libraries + - name: Info GitHub directories + run: | + echo github.workspace: ${{github.workspace}} + echo runner.temp : ${{runner.temp}} + + - name: Install MySQL client libraries and CURL Development libraries run: | sudo apt-get update - sudo apt-get install libmysqlclient-dev + sudo apt-get install libmysqlclient-dev libcurl4-openssl-dev - name: Install MySQL server and MTR if: matrix.config.run_mtr @@ -151,44 +169,100 @@ jobs: - name: Creating deps directory run: mkdir -p ${{runner.temp}}/deps - - name: Cache boost tarball - id: cache-boost-tarball + - name: Cache boost libraries + id: cache-boost-libraries uses: actions/cache@v4 with: - path: ${{runner.temp}}/deps/${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}} - key: boost-tarball + path: ${{runner.temp}}/deps/${{format('boost_{0}_{1}_{2}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH)}} + key: ${{format('boost-libraries-{0}-{1}-{2}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH)}} - - name: Download boost libraries - if: steps.cache-boost-tarball.outputs.cache-hit != 'true' + - name: Download and unpack boost libraries + if: steps.cache-boost-libraries.outputs.cache-hit != 'true' working-directory: ${{runner.temp}}/deps - run: wget --quiet ${{format('https://boostorg.jfrog.io/artifactory/main/release/{0}.{1}.{2}/source/boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}} + run: | + wget --quiet ${{format('https://boostorg.jfrog.io/artifactory/main/release/{0}.{1}.{2}/source/boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}} + tar xf ${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}} + rm -f ${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}} - - name: Unpack boost libraries - working-directory: ${{runner.temp}}/deps - run: tar xf ${{format('boost_{0}_{1}_{2}{3}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH, env.BOOST_EXT)}} + - name: Cache AWS SDK C++ libraries + id: cache-aws-sdk-cpp-libraries + uses: actions/cache@v4 + with: + path: ${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}} + key: ${{format('aws-cpp-sdk-libraries-{0}-{1}-{2}-{3}', env.AWS_SDK_CPP_MAJOR, env.AWS_SDK_CPP_MINOR, env.AWS_SDK_CPP_PATCH, matrix.config.label)}} - - uses: actions/checkout@v4 + - name: Checking out AWS SDK C++ source tree + if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: aws/aws-sdk-cpp + ref: ${{format('{0}.{1}.{2}', env.AWS_SDK_CPP_MAJOR, env.AWS_SDK_CPP_MINOR, env.AWS_SDK_CPP_PATCH)}} + path: aws-sdk-cpp + submodules: recursive + fetch-tags: true + + - name: Configure CMake for AWS SDK C++ + if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true' + run: | + cmake \ + -B ${{github.workspace}}/aws-sdk-cpp-build-${{matrix.config.label}} \ + -S ${{github.workspace}}/aws-sdk-cpp \ + -DCMAKE_INSTALL_PREFIX=${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}} \ + -DCMAKE_BUILD_TYPE=${{matrix.config.build_type}} \ + -DCMAKE_C_COMPILER=${{matrix.config.cc}} \ + -DCMAKE_CXX_COMPILER=${{matrix.config.cxx}} \ + ${{matrix.config.aws_libcxx_cmake_flags}} \ + ${{matrix.config.aws_sanitizer_cmake_flags}} \ + -DCPP_STANDARD=20 \ + -DENABLE_UNITY_BUILD=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DFORCE_SHARED_CRT=OFF \ + -DENABLE_TESTING=OFF \ + -DAUTORUN_UNIT_TESTS=OFF \ + -DBUILD_ONLY=s3-crt + + - name: CMake info for AWS SDK C++ + if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true' + run: cmake -L ${{github.workspace}}/aws-sdk-cpp-build-${{matrix.config.label}} + + - name: Build for AWS SDK C++ + if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true' + run: cmake --build ${{github.workspace}}/aws-sdk-cpp-build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel + + - name: Install for AWS SDK C++ + if: steps.cache-aws-sdk-cpp-libraries.outputs.cache-hit != 'true' + run: cmake --install ${{github.workspace}}/aws-sdk-cpp-build-${{matrix.config.label}} --config ${{matrix.config.build_type}} + + - name: Checking out source tree + uses: actions/checkout@v4 + with: + path: src + fetch-depth: 0 + fetch-tags: true - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type run: | cmake -Wdev -Werror=dev -Wdeprecated -Werror=deprecated \ - -B ${{github.workspace}}/../build-${{matrix.config.label}} \ + -B ${{github.workspace}}/build-${{matrix.config.label}} \ + -S ${{github.workspace}}/src \ -DCMAKE_BUILD_TYPE=${{matrix.config.build_type}} \ -DCMAKE_C_COMPILER=${{matrix.config.cc}} \ -DCMAKE_CXX_COMPILER=${{matrix.config.cxx}} \ + -DCPP_STANDARD=20 \ ${{matrix.config.libcxx_cmake_flags}} \ ${{matrix.config.sanitizer_cmake_flags}} \ + -DCMAKE_PREFIX_PATH=${{runner.temp}}/deps/aws-sdk-cpp-install-${{matrix.config.label}} \ -DBoost_ROOT=${{runner.temp}}/deps/${{format('boost_{0}_{1}_{2}', env.BOOST_MAJOR, env.BOOST_MINOR, env.BOOST_PATCH)}} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - name: CMake info - run: cmake -L ${{github.workspace}}/..//build-${{matrix.config.label}} + run: cmake -L ${{github.workspace}}/build-${{matrix.config.label}} - name: Build # Build your program with the given configuration - run: cmake --build ${{github.workspace}}/../build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel + run: cmake --build ${{github.workspace}}/build-${{matrix.config.label}} --config ${{matrix.config.build_type}} --parallel - name: Info Clang Tidy if: matrix.config.run_clang_tidy @@ -197,7 +271,7 @@ jobs: - name: Clang Tidy if: matrix.config.run_clang_tidy # Run Clang Tidy - run: run-clang-tidy-17 -header-filter=.* -j=${{steps.cpu-cores.outputs.count}} -use-color -p=${{github.workspace}}/../build-${{matrix.config.label}} + run: run-clang-tidy-17 -header-filter=.* -j=${{steps.cpu-cores.outputs.count}} -use-color -p=${{github.workspace}}/build-${{matrix.config.label}} - name: MTR tests if: matrix.config.run_mtr @@ -206,15 +280,21 @@ jobs: # Switching MySQL Server Apparmor profile to "complain" as we are creating a custom data directory sudo aa-complain /usr/sbin/mysqld # Linking the "binlog_streaming" from the source tree into the MTR suits directory on the system - sudo ln -s ${{github.workspace}}/mtr/binlog_streaming /usr/lib/mysql-test/suite/binlog_streaming + sudo ln -s ${{github.workspace}}/src/mtr/binlog_streaming /usr/lib/mysql-test/suite/binlog_streaming # Running MTR from the system package - BINSRV=${{github.workspace}}/../build-${{matrix.config.label}}/binlog_server ./mtr \ - --client-bindir=/usr/lib/mysql-test/bin --vardir=${{github.workspace}}/../mtrvardir \ + BINSRV=${{github.workspace}}/build-${{matrix.config.label}}/binlog_server ./mtr \ + --client-bindir=/usr/lib/mysql-test/bin --vardir=${{runner.temp}}/mtrvardir \ --force --max-test-fail=0 --retry=0 --nounit-tests --big-test --repeat=2 --parallel=${{steps.cpu-cores.outputs.count}} \ --suite=binlog_streaming ${{matrix.config.mtr_options}} - name: CTest - working-directory: ${{github.workspace}}/../build-${{matrix.config.label}} + working-directory: ${{github.workspace}}/build-${{matrix.config.label}} # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail run: ctest -C ${{matrix.config.build_type}} --parallel + + - name: Info Build artefacts + run: | + ls -la ${{github.workspace}} + ls -la ${{runner.temp}} + ls -la ${{runner.temp}}/deps diff --git a/CMakeLists.txt b/CMakeLists.txt index 07fd8b2..8ed8a4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,9 @@ find_package(Boost 1.83.0 EXACT REQUIRED) find_package(MySQL REQUIRED) +find_package(ZLIB REQUIRED) +find_package(AWSSDK REQUIRED COMPONENTS s3-crt) + set(source_files # main application files src/app.cpp @@ -167,6 +170,9 @@ set(source_files src/binsrv/main_config.hpp src/binsrv/main_config.cpp + src/binsrv/s3_storage_backend.hpp + src/binsrv/s3_storage_backend.cpp + src/binsrv/storage_fwd.hpp src/binsrv/storage.hpp src/binsrv/storage.cpp @@ -240,6 +246,7 @@ target_link_libraries(binlog_server binlog_server_compiler_flags PRIVATE Boost::headers MySQL::client + aws-cpp-sdk-s3-crt ) # for some reason it is not possible to propagate CXX_EXTENSIONS and # CXX_STANDARD_REQUIRED via interface library (binlog_server_compiler_flags) diff --git a/src/app.cpp b/src/app.cpp index 91d3e59..27662e0 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -208,6 +208,9 @@ int main(int argc, char *argv[]) { msg += ", path: "; msg += storage_config.get<"path">(); logger->log(binsrv::log_severity::info, msg); + msg = "description: "; + msg += storage_backend->get_description(); + logger->log(binsrv::log_severity::info, msg); binsrv::storage storage{std::move(storage_backend)}; logger->log(binsrv::log_severity::info, diff --git a/src/binsrv/basic_storage_backend.cpp b/src/binsrv/basic_storage_backend.cpp index 119f205..8790176 100644 --- a/src/binsrv/basic_storage_backend.cpp +++ b/src/binsrv/basic_storage_backend.cpp @@ -51,4 +51,8 @@ void basic_storage_backend::close_stream() { stream_opened_ = false; } +[[nodiscard]] std::string basic_storage_backend::get_description() const { + return do_get_description(); +} + } // namespace binsrv diff --git a/src/binsrv/basic_storage_backend.hpp b/src/binsrv/basic_storage_backend.hpp index fb1d41c..fdafcf1 100644 --- a/src/binsrv/basic_storage_backend.hpp +++ b/src/binsrv/basic_storage_backend.hpp @@ -28,6 +28,8 @@ class basic_storage_backend { void write_data_to_stream(util::const_byte_span data); void close_stream(); + [[nodiscard]] std::string get_description() const; + private: bool stream_opened_{false}; @@ -39,6 +41,8 @@ class basic_storage_backend { virtual void do_open_stream(std::string_view name) = 0; virtual void do_write_data_to_stream(util::const_byte_span data) = 0; virtual void do_close_stream() = 0; + + [[nodiscard]] virtual std::string do_get_description() const = 0; }; } // namespace binsrv diff --git a/src/binsrv/filesystem_storage_backend.cpp b/src/binsrv/filesystem_storage_backend.cpp index 9ab2f0c..ed50b52 100644 --- a/src/binsrv/filesystem_storage_backend.cpp +++ b/src/binsrv/filesystem_storage_backend.cpp @@ -117,6 +117,11 @@ void filesystem_storage_backend::do_write_data_to_stream( void filesystem_storage_backend::do_close_stream() { ofs_.close(); } +[[nodiscard]] std::string +filesystem_storage_backend::do_get_description() const { + return "local filesystem"; +} + [[nodiscard]] std::filesystem::path filesystem_storage_backend::get_object_path(std::string_view name) const { auto result{root_path_}; diff --git a/src/binsrv/filesystem_storage_backend.hpp b/src/binsrv/filesystem_storage_backend.hpp index 91d3189..9b8b6a5 100644 --- a/src/binsrv/filesystem_storage_backend.hpp +++ b/src/binsrv/filesystem_storage_backend.hpp @@ -31,6 +31,8 @@ class [[nodiscard]] filesystem_storage_backend : public basic_storage_backend { void do_write_data_to_stream(util::const_byte_span data) override; void do_close_stream() override; + [[nodiscard]] std::string do_get_description() const override; + [[nodiscard]] std::filesystem::path get_object_path(std::string_view name) const; }; diff --git a/src/binsrv/s3_storage_backend.cpp b/src/binsrv/s3_storage_backend.cpp new file mode 100644 index 0000000..b43e8b6 --- /dev/null +++ b/src/binsrv/s3_storage_backend.cpp @@ -0,0 +1,70 @@ +#include "binsrv/s3_storage_backend.hpp" + +#include +#include +#include +#include +#include + +#include + +// TODO: remove this include when switching to clang-18 where transitive +// IWYU 'export' pragmas are handled properly +#include "binsrv/basic_storage_backend_fwd.hpp" + +#include "util/byte_span.hpp" +#include "util/impl_helpers.hpp" + +namespace binsrv { + +using options_deimpl = util::deimpl; + +void s3_storage_backend::options_deleter::operator()(void *ptr) const noexcept { + auto *casted_ptr{static_cast(ptr)}; + Aws::ShutdownAPI(*casted_ptr); + // deleting via std::default_delete to avoid + // cppcoreguidelines-owning-memory warnings + using delete_helper = std::default_delete; + delete_helper{}(casted_ptr); +} + +s3_storage_backend::s3_storage_backend(std::string_view root_path) + : root_path_{root_path}, options_{new Aws::SDKOptions} { + Aws::InitAPI(*options_deimpl::get(options_)); +} + +[[nodiscard]] storage_object_name_container +s3_storage_backend::do_list_objects() { + storage_object_name_container result; + return result; +} + +[[nodiscard]] std::string +s3_storage_backend::do_get_object(std::string_view /*name*/) { + static constexpr std::size_t file_size{8}; + std::string file_content(file_size, 'x'); + return file_content; +} + +void s3_storage_backend::do_put_object(std::string_view /*name*/, + util::const_byte_span /*content*/) {} + +void s3_storage_backend::do_open_stream(std::string_view /*name*/) {} + +void s3_storage_backend::do_write_data_to_stream( + util::const_byte_span /*data*/) {} + +void s3_storage_backend::do_close_stream() {} + +[[nodiscard]] std::string s3_storage_backend::do_get_description() const { + std::string res{"AWS S3 (SDK "}; + const auto &casted_options = *options_deimpl::get(options_); + res += std::to_string(casted_options.sdkVersion.major); + res += '.'; + res += std::to_string(casted_options.sdkVersion.minor); + res += '.'; + res += std::to_string(casted_options.sdkVersion.patch); + res += ')'; + return res; +} +} // namespace binsrv diff --git a/src/binsrv/s3_storage_backend.hpp b/src/binsrv/s3_storage_backend.hpp new file mode 100644 index 0000000..03c0502 --- /dev/null +++ b/src/binsrv/s3_storage_backend.hpp @@ -0,0 +1,42 @@ +#ifndef BINSRV_S3_STORAGE_BACKEND_HPP +#define BINSRV_S3_STORAGE_BACKEND_HPP + +#include +#include + +#include "binsrv/basic_storage_backend.hpp" // IWYU pragma: export + +namespace binsrv { + +class [[nodiscard]] s3_storage_backend : public basic_storage_backend { +public: + explicit s3_storage_backend(std::string_view root_path); + + [[nodiscard]] const std::filesystem::path &get_root_path() const noexcept { + return root_path_; + } + +private: + std::filesystem::path root_path_; + struct options_deleter { + void operator()(void *ptr) const noexcept; + }; + using options_ptr = std::unique_ptr; + options_ptr options_; + + [[nodiscard]] storage_object_name_container do_list_objects() override; + + [[nodiscard]] std::string do_get_object(std::string_view name) override; + void do_put_object(std::string_view name, + util::const_byte_span content) override; + + void do_open_stream(std::string_view name) override; + void do_write_data_to_stream(util::const_byte_span data) override; + void do_close_stream() override; + + [[nodiscard]] std::string do_get_description() const override; +}; + +} // namespace binsrv + +#endif // BINSRV_S3_STORAGE_BACKEND_HPP diff --git a/src/binsrv/storage_backend_factory.cpp b/src/binsrv/storage_backend_factory.cpp index 94586d5..28e3a04 100644 --- a/src/binsrv/storage_backend_factory.cpp +++ b/src/binsrv/storage_backend_factory.cpp @@ -5,6 +5,7 @@ #include "binsrv/basic_storage_backend_fwd.hpp" #include "binsrv/filesystem_storage_backend.hpp" +#include "binsrv/s3_storage_backend.hpp" #include "binsrv/storage_config.hpp" #include "util/exception_location_helpers.hpp" @@ -14,11 +15,14 @@ namespace binsrv { basic_storage_backend_ptr storage_backend_factory::create(const storage_config &config) { const auto &storage_backend_type = config.get<"type">(); - if (storage_backend_type != "fs") { - util::exception_location().raise( - "unknown storage backend type"); + if (storage_backend_type == "fs") { + return std::make_unique(config.get<"path">()); } - return std::make_unique(config.get<"path">()); + if (storage_backend_type == "s3") { + return std::make_unique(config.get<"path">()); + } + util::exception_location().raise( + "unknown storage backend type"); } } // namespace binsrv