diff --git a/.github/workflows/test_pr.yml b/.github/workflows/test_pr.yml index ebd53461..9e17a369 100644 --- a/.github/workflows/test_pr.yml +++ b/.github/workflows/test_pr.yml @@ -15,7 +15,7 @@ jobs: test: name: ${{ matrix.platform }} runs-on: ${{ matrix.platform }} - timeout-minutes: 10 + timeout-minutes: 20 strategy: fail-fast: false matrix: diff --git a/src/3rdParty/CMakeLists.txt b/src/3rdParty/CMakeLists.txt new file mode 100644 index 00000000..34838019 --- /dev/null +++ b/src/3rdParty/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(minio-cpp) \ No newline at end of file diff --git a/src/3rdParty/minio-cpp/CMakeLists.txt b/src/3rdParty/minio-cpp/CMakeLists.txt new file mode 100644 index 00000000..6e408e7a --- /dev/null +++ b/src/3rdParty/minio-cpp/CMakeLists.txt @@ -0,0 +1,45 @@ +set(pwd ${CMAKE_CURRENT_LIST_DIR}) + +set(lib "unknown-minio-cpp-lib") + +if(WIN32) + set(lib "${pwd}/lib/win64/miniocpp.lib") + set(libd "${pwd}/lib/win64/miniocpp-debug.lib") +elseif(APPLE) + set(lib "${pwd}/lib/osx/libminiocpp.a") + set(libd "${pwd}/lib/osx/libminiocpp-debug.a") +elseif(LINUX) + set(lib "${pwd}/lib/linux-amd64/libminiocpp.a") + set(libd "${pwd}/lib/linux-amd64/libminiocpp-debug.a") +endif() + +find_package(OpenSSL REQUIRED) +find_package(unofficial-curlpp CONFIG REQUIRED) +find_package(unofficial-inih CONFIG REQUIRED) +find_package(nlohmann_json CONFIG REQUIRED) +find_package(pugixml CONFIG REQUIRED) +find_package(ZLIB REQUIRED) + +list(APPEND MINIO_CPP_LIBS + unofficial::curlpp::curlpp + unofficial::inih::inireader + nlohmann_json::nlohmann_json + pugixml + OpenSSL::SSL OpenSSL::Crypto + ZLIB::ZLIB +) + +if (WIN32) + list(APPEND MINIO_CPP_LIBS wsock32) + list(APPEND MINIO_CPP_LIBS ws2_32) +endif() + +message(STATUS "minio-cpp: ${pwd}") +set(tgt miniocpp::miniocpp) +add_library(${tgt} STATIC IMPORTED GLOBAL) +target_include_directories(${tgt} INTERFACE ${pwd}/inc) +target_link_libraries(${tgt} INTERFACE ${MINIO_CPP_LIBS}) +set_target_properties(${tgt} PROPERTIES + IMPORTED_LOCATION ${lib} + IMPORTED_LOCATION_DEBUG ${libd} +) \ No newline at end of file diff --git a/src/3rdParty/minio-cpp/README.md b/src/3rdParty/minio-cpp/README.md new file mode 100644 index 00000000..feb03a07 --- /dev/null +++ b/src/3rdParty/minio-cpp/README.md @@ -0,0 +1,50 @@ +# MinIO C++ client + +This repository includes headers and pre-build binaries for the MinIO C++ client SDK. +It was built from +commit [cd4ef14955201d5ce81e54155ef0980f7c644864](https://github.com/minio/minio-cpp/tree/cd4ef14955201d5ce81e54155ef0980f7c644864). +In the base CMakeLists.txt, it was necessary to change `STATIC` to `MODULE` in the `add_library` command to build the +shared library. + +## Building + +### Windows + +To generate the solution file, run the following command: + +```shell +cmake . -B build -DMINIO_CPP_TEST=OFF -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" -DVCPKG_INSTALLED_DIR=vcpkg_installed -DVCPKG_TARGET_TRIPLET=x64-windows-static +``` + +Then, open the solution file in Visual Studio. +Open the Solution Explorer, select `miniocpp` and right-click on it. +Select `Configuration Properties > C/C++ > Code Generation`. +For Release type builds, include RelWithDebInfo, change the `Runtime Library` to `Multi-threaded (/MT)` in order to +statically link the MSVC runtime. +For Debug builds, change the `Runtime Library` to `Multi-threaded Debug (/MTd)`. +Finally, build the solution. + +### macOS + +To generate the universal binary, you will need to build the library for both x86_64 and arm64 architectures. +Run the following commands: + +```shell +# Build for x86_64 +cmake . -B build-x64 -DMINIO_CPP_TEST=OFF -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" -DVCPKG_INSTALLED_DIR=vcpkg-x64 -DVCPKG_TARGET_TRIPLET=x64-osx -DCMAKE_OSX_ARCHITECTURES="x86_64" -DCMAKE_BUILD_TYPE=Release # or Debug +cmake --build build-x64 --config Release # or Debug +# Repeat the process for arm64 +cmake . -B build-arm64 -DMINIO_CPP_TEST=OFF -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" -DVCPKG_INSTALLED_DIR=vcpkg-arm64 -DVCPKG_TARGET_TRIPLET=arm64-osx -DCMAKE_OSX_ARCHITECTURES="arm64" -DCMAKE_BUILD_TYPE=Release # or Debug +cmake --build build-arm64 --config Release # or Debug +# Create the universal binary +lipo -create build-x64/libminiocpp.a build-arm64/libminiocpp.a -output libminiocpp.a +``` + +### Linux + +To generate the shared library, run the following command: + +```shell +cmake . -B build -DMINIO_CPP_TEST=OFF -DCMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" -DCMAKE_BUILD_TYPE=Release # or Debug +cmake --build build --config Release # or Debug +``` \ No newline at end of file diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/args.h b/src/3rdParty/minio-cpp/inc/miniocpp/args.h new file mode 100644 index 00000000..d1c4d129 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/args.h @@ -0,0 +1,606 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_ARGS_H_INCLUDED +#define MINIO_CPP_ARGS_H_INCLUDED + +#include +#include +#include +#include +#include + +#include "error.h" +#include "http.h" +#include "sse.h" +#include "types.h" +#include "utils.h" + +namespace minio::s3 { + +struct BaseArgs { + utils::Multimap extra_headers; + utils::Multimap extra_query_params; + + BaseArgs() = default; + ~BaseArgs() = default; +}; // struct BaseArgs + +struct BucketArgs : public BaseArgs { + std::string bucket; + std::string region; + + BucketArgs() = default; + ~BucketArgs() = default; + + error::Error Validate() const; +}; // struct BucketArgs + +struct ObjectArgs : public BucketArgs { + std::string object; + + ObjectArgs() = default; + ~ObjectArgs() = default; + + error::Error Validate() const; +}; // struct ObjectArgs + +struct ObjectWriteArgs : public ObjectArgs { + utils::Multimap headers; + utils::Multimap user_metadata; + Sse* sse = nullptr; + std::map tags; + Retention* retention = nullptr; + bool legal_hold = false; + + ObjectWriteArgs() = default; + ~ObjectWriteArgs() = default; + + utils::Multimap Headers() const; +}; // struct ObjectWriteArgs + +struct ObjectVersionArgs : public ObjectArgs { + std::string version_id; + + ObjectVersionArgs() = default; + ~ObjectVersionArgs() = default; +}; // struct ObjectVersionArgs + +struct ObjectReadArgs : public ObjectVersionArgs { + SseCustomerKey* ssec = nullptr; + + ObjectReadArgs() = default; + ~ObjectReadArgs() = default; +}; // struct ObjectReadArgs + +struct ObjectConditionalReadArgs : public ObjectReadArgs { + size_t* offset = nullptr; + size_t* length = nullptr; + std::string match_etag; + std::string not_match_etag; + utils::UtcTime modified_since; + utils::UtcTime unmodified_since; + + ObjectConditionalReadArgs() = default; + ~ObjectConditionalReadArgs() = default; + + utils::Multimap Headers() const; + utils::Multimap CopyHeaders() const; +}; // struct ObjectConditionalReadArgs + +struct MakeBucketArgs : public BucketArgs { + bool object_lock = false; + + MakeBucketArgs() = default; + ~MakeBucketArgs() = default; + + error::Error Validate() const; +}; // struct MakeBucketArgs + +using ListBucketsArgs = BaseArgs; + +using BucketExistsArgs = BucketArgs; + +using RemoveBucketArgs = BucketArgs; + +struct AbortMultipartUploadArgs : public ObjectArgs { + std::string upload_id; + + AbortMultipartUploadArgs() = default; + ~AbortMultipartUploadArgs() = default; + + error::Error Validate() const; +}; // struct AbortMultipartUploadArgs + +struct CompleteMultipartUploadArgs : public ObjectArgs { + std::string upload_id; + std::list parts; + + CompleteMultipartUploadArgs() = default; + ~CompleteMultipartUploadArgs() = default; + + error::Error Validate() const; +}; // struct CompleteMultipartUploadArgs + +struct CreateMultipartUploadArgs : public ObjectArgs { + utils::Multimap headers; + + CreateMultipartUploadArgs() = default; + ~CreateMultipartUploadArgs() = default; +}; // struct CreateMultipartUploadArgs + +struct PutObjectBaseArgs : public ObjectWriteArgs { + long object_size = -1; + size_t part_size = 0; + long part_count = 0; + std::string content_type; + + PutObjectBaseArgs() = default; + ~PutObjectBaseArgs() = default; +}; // struct PutObjectBaseArgs + +struct PutObjectApiArgs : public PutObjectBaseArgs { + std::string_view data; + utils::Multimap query_params; + http::ProgressFunction progressfunc = nullptr; + void* progress_userdata = nullptr; + + PutObjectApiArgs() = default; + ~PutObjectApiArgs() = default; +}; // struct PutObjectApiArgs + +struct UploadPartArgs : public ObjectWriteArgs { + std::string upload_id; + unsigned int part_number; + std::string_view data; + http::ProgressFunction progressfunc = nullptr; + void* progress_userdata = nullptr; + + UploadPartArgs() = default; + ~UploadPartArgs() = default; + + error::Error Validate() const; +}; // struct UploadPartArgs + +struct UploadPartCopyArgs : public ObjectWriteArgs { + std::string upload_id; + unsigned int part_number; + utils::Multimap headers; + + UploadPartCopyArgs() = default; + ~UploadPartCopyArgs() = default; + + error::Error Validate() const; +}; // struct UploadPartCopyArgs + +using StatObjectArgs = ObjectConditionalReadArgs; + +using RemoveObjectArgs = ObjectVersionArgs; + +struct DownloadObjectArgs : public ObjectReadArgs { + std::string filename; + bool overwrite; + http::ProgressFunction progressfunc = nullptr; + void* progress_userdata = nullptr; + + DownloadObjectArgs() = default; + ~DownloadObjectArgs() = default; + + error::Error Validate() const; +}; // struct DownloadObjectArgs + +struct GetObjectArgs : public ObjectConditionalReadArgs { + http::DataFunction datafunc; + void* userdata = nullptr; + http::ProgressFunction progressfunc = nullptr; + void* progress_userdata = nullptr; + + GetObjectArgs() = default; + ~GetObjectArgs() = default; + + error::Error Validate() const; +}; // struct GetObjectArgs + +struct ListObjectsArgs : public BucketArgs { + std::string delimiter; + bool use_url_encoding_type = true; + std::string marker; // only for ListObjectsV1. + std::string start_after; // only for ListObjectsV2. + std::string key_marker; // only for GetObjectVersions. + unsigned int max_keys = 1000; + std::string prefix; + std::string continuation_token; // only for ListObjectsV2. + bool fetch_owner = false; // only for ListObjectsV2. + std::string version_id_marker; // only for GetObjectVersions. + bool include_user_metadata = false; // MinIO extension for ListObjectsV2. + bool recursive = false; + bool use_api_v1 = false; + bool include_versions = false; + + ListObjectsArgs() = default; + ~ListObjectsArgs() = default; + + ListObjectsArgs(const ListObjectsArgs&) = default; + ListObjectsArgs& operator=(const ListObjectsArgs&) = default; + + ListObjectsArgs(ListObjectsArgs&&) = default; + ListObjectsArgs& operator=(ListObjectsArgs&&) = default; +}; // struct ListObjectsArgs + +struct ListObjectsCommonArgs : public BucketArgs { + std::string delimiter; + std::string encoding_type; + unsigned int max_keys = 1000; + std::string prefix; + + ListObjectsCommonArgs() = default; + ~ListObjectsCommonArgs() = default; + + ListObjectsCommonArgs(const ListObjectsCommonArgs&) = default; + ListObjectsCommonArgs& operator=(const ListObjectsCommonArgs&) = default; + + ListObjectsCommonArgs(ListObjectsCommonArgs&&) = default; + ListObjectsCommonArgs& operator=(ListObjectsCommonArgs&&) = default; +}; // struct ListObjectsCommonArgs + +struct ListObjectsV1Args : public ListObjectsCommonArgs { + std::string marker; + + ListObjectsV1Args(); + + explicit ListObjectsV1Args(ListObjectsArgs args); + ListObjectsV1Args& operator=(ListObjectsArgs args); + + ~ListObjectsV1Args() = default; + + ListObjectsV1Args(const ListObjectsV1Args&) = default; + ListObjectsV1Args& operator=(const ListObjectsV1Args&) = default; + + ListObjectsV1Args(ListObjectsV1Args&&) = default; + ListObjectsV1Args& operator=(ListObjectsV1Args&&) = default; +}; // struct ListObjectsV1Args + +struct ListObjectsV2Args : public ListObjectsCommonArgs { + std::string start_after; + std::string continuation_token; + bool fetch_owner = false; + bool include_user_metadata = false; + + ListObjectsV2Args(); + + explicit ListObjectsV2Args(ListObjectsArgs args); + ListObjectsV2Args& operator=(ListObjectsArgs args); + + ~ListObjectsV2Args() = default; + + ListObjectsV2Args(const ListObjectsV2Args&) = default; + ListObjectsV2Args& operator=(const ListObjectsV2Args&) = default; + + ListObjectsV2Args(ListObjectsV2Args&&) = default; + ListObjectsV2Args& operator=(ListObjectsV2Args&&) = default; +}; // struct ListObjectsV2Args + +struct ListObjectVersionsArgs : public ListObjectsCommonArgs { + std::string key_marker; + std::string version_id_marker; + + ListObjectVersionsArgs(); + + explicit ListObjectVersionsArgs(ListObjectsArgs args); + ListObjectVersionsArgs& operator=(ListObjectsArgs args); + + ~ListObjectVersionsArgs() = default; + + ListObjectVersionsArgs(const ListObjectVersionsArgs&) = default; + ListObjectVersionsArgs& operator=(const ListObjectVersionsArgs&) = default; + + ListObjectVersionsArgs(ListObjectVersionsArgs&&) = default; + ListObjectVersionsArgs& operator=(ListObjectVersionsArgs&&) = default; +}; // struct ListObjectVersionsArgs + +struct PutObjectArgs : public PutObjectBaseArgs { + std::istream& stream; + http::ProgressFunction progressfunc = nullptr; + void* progress_userdata = nullptr; + + PutObjectArgs(std::istream& stream, long object_size, long part_size); + ~PutObjectArgs() = default; + + error::Error Validate(); +}; // struct PutObjectArgs + +using CopySource = ObjectConditionalReadArgs; + +struct CopyObjectArgs : public ObjectWriteArgs { + CopySource source; + Directive* metadata_directive = nullptr; + Directive* tagging_directive = nullptr; + + CopyObjectArgs() = default; + ~CopyObjectArgs() = default; + + error::Error Validate() const; +}; // struct CopyObjectArgs + +struct ComposeSource : public ObjectConditionalReadArgs { + ComposeSource() = default; + ~ComposeSource() = default; + + error::Error BuildHeaders(size_t object_size, const std::string& etag); + size_t ObjectSize() const; + utils::Multimap Headers() const; + + private: + long object_size_ = -1; + utils::Multimap headers_; +}; // struct ComposeSource + +struct ComposeObjectArgs : public ObjectWriteArgs { + std::list sources; + + ComposeObjectArgs() = default; + ~ComposeObjectArgs() = default; + + error::Error Validate() const; +}; // struct ComposeObjectArgs + +struct UploadObjectArgs : public PutObjectBaseArgs { + std::string filename; + http::ProgressFunction progressfunc = nullptr; + void* progress_userdata = nullptr; + + UploadObjectArgs() = default; + ~UploadObjectArgs() = default; + + error::Error Validate(); +}; // struct UploadObjectArgs + +struct RemoveObjectsApiArgs : public BucketArgs { + bool bypass_governance_mode = false; + bool quiet = true; + std::list objects; + + RemoveObjectsApiArgs() = default; + ~RemoveObjectsApiArgs() = default; +}; // struct RemoveObjectsApiArgs + +using DeleteObjectFunction = std::function; + +struct RemoveObjectsArgs : public BucketArgs { + bool bypass_governance_mode = false; + DeleteObjectFunction func = nullptr; + + RemoveObjectsArgs() = default; + ~RemoveObjectsArgs() = default; + + error::Error Validate() const; +}; // struct RemoveObjectsArgs + +struct SelectObjectContentArgs : public ObjectReadArgs { + SelectRequest& request; + SelectResultFunction resultfunc = nullptr; + + SelectObjectContentArgs(SelectRequest& req, SelectResultFunction func) + : request(req), resultfunc(func) {} + + ~SelectObjectContentArgs() = default; + + error::Error Validate() const; +}; // struct SelectObjectContentArgs + +struct ListenBucketNotificationArgs : public BucketArgs { + std::string prefix; + std::string suffix; + std::list events; + NotificationRecordsFunction func = nullptr; + + ListenBucketNotificationArgs() = default; + ~ListenBucketNotificationArgs() = default; + + error::Error Validate() const; +}; // struct ListenBucketNotificationArgs + +using DeleteBucketPolicyArgs = BucketArgs; + +using GetBucketPolicyArgs = BucketArgs; + +struct SetBucketPolicyArgs : public BucketArgs { + std::string policy; + + SetBucketPolicyArgs() = default; + ~SetBucketPolicyArgs() = default; + + error::Error Validate() const; +}; // struct SetBucketPolicy + +using DeleteBucketNotificationArgs = BucketArgs; + +using GetBucketNotificationArgs = BucketArgs; + +struct SetBucketNotificationArgs : public BucketArgs { + NotificationConfig& config; + + explicit SetBucketNotificationArgs(NotificationConfig& configvalue) + : config(configvalue) {} + + ~SetBucketNotificationArgs() = default; +}; // struct SetBucketNotification + +using DeleteBucketEncryptionArgs = BucketArgs; + +using GetBucketEncryptionArgs = BucketArgs; + +struct SetBucketEncryptionArgs : public BucketArgs { + SseConfig& config; + + explicit SetBucketEncryptionArgs(SseConfig& sseconfig) : config(sseconfig) {} + + ~SetBucketEncryptionArgs() = default; + + error::Error Validate() const; +}; // struct SetBucketEncryption + +using GetBucketVersioningArgs = BucketArgs; + +struct SetBucketVersioningArgs : public BucketArgs { + Boolean status; + Boolean mfa_delete; + + SetBucketVersioningArgs() = default; + ~SetBucketVersioningArgs() = default; + + error::Error Validate() const; +}; // struct SetBucketVersioning + +using DeleteBucketReplicationArgs = BucketArgs; + +using GetBucketReplicationArgs = BucketArgs; + +struct SetBucketReplicationArgs : public BucketArgs { + ReplicationConfig& config; + + explicit SetBucketReplicationArgs(ReplicationConfig& value) : config(value) {} + + ~SetBucketReplicationArgs() = default; +}; // struct SetBucketReplication + +using DeleteBucketLifecycleArgs = BucketArgs; + +using GetBucketLifecycleArgs = BucketArgs; + +struct SetBucketLifecycleArgs : public BucketArgs { + LifecycleConfig& config; + + explicit SetBucketLifecycleArgs(LifecycleConfig& value) : config(value) {} + + ~SetBucketLifecycleArgs() = default; +}; // struct SetBucketLifecycle + +using DeleteBucketTagsArgs = BucketArgs; + +using GetBucketTagsArgs = BucketArgs; + +struct SetBucketTagsArgs : public BucketArgs { + std::map tags; + + SetBucketTagsArgs() = default; + ~SetBucketTagsArgs() = default; + + error::Error Validate() const; +}; // struct SetBucketTags + +using DeleteObjectLockConfigArgs = BucketArgs; + +using GetObjectLockConfigArgs = BucketArgs; + +struct SetObjectLockConfigArgs : public BucketArgs { + ObjectLockConfig config; + + SetObjectLockConfigArgs() = default; + ~SetObjectLockConfigArgs() = default; + + error::Error Validate() const; +}; // struct SetObjectLockConfig + +using DeleteObjectTagsArgs = ObjectVersionArgs; + +using GetObjectTagsArgs = ObjectVersionArgs; + +struct SetObjectTagsArgs : public ObjectVersionArgs { + std::map tags; + + SetObjectTagsArgs() = default; + ~SetObjectTagsArgs() = default; + + error::Error Validate() const; +}; // struct SetObjectTags + +using EnableObjectLegalHoldArgs = ObjectVersionArgs; + +using DisableObjectLegalHoldArgs = ObjectVersionArgs; + +using IsObjectLegalHoldEnabledArgs = ObjectVersionArgs; + +using GetObjectRetentionArgs = ObjectVersionArgs; + +struct SetObjectRetentionArgs : public ObjectVersionArgs { + RetentionMode retention_mode; + utils::UtcTime retain_until_date; + + SetObjectRetentionArgs() = default; + ~SetObjectRetentionArgs() = default; + + error::Error Validate() const; +}; // struct SetObjectRetention + +inline constexpr unsigned int kDefaultExpirySeconds = + (60 * 60 * 24 * 7); // 7 days + +struct GetPresignedObjectUrlArgs : public ObjectVersionArgs { + http::Method method; + unsigned int expiry_seconds = kDefaultExpirySeconds; + utils::UtcTime request_time; + + GetPresignedObjectUrlArgs() = default; + ~GetPresignedObjectUrlArgs() = default; + + error::Error Validate() const; +}; // struct GetPresignedObjectUrlArgs + +struct PostPolicy { + std::string bucket; + std::string region; + + PostPolicy(std::string bucket, utils::UtcTime expiration) + : bucket(std::move(bucket)), expiration_(std::move(expiration)) {} + + ~PostPolicy() = default; + + explicit operator bool() const { return !bucket.empty() && expiration_; } + + error::Error AddEqualsCondition(std::string element, std::string value); + error::Error RemoveEqualsCondition(std::string element); + error::Error AddStartsWithCondition(std::string element, std::string value); + error::Error RemoveStartsWithCondition(std::string element); + error::Error AddContentLengthRangeCondition(size_t lower_limit, + size_t upper_limit); + void RemoveContentLengthRangeCondition(); + + error::Error FormData(std::map& data, + std::string access_key, std::string secret_key, + std::string session_token, std::string region); + + private: + static constexpr const char* eq_ = "eq"; + static constexpr const char* starts_with_ = "starts-with"; + static constexpr const char* algorithm_ = "AWS4-HMAC-SHA256"; + + utils::UtcTime expiration_; + std::map> conditions_; + Integer lower_limit_; + Integer upper_limit_; + + static std::string trimDollar(std::string value); + static std::string getCredentialString(std::string access_key, + utils::UtcTime date, + std::string region); + static bool isReservedElement(std::string element); +}; // struct PostPolicy + +} // namespace minio::s3 + +#endif // _MINIO_CPP_ARGS_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/baseclient.h b/src/3rdParty/minio-cpp/inc/miniocpp/baseclient.h new file mode 100644 index 00000000..87b2c375 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/baseclient.h @@ -0,0 +1,157 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_BASECLIENT_H_INCLUDED +#define MINIO_CPP_BASECLIENT_H_INCLUDED + +#include +#include +#include + +#include "args.h" +#include "config.h" +#include "error.h" +#include "http.h" +#include "providers.h" +#include "request.h" +#include "response.h" +#include "utils.h" + +namespace minio::s3 { + +utils::Multimap GetCommonListObjectsQueryParams( + const std::string& delimiter, const std::string& encoding_type, + unsigned int max_keys, const std::string& prefix); + +/** + * Base client to perform S3 APIs. + */ +class BaseClient { + protected: + BaseUrl base_url_; + creds::Provider* const provider_ = nullptr; + std::map region_map_; + bool debug_ = false; + bool ignore_cert_check_ = false; + std::string ssl_cert_file_; + std::string user_agent_ = DEFAULT_USER_AGENT; + + public: + explicit BaseClient(BaseUrl base_url, + creds::Provider* const provider = nullptr); + + virtual ~BaseClient() = default; + + void Debug(bool flag) { debug_ = flag; } + + void IgnoreCertCheck(bool flag) { ignore_cert_check_ = flag; } + + void SetSslCertFile(std::string ssl_cert_file) { + ssl_cert_file_ = std::move(ssl_cert_file); + } + + error::Error SetAppInfo(std::string_view app_name, + std::string_view app_version); + + void HandleRedirectResponse(std::string& code, std::string& message, + int status_code, http::Method method, + const utils::Multimap& headers, + const std::string& bucket_name, + bool retry = false); + Response GetErrorResponse(http::Response resp, std::string_view resource, + http::Method method, const std::string& bucket_name, + const std::string& object_name); + Response execute(Request& req); + Response Execute(Request& req); + GetRegionResponse GetRegion(const std::string& bucket_name, + const std::string& region); + + AbortMultipartUploadResponse AbortMultipartUpload( + AbortMultipartUploadArgs args); + BucketExistsResponse BucketExists(BucketExistsArgs args); + CompleteMultipartUploadResponse CompleteMultipartUpload( + CompleteMultipartUploadArgs args); + CreateMultipartUploadResponse CreateMultipartUpload( + CreateMultipartUploadArgs args); + DeleteBucketEncryptionResponse DeleteBucketEncryption( + DeleteBucketEncryptionArgs args); + DisableObjectLegalHoldResponse DisableObjectLegalHold( + DisableObjectLegalHoldArgs args); + DeleteBucketLifecycleResponse DeleteBucketLifecycle( + DeleteBucketLifecycleArgs args); + DeleteBucketNotificationResponse DeleteBucketNotification( + DeleteBucketNotificationArgs args); + DeleteBucketPolicyResponse DeleteBucketPolicy(DeleteBucketPolicyArgs args); + DeleteBucketReplicationResponse DeleteBucketReplication( + DeleteBucketReplicationArgs args); + DeleteBucketTagsResponse DeleteBucketTags(DeleteBucketTagsArgs args); + DeleteObjectLockConfigResponse DeleteObjectLockConfig( + DeleteObjectLockConfigArgs args); + DeleteObjectTagsResponse DeleteObjectTags(DeleteObjectTagsArgs args); + EnableObjectLegalHoldResponse EnableObjectLegalHold( + EnableObjectLegalHoldArgs args); + GetBucketEncryptionResponse GetBucketEncryption(GetBucketEncryptionArgs args); + GetBucketLifecycleResponse GetBucketLifecycle(GetBucketLifecycleArgs args); + GetBucketNotificationResponse GetBucketNotification( + GetBucketNotificationArgs args); + GetBucketPolicyResponse GetBucketPolicy(GetBucketPolicyArgs args); + GetBucketReplicationResponse GetBucketReplication( + GetBucketReplicationArgs args); + GetBucketTagsResponse GetBucketTags(GetBucketTagsArgs args); + GetBucketVersioningResponse GetBucketVersioning(GetBucketVersioningArgs args); + GetObjectResponse GetObject(GetObjectArgs args); + GetObjectLockConfigResponse GetObjectLockConfig(GetObjectLockConfigArgs args); + GetObjectRetentionResponse GetObjectRetention(GetObjectRetentionArgs args); + GetObjectTagsResponse GetObjectTags(GetObjectTagsArgs args); + GetPresignedObjectUrlResponse GetPresignedObjectUrl( + GetPresignedObjectUrlArgs args); + GetPresignedPostFormDataResponse GetPresignedPostFormData(PostPolicy policy); + IsObjectLegalHoldEnabledResponse IsObjectLegalHoldEnabled( + IsObjectLegalHoldEnabledArgs args); + ListBucketsResponse ListBuckets(ListBucketsArgs args); + ListBucketsResponse ListBuckets(); + ListenBucketNotificationResponse ListenBucketNotification( + ListenBucketNotificationArgs args); + ListObjectsResponse ListObjectsV1(ListObjectsV1Args args); + ListObjectsResponse ListObjectsV2(ListObjectsV2Args args); + ListObjectsResponse ListObjectVersions(ListObjectVersionsArgs args); + MakeBucketResponse MakeBucket(MakeBucketArgs args); + PutObjectResponse PutObject(PutObjectApiArgs args); + RemoveBucketResponse RemoveBucket(RemoveBucketArgs args); + RemoveObjectResponse RemoveObject(RemoveObjectArgs args); + RemoveObjectsResponse RemoveObjects(RemoveObjectsApiArgs args); + SetBucketEncryptionResponse SetBucketEncryption(SetBucketEncryptionArgs args); + SetBucketLifecycleResponse SetBucketLifecycle(SetBucketLifecycleArgs args); + SetBucketNotificationResponse SetBucketNotification( + SetBucketNotificationArgs args); + SetBucketPolicyResponse SetBucketPolicy(SetBucketPolicyArgs args); + SetBucketReplicationResponse SetBucketReplication( + SetBucketReplicationArgs args); + SetBucketTagsResponse SetBucketTags(SetBucketTagsArgs args); + SetBucketVersioningResponse SetBucketVersioning(SetBucketVersioningArgs args); + SetObjectLockConfigResponse SetObjectLockConfig(SetObjectLockConfigArgs args); + SetObjectRetentionResponse SetObjectRetention(SetObjectRetentionArgs args); + SetObjectTagsResponse SetObjectTags(SetObjectTagsArgs args); + SelectObjectContentResponse SelectObjectContent(SelectObjectContentArgs args); + StatObjectResponse StatObject(StatObjectArgs args); + UploadPartResponse UploadPart(UploadPartArgs args); + UploadPartCopyResponse UploadPartCopy(UploadPartCopyArgs args); +}; // class BaseClient + +} // namespace minio::s3 + +#endif // MINIO_CPP_BASECLIENT_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/client.h b/src/3rdParty/minio-cpp/inc/miniocpp/client.h new file mode 100644 index 00000000..5820e168 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/client.h @@ -0,0 +1,129 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_CLIENT_H_INCLUDED +#define MINIO_CPP_CLIENT_H_INCLUDED + +#include +#include + +#include "args.h" +#include "baseclient.h" +#include "error.h" +#include "providers.h" +#include "request.h" +#include "response.h" + +namespace minio::s3 { + +class Client; + +class ListObjectsResult { + private: + Client* client_ = nullptr; + ListObjectsArgs args_; + bool failed_ = false; + ListObjectsResponse resp_; + std::list::iterator itr_; + + void Populate(); + + public: + explicit ListObjectsResult(error::Error err); + ListObjectsResult(Client* const client, const ListObjectsArgs& args); + ListObjectsResult(Client* const client, ListObjectsArgs&& args); + ~ListObjectsResult() = default; + + Item& operator*() const { return *itr_; } + explicit operator bool() const { return itr_ != resp_.contents.end(); } + + ListObjectsResult& operator++() { + itr_++; + if (!failed_ && itr_ == resp_.contents.end() && resp_.is_truncated) { + Populate(); + } + return *this; + } + + ListObjectsResult operator++(int) { + ListObjectsResult curr = *this; + ++(*this); + return curr; + } +}; // class ListObjectsResult + +class RemoveObjectsResult { + private: + Client* client_ = nullptr; + RemoveObjectsArgs args_; + bool done_ = false; + RemoveObjectsResponse resp_; + std::list::iterator itr_; + + void Populate(); + + public: + explicit RemoveObjectsResult(error::Error err); + RemoveObjectsResult(Client* const client, const RemoveObjectsArgs& args); + RemoveObjectsResult(Client* const client, RemoveObjectsArgs&& args); + ~RemoveObjectsResult() = default; + + DeleteError& operator*() const { return *itr_; } + explicit operator bool() const { return itr_ != resp_.errors.end(); } + RemoveObjectsResult& operator++() { + itr_++; + if (!done_ && itr_ == resp_.errors.end()) { + Populate(); + } + return *this; + } + RemoveObjectsResult operator++(int) { + RemoveObjectsResult curr = *this; + ++(*this); + return curr; + } +}; // class RemoveObjectsResult + +/** + * Simple Storage Service (aka S3) client to perform bucket and object + * operations. + */ +class Client : public BaseClient { + protected: + StatObjectResponse CalculatePartCount(size_t& part_count, + std::list sources); + ComposeObjectResponse ComposeObject(ComposeObjectArgs args, + std::string& upload_id); + PutObjectResponse PutObject(PutObjectArgs args, std::string& upload_id, + char* buf); + + public: + explicit Client(BaseUrl& base_url, creds::Provider* const provider = nullptr); + ~Client() = default; + + ComposeObjectResponse ComposeObject(ComposeObjectArgs args); + CopyObjectResponse CopyObject(CopyObjectArgs args); + DownloadObjectResponse DownloadObject(DownloadObjectArgs args); + ListObjectsResult ListObjects(ListObjectsArgs args); + PutObjectResponse PutObject(PutObjectArgs args); + UploadObjectResponse UploadObject(UploadObjectArgs args); + RemoveObjectsResult RemoveObjects(RemoveObjectsArgs args); +}; // class Client + +} // namespace minio::s3 + +#endif // MINIO_CPP_CLIENT_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/config.h b/src/3rdParty/minio-cpp/inc/miniocpp/config.h new file mode 100644 index 00000000..25bf10ba --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/config.h @@ -0,0 +1,76 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_CONFIG_H_INCLUDED +#define MINIO_CPP_CONFIG_H_INCLUDED + +#define MINIO_CPP_STRINGIFY(x) #x + +#define MINIO_CPP_MAJOR_VERSION 0 +#define MINIO_CPP_MINOR_VERSION 2 +#define MINIO_CPP_PATCH_VERSION 0 + +#define MINIO_CPP_VERSION \ + "" MINIO_CPP_STRINGIFY(MINIO_CPP_MAJOR_VERSION) "." MINIO_CPP_STRINGIFY( \ + MINIO_CPP_MINOR_VERSION) "." MINIO_CPP_STRINGIFY(MINIO_CPP_PATCH_VERSION) + +#if defined(_M_X64) || defined(__x86_64__) +#define MINIO_CPP_ARCH_STRING "x86_64" +#elif defined(_M_IX86) || defined(__X86__) || defined(__i386__) +#define MINIO_CPP_ARCH_STRING "x86" +#elif defined(_M_ARM64) || defined(__arm64__) || defined(__aarch64__) +#define MINIO_CPP_ARCH_STRING "arm64" +#elif defined(_M_ARM) || defined(_M_ARMT) || defined(__arm__) || \ + defined(__thumb__) || defined(__thumb2__) +#define MINIO_CPP_ARCH_STRING "arm32" +#elif defined(_MIPS_ARCH_MIPS64) || defined(__mips64) +#define MINIO_CPP_ARCH_STRING "mips64" +#elif defined(_MIPS_ARCH_MIPS32) || defined(_M_MRX000) || defined(__mips__) +#define MINIO_CPP_ARCH_STRING "mips32" +#elif (defined(__riscv) || defined(__riscv__)) && defined(__riscv_xlen) +#define MINIO_CPP_ARCH_STRING "riscv" MINIO_CPP_STRINGIFY(__riscv_xlen) +#elif defined(__loongarch__) +#define MINIO_CPP_ARCH_STRING "loongarch" +#elif defined(__s390__) || defined(__s390x__) +#define MINIO_CPP_ARCH_STRING "s390" +#else +#define MINIO_CPP_ARCH_STRING "unknown-arch" +#endif + +#if defined(_WIN32) || defined(__CYGWIN__) +#define MINIO_CPP_PLATFORM_STRING "windows" +#elif defined(__ANDROID__) +#define MINIO_CPP_PLATFORM_STRING "android" +#elif defined(__linux__) || defined(__linux) +#define MINIO_CPP_PLATFORM_STRING "linux" +#elif defined(__APPLE__) || defined(__MACH__) +#define MINIO_CPP_PLATFORM_STRING "darwin" +#elif defined(__FreeBSD__) +#define MINIO_CPP_PLATFORM_STRING "freebsd" +#elif defined(__NetBSD__) +#define MINIO_CPP_PLATFORM_STRING "netbsd" +#elif defined(__OpenBSD__) +#define MINIO_CPP_PLATFORM_STRING "openbsd" +#else +#define MINIO_CPP_PLATFORM_STRING "unknown-os" +#endif + +#define DEFAULT_USER_AGENT \ + "MinIO (" MINIO_CPP_PLATFORM_STRING "; " MINIO_CPP_ARCH_STRING \ + ") minio-cpp/" MINIO_CPP_VERSION "" + +#endif // MINIO_CPP_CONFIG_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/credentials.h b/src/3rdParty/minio-cpp/inc/miniocpp/credentials.h new file mode 100644 index 00000000..bcf0fb64 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/credentials.h @@ -0,0 +1,80 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_CREDENTIALS_H_INCLUDED +#define MINIO_CPP_CREDENTIALS_H_INCLUDED + +#include +#include + +#include "error.h" +#include "utils.h" + +namespace minio::creds { + +bool expired(const utils::UtcTime& expiration); + +/** + * Credentials contains access key and secret key with optional session token + * and expiration. + */ +struct Credentials { + error::Error err; + std::string access_key = {}; + std::string secret_key = {}; + std::string session_token = {}; + utils::UtcTime expiration = {}; + + Credentials() = default; + explicit Credentials(error::Error err) : err(std::move(err)) {} + + explicit Credentials(error::Error err, std::string access_key, + std::string secret_key) + : err(std::move(err)), + access_key(std::move(access_key)), + secret_key(std::move(secret_key)) {} + + explicit Credentials(error::Error err, std::string access_key, + std::string secret_key, std::string session_token) + : err(std::move(err)), + access_key(std::move(access_key)), + secret_key(std::move(secret_key)), + session_token(std::move(session_token)) {} + + explicit Credentials(error::Error err, std::string access_key, + std::string secret_key, std::string session_token, + utils::UtcTime expiration) + : err(std::move(err)), + access_key(std::move(access_key)), + secret_key(std::move(secret_key)), + session_token(std::move(session_token)), + expiration(std::move(expiration)) {} + + ~Credentials() = default; + + bool IsExpired() const { return expired(expiration); } + + explicit operator bool() const { + return !err && !access_key.empty() && expired(expiration); + } + + static Credentials ParseXML(std::string_view data, const std::string& root); +}; // class Credentials + +} // namespace minio::creds + +#endif // MINIO_CPP_CREDENTIALS_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/error.h b/src/3rdParty/minio-cpp/inc/miniocpp/error.h new file mode 100644 index 00000000..f90a4734 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/error.h @@ -0,0 +1,61 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_ERROR_H_INCLUDED +#define MINIO_CPP_ERROR_H_INCLUDED + +#include +#include +#include + +namespace minio::error { + +class Error { + private: + std::string msg_; + + public: + Error() = default; + + explicit Error(std::string msg) : msg_(std::move(msg)) {} + + Error(const Error&) = default; + Error(Error&& v) = default; + + Error& operator=(const Error&) = default; + Error& operator=(Error&& v) = default; + + ~Error() = default; + + const std::string& String() const { return msg_; } + explicit operator bool() const { return !msg_.empty(); } + + friend std::ostream& operator<<(std::ostream& s, const Error& e) { + return s << e.msg_; + } +}; // class Error + +extern const Error SUCCESS; + +template +inline T_RESULT make(TA&&... args) { + return T_RESULT{Error(std::forward(args)...)}; +} + +} // namespace minio::error + +#endif // MINIO_CPP_ERROR_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/http.h b/src/3rdParty/minio-cpp/inc/miniocpp/http.h new file mode 100644 index 00000000..a3754a3d --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/http.h @@ -0,0 +1,167 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_HTTP_H_INCLUDED +#define MINIO_CPP_HTTP_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include + +#include "error.h" +#include "utils.h" + +namespace minio::http { + +enum class Method { kGet, kHead, kPost, kPut, kDelete }; + +//! MethodToString converts http Method enum to string. +const char* MethodToString(Method method) noexcept; + +/** + * Url represents HTTP URL and it's components. + */ +struct Url { + bool https = false; + std::string host; + unsigned int port = 0; + std::string path; + std::string query_string; + + Url() = default; + explicit Url(bool https, std::string host, unsigned int port, + std::string path, std::string query_string) + : https(https), + host(std::move(host)), + port(port), + path(std::move(path)), + query_string(std::move(query_string)) {} + ~Url() = default; + + explicit operator bool() const { return !host.empty(); } + + std::string String() const; + std::string HostHeaderValue() const; + static Url Parse(std::string value); +}; // struct Url + +struct DataFunctionArgs; + +using DataFunction = std::function; + +struct ProgressFunctionArgs; + +using ProgressFunction = std::function; + +struct Response; + +struct DataFunctionArgs { + curlpp::Easy* handle = nullptr; + Response* response = nullptr; + std::string datachunk; + void* userdata = nullptr; + + DataFunctionArgs() = default; + DataFunctionArgs(curlpp::Easy* handle, Response* response, void* userdata) + : handle(handle), response(response), userdata(userdata) {} + DataFunctionArgs(curlpp::Easy* handle, Response* response, + std::string datachunk, void* userdata) + : handle(handle), + response(response), + datachunk(std::move(datachunk)), + userdata(userdata) {} + + ~DataFunctionArgs() = default; +}; // struct DataFunctionArgs + +struct ProgressFunctionArgs { + double download_total_bytes = 0.0; + double downloaded_bytes = 0.0; + double upload_total_bytes = 0.0; + double uploaded_bytes = 0.0; + double download_speed = 0.0; + double upload_speed = 0.0; + void* userdata = nullptr; +}; // struct ProgressFunctionArgs + +struct Request { + Method method; + http::Url url; + utils::Multimap headers; + std::string_view body; + DataFunction datafunc = nullptr; + void* userdata = nullptr; + ProgressFunction progressfunc = nullptr; + void* progress_userdata = nullptr; + bool debug = false; + bool ignore_cert_check = false; + std::string ssl_cert_file; + std::string key_file; + std::string cert_file; + + Request(Method method, Url url); + ~Request() = default; + + Response Execute(); + + explicit operator bool() const { + if (method < Method::kGet || method > Method::kDelete) return false; + return static_cast(url); + } + + private: + Response execute(); +}; // struct Request + +struct Response { + std::string error; + DataFunction datafunc = nullptr; + void* userdata = nullptr; + int status_code = 0; + utils::Multimap headers; + std::string body; + + Response() = default; + ~Response() = default; + + size_t ResponseCallback(curlpp::Multi* const requests, + curlpp::Easy* const request, const char* const buffer, + size_t size, size_t length); + + explicit operator bool() const { + return error.empty() && status_code >= 200 && status_code <= 299; + } + + error::Error Error() const; + + private: + std::string response_; + bool continue100_ = false; + bool status_code_read_ = false; + bool headers_read_ = false; + + error::Error ReadStatusCode(); + error::Error ReadHeaders(); +}; // struct Response + +} // namespace minio::http + +#endif // MINIO_CPP_HTTP_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/providers.h b/src/3rdParty/minio-cpp/inc/miniocpp/providers.h new file mode 100644 index 00000000..4ea4444e --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/providers.h @@ -0,0 +1,259 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_PROVIDERS_H_INCLUDED +#define MINIO_CPP_PROVIDERS_H_INCLUDED + +#include +#include +#include +#include + +#include "credentials.h" +#include "error.h" +#include "http.h" + +namespace minio::creds { + +// 1 day. +static constexpr unsigned DEFAULT_DURATION_SECONDS = 60 * 60 * 24; + +// 15 minutes. +static constexpr unsigned MIN_DURATION_SECONDS = 60 * 15; + +// 7 days. +static constexpr unsigned MAX_DURATION_SECONDS = 60 * 60 * 24 * 7; + +struct Jwt { + std::string token; + unsigned int expiry = 0; + + Jwt() = default; + explicit Jwt(std::string token, unsigned int expiry) + : token(std::move(token)), expiry(expiry) {} + ~Jwt() = default; + + explicit operator bool() const { return !token.empty(); } +}; // struct Jwt + +using JwtFunction = std::function; + +error::Error checkLoopbackHost(const std::string& host); + +/** + * Credential provider interface. + */ +class Provider { + protected: + error::Error err_; + Credentials creds_; + + public: + Provider() = default; + virtual ~Provider(); + + explicit operator bool() const { return !err_; } + + virtual Credentials Fetch() = 0; +}; // class Provider + +class ChainedProvider : public Provider { + private: + std::list providers_; + Provider* provider_ = nullptr; + + public: + explicit ChainedProvider(std::list providers) + : providers_(std::move(providers)) {} + + virtual ~ChainedProvider(); + + virtual Credentials Fetch() override; +}; // class ChainedProvider + +/** + * Static credential provider. + */ +class StaticProvider : public Provider { + public: + StaticProvider(std::string access_key, std::string secret_key, + std::string session_token = {}); + virtual ~StaticProvider(); + + virtual Credentials Fetch() override; +}; // class StaticProvider + +class EnvAwsProvider : public Provider { + public: + EnvAwsProvider(); + virtual ~EnvAwsProvider(); + + virtual Credentials Fetch() override; +}; // class EnvAwsProvider + +class EnvMinioProvider : public Provider { + public: + EnvMinioProvider(); + virtual ~EnvMinioProvider(); + + virtual Credentials Fetch() override; +}; // class EnvMinioProvider + +class AwsConfigProvider : public Provider { + public: + explicit AwsConfigProvider(std::string filename = {}, + std::string profile = {}); + virtual ~AwsConfigProvider(); + + virtual Credentials Fetch() override; +}; // class AwsConfigProvider + +class MinioClientConfigProvider : public Provider { + public: + explicit MinioClientConfigProvider(std::string filename = {}, + std::string alias = {}); + virtual ~MinioClientConfigProvider(); + + virtual Credentials Fetch() override; +}; // class MinioClientConfigProvider + +class AssumeRoleProvider : public Provider { + private: + http::Url sts_endpoint_; + std::string access_key_; + std::string secret_key_; + std::string region_; + std::string body_; + std::string content_sha256_; + + public: + AssumeRoleProvider(http::Url sts_endpoint, std::string access_key, + std::string secret_key, unsigned int duration_seconds = 0, + std::string policy = {}, std::string region = {}, + std::string role_arn = {}, + std::string role_session_name = {}, + std::string external_id = {}); + + virtual ~AssumeRoleProvider(); + + virtual Credentials Fetch() override; +}; // class AssumeRoleProvider + +class WebIdentityClientGrantsProvider : public Provider { + private: + JwtFunction jwtfunc_ = nullptr; + http::Url sts_endpoint_; + unsigned int duration_seconds_ = 0; + std::string policy_; + std::string role_arn_; + std::string role_session_name_; + + public: + WebIdentityClientGrantsProvider(JwtFunction jwtfunc, http::Url sts_endpoint, + unsigned int duration_seconds = 0, + std::string policy = {}, + std::string role_arn = {}, + std::string role_session_name = {}); + + virtual ~WebIdentityClientGrantsProvider(); + + virtual bool IsWebIdentity() const = 0; + + unsigned int getDurationSeconds(unsigned int expiry) const; + + virtual Credentials Fetch() override; +}; // class WebIdentityClientGrantsProvider + +class ClientGrantsProvider : public WebIdentityClientGrantsProvider { + public: + ClientGrantsProvider(JwtFunction jwtfunc, http::Url sts_endpoint, + unsigned int duration_seconds = 0, + std::string policy = {}, std::string role_arn = {}, + std::string role_session_name = {}); + + virtual ~ClientGrantsProvider(); + + virtual bool IsWebIdentity() const override; +}; // class ClientGrantsProvider + +class WebIdentityProvider : public WebIdentityClientGrantsProvider { + public: + WebIdentityProvider(JwtFunction jwtfunc, http::Url sts_endpoint, + unsigned int duration_seconds = 0, + std::string policy = {}, std::string role_arn = {}, + std::string role_session_name = {}); + + virtual ~WebIdentityProvider(); + + virtual bool IsWebIdentity() const override; +}; // class WebIdentityProvider + +class IamAwsProvider : public Provider { + private: + http::Url custom_endpoint_; + std::string token_file_; + std::string aws_region_; + std::string role_arn_; + std::string role_session_name_; + std::string relative_uri_; + std::string full_uri_; + + public: + explicit IamAwsProvider(http::Url custom_endpoint = http::Url()); + virtual ~IamAwsProvider(); + + virtual Credentials Fetch() override; + + private: + Credentials fetch(http::Url url); + error::Error getRoleName(std::string& role_name, http::Url url) const; +}; // class IamAwsProvider + +class LdapIdentityProvider : public Provider { + private: + http::Url sts_endpoint_; + + public: + LdapIdentityProvider(http::Url sts_endpoint, std::string ldap_username, + std::string ldap_password); + + virtual ~LdapIdentityProvider(); + + virtual Credentials Fetch() override; +}; // class LdapIdentityProvider + +struct CertificateIdentityProvider : public Provider { + private: + http::Url sts_endpoint_; + std::string key_file_; + std::string cert_file_; + std::string ssl_cert_file_; + + public: + CertificateIdentityProvider(http::Url sts_endpoint, std::string key_file, + std::string cert_file, + std::string ssl_cert_file = {}, + unsigned int duration_seconds = 0); + + virtual ~CertificateIdentityProvider(); + + virtual Credentials Fetch() override; +}; // struct CertificateIdentityProvider + +} // namespace minio::creds + +#endif // MINIO_CPP_PROVIDERS_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/request.h b/src/3rdParty/minio-cpp/inc/miniocpp/request.h new file mode 100644 index 00000000..2932383c --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/request.h @@ -0,0 +1,115 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_REQUEST_H_INCLUDED +#define MINIO_CPP_REQUEST_H_INCLUDED + +#include + +#include "error.h" +#include "http.h" +#include "providers.h" +#include "utils.h" + +namespace minio::s3 { + +error::Error getAwsInfo(const std::string& host, bool https, + std::string& region, std::string& aws_s3_prefix, + std::string& aws_domain_suffix, bool& dualstack); + +std::string extractRegion(const std::string& host); + +struct BaseUrl { + bool https = true; + std::string host; + unsigned int port = 0; + std::string region; + std::string aws_s3_prefix; + std::string aws_domain_suffix; + bool dualstack = false; + bool virtual_style = false; + + BaseUrl() = default; + explicit BaseUrl(std::string host, bool https = true, + std::string region = {}); + ~BaseUrl() = default; + + error::Error BuildUrl(http::Url& url, http::Method method, + const std::string& region, + const utils::Multimap& query_params, + const std::string& bucket_name, + const std::string& object_name); + + explicit operator bool() const { return !err_ && !host.empty(); } + + error::Error Error() const { + if (host.empty() && !err_) { + return error::Error("empty host"); + } + return err_; + } + + private: + error::Error err_; + + error::Error BuildAwsUrl(http::Url& url, const std::string& bucket_name, + bool enforce_path_style, const std::string& region); + void BuildListBucketsUrl(http::Url& url, const std::string& region); +}; // struct Url + +struct Request { + http::Method method; + std::string region; + BaseUrl& base_url; + + std::string user_agent; + + utils::Multimap headers; + utils::Multimap query_params; + + std::string bucket_name; + std::string object_name; + + std::string_view body; + + http::DataFunction datafunc = nullptr; + void* userdata = nullptr; + + http::ProgressFunction progressfunc = nullptr; + void* progress_userdata = nullptr; + + std::string sha256; + utils::UtcTime date; + + bool debug = false; + bool ignore_cert_check = false; + std::string ssl_cert_file; + + Request(http::Method method, std::string region, BaseUrl& baseurl, + utils::Multimap extra_headers, utils::Multimap extra_query_params); + + ~Request() = default; + + http::Request ToHttpRequest(creds::Provider* const provider = nullptr); + + private: + void BuildHeaders(http::Url& url, creds::Provider* const provider); +}; // struct Request + +} // namespace minio::s3 + +#endif // MINIO_CPP_REQUEST_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/response.h b/src/3rdParty/minio-cpp/inc/miniocpp/response.h new file mode 100644 index 00000000..436a4c23 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/response.h @@ -0,0 +1,575 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_RESPONSE_H_INCLUDED +#define MINIO_CPP_RESPONSE_H_INCLUDED + +#include +#include +#include +#include + +#include "error.h" +#include "types.h" +#include "utils.h" + +namespace minio::s3 { + +struct Response { + public: + int status_code = 0; + utils::Multimap headers; + std::string data; + + std::string code; + std::string message; + std::string resource; + std::string request_id; + std::string host_id; + std::string bucket_name; + std::string object_name; + + private: + error::Error err_; + + public: + Response(); + explicit Response(error::Error err) : err_(std::move(err)) {} + + Response(const Response& resp) = default; + Response& operator=(const Response& resp) = default; + + Response(Response&& resp) = default; + Response& operator=(Response&& resp) = default; + + ~Response(); + + explicit operator bool() const { + return !err_ && code.empty() && message.empty() && + (status_code == 0 || (status_code >= 200 && status_code <= 299)); + } + + error::Error Error() const; + + static Response ParseXML(std::string_view data, int status_code, + utils::Multimap headers); +}; // struct Response + +#define MINIO_S3_DERIVE_FROM_RESPONSE(DerivedName) \ + struct DerivedName : public Response { \ + DerivedName() = default; \ + ~DerivedName() = default; \ + \ + DerivedName(const DerivedName&) = default; \ + DerivedName& operator=(const DerivedName&) = default; \ + \ + DerivedName(DerivedName&&) = default; \ + DerivedName& operator=(DerivedName&&) = default; \ + \ + explicit DerivedName(error::Error err) : Response(std::move(err)) {} \ + explicit DerivedName(const Response& resp) : Response(resp) {} \ + }; + +#define MINIO_S3_DERIVE_FROM_PUT_OBJECT_RESPONSE(DerivedName) \ + struct DerivedName : public PutObjectResponse { \ + DerivedName() = default; \ + ~DerivedName() = default; \ + \ + DerivedName(const DerivedName&) = default; \ + DerivedName& operator=(const DerivedName&) = default; \ + \ + DerivedName(DerivedName&&) = default; \ + DerivedName& operator=(DerivedName&&) = default; \ + \ + explicit DerivedName(error::Error err) \ + : PutObjectResponse(std::move(err)) {} \ + explicit DerivedName(const PutObjectResponse& resp) \ + : PutObjectResponse(resp) {} \ + explicit DerivedName(const Response& resp) : PutObjectResponse(resp) {} \ + \ + explicit DerivedName(const CompleteMultipartUploadResponse& resp) \ + : PutObjectResponse(resp) {} \ + }; + +struct GetRegionResponse : public Response { + std::string region; + + explicit GetRegionResponse(std::string region) : region(std::move(region)) {} + + explicit GetRegionResponse(error::Error err) : Response(std::move(err)) {} + + explicit GetRegionResponse(const Response& resp) : Response(resp) {} + + ~GetRegionResponse() = default; +}; // struct GetRegionResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(MakeBucketResponse) + +struct ListBucketsResponse : public Response { + std::list buckets; + + explicit ListBucketsResponse(std::list buckets) + : buckets(std::move(buckets)) {} + + explicit ListBucketsResponse(error::Error err) : Response(std::move(err)) {} + + explicit ListBucketsResponse(const Response& resp) : Response(resp) {} + + ~ListBucketsResponse() = default; + + static ListBucketsResponse ParseXML(std::string_view data); +}; // struct ListBucketsResponse + +struct BucketExistsResponse : public Response { + bool exist = false; + + explicit BucketExistsResponse(bool exist) : exist(exist) {} + + explicit BucketExistsResponse(error::Error err) : Response(std::move(err)) {} + + explicit BucketExistsResponse(const Response& resp) : Response(resp) {} + + ~BucketExistsResponse() = default; +}; // struct BucketExistsResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(RemoveBucketResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(AbortMultipartUploadResponse) + +struct CompleteMultipartUploadResponse : public Response { + std::string location; + std::string etag; + std::string version_id; + + CompleteMultipartUploadResponse() = default; + + explicit CompleteMultipartUploadResponse(error::Error err) + : Response(std::move(err)) {} + + explicit CompleteMultipartUploadResponse(const Response& resp) + : Response(resp) {} + + ~CompleteMultipartUploadResponse() = default; + + static CompleteMultipartUploadResponse ParseXML(std::string_view data, + std::string version_id); +}; // struct CompleteMultipartUploadResponse + +struct CreateMultipartUploadResponse : public Response { + std::string upload_id; + + explicit CreateMultipartUploadResponse(std::string upload_id) + : upload_id(std::move(upload_id)) {} + + explicit CreateMultipartUploadResponse(error::Error err) + : Response(std::move(err)) {} + + explicit CreateMultipartUploadResponse(const Response& resp) + : Response(resp) {} + + ~CreateMultipartUploadResponse() = default; +}; // struct CreateMultipartUploadResponse + +struct PutObjectResponse : public Response { + std::string etag; + std::string version_id; + + PutObjectResponse() = default; + + explicit PutObjectResponse(error::Error err) : Response(std::move(err)) {} + + explicit PutObjectResponse(const Response& resp) : Response(resp) {} + + explicit PutObjectResponse(const CompleteMultipartUploadResponse& resp) + : Response(resp), etag(resp.etag), version_id(resp.version_id) {} + + ~PutObjectResponse() = default; +}; // struct PutObjectResponse + +MINIO_S3_DERIVE_FROM_PUT_OBJECT_RESPONSE(UploadPartResponse) +MINIO_S3_DERIVE_FROM_PUT_OBJECT_RESPONSE(UploadPartCopyResponse) + +struct StatObjectResponse : public Response { + std::string version_id; + std::string etag; + size_t size = 0; + utils::UtcTime last_modified; + RetentionMode retention_mode; + utils::UtcTime retention_retain_until_date; + LegalHold legal_hold; + bool delete_marker; + utils::Multimap user_metadata; + + StatObjectResponse() = default; + + explicit StatObjectResponse(error::Error err) : Response(std::move(err)) {} + + explicit StatObjectResponse(const Response& resp) : Response(resp) {} + + ~StatObjectResponse() = default; +}; // struct StatObjectResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(RemoveObjectResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DownloadObjectResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(GetObjectResponse) + +struct Item : public Response { + std::string etag; // except DeleteMarker + std::string name; + utils::UtcTime last_modified; + std::string owner_id; + std::string owner_name; + size_t size = 0; // except DeleteMarker + std::string storage_class; + bool is_latest = false; // except ListObjects V1/V2 + std::string version_id; // except ListObjects V1/V2 + std::map user_metadata; + bool is_prefix = false; + bool is_delete_marker = false; + std::string encoding_type; + + Item() = default; + + explicit Item(error::Error err) : Response(std::move(err)) {} + + explicit Item(const Response& resp) : Response(resp) {} + + ~Item() = default; +}; // struct Item + +struct ListObjectsResponse : public Response { + // Common + std::string name; + std::string encoding_type; + std::string prefix; + std::string delimiter; + bool is_truncated; + unsigned int max_keys; + std::list contents; + + // ListObjectsV1 + std::string marker; + std::string next_marker; + + // ListObjectsV2 + unsigned int key_count; + std::string start_after; + std::string continuation_token; + std::string next_continuation_token; + + // ListObjectVersions + std::string key_marker; + std::string next_key_marker; + std::string version_id_marker; + std::string next_version_id_marker; + + ListObjectsResponse() = default; + + explicit ListObjectsResponse(error::Error err) : Response(std::move(err)) {} + + explicit ListObjectsResponse(const Response& resp) : Response(resp) {} + + ~ListObjectsResponse() = default; + + static ListObjectsResponse ParseXML(std::string_view data, bool version); +}; // struct ListObjectsResponse + +MINIO_S3_DERIVE_FROM_PUT_OBJECT_RESPONSE(CopyObjectResponse) +MINIO_S3_DERIVE_FROM_PUT_OBJECT_RESPONSE(ComposeObjectResponse) +MINIO_S3_DERIVE_FROM_PUT_OBJECT_RESPONSE(UploadObjectResponse) + +struct DeletedObject : public Response { + std::string name; + std::string version_id; + bool delete_marker; + std::string delete_marker_version_id; + + DeletedObject() = default; + ~DeletedObject() = default; +}; // struct DeletedObject + +struct DeleteError : public Response { + std::string version_id; + + DeleteError() = default; + + explicit DeleteError(error::Error err) : Response(std::move(err)) {} + + explicit DeleteError(const Response& resp) : Response(resp) {} + + ~DeleteError() = default; +}; // struct DeleteError + +struct RemoveObjectsResponse : public Response { + std::list objects; + std::list errors; + + RemoveObjectsResponse() = default; + + explicit RemoveObjectsResponse(error::Error err) : Response(std::move(err)) {} + + explicit RemoveObjectsResponse(const Response& resp) : Response(resp) {} + + ~RemoveObjectsResponse() = default; + + static RemoveObjectsResponse ParseXML(std::string_view data); +}; // struct RemoveObjectsResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SelectObjectContentResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(ListenBucketNotificationResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DeleteBucketPolicyResponse) + +struct GetBucketPolicyResponse : public Response { + std::string policy; + + explicit GetBucketPolicyResponse(std::string policy) + : policy(std::move(policy)) {} + + explicit GetBucketPolicyResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetBucketPolicyResponse(const Response& resp) : Response(resp) {} + + ~GetBucketPolicyResponse() = default; +}; // struct GetBucketPolicyResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetBucketPolicyResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DeleteBucketNotificationResponse) + +struct GetBucketNotificationResponse : public Response { + NotificationConfig config; + + explicit GetBucketNotificationResponse(NotificationConfig config) + : config(std::move(config)) {} + + explicit GetBucketNotificationResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetBucketNotificationResponse(const Response& resp) + : Response(resp) {} + + ~GetBucketNotificationResponse() = default; + + static GetBucketNotificationResponse ParseXML(std::string_view data); +}; // struct GetBucketNotificationResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetBucketNotificationResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DeleteBucketEncryptionResponse) + +struct GetBucketEncryptionResponse : public Response { + SseConfig config; + + explicit GetBucketEncryptionResponse(SseConfig config) + : config(std::move(config)) {} + + explicit GetBucketEncryptionResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetBucketEncryptionResponse(const Response& resp) : Response(resp) {} + + ~GetBucketEncryptionResponse() = default; + + static GetBucketEncryptionResponse ParseXML(std::string_view data); +}; // struct GetBucketEncryptionResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetBucketEncryptionResponse) + +struct GetBucketVersioningResponse : public Response { + Boolean status; + Boolean mfa_delete; + + GetBucketVersioningResponse() = default; + + explicit GetBucketVersioningResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetBucketVersioningResponse(const Response& resp) : Response(resp) {} + + ~GetBucketVersioningResponse() = default; + + std::string Status() const { + if (!status) return "Off"; + return status.Get() ? "Enabled" : "Suspended"; + } + + std::string MfaDelete() const { + if (!mfa_delete) { + return {}; + } + return mfa_delete.Get() ? "Enabled" : "Disabled"; + } +}; // struct GetBucketVersioningResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetBucketVersioningResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DeleteBucketReplicationResponse) + +struct GetBucketReplicationResponse : public Response { + ReplicationConfig config; + + explicit GetBucketReplicationResponse(ReplicationConfig config) + : config(std::move(config)) {} + + explicit GetBucketReplicationResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetBucketReplicationResponse(const Response& resp) + : Response(resp) {} + + ~GetBucketReplicationResponse() = default; + + static GetBucketReplicationResponse ParseXML(std::string_view data); +}; // struct GetBucketReplicationResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetBucketReplicationResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DeleteBucketLifecycleResponse) + +struct GetBucketLifecycleResponse : public Response { + LifecycleConfig config; + + explicit GetBucketLifecycleResponse(LifecycleConfig config) + : config(std::move(config)) {} + + explicit GetBucketLifecycleResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetBucketLifecycleResponse(const Response& resp) : Response(resp) {} + + static GetBucketLifecycleResponse ParseXML(std::string_view data); +}; // struct GetBucketLifecycleResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetBucketLifecycleResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DeleteBucketTagsResponse) + +struct GetBucketTagsResponse : public Response { + std::map tags; + + GetBucketTagsResponse(std::map tags) + : tags(std::move(tags)) {} + + explicit GetBucketTagsResponse(error::Error err) : Response(std::move(err)) {} + + explicit GetBucketTagsResponse(const Response& resp) : Response(resp) {} + + ~GetBucketTagsResponse() = default; + + static GetBucketTagsResponse ParseXML(std::string_view data); +}; // struct GetBucketTagsResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetBucketTagsResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DeleteObjectLockConfigResponse) + +struct GetObjectLockConfigResponse : public Response { + ObjectLockConfig config; + + explicit GetObjectLockConfigResponse(ObjectLockConfig config) + : config(std::move(config)) {} + + explicit GetObjectLockConfigResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetObjectLockConfigResponse(const Response& resp) : Response(resp) {} + + ~GetObjectLockConfigResponse() = default; +}; // struct GetObjectLockConfigResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetObjectLockConfigResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DeleteObjectTagsResponse) + +struct GetObjectTagsResponse : public Response { + std::map tags; + + GetObjectTagsResponse(std::map tags) + : tags(std::move(tags)) {} + + explicit GetObjectTagsResponse(error::Error err) : Response(std::move(err)) {} + + explicit GetObjectTagsResponse(const Response& resp) : Response(resp) {} + + ~GetObjectTagsResponse() = default; + + static GetObjectTagsResponse ParseXML(std::string_view data); +}; // struct GetObjectTagsResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetObjectTagsResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(EnableObjectLegalHoldResponse) +MINIO_S3_DERIVE_FROM_RESPONSE(DisableObjectLegalHoldResponse) + +struct IsObjectLegalHoldEnabledResponse : public Response { + bool enabled = false; + + explicit IsObjectLegalHoldEnabledResponse(bool enabled) : enabled(enabled) {} + + explicit IsObjectLegalHoldEnabledResponse(error::Error err) + : Response(std::move(err)) {} + + explicit IsObjectLegalHoldEnabledResponse(const Response& resp) + : Response(resp) {} + + ~IsObjectLegalHoldEnabledResponse() = default; +}; // struct IsObjectLegalHoldEnabledResponse + +struct GetObjectRetentionResponse : public Response { + RetentionMode retention_mode; + utils::UtcTime retain_until_date; + + GetObjectRetentionResponse() = default; + + explicit GetObjectRetentionResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetObjectRetentionResponse(const Response& resp) : Response(resp) {} + + ~GetObjectRetentionResponse() = default; +}; // struct GetObjectRetentionResponse + +MINIO_S3_DERIVE_FROM_RESPONSE(SetObjectRetentionResponse) + +struct GetPresignedObjectUrlResponse : public Response { + std::string url; + + explicit GetPresignedObjectUrlResponse(std::string url) + : url(std::move(url)) {} + + explicit GetPresignedObjectUrlResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetPresignedObjectUrlResponse(const Response& resp) + : Response(resp) {} + + ~GetPresignedObjectUrlResponse() = default; +}; // struct GetPresignedObjectUrlResponse + +struct GetPresignedPostFormDataResponse : public Response { + std::map form_data; + + GetPresignedPostFormDataResponse(std::map form_data) + : form_data(std::move(form_data)) {} + + explicit GetPresignedPostFormDataResponse(error::Error err) + : Response(std::move(err)) {} + + explicit GetPresignedPostFormDataResponse(const Response& resp) + : Response(resp) {} + + ~GetPresignedPostFormDataResponse() = default; +}; // struct GetPresignedPostFormDataResponse + +#undef MINIO_S3_DERIVE_FROM_PUT_OBJECT_RESPONSE +#undef MINIO_S3_DERIVE_FROM_RESPONSE + +} // namespace minio::s3 + +#endif // MINIO_CPP_RESPONSE_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/select.h b/src/3rdParty/minio-cpp/inc/miniocpp/select.h new file mode 100644 index 00000000..545835ee --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/select.h @@ -0,0 +1,72 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_SELECT_H_INCLUDED +#define MINIO_CPP_SELECT_H_INCLUDED + +#include +#include +#include + +#include "error.h" +#include "http.h" +#include "types.h" + +namespace minio::s3 { + +class SelectHandler { + private: + SelectResultFunction result_func_ = nullptr; + + bool done_ = false; + std::string response_; + + std::string prelude_; + bool prelude_read_ = false; + + std::string prelude_crc_; + bool prelude_crc_read_ = false; + + size_t total_length_ = 0; + + std::string data_; + bool data_read_ = false; + + std::string message_crc_; + bool message_crc_read_ = false; + + void Reset(); + bool ReadPrelude(); + bool ReadPreludeCrc(); + bool ReadData(); + bool ReadMessageCrc(); + error::Error DecodeHeader(std::map& headers, + std::string data); + bool process(const http::DataFunctionArgs& args, bool& cont); + + public: + explicit SelectHandler(SelectResultFunction result_func) + : result_func_(std::move(result_func)) {} + + ~SelectHandler() = default; + + bool DataFunction(const http::DataFunctionArgs& args); +}; // struct SelectHandler + +} // namespace minio::s3 + +#endif // MINIO_CPP_SELECT_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/signer.h b/src/3rdParty/minio-cpp/inc/miniocpp/signer.h new file mode 100644 index 00000000..670a93d1 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/signer.h @@ -0,0 +1,83 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_SIGNER_H_INCLUDED +#define MINIO_CPP_SIGNER_H_INCLUDED + +#include + +#include "http.h" +#include "utils.h" + +namespace minio::signer { + +std::string GetScope(const utils::UtcTime& time, const std::string& region, + const std::string& service_name); +std::string GetCanonicalRequestHash(const std::string& method, + const std::string& uri, + const std::string& query_string, + const std::string& headers, + const std::string& signed_headers, + const std::string& content_sha256); +std::string GetStringToSign(const utils::UtcTime& date, + const std::string& scope, + const std::string& canonical_request_hash); +std::string HmacHash(std::string_view key, std::string_view data); +std::string GetSigningKey(const std::string& secret_key, + const utils::UtcTime& date, std::string_view region, + std::string_view service_name); +std::string GetSignature(std::string_view signing_key, + std::string_view string_to_sign); +std::string GetAuthorization(const std::string& access_key, + const std::string& scope, + const std::string& signed_headers, + const std::string& signature); +utils::Multimap SignV4(const std::string& service_name, http::Method method, + const std::string& uri, const std::string& region, + utils::Multimap& headers, utils::Multimap query_params, + const std::string& access_key, + const std::string& secret_key, + const std::string& content_sha256, + const utils::UtcTime& date); +utils::Multimap SignV4S3(http::Method method, const std::string& uri, + const std::string& region, utils::Multimap& headers, + utils::Multimap query_params, + const std::string& access_key, + const std::string& secret_key, + const std::string& content_sha256, + const utils::UtcTime& date); +utils::Multimap SignV4STS(http::Method method, const std::string& uri, + const std::string& region, utils::Multimap& headers, + utils::Multimap query_params, + const std::string& access_key, + const std::string& secret_key, + const std::string& content_sha256, + const utils::UtcTime& date); +utils::Multimap PresignV4(http::Method method, const std::string& host, + const std::string& uri, const std::string& region, + utils::Multimap& query_params, + const std::string& access_key, + const std::string& secret_key, + const utils::UtcTime& date, unsigned int expires); +std::string PostPresignV4(const std::string& data, + const std::string& secret_key, + const utils::UtcTime& date, + const std::string& region); + +} // namespace minio::signer + +#endif // MINIO_CPP_SIGNER_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/sse.h b/src/3rdParty/minio-cpp/inc/miniocpp/sse.h new file mode 100644 index 00000000..67838792 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/sse.h @@ -0,0 +1,68 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_SSE_H_INCLUDED +#define MINIO_CPP_SSE_H_INCLUDED + +#include + +#include "utils.h" + +namespace minio::s3 { + +class Sse { + protected: + utils::Multimap headers_; + utils::Multimap copy_headers_; + + public: + Sse(); + virtual ~Sse(); + + utils::Multimap Headers() const; + utils::Multimap CopyHeaders() const; + + virtual bool TlsRequired() const = 0; +}; // class Sse + +class SseCustomerKey : public Sse { + public: + explicit SseCustomerKey(std::string_view key); + virtual ~SseCustomerKey(); + + virtual bool TlsRequired() const override; +}; // class SseCustomerKey + +class SseKms : public Sse { + public: + SseKms(std::string_view key, std::string_view context); + virtual ~SseKms(); + + virtual bool TlsRequired() const override; +}; // class SseKms + +class SseS3 : public Sse { + public: + SseS3(); + virtual ~SseS3(); + + virtual bool TlsRequired() const override; +}; // class SseS3 + +} // namespace minio::s3 + +#endif // MINIO_CPP_SSE_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/types.h b/src/3rdParty/minio-cpp/inc/miniocpp/types.h new file mode 100644 index 00000000..ec1ba067 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/types.h @@ -0,0 +1,771 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_TYPES_H_INCLUDED +#define MINIO_CPP_TYPES_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "error.h" +#include "utils.h" + +namespace minio::s3 { + +enum class RetentionMode { kGovernance, kCompliance }; + +// StringToRetentionMode converts string to retention mode enum. +RetentionMode StringToRetentionMode(std::string_view str) noexcept; + +constexpr bool IsRetentionModeValid(RetentionMode retention) { + switch (retention) { + case RetentionMode::kGovernance: + case RetentionMode::kCompliance: + return true; + } + return false; +} + +// RetentionModeToString converts retention mode enum to string. +constexpr const char* RetentionModeToString(RetentionMode retention) noexcept { + switch (retention) { + case RetentionMode::kGovernance: + return "GOVERNANCE"; + case RetentionMode::kCompliance: + return "COMPLIANCE"; + default: { + std::cerr << "ABORT: Unknown retention mode. This should not happen." + << std::endl; + std::terminate(); + } + } + return nullptr; +} + +enum class LegalHold { kOn, kOff }; + +// StringToLegalHold converts string to legal hold enum. +LegalHold StringToLegalHold(std::string_view str) noexcept; + +constexpr bool IsLegalHoldValid(LegalHold legal_hold) { + switch (legal_hold) { + case LegalHold::kOn: + case LegalHold::kOff: + return true; + } + return false; +} + +// LegalHoldToString converts legal hold enum to string. +constexpr const char* LegalHoldToString(LegalHold legal_hold) noexcept { + switch (legal_hold) { + case LegalHold::kOn: + return "ON"; + case LegalHold::kOff: + return "OFF"; + default: { + std::cerr << "ABORT: Unknown legal hold. This should not happen." + << std::endl; + std::terminate(); + } + } + return nullptr; +} + +enum class Directive { kCopy, kReplace }; + +// StringToDirective converts string to directive enum. +Directive StringToDirective(std::string_view str) noexcept; + +// DirectiveToString converts directive enum to string. +constexpr const char* DirectiveToString(Directive directive) noexcept { + switch (directive) { + case Directive::kCopy: + return "COPY"; + case Directive::kReplace: + return "REPLACE"; + default: { + std::cerr << "ABORT: Unknown directive. This should not happen." + << std::endl; + std::terminate(); + } + } + return nullptr; +} + +enum class CompressionType { kNone, kGZip, kBZip2 }; + +// CompressionTypeToString converts compression type enum to string. +constexpr const char* CompressionTypeToString(CompressionType ctype) noexcept { + switch (ctype) { + case CompressionType::kNone: + return "NONE"; + case CompressionType::kGZip: + return "GZIP"; + case CompressionType::kBZip2: + return "BZIP2"; + default: { + std::cerr << "ABORT: Unknown compression type. This should not happen." + << std::endl; + std::terminate(); + } + } + return nullptr; +} + +enum class FileHeaderInfo { kUse, kIgnore, kNone }; + +// FileHeaderInfoToString converts file header info enum to string. +constexpr const char* FileHeaderInfoToString(FileHeaderInfo info) noexcept { + switch (info) { + case FileHeaderInfo::kUse: + return "USE"; + case FileHeaderInfo::kIgnore: + return "IGNORE"; + case FileHeaderInfo::kNone: + return "NONE"; + default: { + std::cerr << "ABORT: Unknown file header info. This should not happen." + << std::endl; + std::terminate(); + } + } + return nullptr; +} + +enum class JsonType { kDocument, kLines }; + +// JsonTypeToString converts JSON type enum to string. +constexpr const char* JsonTypeToString(JsonType jtype) noexcept { + switch (jtype) { + case JsonType::kDocument: + return "DOCUMENT"; + case JsonType::kLines: + return "LINES"; + default: { + std::cerr << "ABORT: Unknown JSON type. This should not happen." + << std::endl; + std::terminate(); + } + } + return nullptr; +} + +enum class QuoteFields { kAlways, kAsNeeded }; + +// QuoteFieldsToString converts quote fields enum to string. +constexpr const char* QuoteFieldsToString(QuoteFields qtype) noexcept { + switch (qtype) { + case QuoteFields::kAlways: + return "ALWAYS"; + case QuoteFields::kAsNeeded: + return "ASNEEDED"; + default: { + std::cerr << "ABORT: Unknown quote fields. This should not happen." + << std::endl; + std::terminate(); + } + } + return nullptr; +} + +struct CsvInputSerialization { + CompressionType* compression_type = nullptr; + bool allow_quoted_record_delimiter = false; + char comments = 0; + char field_delimiter = 0; + FileHeaderInfo* file_header_info = nullptr; + char quote_character = 0; + char quote_escape_character = 0; + char record_delimiter = 0; + + CsvInputSerialization() = default; + ~CsvInputSerialization() = default; +}; // struct CsvInputSerialization + +struct JsonInputSerialization { + CompressionType* compression_type = nullptr; + JsonType* json_type = nullptr; + + JsonInputSerialization() = default; + ~JsonInputSerialization() = default; +}; // struct JsonInputSerialization + +struct ParquetInputSerialization { + ParquetInputSerialization() = default; + ~ParquetInputSerialization() = default; +}; // struct ParquetInputSerialization + +struct CsvOutputSerialization { + char field_delimiter = 0; + char quote_character = 0; + char quote_escape_character = 0; + QuoteFields* quote_fields = nullptr; + char record_delimiter = 0; + + CsvOutputSerialization() = default; + ~CsvOutputSerialization() = default; +}; // struct CsvOutputSerialization + +struct JsonOutputSerialization { + char record_delimiter = 0; + + JsonOutputSerialization() = default; + ~JsonOutputSerialization() = default; +}; // struct JsonOutputSerialization + +struct SelectRequest { + std::string expr; + CsvInputSerialization* csv_input = nullptr; + JsonInputSerialization* json_input = nullptr; + ParquetInputSerialization* parquet_input = nullptr; + CsvOutputSerialization* csv_output = nullptr; + JsonOutputSerialization* json_output = nullptr; + bool request_progress = false; + size_t* scan_start_range = nullptr; + size_t* scan_end_range = nullptr; + + SelectRequest(std::string expression, CsvInputSerialization* csv_input, + CsvOutputSerialization* csv_output) + : expr(std::move(expression)), + csv_input(csv_input), + csv_output(csv_output) {} + + SelectRequest(std::string expression, CsvInputSerialization* csv_input, + JsonOutputSerialization* json_output) + : expr(std::move(expression)), + csv_input(csv_input), + json_output(json_output) {} + + SelectRequest(std::string expression, JsonInputSerialization* json_input, + CsvOutputSerialization* csv_output) + : expr(std::move(expression)), + json_input(json_input), + csv_output(csv_output) {} + + SelectRequest(std::string expression, JsonInputSerialization* json_input, + JsonOutputSerialization* json_output) + : expr(std::move(expression)), + json_input(json_input), + json_output(json_output) {} + + SelectRequest(std::string expression, + ParquetInputSerialization* parquet_input, + CsvOutputSerialization* csv_output) + : expr(std::move(expression)), + parquet_input(parquet_input), + csv_output(csv_output) {} + + SelectRequest(std::string expression, + ParquetInputSerialization* parquet_input, + JsonOutputSerialization* json_output) + : expr(std::move(expression)), + parquet_input(parquet_input), + json_output(json_output) {} + + ~SelectRequest() = default; + + std::string ToXML() const; +}; // struct SelectRequest + +struct SelectResult { + error::Error err = error::SUCCESS; + bool ended = false; + long int bytes_scanned = -1; + long int bytes_processed = -1; + long int bytes_returned = -1; + std::string records; + + SelectResult() : ended(true) {} + + explicit SelectResult(error::Error err) : err(std::move(err)), ended(true) {} + + SelectResult(long int bytes_scanned, long int bytes_processed, + long int bytes_returned) + : bytes_scanned(bytes_scanned), + bytes_processed(bytes_processed), + bytes_returned(bytes_returned) {} + + SelectResult(std::string records) : records(std::move(records)) {} + + ~SelectResult() = default; +}; + +using SelectResultFunction = std::function; + +struct Bucket { + std::string name; + utils::UtcTime creation_date; + + Bucket() = default; + explicit Bucket(std::string name, utils::UtcTime creation_date) + : name(std::move(name)), creation_date(std::move(creation_date)) {} + ~Bucket() = default; +}; // struct Bucket + +struct Part { + unsigned int number; + std::string etag; + utils::UtcTime last_modified = {}; + size_t size = 0; + + Part() = default; + explicit Part(unsigned int number, std::string etag) + : number(number), etag(std::move(etag)) {} + ~Part() = default; +}; // struct Part + +struct Retention { + RetentionMode mode; + utils::UtcTime retain_until_date; + + Retention() = default; + ~Retention() = default; +}; // struct Retention + +struct DeleteObject { + std::string name = {}; + std::string version_id = {}; + + DeleteObject() = default; + ~DeleteObject() = default; +}; // struct DeleteObject + +struct NotificationRecord { + std::string event_version; + std::string event_source; + std::string aws_region; + std::string event_time; + std::string event_name; + struct { + std::string principal_id; + } user_identity; + struct { + std::string principal_id; + std::string region; + std::string source_ip_address; + } request_parameters; + struct { + std::string content_length; + std::string x_amz_request_id; + std::string x_minio_deployment_id; + std::string x_minio_origin_endpoint; + } response_elements; + struct { + std::string s3_schema_version; + std::string configuration_id; + struct { + std::string name; + std::string arn; + struct { + std::string principal_id; + } owner_identity; + } bucket; + struct { + std::string key; + size_t size; + std::string etag; + std::string content_type; + std::map user_metadata; + std::string sequencer; + } object; + } s3; + struct { + std::string host; + std::string port; + std::string user_agent; + } source; + + NotificationRecord() = default; + ~NotificationRecord() = default; + + static NotificationRecord ParseJSON(nlohmann::json j_record); +}; // struct NotificationRecord + +using NotificationRecordsFunction = + std::function)>; + +struct FilterValue { + private: + std::string value_; + bool is_value_set_ = false; + + public: + FilterValue() = default; + + explicit FilterValue(std::string value) + : value_(std::move(value)), is_value_set_(true) {} + + ~FilterValue() = default; + + explicit operator bool() const { return is_value_set_; } + std::string Value() const { return value_; } +}; // struct FilterValue + +struct PrefixFilterRule : public FilterValue { + static constexpr const char* name = "prefix"; + + PrefixFilterRule() = default; + + explicit PrefixFilterRule(std::string value) + : FilterValue(std::move(value)) {} + + ~PrefixFilterRule() = default; +}; // struct PrefixFilterRule + +struct SuffixFilterRule : public FilterValue { + static constexpr const char* name = "suffix"; + + SuffixFilterRule() = default; + + explicit SuffixFilterRule(std::string value) + : FilterValue(std::move(value)) {} + + ~SuffixFilterRule() = default; +}; // struct SuffixFilterRule + +struct NotificationCommonConfig { + std::list events; + std::string id; + PrefixFilterRule prefix_filter_rule; + SuffixFilterRule suffix_filter_rule; + + NotificationCommonConfig() = default; + ~NotificationCommonConfig() = default; +}; // struct NotificationCommonConfig + +struct CloudFuncConfig : public NotificationCommonConfig { + std::string cloud_func; + + CloudFuncConfig() = default; + ~CloudFuncConfig() = default; +}; // struct CloudFuncConfig + +struct QueueConfig : public NotificationCommonConfig { + std::string queue; + + QueueConfig() = default; + ~QueueConfig() = default; +}; // struct QueueConfig + +struct TopicConfig : public NotificationCommonConfig { + std::string topic; + + TopicConfig() = default; + ~TopicConfig() = default; +}; // struct TopicConfig + +struct NotificationConfig { + std::list cloud_func_config_list; + std::list queue_config_list; + std::list topic_config_list; + + NotificationConfig() = default; + ~NotificationConfig() = default; + + std::string ToXML() const; +}; // struct NotificationConfig + +struct SseConfig { + std::string sse_algorithm; + std::string kms_master_key_id; + + SseConfig() = default; + ~SseConfig() = default; + + static SseConfig S3(); + static SseConfig Kms(std::string masterkeyid = {}); + + explicit operator bool() const { return !sse_algorithm.empty(); } +}; // struct SseConfig + +struct Tag { + std::string key; + std::string value; + + Tag() = default; + ~Tag() = default; + + explicit operator bool() const { return !key.empty(); } +}; // struct Tag + +struct Prefix { + private: + std::string value_; + bool is_set_ = false; + + public: + Prefix() = default; + + explicit Prefix(std::string value) + : value_(std::move(value)), is_set_(true) {} + + ~Prefix() = default; + + explicit operator bool() const { return is_set_; } + std::string Get() const { return value_; } + + friend std::ostream& operator<<(std::ostream& s, const Prefix& v) { + return s << v.value_; + } +}; // struct Prefix + +struct Integer { + private: + int value_ = 0; + bool is_set_ = false; + + public: + Integer() = default; + + explicit Integer(int value) : value_(value), is_set_(true) {} + + Integer& operator=(int value) { + value_ = value; + is_set_ = true; + return *this; + } + + Integer(const Integer&) = default; + Integer& operator=(const Integer&) = default; + + Integer(Integer&&) = default; + Integer& operator=(Integer&&) = default; + + ~Integer() = default; + + explicit operator bool() const { return is_set_; } + int Get() const { return value_; } + + friend std::ostream& operator<<(std::ostream& s, const Integer& v) { + return s << v.value_; + } +}; // struct Integer + +struct Boolean { + private: + bool value_ = false; + bool is_set_ = false; + + public: + Boolean() = default; + + explicit Boolean(bool value) : value_(value), is_set_(true) {} + + Boolean& operator=(bool value) { + value_ = value; + is_set_ = true; + return *this; + } + + Boolean(const Boolean&) = default; + Boolean& operator=(const Boolean&) = default; + + Boolean(Boolean&&) = default; + Boolean& operator=(Boolean&&) = default; + + ~Boolean() = default; + + explicit operator bool() const { return is_set_; } + bool Get() const { return value_; } + + friend std::ostream& operator<<(std::ostream& s, const Boolean& v) { + return s << utils::BoolToString(v.value_); + } +}; // struct Boolean + +struct AndOperator { + Prefix prefix; + std::map tags; + + AndOperator() = default; + ~AndOperator() = default; + + explicit operator bool() const { return prefix || !tags.empty(); } +}; // struct AndOperator + +struct Filter { + AndOperator and_operator; + Prefix prefix; + Tag tag; + + Filter() = default; + ~Filter() = default; + + explicit operator bool() const { + return static_cast(and_operator) ^ static_cast(prefix) ^ + static_cast(tag); + } +}; // struct Filter + +struct AccessControlTranslation { + std::string owner = "Destination"; + + AccessControlTranslation() = default; + ~AccessControlTranslation() = default; + + void Enable() { enabled_ = true; } + explicit operator bool() const { return enabled_; } + + private: + bool enabled_ = false; +}; // struct AccessControlTranslation + +struct EncryptionConfig { + std::string replica_kms_key_id; + + EncryptionConfig() = default; + ~EncryptionConfig() = default; + + void Enable() { enabled_ = true; } + explicit operator bool() const { return enabled_; } + + private: + bool enabled_ = false; +}; // struct EncryptionConfig + +struct Metrics { + unsigned int event_threshold_minutes = 15; + bool status = false; + + Metrics() = default; + ~Metrics() = default; + + void Enable() { enabled_ = true; } + explicit operator bool() const { return enabled_; } + + private: + bool enabled_ = false; +}; // struct Metrics + +struct ReplicationTime { + unsigned int time_minutes = 15; + bool status = false; + + ReplicationTime() = default; + ~ReplicationTime() = default; + + void Enable() { enabled_ = true; } + explicit operator bool() const { return enabled_; } + + private: + bool enabled_ = false; +}; // struct ReplicationTime + +struct Destination { + std::string bucket_arn; + AccessControlTranslation access_control_translation; + std::string account; + EncryptionConfig encryption_config; + Metrics metrics; + ReplicationTime replication_time; + std::string storage_class; + + Destination() = default; + ~Destination() = default; +}; // struct Destination + +struct SourceSelectionCriteria { + Boolean sse_kms_encrypted_objects_status; + + SourceSelectionCriteria() = default; + ~SourceSelectionCriteria() = default; + + void Enable() { enabled_ = true; } + explicit operator bool() const { return enabled_; } + + private: + bool enabled_ = false; +}; // struct SourceSelectionCriteria + +struct ReplicationRule { + Destination destination; + Boolean delete_marker_replication_status; + Boolean existing_object_replication_status; + Filter filter; + std::string id; + Prefix prefix; + Integer priority; + SourceSelectionCriteria source_selection_criteria; + Boolean delete_replication_status; + bool status = false; + + ReplicationRule() = default; + ~ReplicationRule() = default; +}; // struct ReplicationRule + +struct ReplicationConfig { + std::string role; + std::list rules; + + ReplicationConfig() = default; + ~ReplicationConfig() = default; + + std::string ToXML() const; +}; // status ReplicationConfig + +struct LifecycleRule { + Integer abort_incomplete_multipart_upload_days_after_initiation; + utils::UtcTime expiration_date; + Integer expiration_days; + Boolean expiration_expired_object_delete_marker; + Filter filter; + std::string id; + Integer noncurrent_version_expiration_noncurrent_days; + Integer noncurrent_version_transition_noncurrent_days; + std::string noncurrent_version_transition_storage_class; + bool status = false; + utils::UtcTime transition_date; + Integer transition_days; + std::string transition_storage_class; + + LifecycleRule() = default; + ~LifecycleRule() = default; + + error::Error Validate() const; +}; // struct LifecycleRule + +struct LifecycleConfig { + std::list rules; + + LifecycleConfig() = default; + ~LifecycleConfig() = default; + + std::string ToXML() const; +}; // struct LifecycleConfig + +struct ObjectLockConfig { + RetentionMode retention_mode; + Integer retention_duration_days; + Integer retention_duration_years; + + ObjectLockConfig() = default; + ~ObjectLockConfig() = default; + + error::Error Validate() const; +}; // struct ObjectLockConfig + +} // namespace minio::s3 + +#endif // MINIO_CPP_TYPES_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/inc/miniocpp/utils.h b/src/3rdParty/minio-cpp/inc/miniocpp/utils.h new file mode 100644 index 00000000..a46ea243 --- /dev/null +++ b/src/3rdParty/minio-cpp/inc/miniocpp/utils.h @@ -0,0 +1,234 @@ +// MinIO C++ Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022-2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef MINIO_CPP_UTILS_H_INCLUDED +#define MINIO_CPP_UTILS_H_INCLUDED + +#include + +#ifndef _WIN32 +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "error.h" + +namespace minio::utils { + +inline constexpr unsigned int kMaxMultipartCount = 10000; // 10000 parts +inline constexpr uint64_t kMaxObjectSize = 5'497'558'138'880; // 5TiB +inline constexpr uint64_t kMaxPartSize = 5'368'709'120; // 5GiB +inline constexpr unsigned int kMinPartSize = 5 * 1024 * 1024; // 5MiB + +// GetEnv copies the environment variable name into var +bool GetEnv(std::string& var, const char* name); + +std::string GetHomeDir(); + +std::string Printable(const std::string& s); + +unsigned long CRC32(std::string_view str); + +unsigned int Int(std::string_view str); + +// FormatTime formats time as per format. +std::string FormatTime(const std::tm& time, const char* format); + +// StringToBool converts string to bool. +bool StringToBool(const std::string& str); + +// BoolToString converts bool to string. +inline const char* BoolToString(bool b) { return b ? "true" : "false"; } + +// Trim trims leading and trailing character of a string. +std::string Trim(std::string_view str, char ch = ' '); + +// CheckNonEmptyString checks whether string is not empty after trimming +// whitespaces. +bool CheckNonEmptyString(std::string_view str); + +// ToLower converts string to lower case. +std::string ToLower(const std::string& str); + +// StartsWith returns whether str starts with prefix or not. +bool StartsWith(std::string_view str, std::string_view prefix); + +// EndsWith returns whether str ends with suffix or not. +bool EndsWith(std::string_view str, std::string_view suffix); + +// Contains returns whether str has ch. +bool Contains(std::string_view str, char ch); + +// Contains returns whether str has substr. +bool Contains(std::string_view str, std::string_view substr); + +// Join returns a string of joined values by delimiter. +std::string Join(const std::list& values, + const std::string& delimiter); + +// Join returns a string of joined values by delimiter. +std::string Join(const std::vector& values, + const std::string& delimiter); + +// EncodePath does URL encoding of path. It also normalizes multiple slashes. +std::string EncodePath(const std::string& path); + +// Sha256hash computes SHA-256 of data and return hash as hex encoded value. +std::string Sha256Hash(std::string_view str); + +// Base64Encode encodes string to base64. +std::string Base64Encode(std::string_view str); + +// Md5sumHash computes MD5 of data and return hash as Base64 encoded value. +std::string Md5sumHash(std::string_view str); + +error::Error CheckBucketName(std::string_view bucket_name, bool strict = false); +error::Error ReadPart(std::istream& stream, char* buf, size_t size, + size_t& bytes_read); +error::Error CalcPartInfo(long object_size, size_t& part_size, + long& part_count); + +/** + * UtcTime represents date and time in UTC timezone. + */ +class UtcTime { + private: + std::time_t secs_ = {}; + long usecs_ = 0L; + + static std::tm auxLocaltime(const std::time_t& time); + std::tm getBrokenDownTime() const { return auxLocaltime(secs_); } + static std::time_t toUtcSeconds(const std::time_t time); + + public: + UtcTime() = default; + + UtcTime(std::time_t secs) : secs_(secs) {} + UtcTime(std::time_t secs, long usecs) : secs_(secs), usecs_(usecs) {} + + ~UtcTime() = default; + + void Add(std::time_t seconds) { secs_ += seconds; } + + void ToLocalTime(std::tm& time); + + std::string ToSignerDate() const; + + std::string ToAmzDate() const; + + std::string ToHttpHeaderValue() const; + + static UtcTime FromHttpHeaderValue(const char* value); + + std::string ToISO8601UTC() const; + + static UtcTime FromISO8601UTC(const char* value); + + static UtcTime Now(); + + explicit operator bool() const { return secs_ != 0 && usecs_ != 0; } + + int Compare(const UtcTime& rhs) const; + + bool Equal(const UtcTime& rhs) const { return Compare(rhs) == 0; } + + bool operator==(const UtcTime& rhs) const { return Equal(rhs); } + + bool operator!=(const UtcTime& rhs) const { return !operator==(rhs); } + + bool operator<(const UtcTime& rhs) const { return Compare(rhs) < 0; } + + bool operator>(const UtcTime& rhs) const { return Compare(rhs) > 0; } + + bool operator<=(const UtcTime& rhs) const { return !operator>(rhs); } + + bool operator>=(const UtcTime& rhs) const { return !operator<(rhs); } + +#if __cplusplus >= 202002L + auto operator<=>(const UtcTime& rhs) const { return Compare(rhs); } +#endif + + friend std::ostream& operator<<(std::ostream& s, const UtcTime& v) { + return s << v.ToISO8601UTC(); + } +}; // class UtcTime + +/** + * Multimap represents dictionary of keys and their multiple values. + */ +class Multimap { + private: + std::map> map_; + std::map> keys_; + + public: + Multimap() = default; + Multimap(const Multimap& headers) = default; + Multimap& operator=(const Multimap& headers) = default; + Multimap(Multimap&& headers) = default; + Multimap& operator=(Multimap&& headers) = default; + ~Multimap() = default; + + void Add(std::string key, std::string value); + + void AddAll(const Multimap& headers); + + std::list ToHttpHeaders() const; + + std::string ToQueryString() const; + + explicit operator bool() const { return !map_.empty(); } + + bool Contains(std::string_view key) const; + + std::list Get(std::string_view key) const; + + std::string GetFront(std::string_view key) const; + + std::list Keys() const; + + void GetCanonicalHeaders(std::string& signed_headers, + std::string& canonical_headers) const; + + std::string GetCanonicalQueryString() const; +}; // class Multimap + +/** + * CharBuffer represents stream buffer wrapping character array and its size. + */ +struct CharBuffer : std::streambuf { + CharBuffer(char* buf, size_t size) { this->setg(buf, buf, buf + size); } + virtual ~CharBuffer(); + + virtual pos_type seekpos(pos_type sp, std::ios_base::openmode which) override; + + virtual pos_type seekoff( + off_type off, std::ios_base::seekdir dir, + std::ios_base::openmode which = std::ios_base::in) override; +}; // struct CharBuffer + +} // namespace minio::utils + +#endif // MINIO_CPP_UTILS_H_INCLUDED diff --git a/src/3rdParty/minio-cpp/lib/linux-amd64/libminiocpp-debug.a b/src/3rdParty/minio-cpp/lib/linux-amd64/libminiocpp-debug.a new file mode 100644 index 00000000..3ea063c1 Binary files /dev/null and b/src/3rdParty/minio-cpp/lib/linux-amd64/libminiocpp-debug.a differ diff --git a/src/3rdParty/minio-cpp/lib/linux-amd64/libminiocpp.a b/src/3rdParty/minio-cpp/lib/linux-amd64/libminiocpp.a new file mode 100644 index 00000000..e8efee4c Binary files /dev/null and b/src/3rdParty/minio-cpp/lib/linux-amd64/libminiocpp.a differ diff --git a/src/3rdParty/minio-cpp/lib/osx/libminiocpp-debug.a b/src/3rdParty/minio-cpp/lib/osx/libminiocpp-debug.a new file mode 100644 index 00000000..71a20679 Binary files /dev/null and b/src/3rdParty/minio-cpp/lib/osx/libminiocpp-debug.a differ diff --git a/src/3rdParty/minio-cpp/lib/osx/libminiocpp.a b/src/3rdParty/minio-cpp/lib/osx/libminiocpp.a new file mode 100644 index 00000000..71a20679 Binary files /dev/null and b/src/3rdParty/minio-cpp/lib/osx/libminiocpp.a differ diff --git a/src/3rdParty/minio-cpp/lib/win64/miniocpp-debug.lib b/src/3rdParty/minio-cpp/lib/win64/miniocpp-debug.lib new file mode 100644 index 00000000..f13e5ad6 Binary files /dev/null and b/src/3rdParty/minio-cpp/lib/win64/miniocpp-debug.lib differ diff --git a/src/3rdParty/minio-cpp/lib/win64/miniocpp.lib b/src/3rdParty/minio-cpp/lib/win64/miniocpp.lib new file mode 100644 index 00000000..56c82bf7 Binary files /dev/null and b/src/3rdParty/minio-cpp/lib/win64/miniocpp.lib differ diff --git a/src/3rdParty/minio-cpp/lib/win64/miniocpp.pdb b/src/3rdParty/minio-cpp/lib/win64/miniocpp.pdb new file mode 100644 index 00000000..0610d77a Binary files /dev/null and b/src/3rdParty/minio-cpp/lib/win64/miniocpp.pdb differ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0ed837ea..2b7823ef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory(3rdParty) + if (NOT TARGET acquire-core-logger) add_subdirectory(../acquire-common/acquire-core-libs ${CMAKE_CURRENT_BINARY_DIR}/acquire-core-libs) endif () @@ -44,6 +46,7 @@ target_link_libraries(${tgt} PRIVATE acquire-device-properties blosc_static nlohmann_json::nlohmann_json + miniocpp::miniocpp ) set_target_properties(${tgt} PROPERTIES MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>" diff --git a/vcpkg.json b/vcpkg.json index bab98275..ed03bcfa 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,12 +1,17 @@ { "dependencies": [ { - "name": "blosc", - "version>=": "1.21.5" + "name": "blosc", + "version>=": "1.21.5" }, { - "name": "nlohmann-json", - "version>=": "3.11.3" - } + "name": "nlohmann-json", + "version>=": "3.11.3" + }, + { "name": "curlpp" }, + { "name": "inih", "features": ["cpp"] }, + { "name": "openssl" }, + { "name": "pugixml" }, + { "name": "zlib", "platform": "windows" } ] -} +} \ No newline at end of file