Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

C.41 compliance: S3 factory constructor #4368

Merged
merged 4 commits into from
Sep 27, 2023
Merged

C.41 compliance: S3 factory constructor #4368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b543265
Class S3: C.41 compliant constructor, remove init
bekadavis9 Sep 14, 2023
e186bb5
Replace S3 config_ member variable with S3Parameters struct
bekadavis9 Sep 18, 2023
e3fb83b
Syntax and doxygen improvements
bekadavis9 Sep 21, 2023
2177a6f
Remove defaults from the S3Parameters struct comments
bekadavis9 Sep 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 30 additions & 28 deletions test/src/unit-s3-no-multipart.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* The MIT License
*
* @copyright Copyright (c) 2017-2021 TileDB, Inc.
* @copyright Copyright (c) 2017-2023 TileDB, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
Expand Down Expand Up @@ -48,34 +48,20 @@ using namespace tiledb::common;
using namespace tiledb::sm;

struct S3DirectFx {
S3DirectFx();
~S3DirectFx();
static Config set_config_params();
static std::string random_name(const std::string& prefix);

const std::string S3_PREFIX = "s3://";
const tiledb::sm::URI S3_BUCKET =
tiledb::sm::URI(S3_PREFIX + random_name("tiledb") + "/");
const std::string TEST_DIR = S3_BUCKET.to_string() + "tiledb_test_dir/";
tiledb::sm::S3 s3_;
ThreadPool thread_pool_{2};

S3DirectFx();
~S3DirectFx();

static std::string random_name(const std::string& prefix);
tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, set_config_params()};
};

S3DirectFx::S3DirectFx() {
// Connect
Config config;
#ifndef TILEDB_TESTS_AWS_S3_CONFIG
REQUIRE(config.set("vfs.s3.endpoint_override", "localhost:9999").ok());
REQUIRE(config.set("vfs.s3.scheme", "https").ok());
REQUIRE(config.set("vfs.s3.use_virtual_addressing", "false").ok());
REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok());
#endif
REQUIRE(config.set("vfs.s3.max_parallel_ops", "1").ok());
// set max buffer size to 10 MB
REQUIRE(config.set("vfs.s3.multipart_part_size", "10000000").ok());
REQUIRE(config.set("vfs.s3.use_multipart_upload", "false").ok());
REQUIRE(s3_.init(&g_helper_stats, config, &thread_pool_).ok());

// Create bucket
bool exists;
REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok());
Expand Down Expand Up @@ -107,6 +93,29 @@ S3DirectFx::~S3DirectFx() {
CHECK(s3_.disconnect().ok());
}

Config S3DirectFx::set_config_params() {
// Connect
Config config;
#ifndef TILEDB_TESTS_AWS_S3_CONFIG
REQUIRE(config.set("vfs.s3.endpoint_override", "localhost:9999").ok());
REQUIRE(config.set("vfs.s3.scheme", "https").ok());
REQUIRE(config.set("vfs.s3.use_virtual_addressing", "false").ok());
REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok());
#endif
REQUIRE(config.set("vfs.s3.max_parallel_ops", "1").ok());
// set max buffer size to 10 MB
REQUIRE(config.set("vfs.s3.multipart_part_size", "10000000").ok());
REQUIRE(config.set("vfs.s3.use_multipart_upload", "false").ok());
return config;
}

std::string S3DirectFx::random_name(const std::string& prefix) {
std::stringstream ss;
ss << prefix << "-" << std::this_thread::get_id() << "-"
<< tiledb::sm::utils::time::timestamp_now_ms();
return ss.str();
}

TEST_CASE_METHOD(
S3DirectFx,
"Test S3 filesystem, file I/O with multipart API disabled",
Expand Down Expand Up @@ -183,11 +192,4 @@ TEST_CASE_METHOD(
auto badbuffer = (char*)malloc(11000000);
CHECK(!(s3_.write(URI(badfile), badbuffer, 11000000).ok()));
}

std::string S3DirectFx::random_name(const std::string& prefix) {
std::stringstream ss;
ss << prefix << "-" << std::this_thread::get_id() << "-"
<< tiledb::sm::utils::time::timestamp_now_ms();
return ss.str();
}
#endif
42 changes: 21 additions & 21 deletions test/src/unit-s3.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* The MIT License
*
* @copyright Copyright (c) 2017-2021 TileDB, Inc.
* @copyright Copyright (c) 2017-2023 TileDB, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
Expand Down Expand Up @@ -48,30 +48,20 @@ using namespace tiledb::sm;
using namespace tiledb::test;

struct S3Fx {
S3Fx();
~S3Fx();
static Config set_config_params();
static std::string random_name(const std::string& prefix);

const std::string S3_PREFIX = "s3://";
const tiledb::sm::URI S3_BUCKET =
tiledb::sm::URI(S3_PREFIX + random_name("tiledb") + "/");
const std::string TEST_DIR = S3_BUCKET.to_string() + "tiledb_test_dir/";
tiledb::sm::S3 s3_;
ThreadPool thread_pool_{2};

S3Fx();
~S3Fx();

static std::string random_name(const std::string& prefix);
tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, set_config_params()};
};

S3Fx::S3Fx() {
// Connect
Config config;
#ifndef TILEDB_TESTS_AWS_S3_CONFIG
REQUIRE(config.set("vfs.s3.endpoint_override", "localhost:9999").ok());
REQUIRE(config.set("vfs.s3.scheme", "https").ok());
REQUIRE(config.set("vfs.s3.use_virtual_addressing", "false").ok());
REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok());
#endif
REQUIRE(s3_.init(&g_helper_stats, config, &thread_pool_).ok());

// Create bucket
bool exists;
REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok());
Expand Down Expand Up @@ -103,6 +93,18 @@ S3Fx::~S3Fx() {
CHECK(s3_.disconnect().ok());
}

Config S3Fx::set_config_params() {
// Connect
Config config;
#ifndef TILEDB_TESTS_AWS_S3_CONFIG
REQUIRE(config.set("vfs.s3.endpoint_override", "localhost:9999").ok());
REQUIRE(config.set("vfs.s3.scheme", "https").ok());
REQUIRE(config.set("vfs.s3.use_virtual_addressing", "false").ok());
REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok());
#endif
return config;
}

std::string S3Fx::random_name(const std::string& prefix) {
std::stringstream ss;
ss << prefix << "-" << std::this_thread::get_id() << "-"
Expand Down Expand Up @@ -369,8 +371,7 @@ TEST_CASE_METHOD(S3Fx, "Test S3 use BucketCannedACL", "[s3]") {
// by string parameter.
auto try_with_bucket_canned_acl = [&](const char* bucket_acl_to_try) {
REQUIRE(config.set("vfs.s3.bucket_canned_acl", bucket_acl_to_try).ok());
tiledb::sm::S3 s3_;
REQUIRE(s3_.init(&g_helper_stats, config, &thread_pool_).ok());
tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, config};

// Create bucket
bool exists;
Expand Down Expand Up @@ -524,8 +525,7 @@ TEST_CASE_METHOD(S3Fx, "Test S3 use Bucket/Object CannedACL", "[s3]") {
REQUIRE(config.set("vfs.s3.bucket_canned_acl", bucket_acl_to_try).ok());
REQUIRE(config.set("vfs.s3.object_canned_acl", object_acl_to_try).ok());

tiledb::sm::S3 s3_;
REQUIRE(s3_.init(&g_helper_stats, config, &thread_pool_).ok());
tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, config};

// Create bucket
bool exists;
Expand Down
4 changes: 4 additions & 0 deletions tiledb/api/c_api/config/config_api_external.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,10 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT;
* The server-side encryption algorithm to use. Supported non-empty
* values are "aes256" and "kms" (AWS key management service). <br>
* **Default**: ""
* - `vfs.s3.sse_kms_key_id` <br>
* The server-side encryption key to use if
* vfs.s3.sse == "kms" (AWS key management service). <br>
* **Default**: ""
* - `vfs.s3.bucket_canned_acl` <br>
* Names of values found in Aws::S3::Model::BucketCannedACL enumeration.
* "NOT_SET"
Expand Down
6 changes: 5 additions & 1 deletion tiledb/sm/cpp_api/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -644,7 +644,7 @@ class Config {
* The AWS SDK logging level. This is a process-global setting. The
* configuration of the most recently constructed context will set
* process state. Log files are written to the process working directory.
* **Default**: off""
* **Default**: "off"
* - `vfs.s3.request_timeout_ms` <br>
* The request timeout in ms. Any `long` value is acceptable. <br>
* **Default**: 3000
Expand Down Expand Up @@ -678,6 +678,10 @@ class Config {
* The server-side encryption algorithm to use. Supported non-empty
* values are "aes256" and "kms" (AWS key management service). <br>
* **Default**: ""
* - `vfs.s3.sse_kms_key_id` <br>
* The server-side encryption key to use if
* vfs.s3.sse == "kms" (AWS key management service). <br>
* **Default**: ""
* - `vfs.s3.bucket_canned_acl` <br>
* Names of values found in Aws::S3::Model::BucketCannedACL enumeration.
* "NOT_SET"
Expand Down
Loading
Loading