Skip to content

Commit

Permalink
force to use http for some cases
Browse files Browse the repository at this point in the history
  • Loading branch information
Hang Zheng committed Oct 16, 2024
1 parent 94623a2 commit 598f146
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 8 deletions.
9 changes: 6 additions & 3 deletions cpp/src/arrow/filesystem/s3_test_util.cc
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,19 @@ struct MinioTestEnvironment::Impl {
}
};

MinioTestEnvironment::MinioTestEnvironment() : impl_(new Impl) {}
MinioTestEnvironment::MinioTestEnvironment(bool enable_tls_if_supported)
: impl_(new Impl), enable_tls_if_supported_(enable_tls_if_supported) {}

MinioTestEnvironment::~MinioTestEnvironment() = default;

void MinioTestEnvironment::SetUp() {
auto pool = ::arrow::internal::GetCpuThreadPool();

auto launch_one_server = []() -> Result<std::shared_ptr<MinioTestServer>> {
auto launch_one_server =
[enable_tls_if_supported =
enable_tls_if_supported_]() -> Result<std::shared_ptr<MinioTestServer>> {
auto server = std::make_shared<MinioTestServer>();
RETURN_NOT_OK(server->Start());
RETURN_NOT_OK(server->Start(enable_tls_if_supported));
return server;
};
impl_->server_generator_ = [pool, launch_one_server]() {
Expand Down
3 changes: 2 additions & 1 deletion cpp/src/arrow/filesystem/s3_test_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ class MinioTestServer {

class MinioTestEnvironment : public ::testing::Environment {
public:
MinioTestEnvironment();
explicit MinioTestEnvironment(bool enable_tls_if_supported = true);
~MinioTestEnvironment();

void SetUp() override;
Expand All @@ -84,6 +84,7 @@ class MinioTestEnvironment : public ::testing::Environment {
protected:
struct Impl;
std::unique_ptr<Impl> impl_;
bool enable_tls_if_supported_ = true; // by default, enable TLS if supported
};

// A global test "environment", to ensure that the S3 API is initialized before
Expand Down
24 changes: 20 additions & 4 deletions cpp/src/arrow/filesystem/s3fs_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,15 @@ ::testing::Environment* s3_env = ::testing::AddGlobalTestEnvironment(new S3Envir
::testing::Environment* minio_env =
::testing::AddGlobalTestEnvironment(new MinioTestEnvironment);

MinioTestEnvironment* GetMinioEnv() {
return ::arrow::internal::checked_cast<MinioTestEnvironment*>(minio_env);
::testing::Environment* minio_env_http =
::testing::AddGlobalTestEnvironment(new MinioTestEnvironment(false));

MinioTestEnvironment* GetMinioEnv(bool enable_tls_if_supported) {
if (enable_tls_if_supported) {
return ::arrow::internal::checked_cast<MinioTestEnvironment*>(minio_env);
} else {
return ::arrow::internal::checked_cast<MinioTestEnvironment*>(minio_env_http);
}
}

class ShortRetryStrategy : public S3RetryStrategy {
Expand Down Expand Up @@ -203,7 +210,7 @@ class S3TestMixin : public AwsTestMixin {

protected:
Status InitServerAndClient() {
ARROW_ASSIGN_OR_RAISE(minio_, GetMinioEnv()->GetOneServer());
ARROW_ASSIGN_OR_RAISE(minio_, GetMinioEnv(enable_tls_if_supported_)->GetOneServer());
client_config_.reset(new Aws::Client::ClientConfiguration());
client_config_->endpointOverride = ToAwsString(minio_->connect_string());
if (minio_->scheme() == "https") {
Expand All @@ -230,6 +237,7 @@ class S3TestMixin : public AwsTestMixin {
std::unique_ptr<Aws::Client::ClientConfiguration> client_config_;
Aws::Auth::AWSCredentials credentials_;
std::unique_ptr<Aws::S3::S3Client> client_;
bool enable_tls_if_supported_ = true;
};

void AssertGetObject(Aws::S3::Model::GetObjectResult& result,
Expand Down Expand Up @@ -914,7 +922,15 @@ TEST_F(TestS3FS, GetFileInfoGenerator) {
// Non-root dir case is tested by generic tests
}

TEST_F(TestS3FS, GetFileInfoGeneratorStress) {
class TestS3FSHTTP : public TestS3FS {
public:
void SetUp() override {
enable_tls_if_supported_ = false;
TestS3FS::SetUp();
}
};

TEST_F(TestS3FSHTTP, GetFileInfoGeneratorStress) {
// This test is slow because it needs to create a bunch of seed files. However, it is
// the only test that stresses listing and deleting when there are more than 1000
// files and paging is required.
Expand Down

0 comments on commit 598f146

Please sign in to comment.