Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 2205 - 4-3-stable #2208

Merged
merged 2 commits into from
Jul 11, 2024
Merged

Issue 2205 - 4-3-stable #2208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a10faf
[#2205] Use shared memory to share number of upload threads
JustinKyleJames Jul 10, 2024
19a7c86
[#2207] Changing filenames in test cases
JustinKyleJames Jul 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 54 additions & 54 deletions packaging/resource_suite_s3_nocache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1190,9 +1190,9 @@ def test_itrim_returns_on_negative_status__ticket_3531(self):

def test_iput_large_file_over_smaller(self):

file1 = "f1"
file2 = "f2"
filename_get = "f3"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
filename_get = f'{inspect.currentframe().f_code.co_name}_get'

file1_size = pow(2,20)
file2_size = 512*pow(2,20)
Expand Down Expand Up @@ -1227,9 +1227,9 @@ def test_iput_large_file_over_smaller(self):

def test_iput_small_file_over_larger(self):

file1 = "f1"
file2 = "f2"
filename_get = "f3"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
filename_get = f'{inspect.currentframe().f_code.co_name}_get'

file1_size = 512*pow(2,20)
file2_size = pow(2,20)
Expand Down Expand Up @@ -1453,7 +1453,7 @@ def test_small_write_read_in_large_file(self):
OUTPUT ruleExecOut
'''.format(**locals())

file1 = 'f1'
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
write_rule_file_path = 'test_small_write.r'
read_rule_file_path = 'test_small_read.r'

Expand Down Expand Up @@ -1493,8 +1493,8 @@ def test_small_write_read_in_large_file(self):
def test_detached_mode(self):

try:
file1 = "f1"
file2 = "f2"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
resource_host = "irods.org"
resource_name = 's3resc1'

Expand Down Expand Up @@ -1547,7 +1547,7 @@ def test_attached_mode_invalid_resource_host(self):
# in non-topology HOSTNAME_3 is just local host so it really doesn't test detached mode
# but in topology it will

file1 = "f1"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
#resource_host = test.settings.HOSTNAME_3
resource_host = "irods.org"

Expand Down Expand Up @@ -1608,8 +1608,8 @@ def test_copy_file_greater_than_chunk_size(self):

try:

file1 = "f1"
file2 = "f2"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
lib.make_file(file1, 128*1024*1024, 'arbitrary')
self.user0.assert_icommand("iput {file1}".format(**locals())) # iput
self.user0.assert_icommand("imv {file1} {file2}".format(**locals())) # imv
Expand Down Expand Up @@ -1640,9 +1640,9 @@ def test_put_get_small_file_in_repl_node(self):
self.admin.assert_icommand("iadmin addchildtoresc s3repl s3resc3", 'EMPTY')

# create and put file
file1 = "f1"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
lib.make_file(file1, 1024, 'arbitrary')
file2 = "f1.get"
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
self.user0.assert_icommand("iput -R s3repl {file1}".format(**locals())) # iput

# get file from first repl
Expand Down Expand Up @@ -1698,8 +1698,8 @@ def test_put_get_large_file_in_repl_node(self):
self.admin.assert_icommand("iadmin addchildtoresc s3repl s3resc3", 'EMPTY')

# create and put file
file1 = "f1"
file2 = "f1.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
lib.make_file(file1, 120*1024*1024)
self.user0.assert_icommand("iput -R s3repl {file1}".format(**locals())) # iput

Expand Down Expand Up @@ -1776,8 +1776,8 @@ def test_put_get_various_file_sizes(self):
320 * 1024 * 1024 + 1, # 16 threads, different sizes, multipart upload, three parts per thread
];

file1 = "f1"
file2 = "f1.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'

for file_size in file_sizes:

Expand Down Expand Up @@ -1814,7 +1814,7 @@ def test_rm_without_force(self):
self.admin.assert_icommand("iadmin mkresc s3resc1 s3 %s:/%s/%s/s3resc1 %s" %
(hostname, self.s3bucketname, hostuser, self.s3_context), 'STDOUT_SINGLELINE', "Creating")

file1 = "f1"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file_size = 120*1024*1024;

# create and put file
Expand Down Expand Up @@ -2235,8 +2235,8 @@ def test_put_get_file_greater_than_4GiB_one_thread(self):
self.admin.assert_icommand("iadmin mkresc s3resc1 s3 %s:/%s/%s/s3resc1 %s" %
(hostname, self.s3bucketname, hostuser, self.s3_context), 'STDOUT_SINGLELINE', "Creating")

file1 = "f1"
file2 = "f1.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'

file_size = 4*1024*1024*1024 + 1 # +1 - make sure one thread handles more than 4 GiB

Expand Down Expand Up @@ -2276,8 +2276,8 @@ def test_put_get_file_greater_than_8GiB_two_threads(self):
self.admin.assert_icommand("iadmin mkresc s3resc1 s3 %s:/%s/%s/s3resc1 %s" %
(hostname, self.s3bucketname, hostuser, self.s3_context), 'STDOUT_SINGLELINE', "Creating")

file1 = "f1"
file2 = "f1.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'

file_size = 8*1024*1024*1024 + 2 # +2 - make sure each thread handles more than 4 GiB

Expand Down Expand Up @@ -2333,8 +2333,8 @@ def test_large_file_put_repl_node(self):
self.admin.assert_icommand("iadmin addchildtoresc replresc s3resc1", 'EMPTY')
self.admin.assert_icommand("iadmin addchildtoresc replresc s3resc2", 'EMPTY')

file1 = "f1"
file2 = "f1.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'

file_size = 1024*1024*1024;

Expand Down Expand Up @@ -2377,8 +2377,8 @@ def test_upload_very_large_file_after_redirect_issues_2104_2114(self):
# in non-topology HOSTNAME_3 is just local host so it really doesn't test detached mode
# but in topology it will

file1 = "f1"
retrieved_file = "f1.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
retrieved_file = f'{inspect.currentframe().f_code.co_name}_get'
resource_host = test.settings.HOSTNAME_3
resource_name = "s3_resc_on_host3"

Expand Down Expand Up @@ -2747,10 +2747,10 @@ def test_put_get_glacier_expedited_retrieval(self):
self.admin.assert_icommand("iadmin mkresc s3resc1 s3 %s:/%s/%s/s3resc1 %s" %
(hostname, self.s3bucketname, hostuser, s3_context), 'STDOUT_SINGLELINE', "Creating")

file1 = "f1"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file1_get = "f1.get"
file2 = "f2"
file2_get = "f2.get"
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
file2_get = f'{file2}_get'

file1_size = 8*1024*1024
file2_size = 32*1024*1024 + 1
Expand Down Expand Up @@ -2818,10 +2818,10 @@ def test_put_get_glacier_standard_retrieval(self):
self.admin.assert_icommand("iadmin mkresc s3resc1 s3 %s:/%s/%s/s3resc1 %s" %
(hostname, self.s3bucketname, hostuser, s3_context), 'STDOUT_SINGLELINE', "Creating")

file1 = "f1"
file1_get = "f1.get"
file2 = "f2"
file2_get = "f2.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file1_get = f'{file1}_get'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
file2_get = f'{file2}_get'

file1_size = 8*1024*1024
file2_size = 32*1024*1024 + 1
Expand Down Expand Up @@ -2879,10 +2879,10 @@ def test_put_get_glacier_bulk_retrieval(self):
self.admin.assert_icommand("iadmin mkresc s3resc1 s3 %s:/%s/%s/s3resc1 %s" %
(hostname, self.s3bucketname, hostuser, s3_context), 'STDOUT_SINGLELINE', "Creating")

file1 = "f1"
file1_get = "f1.get"
file2 = "f2"
file2_get = "f2.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file1_get = f'{file1}_get'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
file2_get = f'{file2}_get'

file1_size = 8*1024*1024
file2_size = 32*1024*1024 + 1
Expand Down Expand Up @@ -2941,10 +2941,10 @@ def test_put_get_deep_archive(self):
self.admin.assert_icommand("iadmin mkresc s3resc1 s3 %s:/%s/%s/s3resc1 %s" %
(hostname, self.s3bucketname, hostuser, s3_context), 'STDOUT_SINGLELINE', "Creating")

file1 = "f1"
file1_get = "f1.get"
file2 = "f2"
file2_get = "f2.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file1_get = f'{file1}_get'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
file2_get = f'{file2}_get'

file1_size = 8*1024*1024
file2_size = 32*1024*1024 + 1
Expand Down Expand Up @@ -3003,10 +3003,10 @@ def test_put_get_glacier_ir(self):
self.admin.assert_icommand("iadmin mkresc s3resc1 s3 %s:/%s/%s/s3resc1 %s" %
(hostname, self.s3bucketname, hostuser, s3_context), 'STDOUT_SINGLELINE', "Creating")

file1 = "f1"
file1_get = "f1.get"
file2 = "f2"
file2_get = "f2.get"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file1_get = f'{file1}_get'
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
file1_get = f'{file2}_get'

file1_size = 8*1024*1024
file2_size = 32*1024*1024 + 1
Expand Down Expand Up @@ -3070,9 +3070,9 @@ class Test_S3_NoCache_Decoupled_Base(Test_S3_NoCache_Base):

def test_decoupled_redirect_issue_2146(self):

file1 = "f1"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file1_size = 2*1024*1024
retrieved_file = "f1.get"
retrieved_file = f'{inspect.currentframe().f_code.co_name}_get'
resource_host = test.settings.HOSTNAME_3
resource_name = "s3_resc_on_host3"

Expand Down Expand Up @@ -3105,11 +3105,11 @@ def test_decoupled_redirect_issue_2146(self):
# This verifies that once the file is written to the DB, whatever mode was used remains in effect as the DB always wins.
def test_resource_updated_from_consistent_to_decoupled_issue_2161(self):

file1 = "f1"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file1_size = 2*1024
file2 = "f2"
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
file2_size = 3*1024
retrieved_file = 'f.get'
retrieved_file = f'{inspect.currentframe().f_code.co_name}_get'
hostname = lib.get_hostname()
resource_name = "s3_resc"

Expand Down Expand Up @@ -3164,11 +3164,11 @@ def test_resource_updated_from_consistent_to_decoupled_issue_2161(self):
# This verifies that once the file is written to the DB, whatever mode was used remains in effect as the DB always wins.
def test_resource_updated_from_decoupled_to_consistent_issue_2161(self):

file1 = "f1"
file1 = f'{inspect.currentframe().f_code.co_name}_f1'
file1_size = 2*1024
file2 = "f2"
file2 = f'{inspect.currentframe().f_code.co_name}_f2'
file2_size = 3*1024
retrieved_file = 'f.get'
retrieved_file = f'{inspect.currentframe().f_code.co_name}_get'
hostname = lib.get_hostname()
resource_name = "s3_resc"

Expand Down
45 changes: 45 additions & 0 deletions s3_resource/include/irods/private/s3_resource/multipart_shared_data.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#ifndef IRODS_S3_RESOURCE_MULTIPART_SHARED_DATA_HPP
#define IRODS_S3_RESOURCE_MULTIPART_SHARED_DATA_HPP

#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/containers/map.hpp>
#include <boost/interprocess/containers/vector.hpp>
#include <boost/interprocess/containers/list.hpp>
#include <boost/interprocess/allocators/allocator.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/containers/string.hpp>
#include <boost/interprocess/sync/named_mutex.hpp>
#include <boost/container/scoped_allocator.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>

#include <iostream>

namespace irods_s3
{
namespace interprocess_types
{
namespace bi = boost::interprocess;
using segment_manager = bi::managed_shared_memory::segment_manager;
using void_allocator = boost::container::scoped_allocator_adaptor<bi::allocator<void, segment_manager> >;
}

// data that needs to be shared among different processes
struct multipart_shared_data
{
explicit multipart_shared_data(const interprocess_types::void_allocator &allocator)
: threads_remaining_to_close{0}
, number_of_threads{0}
, ref_count{0}
JustinKyleJames marked this conversation as resolved.
Show resolved Hide resolved
{}

bool can_delete() {
return threads_remaining_to_close == 0;
}

int threads_remaining_to_close;
int number_of_threads;
int ref_count;
}; // struct multipart_shared_data
} // namespace irods_s3

#endif // IRODS_S3_RESOURCE_MULTIPART_SHARED_DATA_HPP
Loading
Loading