From e9a4ce6c5a86d089bba12a3ac3b9800bf185c43d Mon Sep 17 00:00:00 2001 From: Yuval Lifshitz Date: Mon, 11 Nov 2024 17:48:39 +0000 Subject: [PATCH] test/rgw/logging: add bucket logging test suite to teuthology this is depended with this s3test PR: https://github.com/ceph/s3-tests/pull/576 and the bucket logging PR: https://github.com/ceph/ceph/pull/59808 also point to the s3test of bucket logging. this should be reverted before merging. Signed-off-by: Yuval Lifshitz --- qa/rgw/s3tests-branch.yaml | 4 +-- qa/rgw/service-2.sdk-extras.json | 1 + qa/suites/rgw/bucket-logging/% | 0 qa/suites/rgw/bucket-logging/.qa | 1 + qa/suites/rgw/bucket-logging/0-install.yaml | 13 +++++++++ qa/suites/rgw/bucket-logging/beast.yaml | 1 + qa/suites/rgw/bucket-logging/fixed-1.yaml | 1 + .../ignore-pg-availability.yaml | 1 + qa/suites/rgw/bucket-logging/overrides.yaml | 10 +++++++ .../rgw/bucket-logging/s3tests-branch.yaml | 1 + .../rgw/bucket-logging/supported-distros | 1 + qa/suites/rgw/bucket-logging/tasks/+ | 0 .../rgw/bucket-logging/tasks/s3tests.yaml | 6 ++++ qa/tasks/s3tests.py | 29 ++++++++++++++++++- 14 files changed, 66 insertions(+), 3 deletions(-) create mode 120000 qa/rgw/service-2.sdk-extras.json create mode 100644 qa/suites/rgw/bucket-logging/% create mode 120000 qa/suites/rgw/bucket-logging/.qa create mode 100644 qa/suites/rgw/bucket-logging/0-install.yaml create mode 120000 qa/suites/rgw/bucket-logging/beast.yaml create mode 120000 qa/suites/rgw/bucket-logging/fixed-1.yaml create mode 120000 qa/suites/rgw/bucket-logging/ignore-pg-availability.yaml create mode 100644 qa/suites/rgw/bucket-logging/overrides.yaml create mode 120000 qa/suites/rgw/bucket-logging/s3tests-branch.yaml create mode 120000 qa/suites/rgw/bucket-logging/supported-distros create mode 100644 qa/suites/rgw/bucket-logging/tasks/+ create mode 100644 qa/suites/rgw/bucket-logging/tasks/s3tests.yaml diff --git a/qa/rgw/s3tests-branch.yaml b/qa/rgw/s3tests-branch.yaml index ef6819c87e06a4..476d8fe010910c 100644 --- a/qa/rgw/s3tests-branch.yaml +++ b/qa/rgw/s3tests-branch.yaml @@ -1,4 +1,4 @@ overrides: s3tests: - force-branch: ceph-master - # git_remote: https://github.com/ceph/ + force-branch: wip-yuval-bucket-logging + git_remote: https://github.com/yuvalif/ diff --git a/qa/rgw/service-2.sdk-extras.json b/qa/rgw/service-2.sdk-extras.json new file mode 120000 index 00000000000000..cec0b55a7ef140 --- /dev/null +++ b/qa/rgw/service-2.sdk-extras.json @@ -0,0 +1 @@ +../../examples/rgw/boto3/service-2.sdk-extras.json \ No newline at end of file diff --git a/qa/suites/rgw/bucket-logging/% b/qa/suites/rgw/bucket-logging/% new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/qa/suites/rgw/bucket-logging/.qa b/qa/suites/rgw/bucket-logging/.qa new file mode 120000 index 00000000000000..a602a0353e7510 --- /dev/null +++ b/qa/suites/rgw/bucket-logging/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/rgw/bucket-logging/0-install.yaml b/qa/suites/rgw/bucket-logging/0-install.yaml new file mode 100644 index 00000000000000..fa6e279145c34f --- /dev/null +++ b/qa/suites/rgw/bucket-logging/0-install.yaml @@ -0,0 +1,13 @@ +tasks: +- install: +- ceph: +- openssl_keys: +- rgw: [client.0] +- tox: [client.0] + +overrides: + ceph: + conf: + global: + osd_min_pg_log_entries: 10 + osd_max_pg_log_entries: 10 diff --git a/qa/suites/rgw/bucket-logging/beast.yaml b/qa/suites/rgw/bucket-logging/beast.yaml new file mode 120000 index 00000000000000..09ced62c42aa84 --- /dev/null +++ b/qa/suites/rgw/bucket-logging/beast.yaml @@ -0,0 +1 @@ +.qa/rgw_frontend/beast.yaml \ No newline at end of file diff --git a/qa/suites/rgw/bucket-logging/fixed-1.yaml b/qa/suites/rgw/bucket-logging/fixed-1.yaml new file mode 120000 index 00000000000000..02df5dd0cd0435 --- /dev/null +++ b/qa/suites/rgw/bucket-logging/fixed-1.yaml @@ -0,0 +1 @@ +.qa/clusters/fixed-1.yaml \ No newline at end of file diff --git a/qa/suites/rgw/bucket-logging/ignore-pg-availability.yaml b/qa/suites/rgw/bucket-logging/ignore-pg-availability.yaml new file mode 120000 index 00000000000000..32340b1fa8be86 --- /dev/null +++ b/qa/suites/rgw/bucket-logging/ignore-pg-availability.yaml @@ -0,0 +1 @@ +.qa/rgw/ignore-pg-availability.yaml \ No newline at end of file diff --git a/qa/suites/rgw/bucket-logging/overrides.yaml b/qa/suites/rgw/bucket-logging/overrides.yaml new file mode 100644 index 00000000000000..a448a323d363dd --- /dev/null +++ b/qa/suites/rgw/bucket-logging/overrides.yaml @@ -0,0 +1,10 @@ +overrides: + ceph: + conf: + client: + setuser: ceph + setgroup: ceph + debug rgw: 20 + rgw bucket logging obj roll time: 5 + rgw: + storage classes: LUKEWARM, FROZEN diff --git a/qa/suites/rgw/bucket-logging/s3tests-branch.yaml b/qa/suites/rgw/bucket-logging/s3tests-branch.yaml new file mode 120000 index 00000000000000..bdcaca48ae02b4 --- /dev/null +++ b/qa/suites/rgw/bucket-logging/s3tests-branch.yaml @@ -0,0 +1 @@ +.qa/rgw/s3tests-branch.yaml \ No newline at end of file diff --git a/qa/suites/rgw/bucket-logging/supported-distros b/qa/suites/rgw/bucket-logging/supported-distros new file mode 120000 index 00000000000000..78f2991b407af5 --- /dev/null +++ b/qa/suites/rgw/bucket-logging/supported-distros @@ -0,0 +1 @@ +.qa/distros/supported-random-distro$/ \ No newline at end of file diff --git a/qa/suites/rgw/bucket-logging/tasks/+ b/qa/suites/rgw/bucket-logging/tasks/+ new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/qa/suites/rgw/bucket-logging/tasks/s3tests.yaml b/qa/suites/rgw/bucket-logging/tasks/s3tests.yaml new file mode 100644 index 00000000000000..c1d3b7192e15d1 --- /dev/null +++ b/qa/suites/rgw/bucket-logging/tasks/s3tests.yaml @@ -0,0 +1,6 @@ +tasks: +- s3tests: + client.0: + boto3_extensions: True + rgw_server: client.0 + extra_attrs: ["bucket_logging"] diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py index 6d7b39d5892103..cd35e47acc7dd0 100644 --- a/qa/tasks/s3tests.py +++ b/qa/tasks/s3tests.py @@ -57,6 +57,20 @@ def download(ctx, config): 'git', 'reset', '--hard', sha1, ], ) + if client_config.get('boto3_extensions'): + ctx.cluster.only(client).run( + args=['mkdir', + '-p', + '/home/ubuntu/.aws/models/s3/2006-03-01/'] + ) + ctx.cluster.only(client).run( + args=['cd', '/home/ubuntu/.aws/models/s3/2006-03-01/', + run.Raw('&&'), + 'cp', + '{qadir}/rgw/service-2.sdk-extras.json'.format(qadir=client_config.get('suite_path'), + 'service-2.sdk-extras.json'] + ) + try: yield finally: @@ -70,6 +84,17 @@ def download(ctx, config): '{tdir}/s3-tests-{client}'.format(tdir=testdir, client=client), ], ) + if client_config.get('boto3_extensions'): + ctx.cluster.only(client).run( + args=[ + 'rm', '-rf', '/home/ubuntu/.aws/models/s3/2006-03-01/service-2.sdk-extras.json', + ], + ) + ctx.cluster.only(client).run( + args=[ + 'cd', '/home/ubuntu/', run.Raw('&&'), 'rmdir', '-p', '.aws/models/s3/2006-03-01/', + ], + ) def _config_user(s3tests_conf, section, user, email): @@ -444,8 +469,10 @@ def run_tests(ctx, config): attrs += ['not fails_with_subdomain'] if not client_config.get('with-sse-s3'): attrs += ['not sse_s3'] - + attrs += client_config.get('extra_attrs', []) + if 'bucket_logging' not in attrs: + attrs += ['not bucket_logging'] if 'unit_test_scan' in client_config and client_config['unit_test_scan']: xmlfile_id = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S--") + str(uuid.uuid4()) xmlpath= f'{testdir}/archive/s3test-{xmlfile_id}.xml'