diff --git a/beetmoverscript/src/beetmoverscript/data/artifactMap_beetmover_task_schema.json b/beetmoverscript/src/beetmoverscript/data/artifactMap_beetmover_task_schema.json index 675443455..19a3c183e 100644 --- a/beetmoverscript/src/beetmoverscript/data/artifactMap_beetmover_task_schema.json +++ b/beetmoverscript/src/beetmoverscript/data/artifactMap_beetmover_task_schema.json @@ -89,6 +89,14 @@ "minItems": 1, "uniqueItems": true }, + "exclude": { + "type": "array", + "minItems": 0, + "uniqueItems": false, + "items": { + "type": "string" + } + }, "artifactMap": { "type": "array", "items": { diff --git a/beetmoverscript/src/beetmoverscript/data/beetmover_task_schema.json b/beetmoverscript/src/beetmoverscript/data/beetmover_task_schema.json index a8583803b..024e34546 100644 --- a/beetmoverscript/src/beetmoverscript/data/beetmover_task_schema.json +++ b/beetmoverscript/src/beetmoverscript/data/beetmover_task_schema.json @@ -85,6 +85,14 @@ }, "minItems": 1, "uniqueItems": true + }, + "exclude": { + "type": "array", + "minItems": 0, + "uniqueItems": false, + "items": { + "type": "string" + } } }, "required": ["upload_date", "upstreamArtifacts", "releaseProperties"] diff --git a/beetmoverscript/src/beetmoverscript/data/maven_beetmover_task_schema.json b/beetmoverscript/src/beetmoverscript/data/maven_beetmover_task_schema.json index 702e7528c..70b956613 100644 --- a/beetmoverscript/src/beetmoverscript/data/maven_beetmover_task_schema.json +++ b/beetmoverscript/src/beetmoverscript/data/maven_beetmover_task_schema.json @@ -82,6 +82,14 @@ }, "minItems": 1, "uniqueItems": true + }, + "exclude": { + "type": "array", + "minItems": 0, + "uniqueItems": false, + "items": { + "type": "string" + } } }, "required": ["upstreamArtifacts", "releaseProperties"] diff --git a/beetmoverscript/src/beetmoverscript/data/release_beetmover_task_schema.json b/beetmoverscript/src/beetmoverscript/data/release_beetmover_task_schema.json index a0b7bfe13..0cfe8678c 100644 --- a/beetmoverscript/src/beetmoverscript/data/release_beetmover_task_schema.json +++ b/beetmoverscript/src/beetmoverscript/data/release_beetmover_task_schema.json @@ -53,6 +53,14 @@ "minItems": 0, "uniqueItems": true }, + "exclude": { + "type": "array", + "minItems": 0, + "uniqueItems": false, + "items": { + "type": "string" + } + }, "partners": { "type": "array", "minItems": 0, diff --git a/beetmoverscript/src/beetmoverscript/gcloud.py b/beetmoverscript/src/beetmoverscript/gcloud.py index a464f9913..326dc594d 100644 --- a/beetmoverscript/src/beetmoverscript/gcloud.py +++ b/beetmoverscript/src/beetmoverscript/gcloud.py @@ -203,6 +203,7 @@ async def push_to_releases_gcs(context): # Weed out RELEASE_EXCLUDE matches, but allow partners specified in the payload push_partners = context.task["payload"].get("partners", []) + exclude = context.task["payload"].get("exclude", []) + list(RELEASE_EXCLUDE) for blob_path in candidates_blobs.keys(): if "/partner-repacks/" in blob_path: @@ -214,7 +215,7 @@ async def push_to_releases_gcs(context): ) else: log.debug("Excluding partner repack {}".format(blob_path)) - elif not matches_exclude(blob_path, RELEASE_EXCLUDE): + elif not matches_exclude(blob_path, exclude): blobs_to_copy[blob_path] = blob_path.replace(candidates_prefix, releases_prefix) else: log.debug("Excluding {}".format(blob_path)) diff --git a/beetmoverscript/src/beetmoverscript/script.py b/beetmoverscript/src/beetmoverscript/script.py index f1e600621..158e11aa4 100755 --- a/beetmoverscript/src/beetmoverscript/script.py +++ b/beetmoverscript/src/beetmoverscript/script.py @@ -226,6 +226,8 @@ async def push_to_releases_s3(context): # Weed out RELEASE_EXCLUDE matches, but allow partners specified in the payload push_partners = context.task["payload"].get("partners", []) + exclude = context.task["payload"].get("exclude", []) + list(RELEASE_EXCLUDE) + for k in candidates_keys_checksums.keys(): if "/partner-repacks/" in k: partner_match = get_partner_match(k, candidates_prefix, push_partners) @@ -236,7 +238,7 @@ async def push_to_releases_s3(context): ) else: log.debug("Excluding partner repack {}".format(k)) - elif not matches_exclude(k, RELEASE_EXCLUDE): + elif not matches_exclude(k, exclude): context.artifacts_to_beetmove[k] = k.replace(candidates_prefix, releases_prefix) else: log.debug("Excluding {}".format(k)) diff --git a/beetmoverscript/tests/test_gcloud.py b/beetmoverscript/tests/test_gcloud.py index a4ccc4292..a5e58d611 100644 --- a/beetmoverscript/tests/test_gcloud.py +++ b/beetmoverscript/tests/test_gcloud.py @@ -10,6 +10,7 @@ from scriptworker.exceptions import ScriptWorkerTaskException import beetmoverscript.gcloud +from beetmoverscript.utils import get_candidates_prefix, get_releases_prefix from . import get_fake_valid_task, noop_sync @@ -233,6 +234,41 @@ def fake_list_bucket_objects_gcs_same(client, bucket, prefix): await beetmoverscript.gcloud.push_to_releases_gcs(context) +@pytest.mark.parametrize( + "candidate_blobs,exclude,results", + [ + ({"foo/bar.zip": "md5hash", "foo/baz.exe": "abcd", "foo/qux.js": "shasum"}, [], ["foo/baz.exe", "foo/qux.js"]), + ({"foo/bar.zip": "md5hash", "foo/baz.exe": "abcd", "foo/qux.js": "shasum"}, [r"^.*\.exe$"], ["foo/qux.js"]), + ({"foo/bar.zip": "md5hash", "foo/baz.exe": "abcd", "foo/qux.js": "shasum"}, [r"^.*\.exe$", r"^.*\.js"], []), + ], +) +@pytest.mark.asyncio +async def test_push_to_releases_gcs_exclude(context, monkeypatch, candidate_blobs, exclude, results): + context.gcs_client = FakeClient() + context.task = get_fake_valid_task("task_push_to_releases.json") + + payload = context.task["payload"] + payload["exclude"] = exclude + test_candidate_prefix = get_candidates_prefix(payload["product"], payload["version"], payload["build_number"]) + test_release_prefix = get_releases_prefix(payload["product"], payload["version"]) + + def fake_list_bucket_objects_gcs_same(client, bucket, prefix): + if "candidates" in prefix: + return {f"{prefix}{key}": value for (key, value) in candidate_blobs.items()} + if "releases" in prefix: + return {} + + expect_blobs = {f"{test_candidate_prefix}{key}": f"{test_release_prefix}{key}" for key in results} + + def fake_move_artifacts(client, bucket_name, blobs_to_copy, candidates_blobs, releases_blobs): + assert blobs_to_copy == expect_blobs + + monkeypatch.setattr(beetmoverscript.gcloud, "list_bucket_objects_gcs", fake_list_bucket_objects_gcs_same) + monkeypatch.setattr(beetmoverscript.gcloud, "move_artifacts", fake_move_artifacts) + + await beetmoverscript.gcloud.push_to_releases_gcs(context) + + def test_list_bucket_objects_gcs(): beetmoverscript.gcloud.list_bucket_objects_gcs(FakeClient(), "foobucket", "prefix") diff --git a/beetmoverscript/tests/test_script.py b/beetmoverscript/tests/test_script.py index 076766941..b0a5fffe5 100644 --- a/beetmoverscript/tests/test_script.py +++ b/beetmoverscript/tests/test_script.py @@ -29,7 +29,7 @@ setup_mimetypes, ) from beetmoverscript.task import get_release_props, get_upstream_artifacts -from beetmoverscript.utils import generate_beetmover_manifest, is_promotion_action +from beetmoverscript.utils import get_candidates_prefix, get_releases_prefix, generate_beetmover_manifest, is_promotion_action from . import get_fake_valid_config, get_fake_valid_task, get_test_jinja_env, noop_async, noop_sync @@ -70,6 +70,39 @@ def fake_list(*args): await push_to_releases_s3(context) +# push_to_releases_s3_exclude {{{1 +@pytest.mark.parametrize( + "candidates_keys,exclude,results", + [ + ({"foo.zip": "x", "bar.exe": "y", "baz.js": "z"}, [], ["bar.exe", "baz.js"]), + ({"foo.zip": "x", "bar.exe": "y", "baz.js": "z"}, [r"^.*\.exe$"], ["baz.js"]), + ({"foo.zip": "x", "bar.exe": "y", "baz.js": "z"}, [r"^.*\.exe$", r"^.*\.js"], []), + ], +) +@pytest.mark.asyncio +async def test_push_to_releases_s3_exclude(context, mocker, candidates_keys, exclude, results): + payload = {"product": "devedition", "build_number": 33, "version": "99.0b44", "exclude": exclude} + context.task = {"payload": payload} + test_candidate_prefix = get_candidates_prefix(payload["product"], payload["version"], payload["build_number"]) + test_release_prefix = get_releases_prefix(payload["product"], payload["version"]) + + objects = [{f"{test_candidate_prefix}{key}": candidates_keys[key] for key in candidates_keys}, {}] + + expect_artifacts = {f"{test_candidate_prefix}{key}": f"{test_release_prefix}{key}" for key in results} + + def check(ctx, _, r): + assert ctx.artifacts_to_beetmove == expect_artifacts + + def fake_list(*args): + return objects.pop(0) + + mocker.patch.object(boto3, "resource") + mocker.patch.object(beetmoverscript.script, "list_bucket_objects", new=fake_list) + mocker.patch.object(beetmoverscript.script, "copy_beets", new=check) + + await push_to_releases_s3(context) + + # copy_beets {{{1 @pytest.mark.parametrize("releases_keys,raises", (({}, False), ({"to2": "from2_md5"}, False), ({"to1": "to1_md5"}, True))) def test_copy_beets(context, mocker, releases_keys, raises):