From 3b1dc9daff3fdc44e1d5312c5dc9ce12f20ddbe2 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 10 Apr 2024 09:52:37 +0200 Subject: [PATCH 1/3] add sandbox flag --- .github/workflows/emails_to_chat.yaml | 8 +++++++- .github/workflows/publish.yaml | 8 ++++++-- .github/workflows/retest_all.yaml | 14 +++++++++++--- .github/workflows/stage.yaml | 9 +++++++-- .github/workflows/test.yaml | 7 ++++++- 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/.github/workflows/emails_to_chat.yaml b/.github/workflows/emails_to_chat.yaml index b9081d2a..d841f3a3 100644 --- a/.github/workflows/emails_to_chat.yaml +++ b/.github/workflows/emails_to_chat.yaml @@ -3,6 +3,12 @@ name: forward emails to chat on: workflow_call: + inputs: + sandbox: + description: Flag to use sandbox instead + required: false + default: true + type: boolean # schedule: # - cron: "0 * * * *" # every hour at minute 0 @@ -11,7 +17,7 @@ concurrency: forward-emails-to-chat env: S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} - S3_FOLDER: ${{vars.S3_FOLDER}} + S3_FOLDER: ${{inputs.sandbox && vars.S3_SANDBOX_FOLDER || vars.S3_FOLDER}} S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}} S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}} MAIL_PASSWORD: ${{secrets.MAIL_PASSWORD}} diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 8e120862..3a225f3e 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -16,7 +16,11 @@ on: description: Name of bioimage.io maintainer accepting this resource version required: true type: string - + sandbox: + description: Flag to publish within sandbox instead + required: false + default: false + type: boolean concurrency: ${{inputs.resource_id}} @@ -29,5 +33,5 @@ jobs: reviewer: ${{inputs.reviewer}} S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} - S3_FOLDER: ${{vars.S3_FOLDER}} + S3_FOLDER: ${{inputs.sandbox && vars.S3_SANDBOX_FOLDER || vars.S3_FOLDER}} secrets: inherit diff --git a/.github/workflows/retest_all.yaml b/.github/workflows/retest_all.yaml index 7d132f9d..6dd46dc9 100644 --- a/.github/workflows/retest_all.yaml +++ b/.github/workflows/retest_all.yaml @@ -1,6 +1,14 @@ name: retest all bioimageio resources adding to their logs -on: workflow_dispatch +on: + workflow_dispatch: + inputs: + sandbox: + description: Flag to use sandbox instead + required: false + default: true + type: boolean + concurrency: retest jobs: @@ -10,7 +18,7 @@ jobs: matrix: ${{steps.get_matrix.outputs.matrix}} steps: - - run: wget ${{vars.S3_HOST}}/${{vars.S3_BUCKET}}/${{vars.S3_FOLDER}}/collection.json + - run: wget ${{vars.S3_HOST}}/${{vars.S3_BUCKET}}/${{inputs.sandbox && vars.S3_SANDBOX_FOLDER || vars.S3_FOLDER}}/collection.json - shell: python id: get_matrix run: | @@ -35,7 +43,7 @@ jobs: conclude: 'no' S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} - S3_FOLDER: ${{vars.S3_FOLDER}} + S3_FOLDER: ${{inputs.sandbox && vars.S3_SANDBOX_FOLDER || vars.S3_FOLDER}} secrets: inherit # TODO: call emailer diff --git a/.github/workflows/stage.yaml b/.github/workflows/stage.yaml index e7fbae97..9115ec2a 100644 --- a/.github/workflows/stage.yaml +++ b/.github/workflows/stage.yaml @@ -12,6 +12,11 @@ on: description: "Download URL of the resource package zip-file" required: true type: string + sandbox: + description: Flag to stage to sandbox instead + required: false + default: false + type: boolean concurrency: ${{inputs.resource_id}} @@ -23,7 +28,7 @@ jobs: package_url: ${{inputs.package_url}} S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} - S3_FOLDER: ${{vars.S3_FOLDER}} + S3_FOLDER: ${{inputs.sandbox && vars.S3_SANDBOX_FOLDER || vars.S3_FOLDER}} secrets: inherit test: @@ -34,5 +39,5 @@ jobs: version: ${{needs.stage.outputs.version}} S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} - S3_FOLDER: ${{vars.S3_FOLDER}} + S3_FOLDER: ${{inputs.sandbox && vars.S3_SANDBOX_FOLDER || vars.S3_FOLDER}} secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ce935f0b..6f267ccb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,6 +12,11 @@ on: description: stage number to test required: true type: number + sandbox: + description: Flag to test resource in sandbox instead + required: false + default: false + type: boolean concurrency: ${{inputs.resource_id}} @@ -23,7 +28,7 @@ jobs: version: staged/${{inputs.stage_number}} S3_HOST: ${{vars.S3_HOST}} S3_BUCKET: ${{vars.S3_BUCKET}} - S3_FOLDER: ${{vars.S3_FOLDER}} + S3_FOLDER: ${{inputs.sandbox && vars.S3_SANDBOX_FOLDER || vars.S3_FOLDER}} secrets: inherit # TODO: call emailer From 4eb38fe44b14ce29f95b639d63180c63ed029a1e Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 10 Apr 2024 09:53:02 +0200 Subject: [PATCH 2/3] disallow empty prefix --- bioimageio_collection_backoffice/s3_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bioimageio_collection_backoffice/s3_client.py b/bioimageio_collection_backoffice/s3_client.py index 5fc118e0..021e0b1b 100644 --- a/bioimageio_collection_backoffice/s3_client.py +++ b/bioimageio_collection_backoffice/s3_client.py @@ -41,6 +41,9 @@ class Client: def __post_init__(self): self.prefix = self.prefix.strip("/") + if not self.prefix: + raise ValueError("empty prefix not allowed") + self._client = Minio( self.host, access_key=self.access_key, From 37a514b2695f622244b77c7088c5bdf8724bcb4b Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 10 Apr 2024 09:58:10 +0200 Subject: [PATCH 3/3] remove unused rm_obj --- bioimageio_collection_backoffice/s3_client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bioimageio_collection_backoffice/s3_client.py b/bioimageio_collection_backoffice/s3_client.py index 021e0b1b..661a6dcb 100644 --- a/bioimageio_collection_backoffice/s3_client.py +++ b/bioimageio_collection_backoffice/s3_client.py @@ -181,10 +181,6 @@ def _cp_dir(self, src: str, tgt: str): return objects - def rm_obj(self, name: str) -> None: - """remove single object""" - self._client.remove_object(self.bucket, name) - def _rm_objs( self, objects: Sequence[Object], *, bypass_governance_mode: bool ) -> None: