Merge pull request #33 from bioimage-io/zenodo_backup

Add backup to Zenodo

.github/workflows/backup_call.yaml

@@ -22,7 +22,6 @@ env:
   S3_HOST: ${{inputs.S3_HOST}}
   S3_BUCKET: ${{inputs.S3_BUCKET}}
   S3_FOLDER: ${{inputs.S3_FOLDER}}
-  ZENODO_URL: ${{inputs.ZENODO_URL}}
   S3_ACCESS_KEY_ID: ${{secrets.S3_ACCESS_KEY_ID}}
   S3_SECRET_ACCESS_KEY: ${{secrets.S3_SECRET_ACCESS_KEY}}
   ZENODO_API_ACCESS_TOKEN: ${{secrets.ZENODO_API_ACCESS_TOKEN}}
@@ -37,4 +36,4 @@ jobs:
           python-version: "3.12"
           cache: "pip" # caching pip dependencies
       - run: pip install .
-      - run: backoffice backup
+      - run: backoffice backup  ${{inputs.ZENODO_URL}}

bioimageio_collection_backoffice/_backoffice.py

@@ -1,39 +1,38 @@
-import os
+from datetime import datetime
 from pathlib import Path
 from typing import Literal, Optional, Union
 
 from bioimageio.spec.model.v0_5 import WeightsFormat
-from dotenv import load_dotenv
 from loguru import logger
 
-from .backup import backup
+from ._settings import settings
+from .backup import ZenodoHost, backup
 from .generate_collection_json import generate_collection_json
 from .gh_utils import set_gh_actions_outputs
 from .mailroom import notify_uploader
 from .remote_resource import (
     PublishedVersion,
-    RemoteResource,
+    ResourceConcept,
     get_remote_resource_version,
 )
 from .run_dynamic_tests import run_dynamic_tests
 from .s3_client import Client
 from .s3_structure.chat import Chat, Message
 from .validate_format import validate_format
 
-_ = load_dotenv()
-
 
 class BackOffice:
     """This backoffice aids to maintain the bioimage.io collection"""
 
     def __init__(
         self,
-        host: str = os.environ["S3_HOST"],
-        bucket: str = os.environ["S3_BUCKET"],
-        prefix: str = os.environ["S3_FOLDER"],
+        host: str = settings.s3_host,
+        bucket: str = settings.s3_bucket,
+        prefix: str = settings.s3_folder,
     ) -> None:
         super().__init__()
         self.client = Client(host=host, bucket=bucket, prefix=prefix)
+        logger.info("created backoffice with client {}", self.client)
 
     def wipe(self, subfolder: str = ""):
         """DANGER ZONE: wipes `subfolder` completely, only use for test folders!"""
@@ -46,7 +45,7 @@ def wipe(self, subfolder: str = ""):
 
     def stage(self, resource_id: str, package_url: str):
         """stage a new resourse (version) from `package_url`"""
-        resource = RemoteResource(self.client, resource_id)
+        resource = ResourceConcept(self.client, resource_id)
         staged = resource.stage_new_version(package_url)
         set_gh_actions_outputs(version=staged.version)
 
@@ -132,9 +131,9 @@ def publish(self, resource_id: str, version: str, reviewer: str):
             + "Check it out at https://bioimage.io/#/?id={published.id}\n",  # TODO: link to version
         )
 
-    def backup(self, destination: Optional[str] = None):
+    def backup(self, destination: ZenodoHost):
         """backup the whole collection (to zenodo.org)"""
-        _ = backup(self.client, destination or os.environ["ZENODO_URL"])
+        _ = backup(self.client, destination)
 
     def generate_collection_json(
         self, collection_template: Path = Path("collection_template.json")
@@ -149,6 +148,10 @@ def forward_emails_to_chat(self):
     def add_chat_message(
         self, resource_id: str, version: str, chat_message: str, author: str
     ):
-        chat = Chat(messages=[Message(author=author, text=chat_message)])
+        chat = Chat(
+            messages=[
+                Message(author=author, text=chat_message, timestamp=datetime.now())
+            ]
+        )
         rv = get_remote_resource_version(self.client, resource_id, version)
         rv.extend_chat(chat)

bioimageio_collection_backoffice/_settings.py

@@ -0,0 +1,49 @@
+from typing import Literal, Optional
+
+from loguru import logger
+from pydantic import SecretStr
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings, extra="ignore"):
+    """environment variables for bioimageio.spec"""
+
+    model_config = SettingsConfigDict(
+        case_sensitive=False,
+        env_file=".env",
+        env_file_encoding="utf-8",
+    )
+
+    github_output: Optional[str] = None
+    reviewers: str = (
+        "https://raw.githubusercontent.com/bioimage-io/collection/main/reviewers.json"
+    )
+    s3_bucket: str = "public-datasets"
+    s3_folder: str = "sandbox.bioimage.io"
+    s3_host: str = "uk1s3.embassy.ebi.ac.uk"
+    s3_pytest_folder: str = "testing.bioimage.io/user_pytest"
+    s3_sandbox_folder: str = "sandbox.bioimage.io"
+    s3_test_bucket: str = "public-datasets"
+    s3_test_folder: str = "testing.bioimage.io/user_sandbox"
+    test_package_id: str = "frank-water-buffalo"
+    test_package_url: str = (
+        "https://uk1s3.embassy.ebi.ac.uk/public-datasets/examples.bioimage.io/frank-water-buffalo_v1.zip"
+    )
+    zenodo_test_url: Literal["https://sandbox.zenodo.org"] = (
+        "https://sandbox.zenodo.org"
+    )
+    zenodo_url: Literal["https://sandbox.zenodo.org", "https://zenodo.org"] = (
+        "https://sandbox.zenodo.org"
+    )
+
+    # secrets
+    mail_password: SecretStr = SecretStr("")
+    s3_access_key_id: SecretStr = SecretStr("")
+    s3_secret_access_key: SecretStr = SecretStr("")
+    zenodo_api_access_token: SecretStr = SecretStr("")
+    zenodo_test_api_access_token: SecretStr = SecretStr("")
+    github_pat: SecretStr = SecretStr("")
+
+
+settings = Settings()
+logger.info("settings: {}", settings)