From 44e5251d34ab4f4aab1cda82fb1fab77e9b46e0b Mon Sep 17 00:00:00 2001 From: doggie <3859395+fubuloubu@users.noreply.github.com> Date: Tue, 4 Jun 2024 12:53:39 -0400 Subject: [PATCH 01/54] feat: add support for platform login, cluster API CRUD methods --- setup.py | 2 + silverback/_cli.py | 161 ++++++++++++++++++++++++++++++++ silverback/platform/__init__.py | 1 + silverback/platform/client.py | 138 +++++++++++++++++++++++++++ silverback/platform/types.py | 144 ++++++++++++++++++++++++++++ 5 files changed, 446 insertions(+) create mode 100644 silverback/platform/__init__.py create mode 100644 silverback/platform/client.py create mode 100644 silverback/platform/types.py diff --git a/setup.py b/setup.py index 76f40d5f..3a9be5cc 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,8 @@ "packaging", # Use same version as eth-ape "pydantic_settings", # Use same version as eth-ape "taskiq[metrics]>=0.11.3,<0.12", + "tomlkit>=0.12,<1", # For reading/writing global platform profile + "fief-client[cli]>=0.19,<1", # for platform auth/cluster login ], entry_points={ "console_scripts": ["silverback=silverback._cli:cli"], diff --git a/silverback/_cli.py b/silverback/_cli.py index 17781839..c6abf75d 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -11,11 +11,14 @@ verbosity_option, ) from ape.exceptions import Abort +from fief_client.integrations.cli import FiefAuthNotAuthenticatedError from taskiq import AsyncBroker from taskiq.cli.worker.run import shutdown_broker from taskiq.receiver import Receiver from silverback._importer import import_from_string +from silverback.platform.client import DEFAULT_PROFILE, PlatformClient +from silverback.platform.types import ClusterConfiguration, ClusterTier from silverback.runner import PollingRunner, WebsocketRunner @@ -142,3 +145,161 @@ def run(cli_ctx, account, runner_class, recorder, max_exceptions, path): def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): app = import_from_string(path) asyncio.run(run_worker(app.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) + + +def platform_client(display_userinfo: bool = True): + def get_client(ctx, param, value) -> PlatformClient: + client = PlatformClient(profile_name=value) + + # NOTE: We need to be authenticated to display userinfo + if not display_userinfo: + return client + + try: + userinfo = client.userinfo # cache this + except FiefAuthNotAuthenticatedError as e: + raise click.UsageError("Not authenticated, please use `silverback login` first.") from e + + user_id = userinfo["sub"] + username = userinfo["fields"].get("username") + click.echo( + f"{click.style('INFO', fg='blue')}: " + f"Logged in as '{click.style(username if username else user_id, bold=True)}'" + ) + return client + + return click.option( + "-p", + "--profile", + "platform_client", + default=DEFAULT_PROFILE, + callback=get_client, + help="Profile to use for Authentication and Platform API Host.", + ) + + +class PlatformCommands(click.Group): + # NOTE: Override so we get the list ordered by definition order + def list_commands(self, ctx: click.Context) -> list[str]: + return list(self.commands) + + def command(self, *args, display_userinfo=True, **kwargs): + profile_option = platform_client(display_userinfo=display_userinfo) + outer = super().command + + def decorator(fn): + return outer(*args, **kwargs)(profile_option(fn)) + + return decorator + + +@cli.group(cls=PlatformCommands) +def cluster(): + """Connect to hosted application clusters""" + + +@cluster.command(display_userinfo=False) # Otherwise would fail because not authorized +def login(platform_client: PlatformClient): + """Login to hosted clusters""" + platform_client.auth.authorize() + userinfo = platform_client.userinfo # cache this + user_id = userinfo["sub"] + username = userinfo["fields"]["username"] + click.echo( + f"{click.style('SUCCESS', fg='green')}: Logged in as " + f"'{click.style(username, bold=True)}' (UUID: {user_id})" + ) + + +@cluster.command(name="list") +def list_clusters(platform_client: PlatformClient): + """List available clusters""" + if clusters := platform_client.clusters: + for cluster_name, cluster_info in clusters.items(): + click.echo(f"{cluster_name}:") + click.echo(f" status: {cluster_info.status}") + click.echo(" configuration:") + click.echo(f" cpu: {256 * 2 ** cluster_info.configuration.cpu / 1024} vCPU") + memory_display = ( + f"{cluster_info.configuration.memory} GB" + if cluster_info.configuration.memory > 0 + else "512 MiB" + ) + click.echo(f" memory: {memory_display}") + click.echo(f" networks: {cluster_info.configuration.networks}") + click.echo(f" bots: {cluster_info.configuration.bots}") + click.echo(f" triggers: {cluster_info.configuration.triggers}") + + else: + click.secho("No clusters for this account", bold=True, fg="red") + + +@cluster.command(name="new") +@click.option( + "-n", + "--name", + "cluster_name", + default="", + help="Name for new cluster (Defaults to random)", +) +@click.option( + "-t", + "--tier", + default=ClusterTier.PERSONAL.name, +) +@click.option("-c", "--config", "config_updates", type=(str, str), multiple=True) +def new_cluster( + platform_client: PlatformClient, + cluster_name: str, + tier: str, + config_updates: list[tuple[str, str]], +): + """Create a new cluster""" + base_configuration = getattr(ClusterTier, tier.upper()).configuration() + upgrades = ClusterConfiguration( + **{k: int(v) if v.isnumeric() else v for k, v in config_updates} + ) + cluster = platform_client.create_cluster( + cluster_name=cluster_name, + configuration=base_configuration | upgrades, + ) + # TODO: Create a signature scheme for ClusterInfo + # (ClusterInfo configuration as plaintext, .id as nonce?) + # TODO: Test payment w/ Signature validation of extra data + click.echo(f"{click.style('SUCCESS', fg='green')}: Created '{cluster.name}'") + + +@cluster.command() +@click.option( + "-c", + "--cluster", + "cluster_name", + help="Name of cluster to connect with.", + required=True, +) +def bots(platform_client: PlatformClient, cluster_name: str): + """List all bots in a cluster""" + if not (cluster := platform_client.clusters.get(cluster_name)): + if clusters := "', '".join(platform_client.clusters): + message = f"'{cluster_name}' is not a valid cluster, must be one of: '{clusters}'." + + else: + suggestion = ( + "Check out https://silverback.apeworx.io " + "for more information on how to get started" + ) + message = "You have no valid clusters to chose from\n\n" + click.style( + suggestion, bold=True + ) + raise click.BadOptionUsage( + option_name="cluster_name", + message=message, + ) + + if bots := cluster.bots: + click.echo("Available Bots:") + for bot_name, bot_info in bots.items(): + click.echo(f"- {bot_name} (UUID: {bot_info.id})") + + else: + click.secho("No bots in this cluster", bold=True, fg="red") diff --git a/silverback/platform/__init__.py b/silverback/platform/__init__.py new file mode 100644 index 00000000..c54c74d2 --- /dev/null +++ b/silverback/platform/__init__.py @@ -0,0 +1 @@ +# NOTE: Don't import anything here from `.client` diff --git a/silverback/platform/client.py b/silverback/platform/client.py new file mode 100644 index 00000000..dfe5f5fb --- /dev/null +++ b/silverback/platform/client.py @@ -0,0 +1,138 @@ +import os +from functools import cache +from pathlib import Path +from typing import ClassVar + +import httpx +import tomlkit +from fief_client import Fief, FiefAccessTokenInfo, FiefUserInfo +from fief_client.integrations.cli import FiefAuth + +from silverback.platform.types import ClusterConfiguration +from silverback.version import version + +from .types import BotInfo, ClusterInfo + +CREDENTIALS_FOLDER = Path.home() / ".silverback" +CREDENTIALS_FOLDER.mkdir(exist_ok=True) +DEFAULT_PROFILE = "production" + + +class ClusterClient(ClusterInfo): + # NOTE: Client used only for this SDK + platform_client: ClassVar[httpx.Client | None] = None + + def __hash__(self) -> int: + return int(self.id) + + @property + @cache + def client(self) -> httpx.Client: + assert self.platform_client, "Forgot to link platform client" + # NOTE: DI happens in `PlatformClient.client` + return httpx.Client( + base_url=f"{self.platform_client.base_url}/clusters/{self.name}", + cookies=self.platform_client.cookies, + headers=self.platform_client.headers, + ) + + @property + @cache + def openapi_schema(self) -> dict: + return self.client.get("/openapi.json").json() + + @property + def bots(self) -> dict[str, BotInfo]: + # TODO: Actually connect to cluster and display options + return {} + + +class PlatformClient: + def __init__(self, profile_name: str = DEFAULT_PROFILE): + if not (profile_toml := (CREDENTIALS_FOLDER / "profile.toml")).exists(): + if profile_name != DEFAULT_PROFILE: + raise RuntimeError(f"create '{profile_toml}' to add custom profile") + + # Cache this for later + profile_toml.write_text( + tomlkit.dumps( + { + DEFAULT_PROFILE: { + "auth-domain": "https://account.apeworx.io", + "host-url": "https://silverback.apeworx.io", + "client-id": "lcylrp34lnggGO-E-KKlMJgvAI4Q2Jhf6U2G6CB5uMg", + } + } + ) + ) + + if not (profile := tomlkit.loads(profile_toml.read_text()).get(profile_name)): + raise RuntimeError(f"Unknown profile {profile_name}") + + fief = Fief(profile["auth-domain"], profile["client-id"]) + self.auth = FiefAuth(fief, str(CREDENTIALS_FOLDER / f"{profile_name}.json")) + + # NOTE: Use `SILVERBACK_PLATFORM_HOST=http://127.0.0.1:8000` for local testing + self.base_url = os.environ.get("SILVERBACK_PLATFORM_HOST") or profile["host-url"] + + @property + @cache + def client(self) -> httpx.Client: + client = httpx.Client( + base_url=self.base_url, + # NOTE: Raises `FiefAuthNotAuthenticatedError` if access token not available + cookies={"session": self.access_token_info["access_token"]}, + headers={"User-Agent": f"Silverback SDK/{version}"}, + follow_redirects=True, + ) + + # Detect connection fault early + try: + self.openapi = client.get("/openapi.json").json() + except httpx.ConnectError: + raise RuntimeError(f"No Platform API Host detected at '{self.base_url}'.") + except Exception: + raise RuntimeError(f"Error with API Host at '{self.base_url}'.") + + # DI for `ClusterClient` + ClusterClient.platform_client = client # Connect to client + return client + + @property + def userinfo(self) -> FiefUserInfo: + return self.auth.current_user() + + @property + def access_token_info(self) -> FiefAccessTokenInfo: + return self.auth.access_token_info() + + @property + @cache + def clusters(self) -> dict[str, ClusterClient]: + response = self.client.get("/clusters/") + response.raise_for_status() + clusters = response.json() + # TODO: Support paging + return {cluster.name: cluster for cluster in map(ClusterClient.parse_obj, clusters)} + + def create_cluster( + self, + cluster_name: str = "", + configuration: ClusterConfiguration = ClusterConfiguration(), + ) -> ClusterClient: + if ( + response := self.client.post( + "/clusters/", + params=dict(name=cluster_name), + json=configuration.model_dump(), + ) + ).status_code >= 400: + message = response.text + try: + message = response.json().get("detail", response.text) + except Exception: + pass + + raise RuntimeError(message) + + return ClusterClient.parse_raw(response.text) diff --git a/silverback/platform/types.py b/silverback/platform/types.py new file mode 100644 index 00000000..e361ee64 --- /dev/null +++ b/silverback/platform/types.py @@ -0,0 +1,144 @@ +import enum +import math +import uuid +from datetime import datetime +from typing import Annotated + +from pydantic import BaseModel, Field, field_validator + +# NOTE: All configuration settings must be uint8 integer values +UINT8_MAX = 2**8 - 1 + + +class BotInfo(BaseModel): + id: uuid.UUID + name: str + + +class ClusterConfiguration(BaseModel): + """Configuration of the cluster (represented as 16 byte value)""" + + # NOTE: All defaults should be the minimal end of the scale, + # so that `__or__` works right + + # Version byte (Byte 1) + version: int = 1 + + # Bot Worker Configuration (Bytes 2-3) + cpu: Annotated[int, Field(ge=0, le=16)] = 0 # 0.25 vCPU + """Allocated vCPUs per bot: 0.25 vCPU (0) to 16 vCPU (6)""" + + memory: Annotated[int, Field(ge=0, le=120)] = 0 # 512 MiB + """Total memory per bot (in GB)""" + + # Runner configuration (Bytes 4-6) + networks: Annotated[int, Field(ge=1, le=20)] = 1 + """Maximum number of concurrent network runners""" + + bots: Annotated[int, Field(ge=1, le=250)] = 1 + """Maximum number of concurrent bots running""" + + triggers: Annotated[int, Field(ge=5, le=1000, multiple_of=5)] = 30 + """Maximum number of task triggers across all running bots""" + + # TODO: Recorder configuration + # NOTE: Bytes 7-15 empty + + @field_validator("cpu", mode="before") + def parse_cpu_value(cls, value: str | int) -> int: + if not isinstance(value, str): + return value + + return round(math.log2(float(value.split(" ")[0]) * 1024 / 256)) + + @field_validator("memory", mode="before") + def parse_memory_value(cls, value: str | int) -> int: + if not isinstance(value, str): + return value + + mem, units = value.split(" ") + if units.lower() == "mib": + assert mem == "512" + return 0 + + assert units.lower() == "gb" + return int(mem) + + def __or__(self, other: "ClusterConfiguration") -> "ClusterConfiguration": + # NOTE: Helps combine configurations + assert isinstance(other, ClusterConfiguration) + + new = self.copy() + for field in self.model_fields: + setattr(new, field, max(getattr(self, field), getattr(other, field))) + + return new + + @classmethod + def decode(cls, value: int) -> "ClusterConfiguration": + """Decode the configuration from 16 byte integer value""" + # NOTE: Do not change the order of these, these are not forwards compatible + return cls( + version=value & UINT8_MAX, + cpu=(value >> 8) & UINT8_MAX, + memory=(value >> 16) & UINT8_MAX, + networks=(value >> 24) & UINT8_MAX, + bots=(value >> 32) & UINT8_MAX, + triggers=5 * ((value >> 40) & UINT8_MAX), + ) + + def encode(self) -> int: + """Encode configuration as 16 byte integer value""" + # NOTE: Do not change the order of these, these are not forwards compatible + return ( + self.version + + (self.cpu << 8) + + (self.memory << 16) + + (self.networks << 24) + + (self.bots << 32) + + (self.triggers // 5 << 40) + ) + + +class ClusterTier(enum.IntEnum): + """Default configuration for different tier suggestions""" + + PERSONAL = ClusterConfiguration( + cpu="0.25 vCPU", + memory="512 MiB", + networks=3, + bots=5, + triggers=30, + ).encode() + PROFESSIONAL = ClusterConfiguration( + cpu="1 vCPU", + memory="2 GB", + networks=10, + bots=20, + triggers=120, + ).encode() + + def configuration(self) -> ClusterConfiguration: + return ClusterConfiguration.decode(int(self)) + + +class ClusterStatus(enum.Enum): + CREATED = enum.auto() # User record created, but not paid for yet + STANDUP = enum.auto() # Payment received, provisioning infrastructure + RUNNING = enum.auto() # Paid for and fully deployed by payment handler + TEARDOWN = enum.auto() # User triggered shutdown or payment expiration recorded + REMOVED = enum.auto() # Infrastructure de-provisioning complete + + def __str__(self) -> str: + return self.name.capitalize() + + +class ClusterInfo(BaseModel): + # NOTE: Raw API object (gets exported) + id: uuid.UUID # NOTE: Keep this private, used as a temporary secret key for payment + name: str + configuration: ClusterConfiguration + + created: datetime + status: ClusterStatus + last_updated: datetime From ecf49dfd2335450473de30bbe2cc4db5ebf98d59 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 8 Jul 2024 20:29:19 -0400 Subject: [PATCH 02/54] refactor: add workspaces --- silverback/_cli.py | 52 ++++++++++++++--- silverback/platform/client.py | 105 +++++++++++++++++++++++----------- silverback/platform/types.py | 14 ++++- 3 files changed, 127 insertions(+), 44 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index c6abf75d..5865e7c6 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -211,12 +211,39 @@ def login(platform_client: PlatformClient): ) +@cluster.command() +def workspaces(platform_client: PlatformClient): + """List available workspaces""" + + user_id = platform_client.userinfo["sub"] + if workspaces := platform_client.workspaces: + for workspace_slug, workspace_info in workspaces.items(): + click.echo(f"{workspace_slug}:") + click.echo(f" id: {workspace_info.id}") + click.echo(f" name: {workspace_info.name}") + is_owner = str(workspace_info.owner_id) == user_id + click.echo(f" owner: {is_owner}") + + else: + click.secho( + "No workspaces available for this account. " + "Go to https://silverback.apeworx.io to sign up and create a new workspace", + bold=True, + fg="red", + ) + + @cluster.command(name="list") -def list_clusters(platform_client: PlatformClient): +@click.option("-w", "--workspace", "workspace_name") +def list_clusters(platform_client: PlatformClient, workspace_name: str): """List available clusters""" - if clusters := platform_client.clusters: - for cluster_name, cluster_info in clusters.items(): - click.echo(f"{cluster_name}:") + if not (workspace := platform_client.workspaces.get(workspace_name)): + raise click.BadOptionUsage("workspace_name", f"Unknown workspace '{workspace_name}'") + + if clusters := workspace.clusters: + for cluster_slug, cluster_info in clusters.items(): + click.echo(f"{cluster_slug}:") + click.echo(f" name: {cluster_info.name}") click.echo(f" status: {cluster_info.status}") click.echo(" configuration:") click.echo(f" cpu: {256 * 2 ** cluster_info.configuration.cpu / 1024} vCPU") @@ -235,6 +262,7 @@ def list_clusters(platform_client: PlatformClient): @cluster.command(name="new") +@click.option("-w", "--workspace", "workspace_name") @click.option( "-n", "--name", @@ -250,16 +278,20 @@ def list_clusters(platform_client: PlatformClient): @click.option("-c", "--config", "config_updates", type=(str, str), multiple=True) def new_cluster( platform_client: PlatformClient, + workspace_name: str, cluster_name: str, tier: str, config_updates: list[tuple[str, str]], ): """Create a new cluster""" + if not (workspace := platform_client.workspaces.get(workspace_name)): + raise click.BadOptionUsage("workspace_name", f"Unknown workspace '{workspace_name}'") + base_configuration = getattr(ClusterTier, tier.upper()).configuration() upgrades = ClusterConfiguration( **{k: int(v) if v.isnumeric() else v for k, v in config_updates} ) - cluster = platform_client.create_cluster( + cluster = workspace.create_cluster( cluster_name=cluster_name, configuration=base_configuration | upgrades, ) @@ -270,6 +302,7 @@ def new_cluster( @cluster.command() +@click.option("-w", "--workspace", "workspace_name") @click.option( "-c", "--cluster", @@ -277,10 +310,13 @@ def new_cluster( help="Name of cluster to connect with.", required=True, ) -def bots(platform_client: PlatformClient, cluster_name: str): +def bots(platform_client: PlatformClient, workspace_name: str, cluster_name: str): """List all bots in a cluster""" - if not (cluster := platform_client.clusters.get(cluster_name)): - if clusters := "', '".join(platform_client.clusters): + if not (workspace := platform_client.workspaces.get(workspace_name)): + raise click.BadOptionUsage("workspace_name", f"Unknown workspace '{workspace_name}'") + + if not (cluster := workspace.clusters.get(cluster_name)): + if clusters := "', '".join(workspace.clusters): message = f"'{cluster_name}' is not a valid cluster, must be one of: '{clusters}'." else: diff --git a/silverback/platform/client.py b/silverback/platform/client.py index dfe5f5fb..fb66d4bf 100644 --- a/silverback/platform/client.py +++ b/silverback/platform/client.py @@ -11,7 +11,7 @@ from silverback.platform.types import ClusterConfiguration from silverback.version import version -from .types import BotInfo, ClusterInfo +from .types import BotInfo, ClusterInfo, WorkspaceInfo CREDENTIALS_FOLDER = Path.home() / ".silverback" CREDENTIALS_FOLDER.mkdir(exist_ok=True) @@ -19,8 +19,9 @@ class ClusterClient(ClusterInfo): + workspace: WorkspaceInfo # NOTE: Client used only for this SDK - platform_client: ClassVar[httpx.Client | None] = None + _client: ClassVar[httpx.Client] def __hash__(self) -> int: return int(self.id) @@ -28,12 +29,12 @@ def __hash__(self) -> int: @property @cache def client(self) -> httpx.Client: - assert self.platform_client, "Forgot to link platform client" + assert self._client, "Forgot to link platform client" # NOTE: DI happens in `PlatformClient.client` return httpx.Client( - base_url=f"{self.platform_client.base_url}/clusters/{self.name}", - cookies=self.platform_client.cookies, - headers=self.platform_client.headers, + base_url=f"{self._client.base_url}/{self.workspace.slug}/{self.slug}", + cookies=self._client.cookies, + headers=self._client.headers, ) @property @@ -47,6 +48,60 @@ def bots(self) -> dict[str, BotInfo]: return {} +class WorkspaceClient(WorkspaceInfo): + # NOTE: Client used only for this SDK + # NOTE: DI happens in `PlatformClient.client` + client: ClassVar[httpx.Client] + + def __hash__(self) -> int: + return int(self.id) + + def parse_cluster(self, data: dict) -> ClusterClient: + return ClusterClient.model_validate(dict(**data, workspace=self)) + + @property + @cache + def clusters(self) -> dict[str, ClusterClient]: + response = self.client.get("/clusters", params=dict(org=str(self.id))) + response.raise_for_status() + clusters = response.json() + # TODO: Support paging + return {cluster.slug: cluster for cluster in map(self.parse_cluster, clusters)} + + def create_cluster( + self, + cluster_slug: str = "", + cluster_name: str = "", + configuration: ClusterConfiguration = ClusterConfiguration(), + ) -> ClusterClient: + body: dict = dict(configuration=configuration.model_dump()) + + if cluster_slug: + body["slug"] = cluster_slug + + if cluster_name: + body["name"] = cluster_name + + if ( + response := self.client.post( + "/clusters/", + params=dict(org=str(self.id)), + json=body, + ) + ).status_code >= 400: + message = response.text + try: + message = response.json().get("detail", response.text) + except Exception: + pass + + raise RuntimeError(message) + + new_cluster = ClusterClient.model_validate_json(response.text) + self.clusters.update({new_cluster.slug: new_cluster}) # NOTE: Update cache + return new_cluster + + class PlatformClient: def __init__(self, profile_name: str = DEFAULT_PROFILE): if not (profile_toml := (CREDENTIALS_FOLDER / "profile.toml")).exists(): @@ -94,8 +149,9 @@ def client(self) -> httpx.Client: except Exception: raise RuntimeError(f"Error with API Host at '{self.base_url}'.") - # DI for `ClusterClient` - ClusterClient.platform_client = client # Connect to client + # DI for other client classes + WorkspaceClient.client = client # Connect to client + ClusterClient._client = client # Connect to client return client @property @@ -108,31 +164,12 @@ def access_token_info(self) -> FiefAccessTokenInfo: @property @cache - def clusters(self) -> dict[str, ClusterClient]: - response = self.client.get("/clusters/") + def workspaces(self) -> dict[str, WorkspaceClient]: + response = self.client.get("/organizations") response.raise_for_status() - clusters = response.json() + workspaces = response.json() # TODO: Support paging - return {cluster.name: cluster for cluster in map(ClusterClient.parse_obj, clusters)} - - def create_cluster( - self, - cluster_name: str = "", - configuration: ClusterConfiguration = ClusterConfiguration(), - ) -> ClusterClient: - if ( - response := self.client.post( - "/clusters/", - params=dict(name=cluster_name), - json=configuration.model_dump(), - ) - ).status_code >= 400: - message = response.text - try: - message = response.json().get("detail", response.text) - except Exception: - pass - - raise RuntimeError(message) - - return ClusterClient.parse_raw(response.text) + return { + workspace.slug: workspace + for workspace in map(WorkspaceClient.model_validate, workspaces) + } diff --git a/silverback/platform/types.py b/silverback/platform/types.py index e361ee64..2a5d19a8 100644 --- a/silverback/platform/types.py +++ b/silverback/platform/types.py @@ -10,9 +10,11 @@ UINT8_MAX = 2**8 - 1 -class BotInfo(BaseModel): +class WorkspaceInfo(BaseModel): id: uuid.UUID + owner_id: uuid.UUID name: str + slug: str class ClusterConfiguration(BaseModel): @@ -101,7 +103,7 @@ def encode(self) -> int: class ClusterTier(enum.IntEnum): - """Default configuration for different tier suggestions""" + """Suggestions for different tier configurations""" PERSONAL = ClusterConfiguration( cpu="0.25 vCPU", @@ -137,8 +139,16 @@ class ClusterInfo(BaseModel): # NOTE: Raw API object (gets exported) id: uuid.UUID # NOTE: Keep this private, used as a temporary secret key for payment name: str + slug: str configuration: ClusterConfiguration created: datetime status: ClusterStatus last_updated: datetime + + +class BotInfo(BaseModel): + id: uuid.UUID + name: str + + # TODO: More fields From caad74662c59086a936514d7af6e5fde3d653bc6 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 10 Jul 2024 19:53:40 -0400 Subject: [PATCH 03/54] refactor: use different profile types for platform/cluster features --- silverback/_cli.py | 173 ++++++++++-------- silverback/{platform => cluster}/__init__.py | 0 silverback/cluster/client.py | 141 +++++++++++++++ silverback/cluster/settings.py | 73 ++++++++ silverback/{platform => cluster}/types.py | 0 silverback/platform/client.py | 175 ------------------- 6 files changed, 314 insertions(+), 248 deletions(-) rename silverback/{platform => cluster}/__init__.py (100%) create mode 100644 silverback/cluster/client.py create mode 100644 silverback/cluster/settings.py rename silverback/{platform => cluster}/types.py (100%) delete mode 100644 silverback/platform/client.py diff --git a/silverback/_cli.py b/silverback/_cli.py index 5865e7c6..ca1f312d 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -11,14 +11,22 @@ verbosity_option, ) from ape.exceptions import Abort -from fief_client.integrations.cli import FiefAuthNotAuthenticatedError +from fief_client import Fief +from fief_client.integrations.cli import FiefAuth, FiefAuthNotAuthenticatedError from taskiq import AsyncBroker from taskiq.cli.worker.run import shutdown_broker from taskiq.receiver import Receiver from silverback._importer import import_from_string -from silverback.platform.client import DEFAULT_PROFILE, PlatformClient -from silverback.platform.types import ClusterConfiguration, ClusterTier +from silverback.cluster.client import Client, ClusterClient, PlatformClient +from silverback.cluster.settings import ( + DEFAULT_PROFILE, + PROFILE_PATH, + ClusterProfile, + PlatformProfile, + ProfileSettings, +) +from silverback.cluster.types import ClusterConfiguration, ClusterTier from silverback.runner import PollingRunner, WebsocketRunner @@ -147,34 +155,64 @@ def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): asyncio.run(run_worker(app.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) -def platform_client(display_userinfo: bool = True): - def get_client(ctx, param, value) -> PlatformClient: - client = PlatformClient(profile_name=value) +def get_auth(profile_name: str = DEFAULT_PROFILE) -> FiefAuth: + settings = ProfileSettings.from_config_file() + auth_info = settings.auth[profile_name] + fief = Fief(auth_info.host, auth_info.client_id) + return FiefAuth(fief, str(PROFILE_PATH.parent / f"{profile_name}.json")) - # NOTE: We need to be authenticated to display userinfo - if not display_userinfo: - return client - try: - userinfo = client.userinfo # cache this - except FiefAuthNotAuthenticatedError as e: - raise click.UsageError("Not authenticated, please use `silverback login` first.") from e +def display_login_message(auth: FiefAuth, host: str): + userinfo = auth.current_user() - user_id = userinfo["sub"] - username = userinfo["fields"].get("username") - click.echo( - f"{click.style('INFO', fg='blue')}: " - f"Logged in as '{click.style(username if username else user_id, bold=True)}'" - ) - return client + user_id = userinfo["sub"] + username = userinfo["fields"].get("username") + click.echo( + f"{click.style('INFO', fg='blue')}: " + f"Logged in to '{click.style(host, bold=True)}' as " + f"'{click.style(username if username else user_id, bold=True)}'" + ) + + +def client_option(): + settings = ProfileSettings.from_config_file() + + def get_client_from_profile(ctx, param, value) -> Client: + if not (profile := settings.profile.get(value)): + raise click.BadOptionUsage(option_name=param, message=f"Unknown profile '{value}'.") + + if isinstance(profile, PlatformProfile): + auth = get_auth(profile.auth) + + try: + display_login_message(auth, profile.host) + except FiefAuthNotAuthenticatedError as e: + raise click.UsageError( + "Not authenticated, please use `silverback login` first." + ) from e + + return PlatformClient( + base_url=profile.host, + cookies=dict(session=auth.access_token_info()["access_token"]), + ) + + elif isinstance(profile, ClusterProfile): + click.echo( + f"{click.style('INFO', fg='blue')}: Logged in to " + f"'{click.style(profile.host, bold=True)}' using API Key" + ) + return ClusterClient( + base_url=profile.host, + headers={"X-API-Key": profile.api_key}, + ) return click.option( "-p", "--profile", - "platform_client", + "client", default=DEFAULT_PROFILE, - callback=get_client, - help="Profile to use for Authentication and Platform API Host.", + callback=get_client_from_profile, + help="Profile to use for connecting to Cluster Host.", ) @@ -183,46 +221,42 @@ class PlatformCommands(click.Group): def list_commands(self, ctx: click.Context) -> list[str]: return list(self.commands) - def command(self, *args, display_userinfo=True, **kwargs): - profile_option = platform_client(display_userinfo=display_userinfo) - outer = super().command - - def decorator(fn): - return outer(*args, **kwargs)(profile_option(fn)) - - return decorator - @cli.group(cls=PlatformCommands) def cluster(): """Connect to hosted application clusters""" -@cluster.command(display_userinfo=False) # Otherwise would fail because not authorized -def login(platform_client: PlatformClient): +@cluster.command() +@click.option( + "-a", + "--auth-profile", + "auth", + default=DEFAULT_PROFILE, + callback=lambda ctx, param, value: get_auth(value), + help="Authentication profile to use for Platform login.", +) +def login(auth: FiefAuth): """Login to hosted clusters""" - platform_client.auth.authorize() - userinfo = platform_client.userinfo # cache this - user_id = userinfo["sub"] - username = userinfo["fields"]["username"] - click.echo( - f"{click.style('SUCCESS', fg='green')}: Logged in as " - f"'{click.style(username, bold=True)}' (UUID: {user_id})" - ) + + auth.authorize() + display_login_message(auth, auth.client.base_url) @cluster.command() -def workspaces(platform_client: PlatformClient): +@client_option() +def workspaces(client: Client): """List available workspaces""" - user_id = platform_client.userinfo["sub"] - if workspaces := platform_client.workspaces: + if not isinstance(client, PlatformClient): + raise click.UsageError("This feature is not available when directly connected to a cluster") + + if workspaces := client.workspaces: for workspace_slug, workspace_info in workspaces.items(): click.echo(f"{workspace_slug}:") click.echo(f" id: {workspace_info.id}") click.echo(f" name: {workspace_info.name}") - is_owner = str(workspace_info.owner_id) == user_id - click.echo(f" owner: {is_owner}") + click.echo(f" owner: {workspace_info.owner_id}") else: click.secho( @@ -234,10 +268,15 @@ def workspaces(platform_client: PlatformClient): @cluster.command(name="list") +@client_option() @click.option("-w", "--workspace", "workspace_name") -def list_clusters(platform_client: PlatformClient, workspace_name: str): +def list_clusters(client: Client, workspace_name: str): """List available clusters""" - if not (workspace := platform_client.workspaces.get(workspace_name)): + + if not isinstance(client, PlatformClient): + raise click.UsageError("This feature is not available when directly connected to a cluster") + + if not (workspace := client.workspaces.get(workspace_name)): raise click.BadOptionUsage("workspace_name", f"Unknown workspace '{workspace_name}'") if clusters := workspace.clusters: @@ -262,6 +301,7 @@ def list_clusters(platform_client: PlatformClient, workspace_name: str): @cluster.command(name="new") +@client_option() @click.option("-w", "--workspace", "workspace_name") @click.option( "-n", @@ -277,14 +317,18 @@ def list_clusters(platform_client: PlatformClient, workspace_name: str): ) @click.option("-c", "--config", "config_updates", type=(str, str), multiple=True) def new_cluster( - platform_client: PlatformClient, + client: Client, workspace_name: str, cluster_name: str, tier: str, config_updates: list[tuple[str, str]], ): """Create a new cluster""" - if not (workspace := platform_client.workspaces.get(workspace_name)): + + if not isinstance(client, PlatformClient): + raise click.UsageError("This feature is not available when directly connected to a cluster") + + if not (workspace := client.workspaces.get(workspace_name)): raise click.BadOptionUsage("workspace_name", f"Unknown workspace '{workspace_name}'") base_configuration = getattr(ClusterTier, tier.upper()).configuration() @@ -302,37 +346,20 @@ def new_cluster( @cluster.command() +@client_option() @click.option("-w", "--workspace", "workspace_name") @click.option( "-c", "--cluster", "cluster_name", help="Name of cluster to connect with.", - required=True, ) -def bots(platform_client: PlatformClient, workspace_name: str, cluster_name: str): +def bots(client: Client, workspace_name: str, cluster_name: str): """List all bots in a cluster""" - if not (workspace := platform_client.workspaces.get(workspace_name)): - raise click.BadOptionUsage("workspace_name", f"Unknown workspace '{workspace_name}'") - - if not (cluster := workspace.clusters.get(cluster_name)): - if clusters := "', '".join(workspace.clusters): - message = f"'{cluster_name}' is not a valid cluster, must be one of: '{clusters}'." - - else: - suggestion = ( - "Check out https://silverback.apeworx.io " - "for more information on how to get started" - ) - message = "You have no valid clusters to chose from\n\n" + click.style( - suggestion, bold=True - ) - raise click.BadOptionUsage( - option_name="cluster_name", - message=message, - ) + if not isinstance(client, ClusterClient): + client = client.get_cluster_client(workspace_name, cluster_name) - if bots := cluster.bots: + if bots := client.bots: click.echo("Available Bots:") for bot_name, bot_info in bots.items(): click.echo(f"- {bot_name} (UUID: {bot_info.id})") diff --git a/silverback/platform/__init__.py b/silverback/cluster/__init__.py similarity index 100% rename from silverback/platform/__init__.py rename to silverback/cluster/__init__.py diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py new file mode 100644 index 00000000..060cb3d5 --- /dev/null +++ b/silverback/cluster/client.py @@ -0,0 +1,141 @@ +from functools import cache +from typing import ClassVar + +import httpx +from fief_client import Fief, FiefAccessTokenInfo, FiefUserInfo +from fief_client.integrations.cli import FiefAuth + +from silverback.version import version + +from .settings import ( + DEFAULT_PROFILE, + PROFILE_PATH, + AuthenticationConfig, + PlatformProfile, + ProfileSettings, +) +from .types import BotInfo, ClusterConfiguration, ClusterInfo, WorkspaceInfo + +DEFAULT_HEADERS = {"User-Agent": f"Silverback SDK/{version}"} + + +class ClusterClient(httpx.Client): + def __init__(self, *args, **kwargs): + kwargs["headers"] = {**kwargs.get("headers", {}), **DEFAULT_HEADERS} + super().__init__(*args, **kwargs) + + @property + @cache + def openapi_schema(self) -> dict: + return self.get("/openapi.json").json() + + @property + def bots(self) -> dict[str, BotInfo]: + # TODO: Actually connect to cluster and display options + return {} + + +class Workspace(WorkspaceInfo): + # NOTE: Client used only for this SDK + # NOTE: DI happens in `PlatformClient.client` + client: ClassVar[httpx.Client] + + def __hash__(self) -> int: + return int(self.id) + + def get_cluster_client(self, cluster_name: str) -> ClusterClient: + if not (cluster := self.clusters.get(cluster_name)): + raise ValueError(f"Unknown cluster '{cluster_name}' in workspace '{self.name}'.") + + return ClusterClient( + base_url=f"{self.client.base_url}/{self.slug}/{cluster.slug}", + cookies=self.client.cookies, # NOTE: pass along platform cookies for proxy auth + ) + + @property + @cache + def clusters(self) -> dict[str, ClusterInfo]: + response = self.client.get("/clusters", params=dict(org=str(self.id))) + response.raise_for_status() + clusters = response.json() + # TODO: Support paging + return {cluster.slug: cluster for cluster in map(ClusterInfo.model_validate, clusters)} + + def create_cluster( + self, + cluster_slug: str = "", + cluster_name: str = "", + configuration: ClusterConfiguration = ClusterConfiguration(), + ) -> ClusterInfo: + body: dict = dict(configuration=configuration.model_dump()) + + if cluster_slug: + body["slug"] = cluster_slug + + if cluster_name: + body["name"] = cluster_name + + if ( + response := self.client.post( + "/clusters/", + params=dict(org=str(self.id)), + json=body, + ) + ).status_code >= 400: + message = response.text + try: + message = response.json().get("detail", response.text) + except Exception: + pass + + raise RuntimeError(message) + + new_cluster = ClusterInfo.model_validate_json(response.text) + self.clusters.update({new_cluster.slug: new_cluster}) # NOTE: Update cache + return new_cluster + + +class PlatformClient(httpx.Client): + def __init__(self, *args, **kwargs): + if "follow_redirects" not in kwargs: + kwargs["follow_redirects"] = True + + kwargs["headers"] = {**kwargs.get("headers", {}), **DEFAULT_HEADERS} + super().__init__(*args, **kwargs) + + # DI for other client classes + Workspace.client = self # Connect to platform client + + def get_cluster_client(self, workspace_name: str, cluster_name: str) -> ClusterClient: + if not (workspace := self.workspaces.get(workspace_name)): + raise ValueError(f"Unknown workspace '{workspace_name}'.") + + return workspace.get_cluster_client(cluster_name) + + @property + @cache + def workspaces(self) -> dict[str, Workspace]: + response = self.get("/organizations") + response.raise_for_status() + workspaces = response.json() + # TODO: Support paging + return { + workspace.slug: workspace for workspace in map(Workspace.model_validate, workspaces) + } + + def create_workspace( + self, + workspace_slug: str = "", + workspace_name: str = "", + ) -> Workspace: + response = self.post( + "/organizations", + json=dict(slug=workspace_slug, name=workspace_name), + ) + response.raise_for_status() + new_workspace = Workspace.model_validate_json(response.text) + self.workspaces.update({new_workspace.slug: new_workspace}) # NOTE: Update cache + return new_workspace + + +Client = PlatformClient | ClusterClient diff --git a/silverback/cluster/settings.py b/silverback/cluster/settings.py new file mode 100644 index 00000000..aa9f812d --- /dev/null +++ b/silverback/cluster/settings.py @@ -0,0 +1,73 @@ +from pathlib import Path + +import tomlkit +from pydantic import BaseModel, Field, ValidationError, model_validator +from typing_extensions import Self + +PROFILE_PATH = Path.home() / ".silverback" / "profile.toml" +DEFAULT_PROFILE = "default" + + +class AuthenticationConfig(BaseModel): + """Authentication host configuration information (~/.silverback/profile.toml)""" + + host: str = "https://account.apeworx.io" + client_id: str = Field(default="lcylrp34lnggGO-E-KKlMJgvAI4Q2Jhf6U2G6CB5uMg", alias="client-id") + + +class BaseProfile(BaseModel): + """Profile information (~/.silverback/profile.toml)""" + + host: str + + +class ClusterProfile(BaseProfile): + api_key: str = Field(alias="api-key") # direct access to a cluster + + +class PlatformProfile(BaseProfile): + auth: str # key of `AuthenticationConfig` in authentication section + + +class ProfileSettings(BaseModel): + """Configuration settings for working with Bot Clusters and the Silverback Platform""" + + auth: dict[str, AuthenticationConfig] + profile: dict[str, PlatformProfile | ClusterProfile] + + @model_validator(mode="after") + def ensure_auth_exists_for_profile(self) -> Self: + for profile_name, profile in self.profile.items(): + if isinstance(profile, PlatformProfile) and profile.auth not in self.auth: + auth_names = "', '".join(self.auth) + raise ValidationError( + f"Key `profile.'{profile_name}'.auth` must be one of '{auth_names}'." + ) + + return self + + @classmethod + def from_config_file(cls) -> Self: + # TODO: Figure out why `BaseSettings` doesn't work well (probably uses tomlkit) + settings_dict: dict # NOTE: So mypy knows it's not redefined + + if PROFILE_PATH.exists(): + # NOTE: cast to dict because tomlkit has a bug in it that mutates dicts + settings_dict = dict(tomlkit.loads(PROFILE_PATH.read_text())) + + else: # Write the defaults to disk for next time + settings_dict = dict( + auth={ + DEFAULT_PROFILE: AuthenticationConfig(), + }, + profile={ + DEFAULT_PROFILE: PlatformProfile( + auth=DEFAULT_PROFILE, + host="https://silverback.apeworx.io", + ) + }, + ) + PROFILE_PATH.parent.mkdir(exist_ok=True) + PROFILE_PATH.write_text(tomlkit.dumps(settings_dict)) + + return cls.model_validate(settings_dict) diff --git a/silverback/platform/types.py b/silverback/cluster/types.py similarity index 100% rename from silverback/platform/types.py rename to silverback/cluster/types.py diff --git a/silverback/platform/client.py b/silverback/platform/client.py deleted file mode 100644 index fb66d4bf..00000000 --- a/silverback/platform/client.py +++ /dev/null @@ -1,175 +0,0 @@ -import os -from functools import cache -from pathlib import Path -from typing import ClassVar - -import httpx -import tomlkit -from fief_client import Fief, FiefAccessTokenInfo, FiefUserInfo -from fief_client.integrations.cli import FiefAuth - -from silverback.platform.types import ClusterConfiguration -from silverback.version import version - -from .types import BotInfo, ClusterInfo, WorkspaceInfo - -CREDENTIALS_FOLDER = Path.home() / ".silverback" -CREDENTIALS_FOLDER.mkdir(exist_ok=True) -DEFAULT_PROFILE = "production" - - -class ClusterClient(ClusterInfo): - workspace: WorkspaceInfo - # NOTE: Client used only for this SDK - _client: ClassVar[httpx.Client] - - def __hash__(self) -> int: - return int(self.id) - - @property - @cache - def client(self) -> httpx.Client: - assert self._client, "Forgot to link platform client" - # NOTE: DI happens in `PlatformClient.client` - return httpx.Client( - base_url=f"{self._client.base_url}/{self.workspace.slug}/{self.slug}", - cookies=self._client.cookies, - headers=self._client.headers, - ) - - @property - @cache - def openapi_schema(self) -> dict: - return self.client.get("/openapi.json").json() - - @property - def bots(self) -> dict[str, BotInfo]: - # TODO: Actually connect to cluster and display options - return {} - - -class WorkspaceClient(WorkspaceInfo): - # NOTE: Client used only for this SDK - # NOTE: DI happens in `PlatformClient.client` - client: ClassVar[httpx.Client] - - def __hash__(self) -> int: - return int(self.id) - - def parse_cluster(self, data: dict) -> ClusterClient: - return ClusterClient.model_validate(dict(**data, workspace=self)) - - @property - @cache - def clusters(self) -> dict[str, ClusterClient]: - response = self.client.get("/clusters", params=dict(org=str(self.id))) - response.raise_for_status() - clusters = response.json() - # TODO: Support paging - return {cluster.slug: cluster for cluster in map(self.parse_cluster, clusters)} - - def create_cluster( - self, - cluster_slug: str = "", - cluster_name: str = "", - configuration: ClusterConfiguration = ClusterConfiguration(), - ) -> ClusterClient: - body: dict = dict(configuration=configuration.model_dump()) - - if cluster_slug: - body["slug"] = cluster_slug - - if cluster_name: - body["name"] = cluster_name - - if ( - response := self.client.post( - "/clusters/", - params=dict(org=str(self.id)), - json=body, - ) - ).status_code >= 400: - message = response.text - try: - message = response.json().get("detail", response.text) - except Exception: - pass - - raise RuntimeError(message) - - new_cluster = ClusterClient.model_validate_json(response.text) - self.clusters.update({new_cluster.slug: new_cluster}) # NOTE: Update cache - return new_cluster - - -class PlatformClient: - def __init__(self, profile_name: str = DEFAULT_PROFILE): - if not (profile_toml := (CREDENTIALS_FOLDER / "profile.toml")).exists(): - if profile_name != DEFAULT_PROFILE: - raise RuntimeError(f"create '{profile_toml}' to add custom profile") - - # Cache this for later - profile_toml.write_text( - tomlkit.dumps( - { - DEFAULT_PROFILE: { - "auth-domain": "https://account.apeworx.io", - "host-url": "https://silverback.apeworx.io", - "client-id": "lcylrp34lnggGO-E-KKlMJgvAI4Q2Jhf6U2G6CB5uMg", - } - } - ) - ) - - if not (profile := tomlkit.loads(profile_toml.read_text()).get(profile_name)): - raise RuntimeError(f"Unknown profile {profile_name}") - - fief = Fief(profile["auth-domain"], profile["client-id"]) - self.auth = FiefAuth(fief, str(CREDENTIALS_FOLDER / f"{profile_name}.json")) - - # NOTE: Use `SILVERBACK_PLATFORM_HOST=http://127.0.0.1:8000` for local testing - self.base_url = os.environ.get("SILVERBACK_PLATFORM_HOST") or profile["host-url"] - - @property - @cache - def client(self) -> httpx.Client: - client = httpx.Client( - base_url=self.base_url, - # NOTE: Raises `FiefAuthNotAuthenticatedError` if access token not available - cookies={"session": self.access_token_info["access_token"]}, - headers={"User-Agent": f"Silverback SDK/{version}"}, - follow_redirects=True, - ) - - # Detect connection fault early - try: - self.openapi = client.get("/openapi.json").json() - except httpx.ConnectError: - raise RuntimeError(f"No Platform API Host detected at '{self.base_url}'.") - except Exception: - raise RuntimeError(f"Error with API Host at '{self.base_url}'.") - - # DI for other client classes - WorkspaceClient.client = client # Connect to client - ClusterClient._client = client # Connect to client - return client - - @property - def userinfo(self) -> FiefUserInfo: - return self.auth.current_user() - - @property - def access_token_info(self) -> FiefAccessTokenInfo: - return self.auth.access_token_info() - - @property - @cache - def workspaces(self) -> dict[str, WorkspaceClient]: - response = self.client.get("/organizations") - response.raise_for_status() - workspaces = response.json() - # TODO: Support paging - return { - workspace.slug: workspace - for workspace in map(WorkspaceClient.model_validate, workspaces) - } From cb1110dc8b76bd68551db2aa8133a518ee655b4e Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:51:18 -0400 Subject: [PATCH 04/54] refactor: remove unused imports --- silverback/cluster/client.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 060cb3d5..18cd1498 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -2,18 +2,9 @@ from typing import ClassVar import httpx -from fief_client import Fief, FiefAccessTokenInfo, FiefUserInfo -from fief_client.integrations.cli import FiefAuth from silverback.version import version -from .settings import ( - DEFAULT_PROFILE, - PROFILE_PATH, - AuthenticationConfig, - PlatformProfile, - ProfileSettings, -) from .types import BotInfo, ClusterConfiguration, ClusterInfo, WorkspaceInfo DEFAULT_HEADERS = {"User-Agent": f"Silverback SDK/{version}"} From 0ed98fb771be51ea27f68a7b36df381c7ecc0460 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:51:29 -0400 Subject: [PATCH 05/54] fix: add exception for fall-through case --- silverback/_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/silverback/_cli.py b/silverback/_cli.py index ca1f312d..079f51b8 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -206,6 +206,8 @@ def get_client_from_profile(ctx, param, value) -> Client: headers={"X-API-Key": profile.api_key}, ) + raise NotImplementedError # Should not be possible, but mypy barks + return click.option( "-p", "--profile", From 8628bd3a9a851721cca51adfd4c851fb86f09e25 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:51:53 -0400 Subject: [PATCH 06/54] fix: create dict for writing TOML config file out --- silverback/cluster/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/silverback/cluster/settings.py b/silverback/cluster/settings.py index aa9f812d..3e4fa21e 100644 --- a/silverback/cluster/settings.py +++ b/silverback/cluster/settings.py @@ -58,13 +58,13 @@ def from_config_file(cls) -> Self: else: # Write the defaults to disk for next time settings_dict = dict( auth={ - DEFAULT_PROFILE: AuthenticationConfig(), + DEFAULT_PROFILE: AuthenticationConfig().model_dump(), }, profile={ DEFAULT_PROFILE: PlatformProfile( auth=DEFAULT_PROFILE, host="https://silverback.apeworx.io", - ) + ).model_dump() }, ) PROFILE_PATH.parent.mkdir(exist_ok=True) From 79973a64bcf6e206f0c01ad562cbadbbde594dd3 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:25:54 -0400 Subject: [PATCH 07/54] refactor: make sure that all commands are ordered by listing --- silverback/_cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 079f51b8..7e071a58 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -30,7 +30,13 @@ from silverback.runner import PollingRunner, WebsocketRunner -@click.group() +class OrderedCommands(click.Group): + # NOTE: Override so we get the list ordered by definition order + def list_commands(self, ctx: click.Context) -> list[str]: + return list(self.commands) + + +@click.group(cls=OrderedCommands) def cli(): """Work with Silverback applications in local context (using Ape).""" @@ -218,13 +224,7 @@ def get_client_from_profile(ctx, param, value) -> Client: ) -class PlatformCommands(click.Group): - # NOTE: Override so we get the list ordered by definition order - def list_commands(self, ctx: click.Context) -> list[str]: - return list(self.commands) - - -@cli.group(cls=PlatformCommands) +@cli.group(cls=OrderedCommands) def cluster(): """Connect to hosted application clusters""" From 9045babae050e0062f92371fa8d1cd05ec563629 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:35:57 -0400 Subject: [PATCH 08/54] refactor: move `silverback cluster login` to `silverback login` --- silverback/_cli.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 7e071a58..4a976606 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -170,7 +170,6 @@ def get_auth(profile_name: str = DEFAULT_PROFILE) -> FiefAuth: def display_login_message(auth: FiefAuth, host: str): userinfo = auth.current_user() - user_id = userinfo["sub"] username = userinfo["fields"].get("username") click.echo( @@ -180,6 +179,27 @@ def display_login_message(auth: FiefAuth, host: str): ) +@cli.command() +@click.argument( + "auth", + metavar="PROFILE", + default=DEFAULT_PROFILE, + callback=lambda ctx, param, value: get_auth(value), +) +def login(auth: FiefAuth): + """ + CLI Login to Managed Authorization Service + + Initiate a login in to the configured service using the given auth PROFILE. + Defaults to https://account.apeworx.io if PROFILE not provided. + + NOTE: You likely do not need to use an auth PROFILE here. + """ + + auth.authorize() + display_login_message(auth, auth.client.base_url) + + def client_option(): settings = ProfileSettings.from_config_file() @@ -229,22 +249,6 @@ def cluster(): """Connect to hosted application clusters""" -@cluster.command() -@click.option( - "-a", - "--auth-profile", - "auth", - default=DEFAULT_PROFILE, - callback=lambda ctx, param, value: get_auth(value), - help="Authentication profile to use for Platform login.", -) -def login(auth: FiefAuth): - """Login to hosted clusters""" - - auth.authorize() - display_login_message(auth, auth.client.base_url) - - @cluster.command() @client_option() def workspaces(client: Client): From de2f8f0bc63b007ba6e00c5ec8d73213097277f1 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:21:43 -0400 Subject: [PATCH 09/54] refactor(CLI): Use arguments instead of options for platform --- silverback/_cli.py | 66 ++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 4a976606..25527277 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -252,10 +252,10 @@ def cluster(): @cluster.command() @client_option() def workspaces(client: Client): - """List available workspaces""" + """[Platform Only] List available workspaces""" if not isinstance(client, PlatformClient): - raise click.UsageError("This feature is not available when directly connected to a cluster") + raise click.UsageError("This feature is not available outside of the silverback platform") if workspaces := client.workspaces: for workspace_slug, workspace_info in workspaces.items(): @@ -275,17 +275,17 @@ def workspaces(client: Client): @cluster.command(name="list") @client_option() -@click.option("-w", "--workspace", "workspace_name") -def list_clusters(client: Client, workspace_name: str): - """List available clusters""" +@click.argument("workspace") +def list_clusters(client: Client, workspace: str): + """[Platform Only] List available clusters in WORKSPACE""" - if not isinstance(client, PlatformClient): + if isinstance(client, ClusterClient): raise click.UsageError("This feature is not available when directly connected to a cluster") - if not (workspace := client.workspaces.get(workspace_name)): - raise click.BadOptionUsage("workspace_name", f"Unknown workspace '{workspace_name}'") + if not (workspace_client := client.workspaces.get(workspace)): + raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") - if clusters := workspace.clusters: + if clusters := workspace_client.clusters: for cluster_slug, cluster_info in clusters.items(): click.echo(f"{cluster_slug}:") click.echo(f" name: {cluster_info.name}") @@ -308,7 +308,6 @@ def list_clusters(client: Client, workspace_name: str): @cluster.command(name="new") @client_option() -@click.option("-w", "--workspace", "workspace_name") @click.option( "-n", "--name", @@ -322,48 +321,57 @@ def list_clusters(client: Client, workspace_name: str): default=ClusterTier.PERSONAL.name, ) @click.option("-c", "--config", "config_updates", type=(str, str), multiple=True) +@click.argument("workspace") def new_cluster( client: Client, - workspace_name: str, + workspace: str, cluster_name: str, tier: str, config_updates: list[tuple[str, str]], ): - """Create a new cluster""" + """[Platform Only] Create a new cluster in WORKSPACE""" - if not isinstance(client, PlatformClient): + if isinstance(client, ClusterClient): raise click.UsageError("This feature is not available when directly connected to a cluster") - if not (workspace := client.workspaces.get(workspace_name)): - raise click.BadOptionUsage("workspace_name", f"Unknown workspace '{workspace_name}'") + if not (workspace_client := client.workspaces.get(workspace)): + raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") base_configuration = getattr(ClusterTier, tier.upper()).configuration() upgrades = ClusterConfiguration( **{k: int(v) if v.isnumeric() else v for k, v in config_updates} ) - cluster = workspace.create_cluster( + cluster = workspace_client.create_cluster( cluster_name=cluster_name, configuration=base_configuration | upgrades, ) - # TODO: Create a signature scheme for ClusterInfo - # (ClusterInfo configuration as plaintext, .id as nonce?) - # TODO: Test payment w/ Signature validation of extra data click.echo(f"{click.style('SUCCESS', fg='green')}: Created '{cluster.name}'") +# `silverback cluster pay WORKSPACE/CLUSTER_NAME --account ALIAS --time "10 days"` +# TODO: Create a signature scheme for ClusterInfo +# (ClusterInfo configuration as plaintext, .id as nonce?) +# TODO: Test payment w/ Signature validation of extra data + + @cluster.command() @client_option() -@click.option("-w", "--workspace", "workspace_name") -@click.option( - "-c", - "--cluster", - "cluster_name", - help="Name of cluster to connect with.", -) -def bots(client: Client, workspace_name: str, cluster_name: str): - """List all bots in a cluster""" +@click.argument("cluster", default=None) +def bots(client: Client, cluster: str): + """ + List all bots in a CLUSTER + + For clusters on the Silverback Platform, please provide a name for the cluster to access using + your platform authentication obtained via `silverback login` in `workspace/cluster-name` format + + NOTE: Connecting directly to clusters is supported, but is an advanced use case. + """ if not isinstance(client, ClusterClient): - client = client.get_cluster_client(workspace_name, cluster_name) + workspace_name, cluster_name = cluster.split("/") + try: + client = client.get_cluster_client(workspace_name, cluster_name) + except ValueError as e: + raise click.UsageError(str(e)) if bots := client.bots: click.echo("Available Bots:") From c65f63217b6931c12d3d8a86774ffbdb1254e0cf Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:22:28 -0400 Subject: [PATCH 10/54] refactor: clearer error handling for cluster client --- silverback/cluster/client.py | 60 ++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 18cd1498..688470f7 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -10,16 +10,43 @@ DEFAULT_HEADERS = {"User-Agent": f"Silverback SDK/{version}"} +def handle_error_with_response(response: httpx.Response): + if 400 <= response.status_code < 500: + message = response.text + try: + message = response.json().get("detail", response.text) + except Exception: + pass + + raise RuntimeError(message) + + response.raise_for_status() + + assert response.status_code < 300, "Should follow redirects, so not sure what the issue is" + + class ClusterClient(httpx.Client): def __init__(self, *args, **kwargs): kwargs["headers"] = {**kwargs.get("headers", {}), **DEFAULT_HEADERS} super().__init__(*args, **kwargs) + def send(self, request, *args, **kwargs): + try: + return super().send(request, *args, **kwargs) + + except httpx.ConnectError as e: + raise ValueError(f"{e} '{request.url}'") from e + @property @cache def openapi_schema(self) -> dict: return self.get("/openapi.json").json() + @property + def status(self) -> str: + # NOTE: Just return full response directly to avoid errors + return self.get("/").text + @property def bots(self) -> dict[str, BotInfo]: # TODO: Actually connect to cluster and display options @@ -47,7 +74,7 @@ def get_cluster_client(self, cluster_name: str) -> ClusterClient: @cache def clusters(self) -> dict[str, ClusterInfo]: response = self.client.get("/clusters", params=dict(org=str(self.id))) - response.raise_for_status() + handle_error_with_response(response) clusters = response.json() # TODO: Support paging return {cluster.slug: cluster for cluster in map(ClusterInfo.model_validate, clusters)} @@ -66,21 +93,13 @@ def create_cluster( if cluster_name: body["name"] = cluster_name - if ( - response := self.client.post( - "/clusters/", - params=dict(org=str(self.id)), - json=body, - ) - ).status_code >= 400: - message = response.text - try: - message = response.json().get("detail", response.text) - except Exception: - pass - - raise RuntimeError(message) + response = self.client.post( + "/clusters/", + params=dict(org=str(self.id)), + json=body, + ) + handle_error_with_response(response) new_cluster = ClusterInfo.model_validate_json(response.text) self.clusters.update({new_cluster.slug: new_cluster}) # NOTE: Update cache return new_cluster @@ -97,6 +116,13 @@ def __init__(self, *args, **kwargs): # DI for other client classes Workspace.client = self # Connect to platform client + def send(self, request, *args, **kwargs): + try: + return super().send(request, *args, **kwargs) + + except httpx.ConnectError as e: + raise ValueError(f"{e} '{request.url}'") from e + def get_cluster_client(self, workspace_name: str, cluster_name: str) -> ClusterClient: if not (workspace := self.workspaces.get(workspace_name)): raise ValueError(f"Unknown workspace '{workspace_name}'.") @@ -107,7 +133,7 @@ def get_cluster_client(self, workspace_name: str, cluster_name: str) -> ClusterC @cache def workspaces(self) -> dict[str, Workspace]: response = self.get("/organizations") - response.raise_for_status() + handle_error_with_response(response) workspaces = response.json() # TODO: Support paging return { @@ -123,7 +149,7 @@ def create_workspace( "/organizations", json=dict(slug=workspace_slug, name=workspace_name), ) - response.raise_for_status() + handle_error_with_response(response) new_workspace = Workspace.model_validate_json(response.text) self.workspaces.update({new_workspace.slug: new_workspace}) # NOTE: Update cache return new_workspace From 7886328a553acf23e8ab8ffda53c6d7a232b72eb Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:22:47 -0400 Subject: [PATCH 11/54] feat(CLI): add cluster status command --- silverback/_cli.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/silverback/_cli.py b/silverback/_cli.py index 25527277..cba72d70 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -354,6 +354,28 @@ def new_cluster( # TODO: Test payment w/ Signature validation of extra data +@cluster.command() +@client_option() +@click.argument("cluster", default=None, required=False) +def status(client: Client, cluster: str): + """ + Get Status information about a CLUSTER + + For clusters on the Silverback Platform, please provide a name for the cluster to access using + your platform authentication obtained via `silverback login` in `workspace/cluster-name` format + + NOTE: Connecting directly to clusters is supported, but is an advanced use case. + """ + if not isinstance(client, ClusterClient): + workspace_name, cluster_name = cluster.split("/") + try: + client = client.get_cluster_client(workspace_name, cluster_name) + except ValueError as e: + raise click.UsageError(str(e)) + + click.echo(client.status) + + @cluster.command() @client_option() @click.argument("cluster", default=None) From 833f032329ffdbc5fe9b8b7bed6ff5069ba06b31 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 12 Jul 2024 18:51:32 -0400 Subject: [PATCH 12/54] fix: refine cli errors a bit for cluster optional args --- silverback/_cli.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index cba72d70..b3f63d16 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -367,6 +367,12 @@ def status(client: Client, cluster: str): NOTE: Connecting directly to clusters is supported, but is an advanced use case. """ if not isinstance(client, ClusterClient): + if cluster is None: + raise click.UsageError("CLUSTER is required for a platform-managed cluster") + + elif "/" not in cluster or len(cluster.split("/")) > 2: + raise click.UsageError("CLUSTER should be in format `WORKSPACE-NAME/CLUSTER-NAME`") + workspace_name, cluster_name = cluster.split("/") try: client = client.get_cluster_client(workspace_name, cluster_name) @@ -378,7 +384,7 @@ def status(client: Client, cluster: str): @cluster.command() @client_option() -@click.argument("cluster", default=None) +@click.argument("cluster", default=None, required=False) def bots(client: Client, cluster: str): """ List all bots in a CLUSTER @@ -389,6 +395,12 @@ def bots(client: Client, cluster: str): NOTE: Connecting directly to clusters is supported, but is an advanced use case. """ if not isinstance(client, ClusterClient): + if cluster is None: + raise click.UsageError("CLUSTER is required for a platform-managed cluster") + + elif "/" not in cluster or len(cluster.split("/")) > 2: + raise click.UsageError("CLUSTER should be in format `WORKSPACE-NAME/CLUSTER-NAME`") + workspace_name, cluster_name = cluster.split("/") try: client = client.get_cluster_client(workspace_name, cluster_name) From 703e2fcff85a73b5c4875e13349224a0a1dbd68a Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:33:15 -0400 Subject: [PATCH 13/54] fix: something weird when initializing this value w/ SQLModel --- silverback/cluster/types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index 2a5d19a8..05472589 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -79,6 +79,9 @@ def __or__(self, other: "ClusterConfiguration") -> "ClusterConfiguration": @classmethod def decode(cls, value: int) -> "ClusterConfiguration": """Decode the configuration from 16 byte integer value""" + if isinstance(value, ClusterConfiguration): + return value # TODO: Something weird with SQLModel + # NOTE: Do not change the order of these, these are not forwards compatible return cls( version=value & UINT8_MAX, From 63934fdcba4c8a8ccfe8577e1b3dce861b575707 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:34:19 -0400 Subject: [PATCH 14/54] refactor(cli): use better option defaults, show errors better --- silverback/_cli.py | 55 +++++++++++++++++++++++++++--------- silverback/cluster/client.py | 45 ++++++++++++++++++++--------- silverback/cluster/types.py | 10 ------- 3 files changed, 73 insertions(+), 37 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index b3f63d16..cff13134 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -312,20 +312,34 @@ def list_clusters(client: Client, workspace: str): "-n", "--name", "cluster_name", - default="", help="Name for new cluster (Defaults to random)", ) +@click.option( + "-s", + "--slug", + "cluster_slug", + help="Slug for new cluster (Defaults to name.lower())", +) @click.option( "-t", "--tier", default=ClusterTier.PERSONAL.name, + help="Named set of options to use for cluster (Defaults to PERSONAL)", +) +@click.option( + "-c", + "--config", + "config_updates", + type=(str, str), + multiple=True, + help="Config options to set for cluster (overrides value of -t/--tier)", ) -@click.option("-c", "--config", "config_updates", type=(str, str), multiple=True) @click.argument("workspace") def new_cluster( client: Client, workspace: str, - cluster_name: str, + cluster_name: str | None, + cluster_slug: str | None, tier: str, config_updates: list[tuple[str, str]], ): @@ -337,15 +351,20 @@ def new_cluster( if not (workspace_client := client.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") - base_configuration = getattr(ClusterTier, tier.upper()).configuration() - upgrades = ClusterConfiguration( - **{k: int(v) if v.isnumeric() else v for k, v in config_updates} - ) - cluster = workspace_client.create_cluster( - cluster_name=cluster_name, - configuration=base_configuration | upgrades, - ) - click.echo(f"{click.style('SUCCESS', fg='green')}: Created '{cluster.name}'") + configuration = getattr(ClusterTier, tier.upper()).configuration() + + for k, v in config_updates: + setattr(configuration, k, int(v) if v.isnumeric() else v) + + try: + cluster = workspace_client.create_cluster( + cluster_name=cluster_name, + cluster_slug=cluster_slug, + configuration=configuration, + ) + click.echo(f"{click.style('SUCCESS', fg='green')}: Created '{cluster.name}'") + except RuntimeError as e: + raise click.UsageError(str(e)) # `silverback cluster pay WORKSPACE/CLUSTER_NAME --account ALIAS --time "10 days"` @@ -379,7 +398,10 @@ def status(client: Client, cluster: str): except ValueError as e: raise click.UsageError(str(e)) - click.echo(client.status) + try: + click.echo(client.status) + except RuntimeError as e: + raise click.UsageError(str(e)) @cluster.command() @@ -407,7 +429,12 @@ def bots(client: Client, cluster: str): except ValueError as e: raise click.UsageError(str(e)) - if bots := client.bots: + try: + bots = client.bots + except RuntimeError as e: + raise click.UsageError(str(e)) + + if bots: click.echo("Available Bots:") for bot_name, bot_info in bots.items(): click.echo(f"- {bot_name} (UUID: {bot_info.id})") diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 688470f7..1eadabc9 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -13,11 +13,30 @@ def handle_error_with_response(response: httpx.Response): if 400 <= response.status_code < 500: message = response.text + try: - message = response.json().get("detail", response.text) + message = response.json() except Exception: pass + if isinstance(message, dict): + if detail := message.get("detail"): + if isinstance(detail, list): + + def render_error(error: dict): + location = ".".join(error["loc"]) + return f"- {location}: '{error['msg']}'" + + message = "Multiple validation errors found:\n" + "\n".join( + map(render_error, detail) + ) + + else: + message = detail + + else: + message = response.text + raise RuntimeError(message) response.raise_for_status() @@ -44,11 +63,15 @@ def openapi_schema(self) -> dict: @property def status(self) -> str: + response = self.get("/") + handle_error_with_response(response) # NOTE: Just return full response directly to avoid errors - return self.get("/").text + return response.text @property def bots(self) -> dict[str, BotInfo]: + response = self.get("/bots") + handle_error_with_response(response) # TODO: Actually connect to cluster and display options return {} @@ -81,22 +104,18 @@ def clusters(self) -> dict[str, ClusterInfo]: def create_cluster( self, - cluster_slug: str = "", - cluster_name: str = "", + cluster_slug: str | None = None, + cluster_name: str | None = None, configuration: ClusterConfiguration = ClusterConfiguration(), ) -> ClusterInfo: - body: dict = dict(configuration=configuration.model_dump()) - - if cluster_slug: - body["slug"] = cluster_slug - - if cluster_name: - body["name"] = cluster_name - response = self.client.post( "/clusters/", params=dict(org=str(self.id)), - json=body, + json=dict( + name=cluster_name, + slug=cluster_slug, + configuration=configuration.model_dump(), + ), ) handle_error_with_response(response) diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index 05472589..e5ef35a2 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -66,16 +66,6 @@ def parse_memory_value(cls, value: str | int) -> int: assert units.lower() == "gb" return int(mem) - def __or__(self, other: "ClusterConfiguration") -> "ClusterConfiguration": - # NOTE: Helps combine configurations - assert isinstance(other, ClusterConfiguration) - - new = self.copy() - for field in self.model_fields: - setattr(new, field, max(getattr(self, field), getattr(other, field))) - - return new - @classmethod def decode(cls, value: int) -> "ClusterConfiguration": """Decode the configuration from 16 byte integer value""" From d0217d15f86efb29b5910415ba15080f53e9447c Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:42:29 -0400 Subject: [PATCH 15/54] refactor(platform): use IntEnum for ClusterStatus; select values --- silverback/cluster/types.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index e5ef35a2..41ed505f 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -117,12 +117,13 @@ def configuration(self) -> ClusterConfiguration: return ClusterConfiguration.decode(int(self)) -class ClusterStatus(enum.Enum): - CREATED = enum.auto() # User record created, but not paid for yet - STANDUP = enum.auto() # Payment received, provisioning infrastructure - RUNNING = enum.auto() # Paid for and fully deployed by payment handler - TEARDOWN = enum.auto() # User triggered shutdown or payment expiration recorded - REMOVED = enum.auto() # Infrastructure de-provisioning complete +class ClusterStatus(enum.IntEnum): + # NOTE: Selected integer values with some space for other steps + CREATED = 0 # User record created, but not paid for yet + STANDUP = 3 # Payment received, provisioning infrastructure + RUNNING = 5 # Paid for and fully deployed by payment handler + TEARDOWN = 6 # User triggered shutdown or payment expiration recorded + REMOVED = 9 # Infrastructure de-provisioning complete def __str__(self) -> str: return self.name.capitalize() From 46bb083e400ba25fa79e8665735c3d3fc9f359ef Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:04:46 -0400 Subject: [PATCH 16/54] refactor: use yaml helper --- silverback/_cli.py | 52 +++++++----------------------------- silverback/cluster/client.py | 16 +++++++++++ silverback/cluster/types.py | 42 ++++++++++++++++++++++++++++- 3 files changed, 67 insertions(+), 43 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index cff13134..566b6f1c 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -26,7 +26,7 @@ PlatformProfile, ProfileSettings, ) -from silverback.cluster.types import ClusterConfiguration, ClusterTier +from silverback.cluster.types import ClusterConfiguration, ClusterTier, render_dict_as_yaml from silverback.runner import PollingRunner, WebsocketRunner @@ -257,12 +257,8 @@ def workspaces(client: Client): if not isinstance(client, PlatformClient): raise click.UsageError("This feature is not available outside of the silverback platform") - if workspaces := client.workspaces: - for workspace_slug, workspace_info in workspaces.items(): - click.echo(f"{workspace_slug}:") - click.echo(f" id: {workspace_info.id}") - click.echo(f" name: {workspace_info.name}") - click.echo(f" owner: {workspace_info.owner_id}") + if workspace_display := render_dict_as_yaml(client.workspaces): + click.echo(workspace_display) else: click.secho( @@ -285,22 +281,8 @@ def list_clusters(client: Client, workspace: str): if not (workspace_client := client.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") - if clusters := workspace_client.clusters: - for cluster_slug, cluster_info in clusters.items(): - click.echo(f"{cluster_slug}:") - click.echo(f" name: {cluster_info.name}") - click.echo(f" status: {cluster_info.status}") - click.echo(" configuration:") - click.echo(f" cpu: {256 * 2 ** cluster_info.configuration.cpu / 1024} vCPU") - memory_display = ( - f"{cluster_info.configuration.memory} GB" - if cluster_info.configuration.memory > 0 - else "512 MiB" - ) - click.echo(f" memory: {memory_display}") - click.echo(f" networks: {cluster_info.configuration.networks}") - click.echo(f" bots: {cluster_info.configuration.bots}") - click.echo(f" triggers: {cluster_info.configuration.triggers}") + if cluster_display := render_dict_as_yaml(workspace_client.clusters): + click.echo(cluster_display) else: click.secho("No clusters for this account", bold=True, fg="red") @@ -392,16 +374,8 @@ def status(client: Client, cluster: str): elif "/" not in cluster or len(cluster.split("/")) > 2: raise click.UsageError("CLUSTER should be in format `WORKSPACE-NAME/CLUSTER-NAME`") - workspace_name, cluster_name = cluster.split("/") - try: - client = client.get_cluster_client(workspace_name, cluster_name) - except ValueError as e: - raise click.UsageError(str(e)) + click.echo(render_dict_as_yaml(client.build_display_fields())) - try: - click.echo(client.status) - except RuntimeError as e: - raise click.UsageError(str(e)) @cluster.command() @@ -429,15 +403,9 @@ def bots(client: Client, cluster: str): except ValueError as e: raise click.UsageError(str(e)) - try: - bots = client.bots - except RuntimeError as e: - raise click.UsageError(str(e)) - - if bots: - click.echo("Available Bots:") - for bot_name, bot_info in bots.items(): - click.echo(f"- {bot_name} (UUID: {bot_info.id})") - + if bot_display := render_dict_as_yaml(client.bots): + click.echo(bot_display) else: click.secho("No bots in this cluster", bold=True, fg="red") + + diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 1eadabc9..6f6fd668 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -81,6 +81,22 @@ class Workspace(WorkspaceInfo): # NOTE: DI happens in `PlatformClient.client` client: ClassVar[httpx.Client] + @property + @cache + def owner(self) -> str: + response = self.client.get(f"/users/{self.owner_id}") + handle_error_with_response(response) + return response.json().get("username") + + def build_display_fields(self) -> dict[str, str]: + return dict( + # `.id` is internal + name=self.name, + # `.slug` is index + # `.owner_id` is UUID, use for client lookup instead + owner=self.owner, + ) + def __hash__(self) -> int: return int(self.id) diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index 41ed505f..ee8e948e 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -2,13 +2,32 @@ import math import uuid from datetime import datetime -from typing import Annotated +from typing import Annotated, Any from pydantic import BaseModel, Field, field_validator # NOTE: All configuration settings must be uint8 integer values UINT8_MAX = 2**8 - 1 +TIME_FORMAT_STRING = "{0:%x} {0:%X}" + + +def render_dict_as_yaml(value: Any, prepend: str = "\n") -> str: + if hasattr(value, "build_display_fields"): + return render_dict_as_yaml(value.build_display_fields(), prepend=prepend) + + elif not isinstance(value, dict): + raise ValueError(f"'{type(value)}' is not renderable.") + + return prepend.join( + ( + f"{key}: {value}" + if isinstance(value, str) + else f"{key}:{prepend + ' '}{render_dict_as_yaml(value, prepend=(prepend + ' '))}" + ) + for key, value in value.items() + ) + class WorkspaceInfo(BaseModel): id: uuid.UUID @@ -94,6 +113,15 @@ def encode(self) -> int: + (self.triggers // 5 << 40) ) + def build_display_fields(self) -> dict[str, str]: + return dict( + cpu=f"{256 * 2 ** self.cpu / 1024} vCPU", + memory=(f"{self.memory} GB" if self.memory > 0 else "512 MiB"), + networks=str(self.networks), + bots=str(self.bots), + triggers=str(self.triggers), + ) + class ClusterTier(enum.IntEnum): """Suggestions for different tier configurations""" @@ -140,6 +168,18 @@ class ClusterInfo(BaseModel): status: ClusterStatus last_updated: datetime + def build_display_fields(self) -> dict[str, str | dict[str, str]]: + return dict( + # No `.id`, not visible to client user + name=self.name, + # No `.slug`, primary identifier used in dict + # NOTE: Convert local time + created=TIME_FORMAT_STRING.format(self.created.astimezone()), + last_updated=TIME_FORMAT_STRING.format(self.last_updated.astimezone()), + status=str(self.status), + configuration=self.configuration.build_display_fields(), + ) + class BotInfo(BaseModel): id: uuid.UUID From 4e9884f029268817dd6df19e8b773ca0a7d46009 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:18:20 -0400 Subject: [PATCH 17/54] refactor(cli): re-organize auth so that it we can just use a class --- silverback/_cli.py | 237 +++++++++++++++++++++++++-------------------- 1 file changed, 130 insertions(+), 107 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 566b6f1c..e2645ef4 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -18,15 +18,15 @@ from taskiq.receiver import Receiver from silverback._importer import import_from_string -from silverback.cluster.client import Client, ClusterClient, PlatformClient +from silverback.cluster.client import ClusterClient, PlatformClient from silverback.cluster.settings import ( DEFAULT_PROFILE, PROFILE_PATH, - ClusterProfile, + BaseProfile, PlatformProfile, ProfileSettings, ) -from silverback.cluster.types import ClusterConfiguration, ClusterTier, render_dict_as_yaml +from silverback.cluster.types import ClusterTier, render_dict_as_yaml from silverback.runner import PollingRunner, WebsocketRunner @@ -161,13 +161,6 @@ def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): asyncio.run(run_worker(app.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) -def get_auth(profile_name: str = DEFAULT_PROFILE) -> FiefAuth: - settings = ProfileSettings.from_config_file() - auth_info = settings.auth[profile_name] - fief = Fief(auth_info.host, auth_info.client_id) - return FiefAuth(fief, str(PROFILE_PATH.parent / f"{profile_name}.json")) - - def display_login_message(auth: FiefAuth, host: str): userinfo = auth.current_user() user_id = userinfo["sub"] @@ -179,69 +172,135 @@ def display_login_message(auth: FiefAuth, host: str): ) -@cli.command() -@click.argument( - "auth", - metavar="PROFILE", - default=DEFAULT_PROFILE, - callback=lambda ctx, param, value: get_auth(value), -) -def login(auth: FiefAuth): - """ - CLI Login to Managed Authorization Service +class PlatformCommand(click.Command): + # NOTE: ClassVar assures only loaded once + settings = ProfileSettings.from_config_file() - Initiate a login in to the configured service using the given auth PROFILE. - Defaults to https://account.apeworx.io if PROFILE not provided. + # NOTE: Cache this class-wide + platform_client: PlatformClient | None = None - NOTE: You likely do not need to use an auth PROFILE here. - """ + def get_params(self, ctx: click.Context): + params = super().get_params(ctx) - auth.authorize() - display_login_message(auth, auth.client.base_url) + def get_profile(ctx, param, value) -> BaseProfile: + if not (profile := self.settings.profile.get(value)): + raise click.BadOptionUsage(option_name=param, message=f"Unknown profile '{value}'.") -def client_option(): - settings = ProfileSettings.from_config_file() + return profile - def get_client_from_profile(ctx, param, value) -> Client: - if not (profile := settings.profile.get(value)): - raise click.BadOptionUsage(option_name=param, message=f"Unknown profile '{value}'.") + params.append( + click.Option( + param_decls=("-p", "--profile", "profile"), + metavar="PROFILE", + default=DEFAULT_PROFILE, + callback=get_profile, + ) + ) - if isinstance(profile, PlatformProfile): - auth = get_auth(profile.auth) + params.append( + click.Argument( + param_decls=("cluster",), + metavar="WORKSPACE/CLUSTER", + required=False, + default=None, + ), + ) - try: - display_login_message(auth, profile.host) - except FiefAuthNotAuthenticatedError as e: - raise click.UsageError( - "Not authenticated, please use `silverback login` first." - ) from e + return params - return PlatformClient( - base_url=profile.host, - cookies=dict(session=auth.access_token_info()["access_token"]), + def get_auth(self, profile: BaseProfile) -> FiefAuth: + if not isinstance(profile, PlatformProfile): + raise click.UsageError( + "This feature is not available outside of the Silverback Platform" ) - elif isinstance(profile, ClusterProfile): - click.echo( - f"{click.style('INFO', fg='blue')}: Logged in to " - f"'{click.style(profile.host, bold=True)}' using API Key" - ) - return ClusterClient( - base_url=profile.host, - headers={"X-API-Key": profile.api_key}, - ) + auth_info = self.settings.auth[profile.auth] + fief = Fief(auth_info.host, auth_info.client_id) + return FiefAuth(fief, str(PROFILE_PATH.parent / f"{profile.auth}.json")) - raise NotImplementedError # Should not be possible, but mypy barks + def get_platform_client(self, auth: FiefAuth, profile: PlatformProfile) -> PlatformClient: + try: + display_login_message(auth, profile.host) + except FiefAuthNotAuthenticatedError as e: + raise click.UsageError("Not authenticated, please use `silverback login` first.") from e - return click.option( - "-p", - "--profile", - "client", - default=DEFAULT_PROFILE, - callback=get_client_from_profile, - help="Profile to use for connecting to Cluster Host.", - ) + return PlatformClient( + base_url=profile.host, + cookies=dict(session=auth.access_token_info()["access_token"]), + ) + + def get_cluster_client(self, cluster_path): + assert self.platform_client, "Something parsing out of order" + + if "/" not in cluster_path or len(cluster_path.split("/")) > 2: + raise click.BadArgumentUsage("CLUSTER should be in format `WORKSPACE/CLUSTER-NAME`") + + workspace_name, cluster_name = cluster_path.split("/") + try: + return self.platform_client.get_cluster_client(workspace_name, cluster_name) + except ValueError as e: + raise click.UsageError(str(e)) + + def invoke(self, ctx: click.Context): + callback_params = self.callback.__annotations__ if self.callback else {} + + cluster_path = ctx.params.pop("cluster") + + if "profile" not in callback_params: + profile = ctx.params.pop("profile") + + else: + profile = ctx.params["profile"] + + if "auth" in callback_params: + ctx.params["auth"] = self.get_auth(profile) + + if "client" in callback_params: + client_type_needed = callback_params.get("client") + + if isinstance(profile, PlatformProfile): + self.platform_client = self.get_platform_client( + ctx.params.get("auth", self.get_auth(profile)), profile + ) + + if client_type_needed == PlatformClient: + ctx.params["client"] = self.platform_client + + else: + ctx.params["client"] = self.get_cluster_client(cluster_path) + + elif not client_type_needed == ClusterClient: + raise click.UsageError("A cluster profile can only directly connect to a cluster.") + + else: + click.echo( + f"{click.style('INFO', fg='blue')}: Logged in to " + f"'{click.style(profile.host, bold=True)}' using API Key" + ) + ctx.params["client"] = ClusterClient( + base_url=profile.host, + headers={"X-API-Key": profile.api_key}, + ) + + assert ctx.params["client"], "Something went wrong" + + super().invoke(ctx) + + +@cli.command(cls=PlatformCommand) +def login(auth: FiefAuth): + """ + CLI Login to Managed Authorization Service + + Initiate a login in to the configured service using the given auth PROFILE. + Defaults to https://account.apeworx.io if PROFILE not provided. + + NOTE: You likely do not need to use an auth PROFILE here. + """ + + auth.authorize() + display_login_message(auth, auth.client.base_url) @cli.group(cls=OrderedCommands) @@ -249,14 +308,10 @@ def cluster(): """Connect to hosted application clusters""" -@cluster.command() -@client_option() -def workspaces(client: Client): +@cluster.command(cls=PlatformCommand) +def workspaces(client: PlatformClient): """[Platform Only] List available workspaces""" - if not isinstance(client, PlatformClient): - raise click.UsageError("This feature is not available outside of the silverback platform") - if workspace_display := render_dict_as_yaml(client.workspaces): click.echo(workspace_display) @@ -269,15 +324,11 @@ def workspaces(client: Client): ) -@cluster.command(name="list") -@client_option() +@cluster.command(name="list", cls=PlatformCommand) @click.argument("workspace") -def list_clusters(client: Client, workspace: str): +def list_clusters(client: PlatformClient, workspace: str): """[Platform Only] List available clusters in WORKSPACE""" - if isinstance(client, ClusterClient): - raise click.UsageError("This feature is not available when directly connected to a cluster") - if not (workspace_client := client.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") @@ -288,8 +339,7 @@ def list_clusters(client: Client, workspace: str): click.secho("No clusters for this account", bold=True, fg="red") -@cluster.command(name="new") -@client_option() +@cluster.command(name="new", cls=PlatformCommand) @click.option( "-n", "--name", @@ -318,7 +368,7 @@ def list_clusters(client: Client, workspace: str): ) @click.argument("workspace") def new_cluster( - client: Client, + client: PlatformClient, workspace: str, cluster_name: str | None, cluster_slug: str | None, @@ -327,9 +377,6 @@ def new_cluster( ): """[Platform Only] Create a new cluster in WORKSPACE""" - if isinstance(client, ClusterClient): - raise click.UsageError("This feature is not available when directly connected to a cluster") - if not (workspace_client := client.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") @@ -355,10 +402,8 @@ def new_cluster( # TODO: Test payment w/ Signature validation of extra data -@cluster.command() -@client_option() -@click.argument("cluster", default=None, required=False) -def status(client: Client, cluster: str): +@cluster.command(cls=PlatformCommand) +def status(client: ClusterClient): """ Get Status information about a CLUSTER @@ -367,21 +412,11 @@ def status(client: Client, cluster: str): NOTE: Connecting directly to clusters is supported, but is an advanced use case. """ - if not isinstance(client, ClusterClient): - if cluster is None: - raise click.UsageError("CLUSTER is required for a platform-managed cluster") - - elif "/" not in cluster or len(cluster.split("/")) > 2: - raise click.UsageError("CLUSTER should be in format `WORKSPACE-NAME/CLUSTER-NAME`") - click.echo(render_dict_as_yaml(client.build_display_fields())) - -@cluster.command() -@client_option() -@click.argument("cluster", default=None, required=False) -def bots(client: Client, cluster: str): +@cluster.command(cls=PlatformCommand) +def bots(client: ClusterClient): """ List all bots in a CLUSTER @@ -390,21 +425,9 @@ def bots(client: Client, cluster: str): NOTE: Connecting directly to clusters is supported, but is an advanced use case. """ - if not isinstance(client, ClusterClient): - if cluster is None: - raise click.UsageError("CLUSTER is required for a platform-managed cluster") - - elif "/" not in cluster or len(cluster.split("/")) > 2: - raise click.UsageError("CLUSTER should be in format `WORKSPACE-NAME/CLUSTER-NAME`") - - workspace_name, cluster_name = cluster.split("/") - try: - client = client.get_cluster_client(workspace_name, cluster_name) - except ValueError as e: - raise click.UsageError(str(e)) - if bot_display := render_dict_as_yaml(client.bots): click.echo(bot_display) + else: click.secho("No bots in this cluster", bold=True, fg="red") From 9e9c14e3bc996adbd07152b9e4e9ca499d126816 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:19:19 -0400 Subject: [PATCH 18/54] refactor: display cluster state in a better way --- silverback/cluster/client.py | 25 +++++++++++++------ silverback/cluster/types.py | 47 +++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 6f6fd668..1618c10c 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -5,7 +5,7 @@ from silverback.version import version -from .types import BotInfo, ClusterConfiguration, ClusterInfo, WorkspaceInfo +from .types import BotInfo, ClusterConfiguration, ClusterInfo, ClusterState, WorkspaceInfo DEFAULT_HEADERS = {"User-Agent": f"Silverback SDK/{version}"} @@ -62,18 +62,29 @@ def openapi_schema(self) -> dict: return self.get("/openapi.json").json() @property - def status(self) -> str: + def state(self) -> ClusterState: response = self.get("/") handle_error_with_response(response) - # NOTE: Just return full response directly to avoid errors - return response.text + return ClusterState.model_validate(response.json()) @property def bots(self) -> dict[str, BotInfo]: - response = self.get("/bots") + response = self.get("/bot") # TODO: rename `/bots` handle_error_with_response(response) - # TODO: Actually connect to cluster and display options - return {} + return {bot.slug: bot for bot in map(BotInfo.model_validate, response.json())} + + def build_display_fields(self) -> dict[str, str | dict[str, str]]: + state = self.state + + display_fields: dict[str, str | dict[str, str]] = dict( + version=state.version, + bots=str(len(self.bots)), # TODO: Source this from `ClusterState` + ) + + if state.configuration: + display_fields["configuration"] = state.configuration.build_display_fields() + + return display_fields class Workspace(WorkspaceInfo): diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index ee8e948e..0384d284 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -2,9 +2,10 @@ import math import uuid from datetime import datetime +from hashlib import blake2s from typing import Annotated, Any -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator # NOTE: All configuration settings must be uint8 integer values UINT8_MAX = 2**8 - 1 @@ -181,8 +182,48 @@ def build_display_fields(self) -> dict[str, str | dict[str, str]]: ) +# TODO: Merge `/health` with `/` +class ClusterState(BaseModel): + """ + Cluster Build Information and Configuration, direct from cluster control service + """ + + version: str = Field(alias="cluster_version") # TODO: Rename in cluster + configuration: ClusterConfiguration | None = None # TODO: Add to cluster + # TODO: Add other useful summary fields for frontend use (`bots: int`, `errors: int`, etc.) + + class BotInfo(BaseModel): - id: uuid.UUID + id: uuid.UUID # TODO: Change `.instance_id` field to `id: UUID` + + # TODO: Add `.network`, `.slug`, `.network` fields to cluster model + @model_validator(mode="before") + def set_expected_fields(cls, data: dict) -> dict: + instance_id: str = data.get("instance_id", "random:network:") + name_hash = blake2s(instance_id.encode("utf-8")) + data["id"] = uuid.UUID(bytes=name_hash.digest()[:16]) + ecosystem, network, name = instance_id.split(":") + data["slug"] = name + data["name"] = name.capitalize() + data["network"] = f"{ecosystem}:{network}" + return data + + slug: str name: str + network: str + + # TODO: More config fields (`.description`, `.image`, `.account`, `.environment`) - # TODO: More fields + # Other fields that are currently in there (TODO: Remove) + config_set_name: str + config_set_revision: int + revision: int + terminated: bool + + def build_display_fields(self) -> dict[str, str]: + return dict( + # No `.id`, not visible to client user + # No `.slug`, primary identifier used in dict + name=self.name, + network=self.network, + ) From cca4aad58e6cb460ecb0c7b09b8b62aa419cf546 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:06:01 -0400 Subject: [PATCH 19/54] feat(cluster): add /env support --- silverback/_cli.py | 144 ++++++++++++++++++++++++++++++++++- silverback/cluster/client.py | 45 ++++++++++- silverback/cluster/types.py | 35 +++++++++ 3 files changed, 218 insertions(+), 6 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index e2645ef4..a6ef1fcb 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -415,8 +415,148 @@ def status(client: ClusterClient): click.echo(render_dict_as_yaml(client.build_display_fields())) -@cluster.command(cls=PlatformCommand) -def bots(client: ClusterClient): +@cluster.group(cls=OrderedCommands) +def env(): + """Commands for managing environment variables in CLUSTER""" + + +def parse_envvars(ctx, name, value: list[str]) -> dict[str, str]: + def parse_envar(item: str): + if not ("=" in item and len(item.split("=")) == 2): + raise click.UsageError("Value '{item}' must be in form `NAME=VAL`") + + return item.split("=") + + return dict(parse_envar(item) for item in value) + + +@env.command(cls=PlatformCommand) +@click.option( + "-e", + "--env", + "variables", + multiple=True, + type=str, + metavar="NAME=VAL", + callback=parse_envvars, + help="Environment variable key and value to add (Multiple allowed)", +) +@click.argument("name") +def add(client: ClusterClient, variables: dict, name: str): + """Create a new GROUP of environment variables in CLUSTER""" + if len(variables) == 0: + raise click.UsageError("Must supply at least one var via `-e`") + + try: + click.echo(render_dict_as_yaml(client.new_env(name=name, variables=variables))) + + except RuntimeError as e: + raise click.UsageError(str(e)) + + +@env.command(name="list", cls=PlatformCommand) +def list_envs(client: ClusterClient): + """List latest revisions of all variable groups in CLUSTER""" + if all_envs := render_dict_as_yaml(client.envs): + click.echo(all_envs) + + else: + click.secho("No envs in this cluster", bold=True, fg="red") + + +@env.command(cls=PlatformCommand) +@click.argument("name") +@click.argument("new_name") +def change_name(client: ClusterClient, name: str, new_name: str): + """Change the display name of a variable GROUP in CLUSTER""" + if not (env := client.envs.get(name)): + raise click.UsageError(f"Unknown Variable Group '{name}'") + + click.echo(render_dict_as_yaml(env.update(name=new_name))) + + +@env.command(name="set", cls=PlatformCommand) +@click.option( + "-e", + "--env", + "updated_vars", + multiple=True, + type=str, + metavar="NAME=VAL", + callback=parse_envvars, + help="Environment variable key and value to add/update (Multiple allowed)", +) +@click.option( + "-d", + "--del", + "deleted_vars", + multiple=True, + type=str, + metavar="NAME", + help="Environment variable name to delete (Multiple allowed)", +) +@click.argument("name") +def set_env( + client: ClusterClient, + name: str, + updated_vars: dict[str, str], + deleted_vars: tuple[str], +): + """Create a new revision of GROUP in CLUSTER with updated values""" + if dup := "', '".join(set(updated_vars) & set(deleted_vars)): + raise click.UsageError(f"Cannot update and delete vars at the same time: '{dup}'") + + if not (env := client.envs.get(name)): + raise click.UsageError(f"Unknown Variable Group '{name}'") + + if missing := "', '".join(set(deleted_vars) - set(env.variables)): + raise click.UsageError(f"Cannot delete vars not in env: '{missing}'") + + click.echo( + render_dict_as_yaml( + env.add_revision(dict(**updated_vars, **{v: None for v in deleted_vars})) + ) + ) + + +@env.command(cls=PlatformCommand) +@click.argument("name") +@click.option("-r", "--revision", type=int, help="Revision of GROUP to show (Defaults to latest)") +def show(client: ClusterClient, name: str, revision: int | None): + """Show all variables in latest revision of GROUP in CLUSTER""" + if not (env := client.envs.get(name)): + raise click.UsageError(f"Unknown Variable Group '{name}'") + + for env_info in env.revisions: + if revision is None or env_info.revision == revision: + click.echo(render_dict_as_yaml(env_info)) + return + + raise click.UsageError(f"Revision {revision} of '{name}' not found") + + +@env.command(cls=PlatformCommand) +@click.argument("name") +def rm(client: ClusterClient, name: str): + """ + Remove a variable GROUP from CLUSTER + + NOTE: Cannot delete if any bots reference any revision of GROUP + """ + if not (env := client.envs.get(name)): + raise click.UsageError(f"Unknown Variable Group '{name}'") + + env.rm() + click.secho(f"Variable Group '{env.name}' removed.", fg="green", bold=True) + + +@cluster.group(cls=OrderedCommands) +def bot(): + """Commands for managing bots in a CLUSTER""" + + +@bot.command(name="list", cls=PlatformCommand) +def list_bots(client: ClusterClient): """ List all bots in a CLUSTER diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 1618c10c..6e0b9d0f 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -1,3 +1,4 @@ +import uuid from functools import cache from typing import ClassVar @@ -5,7 +6,7 @@ from silverback.version import version -from .types import BotInfo, ClusterConfiguration, ClusterInfo, ClusterState, WorkspaceInfo +from .types import BotInfo, ClusterConfiguration, ClusterInfo, ClusterState, EnvInfo, WorkspaceInfo DEFAULT_HEADERS = {"User-Agent": f"Silverback SDK/{version}"} @@ -44,11 +45,39 @@ def render_error(error: dict): assert response.status_code < 300, "Should follow redirects, so not sure what the issue is" +class Env(EnvInfo): + # NOTE: Client used only for this SDK + # NOTE: DI happens in `PlatformClient.client` + client: ClassVar[httpx.Client] + + def update(self, name: str | None = None): + response = self.client.put(f"/env/{self.id}", json=dict(name=name)) + handle_error_with_response(response) + + @property + def revisions(self) -> list[EnvInfo]: + response = self.client.get(f"/env/{self.id}") + handle_error_with_response(response) + return [EnvInfo.model_validate(env_info) for env_info in response.json()] + + def add_revision(self, variables: dict[str, str | None]) -> "Env": + response = self.client.post(f"/env/{self.id}", json=dict(variables=variables)) + handle_error_with_response(response) + return Env.model_validate(response.json()) + + def rm(self): + response = self.client.delete(f"/env/{self.id}") + handle_error_with_response(response) + + class ClusterClient(httpx.Client): def __init__(self, *args, **kwargs): kwargs["headers"] = {**kwargs.get("headers", {}), **DEFAULT_HEADERS} super().__init__(*args, **kwargs) + # DI for other client classes + Env.client = self # Connect to cluster client + def send(self, request, *args, **kwargs): try: return super().send(request, *args, **kwargs) @@ -67,6 +96,17 @@ def state(self) -> ClusterState: handle_error_with_response(response) return ClusterState.model_validate(response.json()) + @property + def envs(self) -> dict[str, Env]: + response = self.get("/env") + handle_error_with_response(response) + return {env.name: env for env in map(Env.model_validate, response.json())} + + def new_env(self, name: str, variables: dict[str, str]) -> EnvInfo: + response = self.post("/env", json=dict(name=name, variables=variables)) + handle_error_with_response(response) + return EnvInfo.model_validate(response.json()) + @property def bots(self) -> dict[str, BotInfo]: response = self.get("/bot") # TODO: rename `/bots` @@ -199,6 +239,3 @@ def create_workspace( new_workspace = Workspace.model_validate_json(response.text) self.workspaces.update({new_workspace.slug: new_workspace}) # NOTE: Update cache return new_workspace - - -Client = PlatformClient | ClusterClient diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index 0384d284..a814c13b 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -17,6 +17,14 @@ def render_dict_as_yaml(value: Any, prepend: str = "\n") -> str: if hasattr(value, "build_display_fields"): return render_dict_as_yaml(value.build_display_fields(), prepend=prepend) + elif isinstance(value, str): + return value + + elif isinstance(value, list): + return "- " + f"{prepend}- ".join( + render_dict_as_yaml(i, prepend=f"{prepend} ") for i in value + ) + elif not isinstance(value, dict): raise ValueError(f"'{type(value)}' is not renderable.") @@ -193,6 +201,33 @@ class ClusterState(BaseModel): # TODO: Add other useful summary fields for frontend use (`bots: int`, `errors: int`, etc.) +class EnvInfo(BaseModel): + id: uuid.UUID + + @model_validator(mode="before") + def set_expected_fields(cls, data: dict) -> dict: + name: str = data["name"] + instance_id: str = f"{name}.{data['revision']}" + name_hash = blake2s(instance_id.encode("utf-8")) + data["id"] = uuid.UUID(bytes=name_hash.digest()[:16]) + data["variables"] = list(data["variables"]) + return data + + name: str + revision: int + variables: list[str] # TODO: Change to list + created: datetime + + def build_display_fields(self) -> dict[str, str | list[str]]: + return dict( + # No `.id`, not visible to client user + # '.name` is primary identifier + revision=str(self.revision), + created=TIME_FORMAT_STRING.format(self.created.astimezone()), + variables=self.variables, + ) + + class BotInfo(BaseModel): id: uuid.UUID # TODO: Change `.instance_id` field to `id: UUID` From 0be9b09e5b45fa9cb883791fadf292133f177685 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:41:02 -0400 Subject: [PATCH 20/54] refactor(cli): move `run_broker` helper function to it's own module --- silverback/_cli.py | 26 +------------------------- silverback/worker.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 25 deletions(-) create mode 100644 silverback/worker.py diff --git a/silverback/_cli.py b/silverback/_cli.py index a6ef1fcb..83ce7cdc 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -1,6 +1,5 @@ import asyncio import os -from concurrent.futures import ThreadPoolExecutor import click from ape.cli import ( @@ -13,10 +12,6 @@ from ape.exceptions import Abort from fief_client import Fief from fief_client.integrations.cli import FiefAuth, FiefAuthNotAuthenticatedError -from taskiq import AsyncBroker -from taskiq.cli.worker.run import shutdown_broker -from taskiq.receiver import Receiver - from silverback._importer import import_from_string from silverback.cluster.client import ClusterClient, PlatformClient from silverback.cluster.settings import ( @@ -28,6 +23,7 @@ ) from silverback.cluster.types import ClusterTier, render_dict_as_yaml from silverback.runner import PollingRunner, WebsocketRunner +from silverback.worker import run_worker class OrderedCommands(click.Group): @@ -86,26 +82,6 @@ def _network_callback(ctx, param, val): return val -async def run_worker(broker: AsyncBroker, worker_count=2, shutdown_timeout=90): - try: - tasks = [] - with ThreadPoolExecutor(max_workers=worker_count) as pool: - for _ in range(worker_count): - receiver = Receiver( - broker=broker, - executor=pool, - validate_params=True, - max_async_tasks=1, - max_prefetch=0, - ) - broker.is_worker_process = True - tasks.append(receiver.listen()) - - await asyncio.gather(*tasks) - finally: - await shutdown_broker(broker, shutdown_timeout) - - @cli.command(cls=ConnectedProviderCommand, help="Run Silverback application client") @ape_cli_context() @verbosity_option() diff --git a/silverback/worker.py b/silverback/worker.py new file mode 100644 index 00000000..ba48ba60 --- /dev/null +++ b/silverback/worker.py @@ -0,0 +1,26 @@ +import asyncio +from concurrent.futures import ThreadPoolExecutor + +from taskiq import AsyncBroker +from taskiq.cli.worker.run import shutdown_broker +from taskiq.receiver import Receiver + + +async def run_worker(broker: AsyncBroker, worker_count=2, shutdown_timeout=90): + try: + tasks = [] + with ThreadPoolExecutor(max_workers=worker_count) as pool: + for _ in range(worker_count): + receiver = Receiver( + broker=broker, + executor=pool, + validate_params=True, + max_async_tasks=1, + max_prefetch=0, + ) + broker.is_worker_process = True + tasks.append(receiver.listen()) + + await asyncio.gather(*tasks) + finally: + await shutdown_broker(broker, shutdown_timeout) From 31dc9adaaa3af90c5ef8b2d276f9304a88d748a5 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:48:22 -0400 Subject: [PATCH 21/54] refactor(cli): refactor platform/cluster client logic into click ext --- silverback/_cli.py | 228 +++++++---------------------------- silverback/_click_ext.py | 195 ++++++++++++++++++++++++++++++ silverback/cluster/client.py | 20 +-- 3 files changed, 248 insertions(+), 195 deletions(-) create mode 100644 silverback/_click_ext.py diff --git a/silverback/_cli.py b/silverback/_cli.py index 83ce7cdc..859f1695 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -10,53 +10,29 @@ verbosity_option, ) from ape.exceptions import Abort -from fief_client import Fief -from fief_client.integrations.cli import FiefAuth, FiefAuthNotAuthenticatedError +from fief_client.integrations.cli import FiefAuth + +from silverback._click_ext import ( + AuthCommand, + OrderedCommands, + PlatformGroup, + cls_import_callback, + display_login_message, +) from silverback._importer import import_from_string from silverback.cluster.client import ClusterClient, PlatformClient -from silverback.cluster.settings import ( - DEFAULT_PROFILE, - PROFILE_PATH, - BaseProfile, - PlatformProfile, - ProfileSettings, -) from silverback.cluster.types import ClusterTier, render_dict_as_yaml from silverback.runner import PollingRunner, WebsocketRunner from silverback.worker import run_worker -class OrderedCommands(click.Group): - # NOTE: Override so we get the list ordered by definition order - def list_commands(self, ctx: click.Context) -> list[str]: - return list(self.commands) - - @click.group(cls=OrderedCommands) def cli(): """Work with Silverback applications in local context (using Ape).""" -def _runner_callback(ctx, param, val): - if not val: - return None - - elif runner := import_from_string(val): - return runner - - raise ValueError(f"Failed to import runner '{val}'.") - - -def _recorder_callback(ctx, param, val): - if not val: - return None - - elif recorder := import_from_string(val): - return recorder() - - raise ValueError(f"Failed to import recorder '{val}'.") - - +# TODO: Make `silverback.settings.Settings` (to remove having to set envvars) +# TODO: Use `envvar=...` to be able to set the value of options from correct envvar def _account_callback(ctx, param, val): if val: val = val.alias.replace("dev_", "TEST::") @@ -65,6 +41,8 @@ def _account_callback(ctx, param, val): return val +# TODO: Make `silverback.settings.Settings` (to remove having to set envvars) +# TODO: Use `envvar=...` to be able to set the value of options from correct envvar def _network_callback(ctx, param, val): # NOTE: Make sure both of these have the same setting if env_network_choice := os.environ.get("SILVERBACK_NETWORK_CHOICE"): @@ -94,16 +72,17 @@ def _network_callback(ctx, param, val): "--runner", "runner_class", help="An import str in format ':'", - callback=_runner_callback, + callback=cls_import_callback, ) @click.option( "--recorder", + "recorder_class", help="An import string in format ':'", - callback=_recorder_callback, + callback=cls_import_callback, ) @click.option("-x", "--max-exceptions", type=int, default=3) @click.argument("path") -def run(cli_ctx, account, runner_class, recorder, max_exceptions, path): +def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): if not runner_class: # NOTE: Automatically select runner class if cli_ctx.provider.ws_uri: @@ -116,7 +95,11 @@ def run(cli_ctx, account, runner_class, recorder, max_exceptions, path): ) app = import_from_string(path) - runner = runner_class(app, recorder=recorder, max_exceptions=max_exceptions) + runner = runner_class( + app, + recorder=recorder_class() if recorder_class else None, + max_exceptions=max_exceptions, + ) asyncio.run(runner.run()) @@ -137,134 +120,7 @@ def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): asyncio.run(run_worker(app.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) -def display_login_message(auth: FiefAuth, host: str): - userinfo = auth.current_user() - user_id = userinfo["sub"] - username = userinfo["fields"].get("username") - click.echo( - f"{click.style('INFO', fg='blue')}: " - f"Logged in to '{click.style(host, bold=True)}' as " - f"'{click.style(username if username else user_id, bold=True)}'" - ) - - -class PlatformCommand(click.Command): - # NOTE: ClassVar assures only loaded once - settings = ProfileSettings.from_config_file() - - # NOTE: Cache this class-wide - platform_client: PlatformClient | None = None - - def get_params(self, ctx: click.Context): - params = super().get_params(ctx) - - def get_profile(ctx, param, value) -> BaseProfile: - - if not (profile := self.settings.profile.get(value)): - raise click.BadOptionUsage(option_name=param, message=f"Unknown profile '{value}'.") - - return profile - - params.append( - click.Option( - param_decls=("-p", "--profile", "profile"), - metavar="PROFILE", - default=DEFAULT_PROFILE, - callback=get_profile, - ) - ) - - params.append( - click.Argument( - param_decls=("cluster",), - metavar="WORKSPACE/CLUSTER", - required=False, - default=None, - ), - ) - - return params - - def get_auth(self, profile: BaseProfile) -> FiefAuth: - if not isinstance(profile, PlatformProfile): - raise click.UsageError( - "This feature is not available outside of the Silverback Platform" - ) - - auth_info = self.settings.auth[profile.auth] - fief = Fief(auth_info.host, auth_info.client_id) - return FiefAuth(fief, str(PROFILE_PATH.parent / f"{profile.auth}.json")) - - def get_platform_client(self, auth: FiefAuth, profile: PlatformProfile) -> PlatformClient: - try: - display_login_message(auth, profile.host) - except FiefAuthNotAuthenticatedError as e: - raise click.UsageError("Not authenticated, please use `silverback login` first.") from e - - return PlatformClient( - base_url=profile.host, - cookies=dict(session=auth.access_token_info()["access_token"]), - ) - - def get_cluster_client(self, cluster_path): - assert self.platform_client, "Something parsing out of order" - - if "/" not in cluster_path or len(cluster_path.split("/")) > 2: - raise click.BadArgumentUsage("CLUSTER should be in format `WORKSPACE/CLUSTER-NAME`") - - workspace_name, cluster_name = cluster_path.split("/") - try: - return self.platform_client.get_cluster_client(workspace_name, cluster_name) - except ValueError as e: - raise click.UsageError(str(e)) - - def invoke(self, ctx: click.Context): - callback_params = self.callback.__annotations__ if self.callback else {} - - cluster_path = ctx.params.pop("cluster") - - if "profile" not in callback_params: - profile = ctx.params.pop("profile") - - else: - profile = ctx.params["profile"] - - if "auth" in callback_params: - ctx.params["auth"] = self.get_auth(profile) - - if "client" in callback_params: - client_type_needed = callback_params.get("client") - - if isinstance(profile, PlatformProfile): - self.platform_client = self.get_platform_client( - ctx.params.get("auth", self.get_auth(profile)), profile - ) - - if client_type_needed == PlatformClient: - ctx.params["client"] = self.platform_client - - else: - ctx.params["client"] = self.get_cluster_client(cluster_path) - - elif not client_type_needed == ClusterClient: - raise click.UsageError("A cluster profile can only directly connect to a cluster.") - - else: - click.echo( - f"{click.style('INFO', fg='blue')}: Logged in to " - f"'{click.style(profile.host, bold=True)}' using API Key" - ) - ctx.params["client"] = ClusterClient( - base_url=profile.host, - headers={"X-API-Key": profile.api_key}, - ) - - assert ctx.params["client"], "Something went wrong" - - super().invoke(ctx) - - -@cli.command(cls=PlatformCommand) +@cli.command(cls=AuthCommand) def login(auth: FiefAuth): """ CLI Login to Managed Authorization Service @@ -279,12 +135,12 @@ def login(auth: FiefAuth): display_login_message(auth, auth.client.base_url) -@cli.group(cls=OrderedCommands) +@cli.group(cls=PlatformGroup) def cluster(): """Connect to hosted application clusters""" -@cluster.command(cls=PlatformCommand) +@cluster.command() def workspaces(client: PlatformClient): """[Platform Only] List available workspaces""" @@ -300,7 +156,7 @@ def workspaces(client: PlatformClient): ) -@cluster.command(name="list", cls=PlatformCommand) +@cluster.command(name="list") @click.argument("workspace") def list_clusters(client: PlatformClient, workspace: str): """[Platform Only] List available clusters in WORKSPACE""" @@ -315,7 +171,7 @@ def list_clusters(client: PlatformClient, workspace: str): click.secho("No clusters for this account", bold=True, fg="red") -@cluster.command(name="new", cls=PlatformCommand) +@cluster.command(name="new") @click.option( "-n", "--name", @@ -371,15 +227,17 @@ def new_cluster( except RuntimeError as e: raise click.UsageError(str(e)) + # TODO: Pay for cluster via new stream + -# `silverback cluster pay WORKSPACE/CLUSTER_NAME --account ALIAS --time "10 days"` +# `silverback cluster pay WORKSPACE/NAME --account ALIAS --time "10 days"` # TODO: Create a signature scheme for ClusterInfo # (ClusterInfo configuration as plaintext, .id as nonce?) # TODO: Test payment w/ Signature validation of extra data -@cluster.command(cls=PlatformCommand) -def status(client: ClusterClient): +@cluster.command(name="status") +def cluster_status(client: ClusterClient): """ Get Status information about a CLUSTER @@ -406,7 +264,7 @@ def parse_envar(item: str): return dict(parse_envar(item) for item in value) -@env.command(cls=PlatformCommand) +@env.command(name="new") @click.option( "-e", "--env", @@ -418,7 +276,7 @@ def parse_envar(item: str): help="Environment variable key and value to add (Multiple allowed)", ) @click.argument("name") -def add(client: ClusterClient, variables: dict, name: str): +def new_env(client: ClusterClient, variables: dict, name: str): """Create a new GROUP of environment variables in CLUSTER""" if len(variables) == 0: raise click.UsageError("Must supply at least one var via `-e`") @@ -430,7 +288,7 @@ def add(client: ClusterClient, variables: dict, name: str): raise click.UsageError(str(e)) -@env.command(name="list", cls=PlatformCommand) +@env.command(name="list") def list_envs(client: ClusterClient): """List latest revisions of all variable groups in CLUSTER""" if all_envs := render_dict_as_yaml(client.envs): @@ -440,7 +298,7 @@ def list_envs(client: ClusterClient): click.secho("No envs in this cluster", bold=True, fg="red") -@env.command(cls=PlatformCommand) +@env.command() @click.argument("name") @click.argument("new_name") def change_name(client: ClusterClient, name: str, new_name: str): @@ -451,7 +309,7 @@ def change_name(client: ClusterClient, name: str, new_name: str): click.echo(render_dict_as_yaml(env.update(name=new_name))) -@env.command(name="set", cls=PlatformCommand) +@env.command(name="set") @click.option( "-e", "--env", @@ -495,10 +353,10 @@ def set_env( ) -@env.command(cls=PlatformCommand) +@env.command(name="show") @click.argument("name") @click.option("-r", "--revision", type=int, help="Revision of GROUP to show (Defaults to latest)") -def show(client: ClusterClient, name: str, revision: int | None): +def show_env(client: ClusterClient, name: str, revision: int | None): """Show all variables in latest revision of GROUP in CLUSTER""" if not (env := client.envs.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") @@ -511,9 +369,9 @@ def show(client: ClusterClient, name: str, revision: int | None): raise click.UsageError(f"Revision {revision} of '{name}' not found") -@env.command(cls=PlatformCommand) +@env.command(name="rm") @click.argument("name") -def rm(client: ClusterClient, name: str): +def remove_env(client: ClusterClient, name: str): """ Remove a variable GROUP from CLUSTER @@ -522,7 +380,7 @@ def rm(client: ClusterClient, name: str): if not (env := client.envs.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") - env.rm() + env.remove() click.secho(f"Variable Group '{env.name}' removed.", fg="green", bold=True) @@ -531,7 +389,7 @@ def bot(): """Commands for managing bots in a CLUSTER""" -@bot.command(name="list", cls=PlatformCommand) +@bot.command(name="list") def list_bots(client: ClusterClient): """ List all bots in a CLUSTER diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py new file mode 100644 index 00000000..02fb03b9 --- /dev/null +++ b/silverback/_click_ext.py @@ -0,0 +1,195 @@ +import click +from fief_client import Fief +from fief_client.integrations.cli import FiefAuth, FiefAuthNotAuthenticatedError + +from silverback._importer import import_from_string +from silverback.cluster.client import ClusterClient, PlatformClient +from silverback.cluster.settings import ( + DEFAULT_PROFILE, + PROFILE_PATH, + BaseProfile, + ClusterProfile, + PlatformProfile, + ProfileSettings, +) + +# NOTE: only load once +settings = ProfileSettings.from_config_file() + + +def cls_import_callback(ctx, param, cls_name): + if cls_name is None: + return None # User explicitly provided None + + elif cls := import_from_string(cls_name): + return cls + + # If class not found, `import_from_string` returns `None`, so raise + raise click.BadParameter(message=f"Failed to import {param} class: '{cls_name}'.") + + +class OrderedCommands(click.Group): + # NOTE: Override so we get the list ordered by definition order + def list_commands(self, ctx: click.Context) -> list[str]: + return list(self.commands) + + +def display_login_message(auth: FiefAuth, host: str): + userinfo = auth.current_user() + user_id = userinfo["sub"] + username = userinfo["fields"].get("username") + click.echo( + f"{click.style('INFO', fg='blue')}: " + f"Logged in to '{click.style(host, bold=True)}' as " + f"'{click.style(username if username else user_id, bold=True)}'" + ) + + +class AuthCommand(click.Command): + # NOTE: ClassVar for any command to access + profile: ClusterProfile | PlatformProfile + auth: FiefAuth | None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.params.append( + click.Option( + param_decls=("-p", "--profile", "profile"), + expose_value=False, + metavar="PROFILE", + default=DEFAULT_PROFILE, + callback=self.get_profile, + help="The profile to use to connect with the cluster (Advanced)", + ) + ) + + def get_profile(self, ctx, param, value) -> BaseProfile: + + if not (profile := settings.profile.get(value)): + raise click.BadOptionUsage(option_name=param, message=f"Unknown profile '{value}'.") + + self.profile = profile + self.auth = self.get_auth(profile) + return profile + + def get_auth(self, profile: BaseProfile) -> FiefAuth | None: + if not isinstance(profile, PlatformProfile): + return None + + auth_info = settings.auth[profile.auth] + fief = Fief(auth_info.host, auth_info.client_id) + return FiefAuth(fief, str(PROFILE_PATH.parent / f"{profile.auth}.json")) + + def invoke(self, ctx: click.Context): + callback_params = self.callback.__annotations__ if self.callback else {} + + # HACK: Click commands will fail otherwise if something is in context + # the callback doesn't expect, so delete these: + if "profile" not in callback_params and "profile" in ctx.params: + del ctx.params["profile"] + + if "auth" not in callback_params and "auth" in ctx.params: + del ctx.params["auth"] + + return super().invoke(ctx) + + +class ClientCommand(AuthCommand): + workspace_name: str | None = None + cluster_name: str | None = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.params.append( + click.Option( + param_decls=( + "-c", + "--cluster", + ), + metavar="WORKSPACE/NAME", + expose_value=False, + callback=self.get_cluster_path, + help="[Platform Only] The NAME of the cluster in the WORKSPACE you wish to access", + ) + ) + + def get_cluster_path(self, ctx, param, value) -> str | None: + if isinstance(self.profile, PlatformProfile): + if not value: + return value + + elif "/" not in value or len(parts := value.split("/")) > 2: + raise click.BadParameter("CLUSTER should be in format `WORKSPACE/CLUSTER-NAME`") + + self.workspace_name, self.cluster_name = parts + + elif self.profile and value: + raise click.BadParameter("CLUSTER not needed unless using a platform profile") + + return value + + def get_platform_client(self, auth: FiefAuth, profile: PlatformProfile) -> PlatformClient: + try: + display_login_message(auth, profile.host) + except FiefAuthNotAuthenticatedError as e: + raise click.UsageError("Not authenticated, please use `silverback login` first.") from e + + return PlatformClient( + base_url=profile.host, + cookies=dict(session=auth.access_token_info()["access_token"]), + ) + + def invoke(self, ctx: click.Context): + callback_params = self.callback.__annotations__ if self.callback else {} + + if "client" in callback_params: + client_type_needed = callback_params.get("client") + + if isinstance(self.profile, PlatformProfile): + if not self.auth: + raise click.UsageError( + "This feature is not available outside of the Silverback Platform" + ) + + platform_client = self.get_platform_client(self.auth, self.profile) + + if client_type_needed == PlatformClient: + ctx.params["client"] = platform_client + + elif not self.workspace_name or not self.cluster_name: + raise click.UsageError( + "-c WORKSPACE/NAME should be present when using a Platform profile" + ) + + else: + try: + ctx.params["client"] = platform_client.get_cluster_client( + self.workspace_name, self.cluster_name + ) + except ValueError as e: + raise click.UsageError(str(e)) + + elif not client_type_needed == ClusterClient: + raise click.UsageError("A cluster profile can only directly connect to a cluster.") + + else: + click.echo( + f"{click.style('INFO', fg='blue')}: Logged in to " + f"'{click.style(self.profile.host, bold=True)}' using API Key" + ) + ctx.params["client"] = ClusterClient( + base_url=self.profile.host, + headers={"X-API-Key": self.profile.api_key}, + ) + + return super().invoke(ctx) + + +class PlatformGroup(OrderedCommands): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.command_class = ClientCommand + self.group_class = PlatformGroup diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 6e0b9d0f..f41d7a06 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -47,26 +47,26 @@ def render_error(error: dict): class Env(EnvInfo): # NOTE: Client used only for this SDK - # NOTE: DI happens in `PlatformClient.client` - client: ClassVar[httpx.Client] + # NOTE: DI happens in `ClusterClient.__init__` + cluster: ClassVar["ClusterClient"] def update(self, name: str | None = None): - response = self.client.put(f"/env/{self.id}", json=dict(name=name)) + response = self.cluster.put(f"/variables/{self.id}", json=dict(name=name)) handle_error_with_response(response) @property def revisions(self) -> list[EnvInfo]: - response = self.client.get(f"/env/{self.id}") + response = self.cluster.get(f"/variables/{self.id}") handle_error_with_response(response) return [EnvInfo.model_validate(env_info) for env_info in response.json()] def add_revision(self, variables: dict[str, str | None]) -> "Env": - response = self.client.post(f"/env/{self.id}", json=dict(variables=variables)) + response = self.cluster.post(f"/variables/{self.id}", json=dict(variables=variables)) handle_error_with_response(response) return Env.model_validate(response.json()) - def rm(self): - response = self.client.delete(f"/env/{self.id}") + def remove(self): + response = self.cluster.delete(f"/variables/{self.id}") handle_error_with_response(response) @@ -76,7 +76,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # DI for other client classes - Env.client = self # Connect to cluster client + Env.cluster = self # Connect to cluster client def send(self, request, *args, **kwargs): try: @@ -98,12 +98,12 @@ def state(self) -> ClusterState: @property def envs(self) -> dict[str, Env]: - response = self.get("/env") + response = self.get("/variables") handle_error_with_response(response) return {env.name: env for env in map(Env.model_validate, response.json())} def new_env(self, name: str, variables: dict[str, str]) -> EnvInfo: - response = self.post("/env", json=dict(name=name, variables=variables)) + response = self.post("/variables", json=dict(name=name, variables=variables)) handle_error_with_response(response) return EnvInfo.model_validate(response.json()) From 63381b59c4d1e7bce1c0cc6b4c499aa977295669 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:49:00 -0400 Subject: [PATCH 22/54] feat(cli): bot endpoints for cluster mgmt cli --- silverback/_cli.py | 171 +++++++++++++++++++++++++++++++++++ silverback/cluster/client.py | 85 ++++++++++++++++- 2 files changed, 252 insertions(+), 4 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 859f1695..60811b39 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -389,6 +389,50 @@ def bot(): """Commands for managing bots in a CLUSTER""" +@bot.command(name="new") +@click.option("-n", "--name", required=True) +@click.option("-i", "--image", required=True) +@click.option("-n", "--network", required=True) +@click.option("-a", "--account") +@click.option("-g", "--group", "groups", multiple=True) +def new_bot( + client: ClusterClient, + name: str, + image: str, + network: str, + account: str | None, + groups: list[str], +): + """Create a new bot in CLUSTER""" + + if name in client.bots: + raise click.UsageError(f"Cannot use name '{name}' to create bot") + + environment = list() + rendered_environment = dict() + for env_id in groups: + if "/" in env_id: + env_name, revision = env_id.split("/") + env = client.envs[env_name].revisions[int(revision)] + + else: + env = client.envs[env_id] + + environment.append(env) + + for var_name in env.variables: + rendered_environment[var_name] = f"{env.name}/{env.revision}" + + display = render_dict_as_yaml(rendered_environment, prepend="\n ") + click.echo(f"Environment:\n {display}") + + if not click.confirm("Do you want to create this bot?"): + return + + bot = client.new_bot(name, image, network, account=account, environment=environment) + click.secho(f"Bot ({bot.id}) deploying...", fg="green", bold=True) + + @bot.command(name="list") def list_bots(client: ClusterClient): """ @@ -406,3 +450,130 @@ def list_bots(client: ClusterClient): click.secho("No bots in this cluster", bold=True, fg="red") +@bot.command(name="status") +@click.argument("bot_name", metavar="BOT") +def show_bot_status(client: ClusterClient, bot_name: str): + """Show status of BOT in CLUSTER""" + + if not (bot := client.bots.get(bot_name)): + raise click.UsageError(f"Unknown bot '{bot_name}'.") + + click.echo(render_dict_as_yaml(bot)) + + +@bot.command(name="update") +@click.option("-n", "--name", "new_name") +@click.option("-i", "--image") +@click.option("-n", "--network") +@click.option("-a", "--account") +@click.option("-g", "--group", "groups", multiple=True) +@click.argument("bot_name", metavar="BOT") +def update_bot( + client: ClusterClient, + bot_name: str, + new_name: str, + image: str, + network: str, + account: str | None, + groups: list[str], +): + """Update configuration of BOT in CLUSTER""" + + if new_name in client.bots: + raise click.UsageError(f"Cannot use name '{new_name}' to update bot '{bot_name}'") + + if not (bot := client.bots.get(bot_name)): + raise click.UsageError(f"Unknown bot '{bot_name}'.") + + environment = list() + rendered_environment = dict() + for env_id in groups: + if "/" in env_id: + env_name, revision = env_id.split("/") + env = client.envs[env_name].revisions[int(revision)] + + else: + env = client.envs[env_id] + + environment.append(env) + + for var_name in env.variables: + rendered_environment[var_name] = f"{env.name}/{env.revision}" + + set_environment = True + + if len(environment) == 0: + set_environment = click.confirm("Do you want to clear all environment variables?") + + else: + display = render_dict_as_yaml(rendered_environment, prepend="\n ") + click.echo(f"Environment:\n {display}") + + if not click.confirm("Do you want to create this bot?"): + return + + bot.update( + name=new_name, + image=image, + network=network, + account=account, + environment=environment if set_environment else None, + ) + + +@bot.command(name="rm") +@click.argument("name", metavar="BOT") +def rm_bot(client: ClusterClient, name: str): + """Remove BOT from CLUSTER""" + + if not (bot := client.bots.get(name)): + raise click.UsageError(f"Unknown bot '{name}'.") + + bot.remove() + click.secho(f"Bot '{bot.name}' removed.", fg="green", bold=True) + + +@bot.command(name="start") +@click.argument("name", metavar="BOT") +def start_bot(client: ClusterClient, name: str): + """Start BOT running in CLUSTER (if stopped or terminated)""" + + if not (bot := client.bots.get(name)): + raise click.UsageError(f"Unknown bot '{name}'.") + + bot.start() + + +@bot.command(name="stop") +@click.argument("name", metavar="BOT") +def stop_bot(client: ClusterClient, name: str): + """Stop BOT running in CLUSTER (if running)""" + + if not (bot := client.bots.get(name)): + raise click.UsageError(f"Unknown bot '{name}'.") + + bot.stop() + + +@bot.command(name="logs") +@click.argument("name", metavar="BOT") +def show_bot_logs(client: ClusterClient, name: str): + """Show runtime logs for BOT in CLUSTER""" + + if not (bot := client.bots.get(name)): + raise click.UsageError(f"Unknown bot '{name}'.") + + for log in bot.logs: + click.echo(log) + + +@bot.command(name="errors") +@click.argument("name", metavar="BOT") +def show_bot_errors(client: ClusterClient, name: str): + """Show errors for BOT in CLUSTER""" + + if not (bot := client.bots.get(name)): + raise click.UsageError(f"Unknown bot '{name}'.") + + for log in bot.errors: + click.echo(log) diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index f41d7a06..59661cb6 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -1,4 +1,3 @@ -import uuid from functools import cache from typing import ClassVar @@ -70,6 +69,59 @@ def remove(self): handle_error_with_response(response) +class Bot(BotInfo): + # NOTE: Client used only for this SDK + # NOTE: DI happens in `ClusterClient.__init__` + cluster: ClassVar["ClusterClient"] + + def update( + self, + name: str | None = None, + image: str | None = None, + network: str | None = None, + account: str | None = None, + environment: list[EnvInfo] | None = None, + ): + form: dict = dict( + name=name, + network=network, + account=account, + image=image, + ) + + if environment: + form["environment"] = [ + dict(id=str(env.id), revision=env.revision) for env in environment + ] + + response = self.cluster.put(f"/bots/{self.id}", json=form) + handle_error_with_response(response) + + def stop(self): + response = self.cluster.post(f"/bots/{self.id}/stop") + handle_error_with_response(response) + + def start(self): + response = self.cluster.post(f"/bots/{self.id}/start") + handle_error_with_response(response) + + @property + def errors(self) -> list[str]: + response = self.cluster.get(f"/bots/{self.id}/errors") + handle_error_with_response(response) + return response.json() + + @property + def logs(self) -> list[str]: + response = self.cluster.get(f"/bots/{self.id}/logs") + handle_error_with_response(response) + return response.json() + + def remove(self): + response = self.cluster.delete(f"/bots/{self.id}") + handle_error_with_response(response) + + class ClusterClient(httpx.Client): def __init__(self, *args, **kwargs): kwargs["headers"] = {**kwargs.get("headers", {}), **DEFAULT_HEADERS} @@ -77,6 +129,7 @@ def __init__(self, *args, **kwargs): # DI for other client classes Env.cluster = self # Connect to cluster client + Bot.cluster = self # Connect to cluster client def send(self, request, *args, **kwargs): try: @@ -108,10 +161,34 @@ def new_env(self, name: str, variables: dict[str, str]) -> EnvInfo: return EnvInfo.model_validate(response.json()) @property - def bots(self) -> dict[str, BotInfo]: - response = self.get("/bot") # TODO: rename `/bots` + def bots(self) -> dict[str, Bot]: + response = self.get("/bots") # TODO: rename `/bots` + handle_error_with_response(response) + return {bot.slug: bot for bot in map(Bot.model_validate, response.json())} + + def new_bot( + self, + name: str, + image: str, + network: str, + account: str | None = None, + environment: list[EnvInfo] | None = None, + ) -> Bot: + form: dict = dict( + name=name, + image=image, + network=network, + account=account, + ) + + if environment is not None: + form["environment"] = [ + dict(id=str(env.id), revision=env.revision) for env in environment + ] + + response = self.post("/bots", json=form) handle_error_with_response(response) - return {bot.slug: bot for bot in map(BotInfo.model_validate, response.json())} + return Bot.model_validate(response.json()) def build_display_fields(self) -> dict[str, str | dict[str, str]]: state = self.state From 29b33f09f9b2a301ed00e94638ca85e9c2482821 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 31 Jul 2024 17:55:15 -0400 Subject: [PATCH 23/54] feat: add ability to create "sections" of commands to help w/readability --- silverback/_cli.py | 26 ++++++++++++++-------- silverback/_click_ext.py | 47 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 60811b39..ad530f27 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -14,8 +14,8 @@ from silverback._click_ext import ( AuthCommand, - OrderedCommands, PlatformGroup, + SectionedHelpGroup, cls_import_callback, display_login_message, ) @@ -26,7 +26,7 @@ from silverback.worker import run_worker -@click.group(cls=OrderedCommands) +@click.group(cls=SectionedHelpGroup) def cli(): """Work with Silverback applications in local context (using Ape).""" @@ -60,7 +60,11 @@ def _network_callback(ctx, param, val): return val -@cli.command(cls=ConnectedProviderCommand, help="Run Silverback application client") +@cli.command( + cls=ConnectedProviderCommand, + help="Run Silverback application client", + section="Local Commands", +) @ape_cli_context() @verbosity_option() @network_option( @@ -103,7 +107,11 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): asyncio.run(runner.run()) -@cli.command(cls=ConnectedProviderCommand, help="Run Silverback application task workers") +@cli.command( + cls=ConnectedProviderCommand, + help="Run Silverback application task workers", + section="Local Commands", +) @ape_cli_context() @verbosity_option() @network_option( @@ -120,7 +128,7 @@ def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): asyncio.run(run_worker(app.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) -@cli.command(cls=AuthCommand) +@cli.command(cls=AuthCommand, section="Cloud Commands (https://silverback.apeworx.io)") def login(auth: FiefAuth): """ CLI Login to Managed Authorization Service @@ -135,12 +143,12 @@ def login(auth: FiefAuth): display_login_message(auth, auth.client.base_url) -@cli.group(cls=PlatformGroup) +@cli.group(cls=PlatformGroup, section="Cloud Commands (https://silverback.apeworx.io)") def cluster(): """Connect to hosted application clusters""" -@cluster.command() +@cluster.command(section="Platform Commands (https://silverback.apeworx.io)") def workspaces(client: PlatformClient): """[Platform Only] List available workspaces""" @@ -156,7 +164,7 @@ def workspaces(client: PlatformClient): ) -@cluster.command(name="list") +@cluster.command(name="list", section="Platform Commands (https://silverback.apeworx.io)") @click.argument("workspace") def list_clusters(client: PlatformClient, workspace: str): """[Platform Only] List available clusters in WORKSPACE""" @@ -171,7 +179,7 @@ def list_clusters(client: PlatformClient, workspace: str): click.secho("No clusters for this account", bold=True, fg="red") -@cluster.command(name="new") +@cluster.command(name="new", section="Platform Commands (https://silverback.apeworx.io)") @click.option( "-n", "--name", diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index 02fb03b9..6d480141 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -34,6 +34,51 @@ def list_commands(self, ctx: click.Context) -> list[str]: return list(self.commands) +class SectionedHelpGroup(OrderedCommands): + """Section commands into help groups""" + + sections: dict[str | None, list[click.Command | click.Group]] + + def __init__(self, *args, section=None, **kwargs): + self.section = section or "Commands" + self.sections = kwargs.pop("sections", {}) + commands = {} + + for section, command_list in self.sections.items(): + for cmd in command_list: + cmd.section = section + commands[cmd.name] = cmd + + super().__init__(*args, commands=commands, **kwargs) + + def command(self, *args, **kwargs): + section = kwargs.pop("section", "Commands") + decorator = super().command(*args, **kwargs) + + def new_decorator(f): + cmd = decorator(f) + cmd.section = section + self.sections.setdefault(section, []).append(cmd) + return cmd + + return new_decorator + + def format_commands(self, ctx, formatter): + for section, cmds in self.sections.items(): + rows = [] + for subcommand in self.list_commands(ctx): + cmd = self.get_command(ctx, subcommand) + + if cmd is None or cmd.section != section: + continue + + rows.append((subcommand, cmd.get_short_help_str(formatter.width) or "")) + + if rows: + with formatter.section(section): + formatter.write_dl(rows) + + def display_login_message(auth: FiefAuth, host: str): userinfo = auth.current_user() user_id = userinfo["sub"] @@ -187,7 +232,7 @@ def invoke(self, ctx: click.Context): return super().invoke(ctx) -class PlatformGroup(OrderedCommands): +class PlatformGroup(SectionedHelpGroup): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From e39cf9417799246769a5d73a0079dcc55c00f392 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:02:58 -0400 Subject: [PATCH 24/54] refactor: remove custom yaml render fn --- silverback/_cli.py | 18 ++++++---- silverback/cluster/client.py | 13 ------- silverback/cluster/types.py | 66 +----------------------------------- 3 files changed, 13 insertions(+), 84 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index ad530f27..a5566c18 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -2,6 +2,7 @@ import os import click +import yaml # type: ignore[import-untyped] from ape.cli import ( AccountAliasPromptChoice, ConnectedProviderCommand, @@ -21,7 +22,7 @@ ) from silverback._importer import import_from_string from silverback.cluster.client import ClusterClient, PlatformClient -from silverback.cluster.types import ClusterTier, render_dict_as_yaml +from silverback.cluster.types import ClusterTier from silverback.runner import PollingRunner, WebsocketRunner from silverback.worker import run_worker @@ -152,8 +153,8 @@ def cluster(): def workspaces(client: PlatformClient): """[Platform Only] List available workspaces""" - if workspace_display := render_dict_as_yaml(client.workspaces): - click.echo(workspace_display) + if workspace_names := list(client.workspaces): + click.echo(yaml.safe_dump(workspace_names)) else: click.secho( @@ -172,8 +173,8 @@ def list_clusters(client: PlatformClient, workspace: str): if not (workspace_client := client.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") - if cluster_display := render_dict_as_yaml(workspace_client.clusters): - click.echo(cluster_display) + if cluster_names := list(workspace_client.clusters): + click.echo(yaml.safe_dump(cluster_names)) else: click.secho("No clusters for this account", bold=True, fg="red") @@ -295,6 +296,7 @@ def new_env(client: ClusterClient, variables: dict, name: str): except RuntimeError as e: raise click.UsageError(str(e)) + click.echo(yaml.safe_dump(vg.model_dump(exclude={"id"}))) # NOTE: Skip machine `.id` @env.command(name="list") def list_envs(client: ClusterClient): @@ -314,7 +316,11 @@ def change_name(client: ClusterClient, name: str, new_name: str): if not (env := client.envs.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") - click.echo(render_dict_as_yaml(env.update(name=new_name))) + click.echo( + yaml.safe_dump( + env.update(name=new_slug).model_dump(exclude={"id"}) # NOTE: Skip machine `.id` + ) + ) @env.command(name="set") diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 59661cb6..c4df65d4 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -190,19 +190,6 @@ def new_bot( handle_error_with_response(response) return Bot.model_validate(response.json()) - def build_display_fields(self) -> dict[str, str | dict[str, str]]: - state = self.state - - display_fields: dict[str, str | dict[str, str]] = dict( - version=state.version, - bots=str(len(self.bots)), # TODO: Source this from `ClusterState` - ) - - if state.configuration: - display_fields["configuration"] = state.configuration.build_display_fields() - - return display_fields - class Workspace(WorkspaceInfo): # NOTE: Client used only for this SDK diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index a814c13b..075f4c50 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -3,40 +3,13 @@ import uuid from datetime import datetime from hashlib import blake2s -from typing import Annotated, Any +from typing import Annotated from pydantic import BaseModel, Field, field_validator, model_validator # NOTE: All configuration settings must be uint8 integer values UINT8_MAX = 2**8 - 1 -TIME_FORMAT_STRING = "{0:%x} {0:%X}" - - -def render_dict_as_yaml(value: Any, prepend: str = "\n") -> str: - if hasattr(value, "build_display_fields"): - return render_dict_as_yaml(value.build_display_fields(), prepend=prepend) - - elif isinstance(value, str): - return value - - elif isinstance(value, list): - return "- " + f"{prepend}- ".join( - render_dict_as_yaml(i, prepend=f"{prepend} ") for i in value - ) - - elif not isinstance(value, dict): - raise ValueError(f"'{type(value)}' is not renderable.") - - return prepend.join( - ( - f"{key}: {value}" - if isinstance(value, str) - else f"{key}:{prepend + ' '}{render_dict_as_yaml(value, prepend=(prepend + ' '))}" - ) - for key, value in value.items() - ) - class WorkspaceInfo(BaseModel): id: uuid.UUID @@ -122,15 +95,6 @@ def encode(self) -> int: + (self.triggers // 5 << 40) ) - def build_display_fields(self) -> dict[str, str]: - return dict( - cpu=f"{256 * 2 ** self.cpu / 1024} vCPU", - memory=(f"{self.memory} GB" if self.memory > 0 else "512 MiB"), - networks=str(self.networks), - bots=str(self.bots), - triggers=str(self.triggers), - ) - class ClusterTier(enum.IntEnum): """Suggestions for different tier configurations""" @@ -177,18 +141,6 @@ class ClusterInfo(BaseModel): status: ClusterStatus last_updated: datetime - def build_display_fields(self) -> dict[str, str | dict[str, str]]: - return dict( - # No `.id`, not visible to client user - name=self.name, - # No `.slug`, primary identifier used in dict - # NOTE: Convert local time - created=TIME_FORMAT_STRING.format(self.created.astimezone()), - last_updated=TIME_FORMAT_STRING.format(self.last_updated.astimezone()), - status=str(self.status), - configuration=self.configuration.build_display_fields(), - ) - # TODO: Merge `/health` with `/` class ClusterState(BaseModel): @@ -218,14 +170,6 @@ def set_expected_fields(cls, data: dict) -> dict: variables: list[str] # TODO: Change to list created: datetime - def build_display_fields(self) -> dict[str, str | list[str]]: - return dict( - # No `.id`, not visible to client user - # '.name` is primary identifier - revision=str(self.revision), - created=TIME_FORMAT_STRING.format(self.created.astimezone()), - variables=self.variables, - ) class BotInfo(BaseModel): @@ -254,11 +198,3 @@ def set_expected_fields(cls, data: dict) -> dict: config_set_revision: int revision: int terminated: bool - - def build_display_fields(self) -> dict[str, str]: - return dict( - # No `.id`, not visible to client user - # No `.slug`, primary identifier used in dict - name=self.name, - network=self.network, - ) From 55f2eee1c4fda91fff8f2c8283280372625385d9 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:03:41 -0400 Subject: [PATCH 25/54] fix: remove duplicate `verbosity_option` (comes for free now) --- silverback/_cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index a5566c18..7ff65638 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -8,7 +8,6 @@ ConnectedProviderCommand, ape_cli_context, network_option, - verbosity_option, ) from ape.exceptions import Abort from fief_client.integrations.cli import FiefAuth @@ -67,7 +66,6 @@ def _network_callback(ctx, param, val): section="Local Commands", ) @ape_cli_context() -@verbosity_option() @network_option( default=os.environ.get("SILVERBACK_NETWORK_CHOICE", "auto"), callback=_network_callback, @@ -114,7 +112,6 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): section="Local Commands", ) @ape_cli_context() -@verbosity_option() @network_option( default=os.environ.get("SILVERBACK_NETWORK_CHOICE", "auto"), callback=_network_callback, From 4f4ba0275c5633c8328bee2cf5006504e8d81615 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:14:54 -0400 Subject: [PATCH 26/54] refactor: remove dynamic cluster option when not needed --- silverback/_cli.py | 17 ++++++++++++++--- silverback/_click_ext.py | 25 +++++++++++++------------ 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 7ff65638..58252447 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -146,7 +146,10 @@ def cluster(): """Connect to hosted application clusters""" -@cluster.command(section="Platform Commands (https://silverback.apeworx.io)") +@cluster.command( + section="Platform Commands (https://silverback.apeworx.io)", + disable_cluster_option=True, +) def workspaces(client: PlatformClient): """[Platform Only] List available workspaces""" @@ -162,7 +165,11 @@ def workspaces(client: PlatformClient): ) -@cluster.command(name="list", section="Platform Commands (https://silverback.apeworx.io)") +@cluster.command( + name="list", + section="Platform Commands (https://silverback.apeworx.io)", + disable_cluster_option=True, +) @click.argument("workspace") def list_clusters(client: PlatformClient, workspace: str): """[Platform Only] List available clusters in WORKSPACE""" @@ -177,7 +184,11 @@ def list_clusters(client: PlatformClient, workspace: str): click.secho("No clusters for this account", bold=True, fg="red") -@cluster.command(name="new", section="Platform Commands (https://silverback.apeworx.io)") +@cluster.command( + name="new", + section="Platform Commands (https://silverback.apeworx.io)", + disable_cluster_option=True, +) @click.option( "-n", "--name", diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index 6d480141..d3cde6e9 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -144,21 +144,22 @@ class ClientCommand(AuthCommand): workspace_name: str | None = None cluster_name: str | None = None - def __init__(self, *args, **kwargs): + def __init__(self, *args, disable_cluster_option: bool = False, **kwargs): super().__init__(*args, **kwargs) - self.params.append( - click.Option( - param_decls=( - "-c", - "--cluster", - ), - metavar="WORKSPACE/NAME", - expose_value=False, - callback=self.get_cluster_path, - help="[Platform Only] The NAME of the cluster in the WORKSPACE you wish to access", + if not disable_cluster_option: + self.params.append( + click.Option( + param_decls=( + "-c", + "--cluster", + ), + metavar="WORKSPACE/NAME", + expose_value=False, + callback=self.get_cluster_path, + help="[Platform Only] NAME of the cluster in the WORKSPACE you wish to access", + ) ) - ) def get_cluster_path(self, ctx, param, value) -> str | None: if isinstance(self.profile, PlatformProfile): From a71d826bb50a89bf28063edd0262796b80e0b823 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:21:25 -0400 Subject: [PATCH 27/54] docs(cli): refactor help text --- silverback/_cli.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 58252447..c1ae0376 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -28,7 +28,11 @@ @click.group(cls=SectionedHelpGroup) def cli(): - """Work with Silverback applications in local context (using Ape).""" + """ + Silverback: Build Python apps that react to on-chain events + + To learn more about our cloud offering, please check out https://silverback.apeworx.io + """ # TODO: Make `silverback.settings.Settings` (to remove having to set envvars) @@ -62,7 +66,7 @@ def _network_callback(ctx, param, val): @cli.command( cls=ConnectedProviderCommand, - help="Run Silverback application client", + help="Run local Silverback application", section="Local Commands", ) @ape_cli_context() @@ -108,7 +112,7 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): @cli.command( cls=ConnectedProviderCommand, - help="Run Silverback application task workers", + help="Start Silverback distributed task workers (advanced)", section="Local Commands", ) @ape_cli_context() @@ -128,14 +132,7 @@ def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): @cli.command(cls=AuthCommand, section="Cloud Commands (https://silverback.apeworx.io)") def login(auth: FiefAuth): - """ - CLI Login to Managed Authorization Service - - Initiate a login in to the configured service using the given auth PROFILE. - Defaults to https://account.apeworx.io if PROFILE not provided. - - NOTE: You likely do not need to use an auth PROFILE here. - """ + """Login to ApeWorX Authorization Service (https://account.apeworx.io)""" auth.authorize() display_login_message(auth, auth.client.base_url) @@ -143,7 +140,10 @@ def login(auth: FiefAuth): @cli.group(cls=PlatformGroup, section="Cloud Commands (https://silverback.apeworx.io)") def cluster(): - """Connect to hosted application clusters""" + """Manage a Silverback hosted application cluster + + For clusters on the Silverback Platform, please provide a name for the cluster to access under + your platform account via `-c WORKSPACE/NAME`""" @cluster.command( @@ -151,7 +151,7 @@ def cluster(): disable_cluster_option=True, ) def workspaces(client: PlatformClient): - """[Platform Only] List available workspaces""" + """List available workspaces for your account""" if workspace_names := list(client.workspaces): click.echo(yaml.safe_dump(workspace_names)) @@ -172,7 +172,7 @@ def workspaces(client: PlatformClient): ) @click.argument("workspace") def list_clusters(client: PlatformClient, workspace: str): - """[Platform Only] List available clusters in WORKSPACE""" + """List available clusters in a WORKSPACE""" if not (workspace_client := client.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") @@ -224,7 +224,7 @@ def new_cluster( tier: str, config_updates: list[tuple[str, str]], ): - """[Platform Only] Create a new cluster in WORKSPACE""" + """Create a new cluster in WORKSPACE""" if not (workspace_client := client.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") @@ -395,7 +395,7 @@ def show_env(client: ClusterClient, name: str, revision: int | None): @click.argument("name") def remove_env(client: ClusterClient, name: str): """ - Remove a variable GROUP from CLUSTER + Remove a variable GROUP from a CLUSTER NOTE: Cannot delete if any bots reference any revision of GROUP """ @@ -425,7 +425,7 @@ def new_bot( account: str | None, groups: list[str], ): - """Create a new bot in CLUSTER""" + """Create a new bot in a CLUSTER with the given configuration""" if name in client.bots: raise click.UsageError(f"Cannot use name '{name}' to create bot") @@ -457,8 +457,7 @@ def new_bot( @bot.command(name="list") def list_bots(client: ClusterClient): - """ - List all bots in a CLUSTER + """List all bots in a CLUSTER (Regardless of status)""" For clusters on the Silverback Platform, please provide a name for the cluster to access using your platform authentication obtained via `silverback login` in `workspace/cluster-name` format @@ -499,7 +498,9 @@ def update_bot( account: str | None, groups: list[str], ): - """Update configuration of BOT in CLUSTER""" + """Update configuration of BOT in CLUSTER + + NOTE: Some configuration updates will trigger a redeploy""" if new_name in client.bots: raise click.UsageError(f"Cannot use name '{new_name}' to update bot '{bot_name}'") @@ -569,7 +570,7 @@ def start_bot(client: ClusterClient, name: str): @bot.command(name="stop") @click.argument("name", metavar="BOT") def stop_bot(client: ClusterClient, name: str): - """Stop BOT running in CLUSTER (if running)""" + """Stop BOT from running in CLUSTER (if running)""" if not (bot := client.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") @@ -592,7 +593,7 @@ def show_bot_logs(client: ClusterClient, name: str): @bot.command(name="errors") @click.argument("name", metavar="BOT") def show_bot_errors(client: ClusterClient, name: str): - """Show errors for BOT in CLUSTER""" + """Show unacknowledged errors for BOT in CLUSTER""" if not (bot := client.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") From ea6cb422dc14fdf6a7c6e09a23e5f03235d252a1 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:25:30 -0400 Subject: [PATCH 28/54] refactor: don't try to catch client errors, let them raise --- silverback/_cli.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index c1ae0376..301a1885 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -234,16 +234,12 @@ def new_cluster( for k, v in config_updates: setattr(configuration, k, int(v) if v.isnumeric() else v) - try: - cluster = workspace_client.create_cluster( - cluster_name=cluster_name, - cluster_slug=cluster_slug, - configuration=configuration, - ) - click.echo(f"{click.style('SUCCESS', fg='green')}: Created '{cluster.name}'") - except RuntimeError as e: - raise click.UsageError(str(e)) - + cluster = workspace_client.create_cluster( + cluster_name=cluster_name, + cluster_slug=cluster_slug, + configuration=configuration, + ) + click.echo(f"{click.style('SUCCESS', fg='green')}: Created '{cluster.name}'") # TODO: Pay for cluster via new stream @@ -298,11 +294,8 @@ def new_env(client: ClusterClient, variables: dict, name: str): if len(variables) == 0: raise click.UsageError("Must supply at least one var via `-e`") - try: - click.echo(render_dict_as_yaml(client.new_env(name=name, variables=variables))) + click.echo(render_dict_as_yaml(client.new_env(name=name, variables=variables))) - except RuntimeError as e: - raise click.UsageError(str(e)) click.echo(yaml.safe_dump(vg.model_dump(exclude={"id"}))) # NOTE: Skip machine `.id` From 3a514f783b81be66fc4023554ce747664e182d2f Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:29:36 -0400 Subject: [PATCH 29/54] refactor: platform add `/c` to cluster routes by slug --- silverback/cluster/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index c4df65d4..817385d8 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -220,7 +220,7 @@ def get_cluster_client(self, cluster_name: str) -> ClusterClient: raise ValueError(f"Unknown cluster '{cluster_name}' in workspace '{self.name}'.") return ClusterClient( - base_url=f"{self.client.base_url}/{self.slug}/{cluster.slug}", + base_url=f"{self.client.base_url}/c/{self.slug}/{cluster.slug}", cookies=self.client.cookies, # NOTE: pass along platform cookies for proxy auth ) From f0d91f0d85b9a4b263df3e2e3c1137e4b860da9c Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:34:24 -0400 Subject: [PATCH 30/54] refactor: lots of changes; env -> vg, bot -> bots, api updates, hacks, etc. --- silverback/_cli.py | 322 +++++++++++++++++++---------------- silverback/cluster/client.py | 92 +++++++--- silverback/cluster/types.py | 229 +++++++++++++++++-------- 3 files changed, 398 insertions(+), 245 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 301a1885..947a57aa 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -249,35 +249,43 @@ def new_cluster( # TODO: Test payment w/ Signature validation of extra data -@cluster.command(name="status") -def cluster_status(client: ClusterClient): - """ - Get Status information about a CLUSTER +@cluster.command(name="info") +def cluster_info(client: ClusterClient): + """Get Configuration information about a CLUSTER""" - For clusters on the Silverback Platform, please provide a name for the cluster to access using - your platform authentication obtained via `silverback login` in `workspace/cluster-name` format + # NOTE: This actually doesn't query the cluster's routes, which are protected + click.echo(f"Cluster Version: v{client.version}") - NOTE: Connecting directly to clusters is supported, but is an advanced use case. - """ - click.echo(render_dict_as_yaml(client.build_display_fields())) + if config := client.state.configuration: + click.echo(yaml.safe_dump(config.model_dump())) + + else: + click.secho("No Cluster Configuration detected", fg="yellow", bold=True) -@cluster.group(cls=OrderedCommands) -def env(): - """Commands for managing environment variables in CLUSTER""" +@cluster.command(name="health") +def cluster_health(client: ClusterClient): + """Get Health information about a CLUSTER""" + + click.echo(yaml.safe_dump(client.health.model_dump())) + + +@cluster.group() +def vars(): + """Manage groups of environment variables in a CLUSTER""" def parse_envvars(ctx, name, value: list[str]) -> dict[str, str]: def parse_envar(item: str): if not ("=" in item and len(item.split("=")) == 2): - raise click.UsageError("Value '{item}' must be in form `NAME=VAL`") + raise click.UsageError(f"Value '{item}' must be in form `NAME=VAL`") return item.split("=") return dict(parse_envar(item) for item in value) -@env.command(name="new") +@vars.command(name="new") @click.option( "-e", "--env", @@ -289,42 +297,40 @@ def parse_envar(item: str): help="Environment variable key and value to add (Multiple allowed)", ) @click.argument("name") -def new_env(client: ClusterClient, variables: dict, name: str): - """Create a new GROUP of environment variables in CLUSTER""" +def new_vargroup(client: ClusterClient, variables: dict, name: str): + """Create a new group of environment variables in a CLUSTER""" + if len(variables) == 0: raise click.UsageError("Must supply at least one var via `-e`") - click.echo(render_dict_as_yaml(client.new_env(name=name, variables=variables))) + vg = client.new_variable_group(name=name, variables=variables) + click.echo(yaml.safe_dump(vg.model_dump(exclude={"id"}))) # NOTE: Skip machine `.id` - click.echo(yaml.safe_dump(vg.model_dump(exclude={"id"}))) # NOTE: Skip machine `.id` +@vars.command(name="list") +def list_vargroups(client: ClusterClient): + """List latest revisions of all variable groups in a CLUSTER""" -@env.command(name="list") -def list_envs(client: ClusterClient): - """List latest revisions of all variable groups in CLUSTER""" - if all_envs := render_dict_as_yaml(client.envs): - click.echo(all_envs) + if group_names := list(client.variable_groups): + click.echo(yaml.safe_dump(group_names)) else: - click.secho("No envs in this cluster", bold=True, fg="red") + click.secho("No Variable Groups present in this cluster", bold=True, fg="red") -@env.command() +@vars.command(name="info") @click.argument("name") -@click.argument("new_name") -def change_name(client: ClusterClient, name: str, new_name: str): - """Change the display name of a variable GROUP in CLUSTER""" - if not (env := client.envs.get(name)): +def vargroup_info(client: ClusterClient, name: str): + """Show latest revision of a variable GROUP in a CLUSTER""" + + if not (vg := client.variable_groups.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") - click.echo( - yaml.safe_dump( - env.update(name=new_slug).model_dump(exclude={"id"}) # NOTE: Skip machine `.id` - ) - ) + click.echo(yaml.safe_dump(vg.model_dump(exclude={"id", "name"}))) -@env.command(name="set") +@vars.command(name="update") +@click.option("--new-name", "new_name") # NOTE: No `-n` to match `bots update` @click.option( "-e", "--env", @@ -345,78 +351,73 @@ def change_name(client: ClusterClient, name: str, new_name: str): help="Environment variable name to delete (Multiple allowed)", ) @click.argument("name") -def set_env( +def update_vargroup( client: ClusterClient, name: str, + new_name: str, updated_vars: dict[str, str], deleted_vars: tuple[str], ): - """Create a new revision of GROUP in CLUSTER with updated values""" - if dup := "', '".join(set(updated_vars) & set(deleted_vars)): - raise click.UsageError(f"Cannot update and delete vars at the same time: '{dup}'") + """Update a variable GROUP in CLUSTER - if not (env := client.envs.get(name)): + NOTE: Changing the values of variables in GROUP by create a new revision, since variable groups + are immutable. New revisions do not automatically update bot configuration.""" + + if not (vg := client.variable_groups.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") - if missing := "', '".join(set(deleted_vars) - set(env.variables)): - raise click.UsageError(f"Cannot delete vars not in env: '{missing}'") + if dup := "', '".join(set(updated_vars) & set(deleted_vars)): + raise click.UsageError(f"Cannot update and delete vars at the same time: '{dup}'") + + if missing := "', '".join(set(deleted_vars) - set(vg.variables)): + raise click.UsageError(f"Cannot delete vars not in group: '{missing}'") click.echo( - render_dict_as_yaml( - env.add_revision(dict(**updated_vars, **{v: None for v in deleted_vars})) + yaml.safe_dump( + vg.update( + name=new_name, + # NOTE: Do not update variables if no updates are provided + variables=dict(**updated_vars, **{v: None for v in deleted_vars}) or None, + ).model_dump( + exclude={"id"} + ) # NOTE: Skip machine `.id` ) ) -@env.command(name="show") +@vars.command(name="remove") @click.argument("name") -@click.option("-r", "--revision", type=int, help="Revision of GROUP to show (Defaults to latest)") -def show_env(client: ClusterClient, name: str, revision: int | None): - """Show all variables in latest revision of GROUP in CLUSTER""" - if not (env := client.envs.get(name)): - raise click.UsageError(f"Unknown Variable Group '{name}'") - - for env_info in env.revisions: - if revision is None or env_info.revision == revision: - click.echo(render_dict_as_yaml(env_info)) - return - - raise click.UsageError(f"Revision {revision} of '{name}' not found") - - -@env.command(name="rm") -@click.argument("name") -def remove_env(client: ClusterClient, name: str): +def remove_vargroup(client: ClusterClient, name: str): """ Remove a variable GROUP from a CLUSTER NOTE: Cannot delete if any bots reference any revision of GROUP """ - if not (env := client.envs.get(name)): + if not (vg := client.variable_groups.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") - env.remove() - click.secho(f"Variable Group '{env.name}' removed.", fg="green", bold=True) + vg.remove() # NOTE: No confirmation because can only delete if no references exist + click.secho(f"Variable Group '{vg.name}' removed.", fg="green", bold=True) -@cluster.group(cls=OrderedCommands) -def bot(): - """Commands for managing bots in a CLUSTER""" +@cluster.group() +def bots(): + """Manage bots in a CLUSTER""" -@bot.command(name="new") -@click.option("-n", "--name", required=True) +@bots.command(name="new", section="Configuration Commands") @click.option("-i", "--image", required=True) @click.option("-n", "--network", required=True) @click.option("-a", "--account") @click.option("-g", "--group", "groups", multiple=True) +@click.argument("name") def new_bot( client: ClusterClient, - name: str, image: str, network: str, account: str | None, groups: list[str], + name: str, ): """Create a new bot in a CLUSTER with the given configuration""" @@ -424,132 +425,159 @@ def new_bot( raise click.UsageError(f"Cannot use name '{name}' to create bot") environment = list() - rendered_environment = dict() - for env_id in groups: - if "/" in env_id: - env_name, revision = env_id.split("/") - env = client.envs[env_name].revisions[int(revision)] + for vg_id in groups: + if "/" in vg_id: + vg_name, revision = vg_id.split("/") + vg = client.variable_groups[vg_name].get_revision(int(revision)) else: - env = client.envs[env_id] - - environment.append(env) + vg = client.variable_groups[vg_id] - for var_name in env.variables: - rendered_environment[var_name] = f"{env.name}/{env.revision}" + environment.append(vg) - display = render_dict_as_yaml(rendered_environment, prepend="\n ") - click.echo(f"Environment:\n {display}") + click.echo(f"Name: {name}") + click.echo(f"Image: {image}") + click.echo(f"Network: {network}") + if environment: + click.echo("Environment:") + click.echo(yaml.safe_dump([var for vg in environment for var in vg.variables])) if not click.confirm("Do you want to create this bot?"): - return + bot = client.new_bot(name, image, network, account=account, environment=environment) + click.secho(f"Bot '{bot.name}' ({bot.id}) deploying...", fg="green", bold=True) - bot = client.new_bot(name, image, network, account=account, environment=environment) - click.secho(f"Bot ({bot.id}) deploying...", fg="green", bold=True) - -@bot.command(name="list") +@bots.command(name="list", section="Configuration Commands") def list_bots(client: ClusterClient): """List all bots in a CLUSTER (Regardless of status)""" - For clusters on the Silverback Platform, please provide a name for the cluster to access using - your platform authentication obtained via `silverback login` in `workspace/cluster-name` format - - NOTE: Connecting directly to clusters is supported, but is an advanced use case. - """ - if bot_display := render_dict_as_yaml(client.bots): - click.echo(bot_display) + if bot_names := list(client.bots): + click.echo(yaml.safe_dump(bot_names)) else: click.secho("No bots in this cluster", bold=True, fg="red") -@bot.command(name="status") +@bots.command(name="info", section="Configuration Commands") @click.argument("bot_name", metavar="BOT") -def show_bot_status(client: ClusterClient, bot_name: str): - """Show status of BOT in CLUSTER""" +def bot_info(client: ClusterClient, bot_name: str): + """Get configuration information of a BOT in a CLUSTER""" if not (bot := client.bots.get(bot_name)): raise click.UsageError(f"Unknown bot '{bot_name}'.") - click.echo(render_dict_as_yaml(bot)) + # NOTE: Skip machine `.id`, and we already know it is `.name` + click.echo(yaml.safe_dump(bot.model_dump(exclude={"id", "name", "environment"}))) + if bot.environment: + click.echo("environment:") + click.echo(yaml.safe_dump([var.name for var in bot.environment])) -@bot.command(name="update") -@click.option("-n", "--name", "new_name") +@bots.command(name="update", section="Configuration Commands") +@click.option("--new-name", "new_name") # NOTE: No shorthand, because conflicts w/ `--network` @click.option("-i", "--image") @click.option("-n", "--network") @click.option("-a", "--account") @click.option("-g", "--group", "groups", multiple=True) -@click.argument("bot_name", metavar="BOT") +@click.argument("name", metavar="BOT") def update_bot( client: ClusterClient, - bot_name: str, - new_name: str, - image: str, - network: str, + new_name: str | None, + image: str | None, + network: str | None, account: str | None, groups: list[str], + name: str, ): """Update configuration of BOT in CLUSTER NOTE: Some configuration updates will trigger a redeploy""" if new_name in client.bots: - raise click.UsageError(f"Cannot use name '{new_name}' to update bot '{bot_name}'") + raise click.UsageError(f"Cannot use name '{new_name}' to update bot '{name}'") - if not (bot := client.bots.get(bot_name)): - raise click.UsageError(f"Unknown bot '{bot_name}'.") + if not (bot := client.bots.get(name)): + raise click.UsageError(f"Unknown bot '{name}'.") + + if new_name: + click.echo(f"Name:\n old: {name}\n new: {new_name}") + + if network: + click.echo(f"Network:\n old: {bot.network}\n new: {network}") + + redeploy_required = False + if image: + redeploy_required = True + click.echo(f"Image:\n old: {bot.image}\n new: {image}") environment = list() - rendered_environment = dict() - for env_id in groups: - if "/" in env_id: - env_name, revision = env_id.split("/") - env = client.envs[env_name].revisions[int(revision)] + for vg_id in groups: + if "/" in vg_id: + vg_name, revision = vg_id.split("/") + vg = client.variable_groups[vg_name].get_revision(int(revision)) else: - env = client.envs[env_id] + vg = client.variable_groups[vg_id] - environment.append(env) - - for var_name in env.variables: - rendered_environment[var_name] = f"{env.name}/{env.revision}" + environment.append(vg) set_environment = True - if len(environment) == 0: + if len(environment) == 0 and bot.environment: set_environment = click.confirm("Do you want to clear all environment variables?") - else: - display = render_dict_as_yaml(rendered_environment, prepend="\n ") - click.echo(f"Environment:\n {display}") - - if not click.confirm("Do you want to create this bot?"): - return - - bot.update( - name=new_name, - image=image, - network=network, - account=account, - environment=environment if set_environment else None, - ) + elif environment != bot.environment: + click.echo("old-environment:") + click.echo(yaml.safe_dump([var.name for var in bot.environment])) + click.echo("new-environment:") + click.echo(yaml.safe_dump([var for vg in environment for var in vg.variables])) + + redeploy_required |= set_environment + + if click.confirm( + f"Do you want to update '{name}'?" + if not redeploy_required + else f"Do you want to update and redeploy '{name}'?" + ): + bot = bot.update( + name=new_name, + image=image, + network=network, + account=account, + environment=environment if set_environment else None, + ) + # NOTE: Skip machine `.id` + click.echo(yaml.safe_dump(bot.model_dump(exclude={"id", "environment"}))) + if bot.environment: + click.echo("environment:") + click.echo(yaml.safe_dump([var.name for var in bot.environment])) -@bot.command(name="rm") +@bots.command(name="remove", section="Configuration Commands") @click.argument("name", metavar="BOT") -def rm_bot(client: ClusterClient, name: str): - """Remove BOT from CLUSTER""" +def remove_bot(client: ClusterClient, name: str): + """Remove BOT from CLUSTER (Shutdown if running)""" if not (bot := client.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") - bot.remove() - click.secho(f"Bot '{bot.name}' removed.", fg="green", bold=True) + elif click.confirm(f"Do you want to shutdown and delete '{name}'?"): + bot.remove() + click.secho(f"Bot '{bot.name}' removed.", fg="green", bold=True) + + +@bots.command(name="health", section="Bot Operation Commands") +@click.argument("bot_name", metavar="BOT") +def bot_health(client: ClusterClient, bot_name: str): + """Show current health of BOT in a CLUSTER""" + + if not (bot := client.bots.get(bot_name)): + raise click.UsageError(f"Unknown bot '{bot_name}'.") + + click.echo(yaml.safe_dump(bot.health.model_dump(exclude={"bot_id"}))) -@bot.command(name="start") +@bots.command(name="start", section="Bot Operation Commands") @click.argument("name", metavar="BOT") def start_bot(client: ClusterClient, name: str): """Start BOT running in CLUSTER (if stopped or terminated)""" @@ -557,10 +585,12 @@ def start_bot(client: ClusterClient, name: str): if not (bot := client.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") - bot.start() + elif click.confirm(f"Do you want to start running '{name}'?"): + bot.start() + click.secho(f"Bot '{bot.name}' starting...", fg="green", bold=True) -@bot.command(name="stop") +@bots.command(name="stop", section="Bot Operation Commands") @click.argument("name", metavar="BOT") def stop_bot(client: ClusterClient, name: str): """Stop BOT from running in CLUSTER (if running)""" @@ -568,10 +598,12 @@ def stop_bot(client: ClusterClient, name: str): if not (bot := client.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") - bot.stop() + elif click.confirm(f"Do you want to stop '{name}' from running?"): + bot.stop() + click.secho(f"Bot '{bot.name}' stopping...", fg="green", bold=True) -@bot.command(name="logs") +@bots.command(name="logs", section="Bot Operation Commands") @click.argument("name", metavar="BOT") def show_bot_logs(client: ClusterClient, name: str): """Show runtime logs for BOT in CLUSTER""" @@ -583,7 +615,7 @@ def show_bot_logs(client: ClusterClient, name: str): click.echo(log) -@bot.command(name="errors") +@bots.command(name="errors", section="Bot Operation Commands") @click.argument("name", metavar="BOT") def show_bot_errors(client: ClusterClient, name: str): """Show unacknowledged errors for BOT in CLUSTER""" diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 817385d8..73aba9d5 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -5,7 +5,16 @@ from silverback.version import version -from .types import BotInfo, ClusterConfiguration, ClusterInfo, ClusterState, EnvInfo, WorkspaceInfo +from .types import ( + BotHealth, + BotInfo, + ClusterConfiguration, + ClusterHealth, + ClusterInfo, + ClusterState, + VariableGroupInfo, + WorkspaceInfo, +) DEFAULT_HEADERS = {"User-Agent": f"Silverback SDK/{version}"} @@ -44,25 +53,34 @@ def render_error(error: dict): assert response.status_code < 300, "Should follow redirects, so not sure what the issue is" -class Env(EnvInfo): +class VariableGroup(VariableGroupInfo): # NOTE: Client used only for this SDK # NOTE: DI happens in `ClusterClient.__init__` cluster: ClassVar["ClusterClient"] - def update(self, name: str | None = None): - response = self.cluster.put(f"/variables/{self.id}", json=dict(name=name)) - handle_error_with_response(response) - - @property - def revisions(self) -> list[EnvInfo]: - response = self.cluster.get(f"/variables/{self.id}") - handle_error_with_response(response) - return [EnvInfo.model_validate(env_info) for env_info in response.json()] + def __hash__(self) -> int: + return int(self.id) - def add_revision(self, variables: dict[str, str | None]) -> "Env": - response = self.cluster.post(f"/variables/{self.id}", json=dict(variables=variables)) + def update( + self, name: str | None = None, variables: dict[str, str | None] | None = None + ) -> "VariableGroup": + if name is not None: + # Update metadata + response = self.cluster.put(f"/variables/{self.id}", json=dict(name=name)) + handle_error_with_response(response) + + if variables is not None: + # Create a new revision + response = self.cluster.post(f"/variables/{self.id}", json=dict(variables=variables)) + handle_error_with_response(response) + return VariableGroup.model_validate(response.json()) + + return self + + def get_revision(self, revision: int) -> VariableGroupInfo: + response = self.cluster.get(f"/variables/{self.id}/{revision}") handle_error_with_response(response) - return Env.model_validate(response.json()) + return VariableGroupInfo.model_validate(response.json()) def remove(self): response = self.cluster.delete(f"/variables/{self.id}") @@ -80,13 +98,13 @@ def update( image: str | None = None, network: str | None = None, account: str | None = None, - environment: list[EnvInfo] | None = None, - ): + environment: list[VariableGroupInfo] | None = None, + ) -> "Bot": form: dict = dict( name=name, - network=network, account=account, image=image, + network=network, ) if environment: @@ -96,13 +114,24 @@ def update( response = self.cluster.put(f"/bots/{self.id}", json=form) handle_error_with_response(response) + return Bot.model_validate(response.json()) + + @property + def health(self) -> BotHealth: + response = self.cluster.get("/health") # TODO: Migrate this endpoint + # response = self.cluster.get(f"/bots/{self.id}/health") + handle_error_with_response(response) + raw_health = next(bot for bot in response.json()["bots"] if bot["bot_id"] == str(self.id)) + return BotHealth.model_validate(raw_health) # response.json()) TODO: Migrate this endpoint def stop(self): response = self.cluster.post(f"/bots/{self.id}/stop") handle_error_with_response(response) def start(self): - response = self.cluster.post(f"/bots/{self.id}/start") + # response = self.cluster.post(f"/bots/{self.id}/start") TODO: Add `/start` + # NOTE: Currently, a noop PUT request will trigger a start + response = self.cluster.put(f"/bots/{self.id}", json=dict(name=self.name)) handle_error_with_response(response) @property @@ -128,7 +157,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # DI for other client classes - Env.cluster = self # Connect to cluster client + VariableGroup.cluster = self # Connect to cluster client Bot.cluster = self # Connect to cluster client def send(self, request, *args, **kwargs): @@ -143,6 +172,11 @@ def send(self, request, *args, **kwargs): def openapi_schema(self) -> dict: return self.get("/openapi.json").json() + @property + def version(self) -> str: + # NOTE: Does not call routes + return self.openapi_schema["info"]["version"] + @property def state(self) -> ClusterState: response = self.get("/") @@ -150,21 +184,27 @@ def state(self) -> ClusterState: return ClusterState.model_validate(response.json()) @property - def envs(self) -> dict[str, Env]: + def health(self) -> ClusterHealth: + response = self.get("/health") + handle_error_with_response(response) + return ClusterHealth.model_validate(response.json()) + + @property + def variable_groups(self) -> dict[str, VariableGroup]: response = self.get("/variables") handle_error_with_response(response) - return {env.name: env for env in map(Env.model_validate, response.json())} + return {vg.name: vg for vg in map(VariableGroup.model_validate, response.json())} - def new_env(self, name: str, variables: dict[str, str]) -> EnvInfo: + def new_variable_group(self, name: str, variables: dict[str, str]) -> VariableGroup: response = self.post("/variables", json=dict(name=name, variables=variables)) handle_error_with_response(response) - return EnvInfo.model_validate(response.json()) + return VariableGroup.model_validate(response.json()) @property def bots(self) -> dict[str, Bot]: - response = self.get("/bots") # TODO: rename `/bots` + response = self.get("/bots") handle_error_with_response(response) - return {bot.slug: bot for bot in map(Bot.model_validate, response.json())} + return {bot.name: bot for bot in map(Bot.model_validate, response.json())} def new_bot( self, @@ -172,7 +212,7 @@ def new_bot( image: str, network: str, account: str | None = None, - environment: list[EnvInfo] | None = None, + environment: list[VariableGroupInfo] | None = None, ) -> Bot: form: dict = dict( name=name, diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index 075f4c50..7d9754e2 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -2,13 +2,9 @@ import math import uuid from datetime import datetime -from hashlib import blake2s from typing import Annotated -from pydantic import BaseModel, Field, field_validator, model_validator - -# NOTE: All configuration settings must be uint8 integer values -UINT8_MAX = 2**8 - 1 +from pydantic import BaseModel, Field, computed_field, field_validator class WorkspaceInfo(BaseModel): @@ -21,31 +17,42 @@ class WorkspaceInfo(BaseModel): class ClusterConfiguration(BaseModel): """Configuration of the cluster (represented as 16 byte value)""" + # NOTE: This configuration must be encode-able to a uint64 value for db storage + # and on-chain processing through ApePay + # NOTE: All defaults should be the minimal end of the scale, # so that `__or__` works right - # Version byte (Byte 1) + # Version byte (Byte 0) + # NOTE: Just in-case we change this after release version: int = 1 - # Bot Worker Configuration (Bytes 2-3) + # Bot Worker Configuration (Bytes 1-2) cpu: Annotated[int, Field(ge=0, le=16)] = 0 # 0.25 vCPU """Allocated vCPUs per bot: 0.25 vCPU (0) to 16 vCPU (6)""" memory: Annotated[int, Field(ge=0, le=120)] = 0 # 512 MiB """Total memory per bot (in GB)""" - # Runner configuration (Bytes 4-6) + # NOTE: Configure # of workers based on cpu & memory settings + + # Runner configuration (Bytes 3-5) networks: Annotated[int, Field(ge=1, le=20)] = 1 """Maximum number of concurrent network runners""" bots: Annotated[int, Field(ge=1, le=250)] = 1 """Maximum number of concurrent bots running""" - triggers: Annotated[int, Field(ge=5, le=1000, multiple_of=5)] = 30 + triggers: Annotated[int, Field(ge=50, le=1000, multiple_of=5)] = 50 """Maximum number of task triggers across all running bots""" - # TODO: Recorder configuration - # NOTE: Bytes 7-15 empty + # Recorder configuration (Byte 6) + storage: Annotated[int, Field(ge=0, le=250)] = 0 # 512 GB + """Total task results and metrics parquet storage (in TB)""" + + # Cluster general configuration (Byte 7) + secrets: Annotated[int, Field(ge=10, le=100)] = 10 + """Total managed secrets""" @field_validator("cpu", mode="before") def parse_cpu_value(cls, value: str | int) -> int: @@ -67,32 +74,58 @@ def parse_memory_value(cls, value: str | int) -> int: assert units.lower() == "gb" return int(mem) + @field_validator("storage", mode="before") + def parse_storage_value(cls, value: str | int) -> int: + if not isinstance(value, str): + return value + + storage, units = value.split(" ") + if units.lower() == "gb": + assert storage == "512" + return 0 + + assert units.lower() == "tb" + return int(storage) + + @staticmethod + def _decode_byte(value: int, byte: int) -> int: + # NOTE: All configuration settings must be uint8 integer values when encoded + return (value >> (8 * byte)) & (2**8 - 1) # NOTE: max uint8 + @classmethod def decode(cls, value: int) -> "ClusterConfiguration": - """Decode the configuration from 16 byte integer value""" + """Decode the configuration from 8 byte integer value""" if isinstance(value, ClusterConfiguration): return value # TODO: Something weird with SQLModel # NOTE: Do not change the order of these, these are not forwards compatible return cls( - version=value & UINT8_MAX, - cpu=(value >> 8) & UINT8_MAX, - memory=(value >> 16) & UINT8_MAX, - networks=(value >> 24) & UINT8_MAX, - bots=(value >> 32) & UINT8_MAX, - triggers=5 * ((value >> 40) & UINT8_MAX), + version=cls._decode_byte(value, 0), + cpu=cls._decode_byte(value, 1), + memory=cls._decode_byte(value, 2), + networks=cls._decode_byte(value, 3), + bots=cls._decode_byte(value, 4), + triggers=5 * cls._decode_byte(value, 5), + storage=cls._decode_byte(value, 6), + secrets=cls._decode_byte(value, 7), ) + @staticmethod + def _encode_byte(value: int, byte: int) -> int: + return value << (8 * byte) + def encode(self) -> int: - """Encode configuration as 16 byte integer value""" + """Encode configuration as 8 byte integer value""" # NOTE: Do not change the order of these, these are not forwards compatible return ( - self.version - + (self.cpu << 8) - + (self.memory << 16) - + (self.networks << 24) - + (self.bots << 32) - + (self.triggers // 5 << 40) + self._encode_byte(self.version, 0) + + self._encode_byte(self.cpu, 1) + + self._encode_byte(self.memory, 2) + + self._encode_byte(self.networks, 3) + + self._encode_byte(self.bots, 4) + + self._encode_byte(self.triggers // 5, 5) + + self._encode_byte(self.storage, 6) + + self._encode_byte(self.secrets, 7) ) @@ -104,27 +137,60 @@ class ClusterTier(enum.IntEnum): memory="512 MiB", networks=3, bots=5, - triggers=30, + triggers=50, + storage="512 GB", + secrets=10, ).encode() PROFESSIONAL = ClusterConfiguration( cpu="1 vCPU", memory="2 GB", networks=10, bots=20, - triggers=120, + triggers=400, + storage="5 TB", + secrets=25, ).encode() def configuration(self) -> ClusterConfiguration: return ClusterConfiguration.decode(int(self)) -class ClusterStatus(enum.IntEnum): - # NOTE: Selected integer values with some space for other steps - CREATED = 0 # User record created, but not paid for yet - STANDUP = 3 # Payment received, provisioning infrastructure - RUNNING = 5 # Paid for and fully deployed by payment handler - TEARDOWN = 6 # User triggered shutdown or payment expiration recorded - REMOVED = 9 # Infrastructure de-provisioning complete +class ResourceStatus(enum.IntEnum): + """ + Generic enum that represents that status of any associated resource or service. + + ```{note} + Calling `str(...)` on this will produce a human-readable status for display. + ``` + """ + + CREATED = 0 + """Resource record created, but not provisioning yet (likely awaiting payment)""" + + # NOTE: `1` is reserved + + PROVISIONING = 2 + """Resource is provisioning infrastructure (on payment received)""" + + STARTUP = 3 + """Resource is being put into the RUNNING state""" + + RUNNING = 4 + """Resource is in good health (Resource itself should be reporting status now)""" + + # NOTE: `5` is reserved + + SHUTDOWN = 6 + """Resource is being put into the STOPPED state""" + + STOPPED = 7 + """Resource has stopped (due to errors, user action, or resource contraints)""" + + DEPROVISIONING = 8 + """User removal action or payment expiration event triggered""" + + REMOVED = 9 + """Infrastructure de-provisioning complete (Cannot change from this state)""" def __str__(self) -> str: return self.name.capitalize() @@ -133,16 +199,17 @@ def __str__(self) -> str: class ClusterInfo(BaseModel): # NOTE: Raw API object (gets exported) id: uuid.UUID # NOTE: Keep this private, used as a temporary secret key for payment - name: str - slug: str - configuration: ClusterConfiguration + version: str | None # NOTE: Unprovisioned clusters have no known version yet + configuration: ClusterConfiguration | None = None # NOTE: self-hosted clusters have no config - created: datetime - status: ClusterStatus - last_updated: datetime + name: str # User-friendly display name + slug: str # Shorthand name, for CLI and URI usage + + created: datetime # When the resource was first created + status: ResourceStatus + last_updated: datetime # Last time the resource was changed (upgrade, provisioning, etc.) -# TODO: Merge `/health` with `/` class ClusterState(BaseModel): """ Cluster Build Information and Configuration, direct from cluster control service @@ -150,51 +217,65 @@ class ClusterState(BaseModel): version: str = Field(alias="cluster_version") # TODO: Rename in cluster configuration: ClusterConfiguration | None = None # TODO: Add to cluster - # TODO: Add other useful summary fields for frontend use (`bots: int`, `errors: int`, etc.) + # TODO: Add other useful summary fields for frontend use -class EnvInfo(BaseModel): - id: uuid.UUID +class ServiceHealth(BaseModel): + healthy: bool + - @model_validator(mode="before") - def set_expected_fields(cls, data: dict) -> dict: - name: str = data["name"] - instance_id: str = f"{name}.{data['revision']}" - name_hash = blake2s(instance_id.encode("utf-8")) - data["id"] = uuid.UUID(bytes=name_hash.digest()[:16]) - data["variables"] = list(data["variables"]) - return data +class ClusterHealth(BaseModel): + ars: ServiceHealth = Field(exclude=True) # TODO: Replace w/ cluster + ccs: ServiceHealth = Field(exclude=True) # TODO: Replace w/ cluster + bots: dict[str, ServiceHealth] = {} + @field_validator("bots", mode="before") # TODO: Fix so this is default + def convert_bot_health(cls, bots): + return {b["instance_id"]: ServiceHealth.model_validate(b) for b in bots} + + @computed_field + def cluster(self) -> ServiceHealth: + return ServiceHealth(healthy=self.ars.healthy and self.ccs.healthy) + + +class VariableGroupInfo(BaseModel): + id: uuid.UUID name: str revision: int - variables: list[str] # TODO: Change to list + variables: list[str] created: datetime +class EnvironmentVariable(BaseModel): + name: str + group_id: uuid.UUID + group_revision: int -class BotInfo(BaseModel): - id: uuid.UUID # TODO: Change `.instance_id` field to `id: UUID` - # TODO: Add `.network`, `.slug`, `.network` fields to cluster model - @model_validator(mode="before") - def set_expected_fields(cls, data: dict) -> dict: - instance_id: str = data.get("instance_id", "random:network:") - name_hash = blake2s(instance_id.encode("utf-8")) - data["id"] = uuid.UUID(bytes=name_hash.digest()[:16]) - ecosystem, network, name = instance_id.split(":") - data["slug"] = name - data["name"] = name.capitalize() - data["network"] = f"{ecosystem}:{network}" - return data +class BotTaskStatus(BaseModel): + last_status: str + exit_code: int | None + reason: str | None + started_at: datetime | None + stop_code: str | None + stopped_at: datetime | None + stopped_reason: str | None + + +class BotHealth(BaseModel): + bot_id: uuid.UUID + task_status: BotTaskStatus | None + healthy: bool - slug: str - name: str - network: str - # TODO: More config fields (`.description`, `.image`, `.account`, `.environment`) +class BotInfo(BaseModel): + id: uuid.UUID # TODO: Change `.instance_id` field to `id: UUID` + name: str + created: datetime - # Other fields that are currently in there (TODO: Remove) - config_set_name: str - config_set_revision: int + image: str + network: str + account: str | None revision: int - terminated: bool + + environment: list[EnvironmentVariable] = [] From df54e64643173852c7356a06be772d4caf8e2418 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:16:04 -0400 Subject: [PATCH 31/54] docs(cli): add autogen docs for `silverback login` and `cluster` cmds --- docs/commands/cluster.rst | 21 +++++++++++++++++++++ docs/index.md | 1 + 2 files changed, 22 insertions(+) create mode 100644 docs/commands/cluster.rst diff --git a/docs/commands/cluster.rst b/docs/commands/cluster.rst new file mode 100644 index 00000000..1dafd291 --- /dev/null +++ b/docs/commands/cluster.rst @@ -0,0 +1,21 @@ +Cloud Platform +************** + +.. click:: silverback._cli:login + :prog: silverback login + :nested: none + +.. click:: silverback._cli:cluster + :prog: silverback cluster + :nested: full + :commands: workspaces, new, list, info, health + +.. click:: silverback._cli:vars + :prog: silverback cluster vars + :nested: full + :commands: new, list, info, update, remove + +.. click:: silverback._cli:bots + :prog: silverback cluster bots + :nested: full + :commands: new, list, info, update, remove, health, start, stop, logs, errors diff --git a/docs/index.md b/docs/index.md index d6177851..a25b210f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,6 +15,7 @@ :maxdepth: 1 commands/run.rst + commands/cluster.rst ``` ```{eval-rst} From 23ff0bf375d66f21e80a1850419721dbd9503319 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:16:56 -0400 Subject: [PATCH 32/54] docs(cli): clean up the autogen docs for run; add `silverback worker` --- docs/commands/run.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/commands/run.rst b/docs/commands/run.rst index 75a12f64..fbceda35 100644 --- a/docs/commands/run.rst +++ b/docs/commands/run.rst @@ -1,6 +1,10 @@ -run -*** +Local Development +***************** .. click:: silverback._cli:run - :prog: run + :prog: silverback run + :nested: none + +.. click:: silverback._cli:worker + :prog: silverback worker :nested: none From cbf0c7500c74267465337e7a2af18b544bd4f58d Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:17:31 -0400 Subject: [PATCH 33/54] feat(docs): add documentation for using the Platform and Cluster CLI --- docs/index.md | 1 + docs/userguides/platform.md | 185 ++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 docs/userguides/platform.md diff --git a/docs/index.md b/docs/index.md index a25b210f..1a045cbb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,6 +7,7 @@ userguides/quickstart userguides/development + userguides/platform ``` ```{eval-rst} diff --git a/docs/userguides/platform.md b/docs/userguides/platform.md new file mode 100644 index 00000000..a798768f --- /dev/null +++ b/docs/userguides/platform.md @@ -0,0 +1,185 @@ +# Deploying Applications + +In this guide, we are going to show you more details on how to deploy your application to the [Silverback Platform](https://silverback.apeworx.io). + +## Creating a Cluster + +The Silverback Platform runs your Applications (or "Bots") on dedicated managed application Clusters. +These Clusters will take care to orchestrate infrastructure, monitor, run your triggers, and collect metrics for your applications. +Each Cluster is bespoke for an individual or organization, and isolates your applications from others on different infrastructure. + +Before we deploy our Application, we have to create a Cluster. +If you haven't yet, please sign up for Silverback at [https://silverback.apeworx.io](https://silverback.apeworx.io). + +Once you have signed up, you can actually create (and pay for) your Clusters from the Silverback CLI utility by first +logging in to the Platform using [`silverback login`][silverback-login], +and then using [`silverback cluster new`][silverback-cluster-new] to follow the steps necessary to deploy it. + +[silverback-login]: ../commands/cluster.html#silverback-login +[silverback-cluster-new]: ../commands/cluster.html#silverback-cluster-new + +```{note} +The Platform UI will let you create and manage Clusters using a graphical experience, which may be preferred. +The CLI experience is for those working locally who don't want to visit the website, or are locally developing their applications. +``` + +## Connecting to your Cluster + +To connect to a cluster, you can use commands from the [`silverback cluster`][silverback-cluster] subcommand group. +For instance, to list all your available bots on your cluster, use [`silverback cluster bots list`][silverback-cluster-bots-list]. +To obtain general information about your cluster, just use [`silverback cluster info`][silverback-cluster-info], +or [`silverback cluster health`][silverback-cluster-health] to see the current status of your Cluster. + +If you have no bots, we will first have to containerize our Applications and upload them to a container registry that our Cluster is configured to access. + +```{note} +Building a container for your application can be an advanced topic, we have included the `silverback build` subcommand to help assist in generating Dockerfiles. +``` + +[silverback-cluster]: ../commands/cluster.html#silverback-cluster +[silverback-cluster-info]: ../commands/cluster.html#silverback-cluster-info +[silverback-cluster-health]: ../commands/cluster.html#silverback-cluster-health +[silverback-cluster-bots-list]: ../commands/cluster.html#silverback-cluster-bots-list + +## Building your Bot + +TODO: Add build process and describe `silverback build --autogen` and `silverback build --upgrade` + +TODO: Add how to debug containers using `silverback run` w/ `taskiq-redis` broker + +## Adding Environment Variables + +Once you have created your bot application container image, you might know of some environment variables the image requires to run properly. +Thanks to it's flexible plugin system, ape plugins may also require specific environment variables to load as well. +Silverback Clusters include an environment variable management system for exactly this purpose, +which you can manage using [`silverback cluster vars`][silverback-cluster-vars] subcommand. + +The environment variable management system makes use of a concept called "Variable Groups" which are distinct collections of environment variables meant to be used together. +These variable groups will help in managing the runtime environment of your Applications by allowing you to segregate different variables depending on each bot's needs. + +To create an environment group, use the [`silverback cluster vars new`][silverback-cluster-vars-new] command and give it a name and a set of related variables. +For instance, it may make sense to make a group of variables for your favorite Ape plugins or services, such as RPC Providers, Blockchain Data Indexers, Etherscan, etc. +You might have a database connection that you want all your bots to access. + +```{warning} +All environment variables in Silverback Clusters are private, meaning they cannot be viewed after they are uploaded. +However, your Bots will have full access to their values from within their runtime environment, so be careful that you fully understand what you are sharing with your bots. + +Also, understand your build dependencies within your container and make sure you are not using any vulnerable or malicious packages. + +**NEVER** upload your private key in a plaintext format! + +Use _Ape Account Plugins_ such as [`ape-aws`](https://github.com/ApeWorX/ape-aws) to safely manage access to your hosted keys. +``` + +```{note} +The Etherscan plugin _will not function_ without an API key in the cloud environment. +This will likely create errors running your applications if you use Ape's `Contract` class. +``` + +To list your Variable Groups, use [`silverback cluster vars list`][silverback-cluster-vars-list]. +To see information about a specific Variable Group, including the Environment Variables it includes, use [`silverback cluster vars info`][silverback-cluster-vars-info] +To remove a variable group, use [`silverback cluster vars remove`][silverback-cluster-vars-remove], + +```{note} +You can only remove a Variable Group if it is not referenced by any existing Bot. +``` + +Once you have created all the Variable Group(s) that you need to operate your Bot, you can reference these groups by name when adding your Bot to the cluster. + +[silverback-cluster-vars]: ../commands/cluster.html#silverback-cluster-vars +[silverback-cluster-vars-new]: ../commands/cluster.html#silverback-cluster-vars-new +[silverback-cluster-vars-info]: ../commands/cluster.html#silverback-cluster-vars-info +[silverback-cluster-vars-list]: ../commands/cluster.html#silverback-cluster-vars-list +[silverback-cluster-vars-remove]: ../commands/cluster.html#silverback-cluster-vars-remove + +## Deploying your Bot + +You are finally ready to deploy your bot on the Cluster and get it running! + +To deploy your Bot, use the [`silverback cluster bots new`][silverback-cluster-bots-new] command and give your bot a name, +container image, network to run on, an account alias (if you want to sign transactions w/ `app.signer`), +and any environment Variable Group(s) the bot needs. +If everything validates successfully, the Cluster will begin orchestrating your deployment for you. + +You should monitor the deployment and startup of your bot to make sure it enters the RUNNING state successfully. +You can do this using the [`silverback cluster bots health`][silverback-cluster-bots-health] command. + +```{note} +It usually takes a minute or so for your bot to transition from PROVISIONING to STARTUP to the RUNNING state. +If there are any difficulties in downloading your container image, provisioning your desired infrastructure, or if your application encounters an error during the STARTUP phase, +the Bot will not enter into the RUNNING state and will be shut down gracefully into the STOPPED state. + +Once in the STOPPED state, you can make any adjustments to the environment Variable Group(s) or other runtime parameters in the Bot config; +or, you can make code changes and deploy a new image for the Bot to use. +Once ready, you can use the `silverback cluster bots start` command to re-start your Bot. +``` + +If at any time you want to view the configuration of your bot, you can do so using the [`silverback cluster bots info`][silverback-cluster-bots-info] command. +You can also update metadata or configuration of your bot using the [`silverback cluster bots update`][silverback-cluster-bots-update] command. +Lastly, if you want to shutdown and delete your bot, you can do so using the [`silverback cluster bots remove`][silverback-cluster-bots-remove] command. + +```{note} +Configuration updates do not redeploy your Bots automatically, you must manually stop and restart your bots for changes to take effect. +``` + +```{warning} +Removing a Bot will immediately trigger a SHUTDOWN if the Bot is not already STOPPED. +``` + +[silverback-cluster-bots-new]: ../commands/cluster.html#silverback-cluster-bots-new +[silverback-cluster-bots-info]: ../commands/cluster.html#silverback-cluster-bots-info +[silverback-cluster-bots-update]: ../commands/cluster.html#silverback-cluster-bots-update +[silverback-cluster-bots-remove]: ../commands/cluster.html#silverback-cluster-bots-remove + +[silverback-cluster-bots-health]: ../commands/cluster.html#silverback-cluster-bots-health + +## Monitoring your Bot + +Once your bot is successfully running in the RUNNING state, you can monitor your bot with a series of commands +under the [`silverback cluster bots`][silverback-cluster-bots] subcommand group. +We already saw how you can use the [`silverback cluster bots list`][silverback-cluster-bots-list] command to see all bots managed by your Cluster (running or not). + +To see runtime health information about a specific bot, again use the [`silverback cluster bots health`][silverback-cluster-bots-health] command. +You can view the logs that a specific bot is generating using the [`silverback cluster bots logs`][silverback-cluster-bots-logs] command. +Lastly, you can view unacknowledged errors that your bot has experienced while in the RUNNING state +using the [`silverback cluster bots errors`][silverback-cluster-bots-errors] command. + +```{warning} +Once in the RUNNING state, your Bot will not stop running unless it experiences a certain amount of errors in quick succession. +Any task execution that experiences an error will abort execution (and therefore not produce any metrics) but the Bot **will not** shutdown. + +All errors encountered during task exeuction are reported to the Cluster for later review by any users with appriopiate access. +Tasks do not retry (by default), but updates to `app.state` are maintained up until the point an error occurs. + +It is important to keep track of these errors and ensure that none of them are in fact critical to the operation of your Application, +and to take corrective or preventative action if it is determined that it should be treated as a more critical failure condition. +``` + +```{note} +Your Bots can also be monitored from the Platform UI at [https://silverback.apeworx.io](https://silverback.apeworx.io). +``` + +[silverback-cluster-bots-logs]: ../commands/cluster.html#silverback-cluster-bots-logs +[silverback-cluster-bots-errors]: ../commands/cluster.html#silverback-cluster-bots-errors + +## Controlling your Bot + +As we already saw, once a Bot is configured in a Cluster, we can control it using commands from the [`silverback cluster bots`][silverback-cluster-bots] subcommand group. +For example, we can attempt to start a Bot that is not currently running (after making configuration or code changes) +using the [`silverback cluster bots start`][silverback-cluster-bots-start] command. +We can also stop a bot using [`silverback cluster bots stop`][silverback-cluster-bots-stop] that is currently in the RUNNING state if we desire. + +```{note} +Controlling your bots can be done from the Platform UI at [https://silverback.apeworx.io](https://silverback.apeworx.io), if you have the right permissions to do so. +``` + +TODO: Updating runtime parameters + +[silverback-cluster-bots]: ../commands/cluster.html#silverback-cluster-bots +[silverback-cluster-bots-start]: ../commands/cluster.html#silverback-cluster-bots-start +[silverback-cluster-bots-stop]: ../commands/cluster.html#silverback-cluster-bots-stop + +## Viewing Measured Metrics + +TODO: Downloading metrics from your Bot From 2f5222f813fafd96cb8e2c1cb5b7ca774bf49ae0 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:19:02 -0400 Subject: [PATCH 34/54] docs(cli): add some metavars and clean up some notes --- silverback/_cli.py | 7 +++++-- silverback/_click_ext.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 947a57aa..2791f0e6 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -78,12 +78,14 @@ def _network_callback(ctx, param, val): @click.option( "--runner", "runner_class", + metavar="CLASS_REF", help="An import str in format ':'", callback=cls_import_callback, ) @click.option( "--recorder", "recorder_class", + metavar="CLASS_REF", help="An import string in format ':'", callback=cls_import_callback, ) @@ -199,13 +201,14 @@ def list_clusters(client: PlatformClient, workspace: str): "-s", "--slug", "cluster_slug", - help="Slug for new cluster (Defaults to name.lower())", + help="Slug for new cluster (Defaults to `name.lower()`)", ) @click.option( "-t", "--tier", default=ClusterTier.PERSONAL.name, - help="Named set of options to use for cluster (Defaults to PERSONAL)", + metavar="NAME", + help="Named set of options to use for cluster as a base (Defaults to Personal)", ) @click.option( "-c", diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index d3cde6e9..85fa51b8 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -105,7 +105,7 @@ def __init__(self, *args, **kwargs): metavar="PROFILE", default=DEFAULT_PROFILE, callback=self.get_profile, - help="The profile to use to connect with the cluster (Advanced)", + help="The authentication profile to use (Advanced)", ) ) From ba2678cedc286cc2b6487567c059bab1cf1d2b8a Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:19:28 -0400 Subject: [PATCH 35/54] docs: update development docs, add link to platform userguide --- docs/userguides/development.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/userguides/development.md b/docs/userguides/development.md index a62514d2..4b8f9c63 100644 --- a/docs/userguides/development.md +++ b/docs/userguides/development.md @@ -1,4 +1,4 @@ -# Developing a Silverback Application +# Developing Applications In this guide, we are going to show you more details on how to build an application with Silverback. @@ -182,12 +182,12 @@ export SILVERBACK_RESULT_BACKEND_URI="redis://127.0.0.1:6379" silverback worker -w 2 "example:app" ``` -This will run one client and 2 workers and all queue data will be go through Redis. +The client will send tasks to the 2 worker subprocesses, and all task queue and results data will be go through Redis. ## Testing your Application TODO: Add backtesting mode w/ `silverback test` -## Deploying to the Silverback Platform +## Deploying your Application -TODO: Add packaging and deployment to the Silverback platform, once available. +Check out the [Platform Deployment Userguide](./platform.html) for more information on how to deploy your application to the [Silverback Platform](https://silverback.apeworx.io). From f0048c7ea0ce2c541e20cec94708960db8b27e69 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:20:55 -0400 Subject: [PATCH 36/54] fix(cli): Make sure that all platform/cluster/auth commands work --- silverback/_cli.py | 150 ++++++++++++++++-------------- silverback/_click_ext.py | 192 ++++++++++++++++----------------------- 2 files changed, 161 insertions(+), 181 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 2791f0e6..58d49585 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -13,11 +13,12 @@ from fief_client.integrations.cli import FiefAuth from silverback._click_ext import ( - AuthCommand, - PlatformGroup, SectionedHelpGroup, + auth_required, cls_import_callback, + cluster_client, display_login_message, + platform_client, ) from silverback._importer import import_from_string from silverback.cluster.client import ClusterClient, PlatformClient @@ -132,7 +133,8 @@ def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): asyncio.run(run_worker(app.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) -@cli.command(cls=AuthCommand, section="Cloud Commands (https://silverback.apeworx.io)") +@cli.command(section="Cloud Commands (https://silverback.apeworx.io)") +@auth_required def login(auth: FiefAuth): """Login to ApeWorX Authorization Service (https://account.apeworx.io)""" @@ -140,7 +142,7 @@ def login(auth: FiefAuth): display_login_message(auth, auth.client.base_url) -@cli.group(cls=PlatformGroup, section="Cloud Commands (https://silverback.apeworx.io)") +@cli.group(cls=SectionedHelpGroup, section="Cloud Commands (https://silverback.apeworx.io)") def cluster(): """Manage a Silverback hosted application cluster @@ -148,14 +150,12 @@ def cluster(): your platform account via `-c WORKSPACE/NAME`""" -@cluster.command( - section="Platform Commands (https://silverback.apeworx.io)", - disable_cluster_option=True, -) -def workspaces(client: PlatformClient): +@cluster.command(section="Platform Commands (https://silverback.apeworx.io)") +@platform_client +def workspaces(platform: PlatformClient): """List available workspaces for your account""" - if workspace_names := list(client.workspaces): + if workspace_names := list(platform.workspaces): click.echo(yaml.safe_dump(workspace_names)) else: @@ -167,16 +167,13 @@ def workspaces(client: PlatformClient): ) -@cluster.command( - name="list", - section="Platform Commands (https://silverback.apeworx.io)", - disable_cluster_option=True, -) +@cluster.command(name="list", section="Platform Commands (https://silverback.apeworx.io)") @click.argument("workspace") -def list_clusters(client: PlatformClient, workspace: str): +@platform_client +def list_clusters(platform: PlatformClient, workspace: str): """List available clusters in a WORKSPACE""" - if not (workspace_client := client.workspaces.get(workspace)): + if not (workspace_client := platform.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") if cluster_names := list(workspace_client.clusters): @@ -186,11 +183,7 @@ def list_clusters(client: PlatformClient, workspace: str): click.secho("No clusters for this account", bold=True, fg="red") -@cluster.command( - name="new", - section="Platform Commands (https://silverback.apeworx.io)", - disable_cluster_option=True, -) +@cluster.command(name="new", section="Platform Commands (https://silverback.apeworx.io)") @click.option( "-n", "--name", @@ -208,6 +201,7 @@ def list_clusters(client: PlatformClient, workspace: str): "--tier", default=ClusterTier.PERSONAL.name, metavar="NAME", + callback=lambda tier: getattr(ClusterTier, tier.upper()), help="Named set of options to use for cluster as a base (Defaults to Personal)", ) @click.option( @@ -219,20 +213,21 @@ def list_clusters(client: PlatformClient, workspace: str): help="Config options to set for cluster (overrides value of -t/--tier)", ) @click.argument("workspace") +@platform_client def new_cluster( - client: PlatformClient, + platform: PlatformClient, workspace: str, cluster_name: str | None, cluster_slug: str | None, - tier: str, + tier: ClusterTier, config_updates: list[tuple[str, str]], ): """Create a new cluster in WORKSPACE""" - if not (workspace_client := client.workspaces.get(workspace)): + if not (workspace_client := platform.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") - configuration = getattr(ClusterTier, tier.upper()).configuration() + configuration = tier.configuration() for k, v in config_updates: setattr(configuration, k, int(v) if v.isnumeric() else v) @@ -253,13 +248,14 @@ def new_cluster( @cluster.command(name="info") -def cluster_info(client: ClusterClient): +@cluster_client +def cluster_info(cluster: ClusterClient): """Get Configuration information about a CLUSTER""" # NOTE: This actually doesn't query the cluster's routes, which are protected - click.echo(f"Cluster Version: v{client.version}") + click.echo(f"Cluster Version: v{cluster.version}") - if config := client.state.configuration: + if config := cluster.state.configuration: click.echo(yaml.safe_dump(config.model_dump())) else: @@ -267,13 +263,14 @@ def cluster_info(client: ClusterClient): @cluster.command(name="health") -def cluster_health(client: ClusterClient): +@cluster_client +def cluster_health(cluster: ClusterClient): """Get Health information about a CLUSTER""" - click.echo(yaml.safe_dump(client.health.model_dump())) + click.echo(yaml.safe_dump(cluster.health.model_dump())) -@cluster.group() +@cluster.group(cls=SectionedHelpGroup) def vars(): """Manage groups of environment variables in a CLUSTER""" @@ -300,21 +297,23 @@ def parse_envar(item: str): help="Environment variable key and value to add (Multiple allowed)", ) @click.argument("name") -def new_vargroup(client: ClusterClient, variables: dict, name: str): +@cluster_client +def new_vargroup(cluster: ClusterClient, variables: dict, name: str): """Create a new group of environment variables in a CLUSTER""" if len(variables) == 0: raise click.UsageError("Must supply at least one var via `-e`") - vg = client.new_variable_group(name=name, variables=variables) + vg = cluster.new_variable_group(name=name, variables=variables) click.echo(yaml.safe_dump(vg.model_dump(exclude={"id"}))) # NOTE: Skip machine `.id` @vars.command(name="list") -def list_vargroups(client: ClusterClient): +@cluster_client +def list_vargroups(cluster: ClusterClient): """List latest revisions of all variable groups in a CLUSTER""" - if group_names := list(client.variable_groups): + if group_names := list(cluster.variable_groups): click.echo(yaml.safe_dump(group_names)) else: @@ -323,10 +322,11 @@ def list_vargroups(client: ClusterClient): @vars.command(name="info") @click.argument("name") -def vargroup_info(client: ClusterClient, name: str): +@cluster_client +def vargroup_info(cluster: ClusterClient, name: str): """Show latest revision of a variable GROUP in a CLUSTER""" - if not (vg := client.variable_groups.get(name)): + if not (vg := cluster.variable_groups.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") click.echo(yaml.safe_dump(vg.model_dump(exclude={"id", "name"}))) @@ -354,8 +354,9 @@ def vargroup_info(client: ClusterClient, name: str): help="Environment variable name to delete (Multiple allowed)", ) @click.argument("name") +@cluster_client def update_vargroup( - client: ClusterClient, + cluster: ClusterClient, name: str, new_name: str, updated_vars: dict[str, str], @@ -366,7 +367,7 @@ def update_vargroup( NOTE: Changing the values of variables in GROUP by create a new revision, since variable groups are immutable. New revisions do not automatically update bot configuration.""" - if not (vg := client.variable_groups.get(name)): + if not (vg := cluster.variable_groups.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") if dup := "', '".join(set(updated_vars) & set(deleted_vars)): @@ -390,20 +391,21 @@ def update_vargroup( @vars.command(name="remove") @click.argument("name") -def remove_vargroup(client: ClusterClient, name: str): +@cluster_client +def remove_vargroup(cluster: ClusterClient, name: str): """ Remove a variable GROUP from a CLUSTER NOTE: Cannot delete if any bots reference any revision of GROUP """ - if not (vg := client.variable_groups.get(name)): + if not (vg := cluster.variable_groups.get(name)): raise click.UsageError(f"Unknown Variable Group '{name}'") vg.remove() # NOTE: No confirmation because can only delete if no references exist click.secho(f"Variable Group '{vg.name}' removed.", fg="green", bold=True) -@cluster.group() +@cluster.group(cls=SectionedHelpGroup) def bots(): """Manage bots in a CLUSTER""" @@ -414,8 +416,9 @@ def bots(): @click.option("-a", "--account") @click.option("-g", "--group", "groups", multiple=True) @click.argument("name") +@cluster_client def new_bot( - client: ClusterClient, + cluster: ClusterClient, image: str, network: str, account: str | None, @@ -424,17 +427,17 @@ def new_bot( ): """Create a new bot in a CLUSTER with the given configuration""" - if name in client.bots: + if name in cluster.bots: raise click.UsageError(f"Cannot use name '{name}' to create bot") environment = list() for vg_id in groups: if "/" in vg_id: vg_name, revision = vg_id.split("/") - vg = client.variable_groups[vg_name].get_revision(int(revision)) + vg = cluster.variable_groups[vg_name].get_revision(int(revision)) else: - vg = client.variable_groups[vg_id] + vg = cluster.variable_groups[vg_id] environment.append(vg) @@ -446,15 +449,16 @@ def new_bot( click.echo(yaml.safe_dump([var for vg in environment for var in vg.variables])) if not click.confirm("Do you want to create this bot?"): - bot = client.new_bot(name, image, network, account=account, environment=environment) + bot = cluster.new_bot(name, image, network, account=account, environment=environment) click.secho(f"Bot '{bot.name}' ({bot.id}) deploying...", fg="green", bold=True) @bots.command(name="list", section="Configuration Commands") -def list_bots(client: ClusterClient): +@cluster_client +def list_bots(cluster: ClusterClient): """List all bots in a CLUSTER (Regardless of status)""" - if bot_names := list(client.bots): + if bot_names := list(cluster.bots): click.echo(yaml.safe_dump(bot_names)) else: @@ -463,10 +467,11 @@ def list_bots(client: ClusterClient): @bots.command(name="info", section="Configuration Commands") @click.argument("bot_name", metavar="BOT") -def bot_info(client: ClusterClient, bot_name: str): +@cluster_client +def bot_info(cluster: ClusterClient, bot_name: str): """Get configuration information of a BOT in a CLUSTER""" - if not (bot := client.bots.get(bot_name)): + if not (bot := cluster.bots.get(bot_name)): raise click.UsageError(f"Unknown bot '{bot_name}'.") # NOTE: Skip machine `.id`, and we already know it is `.name` @@ -483,8 +488,9 @@ def bot_info(client: ClusterClient, bot_name: str): @click.option("-a", "--account") @click.option("-g", "--group", "groups", multiple=True) @click.argument("name", metavar="BOT") +@cluster_client def update_bot( - client: ClusterClient, + cluster: ClusterClient, new_name: str | None, image: str | None, network: str | None, @@ -496,10 +502,10 @@ def update_bot( NOTE: Some configuration updates will trigger a redeploy""" - if new_name in client.bots: + if new_name in cluster.bots: raise click.UsageError(f"Cannot use name '{new_name}' to update bot '{name}'") - if not (bot := client.bots.get(name)): + if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") if new_name: @@ -517,10 +523,10 @@ def update_bot( for vg_id in groups: if "/" in vg_id: vg_name, revision = vg_id.split("/") - vg = client.variable_groups[vg_name].get_revision(int(revision)) + vg = cluster.variable_groups[vg_name].get_revision(int(revision)) else: - vg = client.variable_groups[vg_id] + vg = cluster.variable_groups[vg_id] environment.append(vg) @@ -558,10 +564,11 @@ def update_bot( @bots.command(name="remove", section="Configuration Commands") @click.argument("name", metavar="BOT") -def remove_bot(client: ClusterClient, name: str): +@cluster_client +def remove_bot(cluster: ClusterClient, name: str): """Remove BOT from CLUSTER (Shutdown if running)""" - if not (bot := client.bots.get(name)): + if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") elif click.confirm(f"Do you want to shutdown and delete '{name}'?"): @@ -571,10 +578,11 @@ def remove_bot(client: ClusterClient, name: str): @bots.command(name="health", section="Bot Operation Commands") @click.argument("bot_name", metavar="BOT") -def bot_health(client: ClusterClient, bot_name: str): +@cluster_client +def bot_health(cluster: ClusterClient, bot_name: str): """Show current health of BOT in a CLUSTER""" - if not (bot := client.bots.get(bot_name)): + if not (bot := cluster.bots.get(bot_name)): raise click.UsageError(f"Unknown bot '{bot_name}'.") click.echo(yaml.safe_dump(bot.health.model_dump(exclude={"bot_id"}))) @@ -582,10 +590,11 @@ def bot_health(client: ClusterClient, bot_name: str): @bots.command(name="start", section="Bot Operation Commands") @click.argument("name", metavar="BOT") -def start_bot(client: ClusterClient, name: str): +@cluster_client +def start_bot(cluster: ClusterClient, name: str): """Start BOT running in CLUSTER (if stopped or terminated)""" - if not (bot := client.bots.get(name)): + if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") elif click.confirm(f"Do you want to start running '{name}'?"): @@ -595,10 +604,11 @@ def start_bot(client: ClusterClient, name: str): @bots.command(name="stop", section="Bot Operation Commands") @click.argument("name", metavar="BOT") -def stop_bot(client: ClusterClient, name: str): +@cluster_client +def stop_bot(cluster: ClusterClient, name: str): """Stop BOT from running in CLUSTER (if running)""" - if not (bot := client.bots.get(name)): + if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") elif click.confirm(f"Do you want to stop '{name}' from running?"): @@ -608,10 +618,11 @@ def stop_bot(client: ClusterClient, name: str): @bots.command(name="logs", section="Bot Operation Commands") @click.argument("name", metavar="BOT") -def show_bot_logs(client: ClusterClient, name: str): +@cluster_client +def show_bot_logs(cluster: ClusterClient, name: str): """Show runtime logs for BOT in CLUSTER""" - if not (bot := client.bots.get(name)): + if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") for log in bot.logs: @@ -620,10 +631,11 @@ def show_bot_logs(client: ClusterClient, name: str): @bots.command(name="errors", section="Bot Operation Commands") @click.argument("name", metavar="BOT") -def show_bot_errors(client: ClusterClient, name: str): +@cluster_client +def show_bot_errors(cluster: ClusterClient, name: str): """Show unacknowledged errors for BOT in CLUSTER""" - if not (bot := client.bots.get(name)): + if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") for log in bot.errors: diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index 85fa51b8..79d15f5e 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -1,3 +1,5 @@ +from functools import update_wrapper + import click from fief_client import Fief from fief_client.integrations.cli import FiefAuth, FiefAuthNotAuthenticatedError @@ -90,152 +92,118 @@ def display_login_message(auth: FiefAuth, host: str): ) -class AuthCommand(click.Command): - # NOTE: ClassVar for any command to access - profile: ClusterProfile | PlatformProfile - auth: FiefAuth | None - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.params.append( - click.Option( - param_decls=("-p", "--profile", "profile"), - expose_value=False, - metavar="PROFILE", - default=DEFAULT_PROFILE, - callback=self.get_profile, - help="The authentication profile to use (Advanced)", - ) - ) - - def get_profile(self, ctx, param, value) -> BaseProfile: +def profile_option(f): + expose_value = "profile" in f.__annotations__ + def get_profile(ctx: click.Context, param, value) -> BaseProfile: if not (profile := settings.profile.get(value)): raise click.BadOptionUsage(option_name=param, message=f"Unknown profile '{value}'.") - self.profile = profile - self.auth = self.get_auth(profile) + # Add it to context in case we need it elsewhere + ctx.obj = ctx.obj or {} + ctx.obj["profile"] = profile return profile - def get_auth(self, profile: BaseProfile) -> FiefAuth | None: - if not isinstance(profile, PlatformProfile): - return None - - auth_info = settings.auth[profile.auth] - fief = Fief(auth_info.host, auth_info.client_id) - return FiefAuth(fief, str(PROFILE_PATH.parent / f"{profile.auth}.json")) - - def invoke(self, ctx: click.Context): - callback_params = self.callback.__annotations__ if self.callback else {} - - # HACK: Click commands will fail otherwise if something is in context - # the callback doesn't expect, so delete these: - if "profile" not in callback_params and "profile" in ctx.params: - del ctx.params["profile"] + opt = click.option( + "-p", + "--profile", + "profile", + metavar="PROFILE", + default=DEFAULT_PROFILE, + callback=get_profile, + expose_value=expose_value, + help="The authentication profile to use (Advanced)", + ) + return opt(f) - if "auth" not in callback_params and "auth" in ctx.params: - del ctx.params["auth"] - return super().invoke(ctx) +def auth_required(f): + expose_value = "auth" in f.__annotations__ + @profile_option + @click.pass_context + def add_auth(ctx: click.Context, *args, **kwargs): + profile: BaseProfile = ctx.obj["profile"] -class ClientCommand(AuthCommand): - workspace_name: str | None = None - cluster_name: str | None = None + if isinstance(profile, PlatformProfile): + auth_info = settings.auth[profile.auth] + fief = Fief(auth_info.host, auth_info.client_id) + ctx.obj["auth"] = FiefAuth(fief, str(PROFILE_PATH.parent / f"{profile.auth}.json")) - def __init__(self, *args, disable_cluster_option: bool = False, **kwargs): - super().__init__(*args, **kwargs) + if expose_value: + kwargs["auth"] = ctx.obj.get("auth") - if not disable_cluster_option: - self.params.append( - click.Option( - param_decls=( - "-c", - "--cluster", - ), - metavar="WORKSPACE/NAME", - expose_value=False, - callback=self.get_cluster_path, - help="[Platform Only] NAME of the cluster in the WORKSPACE you wish to access", - ) - ) + return ctx.invoke(f, *args, **kwargs) - def get_cluster_path(self, ctx, param, value) -> str | None: - if isinstance(self.profile, PlatformProfile): - if not value: - return value + return update_wrapper(add_auth, f) - elif "/" not in value or len(parts := value.split("/")) > 2: - raise click.BadParameter("CLUSTER should be in format `WORKSPACE/CLUSTER-NAME`") - self.workspace_name, self.cluster_name = parts +def platform_client(f): + expose_value = "platform" in f.__annotations__ - elif self.profile and value: - raise click.BadParameter("CLUSTER not needed unless using a platform profile") + @auth_required + @click.pass_context + def get_platform_client(ctx: click.Context, *args, **kwargs): + if not isinstance(profile := ctx.obj["profile"], PlatformProfile): + raise click.UsageError("This command only works with the Silveback Platform") - return value + auth: FiefAuth = ctx.obj["auth"] - def get_platform_client(self, auth: FiefAuth, profile: PlatformProfile) -> PlatformClient: try: display_login_message(auth, profile.host) except FiefAuthNotAuthenticatedError as e: raise click.UsageError("Not authenticated, please use `silverback login` first.") from e - return PlatformClient( + ctx.obj["platform"] = PlatformClient( base_url=profile.host, cookies=dict(session=auth.access_token_info()["access_token"]), ) - def invoke(self, ctx: click.Context): - callback_params = self.callback.__annotations__ if self.callback else {} - - if "client" in callback_params: - client_type_needed = callback_params.get("client") + if expose_value: + kwargs["platform"] = ctx.obj["platform"] - if isinstance(self.profile, PlatformProfile): - if not self.auth: - raise click.UsageError( - "This feature is not available outside of the Silverback Platform" - ) + return ctx.invoke(f, *args, **kwargs) - platform_client = self.get_platform_client(self.auth, self.profile) + return update_wrapper(get_platform_client, f) - if client_type_needed == PlatformClient: - ctx.params["client"] = platform_client - elif not self.workspace_name or not self.cluster_name: - raise click.UsageError( - "-c WORKSPACE/NAME should be present when using a Platform profile" - ) +def cluster_client(f): - else: - try: - ctx.params["client"] = platform_client.get_cluster_client( - self.workspace_name, self.cluster_name - ) - except ValueError as e: - raise click.UsageError(str(e)) + def inject_cluster(ctx, param, value: str | None): + if isinstance(ctx.obj["profile"], ClusterProfile): + return value # Ignore processing this for cluster clients - elif not client_type_needed == ClusterClient: - raise click.UsageError("A cluster profile can only directly connect to a cluster.") - - else: - click.echo( - f"{click.style('INFO', fg='blue')}: Logged in to " - f"'{click.style(self.profile.host, bold=True)}' using API Key" - ) - ctx.params["client"] = ClusterClient( - base_url=self.profile.host, - headers={"X-API-Key": self.profile.api_key}, - ) + elif value is None or "/" not in value or len(parts := value.split("/")) > 2: + raise click.BadParameter( + param=param, + message="CLUSTER should be in format `WORKSPACE/NAME`", + ) - return super().invoke(ctx) + ctx.obj["cluster_path"] = parts + return parts + + @click.option( + "-c", + "--cluster", + "cluster_path", + metavar="WORKSPACE/NAME", + expose_value=False, # We don't actually need this exposed + callback=inject_cluster, + help="NAME of the cluster in WORKSPACE you wish to access", + ) + @platform_client + @click.pass_context + def get_cluster_client(ctx: click.Context, *args, **kwargs): + if isinstance(profile := ctx.obj["profile"], ClusterProfile): + kwargs["cluster"] = ClusterClient( + base_url=profile.host, + headers={"X-API-Key": profile.api_key}, + ) + else: # profile is PlatformProfile + platform: PlatformClient = ctx.obj["platform"] + kwargs["cluster"] = platform.get_cluster_client(*ctx.obj["cluster_path"]) -class PlatformGroup(SectionedHelpGroup): + return ctx.invoke(f, *args, **kwargs) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.command_class = ClientCommand - self.group_class = PlatformGroup + return update_wrapper(get_cluster_client, f) From fdc519c498d5f317a942cdb2bcc29738b9fa6535 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:52:13 -0400 Subject: [PATCH 37/54] fix(cli): corner case where it didn't work --- silverback/_click_ext.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index 79d15f5e..69a91dd0 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -145,6 +145,9 @@ def platform_client(f): @click.pass_context def get_platform_client(ctx: click.Context, *args, **kwargs): if not isinstance(profile := ctx.obj["profile"], PlatformProfile): + if not expose_value: + return ctx.invoke(f, *args, **kwargs) + raise click.UsageError("This command only works with the Silveback Platform") auth: FiefAuth = ctx.obj["auth"] From e7221664d4861824fa048f2f03d65d3a9d529867 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:53:06 -0400 Subject: [PATCH 38/54] fix(cli): bad callback, revert --- silverback/_cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 58d49585..b449d9d5 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -201,7 +201,6 @@ def list_clusters(platform: PlatformClient, workspace: str): "--tier", default=ClusterTier.PERSONAL.name, metavar="NAME", - callback=lambda tier: getattr(ClusterTier, tier.upper()), help="Named set of options to use for cluster as a base (Defaults to Personal)", ) @click.option( @@ -219,7 +218,7 @@ def new_cluster( workspace: str, cluster_name: str | None, cluster_slug: str | None, - tier: ClusterTier, + tier: str, config_updates: list[tuple[str, str]], ): """Create a new cluster in WORKSPACE""" @@ -227,7 +226,10 @@ def new_cluster( if not (workspace_client := platform.workspaces.get(workspace)): raise click.BadOptionUsage("workspace", f"Unknown workspace '{workspace}'") - configuration = tier.configuration() + if not hasattr(ClusterTier, tier.upper()): + raise click.BadOptionUsage("tier", f"Invalid choice: {tier}") + + configuration = getattr(ClusterTier, tier.upper()).configuration() for k, v in config_updates: setattr(configuration, k, int(v) if v.isnumeric() else v) From 28504988961ac75e77fba850efe45990483ccf6c Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:57:46 -0400 Subject: [PATCH 39/54] feat(cli): show prettier display of config when creating a new cluster --- silverback/_cli.py | 14 +++++++++++++- silverback/cluster/types.py | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index b449d9d5..5d2f0579 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -234,6 +234,18 @@ def new_cluster( for k, v in config_updates: setattr(configuration, k, int(v) if v.isnumeric() else v) + if cluster_name: + click.echo(f"name: {cluster_name}") + click.echo(f"slug: {cluster_slug or cluster_name.lower().replace(' ', '-')}") + + elif cluster_slug: + click.echo(f"slug: {cluster_slug}") + + click.echo(yaml.safe_dump(dict(configuration=configuration.settings_display_dict()))) + + if not click.confirm("Do you want to make a new cluster with this configuration?"): + return + cluster = workspace_client.create_cluster( cluster_name=cluster_name, cluster_slug=cluster_slug, @@ -258,7 +270,7 @@ def cluster_info(cluster: ClusterClient): click.echo(f"Cluster Version: v{cluster.version}") if config := cluster.state.configuration: - click.echo(yaml.safe_dump(config.model_dump())) + click.echo(yaml.safe_dump(config.settings_display_dict())) else: click.secho("No Cluster Configuration detected", fg="yellow", bold=True) diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index 7d9754e2..fa247bb0 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -87,6 +87,26 @@ def parse_storage_value(cls, value: str | int) -> int: assert units.lower() == "tb" return int(storage) + def settings_display_dict(self) -> dict: + return dict( + version=self.version, + bots=dict( + cpu=f"{256 * 2**self.cpu / 1024} vCPU", + memory=f"{self.memory} GB" if self.memory > 0 else "512 MiB", + ), + general=dict( + bots=self.bots, + secrets=self.secrets, + ), + runner=dict( + networks=self.networks, + triggers=self.triggers, + ), + recorder=dict( + storage=f"{self.storage} TB" if self.storage > 0 else "512 GB", + ), + ) + @staticmethod def _decode_byte(value: int, byte: int) -> int: # NOTE: All configuration settings must be uint8 integer values when encoded From bf465f8e1f1cb6520ab5ebf00a97bae33f8b3fe3 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 21:00:59 -0400 Subject: [PATCH 40/54] style: fixup w/ mdformat --- docs/userguides/platform.md | 52 ++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/docs/userguides/platform.md b/docs/userguides/platform.md index a798768f..d6171cb0 100644 --- a/docs/userguides/platform.md +++ b/docs/userguides/platform.md @@ -10,14 +10,11 @@ Each Cluster is bespoke for an individual or organization, and isolates your app Before we deploy our Application, we have to create a Cluster. If you haven't yet, please sign up for Silverback at [https://silverback.apeworx.io](https://silverback.apeworx.io). - + Once you have signed up, you can actually create (and pay for) your Clusters from the Silverback CLI utility by first logging in to the Platform using [`silverback login`][silverback-login], and then using [`silverback cluster new`][silverback-cluster-new] to follow the steps necessary to deploy it. -[silverback-login]: ../commands/cluster.html#silverback-login -[silverback-cluster-new]: ../commands/cluster.html#silverback-cluster-new - ```{note} The Platform UI will let you create and manage Clusters using a graphical experience, which may be preferred. The CLI experience is for those working locally who don't want to visit the website, or are locally developing their applications. @@ -36,11 +33,6 @@ If you have no bots, we will first have to containerize our Applications and upl Building a container for your application can be an advanced topic, we have included the `silverback build` subcommand to help assist in generating Dockerfiles. ``` -[silverback-cluster]: ../commands/cluster.html#silverback-cluster -[silverback-cluster-info]: ../commands/cluster.html#silverback-cluster-info -[silverback-cluster-health]: ../commands/cluster.html#silverback-cluster-health -[silverback-cluster-bots-list]: ../commands/cluster.html#silverback-cluster-bots-list - ## Building your Bot TODO: Add build process and describe `silverback build --autogen` and `silverback build --upgrade` @@ -87,12 +79,6 @@ You can only remove a Variable Group if it is not referenced by any existing Bot Once you have created all the Variable Group(s) that you need to operate your Bot, you can reference these groups by name when adding your Bot to the cluster. -[silverback-cluster-vars]: ../commands/cluster.html#silverback-cluster-vars -[silverback-cluster-vars-new]: ../commands/cluster.html#silverback-cluster-vars-new -[silverback-cluster-vars-info]: ../commands/cluster.html#silverback-cluster-vars-info -[silverback-cluster-vars-list]: ../commands/cluster.html#silverback-cluster-vars-list -[silverback-cluster-vars-remove]: ../commands/cluster.html#silverback-cluster-vars-remove - ## Deploying your Bot You are finally ready to deploy your bot on the Cluster and get it running! @@ -127,13 +113,6 @@ Configuration updates do not redeploy your Bots automatically, you must manually Removing a Bot will immediately trigger a SHUTDOWN if the Bot is not already STOPPED. ``` -[silverback-cluster-bots-new]: ../commands/cluster.html#silverback-cluster-bots-new -[silverback-cluster-bots-info]: ../commands/cluster.html#silverback-cluster-bots-info -[silverback-cluster-bots-update]: ../commands/cluster.html#silverback-cluster-bots-update -[silverback-cluster-bots-remove]: ../commands/cluster.html#silverback-cluster-bots-remove - -[silverback-cluster-bots-health]: ../commands/cluster.html#silverback-cluster-bots-health - ## Monitoring your Bot Once your bot is successfully running in the RUNNING state, you can monitor your bot with a series of commands @@ -160,9 +139,6 @@ and to take corrective or preventative action if it is determined that it should Your Bots can also be monitored from the Platform UI at [https://silverback.apeworx.io](https://silverback.apeworx.io). ``` -[silverback-cluster-bots-logs]: ../commands/cluster.html#silverback-cluster-bots-logs -[silverback-cluster-bots-errors]: ../commands/cluster.html#silverback-cluster-bots-errors - ## Controlling your Bot As we already saw, once a Bot is configured in a Cluster, we can control it using commands from the [`silverback cluster bots`][silverback-cluster-bots] subcommand group. @@ -176,10 +152,28 @@ Controlling your bots can be done from the Platform UI at [https://silverback.ap TODO: Updating runtime parameters -[silverback-cluster-bots]: ../commands/cluster.html#silverback-cluster-bots -[silverback-cluster-bots-start]: ../commands/cluster.html#silverback-cluster-bots-start -[silverback-cluster-bots-stop]: ../commands/cluster.html#silverback-cluster-bots-stop - ## Viewing Measured Metrics TODO: Downloading metrics from your Bot + +[silverback-cluster]: ../commands/cluster.html#silverback-cluster +[silverback-cluster-bots]: ../commands/cluster.html#silverback-cluster-bots +[silverback-cluster-bots-errors]: ../commands/cluster.html#silverback-cluster-bots-errors +[silverback-cluster-bots-health]: ../commands/cluster.html#silverback-cluster-bots-health +[silverback-cluster-bots-info]: ../commands/cluster.html#silverback-cluster-bots-info +[silverback-cluster-bots-list]: ../commands/cluster.html#silverback-cluster-bots-list +[silverback-cluster-bots-logs]: ../commands/cluster.html#silverback-cluster-bots-logs +[silverback-cluster-bots-new]: ../commands/cluster.html#silverback-cluster-bots-new +[silverback-cluster-bots-remove]: ../commands/cluster.html#silverback-cluster-bots-remove +[silverback-cluster-bots-start]: ../commands/cluster.html#silverback-cluster-bots-start +[silverback-cluster-bots-stop]: ../commands/cluster.html#silverback-cluster-bots-stop +[silverback-cluster-bots-update]: ../commands/cluster.html#silverback-cluster-bots-update +[silverback-cluster-health]: ../commands/cluster.html#silverback-cluster-health +[silverback-cluster-info]: ../commands/cluster.html#silverback-cluster-info +[silverback-cluster-new]: ../commands/cluster.html#silverback-cluster-new +[silverback-cluster-vars]: ../commands/cluster.html#silverback-cluster-vars +[silverback-cluster-vars-info]: ../commands/cluster.html#silverback-cluster-vars-info +[silverback-cluster-vars-list]: ../commands/cluster.html#silverback-cluster-vars-list +[silverback-cluster-vars-new]: ../commands/cluster.html#silverback-cluster-vars-new +[silverback-cluster-vars-remove]: ../commands/cluster.html#silverback-cluster-vars-remove +[silverback-login]: ../commands/cluster.html#silverback-login From 053ac15e9d87681bdeebaa02faf96e37ec23ef02 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Thu, 1 Aug 2024 21:24:41 -0400 Subject: [PATCH 41/54] refactor(cli): groups -> vargroups --- silverback/_cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 5d2f0579..58ee0217 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -428,7 +428,7 @@ def bots(): @click.option("-i", "--image", required=True) @click.option("-n", "--network", required=True) @click.option("-a", "--account") -@click.option("-g", "--group", "groups", multiple=True) +@click.option("-g", "--group", "vargroups", multiple=True) @click.argument("name") @cluster_client def new_bot( @@ -436,7 +436,7 @@ def new_bot( image: str, network: str, account: str | None, - groups: list[str], + vargroups: list[str], name: str, ): """Create a new bot in a CLUSTER with the given configuration""" @@ -445,7 +445,7 @@ def new_bot( raise click.UsageError(f"Cannot use name '{name}' to create bot") environment = list() - for vg_id in groups: + for vg_id in vargroups: if "/" in vg_id: vg_name, revision = vg_id.split("/") vg = cluster.variable_groups[vg_name].get_revision(int(revision)) @@ -500,7 +500,7 @@ def bot_info(cluster: ClusterClient, bot_name: str): @click.option("-i", "--image") @click.option("-n", "--network") @click.option("-a", "--account") -@click.option("-g", "--group", "groups", multiple=True) +@click.option("-g", "--group", "vargroups", multiple=True) @click.argument("name", metavar="BOT") @cluster_client def update_bot( @@ -509,7 +509,7 @@ def update_bot( image: str | None, network: str | None, account: str | None, - groups: list[str], + vargroups: list[str], name: str, ): """Update configuration of BOT in CLUSTER @@ -534,7 +534,7 @@ def update_bot( click.echo(f"Image:\n old: {bot.image}\n new: {image}") environment = list() - for vg_id in groups: + for vg_id in vargroups: if "/" in vg_id: vg_name, revision = vg_id.split("/") vg = cluster.variable_groups[vg_name].get_revision(int(revision)) From cfb6f2ee216900f55f46cc75334cccc5c3cc2787 Mon Sep 17 00:00:00 2001 From: El De-dog-lo <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:08:58 -0400 Subject: [PATCH 42/54] fix: set click object to empty dict if DNE --- silverback/_click_ext.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index 69a91dd0..f6269d7b 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -123,6 +123,7 @@ def auth_required(f): @profile_option @click.pass_context def add_auth(ctx: click.Context, *args, **kwargs): + ctx.obj = ctx.obj or {} profile: BaseProfile = ctx.obj["profile"] if isinstance(profile, PlatformProfile): @@ -144,6 +145,7 @@ def platform_client(f): @auth_required @click.pass_context def get_platform_client(ctx: click.Context, *args, **kwargs): + ctx.obj = ctx.obj or {} if not isinstance(profile := ctx.obj["profile"], PlatformProfile): if not expose_value: return ctx.invoke(f, *args, **kwargs) @@ -173,6 +175,7 @@ def get_platform_client(ctx: click.Context, *args, **kwargs): def cluster_client(f): def inject_cluster(ctx, param, value: str | None): + ctx.obj = ctx.obj or {} if isinstance(ctx.obj["profile"], ClusterProfile): return value # Ignore processing this for cluster clients @@ -197,6 +200,7 @@ def inject_cluster(ctx, param, value: str | None): @platform_client @click.pass_context def get_cluster_client(ctx: click.Context, *args, **kwargs): + ctx.obj = ctx.obj or {} if isinstance(profile := ctx.obj["profile"], ClusterProfile): kwargs["cluster"] = ClusterClient( base_url=profile.host, From efd256307c7f406d96c8b89f233b124c6c5245eb Mon Sep 17 00:00:00 2001 From: El De-dog-lo <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:18:45 -0400 Subject: [PATCH 43/54] fix: if `ctx.obj` is empty, handle appropiately --- silverback/_click_ext.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index f6269d7b..61dce124 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -124,15 +124,15 @@ def auth_required(f): @click.pass_context def add_auth(ctx: click.Context, *args, **kwargs): ctx.obj = ctx.obj or {} - profile: BaseProfile = ctx.obj["profile"] + profile: BaseProfile | None = ctx.obj.get("profile") if isinstance(profile, PlatformProfile): auth_info = settings.auth[profile.auth] fief = Fief(auth_info.host, auth_info.client_id) ctx.obj["auth"] = FiefAuth(fief, str(PROFILE_PATH.parent / f"{profile.auth}.json")) - if expose_value: - kwargs["auth"] = ctx.obj.get("auth") + if expose_value: + kwargs["auth"] = ctx.obj["auth"] return ctx.invoke(f, *args, **kwargs) @@ -146,12 +146,13 @@ def platform_client(f): @click.pass_context def get_platform_client(ctx: click.Context, *args, **kwargs): ctx.obj = ctx.obj or {} - if not isinstance(profile := ctx.obj["profile"], PlatformProfile): + if not isinstance(profile := ctx.obj.get("profile"), PlatformProfile): if not expose_value: return ctx.invoke(f, *args, **kwargs) - raise click.UsageError("This command only works with the Silveback Platform") + raise click.UsageError("This command only works with the Silverback Platform") + # NOTE: `auth` should be set if `profile` is set and is `PlatformProfile` auth: FiefAuth = ctx.obj["auth"] try: @@ -176,7 +177,7 @@ def cluster_client(f): def inject_cluster(ctx, param, value: str | None): ctx.obj = ctx.obj or {} - if isinstance(ctx.obj["profile"], ClusterProfile): + if isinstance(ctx.obj.get("profile"), ClusterProfile): return value # Ignore processing this for cluster clients elif value is None or "/" not in value or len(parts := value.split("/")) > 2: @@ -201,16 +202,19 @@ def inject_cluster(ctx, param, value: str | None): @click.pass_context def get_cluster_client(ctx: click.Context, *args, **kwargs): ctx.obj = ctx.obj or {} - if isinstance(profile := ctx.obj["profile"], ClusterProfile): + if isinstance(profile := ctx.obj.get("profile"), ClusterProfile): kwargs["cluster"] = ClusterClient( base_url=profile.host, headers={"X-API-Key": profile.api_key}, ) - else: # profile is PlatformProfile + elif isinstance(profile, PlatformProfile): platform: PlatformClient = ctx.obj["platform"] kwargs["cluster"] = platform.get_cluster_client(*ctx.obj["cluster_path"]) + else: + raise AssertionError("Profile not set, something wrong") + return ctx.invoke(f, *args, **kwargs) return update_wrapper(get_cluster_client, f) From d4a5d74f56c81b8fb02f08f8600f0915f9e97949 Mon Sep 17 00:00:00 2001 From: El De-dog-lo <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:31:28 -0400 Subject: [PATCH 44/54] refactor(cli): make all confirm conditions exit --- silverback/_cli.py | 61 +++++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 58ee0217..472eeda5 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -462,9 +462,11 @@ def new_bot( click.echo("Environment:") click.echo(yaml.safe_dump([var for vg in environment for var in vg.variables])) - if not click.confirm("Do you want to create this bot?"): - bot = cluster.new_bot(name, image, network, account=account, environment=environment) - click.secho(f"Bot '{bot.name}' ({bot.id}) deploying...", fg="green", bold=True) + if not click.confirm("Do you want to create and start running this bot?"): + return + + bot = cluster.new_bot(name, image, network, account=account, environment=environment) + click.secho(f"Bot '{bot.name}' ({bot.id}) deploying...", fg="green", bold=True) @bots.command(name="list", section="Configuration Commands") @@ -557,23 +559,26 @@ def update_bot( redeploy_required |= set_environment - if click.confirm( + if not click.confirm( f"Do you want to update '{name}'?" if not redeploy_required else f"Do you want to update and redeploy '{name}'?" ): - bot = bot.update( - name=new_name, - image=image, - network=network, - account=account, - environment=environment if set_environment else None, - ) - # NOTE: Skip machine `.id` - click.echo(yaml.safe_dump(bot.model_dump(exclude={"id", "environment"}))) - if bot.environment: - click.echo("environment:") - click.echo(yaml.safe_dump([var.name for var in bot.environment])) + return + + bot = bot.update( + name=new_name, + image=image, + network=network, + account=account, + environment=environment if set_environment else None, + ) + + # NOTE: Skip machine `.id` + click.echo(yaml.safe_dump(bot.model_dump(exclude={"id", "environment"}))) + if bot.environment: + click.echo("environment:") + click.echo(yaml.safe_dump([var.name for var in bot.environment])) @bots.command(name="remove", section="Configuration Commands") @@ -585,9 +590,11 @@ def remove_bot(cluster: ClusterClient, name: str): if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") - elif click.confirm(f"Do you want to shutdown and delete '{name}'?"): - bot.remove() - click.secho(f"Bot '{bot.name}' removed.", fg="green", bold=True) + elif not click.confirm(f"Do you want to shutdown and delete '{name}'?"): + return + + bot.remove() + click.secho(f"Bot '{bot.name}' removed.", fg="green", bold=True) @bots.command(name="health", section="Bot Operation Commands") @@ -611,9 +618,11 @@ def start_bot(cluster: ClusterClient, name: str): if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") - elif click.confirm(f"Do you want to start running '{name}'?"): - bot.start() - click.secho(f"Bot '{bot.name}' starting...", fg="green", bold=True) + elif not click.confirm(f"Do you want to start running '{name}'?"): + return + + bot.start() + click.secho(f"Bot '{bot.name}' starting...", fg="green", bold=True) @bots.command(name="stop", section="Bot Operation Commands") @@ -625,9 +634,11 @@ def stop_bot(cluster: ClusterClient, name: str): if not (bot := cluster.bots.get(name)): raise click.UsageError(f"Unknown bot '{name}'.") - elif click.confirm(f"Do you want to stop '{name}' from running?"): - bot.stop() - click.secho(f"Bot '{bot.name}' stopping...", fg="green", bold=True) + elif not click.confirm(f"Do you want to stop '{name}' from running?"): + return + + bot.stop() + click.secho(f"Bot '{bot.name}' stopping...", fg="green", bold=True) @bots.command(name="logs", section="Bot Operation Commands") From 6e0b081511d01b01c7678a698301bfc5e006f51c Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:42:56 -0400 Subject: [PATCH 45/54] style: black --- silverback/_cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index 472eeda5..a6f42465 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -464,7 +464,7 @@ def new_bot( if not click.confirm("Do you want to create and start running this bot?"): return - + bot = cluster.new_bot(name, image, network, account=account, environment=environment) click.secho(f"Bot '{bot.name}' ({bot.id}) deploying...", fg="green", bold=True) @@ -565,7 +565,7 @@ def update_bot( else f"Do you want to update and redeploy '{name}'?" ): return - + bot = bot.update( name=new_name, image=image, @@ -573,7 +573,7 @@ def update_bot( account=account, environment=environment if set_environment else None, ) - + # NOTE: Skip machine `.id` click.echo(yaml.safe_dump(bot.model_dump(exclude={"id", "environment"}))) if bot.environment: @@ -592,7 +592,7 @@ def remove_bot(cluster: ClusterClient, name: str): elif not click.confirm(f"Do you want to shutdown and delete '{name}'?"): return - + bot.remove() click.secho(f"Bot '{bot.name}' removed.", fg="green", bold=True) @@ -620,7 +620,7 @@ def start_bot(cluster: ClusterClient, name: str): elif not click.confirm(f"Do you want to start running '{name}'?"): return - + bot.start() click.secho(f"Bot '{bot.name}' starting...", fg="green", bold=True) @@ -636,7 +636,7 @@ def stop_bot(cluster: ClusterClient, name: str): elif not click.confirm(f"Do you want to stop '{name}' from running?"): return - + bot.stop() click.secho(f"Bot '{bot.name}' stopping...", fg="green", bold=True) From c8288741a8ca45588a78cf584f6ee7bf6707549d Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:52:49 -0400 Subject: [PATCH 46/54] docs(cli): amend names for `silveback run` and `silverback worker` --- silverback/_cli.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index a6f42465..e91df236 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -65,11 +65,7 @@ def _network_callback(ctx, param, val): return val -@cli.command( - cls=ConnectedProviderCommand, - help="Run local Silverback application", - section="Local Commands", -) +@cli.command(cls=ConnectedProviderCommand, section="Local Commands") @ape_cli_context() @network_option( default=os.environ.get("SILVERBACK_NETWORK_CHOICE", "auto"), @@ -93,6 +89,8 @@ def _network_callback(ctx, param, val): @click.option("-x", "--max-exceptions", type=int, default=3) @click.argument("path") def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): + """Run Silverback application""" + if not runner_class: # NOTE: Automatically select runner class if cli_ctx.provider.ws_uri: @@ -113,11 +111,7 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): asyncio.run(runner.run()) -@cli.command( - cls=ConnectedProviderCommand, - help="Start Silverback distributed task workers (advanced)", - section="Local Commands", -) +@cli.command(cls=ConnectedProviderCommand, section="Local Commands") @ape_cli_context() @network_option( default=os.environ.get("SILVERBACK_NETWORK_CHOICE", "auto"), @@ -129,6 +123,8 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): @click.option("-s", "--shutdown_timeout", type=int, default=90) @click.argument("path") def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): + """Run Silverback task workers (advanced)""" + app = import_from_string(path) asyncio.run(run_worker(app.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) From 5f1bad818c7dc50b9d1850b0b15bede39791edc6 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:00:36 -0400 Subject: [PATCH 47/54] fix(cluster): wrong limit for cpu setting (6 not 16) --- silverback/cluster/types.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/silverback/cluster/types.py b/silverback/cluster/types.py index fa247bb0..428cd371 100644 --- a/silverback/cluster/types.py +++ b/silverback/cluster/types.py @@ -28,11 +28,18 @@ class ClusterConfiguration(BaseModel): version: int = 1 # Bot Worker Configuration (Bytes 1-2) - cpu: Annotated[int, Field(ge=0, le=16)] = 0 # 0.25 vCPU - """Allocated vCPUs per bot: 0.25 vCPU (0) to 16 vCPU (6)""" - - memory: Annotated[int, Field(ge=0, le=120)] = 0 # 512 MiB - """Total memory per bot (in GB)""" + cpu: Annotated[int, Field(ge=0, le=6)] = 0 # defaults to 0.25 vCPU + """Allocated vCPUs per bot: + - 0.25 vCPU (0) + - 0.50 vCPU (1) + - 1.00 vCPU (2) + - 2.00 vCPU (3) + - 4.00 vCPU (4) + - 8.00 vCPU (5) + - 16.0 vCPU (6)""" + + memory: Annotated[int, Field(ge=0, le=120)] = 0 # defaults to 512 MiB + """Total memory per bot (in GB, 0 means '512 MiB')""" # NOTE: Configure # of workers based on cpu & memory settings @@ -48,7 +55,7 @@ class ClusterConfiguration(BaseModel): # Recorder configuration (Byte 6) storage: Annotated[int, Field(ge=0, le=250)] = 0 # 512 GB - """Total task results and metrics parquet storage (in TB)""" + """Total task results and metrics parquet storage (in TB, 0 means '512 GB')""" # Cluster general configuration (Byte 7) secrets: Annotated[int, Field(ge=10, le=100)] = 10 From c3f62f958b33cad9809032902e73f36a10c06ec5 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:06:16 -0400 Subject: [PATCH 48/54] fix(cluster): handle errors w/ openapi fetch --- silverback/cluster/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 73aba9d5..0090c7ba 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -170,7 +170,9 @@ def send(self, request, *args, **kwargs): @property @cache def openapi_schema(self) -> dict: - return self.get("/openapi.json").json() + response = self.get("/openapi.json") + handle_error_with_response(response) + return response.json() @property def version(self) -> str: From f0cab490f9c8ca750b070da50205b694305b5dbe Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:54:51 -0400 Subject: [PATCH 49/54] feat(cli): allow setting default workspaces and clusters per profile --- silverback/_click_ext.py | 28 ++++++++++++++++++++++++++-- silverback/cluster/settings.py | 2 ++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index 61dce124..67d2b1f1 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -1,4 +1,5 @@ from functools import update_wrapper +from pathlib import Path import click from fief_client import Fief @@ -177,10 +178,33 @@ def cluster_client(f): def inject_cluster(ctx, param, value: str | None): ctx.obj = ctx.obj or {} - if isinstance(ctx.obj.get("profile"), ClusterProfile): + if isinstance(profile := ctx.obj.get("profile"), ClusterProfile): return value # Ignore processing this for cluster clients - elif value is None or "/" not in value or len(parts := value.split("/")) > 2: + elif profile is None: + raise AssertionError("Shouldn't happen, fix cli") + + elif value is None or "/" not in value: + if not profile.default_workspace: + raise click.UsageError( + "Must provide `-c CLUSTER`, or set `profile..default-workspace` " + f"in your `~/{PROFILE_PATH.relative_to(Path.home())}`" + ) + + if value is None and profile.default_workspace not in profile.default_cluster: + raise click.UsageError( + "Must provide `-c CLUSTER`, or set " + "`profile..default-cluster.` " + f"in your `~/{PROFILE_PATH.relative_to(Path.home())}`" + ) + + parts = [ + profile.default_workspace, + # NOTE: `value` works as cluster selector, if set + value or profile.default_cluster[profile.default_workspace], + ] + + elif len(parts := value.split("/")) > 2: raise click.BadParameter( param=param, message="CLUSTER should be in format `WORKSPACE/NAME`", diff --git a/silverback/cluster/settings.py b/silverback/cluster/settings.py index 3e4fa21e..73c621ea 100644 --- a/silverback/cluster/settings.py +++ b/silverback/cluster/settings.py @@ -27,6 +27,8 @@ class ClusterProfile(BaseProfile): class PlatformProfile(BaseProfile): auth: str # key of `AuthenticationConfig` in authentication section + default_workspace: str = Field(alias="default-workspace", default="") + default_cluster: dict[str, str] = Field(alias="default-cluster", default_factory=dict) class ProfileSettings(BaseModel): From 46fea600d58a46f72fbaf5bcfeb29061a2efd6b9 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 17:35:53 -0400 Subject: [PATCH 50/54] fix(client): follow redirects w/ cluster --- silverback/cluster/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index 0090c7ba..fdb26769 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -154,6 +154,9 @@ def remove(self): class ClusterClient(httpx.Client): def __init__(self, *args, **kwargs): kwargs["headers"] = {**kwargs.get("headers", {}), **DEFAULT_HEADERS} + if "follow_redirects" not in kwargs: + kwargs["follow_redirects"] = True + super().__init__(*args, **kwargs) # DI for other client classes From 8d299c874a108603537d85325d220a837da0c20d Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 17:36:16 -0400 Subject: [PATCH 51/54] refactor(client): allow getting latest revision for variable group --- silverback/cluster/client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/silverback/cluster/client.py b/silverback/cluster/client.py index fdb26769..fd06ae8e 100644 --- a/silverback/cluster/client.py +++ b/silverback/cluster/client.py @@ -1,5 +1,5 @@ from functools import cache -from typing import ClassVar +from typing import ClassVar, Literal import httpx @@ -77,7 +77,11 @@ def update( return self - def get_revision(self, revision: int) -> VariableGroupInfo: + def get_revision(self, revision: int | Literal["latest"] = "latest") -> VariableGroupInfo: + # TODO: Add `/latest` revision route + if revision == "latest": + revision = "" # type: ignore[assignment] + response = self.cluster.get(f"/variables/{self.id}/{revision}") handle_error_with_response(response) return VariableGroupInfo.model_validate(response.json()) From dd87f9ef3f26c51547beabc0c6cdfe00122838b0 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 17:36:50 -0400 Subject: [PATCH 52/54] fix: remove ability to select old variable group revisions --- silverback/_cli.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/silverback/_cli.py b/silverback/_cli.py index e91df236..b234eaea 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -440,16 +440,7 @@ def new_bot( if name in cluster.bots: raise click.UsageError(f"Cannot use name '{name}' to create bot") - environment = list() - for vg_id in vargroups: - if "/" in vg_id: - vg_name, revision = vg_id.split("/") - vg = cluster.variable_groups[vg_name].get_revision(int(revision)) - - else: - vg = cluster.variable_groups[vg_id] - - environment.append(vg) + environment = [cluster.variable_groups[vg_name].get_revision("latest") for vg_name in vargroups] click.echo(f"Name: {name}") click.echo(f"Image: {image}") @@ -531,16 +522,7 @@ def update_bot( redeploy_required = True click.echo(f"Image:\n old: {bot.image}\n new: {image}") - environment = list() - for vg_id in vargroups: - if "/" in vg_id: - vg_name, revision = vg_id.split("/") - vg = cluster.variable_groups[vg_name].get_revision(int(revision)) - - else: - vg = cluster.variable_groups[vg_id] - - environment.append(vg) + environment = [cluster.variable_groups[vg_name].get_revision("latest") for vg_name in vargroups] set_environment = True From c1df33c5d308f1aeaec0ae3be5e4b7c37dc55e62 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 18:00:54 -0400 Subject: [PATCH 53/54] fix(cli): profile not set in order if `-p` option not provided --- silverback/_click_ext.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index 67d2b1f1..f0b969ed 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -113,6 +113,7 @@ def get_profile(ctx: click.Context, param, value) -> BaseProfile: default=DEFAULT_PROFILE, callback=get_profile, expose_value=expose_value, + is_eager=True, # NOTE: Required to ensure that `profile` is always set, even if not provied help="The authentication profile to use (Advanced)", ) return opt(f) @@ -178,12 +179,12 @@ def cluster_client(f): def inject_cluster(ctx, param, value: str | None): ctx.obj = ctx.obj or {} - if isinstance(profile := ctx.obj.get("profile"), ClusterProfile): - return value # Ignore processing this for cluster clients - - elif profile is None: + if not (profile := ctx.obj.get("profile")): raise AssertionError("Shouldn't happen, fix cli") + elif isinstance(profile, ClusterProfile): + return value # Ignore processing this for cluster clients + elif value is None or "/" not in value: if not profile.default_workspace: raise click.UsageError( From 47d24a53e2638396fa9081827ec4debb5f557228 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Fri, 2 Aug 2024 18:10:19 -0400 Subject: [PATCH 54/54] feat(cli): allow selecting default profile in ~/.silverback/profile.toml --- silverback/_click_ext.py | 3 +-- silverback/cluster/settings.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index f0b969ed..1810ec9d 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -8,7 +8,6 @@ from silverback._importer import import_from_string from silverback.cluster.client import ClusterClient, PlatformClient from silverback.cluster.settings import ( - DEFAULT_PROFILE, PROFILE_PATH, BaseProfile, ClusterProfile, @@ -110,7 +109,7 @@ def get_profile(ctx: click.Context, param, value) -> BaseProfile: "--profile", "profile", metavar="PROFILE", - default=DEFAULT_PROFILE, + default=settings.default_profile, callback=get_profile, expose_value=expose_value, is_eager=True, # NOTE: Required to ensure that `profile` is always set, even if not provied diff --git a/silverback/cluster/settings.py b/silverback/cluster/settings.py index 73c621ea..f1b202aa 100644 --- a/silverback/cluster/settings.py +++ b/silverback/cluster/settings.py @@ -36,6 +36,7 @@ class ProfileSettings(BaseModel): auth: dict[str, AuthenticationConfig] profile: dict[str, PlatformProfile | ClusterProfile] + default_profile: str = Field(default=DEFAULT_PROFILE, alias="default-profile") @model_validator(mode="after") def ensure_auth_exists_for_profile(self) -> Self: