diff --git a/daras_ai_v2/manage_api_keys_widget.py b/daras_ai_v2/manage_api_keys_widget.py index d1a5fd8a4..282fda4d7 100644 --- a/daras_ai_v2/manage_api_keys_widget.py +++ b/daras_ai_v2/manage_api_keys_widget.py @@ -1,6 +1,8 @@ import typing import gooey_gui as gui +from django.contrib.humanize.templatetags.humanize import naturaltime + from api_keys.models import ApiKey from app_users.models import AppUser from daras_ai_v2 import icons @@ -25,28 +27,57 @@ def manage_api_keys(workspace: "Workspace", user: AppUser): api_keys = list(workspace.api_keys.order_by("-created_at")) - table_area = gui.div() + table_area = gui.div( + className="table-responsive text-nowrap container-margin-reset" + ) - if gui.button("＋ Create new secret key"): + if gui.button("＋ Create new API Key"): api_key = generate_new_api_key(workspace=workspace, user=user) api_keys.insert(0, api_key) with table_area: - import pandas as pd - - gui.table( - pd.DataFrame.from_records( - columns=["Secret Key (Preview)", "Created At", "Created By"], - data=[ - ( - api_key.preview, - api_key.created_at.strftime("%B %d, %Y at %I:%M:%S %p %Z"), - api_key.created_by and api_key.created_by.full_name() or "", - ) - for api_key in api_keys - ], - ), - ) + with gui.tag("table", className="table table-striped"): + with gui.tag("thead"), gui.tag("tr"): + with gui.tag("th"): + gui.write("Gooey.AI Key") + with gui.tag("th"): + gui.write("Created At") + with gui.tag("th"): + gui.write("Created By") + gui.tag("th") + with gui.tag("tbody"): + for api_key in api_keys: + with gui.tag("tr"): + with gui.tag("td"): + gui.write(f"`{api_key.preview}`") + with gui.tag("td"): + gui.write(str(naturaltime(api_key.created_at))) + with gui.tag("td"): + gui.write( + api_key.created_by + and api_key.created_by.full_name() + or "" + ) + with gui.tag("td"): + delete_dialog = gui.use_confirm_dialog( + key=f"delete_api_key_{api_key.id}" + ) + gui.button_with_confirm_dialog( + ref=delete_dialog, + trigger_label=icons.delete, + trigger_type="tertiary", + trigger_className="text-danger p-0 m-0", + modal_title="### Delete API Key", + modal_content=f"Are you sure you want to delete `{api_key.preview}`?\n\n" + "API requests made using this key will be rejected, " + "which could cause any systems still depending on it to break. " + "Once deleted, you'll no longer be able to view or modify this API key.", + confirm_label="Delete", + confirm_className="border-danger bg-danger text-white", + ) + if delete_dialog.pressed_confirm: + api_key.delete() + gui.rerun() def generate_new_api_key(workspace: "Workspace", user: AppUser) -> ApiKey: @@ -54,12 +85,11 @@ def generate_new_api_key(workspace: "Workspace", user: AppUser) -> ApiKey: gui.success( """ -##### API key generated - +**API key generated** Please save this secret key somewhere safe and accessible. For security reasons, **you won't be able to view it again** through your account. If you lose this secret key, you'll need to generate a new one. - """ + """ ) col1, col2 = gui.columns([3, 1], responsive=False) with col1: diff --git a/daras_ai_v2/settings.py b/daras_ai_v2/settings.py index ce78485a1..348ce2b14 100644 --- a/daras_ai_v2/settings.py +++ b/daras_ai_v2/settings.py @@ -73,6 +73,7 @@ "functions", "workspaces", "api_keys", + "managed_secrets", ] MIDDLEWARE = [ @@ -424,6 +425,8 @@ AZURE_SPEECH_ENDPOINT = f"https://{AZURE_SPEECH_REGION}.api.cognitive.microsoft.com" AZURE_TTS_ENDPOINT = f"https://{AZURE_SPEECH_REGION}.tts.speech.microsoft.com" +AZURE_KEY_VAULT_ENDPOINT = config("AZURE_KEY_VAULT_ENDPOINT", "") + AZURE_OPENAI_ENDPOINT_CA = config("AZURE_OPENAI_ENDPOINT_CA", "") AZURE_OPENAI_KEY_CA = config("AZURE_OPENAI_KEY_CA", "") AZURE_OPENAI_ENDPOINT_EASTUS2 = config("AZURE_OPENAI_ENDPOINT_EASTUS2", "") diff --git a/functions/executor.js b/functions/executor.js index 884e29c2b..a6ddfcb7b 100644 --- a/functions/executor.js +++ b/functions/executor.js @@ -8,12 +8,17 @@ Deno.serve(async (req) => { return new Response("Unauthorized", { status: 401 }); } - let { tag, code, variables } = await req.json(); + let { tag, code, variables, env } = await req.json(); let { mockConsole, logs } = captureConsole(tag); let status, response; try { - let retval = isolatedEval(mockConsole, code, variables); + let retval = isolatedEval({ + console: mockConsole, + code, + variables, + process: { env }, + }); if (retval instanceof Function) { retval = retval(variables); } @@ -31,7 +36,7 @@ Deno.serve(async (req) => { return new Response(body, { status }); }); -function isolatedEval(console, code, variables) { +function isolatedEval({ console, code, variables, process }) { // Hide global objects let Deno, global, self, globalThis, window; return eval(code); diff --git a/managed_secrets/__init__.py b/managed_secrets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/managed_secrets/admin.py b/managed_secrets/admin.py new file mode 100644 index 000000000..a91beaa9f --- /dev/null +++ b/managed_secrets/admin.py @@ -0,0 +1,18 @@ +from django.contrib import admin + +from managed_secrets.models import ManagedSecret + + +@admin.register(ManagedSecret) +class ManagedSecretAdmin(admin.ModelAdmin): + list_display = [ + "__str__", + "workspace", + "created_at", + "updated_at", + "usage_count", + "last_used_at", + ] + autocomplete_fields = ["workspace", "created_by"] + readonly_fields = ["usage_count", "last_used_at", "external_id"] + search_fields = ["name", "external_id"] diff --git a/managed_secrets/apps.py b/managed_secrets/apps.py new file mode 100644 index 000000000..795161c3b --- /dev/null +++ b/managed_secrets/apps.py @@ -0,0 +1,11 @@ +from django.apps import AppConfig + + +class ManagedSecretsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "managed_secrets" + + def ready(self): + from . import signals + + assert signals diff --git a/managed_secrets/migrations/0001_initial.py b/managed_secrets/migrations/0001_initial.py new file mode 100644 index 000000000..6c54b92ee --- /dev/null +++ b/managed_secrets/migrations/0001_initial.py @@ -0,0 +1,35 @@ +# Generated by Django 5.1.3 on 2024-12-23 15:47 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('app_users', '0023_alter_appusertransaction_workspace'), + ('workspaces', '0006_workspace_description'), + ] + + operations = [ + migrations.CreateModel( + name='ManagedSecret', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('external_id', models.UUIDField(default=uuid.uuid4, editable=False)), + ('name', models.CharField(max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('usage_count', models.PositiveIntegerField(db_index=True, default=0)), + ('last_used_at', models.DateTimeField(blank=True, default=None, null=True)), + ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='app_users.appuser')), + ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='managed_secrets', to='workspaces.workspace')), + ], + options={ + 'unique_together': {('workspace', 'name')}, + }, + ), + ] diff --git a/managed_secrets/migrations/__init__.py b/managed_secrets/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/managed_secrets/models.py b/managed_secrets/models.py new file mode 100644 index 000000000..bf08cd2d6 --- /dev/null +++ b/managed_secrets/models.py @@ -0,0 +1,74 @@ +import uuid + +from django.db import models + +from daras_ai_v2 import settings + +'az role assignment create --role "Key Vault Secrets Officer" --assignee "" --scope "/subscriptions//resourceGroups//providers/Microsoft.KeyVault/vaults/"' + + +class ManagedSecretQuerySet(models.QuerySet): + def create(self, *, value: str, **kwargs): + secret = super().create(**kwargs) + secret.store_value(value) + return secret + + +class ManagedSecret(models.Model): + external_id = models.UUIDField(default=uuid.uuid4, editable=False) + + name = models.CharField(max_length=255) + + workspace = models.ForeignKey( + "workspaces.Workspace", on_delete=models.CASCADE, related_name="managed_secrets" + ) + created_by = models.ForeignKey( + "app_users.AppUser", on_delete=models.SET_NULL, null=True + ) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + usage_count = models.PositiveIntegerField(default=0, db_index=True) + last_used_at = models.DateTimeField(null=True, blank=True, default=None) + + objects: ManagedSecretQuerySet = ManagedSecretQuerySet.as_manager() + + value: str | None = None + + class Meta: + unique_together = ("workspace", "name") + + def __str__(self): + return f"{self.name} ({self._external_name()})" + + def store_value(self, value: str): + client = _get_az_secret_client() + client.set_secret(self._external_name(), value) + self.value = value + + def load_value(self): + client = _get_az_secret_client() + self.value = client.get_secret(self._external_name()).value + + def delete_value(self): + import azure.core.exceptions + + client = _get_az_secret_client() + try: + client.begin_delete_secret(self._external_name()) + except azure.core.exceptions.ResourceNotFoundError: + pass + + def _external_name(self): + return f"ms-gooey-{self.external_id}" + + +def _get_az_secret_client(): + from azure.keyvault.secrets import SecretClient + from azure.identity import DefaultAzureCredential + + credential = DefaultAzureCredential() + return SecretClient( + vault_url=settings.AZURE_KEY_VAULT_ENDPOINT, credential=credential + ) diff --git a/managed_secrets/signals.py b/managed_secrets/signals.py new file mode 100644 index 000000000..f34a56dfe --- /dev/null +++ b/managed_secrets/signals.py @@ -0,0 +1,16 @@ +from django.db.models.signals import post_save, post_delete +from django.dispatch import receiver + +from managed_secrets.models import ManagedSecret + + +@receiver(post_save, sender=ManagedSecret) +def on_secret_saved(instance: ManagedSecret, **kwargs): + if instance.value is None: + return + instance.store_value(instance.value) + + +@receiver(post_delete, sender=ManagedSecret) +def on_secret_deleted(instance: ManagedSecret, **kwargs): + instance.delete_value() diff --git a/managed_secrets/tests.py b/managed_secrets/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/managed_secrets/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/managed_secrets/widgets.py b/managed_secrets/widgets.py new file mode 100644 index 000000000..08b4dd935 --- /dev/null +++ b/managed_secrets/widgets.py @@ -0,0 +1,216 @@ +import typing + +import gooey_gui as gui +from absl.flags import ValidationError +from django.contrib.humanize.templatetags.humanize import naturaltime +from django.db import IntegrityError + +from daras_ai_v2 import icons +from managed_secrets.models import ManagedSecret + +if typing.TYPE_CHECKING: + from workspaces.models import Workspace + from bots.models import AppUser + + +def manage_secrets_table(workspace: "Workspace", user: "AppUser"): + gui.write( + "Secrets are protected environment variables that allow you to share Gooey Workflows with private keys that are hidden from viewers of the workflow." + ) + + secrets = list(workspace.managed_secrets.order_by("-created_at")) + + table_area = gui.div( + className="table-responsive text-nowrap container-margin-reset" + ) + + edit_secret_button_with_dialog( + workspace, + user, + trigger_label=f"{icons.add} Add Secret", + trigger_type="secondary", + ) + + with table_area: + with gui.tag("table", className="table table-striped"): + with gui.tag("thead"), gui.tag("tr"): + with gui.tag("th"): + gui.write("Name") + with gui.tag("th"): + gui.write("Value") + with gui.tag("th"): + gui.write("Created At") + with gui.tag("th"): + gui.write("Created By") + gui.tag("th") + with gui.tag("tbody"): + for secret in secrets: + with gui.tag("tr"): + with gui.tag("td"): + gui.write(f"`{secret.name}`") + with gui.tag("td", className="d-flex gap-3"): + if gui.session_state.pop(f"secret:{secret.id}:show", False): + secret.load_value() + if secret.value: + gui.write(f"`{secret.value}`") + else: + gui.write("`" + "*" * 10 + "`") + gui.button( + '', + type="tertiary", + className="m-0 px-1 py-0", + key=f"secret:{secret.id}:show", + ) + with gui.tag("td"): + gui.write(str(naturaltime(secret.created_at))) + with gui.tag("td"): + gui.write( + secret.created_by + and secret.created_by.full_name() + or "" + ) + with gui.tag("td"): + edit_secret_button_with_dialog( + workspace, + user, + secret=secret, + trigger_label='', + trigger_type="link", + trigger_className="p-0 m-0", + ) + + gui.html('') + + delete_dialog = gui.use_confirm_dialog( + key=f"secrets:{secret.id}:delete" + ) + gui.button_with_confirm_dialog( + ref=delete_dialog, + trigger_label=icons.delete, + trigger_type="tertiary", + trigger_className="text-danger p-0 m-0 ", + modal_title="### Delete Secret", + modal_content=f"Are you sure you want to delete `{secret.name}`?\n\n" + "Functions using this secret may throw errors or silently fail, " + "which could cause any systems still depending on it to break. " + "Once deleted, you'll no longer be able to view or modify this Secret.", + confirm_label="Delete", + confirm_className="border-danger bg-danger text-white", + ) + if delete_dialog.pressed_confirm: + secret.delete() + gui.rerun() + + +def edit_secret_button_with_dialog( + workspace: "Workspace", + user: "AppUser", + *, + trigger_label: str, + trigger_type: str, + trigger_className: str = "", + secret: ManagedSecret | None = None, +): + if secret: + key = f"secret:{secret.id}:edit" + else: + key = "secret:create" + dialog = gui.use_confirm_dialog(key=key, close_on_confirm=False) + + if gui.button( + label=trigger_label, + type=trigger_type, + key=dialog.open_btn_key, + className=trigger_className, + ): + gui.session_state.pop("secret:name", None) + gui.session_state.pop("secret:value", None) + dialog.set_open(True) + if not dialog.is_open: + return + + header, body, footer = gui.modal_scaffold() + + with header: + if secret: + title = f'### Edit {secret.name}' + else: + title = "### Add Secret Value" + gui.write(title, unsafe_allow_html=True) + + with body: + gui.caption( + "The values of protected variables are secret and enable workflow sharing without revealing API keys." + ) + + name = gui.text_input( + label="###### Name", + style=dict(textTransform="uppercase", fontFamily="monospace"), + # language=javascript + onKeyUp="setValue(value.replace(/ /g, '_').replace(/[^a-zA-Z0-9_\$]/g, ''))", + key="secret:name", + value=secret and secret.name, + ) + if name and name[0].isdigit(): + gui.error( + "Secret name must be a valid JS variable name and cannot start with a number." + ) + name = None + if secret: + value_label = "###### New Value" + else: + value_label = "###### Value" + value = gui.text_input(label=value_label, key="secret:value") + + if workspace.is_personal: + visible_to = "you" + else: + visible_to = "your workspace members" + gui.caption( + f"Once entered, secret values are encrypted with your login credentials and only visible to {visible_to}." + ) + gui.text_input( + label="###### Workspace", + value=workspace.display_name(user), + disabled=True, + ) + gui.div(className="mb-4") + + with footer: + gui.button( + label="Cancel", + key=dialog.close_btn_key, + type="tertiary", + ) + gui.button( + label=f"{icons.save} Save", + key=dialog.confirm_btn_key, + type="primary", + disabled=not (name and value), + ) + + if not dialog.pressed_confirm: + return + try: + if secret: + secret.name = name + secret.value = value + secret.full_clean() + secret.save() + dialog.set_open(False) + else: + ManagedSecret.objects.create( + workspace=workspace, + created_by=user, + name=name, + value=value, + ) + except ValidationError as e: + gui.error(e.messages[0], icon="") + except IntegrityError: + gui.error( + f"Secret with name `{name}` already exists. Please choose a different name." + ) + else: + dialog.set_open(False) + raise gui.RedirectException diff --git a/pyproject.toml b/pyproject.toml index a04c13c4c..654a04484 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,8 @@ django-safedelete = "^1.4.0" numexpr = "^2.10.1" django-csp = "^3.8" python-pptx = "^1.0.2" +azure-identity = "^1.19.0" +azure-keyvault-secrets = "^4.9.0" [tool.poetry.group.dev.dependencies] watchdog = "^2.1.9" diff --git a/recipes/Functions.py b/recipes/Functions.py index 8472ddade..59edd6082 100644 --- a/recipes/Functions.py +++ b/recipes/Functions.py @@ -5,12 +5,15 @@ from pydantic import BaseModel, Field from bots.models import Workflow -from daras_ai_v2 import settings +from daras_ai_v2 import settings, icons from daras_ai_v2.base import BasePage -from daras_ai_v2.exceptions import raise_for_status -from daras_ai_v2.field_render import field_title +from daras_ai_v2.exceptions import raise_for_status, UserError +from daras_ai_v2.field_render import field_title, field_desc +from daras_ai_v2.functional import map_parallel from daras_ai_v2.variables_widget import variables_input from functions.models import CalledFunction, VariableSchema +from managed_secrets.models import ManagedSecret +from managed_secrets.widgets import edit_secret_button_with_dialog class ConsoleLogs(BaseModel): @@ -26,7 +29,7 @@ class FunctionsPage(BasePage): price = 1 class RequestModel(BaseModel): - code: str = Field( + code: str | None = Field( None, title="Code", description="The JS code to be executed.", @@ -41,6 +44,13 @@ class RequestModel(BaseModel): title="⌥ Variables Schema", description="Schema for variables to be used in the variables input", ) + secrets: list[str] | None = Field( + None, + title="Secrets", + description="Secrets enable workflow sharing without revealing sensitive environment variables like API keys.\n" + "Use them in your functions from nodejs standard `process.env.SECRET_NAME`\n\n" + "Manage your secrets in the [account keys](/account/api-keys/) section.", + ) class ResponseModel(BaseModel): return_value: typing.Any = Field( @@ -67,12 +77,24 @@ def run_v2( sr = self.current_sr tag = f"run_id={sr.run_id}&uid={sr.uid}" + if request.secrets: + yield "Decrypting secrets..." + secret_values = map_parallel(self._load_secret, request.secrets) + env = dict(zip(request.secrets, secret_values)) + else: + env = None + yield "Running your code..." # this will run functions/executor.js in deno deploy r = requests.post( settings.DENO_FUNCTIONS_URL, headers={"Authorization": f"Basic {settings.DENO_FUNCTIONS_AUTH_TOKEN}"}, - json=dict(code=request.code, variables=request.variables or {}, tag=tag), + json=dict( + code=request.code, + variables=request.variables or {}, + tag=tag, + env=env, + ), ) raise_for_status(r) data = r.json() @@ -80,6 +102,18 @@ def run_v2( response.return_value = data.get("retval") response.error = data.get("error") + def _load_secret(self, name: str) -> str: + try: + secret = ManagedSecret.objects.get( + workspace=self.current_workspace, name=name + ) + except ManagedSecret.DoesNotExist: + raise UserError( + f"Secret `{name}` not found. Please go to your [account keys](/account/api-keys/) section and provide this value." + ) + secret.load_value() + return secret.value + def render_form_v2(self): gui.code_editor( label="##### " + field_title(self.RequestModel, "code"), @@ -103,6 +137,36 @@ def render_variables(self): description="Pass custom parameters to your function and access the parent workflow data. " "Variables will be passed down as the first argument to your anonymous JS function.", ) + with gui.div(className="d-flex align-items-center gap-3 mb-2"): + gui.markdown( + "###### " + + ' ' + + field_title(self.RequestModel, "secrets"), + help=field_desc(self.RequestModel, "secrets"), + unsafe_allow_html=True, + ) + edit_secret_button_with_dialog( + self.current_workspace, + self.request.user, + trigger_label=f"{icons.add} Add", + trigger_type="tertiary", + trigger_className="p-1 mb-2", + ) + + options = list( + set( + self.current_workspace.managed_secrets.order_by( + "-created_at" + ).values_list("name", flat=True) + ) + | set(gui.session_state.get("secrets", [])) + ) + gui.multiselect( + label="", + options=options, + key="secrets", + allow_none=True, + ) def render_output(self): if error := gui.session_state.get("error"): diff --git a/routers/account.py b/routers/account.py index c09c83c67..188ab8932 100644 --- a/routers/account.py +++ b/routers/account.py @@ -19,6 +19,7 @@ from daras_ai_v2.meta_content import raw_build_meta_tags from daras_ai_v2.profiles import edit_user_profile_page from daras_ai_v2.urls import paginate_queryset, paginate_button +from managed_secrets.widgets import manage_secrets_table from payments.webhooks import PaypalWebhookHandler from routers.custom_api_router import CustomAPIRouter from routers.root import explore_page, page_wrapper, get_og_url_path @@ -356,10 +357,14 @@ def _render_run(pr: PublishedRun): def api_keys_tab(request: Request): - gui.write("# 🔐 API Keys") workspace = get_current_workspace(request.user, request.session) + + gui.write("## 🔐 API Keys") manage_api_keys(workspace=workspace, user=request.user) + gui.write("## 🛡 Secrets") + manage_secrets_table(workspace, request.user) + @contextmanager def account_page_wrapper(request: Request, current_tab: TabData):