[Tock Studio] Export and import of ia gen settings (#1760)

* Wip * Rag settings export * Rag settings import * Sentence generation settings export and import * Observability settings export and import * Vector DB settings export and import
theopenconversationkit · Nov 19, 2024 · e7a9428 · e7a9428
1 parent d45d788
commit e7a9428
Show file tree

Hide file tree

Showing 17 changed files with 1,273 additions and 85 deletions.
diff --git a/bot/admin/web/src/app/configuration/observability-settings/models/providers-configuration.ts b/bot/admin/web/src/app/configuration/observability-settings/models/providers-configuration.ts
@@ -9,6 +9,7 @@ export interface ProvidersConfigurationParam {
   source?: string[];
   inputScale?: 'default' | 'fullwidth';
   defaultValue?: string;
+  confirmExport?: boolean;
 }
 
 export interface ProvidersConfiguration {
@@ -23,7 +24,7 @@ export const ProvidersConfigurations: ProvidersConfiguration[] = [
     key: ObservabilityProvider.Langfuse,
     params: [
       { key: 'publicKey', label: 'Public key', type: 'obfuscated' },
-      { key: 'secretKey', label: 'Secret key', type: 'obfuscated' },
+      { key: 'secretKey', label: 'Secret key', type: 'obfuscated', confirmExport: true },
       { key: 'url', label: 'Url', type: 'obfuscated' }
     ]
   }

diff --git a/...in/web/src/app/configuration/observability-settings/observability-settings.component.html b/...in/web/src/app/configuration/observability-settings/observability-settings.component.html
@@ -3,6 +3,26 @@
     <h1 class="flex-grow-1">Observability settings</h1>
 
     <section class="grid-actions">
+      <button
+        [disabled]="!hasExportableData"
+        nbButton
+        ghost
+        shape="round"
+        nbTooltip="Export observability settings dump"
+        (click)="exportSettings()"
+      >
+        <nb-icon icon="download"></nb-icon>
+      </button>
+      <button
+        nbButton
+        ghost
+        shape="round"
+        nbTooltip="Import observability settings dump"
+        (click)="importSettings()"
+      >
+        <nb-icon icon="upload"></nb-icon>
+      </button>
+
       <button
         *ngIf="settingsBackup && form.dirty"
         nbButton
@@ -115,3 +135,116 @@ <h5 class="section-title mt-2">Settings deletion</h5>
     </nb-card-body>
   </nb-card>
 </form>
+
+<ng-template #exportConfirmationModal>
+  <nb-card class="help-modal">
+    <nb-card-header class="d-flex justify-content-between align-items-start gap-1">
+      Confirmation of sensitive data export
+      <button
+        nbButton
+        ghost
+        shape="round"
+        nbTooltip="Cancel"
+        (click)="closeExportConfirmationModal()"
+      >
+        <nb-icon icon="x-lg"></nb-icon>
+      </button>
+    </nb-card-header>
+
+    <nb-card-body>
+      <div class="mb-2">Include the following sensitive data:</div>
+      <div *ngFor="let sensitiveParam of sensitiveParams">
+        <nb-checkbox
+          status="basic"
+          [(ngModel)]="sensitiveParam.include"
+        >
+          {{ sensitiveParam.label }}
+          {{ sensitiveParam.param.label }}
+        </nb-checkbox>
+      </div>
+    </nb-card-body>
+
+    <nb-card-footer class="card-footer-actions">
+      <button
+        nbButton
+        ghost
+        size="small"
+        (click)="closeExportConfirmationModal()"
+      >
+        Cancel
+      </button>
+      <button
+        type="button"
+        nbButton
+        status="primary"
+        size="small"
+        (click)="confirmExportSettings()"
+      >
+        Export
+      </button>
+    </nb-card-footer>
+  </nb-card>
+</ng-template>
+
+<ng-template #importModal>
+  <nb-card class="help-modal">
+    <nb-card-header class="d-flex justify-content-between align-items-start gap-1">
+      Import observability settings dump
+      <button
+        nbButton
+        ghost
+        shape="round"
+        nbTooltip="Cancel"
+        (click)="closeImportModal()"
+      >
+        <nb-icon icon="x-lg"></nb-icon>
+      </button>
+    </nb-card-header>
+
+    <nb-card-body>
+      <form
+        [formGroup]="importForm"
+        (submit)="submitImportSettings()"
+      >
+        <tock-form-control
+          label="Observability settings dump file"
+          name="importFile"
+          [required]="true"
+          [controls]="fileSource"
+          [showError]="isImportSubmitted"
+        >
+          <tock-file-upload
+            id="importFile"
+            formControlName="fileSource"
+            [autofocus]="true"
+            [fullWidth]="true"
+            [multiple]="false"
+            [fileTypeAccepted]="['json']"
+          ></tock-file-upload>
+        </tock-form-control>
+      </form>
+    </nb-card-body>
+
+    <nb-card-footer class="card-footer-actions">
+      <button
+        nbButton
+        ghost
+        size="small"
+        (click)="closeImportModal()"
+      >
+        Cancel
+      </button>
+      <button
+        type="button"
+        nbButton
+        status="primary"
+        size="small"
+        (click)="submitImportSettings()"
+      >
+        Import
+      </button>
+    </nb-card-footer>
+  </nb-card>
+</ng-template>
+
+<tock-scroll-top-button></tock-scroll-top-button>
diff --git a/...in/web/src/app/configuration/observability-settings/observability-settings.component.scss b/...in/web/src/app/configuration/observability-settings/observability-settings.component.scss
@@ -0,0 +1,7 @@
+.grid-actions {
+  display: grid;
+  grid-gap: 0.5rem;
+  grid-auto-flow: column;
+  align-items: center;
+  justify-content: end;
+}
diff --git a/...dmin/web/src/app/configuration/observability-settings/observability-settings.component.ts b/...dmin/web/src/app/configuration/observability-settings/observability-settings.component.ts
@@ -1,15 +1,22 @@
-import { Component, OnDestroy, OnInit } from '@angular/core';
+import { Component, OnDestroy, OnInit, TemplateRef, ViewChild } from '@angular/core';
 import { StateService } from '../../core-nlp/state.service';
 import { RestService } from '../../core-nlp/rest/rest.service';
 import { NbDialogService, NbToastrService, NbWindowService } from '@nebular/theme';
 import { BotConfigurationService } from '../../core/bot-configuration.service';
 import { Observable, Subject, debounceTime, takeUntil } from 'rxjs';
 import { BotApplicationConfiguration } from '../../core/model/configuration';
 import { FormControl, FormGroup, Validators } from '@angular/forms';
-import { ObservabilityProvider, ProvidersConfiguration, ProvidersConfigurations } from './models/providers-configuration';
+import {
+  ObservabilityProvider,
+  ProvidersConfiguration,
+  ProvidersConfigurationParam,
+  ProvidersConfigurations
+} from './models/providers-configuration';
 import { ObservabilitySettings } from './models/observability-settings';
-import { deepCopy } from '../../shared/utils';
+import { deepCopy, getExportFileName, readFileAsText } from '../../shared/utils';
 import { ChoiceDialogComponent, DebugViewerWindowComponent } from '../../shared/components';
+import { saveAs } from 'file-saver-es';
+import { FileValidators } from '../../shared/validators';
 
 interface ObservabilitySettingsForm {
   id: FormControl<string>;
@@ -36,6 +43,9 @@ export class ObservabilitySettingsComponent implements OnInit, OnDestroy {
 
   settingsBackup: ObservabilitySettings;
 
+  @ViewChild('exportConfirmationModal') exportConfirmationModal: TemplateRef<any>;
+  @ViewChild('importModal') importModal: TemplateRef<any>;
+
   constructor(
     private state: StateService,
     private rest: RestService,
@@ -59,9 +69,14 @@ export class ObservabilitySettingsComponent implements OnInit, OnDestroy {
 
     this.botConfiguration.configurations.pipe(takeUntil(this.destroy$)).subscribe((confs: BotApplicationConfiguration[]) => {
       delete this.settingsBackup;
+
+      // Reset form on configuration change
+      this.form.reset();
+      // Reset formGroup control too, if any
+      this.resetFormGroupControls();
+
       this.loading = true;
       this.configurations = confs;
-      this.form.reset();
 
       if (confs.length) {
         this.getObservabilitySettingsLoader().subscribe((res) => {
@@ -136,10 +151,7 @@ export class ObservabilitySettingsComponent implements OnInit, OnDestroy {
 
     if (requiredConfiguration) {
       // Purge existing controls that may contain values incompatible with a new control with the same name if provider change
-      const existingGroupKeys = Object.keys(this.form.controls['setting'].controls);
-      existingGroupKeys.forEach((key) => {
-        this.form.controls['setting'].removeControl(key);
-      });
+      this.resetFormGroupControls();
 
       requiredConfiguration.params.forEach((param) => {
         this.form.controls['setting'].addControl(param.key, new FormControl(param.defaultValue, Validators.required));
@@ -149,6 +161,13 @@ export class ObservabilitySettingsComponent implements OnInit, OnDestroy {
     }
   }
 
+  resetFormGroupControls() {
+    const existingGroupKeys = Object.keys(this.form.controls['setting'].controls);
+    existingGroupKeys.forEach((key) => {
+      this.form.controls['setting'].removeControl(key);
+    });
+  }
+
   cancel(): void {
     this.initForm(this.settingsBackup);
   }
@@ -196,6 +215,144 @@ export class ObservabilitySettingsComponent implements OnInit, OnDestroy {
     }
   }
 
+  get hasExportableData(): boolean {
+    if (this.observabilityProvider.value) return true;
+
+    const formValue: ObservabilitySettings = deepCopy(this.form.value) as unknown as ObservabilitySettings;
+
+    return Object.values(formValue).some((entry) => {
+      return entry && (typeof entry !== 'object' || Object.keys(entry).length !== 0);
+    });
+  }
+
+  sensitiveParams: { label: string; key: string; include: boolean; param: ProvidersConfigurationParam }[];
+
+  exportSettings() {
+    this.sensitiveParams = [];
+
+    const shouldConfirm =
+      this.observabilityProvider.value &&
+      this.currentObservabilityProvider.params.some((entry) => {
+        return entry.confirmExport;
+      });
+
+    if (shouldConfirm) {
+      this.currentObservabilityProvider.params.forEach((entry) => {
+        if (entry.confirmExport) {
+          this.sensitiveParams.push({ label: 'Observability provider', key: 'setting', include: false, param: entry });
+        }
+      });
+
+      this.exportConfirmationModalRef = this.nbDialogService.open(this.exportConfirmationModal);
+    } else {
+      this.downloadSettings();
+    }
+  }
+
+  exportConfirmationModalRef;
+
+  closeExportConfirmationModal() {
+    this.exportConfirmationModalRef.close();
+  }
+
+  confirmExportSettings() {
+    this.downloadSettings();
+    this.closeExportConfirmationModal();
+  }
+
+  downloadSettings() {
+    const formValue: ObservabilitySettings = deepCopy(this.form.value) as unknown as ObservabilitySettings;
+    delete formValue['observabilityProvider'];
+    delete formValue['id'];
+    delete formValue['enabled'];
+
+    if (this.sensitiveParams?.length) {
+      this.sensitiveParams.forEach((sensitiveParam) => {
+        if (!sensitiveParam.include) {
+          delete formValue[sensitiveParam.key][sensitiveParam.param.key];
+        }
+      });
+    }
+
+    const jsonBlob = new Blob([JSON.stringify(formValue)], {
+      type: 'application/json'
+    });
+
+    const exportFileName = getExportFileName(
+      this.state.currentApplication.namespace,
+      this.state.currentApplication.name,
+      'Observability settings',
+      'json'
+    );
+
+    saveAs(jsonBlob, exportFileName);
+
+    this.toastrService.show(`Observability settings dump provided`, 'Observability settings dump', {
+      duration: 3000,
+      status: 'success'
+    });
+  }
+
+  importModalRef;
+
+  importSettings() {
+    this.isImportSubmitted = false;
+    this.importForm.reset();
+    this.importModalRef = this.nbDialogService.open(this.importModal);
+  }
+
+  closeImportModal() {
+    this.importModalRef.close();
+  }
+
+  isImportSubmitted: boolean = false;
+
+  importForm: FormGroup = new FormGroup({
+    fileSource: new FormControl<File[]>([], {
+      nonNullable: true,
+      validators: [Validators.required, FileValidators.mimeTypeSupported(['application/json'])]
+    })
+  });
+
+  get fileSource(): FormControl {
+    return this.importForm.get('fileSource') as FormControl;
+  }
+
+  get canSaveImport(): boolean {
+    return this.isImportSubmitted ? this.importForm.valid : this.importForm.dirty;
+  }
+
+  submitImportSettings() {
+    this.isImportSubmitted = true;
+    if (this.canSaveImport) {
+      const file = this.fileSource.value[0];
+
+      readFileAsText(file).then((fileContent) => {
+        const settings = JSON.parse(fileContent.data);
+
+        const hasCompatibleProvider =
+          settings.setting?.provider && Object.values(ObservabilityProvider).includes(settings.setting.provider);
+
+        if (!hasCompatibleProvider) {
+          this.toastrService.show(
+            `The file supplied does not reference a compatible provider. Please check the file.`,
+            'Observability settings import fails',
+            {
+              duration: 6000,
+              status: 'danger'
+            }
+          );
+          return;
+        }
+
+        this.initForm(settings);
+        this.form.markAsDirty();
+
+        this.closeImportModal();
+      });
+    }
+  }
+
   confirmSettingsDeletion() {
     const confirmAction = 'Delete';
     const cancelAction = 'Cancel';