From 10540b261e9903026e8c0d33eb77325bfc54b5fa Mon Sep 17 00:00:00 2001 From: Svilen Velikov <51084653+svilenvelikov@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:44:12 +0300 Subject: [PATCH] GDB-10937 show warnings in agent settings modal (#1622) ## What * Show warning when the user sets temperature higher than 1. * Show warning and an alert to the user when he tries to override the base instructions. ## Why * Because it would help the user to receive more meaningful results from the agent. * Because this would prevent unexpected results if the user provides incorrect instructions. ## How * Implemented icon with tooltip next to the temperature field label, and a handler for the property change to toggle the warning. * Implemented icon with tooltip and an alert that is displayed when the field value becomes different from empty string. Add tests to cover the new features fix test --- src/css/ttyg/agent-settings-modal.css | 8 ++- src/i18n/locale-en.json | 7 ++- src/i18n/locale-fr.json | 7 ++- .../agent-settings-modal.controller.js | 41 +++++++++++++ .../templates/modal/agent-settings-modal.html | 35 +++++++---- test-cypress/fixtures/locale-en.json | 9 ++- .../ttyg/agent/get-agent-defaults.json | 2 +- .../integration/ttyg/create-agent.spec.js | 61 ++++++++++++++++++- .../ttyg/ttyg-agent-settings-modal.steps.js | 18 +++++- 9 files changed, 166 insertions(+), 22 deletions(-) diff --git a/src/css/ttyg/agent-settings-modal.css b/src/css/ttyg/agent-settings-modal.css index 1b49191ec..53ac14b48 100644 --- a/src/css/ttyg/agent-settings-modal.css +++ b/src/css/ttyg/agent-settings-modal.css @@ -1,5 +1,7 @@ :root { --field-label-weight: 500; + --error-color: #FA787E; + --warning-color: #8a6d3b; } .agent-settings-modal .modal-footer .creating-agent-loader { @@ -18,7 +20,11 @@ } .agent-settings-modal .agent-settings-form .has-error { - outline: 1px solid #FA787E; + outline: 1px solid var(--error-color); +} + +.agent-settings-modal .agent-settings-form .has-warning { + outline: 2px solid var(--warning-color); } .agent-settings-modal .agent-settings-form .extraction-methods .extraction-method-heading { diff --git a/src/i18n/locale-en.json b/src/i18n/locale-en.json index 3eb187f86..a001fa3d1 100644 --- a/src/i18n/locale-en.json +++ b/src/i18n/locale-en.json @@ -383,7 +383,8 @@ }, "temperature": { "label": "Temperature", - "tooltip": "The sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic." + "tooltip": "The sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.", + "high_temperature_warning": "A temperature above 1.0 may cause the model to generate anything from creative responses to overdriven, incoherent outputs. Consider lowering the temperature for more reliable results." }, "top_p": { "label": "Top P", @@ -486,6 +487,10 @@ "label": "Base instructions", "placeholder": "Enter base instructions", "tooltip": "Base instructions for the agent. The default value provides good results in most setups so there is no need to change it.", + "overriding_system_instruction_warning": { + "title": "Overriding base instructions", + "body": "Modifying the base instructions may result in suboptimal performance. Only proceed if you are confident in your adjustments." + }, "btn": { "copy_instruction": { "tooltip": "Copy base instructions" diff --git a/src/i18n/locale-fr.json b/src/i18n/locale-fr.json index 87ef02ec1..6d5be01df 100644 --- a/src/i18n/locale-fr.json +++ b/src/i18n/locale-fr.json @@ -383,7 +383,8 @@ }, "temperature": { "label": "Température", - "tooltip": "La température d'échantillonnage à utiliser, entre 0 et 2. Des valeurs plus élevées comme 0,8 rendront la sortie plus aléatoire, tandis que des valeurs plus faibles comme 0,2 la rendront plus ciblée et déterministe." + "tooltip": "La température d'échantillonnage à utiliser, entre 0 et 2. Des valeurs plus élevées comme 0,8 rendront la sortie plus aléatoire, tandis que des valeurs plus faibles comme 0,2 la rendront plus ciblée et déterministe.", + "high_temperature_warning": "Une température supérieure à 1,0 peut amener le modèle à générer des réponses allant de créatives à incohérentes, voire incompréhensibles. Pensez à baisser la température pour des résultats plus fiables." }, "top_p": { "label": "Top P", @@ -486,6 +487,10 @@ "label": "Instructions de base", "placeholder": "Entrez les instructions de base", "tooltip": "Instructions de base pour l'agent. La valeur par défaut fournit de bons résultats dans la plupart des configurations, il n'est donc pas nécessaire de la modifier.", + "overriding_system_instruction_warning": { + "title": "Remplacement des instructions de base", + "body": "Modifier les instructions de base peut entraîner des résultats sous-optimaux. N’effectuez ces changements que si vous êtes sûr de ce que vous faites." + }, "btn": { "copy_instruction": { "tooltip": "Copier les instructions du système" diff --git a/src/js/angular/ttyg/controllers/agent-settings-modal.controller.js b/src/js/angular/ttyg/controllers/agent-settings-modal.controller.js index d0494e9c7..313bac87f 100644 --- a/src/js/angular/ttyg/controllers/agent-settings-modal.controller.js +++ b/src/js/angular/ttyg/controllers/agent-settings-modal.controller.js @@ -19,6 +19,7 @@ angular AgentSettingsModalController.$inject = [ '$scope', '$uibModalInstance', + 'ModalService', 'SimilarityService', 'ConnectorsService', 'RepositoriesRestService', @@ -33,6 +34,7 @@ AgentSettingsModalController.$inject = [ function AgentSettingsModalController( $scope, $uibModalInstance, + ModalService, SimilarityService, ConnectorsService, RepositoriesRestService, @@ -106,6 +108,18 @@ function AgentSettingsModalController( */ $scope.showAdvancedSettings = false; + /** + * Flag used to show/hide the high temperature warning in the modal. + * @type {boolean} + */ + $scope.showHighTemperatureWarning = false; + + /** + * Flag used to control the visibility of the system instruction warning. + * @type {boolean} + */ + $scope.showSystemInstructionWarning = false; + /** * The similarity indexes to be used for the similarity extraction method as select menu options. * @type {SelectMenuOptionsModel[]} @@ -283,6 +297,33 @@ function AgentSettingsModalController( $scope.agentFormModel.instructions.userInstruction = $scope.agentFormModel.instructions.userInstructionCopy; }; + /** + * Handles the change in the temperature field. This is needed because the high temperature warning should be shown + * when the temperature is higher than 1. + */ + $scope.onTemperatureChange = () => { + $scope.showHighTemperatureWarning = $scope.agentFormModel.temperature.value > 1; + }; + + /** + * Handles the change in the system instructions field. + */ + $scope.onSystemInstructionChange = () => { + if ($scope.agentFormModel.instructions.systemInstruction !== '' && !$scope.showSystemInstructionWarning) { + $scope.showSystemInstructionWarning = true; + ModalService.openModalAlert({ + title: $translate.instant('ttyg.agent.create_agent_modal.form.system_instruction.overriding_system_instruction_warning.title'), + message: $translate.instant('ttyg.agent.create_agent_modal.form.system_instruction.overriding_system_instruction_warning.body') + }).result + .then(function () { + // Do nothing, just warning the user + }); + } + if ($scope.agentFormModel.instructions.systemInstruction === '') { + $scope.showSystemInstructionWarning = false; + } + }; + // ========================= // Private functions // ========================= diff --git a/src/js/angular/ttyg/templates/modal/agent-settings-modal.html b/src/js/angular/ttyg/templates/modal/agent-settings-modal.html index 24c68c35d..8b265299a 100644 --- a/src/js/angular/ttyg/templates/modal/agent-settings-modal.html +++ b/src/js/angular/ttyg/templates/modal/agent-settings-modal.html @@ -321,7 +321,7 @@

{{( {{'ttyg.agent.create_agent_modal.form.additional_query_methods.method.' + + popover-trigger="mouseenter">{{'ttyg.agent.create_agent_modal.form.additional_query_methods.method.' + additionalExtractionMethod.method + '.label' | translate}} @@ -329,7 +329,8 @@

{{(
+ popover-trigger="mouseenter">{{'ttyg.agent.create_agent_modal.form.model.label' | + translate}} {{(
- - + {{'ttyg.agent.create_agent_modal.form.temperature.label' | translate}} + + +
@@ -406,10 +412,13 @@

{{(
-
diff --git a/test-cypress/fixtures/locale-en.json b/test-cypress/fixtures/locale-en.json index 53ac6cf84..a001fa3d1 100644 --- a/test-cypress/fixtures/locale-en.json +++ b/test-cypress/fixtures/locale-en.json @@ -383,7 +383,8 @@ }, "temperature": { "label": "Temperature", - "tooltip": "The sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic." + "tooltip": "The sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.", + "high_temperature_warning": "A temperature above 1.0 may cause the model to generate anything from creative responses to overdriven, incoherent outputs. Consider lowering the temperature for more reliable results." }, "top_p": { "label": "Top P", @@ -486,6 +487,10 @@ "label": "Base instructions", "placeholder": "Enter base instructions", "tooltip": "Base instructions for the agent. The default value provides good results in most setups so there is no need to change it.", + "overriding_system_instruction_warning": { + "title": "Overriding base instructions", + "body": "Modifying the base instructions may result in suboptimal performance. Only proceed if you are confident in your adjustments." + }, "btn": { "copy_instruction": { "tooltip": "Copy base instructions" @@ -599,7 +604,7 @@ "help_1": "Talk to your data in natural language. All you need to do is create and instruct your own AI Agent to assist you.", "help_2": "The Agent is an AI-powered conversational agent designed to help with a wide range of tasks, from answering questions and providing information to assisting with creative and technical projects. It leverages advanced natural language processing to understand and respond to user inputs in a human-like manner, making interactions intuitive and efficient.", "token_prop_warn1": "To enable this feature, set the config property", - "token_prop_warn2": "should be set to your GPT token in", + "token_prop_warn2": "to your GPT token in", "token_prop_warn3": "file.", "token_prop_warn4": "Configuring your use of GPT models", "error_retrieval_connectors_loading": "Error loading retrieval connectors", diff --git a/test-cypress/fixtures/ttyg/agent/get-agent-defaults.json b/test-cypress/fixtures/ttyg/agent/get-agent-defaults.json index a68ddbe38..ceb38fa8b 100644 --- a/test-cypress/fixtures/ttyg/agent/get-agent-defaults.json +++ b/test-cypress/fixtures/ttyg/agent/get-agent-defaults.json @@ -7,7 +7,7 @@ "seed": 0, "repositoryId": "test-repository", "instructions": { - "systemInstruction": "You are a helpful, knowledgeable, and friendly assistant. Your goal is to provide clear and accurate information while being polite, respectful, and professional.", + "systemInstruction": "", "userInstruction": "If you need to write a SPARQL query, use only the classes and properties provided in the schema and don't invent or guess any. Always try to return human-readable names or labels and not only the IRIs. If SPARQL fails to provide the necessary information you can try another tool too." }, "assistantExtractionMethods": [ diff --git a/test-cypress/integration/ttyg/create-agent.spec.js b/test-cypress/integration/ttyg/create-agent.spec.js index 1d966c633..e91072f35 100644 --- a/test-cypress/integration/ttyg/create-agent.spec.js +++ b/test-cypress/integration/ttyg/create-agent.spec.js @@ -6,6 +6,7 @@ import {SimilarityIndexStubs} from "../../stubs/similarity-index-stubs"; import {ConnectorStubs} from "../../stubs/connector-stubs"; import {ModalDialogSteps} from "../../steps/modal-dialog-steps"; import {RepositoriesStub} from "../../stubs/repositories-stub"; +import {AlertDialogSteps} from "../../steps/alert-dialog-steps"; describe('TTYG create new agent', () => { const repositoryId = 'starwars'; @@ -109,9 +110,8 @@ describe('TTYG create new agent', () => { TtygAgentSettingsModalSteps.typeGptModel('gpt-4o'); // temperature - TtygAgentSettingsModalSteps.getTemperatureField().should('have.value', '0.7'); TtygAgentSettingsModalSteps.setTemperature('0.2'); - TtygAgentSettingsModalSteps.getTemperatureField().should('have.value', '0.2'); + TtygAgentSettingsModalSteps.getTemperatureSliderField().should('have.value', '0.2'); // Top P TtygAgentSettingsModalSteps.getTopPField().should('have.value', '1'); @@ -129,7 +129,7 @@ describe('TTYG create new agent', () => { // Validate the advanced settings // System instructions - TtygAgentSettingsModalSteps.getSystemInstructionsField().should('have.value', 'You are a helpful, knowledgeable, and friendly assistant. Your goal is to provide clear and accurate information while being polite, respectful, and professional.'); + TtygAgentSettingsModalSteps.getSystemInstructionsField().should('have.value', ''); // User instructions TtygAgentSettingsModalSteps.getUserInstructionsField().should('have.value', 'If you need to write a SPARQL query, use only the classes and properties provided in the schema and don\'t invent or guess any. Always try to return human-readable names or labels and not only the IRIs. If SPARQL fails to provide the necessary information you can try another tool too.'); @@ -435,6 +435,61 @@ describe('TTYG create new agent', () => { // Then I expect a confirm dialog displayed. ModalDialogSteps.getDialogBody().contains('If you proceed with creating the similarity index, GraphDB will open in a new tab and switch to the ttyg-repo-1725518186812 repository.'); }); + + it('Should warn the user when temperature is set above given treshold', () => { + RepositoriesStubs.stubGetRepositoryConfig(repositoryId, '/repositories/get-repository-config-starwars-enabled-fts.json'); + TTYGStubs.stubChatsListGetNoResults(); + TTYGStubs.stubAgentListGet('/ttyg/agent/get-agent-list-0.json'); + // Given I have opened the ttyg page + TTYGViewSteps.visit(); + cy.wait('@get-all-repositories'); + cy.wait('@get-agent-list'); + // When I open the agent settings dialog + TTYGViewSteps.createFirstAgent(); + TtygAgentSettingsModalSteps.typeAgentName('Test Agent'); + // Then I expect that the high temperature warning is not visible + TtygAgentSettingsModalSteps.getTemperatureWarning().should('not.exist'); + TtygAgentSettingsModalSteps.getTemperatureField().should('not.have.class', 'has-warning'); + // And I change the temperature to value above 1.0 + TtygAgentSettingsModalSteps.setTemperature('1.2'); + TtygAgentSettingsModalSteps.getTemperatureSliderField().should('have.value', '1.2'); + // Then I should see a warning message + TtygAgentSettingsModalSteps.getTemperatureWarning().should('be.visible'); + TtygAgentSettingsModalSteps.getTemperatureField().should('have.class', 'has-warning'); + // When I change the temperature to value below 1.0 + TtygAgentSettingsModalSteps.setTemperature('0.9'); + TtygAgentSettingsModalSteps.getTemperatureSliderField().should('have.value', '0.9'); + // Then The high temperature warning should be hidden + TtygAgentSettingsModalSteps.getTemperatureWarning().should('not.exist'); + TtygAgentSettingsModalSteps.getTemperatureField().should('not.have.class', 'has-warning'); + }); + + it('Should warn the user when he changes the default value of the base instruction', () => { + RepositoriesStubs.stubGetRepositoryConfig(repositoryId, '/repositories/get-repository-config-starwars-enabled-fts.json'); + TTYGStubs.stubChatsListGetNoResults(); + TTYGStubs.stubAgentListGet('/ttyg/agent/get-agent-list-0.json'); + // Given I have opened the ttyg page + TTYGViewSteps.visit(); + cy.wait('@get-all-repositories'); + cy.wait('@get-agent-list'); + // When I open the agent settings dialog + TTYGViewSteps.createFirstAgent(); + TtygAgentSettingsModalSteps.typeAgentName('Test Agent'); + // Then I expect that the overriding base instruction warning is not visible + TtygAgentSettingsModalSteps.toggleAdvancedSettings(); + TtygAgentSettingsModalSteps.getSystemInstructionsWarning().should('not.exist'); + // When I change the base instruction + TtygAgentSettingsModalSteps.typeSystemInstructions('New'); + // Then I should see a warning alert + AlertDialogSteps.getDialog().should('be.visible'); + AlertDialogSteps.acceptAlert(); + // And the warning should be visible + TtygAgentSettingsModalSteps.getSystemInstructionsWarning().should('be.visible'); + // When I revert the base instruction to the default value + TtygAgentSettingsModalSteps.clearSystemInstructions(); + // Then the warning should be hidden + TtygAgentSettingsModalSteps.getSystemInstructionsWarning().should('not.exist'); + }); }); function fillAgentName(name) { diff --git a/test-cypress/steps/ttyg/ttyg-agent-settings-modal.steps.js b/test-cypress/steps/ttyg/ttyg-agent-settings-modal.steps.js index c2a9a9536..4e14fe309 100644 --- a/test-cypress/steps/ttyg/ttyg-agent-settings-modal.steps.js +++ b/test-cypress/steps/ttyg/ttyg-agent-settings-modal.steps.js @@ -322,8 +322,16 @@ export class TtygAgentSettingsModalSteps extends ModalDialogSteps { return this.getTemperatureFormGroup().find('#temperature'); } + static getTemperatureSliderField() { + return this.getTemperatureFormGroup().find('#temperatureSlider'); + } + static setTemperature(value) { - this.getTemperatureField().invoke('val', value).trigger('input'); + this.getTemperatureSliderField().invoke('val', value).trigger('input'); + } + + static getTemperatureWarning() { + return this.getTemperatureFormGroup().find('.high-temperature-warning'); } // Top P @@ -382,6 +390,10 @@ export class TtygAgentSettingsModalSteps extends ModalDialogSteps { return this.getSystemInstructionsFormGroup().find('#systemInstruction'); } + static getSystemInstructionsWarning() { + return this.getSystemInstructionsFormGroup().find('.overriding-system-instructions-warning'); + } + static clearSystemInstructions() { this.getSystemInstructionsField().clear(); } @@ -408,6 +420,10 @@ export class TtygAgentSettingsModalSteps extends ModalDialogSteps { return this.getUserInstructionsField().type(value); } + static toggleAdvancedSettings() { + this.getDialog().find('.toggle-advanced-settings').click(); + } + static getCreatingAgentLoader() { return this.getDialog().find('.saving-agent-loader'); }