From f394c3c20907763cfca6f0d01c550a55335a17dc Mon Sep 17 00:00:00 2001 From: Victor Alfaro Date: Tue, 27 Aug 2024 09:56:43 -0600 Subject: [PATCH] feat(dotAI): Adding fallback mechanism when it comes to send models. (#29748) Adding multi model support with a more robust class thans just String to AIModel class. Introducing README file to document new usage of AIClient instead of OpenAIRequest class. Adding AI testing utils as well as necessary Wiremock templates for IT and postman tests. Refs: #29284 --- .../java/com/dotcms/ai/app/AIAppUtil.java | 30 +++- .../main/java/com/dotcms/ai/app/AIModel.java | 128 +++++++++------- .../main/java/com/dotcms/ai/app/AIModels.java | 16 +- .../java/com/dotcms/ai/app/AppConfig.java | 32 ++-- .../java/com/dotcms/ai/app/ConfigService.java | 8 +- .../dotcms/ai/client/AIClientStrategy.java | 2 + .../com/dotcms/ai/client/AIProxiedClient.java | 2 + .../com/dotcms/ai/client/AIProxyClient.java | 1 + .../main/java/com/dotcms/ai/client/README.md | 61 ++++++++ .../java/com/dotcms/ai/db/EmbeddingsDTO.java | 3 +- .../ai/{client => domain}/AIResponse.java | 2 +- .../com/dotcms/ai/listener/AIAppListener.java | 21 ++- .../dotcms/ai/rest/forms/CompletionsForm.java | 19 ++- .../com/dotmarketing/util/UtilMethods.java | 11 ++ .../java/com/dotcms/ai/app/AIAppUtilTest.java | 9 +- .../ai/client/openai/AIProxiedClientTest.java | 102 +++++++++++++ .../openai/OpenAIResponseEvaluatorTest.java | 143 ++++++++++++++++++ .../ai/service/OpenAIChatServiceImplTest.java | 2 +- .../service/OpenAIImageServiceImplTest.java | 3 +- .../dotmarketing/util/UtilMethodsTest.java | 28 ++++ .../src/test/java/com/dotcms/ai/AiTest.java | 60 ++++++++ .../com/dotcms/ai/app/ConfigServiceTest.java | 101 +++++++++++++ .../mappings/apollo-space-program.json | 41 +++++ .../mappings/decommissioned-model.json | 30 ++++ .../resources/mappings/invalid-model.json | 30 ++++ .../postman/AI.postman_collection.json | 20 +-- .../mappings/apollo-space-program.json | 41 +++++ 27 files changed, 836 insertions(+), 110 deletions(-) create mode 100644 dotCMS/src/main/java/com/dotcms/ai/client/README.md rename dotCMS/src/main/java/com/dotcms/ai/{client => domain}/AIResponse.java (97%) create mode 100644 dotCMS/src/test/java/com/dotcms/ai/client/openai/AIProxiedClientTest.java create mode 100644 dotCMS/src/test/java/com/dotcms/ai/client/openai/OpenAIResponseEvaluatorTest.java create mode 100644 dotcms-integration/src/test/java/com/dotcms/ai/app/ConfigServiceTest.java create mode 100644 dotcms-integration/src/test/resources/mappings/apollo-space-program.json create mode 100644 dotcms-integration/src/test/resources/mappings/decommissioned-model.json create mode 100644 dotcms-integration/src/test/resources/mappings/invalid-model.json create mode 100644 dotcms-postman/src/test/resources/mappings/apollo-space-program.json diff --git a/dotCMS/src/main/java/com/dotcms/ai/app/AIAppUtil.java b/dotCMS/src/main/java/com/dotcms/ai/app/AIAppUtil.java index a4f6d2c8fb12..3cb2a1dad162 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/app/AIAppUtil.java +++ b/dotCMS/src/main/java/com/dotcms/ai/app/AIAppUtil.java @@ -6,6 +6,7 @@ import com.liferay.util.StringPool; import io.vavr.Lazy; import io.vavr.control.Try; +import org.apache.commons.collections4.CollectionUtils; import java.util.Arrays; import java.util.List; @@ -40,9 +41,14 @@ public static AIAppUtil get() { * @return the created text model instance */ public AIModel createTextModel(final Map secrets) { + final List modelNames = splitDiscoveredSecret(secrets, AppKeys.TEXT_MODEL_NAMES); + if (CollectionUtils.isEmpty(modelNames)) { + return AIModel.NOOP_MODEL; + } + return AIModel.builder() .withType(AIModelType.TEXT) - .withNames(splitDiscoveredSecret(secrets, AppKeys.TEXT_MODEL_NAMES)) + .withModelNames(modelNames) .withTokensPerMinute(discoverIntSecret(secrets, AppKeys.TEXT_MODEL_TOKENS_PER_MINUTE)) .withApiPerMinute(discoverIntSecret(secrets, AppKeys.TEXT_MODEL_API_PER_MINUTE)) .withMaxTokens(discoverIntSecret(secrets, AppKeys.TEXT_MODEL_MAX_TOKENS)) @@ -57,9 +63,14 @@ public AIModel createTextModel(final Map secrets) { * @return the created image model instance */ public AIModel createImageModel(final Map secrets) { + final List modelNames = splitDiscoveredSecret(secrets, AppKeys.IMAGE_MODEL_NAMES); + if (CollectionUtils.isEmpty(modelNames)) { + return AIModel.NOOP_MODEL; + } + return AIModel.builder() .withType(AIModelType.IMAGE) - .withNames(splitDiscoveredSecret(secrets, AppKeys.IMAGE_MODEL_NAMES)) + .withModelNames(modelNames) .withTokensPerMinute(discoverIntSecret(secrets, AppKeys.IMAGE_MODEL_TOKENS_PER_MINUTE)) .withApiPerMinute(discoverIntSecret(secrets, AppKeys.IMAGE_MODEL_API_PER_MINUTE)) .withMaxTokens(discoverIntSecret(secrets, AppKeys.IMAGE_MODEL_MAX_TOKENS)) @@ -74,9 +85,14 @@ public AIModel createImageModel(final Map secrets) { * @return the created embeddings model instance */ public AIModel createEmbeddingsModel(final Map secrets) { + final List modelNames = splitDiscoveredSecret(secrets, AppKeys.EMBEDDINGS_MODEL_NAMES); + if (CollectionUtils.isEmpty(modelNames)) { + return AIModel.NOOP_MODEL; + } + return AIModel.builder() .withType(AIModelType.EMBEDDINGS) - .withNames(splitDiscoveredSecret(secrets, AppKeys.EMBEDDINGS_MODEL_NAMES)) + .withModelNames(modelNames) .withTokensPerMinute(discoverIntSecret(secrets, AppKeys.EMBEDDINGS_MODEL_TOKENS_PER_MINUTE)) .withApiPerMinute(discoverIntSecret(secrets, AppKeys.EMBEDDINGS_MODEL_API_PER_MINUTE)) .withMaxTokens(discoverIntSecret(secrets, AppKeys.EMBEDDINGS_MODEL_MAX_TOKENS)) @@ -117,9 +133,11 @@ public String discoverSecret(final Map secrets, final AppKeys ke * @return the list of split secret values */ public List splitDiscoveredSecret(final Map secrets, final AppKeys key) { - return Arrays.stream(Optional.ofNullable(discoverSecret(secrets, key)).orElse(StringPool.BLANK).split(",")) - .map(String::trim) - .map(String::toLowerCase) + return Arrays + .stream(Optional + .ofNullable(discoverSecret(secrets, key)) + .map(secret -> secret.split(StringPool.COMMA)) + .orElse(new String[0])) .collect(Collectors.toList()); } diff --git a/dotCMS/src/main/java/com/dotcms/ai/app/AIModel.java b/dotCMS/src/main/java/com/dotcms/ai/app/AIModel.java index efbcc09a0872..3e98b716ce04 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/app/AIModel.java +++ b/dotCMS/src/main/java/com/dotcms/ai/app/AIModel.java @@ -1,12 +1,15 @@ package com.dotcms.ai.app; +import com.dotcms.ai.domain.Model; +import com.dotcms.ai.exception.DotAIModelNotFoundException; import com.dotcms.util.DotPreconditions; import com.dotmarketing.util.Logger; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * Represents an AI model with various attributes such as type, names, tokens per minute, @@ -18,43 +21,38 @@ */ public class AIModel { + private static final int NOOP_INDEX = -1; + public static final AIModel NOOP_MODEL = AIModel.builder() .withType(AIModelType.UNKNOWN) - .withNames(List.of()) + .withModelNames(List.of()) .build(); private final AIModelType type; - private final List names; + private final List models; private final int tokensPerMinute; private final int apiPerMinute; private final int maxTokens; private final boolean isCompletion; - private final AtomicInteger current; - private final AtomicBoolean decommissioned; - - private AIModel(final AIModelType type, - final List names, - final int tokensPerMinute, - final int apiPerMinute, - final int maxTokens, - final boolean isCompletion) { - DotPreconditions.checkNotNull(type, "type cannot be null"); - this.type = type; - this.names = Optional.ofNullable(names).orElse(List.of()); - this.tokensPerMinute = tokensPerMinute; - this.apiPerMinute = apiPerMinute; - this.maxTokens = maxTokens; - this.isCompletion = isCompletion; - current = new AtomicInteger(this.names.isEmpty() ? -1 : 0); - decommissioned = new AtomicBoolean(false); + private final AtomicInteger currentModelIndex; + + private AIModel(final Builder builder) { + DotPreconditions.checkNotNull(builder.type, "type cannot be null"); + this.type = builder.type; + this.models = builder.models; + this.tokensPerMinute = builder.tokensPerMinute; + this.apiPerMinute = builder.apiPerMinute; + this.maxTokens = builder.maxTokens; + this.isCompletion = builder.isCompletion; + currentModelIndex = new AtomicInteger(this.models.isEmpty() ? NOOP_INDEX : 0); } public AIModelType getType() { return type; } - public List getNames() { - return names; + public List getModels() { + return models; } public int getTokensPerMinute() { @@ -73,38 +71,49 @@ public boolean isCompletion() { return isCompletion; } - public int getCurrent() { - return current.get(); + public int getCurrentModelIndex() { + return currentModelIndex.get(); } - public void setCurrent(final int current) { - if (!isCurrentValid(current)) { - logInvalidModelMessage(); - return; - } - this.current.set(current); - } - - public boolean isDecommissioned() { - return decommissioned.get(); - } - - public void setDecommissioned(final boolean decommissioned) { - this.decommissioned.set(decommissioned); + public void setCurrentModelIndex(final int currentModelIndex) { + this.currentModelIndex.set(currentModelIndex); } public boolean isOperational() { - return this != NOOP_MODEL; + return this != NOOP_MODEL && models.stream().anyMatch(Model::isOperational); } - public String getCurrentModel() { - final int currentIndex = this.current.get(); + public Model getCurrent() { + final int currentIndex = currentModelIndex.get(); if (!isCurrentValid(currentIndex)) { logInvalidModelMessage(); return null; } + return models.get(currentIndex); + } - return names.get(currentIndex); + public String getCurrentModel() { + return Optional.ofNullable(getCurrent()).map(Model::getName).orElse(null); + } + + public Model getModel(final String modelName) { + final String normalized = modelName.trim().toLowerCase(); + return models.stream() + .filter(model -> normalized.equals(model.getName())) + .findFirst() + .orElseThrow(() -> new DotAIModelNotFoundException(String.format("Model [%s] not found", modelName))); + } + + public void repairCurrentIndexIfNeeded() { + if (getCurrentModelIndex() != NOOP_INDEX) { + return; + } + + setCurrentModelIndex( + getModels() + .stream() + .filter(Model::isOperational).findFirst().map(Model::getIndex) + .orElse(NOOP_INDEX)); } public long minIntervalBetweenCalls() { @@ -115,22 +124,21 @@ public long minIntervalBetweenCalls() { public String toString() { return "AIModel{" + "type=" + type + - ", names=" + names + + ", models='" + models + '\'' + ", tokensPerMinute=" + tokensPerMinute + ", apiPerMinute=" + apiPerMinute + ", maxTokens=" + maxTokens + ", isCompletion=" + isCompletion + - ", current=" + current + - ", decommissioned=" + decommissioned + + ", currentModelIndex=" + currentModelIndex.get() + '}'; } private boolean isCurrentValid(final int current) { - return !names.isEmpty() && current >= 0 && current < names.size(); + return !models.isEmpty() && current >= 0 && current < models.size(); } private void logInvalidModelMessage() { - Logger.debug(getClass(), String.format("Current model index must be between 0 and %d", names.size())); + Logger.debug(getClass(), String.format("Current model index must be between 0 and %d", models.size())); } public static Builder builder() { @@ -140,7 +148,7 @@ public static Builder builder() { public static class Builder { private AIModelType type; - private List names; + private List models; private int tokensPerMinute; private int apiPerMinute; private int maxTokens; @@ -154,13 +162,25 @@ public Builder withType(final AIModelType type) { return this; } - public Builder withNames(final List names) { - this.names = names; + public Builder withModels(final List models) { + this.models = Optional.ofNullable(models).orElse(List.of()); return this; } - public Builder withNames(final String... names) { - return withNames(List.of(names)); + public Builder withModelNames(final List names) { + return withModels( + Optional.ofNullable(names) + .map(modelNames -> IntStream.range(0, modelNames.size()) + .mapToObj(index -> Model.builder() + .withName(modelNames.get(index)) + .withIndex(index) + .build()) + .collect(Collectors.toList())) + .orElse(List.of())); + } + + public Builder withModelNames(final String... names) { + return withModelNames(List.of(names)); } public Builder withTokensPerMinute(final int tokensPerMinute) { @@ -184,7 +204,7 @@ public Builder withIsCompletion(final boolean isCompletion) { } public AIModel build() { - return new AIModel(type, names, tokensPerMinute, apiPerMinute, maxTokens, isCompletion); + return new AIModel(this); } } diff --git a/dotCMS/src/main/java/com/dotcms/ai/app/AIModels.java b/dotCMS/src/main/java/com/dotcms/ai/app/AIModels.java index 63ffac55695e..7c25179bf886 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/app/AIModels.java +++ b/dotCMS/src/main/java/com/dotcms/ai/app/AIModels.java @@ -81,22 +81,22 @@ public void loadModels(final String host, final List loading) { loading.stream() .map(model -> Tuple.of(model.getType(), model)) .collect(Collectors.toList()))); - loading.forEach(model -> model - .getNames() - .forEach(name -> { + loading.forEach(aiModel -> aiModel + .getModels() + .forEach(model -> { final Tuple2 key = Tuple.of( host, - name.toLowerCase().trim()); + model.getName().toLowerCase().trim()); if (modelsByName.containsKey(key)) { Logger.debug( this, String.format( "Model [%s] already exists for host [%s], ignoring it", - name, + model.getName(), host)); return; } - modelsByName.putIfAbsent(key, model); + modelsByName.putIfAbsent(key, aiModel); })); } @@ -192,7 +192,9 @@ public List getAvailableModels() { .stream() .flatMap(entry -> entry.getValue().stream()) .map(Tuple2::_2) - .flatMap(model -> model.getNames().stream().map(name -> new SimpleModel(name, model.getType()))) + .flatMap(aiModel -> aiModel.getModels() + .stream() + .map(model -> new SimpleModel(model.getName(), aiModel.getType()))) .collect(Collectors.toSet()); final Set supported = getOrPullSupportedModels() .stream() diff --git a/dotCMS/src/main/java/com/dotcms/ai/app/AppConfig.java b/dotCMS/src/main/java/com/dotcms/ai/app/AppConfig.java index 0d3d5cb6bc3f..bd8515e55834 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/app/AppConfig.java +++ b/dotCMS/src/main/java/com/dotcms/ai/app/AppConfig.java @@ -1,5 +1,6 @@ package com.dotcms.ai.app; +import com.dotcms.ai.exception.DotAIModelNotFoundException; import com.dotcms.security.apps.Secret; import com.dotmarketing.exception.DotRuntimeException; import com.dotmarketing.util.Logger; @@ -113,6 +114,15 @@ public static void setSystemHostConfig(final AppConfig systemHostConfig) { AppConfig.SYSTEM_HOST_CONFIG.set(systemHostConfig); } + /** + * Retrieves the host. + * + * @return the host + */ + public String getHost() { + return host; + } + /** * Retrieves the API URL. * @@ -134,7 +144,7 @@ public String getApiImageUrl() { /** * Retrieves the API Embeddings URL. * - * @return + * @return the API Embeddings URL */ public String getApiEmbeddingsUrl() { return UtilMethods.isEmpty(apiEmbeddingsUrl) ? AppKeys.API_EMBEDDINGS_URL.defaultValue : apiEmbeddingsUrl; @@ -293,24 +303,10 @@ public AIModel resolveModel(final AIModelType type) { * @param modelName the name of the model to find */ public AIModel resolveModelOrThrow(final String modelName) { - final AIModel aiModel = AIModels.get() + return AIModels.get() .findModel(host, modelName) - .orElseThrow(() -> { - final String supported = String.join(", ", AIModels.get().getOrPullSupportedModels()); - return new DotRuntimeException( - "Unable to find model: [" + modelName + "]. Only [" + supported + "] are supported "); - }); - - if (!aiModel.isOperational()) { - debugLogger( - AppConfig.class, - () -> String.format( - "Resolved model [%s] is not operational, avoiding its usage", - aiModel.getCurrentModel())); - throw new DotRuntimeException(String.format("Model [%s] is not operational", aiModel.getCurrentModel())); - } - - return aiModel; + .orElseThrow(() -> + new DotAIModelNotFoundException(String.format("Unable to find model: [%s].", modelName))); } /** diff --git a/dotCMS/src/main/java/com/dotcms/ai/app/ConfigService.java b/dotCMS/src/main/java/com/dotcms/ai/app/ConfigService.java index 115439388cc2..50f70eea4ad7 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/app/ConfigService.java +++ b/dotCMS/src/main/java/com/dotcms/ai/app/ConfigService.java @@ -7,6 +7,7 @@ import com.dotmarketing.business.APILocator; import com.dotmarketing.business.web.WebAPILocator; import com.dotmarketing.util.Logger; +import com.google.common.annotations.VisibleForTesting; import com.liferay.portal.model.User; import io.vavr.control.Try; @@ -23,7 +24,12 @@ public class ConfigService { private final LicenseValiditySupplier licenseValiditySupplier; private ConfigService() { - licenseValiditySupplier = new LicenseValiditySupplier() {}; + this(new LicenseValiditySupplier() {}); + } + + @VisibleForTesting + ConfigService(final LicenseValiditySupplier licenseValiditySupplier) { + this.licenseValiditySupplier = licenseValiditySupplier; } /** diff --git a/dotCMS/src/main/java/com/dotcms/ai/client/AIClientStrategy.java b/dotCMS/src/main/java/com/dotcms/ai/client/AIClientStrategy.java index 36c520c6b775..6ac784ef2a2a 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/client/AIClientStrategy.java +++ b/dotCMS/src/main/java/com/dotcms/ai/client/AIClientStrategy.java @@ -1,5 +1,7 @@ package com.dotcms.ai.client; +import com.dotcms.ai.domain.AIResponse; + import java.io.OutputStream; import java.io.Serializable; diff --git a/dotCMS/src/main/java/com/dotcms/ai/client/AIProxiedClient.java b/dotCMS/src/main/java/com/dotcms/ai/client/AIProxiedClient.java index 6cedcdc47712..73d675a3b90e 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/client/AIProxiedClient.java +++ b/dotCMS/src/main/java/com/dotcms/ai/client/AIProxiedClient.java @@ -1,5 +1,7 @@ package com.dotcms.ai.client; +import com.dotcms.ai.domain.AIResponse; + import java.io.ByteArrayOutputStream; import java.io.OutputStream; import java.io.Serializable; diff --git a/dotCMS/src/main/java/com/dotcms/ai/client/AIProxyClient.java b/dotCMS/src/main/java/com/dotcms/ai/client/AIProxyClient.java index 0f91c0cee1e7..b8776fe6db6c 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/client/AIProxyClient.java +++ b/dotCMS/src/main/java/com/dotcms/ai/client/AIProxyClient.java @@ -3,6 +3,7 @@ import com.dotcms.ai.client.openai.OpenAIClient; import com.dotcms.ai.client.openai.OpenAIResponseEvaluator; import com.dotcms.ai.domain.AIProvider; +import com.dotcms.ai.domain.AIResponse; import io.vavr.Lazy; import java.io.OutputStream; diff --git a/dotCMS/src/main/java/com/dotcms/ai/client/README.md b/dotCMS/src/main/java/com/dotcms/ai/client/README.md new file mode 100644 index 000000000000..796dfbdd5b2f --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/ai/client/README.md @@ -0,0 +1,61 @@ +# AI Clients + +This initiative that is introduced to dotCMS in several commits but the main idea is to provide a way to interact with AI services in a more generic way. + +## From OpenAIRequest to AIClient +Basically we use to have a class called `OpenAIRequest` that was used to interact with OpenAI services. This class was a bit coupled with OpenAI services and we decided to create a more generic way to interact with AI services. +So we created the `AIClient` interface which will be implemented by any component that needs to interact with an AI Provider. +Since OpenAI is the first AI provider we are integrating with, we created the `OpenAIClient` class that implements the `AIClient` interface. + +Usually, the `AIClient` will have a method called `sendRequest` that will receive a `Request` object and return a `Response` object. +That's it. You should not have to worry about instantiating the `AIClient` implementation, since this is managed by a higher level component called `AIClientProxy`. + +## AIClientProxy +This class will hold an internal structure that maps the AI provider identification to something called `AIProxiedClient`. + +The `AIClientProxy` is a class that will manage the instantiation of the `AIClient` implementation. + +### AIProxiedClient +The mere purpose of this class it to hold the `AIClient` implementation, an `AIClientStrategy` implementation and a `AIResponseEvaluator` implementation. +It's like a intersection point between these three classes and to keep decoupled enough the concepts of a AI client and the strategy used to send requests to the AI provider and finally how to interpret/evaluate the response from the AI provider. +It provides the flexibility of decouple a strategy from the client and from the response evaluator. + +### AIClientStrategy +This class is responsible for defining the strategy that the `AIClient` will use to interact with the AI provider. +Currently there are 2 strategies: +- `AIDefaultStrategy`: This strategy will be used when the desired behaviour is to just try to send a request. +- `AIModelFallbackStrategy`: This strategy will be used when the desired behaviour is to try to send a request to a model and if it fails, try to send the request to another model and so on until options are exhausted. + +### AIResponseEvaluator +This class is responsible for defining how the response from the AI provider will be evaluated. We will look for specific contents in the response to resolve (if any) the type of error and how to handle it. + +## How to use the whole thing +`AIClientProxy` is a singleton class that is the entrypoint for any other dotCMS component which needs to consume AI output. +It used to be the `OpenAIRequest` class but now it should be the `AIClientProxy` class since it provides multi-provider and multi-strategy support. + +Given a `AIRequest` instance (which will be built before calling this entrypoint and that for the sake of this example it will be built inline) and an `OutputStream` subclass, this is how you use it in your code: + +```java +final AIReponse response = AIProxyClient.get() + .sendRequest( + AIProvider.OPEN_AI.name(), + OpenAIRequest.builder() + .model("gpt-3.5-turbo") + .prompt("Once upon a time") + .build(), + output); +``` +Based on the provider name, behind curtains, the `AIClientProxy` will look for the `AIProxiedClient` instance that is registered with the provider name and will use the strategy and response evaluator that are registered with the `AIProxiedClient` instance. + +You can register instances of `AIProxiedClient` that will be used by the `AIClientProxy` like this: + +```java +AIProxyClient.get().addClient( + AIProvider.OPEN_AI.name(), + AIProxiedClient.of( + OpenAIClient.get(), + AIProxyStrategy.MODEL_FALLBACK, + OpenAIResponseEvaluator.get())); +``` +Here we can see how the three main elements are conjugated in a single `AIProxiedClient` instance. +Here we are registering an `OpenAIClient` instance with the `MODEL_FALLBACK` strategy and the `OpenAIResponseEvaluator` instance and it means that whenvever \ No newline at end of file diff --git a/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsDTO.java b/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsDTO.java index a5afa4b2bdc9..d621b2e0cd57 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsDTO.java +++ b/dotCMS/src/main/java/com/dotcms/ai/db/EmbeddingsDTO.java @@ -87,7 +87,8 @@ public static Builder from(final CompletionsForm form) { .withOperator(form.operator) .withThreshold(form.threshold) .withTemperature(form.temperature) - .withTokenCount(form.responseLengthTokens); + .withTokenCount(form.responseLengthTokens) + .withUser(form.user); } public static Builder from(final Map form) { diff --git a/dotCMS/src/main/java/com/dotcms/ai/client/AIResponse.java b/dotCMS/src/main/java/com/dotcms/ai/domain/AIResponse.java similarity index 97% rename from dotCMS/src/main/java/com/dotcms/ai/client/AIResponse.java rename to dotCMS/src/main/java/com/dotcms/ai/domain/AIResponse.java index 07c122abf126..8d9887b24571 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/client/AIResponse.java +++ b/dotCMS/src/main/java/com/dotcms/ai/domain/AIResponse.java @@ -1,4 +1,4 @@ -package com.dotcms.ai.client; +package com.dotcms.ai.domain; /** * Represents a response from an AI service. diff --git a/dotCMS/src/main/java/com/dotcms/ai/listener/AIAppListener.java b/dotCMS/src/main/java/com/dotcms/ai/listener/AIAppListener.java index 32bd759b58b2..a95b8f8df520 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/listener/AIAppListener.java +++ b/dotCMS/src/main/java/com/dotcms/ai/listener/AIAppListener.java @@ -1,8 +1,10 @@ package com.dotcms.ai.listener; import com.dotcms.ai.app.AIModels; +import com.dotcms.ai.app.AppConfig; import com.dotcms.ai.app.AppKeys; import com.dotcms.ai.app.ConfigService; +import com.dotcms.ai.validator.AIAppValidator; import com.dotcms.security.apps.AppSecretSavedEvent; import com.dotcms.system.event.local.model.EventSubscriber; import com.dotcms.system.event.local.model.KeyFilterable; @@ -35,6 +37,21 @@ public AIAppListener() { this(APILocator.getHostAPI()); } + /** + * Notifies the listener of an {@link AppSecretSavedEvent}. + * + *

+ * This method is called when an {@link AppSecretSavedEvent} occurs. It performs the following actions: + *

    + *
  • Logs a debug message if the event is null or the event's host identifier is blank.
    • + *
  • Finds the host associated with the event's host identifier.
    • + *
  • Resets the AI models for the found host's hostname.
    • + *
  • Validates the AI configuration using the {@link AIAppValidator}.
    • + *
+ *

+ * + * @param event the {@link AppSecretSavedEvent} that triggered the notification + */ @Override public void notify(final AppSecretSavedEvent event) { if (Objects.isNull(event)) { @@ -51,7 +68,9 @@ public void notify(final AppSecretSavedEvent event) { final Host host = Try.of(() -> hostAPI.find(hostId, APILocator.systemUser(), false)).getOrNull(); Optional.ofNullable(host).ifPresent(found -> AIModels.get().resetModels(found.getHostname())); - ConfigService.INSTANCE.config(host); + final AppConfig appConfig = ConfigService.INSTANCE.config(host); + + AIAppValidator.get().validateAIConfig(appConfig, event.getUserId()); } @Override diff --git a/dotCMS/src/main/java/com/dotcms/ai/rest/forms/CompletionsForm.java b/dotCMS/src/main/java/com/dotcms/ai/rest/forms/CompletionsForm.java index f4eb199d4bf2..2e1f58923556 100644 --- a/dotCMS/src/main/java/com/dotcms/ai/rest/forms/CompletionsForm.java +++ b/dotCMS/src/main/java/com/dotcms/ai/rest/forms/CompletionsForm.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.annotation.Nulls; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.liferay.portal.model.User; import io.vavr.control.Try; import javax.validation.constraints.Max; @@ -49,6 +50,7 @@ public class CompletionsForm { public final String model; public final String operator; public final String site; + public final User user; @Override public boolean equals(final Object o) { @@ -88,6 +90,7 @@ public String toString() { ", operator='" + operator + '\'' + ", site='" + site + '\'' + ", contentType=" + Arrays.toString(contentType) + + ", user=" + user + '}'; } @@ -118,6 +121,7 @@ private CompletionsForm(final Builder builder) { this.temperature = builder.temperature >= 2 ? 2 : builder.temperature; } this.model = UtilMethods.isSet(builder.model) ? builder.model : ConfigService.INSTANCE.config().getModel().getCurrentModel(); + this.user = builder.user; } private String validateBuilderQuery(final String query) { @@ -131,7 +135,6 @@ private long validateLanguage(final String language) { return Try.of(() -> Long.parseLong(language)) .recover(x -> APILocator.getLanguageAPI().getLanguage(language).getId()) .getOrElseTry(() -> APILocator.getLanguageAPI().getDefaultLanguage().getId()); - } public static Builder copy(final CompletionsForm form) { @@ -149,7 +152,8 @@ public static Builder copy(final CompletionsForm form) { .operator(form.operator) .indexName(form.indexName) .threshold(form.threshold) - .stream(form.stream); + .stream(form.stream) + .user(form.user); } public static final class Builder { @@ -182,6 +186,8 @@ public static final class Builder { private String operator = "cosine"; @JsonSetter(nulls = Nulls.SKIP) private String site; + @JsonSetter(nulls = Nulls.SKIP) + private User user; public Builder prompt(String queryOrPrompt) { this.prompt = queryOrPrompt; @@ -224,7 +230,7 @@ public Builder fieldVar(String fieldVar) { } public Builder model(String model) { - this.model =model; + this.model = model; return this; } @@ -254,7 +260,12 @@ public Builder operator(String operator) { } public Builder site(String site) { - this.site =site; + this.site = site; + return this; + } + + public Builder user(User user) { + this.user = user; return this; } diff --git a/dotCMS/src/main/java/com/dotmarketing/util/UtilMethods.java b/dotCMS/src/main/java/com/dotmarketing/util/UtilMethods.java index 61085e6bb0f7..0dfe0d2adc27 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/UtilMethods.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/UtilMethods.java @@ -3678,4 +3678,15 @@ public static T isSetOrGet(final T toEvaluate, final T defaultValue){ public static boolean exceedsMaxLength(final T value, final int maxLength) { return value != null && value.length() > maxLength; } + + /** + * Extracts the user id from a User object or returns null if the object is null + * + * @param user User object + * @return User id or null + */ + public static String extractUserIdOrNull(final User user) { + return Optional.ofNullable(user).map(User::getUserId).orElse(null); + } + } \ No newline at end of file diff --git a/dotCMS/src/test/java/com/dotcms/ai/app/AIAppUtilTest.java b/dotCMS/src/test/java/com/dotcms/ai/app/AIAppUtilTest.java index c4d5c93b7627..8c1cd1e79c4e 100644 --- a/dotCMS/src/test/java/com/dotcms/ai/app/AIAppUtilTest.java +++ b/dotCMS/src/test/java/com/dotcms/ai/app/AIAppUtilTest.java @@ -1,10 +1,12 @@ package com.dotcms.ai.app; +import com.dotcms.ai.domain.Model; import com.dotcms.security.apps.Secret; import org.junit.Before; import org.junit.Test; import java.util.Map; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -127,7 +129,7 @@ public void testCreateTextModel() { AIModel model = aiAppUtil.createTextModel(secrets); assertNotNull(model); assertEquals(AIModelType.TEXT, model.getType()); - assertTrue(model.getNames().contains("textmodel")); + assertTrue(model.getModels().stream().map(Model::getName).collect(Collectors.toList()).contains("textmodel")); } /** @@ -143,7 +145,7 @@ public void testCreateImageModel() { AIModel model = aiAppUtil.createImageModel(secrets); assertNotNull(model); assertEquals(AIModelType.IMAGE, model.getType()); - assertTrue(model.getNames().contains("imagemodel")); + assertTrue(model.getModels().stream().map(Model::getName).collect(Collectors.toList()).contains("imagemodel")); } /** @@ -159,7 +161,8 @@ public void testCreateEmbeddingsModel() { AIModel model = aiAppUtil.createEmbeddingsModel(secrets); assertNotNull(model); assertEquals(AIModelType.EMBEDDINGS, model.getType()); - assertTrue(model.getNames().contains("embeddingsmodel")); + assertTrue(model.getModels().stream().map(Model::getName).collect(Collectors.toList()) + .contains("embeddingsmodel")); } @Test diff --git a/dotCMS/src/test/java/com/dotcms/ai/client/openai/AIProxiedClientTest.java b/dotCMS/src/test/java/com/dotcms/ai/client/openai/AIProxiedClientTest.java new file mode 100644 index 000000000000..86ca35f3290a --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/ai/client/openai/AIProxiedClientTest.java @@ -0,0 +1,102 @@ +package com.dotcms.ai.client.openai; + +import com.dotcms.ai.client.AIClient; +import com.dotcms.ai.client.AIClientStrategy; +import com.dotcms.ai.client.AIProxiedClient; +import com.dotcms.ai.client.AIProxyStrategy; +import com.dotcms.ai.client.AIRequest; +import com.dotcms.ai.domain.AIResponse; +import com.dotcms.ai.client.AIResponseEvaluator; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.io.Serializable; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for the AIProxiedClient class. + * + * @author vico + */ +public class AIProxiedClientTest { + + private AIClient mockClient; + private AIProxyStrategy mockProxyStrategy; + private AIClientStrategy mockClientStrategy; + private AIResponseEvaluator mockResponseEvaluator; + private AIProxiedClient proxiedClient; + + @Before + public void setUp() { + mockClient = mock(AIClient.class); + mockProxyStrategy = mock(AIProxyStrategy.class); + mockClientStrategy = mock(AIClientStrategy.class); + when(mockProxyStrategy.getStrategy()).thenReturn(mockClientStrategy); + mockResponseEvaluator = mock(AIResponseEvaluator.class); + proxiedClient = AIProxiedClient.of(mockClient, mockProxyStrategy, mockResponseEvaluator); + } + + /** + * Scenario: Sending a valid AI request + * Given a valid AI request + * When the request is sent to the AI service + * Then the strategy should be applied + * And the response should be written to the output stream + */ + @Test + public void testSendToAI_withValidRequest() { + AIRequest request = mock(AIRequest.class); + OutputStream output = mock(OutputStream.class); + + AIResponse response = proxiedClient.sendToAI(request, output); + + verify(mockClientStrategy).applyStrategy(mockClient, mockResponseEvaluator, request, output); + assertEquals(AIResponse.EMPTY, response); + } + + /** + * Scenario: Sending an AI request with null output stream + * Given a valid AI request and a null output stream + * When the request is sent to the AI service + * Then the strategy should be applied + * And the response should be returned as a string + */ + @Test + public void testSendToAI_withNullOutput() { + AIRequest request = mock(AIRequest.class); + AIResponse response = proxiedClient.sendToAI(request, null); + + verify(mockClientStrategy).applyStrategy( + eq(mockClient), + eq(mockResponseEvaluator), + eq(request), + any(OutputStream.class)); + assertEquals("", response.getResponse()); + } + + /** + * Scenario: Sending an AI request with NOOP client + * Given a valid AI request and a NOOP client + * When the request is sent to the AI service + * Then no operations should be performed + * And the response should be empty + */ + @Test + public void testSendToAI_withNoopClient() { + proxiedClient = AIProxiedClient.NOOP; + AIRequest request = AIRequest.builder().build(); + OutputStream output = new ByteArrayOutputStream(); + + AIResponse response = proxiedClient.sendToAI(request, output); + + assertEquals(AIResponse.EMPTY, response); + } +} \ No newline at end of file diff --git a/dotCMS/src/test/java/com/dotcms/ai/client/openai/OpenAIResponseEvaluatorTest.java b/dotCMS/src/test/java/com/dotcms/ai/client/openai/OpenAIResponseEvaluatorTest.java new file mode 100644 index 000000000000..4823d35aa7ba --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/ai/client/openai/OpenAIResponseEvaluatorTest.java @@ -0,0 +1,143 @@ +package com.dotcms.ai.client.openai; + +import com.dotcms.ai.domain.AIResponseData; +import com.dotcms.ai.domain.ModelStatus; +import com.dotcms.ai.exception.DotAIModelNotFoundException; +import com.dotmarketing.exception.DotRuntimeException; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Tests for the OpenAIResponseEvaluator class. + * + * @author vico + */ +public class OpenAIResponseEvaluatorTest { + + private OpenAIResponseEvaluator evaluator; + + @Before + public void setUp() { + evaluator = OpenAIResponseEvaluator.get(); + } + + /** + * Scenario: Processing a response with an error + * Given a response with an error message "Model has been deprecated" + * When the response is processed + * Then the metadata should contain the error message "Model has been deprecated" + * And the status should be set to DECOMMISSIONED + */ + @Test + public void testFromResponse_withError() { + String response = new JSONObject() + .put("error", new JSONObject().put("message", "Model has been deprecated")) + .toString(); + AIResponseData metadata = new AIResponseData(); + + evaluator.fromResponse(response, metadata, true); + + assertEquals("Model has been deprecated", metadata.getError()); + assertEquals(ModelStatus.DECOMMISSIONED, metadata.getStatus()); + } + + /** + * Scenario: Processing a response with an error + * Given a response with an error message "Model has been deprecated" + * When the response is processed as no JSON + * Then the metadata should contain the error message "Model has been deprecated" + * And the status should be set to DECOMMISSIONED + */ + @Test + public void testFromResponse_withErrorNoJson() { + String response = new JSONObject() + .put("error", new JSONObject().put("message", "Model has been deprecated")) + .toString(); + AIResponseData metadata = new AIResponseData(); + + evaluator.fromResponse(response, metadata, false); + + assertEquals("Model has been deprecated", metadata.getError()); + assertEquals(ModelStatus.DECOMMISSIONED, metadata.getStatus()); + } + + /** + * Scenario: Processing a response with an error + * Given a response with an error message "Model has been deprecated" + * When the response is processed as no JSON + * Then the metadata should contain the error message "Model has been deprecated" + * And the status should be set to DECOMMISSIONED + */ + @Test + public void testFromResponse_withoutErrorNoJson() { + String response = "not a json response"; + AIResponseData metadata = new AIResponseData(); + + evaluator.fromResponse(response, metadata, false); + + assertNull(metadata.getError()); + assertNull(metadata.getStatus()); + } + + /** + * Scenario: Processing a response without an error + * Given a response without an error message + * When the response is processed + * Then the metadata should not contain any error message + * And the status should be null + */ + @Test + public void testFromResponse_withoutError() { + String response = new JSONObject().put("data", "some data").toString(); + AIResponseData metadata = new AIResponseData(); + + evaluator.fromResponse(response, metadata, true); + + assertNull(metadata.getError()); + assertNull(metadata.getStatus()); + } + + /** + * Scenario: Processing an exception of type DotRuntimeException + * Given an exception of type DotAIModelNotFoundException with message "Model not found" + * When the exception is processed + * Then the metadata should contain the error message "Model not found" + * And the status should be set to INVALID + * And the exception should be set to the given DotRuntimeException + */ + @Test + public void testFromException_withDotRuntimeException() { + DotRuntimeException exception = new DotAIModelNotFoundException("Model not found"); + AIResponseData metadata = new AIResponseData(); + + evaluator.fromException(exception, metadata); + + assertEquals("Model not found", metadata.getError()); + assertEquals(ModelStatus.INVALID, metadata.getStatus()); + assertEquals(exception, metadata.getException()); + } + + /** + * Scenario: Processing a general exception + * Given a general exception with message "General error" + * When the exception is processed + * Then the metadata should contain the error message "General error" + * And the status should be set to UNKNOWN + * And the exception should be wrapped in a DotRuntimeException + */ + @Test + public void testFromException_withOtherException() { + Exception exception = new Exception("General error"); + AIResponseData metadata = new AIResponseData(); + + evaluator.fromException(exception, metadata); + + assertEquals("General error", metadata.getError()); + assertEquals(ModelStatus.UNKNOWN, metadata.getStatus()); + assertEquals(DotRuntimeException.class, metadata.getException().getClass()); + } +} diff --git a/dotCMS/src/test/java/com/dotcms/ai/service/OpenAIChatServiceImplTest.java b/dotCMS/src/test/java/com/dotcms/ai/service/OpenAIChatServiceImplTest.java index e4c43486c3f1..3fe155f2d01b 100644 --- a/dotCMS/src/test/java/com/dotcms/ai/service/OpenAIChatServiceImplTest.java +++ b/dotCMS/src/test/java/com/dotcms/ai/service/OpenAIChatServiceImplTest.java @@ -66,7 +66,7 @@ public String doRequest(final String urlIn, final JSONObject json) { private JSONObject prepareJsonObject(final String prompt) { when(config.getModel()) - .thenReturn(AIModel.builder().withType(AIModelType.TEXT).withNames("some-model").build()); + .thenReturn(AIModel.builder().withType(AIModelType.TEXT).withModelNames("some-model").build()); when(config.getConfigFloat(AppKeys.COMPLETION_TEMPERATURE)).thenReturn(123.321F); when(config.getRolePrompt()).thenReturn("some-role-prompt"); diff --git a/dotCMS/src/test/java/com/dotcms/ai/service/OpenAIImageServiceImplTest.java b/dotCMS/src/test/java/com/dotcms/ai/service/OpenAIImageServiceImplTest.java index 6c3fc6822473..4d0afb444b88 100644 --- a/dotCMS/src/test/java/com/dotcms/ai/service/OpenAIImageServiceImplTest.java +++ b/dotCMS/src/test/java/com/dotcms/ai/service/OpenAIImageServiceImplTest.java @@ -220,7 +220,8 @@ public AIImageRequestDTO.Builder getDtoBuilder() { } private JSONObject prepareJsonObject(final String prompt, final boolean tempFileError) throws Exception { - when(config.getImageModel()).thenReturn(AIModel.builder().withType(AIModelType.IMAGE).withNames("some-image-model").build()); + when(config.getImageModel()) + .thenReturn(AIModel.builder().withType(AIModelType.IMAGE).withModelNames("some-image-model").build()); when(config.getImageSize()).thenReturn("some-image-size"); final File file = mock(File.class); when(file.getName()).thenReturn(UUIDGenerator.shorty()); diff --git a/dotCMS/src/test/java/com/dotmarketing/util/UtilMethodsTest.java b/dotCMS/src/test/java/com/dotmarketing/util/UtilMethodsTest.java index 6a4e474aac4d..f6f3d4fd8213 100644 --- a/dotCMS/src/test/java/com/dotmarketing/util/UtilMethodsTest.java +++ b/dotCMS/src/test/java/com/dotmarketing/util/UtilMethodsTest.java @@ -5,9 +5,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.dotcms.UnitTestBase; import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.liferay.portal.model.User; import org.junit.Test; /** @@ -208,4 +211,29 @@ public void test_isImage_method(){ } } + /** + * Scenario: Extracting user ID from a User object + * Given a null User object + * When the user ID is extracted + * Then the result should be null + * + * Given a mocked User object with no user ID + * When the user ID is extracted + * Then the result should be null + * + * Given a mocked User object with a user ID "userId" + * When the user ID is extracted + * Then the result should be "userId" + */ + @Test + public void test_extractUserIdOrNull(){ + assertNull(UtilMethods.extractUserIdOrNull(null)); + + final User user = mock(User.class); + assertNull(UtilMethods.extractUserIdOrNull(user)); + + when(user.getUserId()).thenReturn("userId"); + assertEquals("userId", UtilMethods.extractUserIdOrNull(user)); + } + } diff --git a/dotcms-integration/src/test/java/com/dotcms/ai/AiTest.java b/dotcms-integration/src/test/java/com/dotcms/ai/AiTest.java index 855f61ad4572..0c790bb24d74 100644 --- a/dotcms-integration/src/test/java/com/dotcms/ai/AiTest.java +++ b/dotcms-integration/src/test/java/com/dotcms/ai/AiTest.java @@ -11,6 +11,8 @@ import com.github.tomakehurst.wiremock.WireMockServer; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; public interface AiTest { @@ -82,4 +84,62 @@ static void removeSecrets(final Host host) throws DotDataException, DotSecurityE APILocator.getAppsAPI().removeSecretsForSite(host, APILocator.systemUser()); } + + + + // TODO: pr-split -> remove methods below + static Map aiAppSecrets(final Host host, + final String apiKey, + final String textModels, + final String imageModels, + final String embeddingsModel) throws Exception { + final AppSecrets.Builder builder = new AppSecrets.Builder() + .withKey(AppKeys.APP_KEY) + .withSecret(AppKeys.API_URL.key, String.format(API_URL, PORT)) + .withSecret(AppKeys.API_IMAGE_URL.key, String.format(API_IMAGE_URL, PORT)) + .withSecret(AppKeys.API_EMBEDDINGS_URL.key, String.format(API_EMBEDDINGS_URL, PORT)) + .withHiddenSecret(AppKeys.API_KEY.key, apiKey) + .withSecret(AppKeys.IMAGE_SIZE.key, IMAGE_SIZE) + .withSecret(AppKeys.LISTENER_INDEXER.key, "{\"default\":\"blog\"}") + .withSecret(AppKeys.COMPLETION_ROLE_PROMPT.key, AppKeys.COMPLETION_ROLE_PROMPT.defaultValue) + .withSecret(AppKeys.COMPLETION_TEXT_PROMPT.key, AppKeys.COMPLETION_TEXT_PROMPT.defaultValue); + + if (Objects.nonNull(textModels)) { + builder.withSecret(AppKeys.TEXT_MODEL_NAMES.key, textModels); + } + if (Objects.nonNull(imageModels)) { + builder.withSecret(AppKeys.IMAGE_MODEL_NAMES.key, imageModels); + } + if (Objects.nonNull(embeddingsModel)) { + builder.withSecret(AppKeys.EMBEDDINGS_MODEL_NAMES.key, embeddingsModel); + } + + final AppSecrets appSecrets = builder.build(); + APILocator.getAppsAPI().saveSecrets(appSecrets, host, APILocator.systemUser()); + TimeUnit.SECONDS.sleep(1); + return appSecrets.getSecrets(); + } + + static Map aiAppSecrets(final Host host, final String apiKey) throws Exception { + return aiAppSecrets(host, apiKey, MODEL, IMAGE_MODEL, EMBEDDINGS_MODEL); + } + + static Map aiAppSecrets(final Host host, + final String textModels, + final String imageModels, + final String embeddingsModel) throws Exception { + return aiAppSecrets(host, API_KEY, textModels, imageModels, embeddingsModel); + } + + static Map aiAppSecrets(final Host host) throws Exception { + + return aiAppSecrets(host, MODEL, IMAGE_MODEL, EMBEDDINGS_MODEL); + } + + static void removeAiAppSecrets(final Host host) throws Exception { + APILocator.getAppsAPI().deleteSecrets(AppKeys.APP_KEY, host, APILocator.systemUser()); + } + + + } diff --git a/dotcms-integration/src/test/java/com/dotcms/ai/app/ConfigServiceTest.java b/dotcms-integration/src/test/java/com/dotcms/ai/app/ConfigServiceTest.java new file mode 100644 index 000000000000..2e6143037095 --- /dev/null +++ b/dotcms-integration/src/test/java/com/dotcms/ai/app/ConfigServiceTest.java @@ -0,0 +1,101 @@ +package com.dotcms.ai.app; + +import com.dotcms.ai.AiTest; +import com.dotcms.datagen.SiteDataGen; +import com.dotcms.util.IntegrationTestInitService; +import com.dotcms.util.LicenseValiditySupplier; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for the ConfigService class. + * + *

+ * This class contains tests to verify the behavior of the ConfigService, + * including scenarios with valid and invalid licenses, and configurations + * with and without secrets. + *

+ * + *

+ * The tests ensure that the ConfigService correctly initializes and + * configures the AppConfig based on the provided Host and license validity. + *

+ * + * @author vico + */ +public class ConfigServiceTest { + + private Host host; + private ConfigService configService; + + @BeforeClass + public static void beforeClass() throws Exception { + IntegrationTestInitService.getInstance().init(); + } + + @Before + public void before() { + host = new SiteDataGen().nextPersisted(); + configService = ConfigService.INSTANCE; + } + + /** + * Given a ConfigService with an invalid license + * When the config method is called with a host + * Then the models should not be operational. + */ + @Test + public void test_invalidLicense() { + configService = new ConfigService(new LicenseValiditySupplier() { + @Override + public boolean hasValidLicense() { + return false; + } + }); + final AppConfig appConfig = configService.config(host); + + assertFalse(appConfig.getModel().isOperational()); + assertFalse(appConfig.getImageModel().isOperational()); + assertFalse(appConfig.getEmbeddingsModel().isOperational()); + } + + /** + * Given a host with secrets and a ConfigService + * When the config method is called with the host + * Then the models should be operational and the host should be correctly set in the AppConfig. + */ + @Test + public void test_config_hostWithSecrets() throws Exception { + AiTest.aiAppSecrets(host, "text-model-0", "image-model-1", "embeddings-model-2"); + final AppConfig appConfig = configService.config(host); + + assertTrue(appConfig.getModel().isOperational()); + assertTrue(appConfig.getImageModel().isOperational()); + assertTrue(appConfig.getEmbeddingsModel().isOperational()); + assertEquals(host.getHostname(), appConfig.getHost()); + } + + /** + * Given a host without secrets and a ConfigService + * When the config method is called with the host + * Then the models should be operational and the host should be set to "System Host" in the AppConfig. + */ + @Test + public void test_config_hostWithoutSecrets() throws Exception { + AiTest.aiAppSecrets(APILocator.systemHost(), "text-model-10", "image-model-11", "embeddings-model-12"); + final AppConfig appConfig = configService.config(host); + + assertTrue(appConfig.getModel().isOperational()); + assertTrue(appConfig.getImageModel().isOperational()); + assertTrue(appConfig.getEmbeddingsModel().isOperational()); + assertEquals("System Host", appConfig.getHost()); + } + +} diff --git a/dotcms-integration/src/test/resources/mappings/apollo-space-program.json b/dotcms-integration/src/test/resources/mappings/apollo-space-program.json new file mode 100644 index 000000000000..7118ba306929 --- /dev/null +++ b/dotcms-integration/src/test/resources/mappings/apollo-space-program.json @@ -0,0 +1,41 @@ +{ + "request": { + "method": "POST", + "url": "/c", + "headers": { + "Content-Type": { + "equalTo": "application/json" + }, + "Authorization": { + "equalTo": "Bearer some-api-key-1a2bc3" + } + }, + "bodyPatterns": [ + { + "matches": ".*\"model\":\"gpt-4o-mini\".*\"content\":\"What are the major achievements of the Apollo space program.*" + } + ] + }, + "response": { + "status": 200, + "jsonBody": { + "id": "cmpl-11", + "object": "text_completion", + "created": 1699999999, + "model": "gpt-4o-mini", + "choices": [ + { + "text": "The Apollo space program, conducted by NASA, achieved several major milestones in space exploration. Its most significant achievement was the successful landing of humans on the Moon. Apollo 11, in 1969, saw astronauts Neil Armstrong and Buzz Aldrin become the first humans to set foot on the lunar surface. The program also provided extensive scientific data and technological advancements.", + "index": 0, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 57, + "total_tokens": 67 + } + } + } +} diff --git a/dotcms-integration/src/test/resources/mappings/decommissioned-model.json b/dotcms-integration/src/test/resources/mappings/decommissioned-model.json new file mode 100644 index 000000000000..46f83ccb4b0b --- /dev/null +++ b/dotcms-integration/src/test/resources/mappings/decommissioned-model.json @@ -0,0 +1,30 @@ +{ + "request": { + "method": "POST", + "url": "/c", + "headers": { + "Content-Type": { + "equalTo": "application/json" + }, + "Authorization": { + "equalTo": "Bearer some-api-key-1a2bc3" + } + }, + "bodyPatterns": [ + { + "matches": ".*\"model\":\"some-decommissioned-model-..\".*" + } + ] + }, + "response": { + "status": 200, + "jsonBody": { + "error": { + "message": "The model `some-decommissioned-model` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations", + "type": "invalid_request_error", + "param": null, + "code": "model_not_found" + } + } + } +} diff --git a/dotcms-integration/src/test/resources/mappings/invalid-model.json b/dotcms-integration/src/test/resources/mappings/invalid-model.json new file mode 100644 index 000000000000..810073b0b0ec --- /dev/null +++ b/dotcms-integration/src/test/resources/mappings/invalid-model.json @@ -0,0 +1,30 @@ +{ + "request": { + "method": "POST", + "url": "/c", + "headers": { + "Content-Type": { + "equalTo": "application/json" + }, + "Authorization": { + "equalTo": "Bearer some-api-key-1a2bc3" + } + }, + "bodyPatterns": [ + { + "matches": ".*\"model\":\"some-made-up-model-..\".*" + } + ] + }, + "response": { + "status": 200, + "jsonBody": { + "error": { + "message": "The model `some-made-up-mode` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "param": null, + "code": "model_not_found" + } + } + } +} diff --git a/dotcms-postman/src/main/resources/postman/AI.postman_collection.json b/dotcms-postman/src/main/resources/postman/AI.postman_collection.json index 9a49e8bcd38e..dd1b0b760d92 100644 --- a/dotcms-postman/src/main/resources/postman/AI.postman_collection.json +++ b/dotcms-postman/src/main/resources/postman/AI.postman_collection.json @@ -60,7 +60,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"apiKey\": {\n \"value\": \"some-api-key-1a2bc3\"\n },\n \"textModelNames\": {\n \"value\": \"gpt-4o\"\n },\n \"textModelMaxTokens\": {\n \"value\":\"16384\"\n },\n \"imageModelNames\": {\n \"value\": \"dall-e-3\"\n },\n \"imageSize\": {\n \"value\": \"1024x1024\"\n },\n \"imageModelMaxTokens\": {\n \"value\":\"0\"\n },\n \"embeddingsModelNames\": {\n \"value\": \"text-embedding-ada-002\"\n },\n \"embeddingsModelMaxTokens\": {\n \"value\":\"8191\"\n },\n \"listenerIndexer\": {\n \"value\": \"{\\\"default\\\":\\\"blog,dotcmsdocumentation,feature,ProductBriefs,news,report.file,builds,casestudy\\\",\\\"documentation\\\":\\\"dotcmsdocumentation\\\"}\"\n }\n}\n" + "raw": "{\n \"apiKey\": {\n \"value\": \"some-api-key-1a2bc3\"\n },\n \"textModelNames\": {\n \"value\": \"gpt-4o-mini\"\n },\n \"textModelMaxTokens\": {\n \"value\":\"16384\"\n },\n \"imageModelNames\": {\n \"value\": \"dall-e-3\"\n },\n \"imageSize\": {\n \"value\": \"1024x1024\"\n },\n \"imageModelMaxTokens\": {\n \"value\":\"0\"\n },\n \"embeddingsModelNames\": {\n \"value\": \"text-embedding-ada-002\"\n },\n \"embeddingsModelMaxTokens\": {\n \"value\":\"8191\"\n },\n \"listenerIndexer\": {\n \"value\": \"{\\\"default\\\":\\\"blog,dotcmsdocumentation,feature,ProductBriefs,news,report.file,builds,casestudy\\\",\\\"documentation\\\":\\\"dotcmsdocumentation\\\"}\"\n }\n}\n" }, "url": { "raw": "{{serverURL}}/api/v1/apps/dotAI/SYSTEM_HOST", @@ -3035,7 +3035,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"prompt\": \"{{seoText}}\",\n \"model\": \"text-embedding-ada-002\",\n \"responseLengthTokens\": 1\n}", + "raw": "{\n \"prompt\": \"{{seoText}}\",\n \"responseLengthTokens\": 1\n}", "options": { "raw": { "language": "json" @@ -3111,7 +3111,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"prompt\": \"{{seoText}}\",\n \"responseLengthTokens\": 1,\n \"model\": \"text-embedding-ada-002\",\n \"stream\": true\n}", + "raw": "{\n \"prompt\": \"{{seoText}}\",\n \"responseLengthTokens\": 1,\n \"stream\": true\n}", "options": { "raw": { "language": "json" @@ -3140,7 +3140,7 @@ "listen": "test", "script": { "exec": [ - "pm.test('Status code should be ok 200', function () {", + "pm.test('Status code should be ok 20', function () {", " pm.response.to.have.status(200);", "});", "", @@ -3430,20 +3430,16 @@ } ], "variable": [ - { - "key": "seoIndex", - "value": "" - }, { "key": "seoId", "value": "" }, { - "key": "seoContentTypeId", + "key": "seoText", "value": "" }, { - "key": "seoContentTypeVar", + "key": "seoIndex", "value": "" }, { @@ -3451,11 +3447,11 @@ "value": "" }, { - "key": "seoText", + "key": "seoContentTypeId", "value": "" }, { - "key": "key", + "key": "seoContentTypeVar", "value": "" } ] diff --git a/dotcms-postman/src/test/resources/mappings/apollo-space-program.json b/dotcms-postman/src/test/resources/mappings/apollo-space-program.json new file mode 100644 index 000000000000..74c682e0f7e4 --- /dev/null +++ b/dotcms-postman/src/test/resources/mappings/apollo-space-program.json @@ -0,0 +1,41 @@ +{ + "request": { + "method": "POST", + "url": "/c", + "headers": { + "Content-Type": { + "equalTo": "application/json" + }, + "Authorization": { + "equalTo": "Bearer some-api-key-1a2bc3" + } + }, + "bodyPatterns": [ + { + "matches": ".*\"content\":\"What are the major achievements of the Apollo space program.*" + } + ] + }, + "response": { + "status": 200, + "jsonBody": { + "id": "cmpl-11", + "object": "text_completion", + "created": 1699999999, + "model": "gpt-3.5-turbo-16k", + "choices": [ + { + "text": "The Apollo space program, conducted by NASA, achieved several major milestones in space exploration. Its most significant achievement was the successful landing of humans on the Moon. Apollo 11, in 1969, saw astronauts Neil Armstrong and Buzz Aldrin become the first humans to set foot on the lunar surface. The program also provided extensive scientific data and technological advancements.", + "index": 0, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 57, + "total_tokens": 67 + } + } + } +}