From 9bfc984782331e3671b9f016ef60d4cd54623631 Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Mon, 21 Oct 2024 10:20:15 +0200 Subject: [PATCH 01/10] add token usage logging for Athena --- .../artemis/athena/dto/ResponseMetaDTO.java | 12 ++++++ .../athena/dto/ResponseMetaLLMCallDTO.java | 10 +++++ .../athena/dto/ResponseMetaTotalUsageDTO.java | 10 +++++ .../AthenaFeedbackSuggestionsService.java | 42 ++++++++++++++++--- .../core/service/LLMTokenUsageService.java | 32 ++++++++++++++ 5 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java create mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java create mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java new file mode 100644 index 000000000000..c2b4c1c0c160 --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java @@ -0,0 +1,12 @@ +package de.tum.cit.aet.artemis.athena.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * DTO representing the meta information in the Athena response. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ResponseMetaDTO(ResponseMetaTotalUsageDTO totalUsage, List llmCalls) { +} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java new file mode 100644 index 000000000000..e4da493dd4df --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java @@ -0,0 +1,10 @@ +package de.tum.cit.aet.artemis.athena.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * DTO representing an individual LLM call. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ResponseMetaLLMCallDTO(String modelName, Integer inputTokens, Integer outputTokens, Integer totalTokens) { +} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java new file mode 100644 index 000000000000..55fa8b858e4b --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java @@ -0,0 +1,10 @@ +package de.tum.cit.aet.artemis.athena.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * DTO representing the total usage metrics. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ResponseMetaTotalUsageDTO(Integer inputTokens, Integer outputTokens, Integer totalTokens) { +} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java index d9c81849b396..cff02c9fed1c 100644 --- a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java +++ b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java @@ -17,10 +17,16 @@ import de.tum.cit.aet.artemis.athena.dto.ExerciseBaseDTO; import de.tum.cit.aet.artemis.athena.dto.ModelingFeedbackDTO; import de.tum.cit.aet.artemis.athena.dto.ProgrammingFeedbackDTO; +import de.tum.cit.aet.artemis.athena.dto.ResponseMetaDTO; import de.tum.cit.aet.artemis.athena.dto.SubmissionBaseDTO; import de.tum.cit.aet.artemis.athena.dto.TextFeedbackDTO; +import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.exception.ConflictException; import de.tum.cit.aet.artemis.core.exception.NetworkingException; +import de.tum.cit.aet.artemis.core.service.LLMTokenUsageService; +import de.tum.cit.aet.artemis.exercise.domain.Exercise; +import de.tum.cit.aet.artemis.exercise.domain.Submission; +import de.tum.cit.aet.artemis.exercise.domain.participation.StudentParticipation; import de.tum.cit.aet.artemis.modeling.domain.ModelingExercise; import de.tum.cit.aet.artemis.modeling.domain.ModelingSubmission; import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise; @@ -48,20 +54,24 @@ public class AthenaFeedbackSuggestionsService { private final AthenaDTOConverterService athenaDTOConverterService; + private final LLMTokenUsageService llmTokenUsageService; + /** * Create a new AthenaFeedbackSuggestionsService to receive feedback suggestions from the Athena service. * * @param athenaRestTemplate REST template used for the communication with Athena * @param athenaModuleService Athena module serviced used to determine the urls for different modules - * @param athenaDTOConverterService Service to convert exr + * @param athenaDTOConverterService Service to convert exrcises and submissions to DTOs + * @param llmTokenUsageService Service to store the usage of LLM tokens */ public AthenaFeedbackSuggestionsService(@Qualifier("athenaRestTemplate") RestTemplate athenaRestTemplate, AthenaModuleService athenaModuleService, - AthenaDTOConverterService athenaDTOConverterService) { + AthenaDTOConverterService athenaDTOConverterService, LLMTokenUsageService llmTokenUsageService) { textAthenaConnector = new AthenaConnector<>(athenaRestTemplate, ResponseDTOText.class); programmingAthenaConnector = new AthenaConnector<>(athenaRestTemplate, ResponseDTOProgramming.class); modelingAthenaConnector = new AthenaConnector<>(athenaRestTemplate, ResponseDTOModeling.class); this.athenaDTOConverterService = athenaDTOConverterService; this.athenaModuleService = athenaModuleService; + this.llmTokenUsageService = llmTokenUsageService; } @JsonInclude(JsonInclude.Include.NON_EMPTY) @@ -69,15 +79,15 @@ private record RequestDTO(ExerciseBaseDTO exercise, SubmissionBaseDTO submission } @JsonInclude(JsonInclude.Include.NON_EMPTY) - private record ResponseDTOText(List data) { + private record ResponseDTOText(List data, ResponseMetaDTO meta) { } @JsonInclude(JsonInclude.Include.NON_EMPTY) - private record ResponseDTOProgramming(List data) { + private record ResponseDTOProgramming(List data, ResponseMetaDTO meta) { } @JsonInclude(JsonInclude.Include.NON_EMPTY) - private record ResponseDTOModeling(List data) { + private record ResponseDTOModeling(List data, ResponseMetaDTO meta) { } /** @@ -100,6 +110,7 @@ public List getTextFeedbackSuggestions(TextExercise exercise, T final RequestDTO request = new RequestDTO(athenaDTOConverterService.ofExercise(exercise), athenaDTOConverterService.ofSubmission(exercise.getId(), submission), isGraded); ResponseDTOText response = textAthenaConnector.invokeWithRetry(athenaModuleService.getAthenaModuleUrl(exercise) + "/feedback_suggestions", request, 0); log.info("Athena responded to '{}' feedback suggestions request: {}", isGraded ? "Graded" : "Non Graded", response.data); + storeTokenUsage(exercise, submission, response.meta, !isGraded); return response.data.stream().toList(); } @@ -117,6 +128,7 @@ public List getProgrammingFeedbackSuggestions(Programmin final RequestDTO request = new RequestDTO(athenaDTOConverterService.ofExercise(exercise), athenaDTOConverterService.ofSubmission(exercise.getId(), submission), isGraded); ResponseDTOProgramming response = programmingAthenaConnector.invokeWithRetry(athenaModuleService.getAthenaModuleUrl(exercise) + "/feedback_suggestions", request, 0); log.info("Athena responded to '{}' feedback suggestions request: {}", isGraded ? "Graded" : "Non Graded", response.data); + storeTokenUsage(exercise, submission, response.meta, !isGraded); return response.data.stream().toList(); } @@ -139,6 +151,26 @@ public List getModelingFeedbackSuggestions(ModelingExercise final RequestDTO request = new RequestDTO(athenaDTOConverterService.ofExercise(exercise), athenaDTOConverterService.ofSubmission(exercise.getId(), submission), isGraded); ResponseDTOModeling response = modelingAthenaConnector.invokeWithRetry(athenaModuleService.getAthenaModuleUrl(exercise) + "/feedback_suggestions", request, 0); log.info("Athena responded to '{}' feedback suggestions request: {}", isGraded ? "Graded" : "Non Graded", response.data); + storeTokenUsage(exercise, submission, response.meta, !isGraded); return response.data; } + + /** + * Store the usage of LLM tokens for a given submission + * + * @param exercise the exercise the submission belongs to + * @param submission the submission for which the tokens were used + * @param meta the meta information of the response from Athena + * @param isPreliminaryFeedback whether the feedback is preliminary or not + */ + private void storeTokenUsage(Exercise exercise, Submission submission, ResponseMetaDTO meta, Boolean isPreliminaryFeedback) { + if (meta == null) { + return; + } + Long courseId = exercise.getCourseViaExerciseGroupOrCourseMember().getId(); + Long exerciseId = exercise.getId(); + Long userId = ((StudentParticipation) submission.getParticipation()).getStudent().map(User::getId).orElse(null); + + llmTokenUsageService.saveAthenaTokenUsage(courseId, exerciseId, userId, meta, isPreliminaryFeedback); + } } diff --git a/src/main/java/de/tum/cit/aet/artemis/core/service/LLMTokenUsageService.java b/src/main/java/de/tum/cit/aet/artemis/core/service/LLMTokenUsageService.java index 897ca8278462..65fa9b957d79 100644 --- a/src/main/java/de/tum/cit/aet/artemis/core/service/LLMTokenUsageService.java +++ b/src/main/java/de/tum/cit/aet/artemis/core/service/LLMTokenUsageService.java @@ -5,12 +5,16 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.function.Function; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; +import de.tum.cit.aet.artemis.athena.dto.ResponseMetaDTO; +import de.tum.cit.aet.artemis.athena.dto.ResponseMetaLLMCallDTO; import de.tum.cit.aet.artemis.core.domain.Course; +import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.LLMTokenUsage; import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.repository.LLMTokenUsageRepository; @@ -32,6 +36,34 @@ public LLMTokenUsageService(LLMTokenUsageRepository llmTokenUsageRepository) { this.llmTokenUsageRepository = llmTokenUsageRepository; } + /** + * Method saves the token usage for the Athena response meta to the database + * + * @param courseId of type Long + * @param exerciseId of type Long + * @param userId of type Long + * @param meta of type ResponseMetaDTO + * @param isPreliminaryFeedback of type Boolean + */ + public void saveAthenaTokenUsage(Long courseId, Long exerciseId, Long userId, ResponseMetaDTO meta, Boolean isPreliminaryFeedback) { + String traceId = UUID.randomUUID().toString(); + LLMServiceType serviceType = isPreliminaryFeedback ? LLMServiceType.ATHENA_PRELIMINARY_FEEDBACK : LLMServiceType.ATHENA_FEEDBACK_SUGGESTION; + + List tokenUsages = new ArrayList<>(); + for (ResponseMetaLLMCallDTO llmCall : meta.llmCalls()) { + LLMTokenUsage llmTokenUsage = new LLMTokenUsage(); + llmTokenUsage.setTraceId(traceId); + llmTokenUsage.setCourseId(courseId); + llmTokenUsage.setExerciseId(exerciseId); + llmTokenUsage.setUserId(userId); + llmTokenUsage.setServiceType(serviceType.name()); + llmTokenUsage.setNumInputTokens(llmCall.inputTokens()); + llmTokenUsage.setNumOutputTokens(llmCall.outputTokens()); + llmTokenUsage.setModel(llmCall.modelName()); + } + llmTokenUsageRepository.saveAll(tokenUsages); + } + /** * method saves the token usage to the database with a link to the IrisMessage * messages of the same job are grouped together by saving the job id as a trace id From 0719524aa84b31f0f3fb4bf6dec31c9b828021f7 Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Mon, 21 Oct 2024 12:03:13 +0200 Subject: [PATCH 02/10] save llm token usage --- .../AthenaFeedbackSuggestionsService.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java index cff02c9fed1c..046bba71da2c 100644 --- a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java +++ b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java @@ -20,6 +20,9 @@ import de.tum.cit.aet.artemis.athena.dto.ResponseMetaDTO; import de.tum.cit.aet.artemis.athena.dto.SubmissionBaseDTO; import de.tum.cit.aet.artemis.athena.dto.TextFeedbackDTO; +import de.tum.cit.aet.artemis.core.domain.Course; +import de.tum.cit.aet.artemis.core.domain.LLMRequest; +import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.exception.ConflictException; import de.tum.cit.aet.artemis.core.exception.NetworkingException; @@ -167,10 +170,16 @@ private void storeTokenUsage(Exercise exercise, Submission submission, ResponseM if (meta == null) { return; } - Long courseId = exercise.getCourseViaExerciseGroupOrCourseMember().getId(); - Long exerciseId = exercise.getId(); - Long userId = ((StudentParticipation) submission.getParticipation()).getStudent().map(User::getId).orElse(null); - llmTokenUsageService.saveAthenaTokenUsage(courseId, exerciseId, userId, meta, isPreliminaryFeedback); + Course course = exercise.getCourseViaExerciseGroupOrCourseMember(); + User user = ((StudentParticipation) submission.getParticipation()).getStudent().orElse(null); + + // Temporary pipelineId until Athena provides it in the response + String pipelineId = isPreliminaryFeedback ? "PRELIMINARY_FEEDBACK" : "FEEDBACK_SUGGESTION"; + + List llmRequests = meta.llmCalls().stream().map(llmCall -> new LLMRequest(llmCall.modelName(), llmCall.inputTokens(), 0, llmCall.outputTokens(), 0, pipelineId)) + .toList(); + llmTokenUsageService.saveLLMTokenUsage(llmRequests, LLMServiceType.ATHENA, + (llmTokenUsageBuilder -> llmTokenUsageBuilder.withCourse(course).withExercise(exercise).withUser(user))); } } From 9f4cccd936ad80a42f0fa722c1cd41950f14521f Mon Sep 17 00:00:00 2001 From: Patrick Bassner Date: Mon, 21 Oct 2024 16:58:28 +0200 Subject: [PATCH 03/10] Refactored token usage tracking and improved session-based job handling --- .../core/service/LLMTokenUsageService.java | 89 +++++++++---------- .../IrisCompetencyGenerationService.java | 14 ++- .../pyris/job/CompetencyExtractionJob.java | 5 +- .../iris/service/pyris/job/CourseChatJob.java | 2 +- .../service/pyris/job/ExerciseChatJob.java | 2 +- .../pyris/job/SessionBasedPyrisJob.java | 9 ++ .../AbstractIrisChatSessionService.java | 74 ++++++++++++++- .../session/IrisCourseChatSessionService.java | 45 +--------- .../IrisExerciseChatSessionService.java | 61 ++----------- ...isCompetencyGenerationIntegrationTest.java | 2 +- 10 files changed, 150 insertions(+), 153 deletions(-) create mode 100644 src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/SessionBasedPyrisJob.java diff --git a/src/main/java/de/tum/cit/aet/artemis/core/service/LLMTokenUsageService.java b/src/main/java/de/tum/cit/aet/artemis/core/service/LLMTokenUsageService.java index 2d48bfcf73fd..5ffe5f379ff5 100644 --- a/src/main/java/de/tum/cit/aet/artemis/core/service/LLMTokenUsageService.java +++ b/src/main/java/de/tum/cit/aet/artemis/core/service/LLMTokenUsageService.java @@ -2,24 +2,20 @@ import static de.tum.cit.aet.artemis.core.config.Constants.PROFILE_CORE; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import de.tum.cit.aet.artemis.core.domain.Course; import de.tum.cit.aet.artemis.core.domain.LLMRequest; import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.LLMTokenUsageRequest; import de.tum.cit.aet.artemis.core.domain.LLMTokenUsageTrace; -import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.repository.LLMTokenUsageRequestRepository; import de.tum.cit.aet.artemis.core.repository.LLMTokenUsageTraceRepository; -import de.tum.cit.aet.artemis.exercise.domain.Exercise; /** * Service for managing the LLMTokenUsage by all LLMs in Artemis @@ -38,12 +34,17 @@ public LLMTokenUsageService(LLMTokenUsageTraceRepository llmTokenUsageTraceRepos } /** - * method saves the token usage to the database + * Saves the token usage to the database. + * This method records the usage of tokens by various LLM services in the system. * - * @param llmRequests List of LLM requests - * @param serviceType type of the LLM service - * @param builderFunction of type Function using IrisTokenUsageBuilder - * @return saved LLMTokenUsage as a List + * @param llmRequests List of LLM requests containing details about the token usage. + * @param serviceType Type of the LLM service (e.g., IRIS, GPT-3). + * @param builderFunction A function that takes an LLMTokenUsageBuilder and returns a modified LLMTokenUsageBuilder. + * This function is used to set additional properties on the LLMTokenUsageTrace object, such as + * the course ID, user ID, exercise ID, and Iris message ID. + * Example usage: + * builder -> builder.withCourse(courseId).withUser(userId) + * @return The saved LLMTokenUsageTrace object, which includes the details of the token usage. */ // TODO: this should ideally be done Async public LLMTokenUsageTrace saveLLMTokenUsage(List llmRequests, LLMServiceType serviceType, Function builderFunction) { @@ -52,34 +53,32 @@ public LLMTokenUsageTrace saveLLMTokenUsage(List llmRequests, LLMSer LLMTokenUsageBuilder builder = builderFunction.apply(new LLMTokenUsageBuilder()); builder.getIrisMessageID().ifPresent(llmTokenUsageTrace::setIrisMessageId); - builder.getUser().ifPresent(user -> llmTokenUsageTrace.setUserId(user.getId())); - builder.getExercise().ifPresent(exercise -> llmTokenUsageTrace.setExerciseId(exercise.getId())); - builder.getCourse().ifPresent(course -> llmTokenUsageTrace.setCourseId(course.getId())); + builder.getCourseID().ifPresent(llmTokenUsageTrace::setCourseId); + builder.getExerciseID().ifPresent(llmTokenUsageTrace::setExerciseId); + builder.getUserID().ifPresent(llmTokenUsageTrace::setUserId); + + llmTokenUsageTrace.setLlmRequests(llmRequests.stream().map(LLMTokenUsageService::convertLLMRequestToLLMTokenUsageRequest) + .peek(llmTokenUsageRequest -> llmTokenUsageRequest.setTrace(llmTokenUsageTrace)).collect(Collectors.toSet())); - Set llmRequestsSet = llmTokenUsageTrace.getLLMRequests(); - setLLMTokenUsageRequests(llmRequests, llmTokenUsageTrace, llmRequestsSet); return llmTokenUsageTraceRepository.save(llmTokenUsageTrace); } - private void setLLMTokenUsageRequests(List llmRequests, LLMTokenUsageTrace llmTokenUsageTrace, Set llmRequestsSet) { - for (LLMRequest llmRequest : llmRequests) { - LLMTokenUsageRequest llmTokenUsageRequest = new LLMTokenUsageRequest(); - llmTokenUsageRequest.setModel(llmRequest.model()); - llmTokenUsageRequest.setNumInputTokens(llmRequest.numInputTokens()); - llmTokenUsageRequest.setNumOutputTokens(llmRequest.numOutputTokens()); - llmTokenUsageRequest.setCostPerMillionInputTokens(llmRequest.costPerMillionInputToken()); - llmTokenUsageRequest.setCostPerMillionOutputTokens(llmRequest.costPerMillionOutputToken()); - llmTokenUsageRequest.setServicePipelineId(llmRequest.pipelineId()); - llmTokenUsageRequest.setTrace(llmTokenUsageTrace); - llmRequestsSet.add(llmTokenUsageRequest); - } + private static LLMTokenUsageRequest convertLLMRequestToLLMTokenUsageRequest(LLMRequest llmRequest) { + LLMTokenUsageRequest llmTokenUsageRequest = new LLMTokenUsageRequest(); + llmTokenUsageRequest.setModel(llmRequest.model()); + llmTokenUsageRequest.setNumInputTokens(llmRequest.numInputTokens()); + llmTokenUsageRequest.setNumOutputTokens(llmRequest.numOutputTokens()); + llmTokenUsageRequest.setCostPerMillionInputTokens(llmRequest.costPerMillionInputToken()); + llmTokenUsageRequest.setCostPerMillionOutputTokens(llmRequest.costPerMillionOutputToken()); + llmTokenUsageRequest.setServicePipelineId(llmRequest.pipelineId()); + return llmTokenUsageRequest; } // TODO: this should ideally be done Async public void appendRequestsToTrace(List requests, LLMTokenUsageTrace trace) { - Set llmRequestsSet = new HashSet<>(); - setLLMTokenUsageRequests(requests, trace, llmRequestsSet); - llmTokenUsageRequestRepository.saveAll(llmRequestsSet); + var requestSet = requests.stream().map(LLMTokenUsageService::convertLLMRequestToLLMTokenUsageRequest).peek(llmTokenUsageRequest -> llmTokenUsageRequest.setTrace(trace)) + .collect(Collectors.toSet()); + llmTokenUsageRequestRepository.saveAll(requestSet); } /** @@ -87,16 +86,16 @@ public void appendRequestsToTrace(List requests, LLMTokenUsageTrace */ public static class LLMTokenUsageBuilder { - private Optional course = Optional.empty(); + private Optional courseID = Optional.empty(); private Optional irisMessageID = Optional.empty(); - private Optional exercise = Optional.empty(); + private Optional exerciseID = Optional.empty(); - private Optional user = Optional.empty(); + private Optional userID = Optional.empty(); - public LLMTokenUsageBuilder withCourse(Course course) { - this.course = Optional.ofNullable(course); + public LLMTokenUsageBuilder withCourse(Long courseID) { + this.courseID = Optional.ofNullable(courseID); return this; } @@ -105,30 +104,30 @@ public LLMTokenUsageBuilder withIrisMessageID(Long irisMessageID) { return this; } - public LLMTokenUsageBuilder withExercise(Exercise exercise) { - this.exercise = Optional.ofNullable(exercise); + public LLMTokenUsageBuilder withExercise(Long exerciseID) { + this.exerciseID = Optional.ofNullable(exerciseID); return this; } - public LLMTokenUsageBuilder withUser(User user) { - this.user = Optional.ofNullable(user); + public LLMTokenUsageBuilder withUser(Long userID) { + this.userID = Optional.ofNullable(userID); return this; } - public Optional getCourse() { - return course; + public Optional getCourseID() { + return courseID; } public Optional getIrisMessageID() { return irisMessageID; } - public Optional getExercise() { - return exercise; + public Optional getExerciseID() { + return exerciseID; } - public Optional getUser() { - return user; + public Optional getUserID() { + return userID; } } } diff --git a/src/main/java/de/tum/cit/aet/artemis/iris/service/IrisCompetencyGenerationService.java b/src/main/java/de/tum/cit/aet/artemis/iris/service/IrisCompetencyGenerationService.java index e307366db077..f8d2a0201198 100644 --- a/src/main/java/de/tum/cit/aet/artemis/iris/service/IrisCompetencyGenerationService.java +++ b/src/main/java/de/tum/cit/aet/artemis/iris/service/IrisCompetencyGenerationService.java @@ -10,6 +10,7 @@ import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.repository.CourseRepository; +import de.tum.cit.aet.artemis.core.repository.UserRepository; import de.tum.cit.aet.artemis.core.service.LLMTokenUsageService; import de.tum.cit.aet.artemis.iris.service.pyris.PyrisJobService; import de.tum.cit.aet.artemis.iris.service.pyris.PyrisPipelineService; @@ -36,13 +37,16 @@ public class IrisCompetencyGenerationService { private final PyrisJobService pyrisJobService; + private final UserRepository userRepository; + public IrisCompetencyGenerationService(PyrisPipelineService pyrisPipelineService, LLMTokenUsageService llmTokenUsageService, CourseRepository courseRepository, - IrisWebsocketService websocketService, PyrisJobService pyrisJobService) { + IrisWebsocketService websocketService, PyrisJobService pyrisJobService, UserRepository userRepository) { this.pyrisPipelineService = pyrisPipelineService; this.llmTokenUsageService = llmTokenUsageService; this.courseRepository = courseRepository; this.websocketService = websocketService; this.pyrisJobService = pyrisJobService; + this.userRepository = userRepository; } /** @@ -58,7 +62,7 @@ public void executeCompetencyExtractionPipeline(User user, Course course, String pyrisPipelineService.executePipeline( "competency-extraction", "default", - pyrisJobService.createTokenForJob(token -> new CompetencyExtractionJob(token, course.getId(), user)), + pyrisJobService.createTokenForJob(token -> new CompetencyExtractionJob(token, course.getId(), user.getId())), executionDto -> new PyrisCompetencyExtractionPipelineExecutionDTO(executionDto, courseDescription, currentCompetencies, CompetencyTaxonomy.values(), 5), stages -> websocketService.send(user.getLogin(), websocketTopic(course.getId()), new PyrisCompetencyStatusUpdateDTO(stages, null, null)) ); @@ -74,9 +78,11 @@ public void executeCompetencyExtractionPipeline(User user, Course course, String public void handleStatusUpdate(CompetencyExtractionJob job, PyrisCompetencyStatusUpdateDTO statusUpdate) { Course course = courseRepository.findByIdForUpdateElseThrow(job.courseId()); if (statusUpdate.tokens() != null && !statusUpdate.tokens().isEmpty()) { - llmTokenUsageService.saveLLMTokenUsage(statusUpdate.tokens(), LLMServiceType.IRIS, builder -> builder.withCourse(course).withUser(job.user())); + llmTokenUsageService.saveLLMTokenUsage(statusUpdate.tokens(), LLMServiceType.IRIS, builder -> builder.withCourse(course.getId()).withUser(job.userId())); } - websocketService.send(job.user().getLogin(), websocketTopic(job.courseId()), statusUpdate); + + var user = userRepository.findById(job.userId()).orElseThrow(); + websocketService.send(user.getLogin(), websocketTopic(job.courseId()), statusUpdate); } private static String websocketTopic(long courseId) { diff --git a/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/CompetencyExtractionJob.java b/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/CompetencyExtractionJob.java index 136a1a5ae243..b50d8e70b8c9 100644 --- a/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/CompetencyExtractionJob.java +++ b/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/CompetencyExtractionJob.java @@ -3,17 +3,16 @@ import com.fasterxml.jackson.annotation.JsonInclude; import de.tum.cit.aet.artemis.core.domain.Course; -import de.tum.cit.aet.artemis.core.domain.User; /** * A pyris job that extracts competencies from a course description. * * @param jobId the job id * @param courseId the course in which the competencies are being extracted - * @param user the user who started the job + * @param userId the user who started the job */ @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record CompetencyExtractionJob(String jobId, long courseId, User user) implements PyrisJob { +public record CompetencyExtractionJob(String jobId, long courseId, long userId) implements PyrisJob { @Override public boolean canAccess(Course course) { diff --git a/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/CourseChatJob.java b/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/CourseChatJob.java index fb4b93a28854..c05cbf9b94ea 100644 --- a/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/CourseChatJob.java +++ b/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/CourseChatJob.java @@ -9,7 +9,7 @@ * This job is used to reference the details of a course chat session when Pyris sends a status update. */ @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record CourseChatJob(String jobId, long courseId, long sessionId) implements PyrisJob { +public record CourseChatJob(String jobId, long courseId, long sessionId) implements SessionBasedPyrisJob { @Override public boolean canAccess(Course course) { diff --git a/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/ExerciseChatJob.java b/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/ExerciseChatJob.java index 302ae274d8e2..1c2278cb2697 100644 --- a/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/ExerciseChatJob.java +++ b/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/ExerciseChatJob.java @@ -10,7 +10,7 @@ * This job is used to reference the details of a exercise chat session when Pyris sends a status update. */ @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record ExerciseChatJob(String jobId, long courseId, long exerciseId, long sessionId) implements PyrisJob { +public record ExerciseChatJob(String jobId, long courseId, long exerciseId, long sessionId) implements SessionBasedPyrisJob { @Override public boolean canAccess(Course course) { diff --git a/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/SessionBasedPyrisJob.java b/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/SessionBasedPyrisJob.java new file mode 100644 index 000000000000..03c2e4007838 --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/job/SessionBasedPyrisJob.java @@ -0,0 +1,9 @@ +package de.tum.cit.aet.artemis.iris.service.pyris.job; + +/** + * An interface Pyris job that is associated with a session. + */ +public interface SessionBasedPyrisJob extends PyrisJob { + + long sessionId(); +} diff --git a/src/main/java/de/tum/cit/aet/artemis/iris/service/session/AbstractIrisChatSessionService.java b/src/main/java/de/tum/cit/aet/artemis/iris/service/session/AbstractIrisChatSessionService.java index 559e21668775..16df99a68337 100644 --- a/src/main/java/de/tum/cit/aet/artemis/iris/service/session/AbstractIrisChatSessionService.java +++ b/src/main/java/de/tum/cit/aet/artemis/iris/service/session/AbstractIrisChatSessionService.java @@ -6,21 +6,40 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.LLMTokenUsageTrace; +import de.tum.cit.aet.artemis.core.service.LLMTokenUsageService; +import de.tum.cit.aet.artemis.iris.domain.message.IrisMessage; +import de.tum.cit.aet.artemis.iris.domain.message.IrisMessageSender; +import de.tum.cit.aet.artemis.iris.domain.message.IrisTextMessageContent; import de.tum.cit.aet.artemis.iris.domain.session.IrisChatSession; import de.tum.cit.aet.artemis.iris.repository.IrisSessionRepository; +import de.tum.cit.aet.artemis.iris.service.IrisMessageService; +import de.tum.cit.aet.artemis.iris.service.pyris.dto.chat.PyrisChatStatusUpdateDTO; +import de.tum.cit.aet.artemis.iris.service.pyris.job.SessionBasedPyrisJob; +import de.tum.cit.aet.artemis.iris.service.websocket.IrisChatWebsocketService; public abstract class AbstractIrisChatSessionService implements IrisChatBasedFeatureInterface, IrisRateLimitedFeatureInterface { private final IrisSessionRepository irisSessionRepository; + private final IrisMessageService irisMessageService; + + private final IrisChatWebsocketService irisChatWebsocketService; + + private final LLMTokenUsageService llmTokenUsageService; + private final ObjectMapper objectMapper; protected final HashMap traces = new HashMap<>(); - public AbstractIrisChatSessionService(IrisSessionRepository irisSessionRepository, ObjectMapper objectMapper) { + public AbstractIrisChatSessionService(IrisSessionRepository irisSessionRepository, ObjectMapper objectMapper, IrisMessageService irisMessageService, + IrisChatWebsocketService irisChatWebsocketService, LLMTokenUsageService llmTokenUsageService) { this.irisSessionRepository = irisSessionRepository; this.objectMapper = objectMapper; + this.irisMessageService = irisMessageService; + this.irisChatWebsocketService = irisChatWebsocketService; + this.llmTokenUsageService = llmTokenUsageService; } /** @@ -44,4 +63,57 @@ protected void updateLatestSuggestions(S session, List latestSuggestions throw new RuntimeException("Could not update latest suggestions for session " + session.getId(), e); } } + + /** + * Handles the status update of a ExerciseChatJob by sending the result to the student via the Websocket. + * + * @param job The job that was executed + * @param statusUpdate The status update of the job + */ + public void handleStatusUpdate(SessionBasedPyrisJob job, PyrisChatStatusUpdateDTO statusUpdate) { + var session = (S) irisSessionRepository.findByIdWithMessagesAndContents(job.sessionId()); + IrisMessage savedMessage; + if (statusUpdate.result() != null) { + var message = new IrisMessage(); + message.addContent(new IrisTextMessageContent(statusUpdate.result())); + savedMessage = irisMessageService.saveMessage(message, session, IrisMessageSender.LLM); + irisChatWebsocketService.sendMessage(session, savedMessage, statusUpdate.stages()); + } + else { + savedMessage = null; + irisChatWebsocketService.sendStatusUpdate(session, statusUpdate.stages(), statusUpdate.suggestions(), statusUpdate.tokens()); + } + + if (statusUpdate.tokens() != null && !statusUpdate.tokens().isEmpty()) { + if (savedMessage != null) { + // generated message is first sent and generated trace is saved + var llmTokenUsageTrace = llmTokenUsageService.saveLLMTokenUsage(statusUpdate.tokens(), LLMServiceType.IRIS, builder -> { + builder.withIrisMessageID(savedMessage.getId()).withUser(session.getUser().getId()); + this.setLLMTokenUsageParameters(builder, session); + return builder; + }); + traces.put(job.jobId(), llmTokenUsageTrace); + } + else { + // interaction suggestion is sent and appended to the generated trace if it exists, trace is then removed, + // because interaction suggestion is the last message from Iris in the pipeline + if (traces.containsKey(job.jobId())) { + var trace = traces.get(job.jobId()); + llmTokenUsageService.appendRequestsToTrace(statusUpdate.tokens(), trace); + traces.remove(job.jobId()); + } + else { + llmTokenUsageService.saveLLMTokenUsage(statusUpdate.tokens(), LLMServiceType.IRIS, builder -> { + builder.withUser(session.getUser().getId()); + this.setLLMTokenUsageParameters(builder, session); + return builder; + }); + } + } + } + + updateLatestSuggestions(session, statusUpdate.suggestions()); + } + + protected abstract void setLLMTokenUsageParameters(LLMTokenUsageService.LLMTokenUsageBuilder builder, S session); } diff --git a/src/main/java/de/tum/cit/aet/artemis/iris/service/session/IrisCourseChatSessionService.java b/src/main/java/de/tum/cit/aet/artemis/iris/service/session/IrisCourseChatSessionService.java index 75c8eb1430da..d2743c2e71a5 100644 --- a/src/main/java/de/tum/cit/aet/artemis/iris/service/session/IrisCourseChatSessionService.java +++ b/src/main/java/de/tum/cit/aet/artemis/iris/service/session/IrisCourseChatSessionService.java @@ -15,15 +15,12 @@ import de.tum.cit.aet.artemis.atlas.domain.competency.CompetencyJol; import de.tum.cit.aet.artemis.core.domain.Course; -import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.exception.AccessForbiddenException; import de.tum.cit.aet.artemis.core.security.Role; import de.tum.cit.aet.artemis.core.service.AuthorizationCheckService; import de.tum.cit.aet.artemis.core.service.LLMTokenUsageService; import de.tum.cit.aet.artemis.iris.domain.message.IrisMessage; -import de.tum.cit.aet.artemis.iris.domain.message.IrisMessageSender; -import de.tum.cit.aet.artemis.iris.domain.message.IrisTextMessageContent; import de.tum.cit.aet.artemis.iris.domain.session.IrisCourseChatSession; import de.tum.cit.aet.artemis.iris.domain.settings.IrisSubSettingsType; import de.tum.cit.aet.artemis.iris.repository.IrisCourseChatSessionRepository; @@ -31,8 +28,6 @@ import de.tum.cit.aet.artemis.iris.service.IrisMessageService; import de.tum.cit.aet.artemis.iris.service.IrisRateLimitService; import de.tum.cit.aet.artemis.iris.service.pyris.PyrisPipelineService; -import de.tum.cit.aet.artemis.iris.service.pyris.dto.chat.PyrisChatStatusUpdateDTO; -import de.tum.cit.aet.artemis.iris.service.pyris.job.CourseChatJob; import de.tum.cit.aet.artemis.iris.service.settings.IrisSettingsService; import de.tum.cit.aet.artemis.iris.service.websocket.IrisChatWebsocketService; @@ -43,10 +38,6 @@ @Profile(PROFILE_IRIS) public class IrisCourseChatSessionService extends AbstractIrisChatSessionService { - private final IrisMessageService irisMessageService; - - private final LLMTokenUsageService llmTokenUsageService; - private final IrisSettingsService irisSettingsService; private final IrisChatWebsocketService irisChatWebsocketService; @@ -65,9 +56,7 @@ public IrisCourseChatSessionService(IrisMessageService irisMessageService, LLMTo IrisChatWebsocketService irisChatWebsocketService, AuthorizationCheckService authCheckService, IrisSessionRepository irisSessionRepository, IrisRateLimitService rateLimitService, IrisCourseChatSessionRepository irisCourseChatSessionRepository, PyrisPipelineService pyrisPipelineService, ObjectMapper objectMapper) { - super(irisSessionRepository, objectMapper); - this.irisMessageService = irisMessageService; - this.llmTokenUsageService = llmTokenUsageService; + super(irisSessionRepository, objectMapper, irisMessageService, irisChatWebsocketService, llmTokenUsageService); this.irisSettingsService = irisSettingsService; this.irisChatWebsocketService = irisChatWebsocketService; this.authCheckService = authCheckService; @@ -132,35 +121,9 @@ private void requestAndHandleResponse(IrisCourseChatSession session, String vari pyrisPipelineService.executeCourseChatPipeline(variant, chatSession, competencyJol); } - /** - * Handles the status update of a CourseChatJob by sending the result to the student via the Websocket. - * - * @param job The job that was executed - * @param statusUpdate The status update of the job - */ - public void handleStatusUpdate(CourseChatJob job, PyrisChatStatusUpdateDTO statusUpdate) { - var session = (IrisCourseChatSession) irisSessionRepository.findByIdWithMessagesAndContents(job.sessionId()); - IrisMessage savedMessage; - if (statusUpdate.result() != null) { - var message = new IrisMessage(); - message.addContent(new IrisTextMessageContent(statusUpdate.result())); - savedMessage = irisMessageService.saveMessage(message, session, IrisMessageSender.LLM); - irisChatWebsocketService.sendMessage(session, savedMessage, statusUpdate.stages()); - } - else { - savedMessage = null; - irisChatWebsocketService.sendStatusUpdate(session, statusUpdate.stages(), statusUpdate.suggestions(), statusUpdate.tokens()); - } - if (statusUpdate.tokens() != null && !statusUpdate.tokens().isEmpty()) { - if (savedMessage != null) { - llmTokenUsageService.saveLLMTokenUsage(statusUpdate.tokens(), LLMServiceType.IRIS, - builder -> builder.withIrisMessageID(savedMessage.getId()).withUser(session.getUser()).withCourse(session.getCourse())); - } - else { - llmTokenUsageService.saveLLMTokenUsage(statusUpdate.tokens(), LLMServiceType.IRIS, builder -> builder.withUser(session.getUser()).withCourse(session.getCourse())); - } - } - updateLatestSuggestions(session, statusUpdate.suggestions()); + @Override + protected void setLLMTokenUsageParameters(LLMTokenUsageService.LLMTokenUsageBuilder builder, IrisCourseChatSession session) { + builder.withCourse(session.getCourse().getId()); } /** diff --git a/src/main/java/de/tum/cit/aet/artemis/iris/service/session/IrisExerciseChatSessionService.java b/src/main/java/de/tum/cit/aet/artemis/iris/service/session/IrisExerciseChatSessionService.java index 93db671f2783..a51f1730e98c 100644 --- a/src/main/java/de/tum/cit/aet/artemis/iris/service/session/IrisExerciseChatSessionService.java +++ b/src/main/java/de/tum/cit/aet/artemis/iris/service/session/IrisExerciseChatSessionService.java @@ -10,7 +10,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; -import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.exception.AccessForbiddenException; import de.tum.cit.aet.artemis.core.exception.ConflictException; @@ -19,16 +18,12 @@ import de.tum.cit.aet.artemis.core.service.LLMTokenUsageService; import de.tum.cit.aet.artemis.exercise.domain.Submission; import de.tum.cit.aet.artemis.iris.domain.message.IrisMessage; -import de.tum.cit.aet.artemis.iris.domain.message.IrisMessageSender; -import de.tum.cit.aet.artemis.iris.domain.message.IrisTextMessageContent; import de.tum.cit.aet.artemis.iris.domain.session.IrisExerciseChatSession; import de.tum.cit.aet.artemis.iris.domain.settings.IrisSubSettingsType; import de.tum.cit.aet.artemis.iris.repository.IrisSessionRepository; import de.tum.cit.aet.artemis.iris.service.IrisMessageService; import de.tum.cit.aet.artemis.iris.service.IrisRateLimitService; import de.tum.cit.aet.artemis.iris.service.pyris.PyrisPipelineService; -import de.tum.cit.aet.artemis.iris.service.pyris.dto.chat.PyrisChatStatusUpdateDTO; -import de.tum.cit.aet.artemis.iris.service.pyris.job.ExerciseChatJob; import de.tum.cit.aet.artemis.iris.service.settings.IrisSettingsService; import de.tum.cit.aet.artemis.iris.service.websocket.IrisChatWebsocketService; import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise; @@ -44,10 +39,6 @@ @Profile(PROFILE_IRIS) public class IrisExerciseChatSessionService extends AbstractIrisChatSessionService implements IrisRateLimitedFeatureInterface { - private final IrisMessageService irisMessageService; - - private final LLMTokenUsageService llmTokenUsageService; - private final IrisSettingsService irisSettingsService; private final IrisChatWebsocketService irisChatWebsocketService; @@ -71,9 +62,7 @@ public IrisExerciseChatSessionService(IrisMessageService irisMessageService, LLM ProgrammingExerciseStudentParticipationRepository programmingExerciseStudentParticipationRepository, ProgrammingSubmissionRepository programmingSubmissionRepository, IrisRateLimitService rateLimitService, PyrisPipelineService pyrisPipelineService, ProgrammingExerciseRepository programmingExerciseRepository, ObjectMapper objectMapper) { - super(irisSessionRepository, objectMapper); - this.irisMessageService = irisMessageService; - this.llmTokenUsageService = llmTokenUsageService; + super(irisSessionRepository, objectMapper, irisMessageService, irisChatWebsocketService, llmTokenUsageService); this.irisSettingsService = irisSettingsService; this.irisChatWebsocketService = irisChatWebsocketService; this.authCheckService = authCheckService; @@ -163,49 +152,9 @@ private Optional getLatestSubmissionIfExists(ProgrammingE .flatMap(sub -> programmingSubmissionRepository.findWithEagerResultsAndFeedbacksAndBuildLogsById(sub.getId())); } - /** - * Handles the status update of a ExerciseChatJob by sending the result to the student via the Websocket. - * - * @param job The job that was executed - * @param statusUpdate The status update of the job - */ - public void handleStatusUpdate(ExerciseChatJob job, PyrisChatStatusUpdateDTO statusUpdate) { - var session = (IrisExerciseChatSession) irisSessionRepository.findByIdWithMessagesAndContents(job.sessionId()); - IrisMessage savedMessage; - if (statusUpdate.result() != null) { - var message = new IrisMessage(); - message.addContent(new IrisTextMessageContent(statusUpdate.result())); - savedMessage = irisMessageService.saveMessage(message, session, IrisMessageSender.LLM); - irisChatWebsocketService.sendMessage(session, savedMessage, statusUpdate.stages()); - } - else { - savedMessage = null; - irisChatWebsocketService.sendStatusUpdate(session, statusUpdate.stages(), statusUpdate.suggestions(), statusUpdate.tokens()); - } - - if (statusUpdate.tokens() != null && !statusUpdate.tokens().isEmpty()) { - if (savedMessage != null) { - // generated message is first sent and generated trace is saved - var llmTokenUsageTrace = llmTokenUsageService.saveLLMTokenUsage(statusUpdate.tokens(), LLMServiceType.IRIS, - builder -> builder.withIrisMessageID(savedMessage.getId()).withExercise(session.getExercise()).withUser(session.getUser()) - .withCourse(session.getExercise().getCourseViaExerciseGroupOrCourseMember())); - traces.put(job.jobId(), llmTokenUsageTrace); - } - else { - // interaction suggestion is sent and appended to the generated trace if it exists, trace is then removed, - // because interaction suggestion is the last message from Iris in the pipeline - if (traces.containsKey(job.jobId())) { - var trace = traces.get(job.jobId()); - llmTokenUsageService.appendRequestsToTrace(statusUpdate.tokens(), trace); - traces.remove(job.jobId()); - } - else { - llmTokenUsageService.saveLLMTokenUsage(statusUpdate.tokens(), LLMServiceType.IRIS, builder -> builder.withExercise(session.getExercise()) - .withUser(session.getUser()).withCourse(session.getExercise().getCourseViaExerciseGroupOrCourseMember())); - } - } - } - - updateLatestSuggestions(session, statusUpdate.suggestions()); + @Override + protected void setLLMTokenUsageParameters(LLMTokenUsageService.LLMTokenUsageBuilder builder, IrisExerciseChatSession session) { + var exercise = session.getExercise(); + builder.withCourse(exercise.getCourseViaExerciseGroupOrCourseMember().getId()).withExercise(exercise.getId()); } } diff --git a/src/test/java/de/tum/cit/aet/artemis/iris/IrisCompetencyGenerationIntegrationTest.java b/src/test/java/de/tum/cit/aet/artemis/iris/IrisCompetencyGenerationIntegrationTest.java index 24085a97f70c..7b7279a25053 100644 --- a/src/test/java/de/tum/cit/aet/artemis/iris/IrisCompetencyGenerationIntegrationTest.java +++ b/src/test/java/de/tum/cit/aet/artemis/iris/IrisCompetencyGenerationIntegrationTest.java @@ -69,7 +69,7 @@ void generateCompetencies_asEditor_shouldSucceed() throws Exception { // In the real system, this would be triggered by Pyris via a REST call to the Artemis server String jobId = "testJobId"; String userLogin = TEST_PREFIX + "editor1"; - CompetencyExtractionJob job = new CompetencyExtractionJob(jobId, course.getId(), userUtilService.getUserByLogin(userLogin)); + CompetencyExtractionJob job = new CompetencyExtractionJob(jobId, course.getId(), userUtilService.getUserByLogin(userLogin).getId()); irisCompetencyGenerationService.handleStatusUpdate(job, new PyrisCompetencyStatusUpdateDTO(stages, recommendations, null)); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PyrisCompetencyStatusUpdateDTO.class); From e437c71665599f240afff627137461d48082d14c Mon Sep 17 00:00:00 2001 From: Alexander Joham Date: Mon, 21 Oct 2024 17:39:10 +0200 Subject: [PATCH 04/10] Update tests to work with new changes --- .../aet/artemis/iris/IrisChatTokenTrackingIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/de/tum/cit/aet/artemis/iris/IrisChatTokenTrackingIntegrationTest.java b/src/test/java/de/tum/cit/aet/artemis/iris/IrisChatTokenTrackingIntegrationTest.java index 6ae7ebf00400..adb5b009809f 100644 --- a/src/test/java/de/tum/cit/aet/artemis/iris/IrisChatTokenTrackingIntegrationTest.java +++ b/src/test/java/de/tum/cit/aet/artemis/iris/IrisChatTokenTrackingIntegrationTest.java @@ -160,7 +160,7 @@ void testTokenTrackingSavedExerciseChat() { irisMessageRepository.save(irisMessage); var tokens = getMockLLMCosts(); LLMTokenUsageTrace tokenUsageTrace = llmTokenUsageService.saveLLMTokenUsage(tokens, LLMServiceType.IRIS, - builder -> builder.withIrisMessageID(irisMessage.getId()).withExercise(exercise).withUser(irisSession.getUser()).withCourse(course)); + builder -> builder.withIrisMessageID(irisMessage.getId()).withExercise(exercise.getId()).withUser(irisSession.getUser().getId()).withCourse(course.getId())); assertThat(tokenUsageTrace.getServiceType()).isEqualTo(LLMServiceType.IRIS); assertThat(tokenUsageTrace.getIrisMessageId()).isEqualTo(irisMessage.getId()); assertThat(tokenUsageTrace.getExerciseId()).isEqualTo(exercise.getId()); From e974c4e06ff16e4d15c7ac7b5932a09a8253e0ad Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Mon, 21 Oct 2024 10:20:15 +0200 Subject: [PATCH 05/10] add token usage logging for Athena --- .../artemis/athena/dto/ResponseMetaDTO.java | 12 ++++++ .../athena/dto/ResponseMetaLLMCallDTO.java | 10 +++++ .../athena/dto/ResponseMetaTotalUsageDTO.java | 10 +++++ .../AthenaFeedbackSuggestionsService.java | 42 ++++++++++++++++--- 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java create mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java create mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java new file mode 100644 index 000000000000..c2b4c1c0c160 --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java @@ -0,0 +1,12 @@ +package de.tum.cit.aet.artemis.athena.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * DTO representing the meta information in the Athena response. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ResponseMetaDTO(ResponseMetaTotalUsageDTO totalUsage, List llmCalls) { +} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java new file mode 100644 index 000000000000..e4da493dd4df --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java @@ -0,0 +1,10 @@ +package de.tum.cit.aet.artemis.athena.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * DTO representing an individual LLM call. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ResponseMetaLLMCallDTO(String modelName, Integer inputTokens, Integer outputTokens, Integer totalTokens) { +} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java new file mode 100644 index 000000000000..55fa8b858e4b --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java @@ -0,0 +1,10 @@ +package de.tum.cit.aet.artemis.athena.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * DTO representing the total usage metrics. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ResponseMetaTotalUsageDTO(Integer inputTokens, Integer outputTokens, Integer totalTokens) { +} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java index d9c81849b396..cff02c9fed1c 100644 --- a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java +++ b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java @@ -17,10 +17,16 @@ import de.tum.cit.aet.artemis.athena.dto.ExerciseBaseDTO; import de.tum.cit.aet.artemis.athena.dto.ModelingFeedbackDTO; import de.tum.cit.aet.artemis.athena.dto.ProgrammingFeedbackDTO; +import de.tum.cit.aet.artemis.athena.dto.ResponseMetaDTO; import de.tum.cit.aet.artemis.athena.dto.SubmissionBaseDTO; import de.tum.cit.aet.artemis.athena.dto.TextFeedbackDTO; +import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.exception.ConflictException; import de.tum.cit.aet.artemis.core.exception.NetworkingException; +import de.tum.cit.aet.artemis.core.service.LLMTokenUsageService; +import de.tum.cit.aet.artemis.exercise.domain.Exercise; +import de.tum.cit.aet.artemis.exercise.domain.Submission; +import de.tum.cit.aet.artemis.exercise.domain.participation.StudentParticipation; import de.tum.cit.aet.artemis.modeling.domain.ModelingExercise; import de.tum.cit.aet.artemis.modeling.domain.ModelingSubmission; import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise; @@ -48,20 +54,24 @@ public class AthenaFeedbackSuggestionsService { private final AthenaDTOConverterService athenaDTOConverterService; + private final LLMTokenUsageService llmTokenUsageService; + /** * Create a new AthenaFeedbackSuggestionsService to receive feedback suggestions from the Athena service. * * @param athenaRestTemplate REST template used for the communication with Athena * @param athenaModuleService Athena module serviced used to determine the urls for different modules - * @param athenaDTOConverterService Service to convert exr + * @param athenaDTOConverterService Service to convert exrcises and submissions to DTOs + * @param llmTokenUsageService Service to store the usage of LLM tokens */ public AthenaFeedbackSuggestionsService(@Qualifier("athenaRestTemplate") RestTemplate athenaRestTemplate, AthenaModuleService athenaModuleService, - AthenaDTOConverterService athenaDTOConverterService) { + AthenaDTOConverterService athenaDTOConverterService, LLMTokenUsageService llmTokenUsageService) { textAthenaConnector = new AthenaConnector<>(athenaRestTemplate, ResponseDTOText.class); programmingAthenaConnector = new AthenaConnector<>(athenaRestTemplate, ResponseDTOProgramming.class); modelingAthenaConnector = new AthenaConnector<>(athenaRestTemplate, ResponseDTOModeling.class); this.athenaDTOConverterService = athenaDTOConverterService; this.athenaModuleService = athenaModuleService; + this.llmTokenUsageService = llmTokenUsageService; } @JsonInclude(JsonInclude.Include.NON_EMPTY) @@ -69,15 +79,15 @@ private record RequestDTO(ExerciseBaseDTO exercise, SubmissionBaseDTO submission } @JsonInclude(JsonInclude.Include.NON_EMPTY) - private record ResponseDTOText(List data) { + private record ResponseDTOText(List data, ResponseMetaDTO meta) { } @JsonInclude(JsonInclude.Include.NON_EMPTY) - private record ResponseDTOProgramming(List data) { + private record ResponseDTOProgramming(List data, ResponseMetaDTO meta) { } @JsonInclude(JsonInclude.Include.NON_EMPTY) - private record ResponseDTOModeling(List data) { + private record ResponseDTOModeling(List data, ResponseMetaDTO meta) { } /** @@ -100,6 +110,7 @@ public List getTextFeedbackSuggestions(TextExercise exercise, T final RequestDTO request = new RequestDTO(athenaDTOConverterService.ofExercise(exercise), athenaDTOConverterService.ofSubmission(exercise.getId(), submission), isGraded); ResponseDTOText response = textAthenaConnector.invokeWithRetry(athenaModuleService.getAthenaModuleUrl(exercise) + "/feedback_suggestions", request, 0); log.info("Athena responded to '{}' feedback suggestions request: {}", isGraded ? "Graded" : "Non Graded", response.data); + storeTokenUsage(exercise, submission, response.meta, !isGraded); return response.data.stream().toList(); } @@ -117,6 +128,7 @@ public List getProgrammingFeedbackSuggestions(Programmin final RequestDTO request = new RequestDTO(athenaDTOConverterService.ofExercise(exercise), athenaDTOConverterService.ofSubmission(exercise.getId(), submission), isGraded); ResponseDTOProgramming response = programmingAthenaConnector.invokeWithRetry(athenaModuleService.getAthenaModuleUrl(exercise) + "/feedback_suggestions", request, 0); log.info("Athena responded to '{}' feedback suggestions request: {}", isGraded ? "Graded" : "Non Graded", response.data); + storeTokenUsage(exercise, submission, response.meta, !isGraded); return response.data.stream().toList(); } @@ -139,6 +151,26 @@ public List getModelingFeedbackSuggestions(ModelingExercise final RequestDTO request = new RequestDTO(athenaDTOConverterService.ofExercise(exercise), athenaDTOConverterService.ofSubmission(exercise.getId(), submission), isGraded); ResponseDTOModeling response = modelingAthenaConnector.invokeWithRetry(athenaModuleService.getAthenaModuleUrl(exercise) + "/feedback_suggestions", request, 0); log.info("Athena responded to '{}' feedback suggestions request: {}", isGraded ? "Graded" : "Non Graded", response.data); + storeTokenUsage(exercise, submission, response.meta, !isGraded); return response.data; } + + /** + * Store the usage of LLM tokens for a given submission + * + * @param exercise the exercise the submission belongs to + * @param submission the submission for which the tokens were used + * @param meta the meta information of the response from Athena + * @param isPreliminaryFeedback whether the feedback is preliminary or not + */ + private void storeTokenUsage(Exercise exercise, Submission submission, ResponseMetaDTO meta, Boolean isPreliminaryFeedback) { + if (meta == null) { + return; + } + Long courseId = exercise.getCourseViaExerciseGroupOrCourseMember().getId(); + Long exerciseId = exercise.getId(); + Long userId = ((StudentParticipation) submission.getParticipation()).getStudent().map(User::getId).orElse(null); + + llmTokenUsageService.saveAthenaTokenUsage(courseId, exerciseId, userId, meta, isPreliminaryFeedback); + } } From faeac2949e4cebab10b9f4f6cb0e135caa111217 Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Mon, 21 Oct 2024 12:03:13 +0200 Subject: [PATCH 06/10] save llm token usage --- .../AthenaFeedbackSuggestionsService.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java index cff02c9fed1c..046bba71da2c 100644 --- a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java +++ b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java @@ -20,6 +20,9 @@ import de.tum.cit.aet.artemis.athena.dto.ResponseMetaDTO; import de.tum.cit.aet.artemis.athena.dto.SubmissionBaseDTO; import de.tum.cit.aet.artemis.athena.dto.TextFeedbackDTO; +import de.tum.cit.aet.artemis.core.domain.Course; +import de.tum.cit.aet.artemis.core.domain.LLMRequest; +import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.User; import de.tum.cit.aet.artemis.core.exception.ConflictException; import de.tum.cit.aet.artemis.core.exception.NetworkingException; @@ -167,10 +170,16 @@ private void storeTokenUsage(Exercise exercise, Submission submission, ResponseM if (meta == null) { return; } - Long courseId = exercise.getCourseViaExerciseGroupOrCourseMember().getId(); - Long exerciseId = exercise.getId(); - Long userId = ((StudentParticipation) submission.getParticipation()).getStudent().map(User::getId).orElse(null); - llmTokenUsageService.saveAthenaTokenUsage(courseId, exerciseId, userId, meta, isPreliminaryFeedback); + Course course = exercise.getCourseViaExerciseGroupOrCourseMember(); + User user = ((StudentParticipation) submission.getParticipation()).getStudent().orElse(null); + + // Temporary pipelineId until Athena provides it in the response + String pipelineId = isPreliminaryFeedback ? "PRELIMINARY_FEEDBACK" : "FEEDBACK_SUGGESTION"; + + List llmRequests = meta.llmCalls().stream().map(llmCall -> new LLMRequest(llmCall.modelName(), llmCall.inputTokens(), 0, llmCall.outputTokens(), 0, pipelineId)) + .toList(); + llmTokenUsageService.saveLLMTokenUsage(llmRequests, LLMServiceType.ATHENA, + (llmTokenUsageBuilder -> llmTokenUsageBuilder.withCourse(course).withExercise(exercise).withUser(user))); } } From 2bcfc2b42798b6a164754a6dea7559ce6ba42ba2 Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Mon, 21 Oct 2024 18:56:45 +0200 Subject: [PATCH 07/10] update athena api usage --- .../cit/aet/artemis/athena/dto/ResponseMetaDTO.java | 7 ++++++- .../aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java | 10 ---------- .../artemis/athena/dto/ResponseMetaTotalUsageDTO.java | 10 ---------- .../service/AthenaFeedbackSuggestionsService.java | 7 +------ 4 files changed, 7 insertions(+), 27 deletions(-) delete mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java delete mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java index c2b4c1c0c160..e80830620b37 100644 --- a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaDTO.java @@ -4,9 +4,14 @@ import com.fasterxml.jackson.annotation.JsonInclude; +import de.tum.cit.aet.artemis.core.domain.LLMRequest; + /** * DTO representing the meta information in the Athena response. */ @JsonInclude(JsonInclude.Include.NON_NULL) -public record ResponseMetaDTO(ResponseMetaTotalUsageDTO totalUsage, List llmCalls) { +public record ResponseMetaDTO(TotalUsage totalUsage, List llmRequests) { + + public record TotalUsage(Integer numInputTokens, Integer numOutputTokens, Integer numTotalTokens, Float cost) { + } } diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java deleted file mode 100644 index e4da493dd4df..000000000000 --- a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java +++ /dev/null @@ -1,10 +0,0 @@ -package de.tum.cit.aet.artemis.athena.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; - -/** - * DTO representing an individual LLM call. - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public record ResponseMetaLLMCallDTO(String modelName, Integer inputTokens, Integer outputTokens, Integer totalTokens) { -} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java deleted file mode 100644 index 55fa8b858e4b..000000000000 --- a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java +++ /dev/null @@ -1,10 +0,0 @@ -package de.tum.cit.aet.artemis.athena.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; - -/** - * DTO representing the total usage metrics. - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public record ResponseMetaTotalUsageDTO(Integer inputTokens, Integer outputTokens, Integer totalTokens) { -} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java index 046bba71da2c..7095a4be329e 100644 --- a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java +++ b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java @@ -173,12 +173,7 @@ private void storeTokenUsage(Exercise exercise, Submission submission, ResponseM Course course = exercise.getCourseViaExerciseGroupOrCourseMember(); User user = ((StudentParticipation) submission.getParticipation()).getStudent().orElse(null); - - // Temporary pipelineId until Athena provides it in the response - String pipelineId = isPreliminaryFeedback ? "PRELIMINARY_FEEDBACK" : "FEEDBACK_SUGGESTION"; - - List llmRequests = meta.llmCalls().stream().map(llmCall -> new LLMRequest(llmCall.modelName(), llmCall.inputTokens(), 0, llmCall.outputTokens(), 0, pipelineId)) - .toList(); + List llmRequests = meta.llmRequests(); llmTokenUsageService.saveLLMTokenUsage(llmRequests, LLMServiceType.ATHENA, (llmTokenUsageBuilder -> llmTokenUsageBuilder.withCourse(course).withExercise(exercise).withUser(user))); } From d070d2c08ad59c2b6f3d3ca10d18505478b90ced Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Mon, 21 Oct 2024 19:00:11 +0200 Subject: [PATCH 08/10] fix remaining issues --- .../service/AthenaFeedbackSuggestionsService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java index 7095a4be329e..d3632f209ca0 100644 --- a/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java +++ b/src/main/java/de/tum/cit/aet/artemis/athena/service/AthenaFeedbackSuggestionsService.java @@ -20,7 +20,6 @@ import de.tum.cit.aet.artemis.athena.dto.ResponseMetaDTO; import de.tum.cit.aet.artemis.athena.dto.SubmissionBaseDTO; import de.tum.cit.aet.artemis.athena.dto.TextFeedbackDTO; -import de.tum.cit.aet.artemis.core.domain.Course; import de.tum.cit.aet.artemis.core.domain.LLMRequest; import de.tum.cit.aet.artemis.core.domain.LLMServiceType; import de.tum.cit.aet.artemis.core.domain.User; @@ -170,11 +169,14 @@ private void storeTokenUsage(Exercise exercise, Submission submission, ResponseM if (meta == null) { return; } - - Course course = exercise.getCourseViaExerciseGroupOrCourseMember(); - User user = ((StudentParticipation) submission.getParticipation()).getStudent().orElse(null); + Long courseId = exercise.getCourseViaExerciseGroupOrCourseMember().getId(); + Long userId = ((StudentParticipation) submission.getParticipation()).getStudent().map(User::getId).orElse(null); List llmRequests = meta.llmRequests(); + if (llmRequests == null) { + return; + } + llmTokenUsageService.saveLLMTokenUsage(llmRequests, LLMServiceType.ATHENA, - (llmTokenUsageBuilder -> llmTokenUsageBuilder.withCourse(course).withExercise(exercise).withUser(user))); + (llmTokenUsageBuilder -> llmTokenUsageBuilder.withCourse(courseId).withExercise(exercise.getId()).withUser(userId))); } } From bc84ff0a4cbb4b46bfcec96d1dce1dcf8aefe733 Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Mon, 21 Oct 2024 10:20:15 +0200 Subject: [PATCH 09/10] add token usage logging for Athena --- .../aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java | 10 ++++++++++ .../artemis/athena/dto/ResponseMetaTotalUsageDTO.java | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java create mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java new file mode 100644 index 000000000000..e4da493dd4df --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java @@ -0,0 +1,10 @@ +package de.tum.cit.aet.artemis.athena.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * DTO representing an individual LLM call. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ResponseMetaLLMCallDTO(String modelName, Integer inputTokens, Integer outputTokens, Integer totalTokens) { +} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java new file mode 100644 index 000000000000..55fa8b858e4b --- /dev/null +++ b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java @@ -0,0 +1,10 @@ +package de.tum.cit.aet.artemis.athena.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * DTO representing the total usage metrics. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ResponseMetaTotalUsageDTO(Integer inputTokens, Integer outputTokens, Integer totalTokens) { +} From ab5195180700bda7e07c099326223f7264ca19b8 Mon Sep 17 00:00:00 2001 From: "Felix T.J. Dietrich" Date: Mon, 21 Oct 2024 19:07:10 +0200 Subject: [PATCH 10/10] remove unused files --- .../aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java | 10 ---------- .../artemis/athena/dto/ResponseMetaTotalUsageDTO.java | 10 ---------- 2 files changed, 20 deletions(-) delete mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java delete mode 100644 src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java deleted file mode 100644 index e4da493dd4df..000000000000 --- a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaLLMCallDTO.java +++ /dev/null @@ -1,10 +0,0 @@ -package de.tum.cit.aet.artemis.athena.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; - -/** - * DTO representing an individual LLM call. - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public record ResponseMetaLLMCallDTO(String modelName, Integer inputTokens, Integer outputTokens, Integer totalTokens) { -} diff --git a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java b/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java deleted file mode 100644 index 55fa8b858e4b..000000000000 --- a/src/main/java/de/tum/cit/aet/artemis/athena/dto/ResponseMetaTotalUsageDTO.java +++ /dev/null @@ -1,10 +0,0 @@ -package de.tum.cit.aet.artemis.athena.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; - -/** - * DTO representing the total usage metrics. - */ -@JsonInclude(JsonInclude.Include.NON_NULL) -public record ResponseMetaTotalUsageDTO(Integer inputTokens, Integer outputTokens, Integer totalTokens) { -}