From d80da3177146fdbcfeab45a0371b4c38a57d88f9 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Tue, 9 Jul 2024 13:30:44 +0200 Subject: [PATCH 01/95] First UI addon's mockup for now --- .../domain/lecture/AttachmentUnit.java | 15 + .../artemis/service/LectureUnitService.java | 13 + .../pyris/PyrisStatusUpdateService.java | 50 ++- .../connectors/pyris/PyrisWebhookService.java | 14 +- .../PyrisLectureIngestionStatusUpdateDTO.java | 2 +- .../PyrisLectureUnitWebhookDTO.java | 4 +- .../PyrisLectureUnitsStatusUpdate.java | 13 + .../pyris/dto/status/IngestionState.java | 8 + .../web/rest/lecture/LectureUnitResource.java | 15 + .../changelog/20240625000000_changelog.xml | 14 + .../resources/config/liquibase/master.xml | 2 + .../competencies-popover.component.scss | 3 +- .../lecture-unit/attachmentUnit.model.ts | 9 + src/main/webapp/app/entities/lecture.model.ts | 34 +- .../lecture-unit-management.component.html | 20 +- .../lecture-unit-management.component.ts | 113 +++++-- .../lectureUnit.service.ts | 12 + .../webapp/app/lecture/lecture.component.html | 5 + .../webapp/app/lecture/lecture.component.ts | 9 +- .../app/overview/course-overview.service.ts | 2 +- .../webapp/app/shared/sort/sort.directive.ts | 14 + src/main/webapp/i18n/de/lecture.json | 1 + src/main/webapp/i18n/en/lecture.json | 1 + .../iris/PyrisLectureIngestionTest.java | 304 ------------------ 24 files changed, 311 insertions(+), 366 deletions(-) create mode 100644 src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitsStatusUpdate.java create mode 100644 src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/IngestionState.java create mode 100644 src/main/resources/config/liquibase/changelog/20240625000000_changelog.xml diff --git a/src/main/java/de/tum/in/www1/artemis/domain/lecture/AttachmentUnit.java b/src/main/java/de/tum/in/www1/artemis/domain/lecture/AttachmentUnit.java index da84dab34bd5..79149530fa56 100644 --- a/src/main/java/de/tum/in/www1/artemis/domain/lecture/AttachmentUnit.java +++ b/src/main/java/de/tum/in/www1/artemis/domain/lecture/AttachmentUnit.java @@ -8,6 +8,8 @@ import jakarta.persistence.Column; import jakarta.persistence.DiscriminatorValue; import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; import jakarta.persistence.FetchType; import jakarta.persistence.OneToMany; import jakarta.persistence.OneToOne; @@ -16,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import de.tum.in.www1.artemis.domain.Attachment; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; @Entity @DiscriminatorValue("A") @@ -26,6 +29,10 @@ public class AttachmentUnit extends LectureUnit { @Column(name = "description") private String description; + @Enumerated(EnumType.STRING) + @Column(name = "pyris_ingestion_state") + private IngestionState pyrisIngestionState = IngestionState.NOT_STARTED; + @OneToOne(mappedBy = "attachmentUnit", cascade = CascadeType.REMOVE, orphanRemoval = true) @JsonIgnoreProperties(value = "attachmentUnit", allowSetters = true) private Attachment attachment; @@ -60,6 +67,14 @@ public void setAttachment(Attachment attachment) { this.attachment = attachment; } + public IngestionState getPyrisIngestionState() { + return pyrisIngestionState; + } + + public void setPyrisIngestionState(IngestionState pyrisIngestionState) { + this.pyrisIngestionState = pyrisIngestionState; + } + public List getSlides() { return slides; } diff --git a/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java b/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java index 23e6b3b849de..7be6c26e9d3a 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java @@ -232,4 +232,17 @@ public URL validateUrlStringAndReturnUrl(String urlString) { throw new BadRequestException(); } } + + /** + * Ingest the lectureUnit when triggered by the ingest lectureUnit button + * + * @param lectureUnit lectureUnit to be ingested + * @return returns the job token if the operation is successful else it returns null + */ + public boolean ingestLectureUnitInPyris(AttachmentUnit lectureUnit) { + if (pyrisWebhookService.isPresent()) { + return pyrisWebhookService.get().addLectureUnitsToPyrisDB(List.of(lectureUnit)) != null; + } + return false; + } } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java index 759e62ca9e35..2f8d1fb19013 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java @@ -1,12 +1,15 @@ package de.tum.in.www1.artemis.service.connectors.pyris; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.List; + import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; +import de.tum.in.www1.artemis.domain.lecture.AttachmentUnit; +import de.tum.in.www1.artemis.repository.AttachmentUnitRepository; import de.tum.in.www1.artemis.service.connectors.pyris.dto.chat.PyrisChatStatusUpdateDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureIngestionStatusUpdateDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState; import de.tum.in.www1.artemis.service.connectors.pyris.job.CourseChatJob; @@ -25,13 +28,15 @@ public class PyrisStatusUpdateService { private final IrisCourseChatSessionService courseChatSessionService; - private static final Logger log = LoggerFactory.getLogger(PyrisStatusUpdateService.class); + private final AttachmentUnitRepository attachmentUnitRepository; - public PyrisStatusUpdateService(PyrisJobService pyrisJobService, IrisExerciseChatSessionService irisExerciseChatSessionService, - IrisCourseChatSessionService courseChatSessionService) { + public PyrisStatusUpdateService(PyrisJobService pyrisJobService, AttachmentUnitRepository attachmentUnitRepository, + IrisExerciseChatSessionService irisExerciseChatSessionService, IrisCourseChatSessionService courseChatSessionService) { this.pyrisJobService = pyrisJobService; this.irisExerciseChatSessionService = irisExerciseChatSessionService; this.courseChatSessionService = courseChatSessionService; + this.attachmentUnitRepository = attachmentUnitRepository; + } /** @@ -43,7 +48,7 @@ public PyrisStatusUpdateService(PyrisJobService pyrisJobService, IrisExerciseCha public void handleStatusUpdate(ExerciseChatJob job, PyrisChatStatusUpdateDTO statusUpdate) { irisExerciseChatSessionService.handleStatusUpdate(job, statusUpdate); - removeJobIfTerminated(statusUpdate, job.jobId()); + removeJobIfTerminated(statusUpdate.stages(), job.jobId()); } /** @@ -56,7 +61,7 @@ public void handleStatusUpdate(ExerciseChatJob job, PyrisChatStatusUpdateDTO sta public void handleStatusUpdate(CourseChatJob job, PyrisChatStatusUpdateDTO statusUpdate) { courseChatSessionService.handleStatusUpdate(job, statusUpdate); - removeJobIfTerminated(statusUpdate, job.jobId()); + removeJobIfTerminated(statusUpdate.stages(), job.jobId()); } /** @@ -66,29 +71,40 @@ public void handleStatusUpdate(CourseChatJob job, PyrisChatStatusUpdateDTO statu * * @see PyrisStageState#isTerminal() * - * @param statusUpdate the status update - * @param job the job to remove + * @param stages the stages of the status update + * @param job the job to remove */ - private void removeJobIfTerminated(PyrisChatStatusUpdateDTO statusUpdate, String job) { - var isDone = statusUpdate.stages().stream().map(PyrisStageDTO::state).allMatch(PyrisStageState::isTerminal); + private boolean removeJobIfTerminated(List stages, String job) { + var isDone = stages.stream().map(PyrisStageDTO::state).allMatch(PyrisStageState::isTerminal); if (isDone) { pyrisJobService.removeJob(job); + return true; } + return false; } /** * Handles the status update of a lecture ingestion job and logs the results for now => will change later - * TODO: Update this method to handle changes beyond logging + * * * @param job the job that is updated * @param statusUpdate the status update */ public void handleStatusUpdate(IngestionWebhookJob job, PyrisLectureIngestionStatusUpdateDTO statusUpdate) { - statusUpdate.stages().forEach(stage -> log.info(stage.name() + ":" + stage.message())); - boolean isDone = statusUpdate.stages().stream().map(PyrisStageDTO::state) - .allMatch(state -> state == PyrisStageState.DONE || state == PyrisStageState.ERROR || state == PyrisStageState.SKIPPED); - if (isDone) { - pyrisJobService.removeJob(job.jobId()); + if (removeJobIfTerminated(statusUpdate.stages(), job.jobId())) { + for (Long id : statusUpdate.ids()) { + if (attachmentUnitRepository.findById(id).isPresent()) { + AttachmentUnit unit = attachmentUnitRepository.findById(id).get(); + if (statusUpdate.stages().getLast().state() == PyrisStageState.DONE) { + unit.setPyrisIngestionState(IngestionState.DONE); + attachmentUnitRepository.save(unit); + } + else if (statusUpdate.stages().getLast().state() == PyrisStageState.ERROR) { + unit.setPyrisIngestionState(IngestionState.ERROR); + attachmentUnitRepository.save(unit); + } + } + } } } } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java index 6d8d18b3ae20..23b811258036 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java @@ -18,11 +18,13 @@ import de.tum.in.www1.artemis.domain.enumeration.AttachmentType; import de.tum.in.www1.artemis.domain.iris.settings.IrisCourseSettings; import de.tum.in.www1.artemis.domain.lecture.AttachmentUnit; +import de.tum.in.www1.artemis.repository.LectureUnitRepository; import de.tum.in.www1.artemis.repository.iris.IrisSettingsRepository; import de.tum.in.www1.artemis.service.FilePathService; import de.tum.in.www1.artemis.service.connectors.pyris.dto.PyrisPipelineExecutionSettingsDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureUnitWebhookDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; import de.tum.in.www1.artemis.service.iris.exception.IrisInternalPyrisErrorException; import de.tum.in.www1.artemis.service.iris.settings.IrisSettingsService; @@ -40,15 +42,18 @@ public class PyrisWebhookService { private final IrisSettingsRepository irisSettingsRepository; + private final LectureUnitRepository lectureUnitRepository; + @Value("${server.url}") private String artemisBaseUrl; public PyrisWebhookService(PyrisConnectorService pyrisConnectorService, PyrisJobService pyrisJobService, IrisSettingsService irisSettingsService, - IrisSettingsRepository irisSettingsRepository) { + IrisSettingsRepository irisSettingsRepository, LectureUnitRepository lectureUnitRepository) { this.pyrisConnectorService = pyrisConnectorService; this.pyrisJobService = pyrisJobService; this.irisSettingsService = irisSettingsService; this.irisSettingsRepository = irisSettingsRepository; + this.lectureUnitRepository = lectureUnitRepository; } private boolean lectureIngestionEnabled(Course course) { @@ -76,15 +81,16 @@ private PyrisLectureUnitWebhookDTO processAttachmentForUpdate(AttachmentUnit att String courseTitle = attachmentUnit.getLecture().getCourse().getTitle(); String courseDescription = attachmentUnit.getLecture().getCourse().getDescription() == null ? "" : attachmentUnit.getLecture().getCourse().getDescription(); String base64EncodedPdf = attachmentToBase64(attachmentUnit); - return new PyrisLectureUnitWebhookDTO(true, artemisBaseUrl, base64EncodedPdf, lectureUnitId, lectureUnitName, lectureId, lectureTitle, courseId, courseTitle, - courseDescription); + attachmentUnit.setPyrisIngestionState(IngestionState.IN_PROGRESS); + lectureUnitRepository.save(attachmentUnit); + return new PyrisLectureUnitWebhookDTO(true, base64EncodedPdf, lectureUnitId, lectureUnitName, lectureId, lectureTitle, courseId, courseTitle, courseDescription); } private PyrisLectureUnitWebhookDTO processAttachmentForDeletion(AttachmentUnit attachmentUnit) { Long lectureUnitId = attachmentUnit.getId(); Long lectureId = attachmentUnit.getLecture().getId(); Long courseId = attachmentUnit.getLecture().getCourse().getId(); - return new PyrisLectureUnitWebhookDTO(false, artemisBaseUrl, "", lectureUnitId, "", lectureId, "", courseId, "", ""); + return new PyrisLectureUnitWebhookDTO(false, "", lectureUnitId, "", lectureId, "", courseId, "", ""); } /** diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java index 959862d3f3f2..0fe0c07813b4 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java @@ -7,5 +7,5 @@ import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record PyrisLectureIngestionStatusUpdateDTO(String result, List stages) { +public record PyrisLectureIngestionStatusUpdateDTO(String result, List stages, List ids) { } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitWebhookDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitWebhookDTO.java index b5a956c2463a..6a36b867d16c 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitWebhookDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitWebhookDTO.java @@ -8,6 +8,6 @@ * providing necessary details such as lecture and course identifiers, names, and descriptions. */ @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record PyrisLectureUnitWebhookDTO(Boolean toUpdate, String artemisBaseUrl, String pdfFile, Long lectureUnitId, String lectureUnitName, Long lectureId, String lectureName, - Long courseId, String courseName, String courseDescription) { +public record PyrisLectureUnitWebhookDTO(Boolean toUpdate, String pdfFile, Long lectureUnitId, String lectureUnitName, Long lectureId, String lectureName, Long courseId, + String courseName, String courseDescription) { } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitsStatusUpdate.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitsStatusUpdate.java new file mode 100644 index 000000000000..2c7312ec6415 --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitsStatusUpdate.java @@ -0,0 +1,13 @@ +package de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * Represents a webhook data transfer object for lecture units in the Pyris system. + * This DTO is used to encapsulate the information related to updates of lecture units, + * providing necessary details such as lecture and course identifiers, names, and descriptions. + */ +@JsonInclude(JsonInclude.Include.NON_EMPTY) + +public record PyrisLectureUnitsStatusUpdate(Long lectureUnitId, Long lectureId, Long courseId) { +} diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/IngestionState.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/IngestionState.java new file mode 100644 index 000000000000..9090d887a78a --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/IngestionState.java @@ -0,0 +1,8 @@ +package de.tum.in.www1.artemis.service.connectors.pyris.dto.status; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public enum IngestionState { + NOT_STARTED, IN_PROGRESS, DONE, ERROR +} diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java index d49ec66c1edd..f0e69d0483a6 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java @@ -22,6 +22,7 @@ import de.tum.in.www1.artemis.domain.Lecture; import de.tum.in.www1.artemis.domain.User; +import de.tum.in.www1.artemis.domain.lecture.AttachmentUnit; import de.tum.in.www1.artemis.domain.lecture.LectureUnit; import de.tum.in.www1.artemis.repository.LectureRepository; import de.tum.in.www1.artemis.repository.LectureUnitRepository; @@ -188,4 +189,18 @@ public ResponseEntity getLectureUnitFo authorizationCheckService.checkHasAtLeastRoleInCourseElseThrow(Role.STUDENT, lectureUnit.getLecture().getCourse(), null); return ResponseEntity.ok(LectureUnitForLearningPathNodeDetailsDTO.of(lectureUnit)); } + + /** + * POST /lectureUnit/{lectureUnitId}/ingest + * This endpoint is for starting the ingestion of one lecture Unit. + * + * @param lectureUnitId The Id of the lecture unit to be ingested. + * @return the ResponseEntity with status 200 (OK) and a message success or null if the operation failed + */ + @PostMapping("lecture-units/{lectureUnitId}/ingest") + public ResponseEntity ingestLectureUnit(@PathVariable Long lectureUnitId) { + log.debug("REST request to ingest lectures of course : {}", lectureUnitId); + LectureUnit lectureUnit = lectureUnitRepository.findByIdWithCompetenciesAndSlidesElseThrow(lectureUnitId); + return ResponseEntity.ok().body(lectureUnitService.ingestLectureUnitInPyris((AttachmentUnit) lectureUnit)); + } } diff --git a/src/main/resources/config/liquibase/changelog/20240625000000_changelog.xml b/src/main/resources/config/liquibase/changelog/20240625000000_changelog.xml new file mode 100644 index 000000000000..20ecf0a3bd52 --- /dev/null +++ b/src/main/resources/config/liquibase/changelog/20240625000000_changelog.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + diff --git a/src/main/resources/config/liquibase/master.xml b/src/main/resources/config/liquibase/master.xml index 729d22ca2a4d..11e6be5bf311 100644 --- a/src/main/resources/config/liquibase/master.xml +++ b/src/main/resources/config/liquibase/master.xml @@ -17,6 +17,8 @@ + + diff --git a/src/main/webapp/app/course/competencies/competencies-popover/competencies-popover.component.scss b/src/main/webapp/app/course/competencies/competencies-popover/competencies-popover.component.scss index af89b79ffb43..508f5f543f01 100644 --- a/src/main/webapp/app/course/competencies/competencies-popover/competencies-popover.component.scss +++ b/src/main/webapp/app/course/competencies/competencies-popover/competencies-popover.component.scss @@ -1,5 +1,6 @@ .competency-button { - width: 100%; + width: 39px !important; + height: 30px !important; } .competency-popover { diff --git a/src/main/webapp/app/entities/lecture-unit/attachmentUnit.model.ts b/src/main/webapp/app/entities/lecture-unit/attachmentUnit.model.ts index 9d1e9986329e..d83c834fe83a 100644 --- a/src/main/webapp/app/entities/lecture-unit/attachmentUnit.model.ts +++ b/src/main/webapp/app/entities/lecture-unit/attachmentUnit.model.ts @@ -6,8 +6,17 @@ export class AttachmentUnit extends LectureUnit { public description?: string; public attachment?: Attachment; public slides?: Slide[]; + public pyrisIngestionState?: IngestionState; constructor() { super(LectureUnitType.ATTACHMENT); } } + +export enum IngestionState { + NOT_STARTED = 'NOT_STARTED', + IN_PROGRESS = 'IN_PROGRESS', + DONE = 'DONE', + ERROR = 'ERROR', + PARTIALLY_INGESTED = 'PARTIALLY_INGESTED', +} diff --git a/src/main/webapp/app/entities/lecture.model.ts b/src/main/webapp/app/entities/lecture.model.ts index 3f039edafe40..989d79852bf2 100644 --- a/src/main/webapp/app/entities/lecture.model.ts +++ b/src/main/webapp/app/entities/lecture.model.ts @@ -4,6 +4,7 @@ import { Attachment } from 'app/entities/attachment.model'; import { Post } from 'app/entities/metis/post.model'; import { Course } from 'app/entities/course.model'; import { LectureUnit } from 'app/entities/lecture-unit/lectureUnit.model'; +import { AttachmentUnit, IngestionState } from 'app/entities/lecture-unit/attachmentUnit.model'; export class Lecture implements BaseEntity { id?: number; @@ -21,6 +22,37 @@ export class Lecture implements BaseEntity { channelName?: string; isAtLeastEditor?: boolean; isAtLeastInstructor?: boolean; + ingested?: IngestionState; - constructor() {} + constructor() { + this.ingested = this.checkIngestionState(); + } + + private checkIngestionState(): IngestionState { + if (!this.lectureUnits) { + return IngestionState.NOT_STARTED; + } + + const attachmentUnits = this.lectureUnits.filter((unit) => unit.type === 'attachment') as AttachmentUnit[]; + const allDone = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.DONE); + const allNotStarted = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.NOT_STARTED); + const allFailed = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.ERROR); + + if (allDone) { + return IngestionState.DONE; + } + if (allFailed) { + return IngestionState.ERROR; + } + + if (allNotStarted) { + return IngestionState.NOT_STARTED; + } + + return IngestionState.PARTIALLY_INGESTED; + } + + public updateIngestionState(): void { + this.ingested = this.checkIngestionState(); + } } diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html index ba9ab05310d2..a8bc88b69797 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html @@ -66,9 +66,23 @@ }
- @if (lecture.course?.id && showCompetencies) { - - } +
+ @if (lecture.course?.id && showCompetencies) { + + } + +
@if (this.emitEditEvents) { @if (editButtonAvailable(lectureUnit)) { diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts index 6b2d25e8a991..596300223022 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts @@ -2,7 +2,7 @@ import { Component, EventEmitter, Input, OnDestroy, OnInit, Output } from '@angu import { ActivatedRoute, NavigationEnd, Router } from '@angular/router'; import { Lecture } from 'app/entities/lecture.model'; import { LectureService } from 'app/lecture/lecture.service'; -import { debounceTime, filter, finalize, map } from 'rxjs/operators'; +import { debounceTime, filter, finalize, map, switchMap } from 'rxjs/operators'; import { HttpErrorResponse, HttpResponse } from '@angular/common/http'; import { LectureUnit, LectureUnitType } from 'app/entities/lecture-unit/lectureUnit.model'; import { AlertService } from 'app/core/util/alert.service'; @@ -10,10 +10,13 @@ import { onError } from 'app/shared/util/global.utils'; import { Subject, Subscription } from 'rxjs'; import { LectureUnitService } from 'app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service'; import { ActionType } from 'app/shared/delete-dialog/delete-dialog.model'; -import { AttachmentUnit } from 'app/entities/lecture-unit/attachmentUnit.model'; +import { AttachmentUnit, IngestionState } from 'app/entities/lecture-unit/attachmentUnit.model'; import { ExerciseUnit } from 'app/entities/lecture-unit/exerciseUnit.model'; -import { faPencilAlt, faTrash } from '@fortawesome/free-solid-svg-icons'; +import { IconDefinition, faCheckCircle, faFileExport, faPencilAlt, faRepeat, faSpinner, faTrash } from '@fortawesome/free-solid-svg-icons'; import { CdkDragDrop, moveItemInArray } from '@angular/cdk/drag-drop'; +import { PROFILE_IRIS } from 'app/app.constants'; +import { ProfileService } from 'app/shared/layouts/profiles/profile.service'; +import { IrisSettingsService } from 'app/iris/settings/shared/iris-settings.service'; @Component({ selector: 'jhi-lecture-unit-management', @@ -29,7 +32,6 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { @Output() onEditLectureUnitClicked: EventEmitter = new EventEmitter(); - lectureUnits: LectureUnit[] = []; lecture: Lecture; isLoading = false; @@ -40,6 +42,9 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { readonly ActionType = ActionType; private dialogErrorSource = new Subject(); dialogError$ = this.dialogErrorSource.asObservable(); + private profileInfoSubscription: Subscription; + irisEnabled = false; + lectureIngestionEnabled = false; routerEditLinksBase = { [LectureUnitType.ATTACHMENT]: 'attachment-units', @@ -51,6 +56,10 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { // Icons faTrash = faTrash; faPencilAlt = faPencilAlt; + sendToIris = faFileExport; + resendToIris = faRepeat; + done = faCheckCircle; + loading = faSpinner; constructor( private activatedRoute: ActivatedRoute, @@ -58,6 +67,8 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { private lectureService: LectureService, private alertService: AlertService, public lectureUnitService: LectureUnitService, + private profileService: ProfileService, + private irisSettingsService: IrisSettingsService, ) {} ngOnDestroy(): void { @@ -65,6 +76,7 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { this.updateOrderSubjectSubscription.unsubscribe(); this.dialogErrorSource.unsubscribe(); this.navigationEndSubscription.unsubscribe(); + this.profileInfoSubscription?.unsubscribe(); } ngOnInit(): void { @@ -73,13 +85,17 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { }); this.updateOrderSubject = new Subject(); - this.activatedRoute.parent!.params.subscribe((params) => { - this.lectureId ??= +params['lectureId']; - if (this.lectureId) { - // TODO: the lecture (without units) is already available through the lecture.route.ts resolver, it's not really good that we load it twice - this.loadData(); - } - }); + this.activatedRoute + .parent!.params.pipe( + map((params) => +params['lectureId']), + switchMap((lectureId) => { + this.lectureId = lectureId; + return this.loadData(); + }), + ) + .subscribe(() => { + this.initializeProfileInfo(); + }); // debounceTime limits the amount of put requests sent for updating the lecture unit order this.updateOrderSubjectSubscription = this.updateOrderSubject.pipe(debounceTime(1000)).subscribe(() => { @@ -91,25 +107,17 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { this.isLoading = true; // TODO: we actually would like to have the lecture with all units! Posts and competencies are not required here // we could also simply load all units for the lecture (as the lecture is already available through the route, see TODO above) - this.lectureService - .findWithDetails(this.lectureId!) - .pipe( - map((response: HttpResponse) => response.body!), - finalize(() => { - this.isLoading = false; - }), - ) - .subscribe({ - next: (lecture) => { - this.lecture = lecture; - if (lecture?.lectureUnits) { - this.lectureUnits = lecture?.lectureUnits; - } else { - this.lectureUnits = []; - } - }, - error: (errorResponse: HttpErrorResponse) => onError(this.alertService, errorResponse), - }); + return this.lectureService.findWithDetails(this.lectureId!).pipe( + map((response: HttpResponse) => response.body!), + finalize(() => { + this.isLoading = false; + }), + switchMap((lecture) => { + this.lecture = lecture; + this.lectureUnits = lecture?.lectureUnits || []; + return []; + }), + ); } updateOrder() { @@ -125,6 +133,17 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { }); } + initializeProfileInfo() { + this.profileInfoSubscription = this.profileService.getProfileInfo().subscribe(async (profileInfo) => { + this.irisEnabled = profileInfo.activeProfiles.includes(PROFILE_IRIS); + if (this.irisEnabled) { + this.irisSettingsService.getCombinedCourseSettings(this.lecture.course!.id!).subscribe((settings) => { + this.lectureIngestionEnabled = settings?.irisLectureIngestionSettings?.enabled || false; + }); + } + }); + } + drop(event: CdkDragDrop) { moveItemInArray(this.lectureUnits, event.previousIndex, event.currentIndex); this.updateOrderSubject.next(''); @@ -201,6 +220,40 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { onEditButtonClicked(lectureUnit: LectureUnit) { this.onEditLectureUnitClicked.emit(lectureUnit); } + onIngestButtonClicked(lectureUnit: LectureUnit) { + this.lectureUnitService.ingestLectureUnitInPyris(lectureUnit.id!).subscribe({ + error: (error) => console.error('Failed to send Ingestion request', error), + }); + } + getIcon(attachmentUnit: AttachmentUnit): IconDefinition { + console.log(`pyrisIngestionState: ${attachmentUnit.pyrisIngestionState}`); + console.log(`Type of pyrisIngestionState: ${typeof attachmentUnit.pyrisIngestionState}`); + + // @ts-ignore + if (Object.values(IngestionState).includes(attachmentUnit.pyrisIngestionState)) { + console.log('Valid ingestion state'); + } else { + console.log('Invalid ingestion state'); + } + + switch (attachmentUnit.pyrisIngestionState) { + case IngestionState.NOT_STARTED: + console.log('NOT_STARTED'); + return this.sendToIris; + case IngestionState.IN_PROGRESS: + console.log('IN_PROGRESS'); + return this.loading; + case IngestionState.DONE: + console.log('DONE'); + return this.done; + case IngestionState.ERROR: + console.log('ERROR'); + return this.resendToIris; + default: + console.log('DEFAULT'); + return this.sendToIris; + } + } getLectureUnitReleaseDate(lectureUnit: LectureUnit) { switch (lectureUnit.type) { diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts index 66b8b1c849e7..7011e3dae036 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts @@ -160,4 +160,16 @@ export class LectureUnitService { observe: 'response', }); } + /** + * triggers the ingestion of one Lecture Unit + * + * @param lectureId The lecture to be ingested in pyris + */ + ingestLectureUnitInPyris(lectureUnitId: number): Observable> { + const params = new HttpParams(); + return this.httpClient.post(`api/lecture-units/${lectureUnitId}/ingest`, null, { + params: params, + observe: 'response', + }); + } } diff --git a/src/main/webapp/app/lecture/lecture.component.html b/src/main/webapp/app/lecture/lecture.component.html index e507242d7a13..6d69f5dbddab 100644 --- a/src/main/webapp/app/lecture/lecture.component.html +++ b/src/main/webapp/app/lecture/lecture.component.html @@ -118,6 +118,10 @@

+ + + + @@ -134,6 +138,7 @@

{{ lecture.visibleDate | artemisDate }} {{ lecture.startDate | artemisDate }} {{ lecture.endDate | artemisDate }} + {{ lecture.ingested?.toString() }}
diff --git a/src/main/webapp/app/lecture/lecture.component.ts b/src/main/webapp/app/lecture/lecture.component.ts index 6f4d832c5167..cd35af947686 100644 --- a/src/main/webapp/app/lecture/lecture.component.ts +++ b/src/main/webapp/app/lecture/lecture.component.ts @@ -150,14 +150,19 @@ export class LectureComponent implements OnInit, OnDestroy { private loadAll() { this.lectureService - .findAllByCourseId(this.courseId) + .findAllByCourseIdWithSlides(this.courseId) .pipe( filter((res: HttpResponse) => res.ok), map((res: HttpResponse) => res.body), ) .subscribe({ next: (res: Lecture[]) => { - this.lectures = res; + this.lectures = res.map((lectureData) => { + const lecture = new Lecture(); + Object.assign(lecture, lectureData); + lecture.updateIngestionState(); + return lecture; + }); this.applyFilters(); }, error: (res: HttpErrorResponse) => onError(this.alertService, res), diff --git a/src/main/webapp/app/overview/course-overview.service.ts b/src/main/webapp/app/overview/course-overview.service.ts index 79703cada3c8..9fc6d13873ea 100644 --- a/src/main/webapp/app/overview/course-overview.service.ts +++ b/src/main/webapp/app/overview/course-overview.service.ts @@ -172,7 +172,7 @@ export class CourseOverviewService { mapLecturesToSidebarCardElements(lectures: Lecture[]) { return lectures.map((lecture) => this.mapLectureToSidebarCardElement(lecture)); } - mapTutorialGroupsToSidebarCardElements(tutorialGroups: Lecture[]) { + mapTutorialGroupsToSidebarCardElements(tutorialGroups: TutorialGroup[]) { return tutorialGroups.map((tutorialGroup) => this.mapTutorialGroupToSidebarCardElement(tutorialGroup)); } diff --git a/src/main/webapp/app/shared/sort/sort.directive.ts b/src/main/webapp/app/shared/sort/sort.directive.ts index 7633048b9b69..8da860084700 100644 --- a/src/main/webapp/app/shared/sort/sort.directive.ts +++ b/src/main/webapp/app/shared/sort/sort.directive.ts @@ -36,4 +36,18 @@ export class SortDirective { this.ascendingChange.emit(this.ascending); this.sortChange.emit({ predicate: this.predicate, ascending: this.ascending }); } + + getSortedData(data: any[]): any[] { + if (typeof this.predicate === 'function') { + return [...data].sort((a, b) => { + const result = (this.predicate as any)(a) < (this.predicate as any)(b) ? -1 : 1; + return this.ascending ? result : -result; + }); + } else { + return [...data].sort((a, b) => { + const result = a[this.predicate] < b[this.predicate] ? -1 : 1; + return this.ascending ? result : -result; + }); + } + } } diff --git a/src/main/webapp/i18n/de/lecture.json b/src/main/webapp/i18n/de/lecture.json index 61e68defd8e9..9cf0c411de72 100644 --- a/src/main/webapp/i18n/de/lecture.json +++ b/src/main/webapp/i18n/de/lecture.json @@ -31,6 +31,7 @@ "endDate": "Ende", "visibleDate": "Sichtbar ab", "course": "Kurs", + "ingestionState": "Vorlesungsimport in Pyris", "detail": { "title": "Vorlesung", "sections": { diff --git a/src/main/webapp/i18n/en/lecture.json b/src/main/webapp/i18n/en/lecture.json index bb8ddb7ad737..a0a58105f6d0 100644 --- a/src/main/webapp/i18n/en/lecture.json +++ b/src/main/webapp/i18n/en/lecture.json @@ -31,6 +31,7 @@ "endDate": "End Date", "visibleDate": "Visible from", "course": "Course", + "ingestionState": "Ingestion State", "detail": { "title": "Lecture", "sections": { diff --git a/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java b/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java index 10ced00f569b..e69de29bb2d1 100644 --- a/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java +++ b/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java @@ -1,304 +0,0 @@ -package de.tum.in.www1.artemis.iris; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.util.LinkedMultiValueMap; - -import de.tum.in.www1.artemis.course.CourseUtilService; -import de.tum.in.www1.artemis.domain.Course; -import de.tum.in.www1.artemis.domain.Lecture; -import de.tum.in.www1.artemis.domain.iris.settings.IrisCourseSettings; -import de.tum.in.www1.artemis.domain.lecture.AttachmentUnit; -import de.tum.in.www1.artemis.lecture.LectureUtilService; -import de.tum.in.www1.artemis.repository.CourseRepository; -import de.tum.in.www1.artemis.repository.LectureRepository; -import de.tum.in.www1.artemis.repository.iris.IrisSettingsRepository; -import de.tum.in.www1.artemis.service.connectors.pyris.PyrisJobService; -import de.tum.in.www1.artemis.service.connectors.pyris.PyrisStatusUpdateService; -import de.tum.in.www1.artemis.service.connectors.pyris.PyrisWebhookService; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureIngestionStatusUpdateDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState; -import de.tum.in.www1.artemis.user.UserUtilService; - -class PyrisLectureIngestionTest extends AbstractIrisIntegrationTest { - - private static final String TEST_PREFIX = "pyrislectureingestiontest"; - - @Autowired - private PyrisWebhookService pyrisWebhookService; - - @Autowired - private CourseRepository courseRepository; - - @Autowired - private LectureRepository lectureRepository; - - @Autowired - private UserUtilService userUtilService; - - @Autowired - private CourseUtilService courseUtilService; - - @Autowired - private LectureUtilService lectureUtilService; - - @Autowired - protected PyrisStatusUpdateService pyrisStatusUpdateService; - - @Autowired - protected PyrisJobService pyrisJobService; - - @Autowired - protected IrisSettingsRepository irisSettingsRepository; - - private Lecture lecture1; - - @BeforeEach - void initTestCase() throws Exception { - userUtilService.addUsers(TEST_PREFIX, 1, 1, 0, 1); - List courses = courseUtilService.createCoursesWithExercisesAndLectures(TEST_PREFIX, true, true, 1); - Course course1 = this.courseRepository.findByIdWithExercisesAndExerciseDetailsAndLecturesElseThrow(courses.getFirst().getId()); - this.lecture1 = course1.getLectures().stream().findFirst().orElseThrow(); - this.lecture1.setTitle("Lecture " + lecture1.getId()); // needed for search by title - this.lecture1 = lectureRepository.save(this.lecture1); - // Add users that are not in the course - userUtilService.createAndSaveUser(TEST_PREFIX + "student42"); - userUtilService.createAndSaveUser(TEST_PREFIX + "tutor42"); - userUtilService.createAndSaveUser(TEST_PREFIX + "instructor42"); - - int numberOfSlides = 2; - AttachmentUnit pdfAttachmentUnitWithSlides = lectureUtilService.createAttachmentUnitWithSlidesAndFile(numberOfSlides, true); - AttachmentUnit imageAttachmentUnitWithSlides = lectureUtilService.createAttachmentUnitWithSlidesAndFile(numberOfSlides, false); - lecture1 = lectureUtilService.addLectureUnitsToLecture(lecture1, List.of(pdfAttachmentUnitWithSlides, imageAttachmentUnitWithSlides)); - this.lecture1 = lectureRepository.findByIdWithLectureUnitsAndAttachmentsElseThrow(lecture1.getId()); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAutoUpdateAttachmentUnitsWithAutoUpdateEnabled() { - activateIrisFor(lecture1.getCourse()); - IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); - courseSettings.getIrisLectureIngestionSettings().setAutoIngestOnLectureAttachmentUpload(true); - this.irisSettingsRepository.save(courseSettings); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - assertThat(pyrisWebhookService.autoUpdateAttachmentUnitsInPyris(lecture1.getCourse().getId(), List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst()))).isTrue(); - assertThat(pyrisWebhookService.autoUpdateAttachmentUnitsInPyris(lecture1.getCourse().getId(), List.of((AttachmentUnit) lecture1.getLectureUnits().getLast()))).isFalse(); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAutoUpdateAttachmentUnitsWithAutoUpdateDisabled() { - activateIrisFor(lecture1.getCourse()); - IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); - courseSettings.getIrisLectureIngestionSettings().setAutoIngestOnLectureAttachmentUpload(false); - this.irisSettingsRepository.save(courseSettings); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - assertThat(pyrisWebhookService.autoUpdateAttachmentUnitsInPyris(lecture1.getCourse().getId(), List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst()))).isFalse(); - assertThat(pyrisWebhookService.autoUpdateAttachmentUnitsInPyris(lecture1.getCourse().getId(), List.of((AttachmentUnit) lecture1.getLectureUnits().getLast()))).isFalse(); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAddLectureToPyrisDatabaseWithCourseSettingsDisabled() { - activateIrisFor(lecture1.getCourse()); - IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); - courseSettings.getIrisLectureIngestionSettings().setEnabled(false); - this.irisSettingsRepository.save(courseSettings); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst())); - assertThat(jobToken).isNull(); - jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getLast())); - assertThat(jobToken).isNull(); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testDeleteLecturefromPyrisDatabaseWithCourseSettingsDisabled() { - activateIrisFor(lecture1.getCourse()); - IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); - courseSettings.getIrisLectureIngestionSettings().setEnabled(false); - this.irisSettingsRepository.save(courseSettings); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst())); - assertThat(jobToken).isNotNull(); - jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getLast())); - assertThat(jobToken).isNull(); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testIngestLecturesButtonInPyris() throws Exception { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - Boolean response = request.postWithResponseBody("/api/courses/" + lecture1.getCourse().getId() + "/ingest", Optional.empty(), boolean.class, HttpStatus.OK); - assertThat(response).isTrue(); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testDeleteLecturefromPyrisDatabaseWithCourseSettingsEnabled() { - activateIrisFor(lecture1.getCourse()); - IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); - this.irisSettingsRepository.save(courseSettings); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst())); - assertThat(jobToken).isNotNull(); - jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getLast())); - assertThat(jobToken).isNull(); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAddLectureToPyrisDBAddJobWithCourseSettingsEnabled() { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst())); - assertThat(jobToken).isNotNull(); - jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getLast())); - assertThat(jobToken).isNull(); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAllStagesDoneRemovesAdditionIngestionJob() throws Exception { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of(attachmentUnit)); - PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage)); - var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); - request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); - assertThat(pyrisJobService.getJob(jobToken)).isNull(); - } - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAllStagesDoneRemovesDeletionIngestionJob() throws Exception { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(attachmentUnit)); - PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage)); - var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); - request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); - assertThat(pyrisJobService.getJob(jobToken)).isNull(); - } - } - - @Test - void testStageNotDoneKeepsAdditionIngestionJob() throws Exception { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of(attachmentUnit)); - PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisStageDTO inProgressStage = new PyrisStageDTO("inProgressStage", 1, PyrisStageState.IN_PROGRESS, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage)); - var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); - request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); - assertThat(pyrisJobService.getJob(jobToken)).isNotNull(); - } - } - - @Test - void testStageNotDoneKeepsDeletionIngetionJob() throws Exception { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(attachmentUnit)); - PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisStageDTO inProgressStage = new PyrisStageDTO("inProgressStage", 1, PyrisStageState.IN_PROGRESS, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage)); - var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); - request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); - assertThat(pyrisJobService.getJob(jobToken)).isNotNull(); - } - } - - @Test - void testErrorStageRemovesDeletionIngetionJob() throws Exception { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(attachmentUnit)); - PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage)); - var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); - request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); - assertThat(pyrisJobService.getJob(jobToken)).isNull(); - } - } - - @Test - void testErrorStageRemovesAdditionIngetionJob() throws Exception { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of(attachmentUnit)); - PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage)); - var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); - request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); - assertThat(pyrisJobService.getJob(jobToken)).isNull(); - } - } - - @Test - void testRunIdIngestionJob() throws Exception { - activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - String newJobToken = pyrisJobService.addIngestionWebhookJob(); - String chatJobToken = pyrisJobService.addCourseChatJob(123L, 123L); - PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage)); - var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + chatJobToken)))); - MockHttpServletResponse response = request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + newJobToken + "/status", statusUpdate, - HttpStatus.CONFLICT, headers); - assertThat(response.getContentAsString()).contains("Run ID in URL does not match run ID in request body"); - response = request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + chatJobToken + "/status", statusUpdate, HttpStatus.CONFLICT, headers); - assertThat(response.getContentAsString()).contains("Run ID is not an ingestion job"); - } -} From d5fb0c0415e0209cea74674bf1abc6842ccd5b8b Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Thu, 18 Jul 2024 02:07:20 +0200 Subject: [PATCH 02/95] Add links to citations --- .../artemis/service/LectureUnitService.java | 6 +-- .../pyris/PyrisConnectorService.java | 46 ++++++++++++++++++- .../pyris/PyrisStatusUpdateService.java | 22 ++++----- .../connectors/pyris/PyrisWebhookService.java | 42 ++++++++++++----- .../PyrisLectureIngestionStatusUpdateDTO.java | 3 +- .../PyrisLectureUnitWebhookDTO.java | 4 +- ...risWebhookLectureDeletionExecutionDTO.java | 13 ++++++ ...isWebhookLectureIngestionExecutionDTO.java | 3 +- .../web/rest/lecture/LectureUnitResource.java | 14 ++++++ .../lecture-unit-management.component.html | 2 +- .../lecture-unit-management.component.ts | 4 +- .../lectureUnit.service.ts | 10 ++++ 12 files changed, 132 insertions(+), 37 deletions(-) create mode 100644 src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisWebhookLectureDeletionExecutionDTO.java diff --git a/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java b/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java index 37f2bb59548f..4979e12d322d 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java @@ -248,9 +248,9 @@ public URL validateUrlStringAndReturnUrl(String urlString) { * @param lectureUnit lectureUnit to be ingested * @return returns the job token if the operation is successful else it returns null */ - public boolean ingestLectureUnitInPyris(AttachmentUnit lectureUnit) { - if (pyrisWebhookService.isPresent()) { - return pyrisWebhookService.get().addLectureUnitsToPyrisDB(List.of(lectureUnit)) != null; + public boolean ingestLectureUnitInPyris(LectureUnit lectureUnit) { + if (lectureUnit instanceof AttachmentUnit) { + return pyrisWebhookService.filter(webhookService -> webhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lectureUnit)) != null).isPresent(); } return false; } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java index bf241bc28837..0e5944eb6e0c 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java @@ -2,6 +2,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,8 +18,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import de.tum.in.www1.artemis.domain.lecture.AttachmentUnit; +import de.tum.in.www1.artemis.domain.lecture.LectureUnit; +import de.tum.in.www1.artemis.repository.LectureUnitRepository; import de.tum.in.www1.artemis.service.connectors.pyris.dto.PyrisModelDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureDeletionExecutionDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; import de.tum.in.www1.artemis.service.iris.exception.IrisException; import de.tum.in.www1.artemis.service.iris.exception.IrisForbiddenException; import de.tum.in.www1.artemis.service.iris.exception.IrisInternalPyrisErrorException; @@ -39,12 +45,16 @@ public class PyrisConnectorService { private final ObjectMapper objectMapper; + private final LectureUnitRepository lectureUnitRepository; + @Value("${artemis.iris.url}") private String pyrisUrl; - public PyrisConnectorService(@Qualifier("pyrisRestTemplate") RestTemplate restTemplate, MappingJackson2HttpMessageConverter springMvcJacksonConverter) { + public PyrisConnectorService(@Qualifier("pyrisRestTemplate") RestTemplate restTemplate, MappingJackson2HttpMessageConverter springMvcJacksonConverter, + LectureUnitRepository lectureUnitRepository) { this.restTemplate = restTemplate; this.objectMapper = springMvcJacksonConverter.getObjectMapper(); + this.lectureUnitRepository = lectureUnitRepository; } /** @@ -93,11 +103,43 @@ public void executePipeline(String feature, String variant, Object executionDTO) * @param variant The variant of the feature to execute * @param executionDTO The DTO sent as a body for the execution */ - public void executeLectureWebhook(String variant, PyrisWebhookLectureIngestionExecutionDTO executionDTO) { + public void executeLectureAddtionWebhook(String variant, PyrisWebhookLectureIngestionExecutionDTO executionDTO) { var endpoint = "/api/v1/webhooks/lectures/" + variant; try { restTemplate.postForEntity(pyrisUrl + endpoint, objectMapper.valueToTree(executionDTO), Void.class); } + catch (HttpStatusCodeException e) { + setIngestionStateToError(executionDTO, e); + throw toIrisException(e); + } + catch (RestClientException | IllegalArgumentException e) { + setIngestionStateToError(executionDTO, e); + throw new PyrisConnectorException("Could not fetch response from Pyris"); + } + } + + private void setIngestionStateToError(PyrisWebhookLectureIngestionExecutionDTO executionDTO, RuntimeException e) { + log.error("Failed to send lectures to Pyris", e); + Optional optionalUnit = lectureUnitRepository.findById(executionDTO.pyrisLectureUnit().lectureUnitId()); + optionalUnit.ifPresent(unit -> { + if (unit instanceof AttachmentUnit) { + AttachmentUnit attachmentUnit = (AttachmentUnit) unit; + attachmentUnit.setPyrisIngestionState(IngestionState.ERROR); + lectureUnitRepository.save(attachmentUnit); + } + }); + } + + /** + * Executes a webhook and send lectures to the webhook with the given variant + * + * @param executionDTO The DTO sent as a body for the execution + */ + public void executeLectureDeletionWebhook(PyrisWebhookLectureDeletionExecutionDTO executionDTO) { + var endpoint = "/api/v1/webhooks/lectures/delete"; + try { + restTemplate.postForEntity(pyrisUrl + endpoint, objectMapper.valueToTree(executionDTO), Void.class); + } catch (HttpStatusCodeException e) { log.error("Failed to send lectures to Pyris", e); throw toIrisException(e); diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java index 2f8d1fb19013..a0e6c629bd12 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java @@ -91,18 +91,16 @@ private boolean removeJobIfTerminated(List stages, String job) { * @param statusUpdate the status update */ public void handleStatusUpdate(IngestionWebhookJob job, PyrisLectureIngestionStatusUpdateDTO statusUpdate) { - if (removeJobIfTerminated(statusUpdate.stages(), job.jobId())) { - for (Long id : statusUpdate.ids()) { - if (attachmentUnitRepository.findById(id).isPresent()) { - AttachmentUnit unit = attachmentUnitRepository.findById(id).get(); - if (statusUpdate.stages().getLast().state() == PyrisStageState.DONE) { - unit.setPyrisIngestionState(IngestionState.DONE); - attachmentUnitRepository.save(unit); - } - else if (statusUpdate.stages().getLast().state() == PyrisStageState.ERROR) { - unit.setPyrisIngestionState(IngestionState.ERROR); - attachmentUnitRepository.save(unit); - } + if (removeJobIfTerminated(statusUpdate.stages(), job.jobId()) && statusUpdate.id().isPresent()) { + if (attachmentUnitRepository.findById(statusUpdate.id().get()).isPresent()) { + AttachmentUnit unit = attachmentUnitRepository.findById(statusUpdate.id().get()).get(); + if (statusUpdate.stages().getLast().state() == PyrisStageState.DONE) { + unit.setPyrisIngestionState(IngestionState.DONE); + attachmentUnitRepository.save(unit); + } + else if (statusUpdate.stages().getLast().state() == PyrisStageState.ERROR || statusUpdate.stages().getLast().state() == PyrisStageState.SKIPPED) { + unit.setPyrisIngestionState(IngestionState.ERROR); + attachmentUnitRepository.save(unit); } } } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java index 23b811258036..3b27136c1c45 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java @@ -23,6 +23,7 @@ import de.tum.in.www1.artemis.service.FilePathService; import de.tum.in.www1.artemis.service.connectors.pyris.dto.PyrisPipelineExecutionSettingsDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureUnitWebhookDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureDeletionExecutionDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; import de.tum.in.www1.artemis.service.iris.exception.IrisInternalPyrisErrorException; @@ -83,14 +84,14 @@ private PyrisLectureUnitWebhookDTO processAttachmentForUpdate(AttachmentUnit att String base64EncodedPdf = attachmentToBase64(attachmentUnit); attachmentUnit.setPyrisIngestionState(IngestionState.IN_PROGRESS); lectureUnitRepository.save(attachmentUnit); - return new PyrisLectureUnitWebhookDTO(true, base64EncodedPdf, lectureUnitId, lectureUnitName, lectureId, lectureTitle, courseId, courseTitle, courseDescription); + return new PyrisLectureUnitWebhookDTO(base64EncodedPdf, lectureUnitId, lectureUnitName, lectureId, lectureTitle, courseId, courseTitle, courseDescription); } private PyrisLectureUnitWebhookDTO processAttachmentForDeletion(AttachmentUnit attachmentUnit) { Long lectureUnitId = attachmentUnit.getId(); Long lectureId = attachmentUnit.getLecture().getId(); Long courseId = attachmentUnit.getLecture().getCourse().getId(); - return new PyrisLectureUnitWebhookDTO(false, "", lectureUnitId, "", lectureId, "", courseId, "", ""); + return new PyrisLectureUnitWebhookDTO("", lectureUnitId, "", lectureId, "", courseId, "", ""); } /** @@ -123,7 +124,7 @@ public String deleteLectureFromPyrisDB(List attachmentUnits) { toUpdateAttachmentUnits.add(processAttachmentForDeletion(unit)); }); if (!toUpdateAttachmentUnits.isEmpty()) { - return executeLectureWebhook(toUpdateAttachmentUnits); + return executeLectureDeletionWebhook(toUpdateAttachmentUnits); } } catch (Exception e) { @@ -143,32 +144,49 @@ public String addLectureUnitsToPyrisDB(List attachmentUnits) { return null; } try { - List toUpdateAttachmentUnits = new ArrayList<>(); attachmentUnits.stream().filter(unit -> unit.getAttachment().getAttachmentType() == AttachmentType.FILE && unit.getAttachment().getLink().endsWith(".pdf")) .forEach(unit -> { - toUpdateAttachmentUnits.add(processAttachmentForUpdate(unit)); + executeLectureAdditionWebhook(processAttachmentForUpdate(unit)); }); - if (!toUpdateAttachmentUnits.isEmpty()) { - return executeLectureWebhook(toUpdateAttachmentUnits); - } + } catch (Exception e) { log.error(e.getMessage()); + attachmentUnits.stream().filter(unit -> unit.getAttachment().getAttachmentType() == AttachmentType.FILE && unit.getAttachment().getLink().endsWith(".pdf")) + .forEach(unit -> { + unit.setPyrisIngestionState(IngestionState.ERROR); + lectureUnitRepository.save(unit); + }); } + ; return null; } /** * executes executeLectureWebhook add or delete lectures from to the vector database on pyris * - * @param toUpdateAttachmentUnits The attachmentUnit that are goin to be Updated / deleted + * @param toUpdateAttachmentUnits The attachmentUnit that are goin to be deleted + * @return jobToken if the job was created + */ + private String executeLectureDeletionWebhook(List toUpdateAttachmentUnits) { + String jobToken = pyrisJobService.addIngestionWebhookJob(); + PyrisPipelineExecutionSettingsDTO settingsDTO = new PyrisPipelineExecutionSettingsDTO(jobToken, List.of(), artemisBaseUrl); + PyrisWebhookLectureDeletionExecutionDTO executionDTO = new PyrisWebhookLectureDeletionExecutionDTO(toUpdateAttachmentUnits, settingsDTO, List.of()); + pyrisConnectorService.executeLectureDeletionWebhook(executionDTO); + return jobToken; + } + + /** + * executes executeLectureAdditionWebhook add lecture from to the vector database on pyris + * + * @param toUpdateAttachmentUnit The attachmentUnit that are going to be Updated * @return jobToken if the job was created */ - private String executeLectureWebhook(List toUpdateAttachmentUnits) { + private String executeLectureAdditionWebhook(PyrisLectureUnitWebhookDTO toUpdateAttachmentUnit) { String jobToken = pyrisJobService.addIngestionWebhookJob(); PyrisPipelineExecutionSettingsDTO settingsDTO = new PyrisPipelineExecutionSettingsDTO(jobToken, List.of(), artemisBaseUrl); - PyrisWebhookLectureIngestionExecutionDTO executionDTO = new PyrisWebhookLectureIngestionExecutionDTO(toUpdateAttachmentUnits, settingsDTO, List.of()); - pyrisConnectorService.executeLectureWebhook("fullIngestion", executionDTO); + PyrisWebhookLectureIngestionExecutionDTO executionDTO = new PyrisWebhookLectureIngestionExecutionDTO(toUpdateAttachmentUnit, settingsDTO, List.of()); + pyrisConnectorService.executeLectureAddtionWebhook("fullIngestion", executionDTO); return jobToken; } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java index 0fe0c07813b4..444056c6626c 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java @@ -1,11 +1,12 @@ package de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook; import java.util.List; +import java.util.Optional; import com.fasterxml.jackson.annotation.JsonInclude; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record PyrisLectureIngestionStatusUpdateDTO(String result, List stages, List ids) { +public record PyrisLectureIngestionStatusUpdateDTO(Optional id, String result, List stages) { } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitWebhookDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitWebhookDTO.java index 6a36b867d16c..b990affda038 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitWebhookDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitWebhookDTO.java @@ -8,6 +8,6 @@ * providing necessary details such as lecture and course identifiers, names, and descriptions. */ @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record PyrisLectureUnitWebhookDTO(Boolean toUpdate, String pdfFile, Long lectureUnitId, String lectureUnitName, Long lectureId, String lectureName, Long courseId, - String courseName, String courseDescription) { +public record PyrisLectureUnitWebhookDTO(String pdfFile, Long lectureUnitId, String lectureUnitName, Long lectureId, String lectureName, Long courseId, String courseName, + String courseDescription) { } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisWebhookLectureDeletionExecutionDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisWebhookLectureDeletionExecutionDTO.java new file mode 100644 index 000000000000..20b70e4d7f3a --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisWebhookLectureDeletionExecutionDTO.java @@ -0,0 +1,13 @@ +package de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import de.tum.in.www1.artemis.service.connectors.pyris.dto.PyrisPipelineExecutionSettingsDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public record PyrisWebhookLectureDeletionExecutionDTO(List pyrisLectureUnits, PyrisPipelineExecutionSettingsDTO settings, + List initialStages) { +} diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisWebhookLectureIngestionExecutionDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisWebhookLectureIngestionExecutionDTO.java index db1741183826..7b39d640fa59 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisWebhookLectureIngestionExecutionDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisWebhookLectureIngestionExecutionDTO.java @@ -8,6 +8,5 @@ import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record PyrisWebhookLectureIngestionExecutionDTO(List pyrisLectureUnitWebhookDTOS, PyrisPipelineExecutionSettingsDTO settings, - List initialStages) { +public record PyrisWebhookLectureIngestionExecutionDTO(PyrisLectureUnitWebhookDTO pyrisLectureUnit, PyrisPipelineExecutionSettingsDTO settings, List initialStages) { } diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java index 3e3e589c4203..e3048cb23569 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java @@ -207,4 +207,18 @@ public ResponseEntity getLectureUnitById(@PathVariable @Valid Long lectureUnit.setCompleted(lectureUnit.isCompletedFor(userRepository.getUser())); return ResponseEntity.ok(lectureUnit); } + + /** + * POST /courses/{courseId}/ingest + * This endpooint is for starting the ingestion of all lectures or only one lecture when triggered in Artemis. + * + * @param lectureId the ID of the Lecture in which the lecture unit is + * @param lectureUnitId The Id of the Lecture Unit that is ingestLectureUnit(@PathVariable Long lectureId, @PathVariable Long lectureUnitId) { + LectureUnit lectureUnit = this.lectureUnitRepository.findByIdWithCompetenciesAndSlidesElseThrow(lectureUnitId); + return ResponseEntity.ok().body(lectureUnitService.ingestLectureUnitInPyris(lectureUnit)); + } } diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html index d45dd226a097..5b5471279b35 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html @@ -79,7 +79,7 @@ error: getIcon(lectureUnit) === resendToIris, 'btn-success': getIcon(lectureUnit) === done, }" - (click)="onIngestButtonClicked(lectureUnit)" + (click)="onIngestButtonClicked(lectureUnit.id!)" [ngbTooltip]="'entity.action.sendToIris' | artemisTranslate" > diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts index 596300223022..4e0d47ccc24e 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts @@ -220,8 +220,8 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { onEditButtonClicked(lectureUnit: LectureUnit) { this.onEditLectureUnitClicked.emit(lectureUnit); } - onIngestButtonClicked(lectureUnit: LectureUnit) { - this.lectureUnitService.ingestLectureUnitInPyris(lectureUnit.id!).subscribe({ + onIngestButtonClicked(lectureUnitId: number) { + this.lectureUnitService.ingestLectureUnitInPyris(lectureUnitId, this.lecture.id!).subscribe({ error: (error) => console.error('Failed to send Ingestion request', error), }); } diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts index 2825fed950e2..df302a87b14c 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts @@ -167,4 +167,14 @@ export class LectureUnitService { getLectureUnitById(lectureUnitId: number): Observable { return this.httpClient.get(`${this.resourceURL}/lecture-units/${lectureUnitId}`); } + /** + * Triggers the ingestion of one lecture unit + */ + ingestLectureUnitInPyris(lectureUnitId: number, lectureId: number): Observable> { + const params = new HttpParams(); + return this.httpClient.post(`${this.resourceURL}/lectures/${lectureId}/lecture-units/${lectureUnitId}/ingest`, null, { + params: params, + observe: 'response', + }); + } } From cadcf14381b2f20c4a6a3c2c0bbd29ec82d78890 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Fri, 19 Jul 2024 16:53:38 +0200 Subject: [PATCH 03/95] Add Ingestion Status server updates. --- .../www1/artemis/service/LectureService.java | 8 +- .../artemis/service/LectureUnitService.java | 8 +- .../pyris/PyrisConnectorService.java | 18 +- .../connectors/pyris/PyrisWebhookService.java | 44 ++- .../PyrisLectureIngestionStatusUpdateDTO.java | 2 +- .../artemis/web/rest/LectureResource.java | 6 +- .../web/rest/lecture/LectureUnitResource.java | 6 +- .../connector/IrisRequestMockProvider.java | 18 ++ .../iris/PyrisConnectorServiceTest.java | 9 +- .../iris/PyrisLectureIngestionTest.java | 263 +++++++++++++----- 10 files changed, 264 insertions(+), 118 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/service/LectureService.java b/src/main/java/de/tum/in/www1/artemis/service/LectureService.java index d722fd2673da..166a142f3055 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/LectureService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/LectureService.java @@ -168,16 +168,14 @@ public void delete(Lecture lecture, boolean updateCompetencyProgress) { * Ingest the lectures when triggered by the ingest lectures button * * @param lectures set of lectures to be ingested - * @return returns the job token if the operation is successful else it returns null */ - public boolean ingestLecturesInPyris(Set lectures) { + public void ingestLecturesInPyris(Set lectures) { if (pyrisWebhookService.isPresent()) { List attachmentUnitList = lectures.stream().flatMap(lec -> lec.getLectureUnits().stream()).filter(unit -> unit instanceof AttachmentUnit) .map(unit -> (AttachmentUnit) unit).toList(); - if (!attachmentUnitList.isEmpty()) { - return pyrisWebhookService.get().addLectureUnitsToPyrisDB(attachmentUnitList) != null; + for (AttachmentUnit attachmentUnit : attachmentUnitList) { + pyrisWebhookService.get().addLectureUnitToPyrisDB(attachmentUnit); } } - return false; } } diff --git a/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java b/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java index 4979e12d322d..544d2bbb902f 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/LectureUnitService.java @@ -246,12 +246,10 @@ public URL validateUrlStringAndReturnUrl(String urlString) { * Ingest the lectureUnit when triggered by the ingest lectureUnit button * * @param lectureUnit lectureUnit to be ingested - * @return returns the job token if the operation is successful else it returns null */ - public boolean ingestLectureUnitInPyris(LectureUnit lectureUnit) { - if (lectureUnit instanceof AttachmentUnit) { - return pyrisWebhookService.filter(webhookService -> webhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lectureUnit)) != null).isPresent(); + public void ingestLectureUnitInPyris(LectureUnit lectureUnit) { + if (lectureUnit instanceof AttachmentUnit && pyrisWebhookService.isPresent()) { + pyrisWebhookService.get().addLectureUnitToPyrisDB((AttachmentUnit) lectureUnit); } - return false; } } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java index 0e5944eb6e0c..36c4956a2745 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java @@ -120,14 +120,15 @@ public void executeLectureAddtionWebhook(String variant, PyrisWebhookLectureInge private void setIngestionStateToError(PyrisWebhookLectureIngestionExecutionDTO executionDTO, RuntimeException e) { log.error("Failed to send lectures to Pyris", e); - Optional optionalUnit = lectureUnitRepository.findById(executionDTO.pyrisLectureUnit().lectureUnitId()); - optionalUnit.ifPresent(unit -> { - if (unit instanceof AttachmentUnit) { - AttachmentUnit attachmentUnit = (AttachmentUnit) unit; - attachmentUnit.setPyrisIngestionState(IngestionState.ERROR); - lectureUnitRepository.save(attachmentUnit); - } - }); + if (executionDTO != null) { + Optional optionalUnit = lectureUnitRepository.findById(executionDTO.pyrisLectureUnit().lectureUnitId()); + optionalUnit.ifPresent(unit -> { + if (unit instanceof AttachmentUnit attachmentUnit) { + attachmentUnit.setPyrisIngestionState(IngestionState.ERROR); + lectureUnitRepository.save(attachmentUnit); + } + }); + } } /** @@ -143,7 +144,6 @@ public void executeLectureDeletionWebhook(PyrisWebhookLectureDeletionExecutionDT catch (HttpStatusCodeException e) { log.error("Failed to send lectures to Pyris", e); throw toIrisException(e); - // TODO : add error ingestion UI. } catch (RestClientException | IllegalArgumentException e) { log.error("Failed to send lectures to Pyris", e); diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java index 3b27136c1c45..d00fae8c7391 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java @@ -99,15 +99,15 @@ private PyrisLectureUnitWebhookDTO processAttachmentForDeletion(AttachmentUnit a * * @param courseId Id of the course where the attachment is added * @param newAttachmentUnits the new attachment Units to be sent to pyris for ingestion - * @return true if the units were sent to pyris */ - public boolean autoUpdateAttachmentUnitsInPyris(Long courseId, List newAttachmentUnits) { + public void autoUpdateAttachmentUnitsInPyris(Long courseId, List newAttachmentUnits) { IrisCourseSettings courseSettings = irisSettingsRepository.findCourseSettings(courseId).isPresent() ? irisSettingsRepository.findCourseSettings(courseId).get() : null; if (courseSettings != null && courseSettings.getIrisLectureIngestionSettings() != null && courseSettings.getIrisLectureIngestionSettings().isEnabled() && courseSettings.getIrisLectureIngestionSettings().getAutoIngestOnLectureAttachmentUpload()) { - return addLectureUnitsToPyrisDB(newAttachmentUnits) != null; + for (AttachmentUnit attachmentUnit : newAttachmentUnits) { + addLectureUnitToPyrisDB(attachmentUnit); + } } - return false; } /** @@ -136,29 +136,23 @@ public String deleteLectureFromPyrisDB(List attachmentUnits) { /** * adds the lectures to the vector database on pyris * - * @param attachmentUnits The attachmentUnit that got Updated / erased - * @return jobToken if the job was created + * @param attachmentUnit The attachmentUnit that got Updated / erased */ - public String addLectureUnitsToPyrisDB(List attachmentUnits) { - if (!lectureIngestionEnabled(attachmentUnits.getFirst().getLecture().getCourse())) { - return null; - } - try { - attachmentUnits.stream().filter(unit -> unit.getAttachment().getAttachmentType() == AttachmentType.FILE && unit.getAttachment().getLink().endsWith(".pdf")) - .forEach(unit -> { - executeLectureAdditionWebhook(processAttachmentForUpdate(unit)); - }); - - } - catch (Exception e) { - log.error(e.getMessage()); - attachmentUnits.stream().filter(unit -> unit.getAttachment().getAttachmentType() == AttachmentType.FILE && unit.getAttachment().getLink().endsWith(".pdf")) - .forEach(unit -> { - unit.setPyrisIngestionState(IngestionState.ERROR); - lectureUnitRepository.save(unit); - }); + public String addLectureUnitToPyrisDB(AttachmentUnit attachmentUnit) { + if (lectureIngestionEnabled(attachmentUnit.getLecture().getCourse())) { + try { + if (attachmentUnit.getAttachment().getAttachmentType() == AttachmentType.FILE && attachmentUnit.getAttachment().getLink().endsWith(".pdf")) { + return executeLectureAdditionWebhook(processAttachmentForUpdate(attachmentUnit)); + } + } + catch (Exception e) { + log.error(e.getMessage()); + if (attachmentUnit.getAttachment().getAttachmentType() == AttachmentType.FILE && attachmentUnit.getAttachment().getLink().endsWith(".pdf")) { + attachmentUnit.setPyrisIngestionState(IngestionState.ERROR); + lectureUnitRepository.save(attachmentUnit); + } + } } - ; return null; } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java index 444056c6626c..f871570fd6ea 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java @@ -8,5 +8,5 @@ import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record PyrisLectureIngestionStatusUpdateDTO(Optional id, String result, List stages) { +public record PyrisLectureIngestionStatusUpdateDTO(String result, List stages, Optional id) { } diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/LectureResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/LectureResource.java index 9aa73d603557..249a2c2b31ec 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/LectureResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/LectureResource.java @@ -277,13 +277,15 @@ public ResponseEntity ingestLectures(@PathVariable Long courseId, @Requ if (lectureToIngest.isPresent()) { Set lecturesToIngest = new HashSet<>(); lecturesToIngest.add(lectureToIngest.get()); - return ResponseEntity.ok().body(lectureService.ingestLecturesInPyris(lecturesToIngest)); + lectureService.ingestLecturesInPyris(lecturesToIngest); + return ResponseEntity.ok().body(true); } return ResponseEntity.badRequest() .headers(HeaderUtil.createAlert(applicationName, "Could not send lecture to Iris, no lecture found with the provided id.", "idExists")).body(null); } - return ResponseEntity.ok().body(lectureService.ingestLecturesInPyris(course.getLectures())); + lectureService.ingestLecturesInPyris(course.getLectures()); + return ResponseEntity.ok().body(true); } /** diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java index e3048cb23569..1b503f4f82fb 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java @@ -212,13 +212,13 @@ public ResponseEntity getLectureUnitById(@PathVariable @Valid Long * POST /courses/{courseId}/ingest * This endpooint is for starting the ingestion of all lectures or only one lecture when triggered in Artemis. * - * @param lectureId the ID of the Lecture in which the lecture unit is * @param lectureUnitId The Id of the Lecture Unit that is ingestLectureUnit(@PathVariable Long lectureId, @PathVariable Long lectureUnitId) { + public ResponseEntity ingestLectureUnit(@PathVariable Long lectureUnitId) { LectureUnit lectureUnit = this.lectureUnitRepository.findByIdWithCompetenciesAndSlidesElseThrow(lectureUnitId); - return ResponseEntity.ok().body(lectureUnitService.ingestLectureUnitInPyris(lectureUnit)); + lectureUnitService.ingestLectureUnitInPyris(lectureUnit); + return ResponseEntity.ok().body(true); } } diff --git a/src/test/java/de/tum/in/www1/artemis/connector/IrisRequestMockProvider.java b/src/test/java/de/tum/in/www1/artemis/connector/IrisRequestMockProvider.java index 98f3ad4db3eb..09ed5dee613d 100644 --- a/src/test/java/de/tum/in/www1/artemis/connector/IrisRequestMockProvider.java +++ b/src/test/java/de/tum/in/www1/artemis/connector/IrisRequestMockProvider.java @@ -105,6 +105,15 @@ public void mockIngestionWebhookRunResponse(Consumer responseConsumer) { + mockServer.expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/lectures/delete")).andExpect(method(HttpMethod.POST)).andRespond(request -> { + var mockRequest = (MockClientHttpRequest) request; + var dto = mapper.readValue(mockRequest.getBodyAsString(), PyrisWebhookLectureIngestionExecutionDTO.class); + responseConsumer.accept(dto); + return MockRestResponseCreators.withRawStatus(HttpStatus.ACCEPTED.value()).createResponse(request); + }); + } + public void mockRunError(int httpStatus) { // @formatter:off mockServer @@ -123,6 +132,15 @@ public void mockIngestionWebhookRunError(int httpStatus) { // @formatter:on } + public void mockDeletionWebhookRunError(int httpStatus) { + // @formatter:off + mockServer + .expect(ExpectedCount.once(), requestTo(webhooksApiURL + "/lectures/delete")) + .andExpect(method(HttpMethod.POST)) + .andRespond(withStatus(HttpStatus.valueOf(httpStatus))); + // @formatter:on + } + public void mockModelsResponse() throws JsonProcessingException { var irisModelDTO = new PyrisModelDTO("TEST_MODEL", "Test model", "Test description"); var irisModelDTOArray = new PyrisModelDTO[] { irisModelDTO }; diff --git a/src/test/java/de/tum/in/www1/artemis/iris/PyrisConnectorServiceTest.java b/src/test/java/de/tum/in/www1/artemis/iris/PyrisConnectorServiceTest.java index c1ff9d381d47..42ad834cf2a9 100644 --- a/src/test/java/de/tum/in/www1/artemis/iris/PyrisConnectorServiceTest.java +++ b/src/test/java/de/tum/in/www1/artemis/iris/PyrisConnectorServiceTest.java @@ -46,7 +46,14 @@ void testExceptionV2(int httpStatus, Class exceptionClass) { @MethodSource("irisExceptions") void testExceptionIngestionV2(int httpStatus, Class exceptionClass) { irisRequestMockProvider.mockIngestionWebhookRunError(httpStatus); - assertThatThrownBy(() -> pyrisConnectorService.executeLectureWebhook("fullIngestion", null)).isInstanceOf(exceptionClass); + assertThatThrownBy(() -> pyrisConnectorService.executeLectureAddtionWebhook("fullIngestion", null)).isInstanceOf(exceptionClass); + } + + @ParameterizedTest + @MethodSource("irisExceptions") + void testExceptionLectureDeletionV2(int httpStatus, Class exceptionClass) { + irisRequestMockProvider.mockDeletionWebhookRunError(httpStatus); + assertThatThrownBy(() -> pyrisConnectorService.executeLectureDeletionWebhook(null)).isInstanceOf(exceptionClass); } @Test diff --git a/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java b/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java index 4556660764d3..88d2c1dd6c70 100644 --- a/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java +++ b/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java @@ -1,32 +1,55 @@ package de.tum.in.www1.artemis.iris; import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Optional; +import jakarta.validation.constraints.NotNull; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.mock.web.MockMultipartFile; import org.springframework.security.test.context.support.WithMockUser; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.util.LinkedMultiValueMap; +import com.fasterxml.jackson.databind.ObjectMapper; + import de.tum.in.www1.artemis.course.CourseUtilService; +import de.tum.in.www1.artemis.domain.Attachment; import de.tum.in.www1.artemis.domain.Course; import de.tum.in.www1.artemis.domain.Lecture; import de.tum.in.www1.artemis.domain.iris.settings.IrisCourseSettings; import de.tum.in.www1.artemis.domain.lecture.AttachmentUnit; +import de.tum.in.www1.artemis.domain.lecture.LectureUnit; +import de.tum.in.www1.artemis.lecture.LectureFactory; import de.tum.in.www1.artemis.lecture.LectureUtilService; +import de.tum.in.www1.artemis.repository.AttachmentUnitRepository; import de.tum.in.www1.artemis.repository.LectureRepository; +import de.tum.in.www1.artemis.repository.LectureUnitRepository; import de.tum.in.www1.artemis.repository.iris.IrisSettingsRepository; import de.tum.in.www1.artemis.service.connectors.pyris.PyrisJobService; import de.tum.in.www1.artemis.service.connectors.pyris.PyrisStatusUpdateService; import de.tum.in.www1.artemis.service.connectors.pyris.PyrisWebhookService; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureIngestionStatusUpdateDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState; import de.tum.in.www1.artemis.user.UserUtilService; @@ -59,6 +82,17 @@ class PyrisLectureIngestionTest extends AbstractIrisIntegrationTest { @Autowired protected IrisSettingsRepository irisSettingsRepository; + @Autowired + protected LectureUnitRepository lectureUnitRepository; + + @Autowired + private ObjectMapper mapper; + + @Autowired + private AttachmentUnitRepository attachmentUnitRepository; + + private Attachment attachment; + private Lecture lecture1; @BeforeEach @@ -83,7 +117,13 @@ void initTestCase() throws Exception { @Test @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAutoUpdateAttachmentUnitsWithAutoUpdateEnabled() { + void autoIngestionWhenAttachmentUnitCreatedAndAutoUpdateEnabled() throws Exception { + this.attachment = LectureFactory.generateAttachment(null); + this.attachment.setName(" LoremIpsum "); + this.attachment.setLink("/api/files/temp/example.txt"); + this.lecture1 = lectureUtilService.createCourseWithLecture(true); + AttachmentUnit attachmentUnit = new AttachmentUnit(); + attachmentUnit.setDescription("Lorem Ipsum"); activateIrisFor(lecture1.getCourse()); IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); courseSettings.getIrisLectureIngestionSettings().setAutoIngestOnLectureAttachmentUpload(true); @@ -91,65 +131,61 @@ void testAutoUpdateAttachmentUnitsWithAutoUpdateEnabled() { irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - assertThat(pyrisWebhookService.autoUpdateAttachmentUnitsInPyris(lecture1.getCourse().getId(), List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst()))).isTrue(); - assertThat(pyrisWebhookService.autoUpdateAttachmentUnitsInPyris(lecture1.getCourse().getId(), List.of((AttachmentUnit) lecture1.getLectureUnits().getLast()))).isFalse(); - } - - @Test - @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAutoUpdateAttachmentUnitsWithAutoUpdateDisabled() { - activateIrisFor(lecture1.getCourse()); - IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); - courseSettings.getIrisLectureIngestionSettings().setAutoIngestOnLectureAttachmentUpload(false); - this.irisSettingsRepository.save(courseSettings); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { - assertThat(dto.settings().authenticationToken()).isNotNull(); - }); - assertThat(pyrisWebhookService.autoUpdateAttachmentUnitsInPyris(lecture1.getCourse().getId(), List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst()))).isFalse(); - assertThat(pyrisWebhookService.autoUpdateAttachmentUnitsInPyris(lecture1.getCourse().getId(), List.of((AttachmentUnit) lecture1.getLectureUnits().getLast()))).isFalse(); + var result = request.performMvcRequest(buildCreateAttachmentUnit(attachmentUnit, attachment)).andExpect(status().isCreated()).andReturn(); + var persistedAttachmentUnit = mapper.readValue(result.getResponse().getContentAsString(), AttachmentUnit.class); + var updatedAttachmentUnit = attachmentUnitRepository.findById(persistedAttachmentUnit.getId()).orElseThrow(); + assertThat(updatedAttachmentUnit.getPyrisIngestionState()).isEqualTo(IngestionState.IN_PROGRESS); } @Test @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAddLectureToPyrisDatabaseWithCourseSettingsDisabled() { - activateIrisFor(lecture1.getCourse()); - IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); - courseSettings.getIrisLectureIngestionSettings().setEnabled(false); - this.irisSettingsRepository.save(courseSettings); + void noAutoIngestionWhenAttachmentUnitCreatedAndAutoUpdateDisabled() throws Exception { + this.attachment = LectureFactory.generateAttachment(null); + this.attachment.setName(" LoremIpsum "); + this.attachment.setLink("/api/files/temp/example.txt"); + this.lecture1 = lectureUtilService.createCourseWithLecture(true); + AttachmentUnit attachmentUnit = new AttachmentUnit(); + attachmentUnit.setDescription("Lorem Ipsum"); irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst())); - assertThat(jobToken).isNull(); - jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getLast())); - assertThat(jobToken).isNull(); + var result = request.performMvcRequest(buildCreateAttachmentUnit(attachmentUnit, attachment)).andExpect(status().isCreated()).andReturn(); + var persistedAttachmentUnit = mapper.readValue(result.getResponse().getContentAsString(), AttachmentUnit.class); + var updatedAttachmentUnit = attachmentUnitRepository.findById(persistedAttachmentUnit.getId()).orElseThrow(); + assertThat(updatedAttachmentUnit.getPyrisIngestionState()).isEqualTo(IngestionState.NOT_STARTED); } @Test @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testDeleteLecturefromPyrisDatabaseWithCourseSettingsDisabled() { + void testIngestLecturesButtonInPyris() throws Exception { activateIrisFor(lecture1.getCourse()); - IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); - courseSettings.getIrisLectureIngestionSettings().setEnabled(false); - this.irisSettingsRepository.save(courseSettings); irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst())); - assertThat(jobToken).isNotNull(); - jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getLast())); - assertThat(jobToken).isNull(); + request.postWithResponseBody("/api/courses/" + lecture1.getCourse().getId() + "/ingest", Optional.empty(), boolean.class, HttpStatus.OK); + Optional optionalUnit = lectureUnitRepository.findById(lecture1.getLectureUnits().getFirst().getId()); + if (optionalUnit.isPresent() && optionalUnit.get() instanceof AttachmentUnit attachmentUnit) { + assertThat(attachmentUnit.getPyrisIngestionState()).isEqualTo(IngestionState.IN_PROGRESS); + } + optionalUnit = lectureUnitRepository.findById(lecture1.getLectureUnits().getLast().getId()); + if (optionalUnit.isPresent() && optionalUnit.get() instanceof AttachmentUnit attachmentUnit) { + assertThat(attachmentUnit.getPyrisIngestionState()).isEqualTo(IngestionState.NOT_STARTED); + } } @Test @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testIngestLecturesButtonInPyris() throws Exception { + void testIngestLectureUnitButtonInPyris() throws Exception { activateIrisFor(lecture1.getCourse()); irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - Boolean response = request.postWithResponseBody("/api/courses/" + lecture1.getCourse().getId() + "/ingest", Optional.empty(), boolean.class, HttpStatus.OK); - assertThat(response).isTrue(); + request.postWithResponseBody("/api/lectures/" + lecture1.getId() + "/lecture-units/" + lecture1.getLectureUnits().getFirst().getId() + "/ingest", Optional.empty(), + boolean.class, HttpStatus.OK); + if (lectureUnitRepository.findById(lecture1.getLectureUnits().getFirst().getId()).isPresent()) { + AttachmentUnit attachmentUnit = (AttachmentUnit) lectureUnitRepository.findById(lecture1.getLectureUnits().getFirst().getId()).get(); + assertThat(attachmentUnit.getPyrisIngestionState() == IngestionState.IN_PROGRESS).isTrue(); + } } @Test @@ -158,7 +194,7 @@ void testDeleteLecturefromPyrisDatabaseWithCourseSettingsEnabled() { activateIrisFor(lecture1.getCourse()); IrisCourseSettings courseSettings = irisSettingsService.getRawIrisSettingsFor(lecture1.getCourse()); this.irisSettingsRepository.save(courseSettings); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { + irisRequestMockProvider.mockDeletionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst())); @@ -169,31 +205,40 @@ void testDeleteLecturefromPyrisDatabaseWithCourseSettingsEnabled() { @Test @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAddLectureToPyrisDBAddJobWithCourseSettingsEnabled() { + void testAddLectureToPyrisDBAddJobWithCourseSettingsEnabled() throws Exception { activateIrisFor(lecture1.getCourse()); irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getFirst())); - assertThat(jobToken).isNotNull(); - jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of((AttachmentUnit) lecture1.getLectureUnits().getLast())); - assertThat(jobToken).isNull(); + request.postWithResponseBody("/api/courses/" + lecture1.getCourse().getId() + "/ingest", Optional.empty(), boolean.class, HttpStatus.OK); + Optional optionalUnit = lectureUnitRepository.findById(lecture1.getLectureUnits().getFirst().getId()); + if (optionalUnit.isPresent() && optionalUnit.get() instanceof AttachmentUnit unit) { + assertThat(unit.getPyrisIngestionState()).isEqualTo(IngestionState.IN_PROGRESS); + } + optionalUnit = lectureUnitRepository.findById(lecture1.getLectureUnits().getLast().getId()); + if (optionalUnit.isPresent() && optionalUnit.get() instanceof AttachmentUnit unit) { + assertThat(unit.getPyrisIngestionState()).isEqualTo(IngestionState.NOT_STARTED); + } } @Test @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") - void testAllStagesDoneRemovesAdditionIngestionJob() throws Exception { + void testAllStagesDoneIngestionStateDone() throws Exception { activateIrisFor(lecture1.getCourse()); irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of(attachmentUnit)); + if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { + String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage)); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), + Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); - assertThat(pyrisJobService.getJob(jobToken)).isNull(); + Optional optionalUnit = lectureUnitRepository.findById(unit.getId()); + if (optionalUnit.isPresent() && optionalUnit.get() instanceof AttachmentUnit attachUnit) { + assertThat(attachUnit.getPyrisIngestionState()).isEqualTo(IngestionState.DONE); + } } } @@ -201,13 +246,14 @@ void testAllStagesDoneRemovesAdditionIngestionJob() throws Exception { @WithMockUser(username = TEST_PREFIX + "instructor1", roles = "INSTRUCTOR") void testAllStagesDoneRemovesDeletionIngestionJob() throws Exception { activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { + irisRequestMockProvider.mockDeletionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(attachmentUnit)); + if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { + String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage)); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), + Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); @@ -220,11 +266,12 @@ void testStageNotDoneKeepsAdditionIngestionJob() throws Exception { irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of(attachmentUnit)); + if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { + String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); PyrisStageDTO inProgressStage = new PyrisStageDTO("inProgressStage", 1, PyrisStageState.IN_PROGRESS, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage)); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), + Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNotNull(); @@ -234,14 +281,15 @@ void testStageNotDoneKeepsAdditionIngestionJob() throws Exception { @Test void testStageNotDoneKeepsDeletionIngetionJob() throws Exception { activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { + irisRequestMockProvider.mockDeletionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(attachmentUnit)); + if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { + String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); PyrisStageDTO inProgressStage = new PyrisStageDTO("inProgressStage", 1, PyrisStageState.IN_PROGRESS, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage)); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), + Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNotNull(); @@ -251,16 +299,21 @@ void testStageNotDoneKeepsDeletionIngetionJob() throws Exception { @Test void testErrorStageRemovesDeletionIngetionJob() throws Exception { activateIrisFor(lecture1.getCourse()); - irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { + irisRequestMockProvider.mockDeletionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(attachmentUnit)); + if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { + String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage)); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), + Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); + Optional optionalUnit = lectureUnitRepository.findById(unit.getId()); + if (optionalUnit.isPresent() && optionalUnit.get() instanceof AttachmentUnit attachUnit) { + assertThat(attachUnit.getPyrisIngestionState()).isEqualTo(IngestionState.ERROR); + } } } @@ -270,13 +323,18 @@ void testErrorStageRemovesAdditionIngetionJob() throws Exception { irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { assertThat(dto.settings().authenticationToken()).isNotNull(); }); - if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit attachmentUnit) { - String jobToken = pyrisWebhookService.addLectureUnitsToPyrisDB(List.of(attachmentUnit)); + if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { + String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage)); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), + Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); + Optional optionalUnit = lectureUnitRepository.findById(unit.getId()); + if (optionalUnit.isPresent() && optionalUnit.get() instanceof AttachmentUnit atUnit) { + assertThat(atUnit.getPyrisIngestionState()).isEqualTo(IngestionState.ERROR); + } } } @@ -289,7 +347,8 @@ void testRunIdIngestionJob() throws Exception { String newJobToken = pyrisJobService.addIngestionWebhookJob(); String chatJobToken = pyrisJobService.addCourseChatJob(123L, 123L); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage)); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), + Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + chatJobToken)))); MockHttpServletResponse response = request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + newJobToken + "/status", statusUpdate, HttpStatus.CONFLICT, headers); @@ -297,4 +356,74 @@ void testRunIdIngestionJob() throws Exception { response = request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + chatJobToken + "/status", statusUpdate, HttpStatus.CONFLICT, headers); assertThat(response.getContentAsString()).contains("Run ID is not an ingestion job"); } + + @Test + void testIngestionJobDone() throws Exception { + activateIrisFor(lecture1.getCourse()); + irisRequestMockProvider.mockIngestionWebhookRunResponse(dto -> { + assertThat(dto.settings().authenticationToken()).isNotNull(); + }); + String newJobToken = pyrisJobService.addIngestionWebhookJob(); + String chatJobToken = pyrisJobService.addCourseChatJob(123L, 123L); + PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), + Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + chatJobToken)))); + MockHttpServletResponse response = request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + newJobToken + "/status", statusUpdate, + HttpStatus.CONFLICT, headers); + assertThat(response.getContentAsString()).contains("Run ID in URL does not match run ID in request body"); + response = request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + chatJobToken + "/status", statusUpdate, HttpStatus.CONFLICT, headers); + assertThat(response.getContentAsString()).contains("Run ID is not an ingestion job"); + } + + /** + * Generates an attachment unit pdf file with 5 pages + * + * @return MockMultipartFile attachment unit pdf file + */ + private MockMultipartFile createAttachmentUnitPdf() throws IOException { + + var font = new PDType1Font(Standard14Fonts.FontName.TIMES_ROMAN); + + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); PDDocument document = new PDDocument()) { + + for (int i = 1; i <= 3; i++) { + document.addPage(new PDPage()); + PDPageContentStream contentStream = new PDPageContentStream(document, document.getPage(i - 1)); + + if (i == 2) { + contentStream.beginText(); + contentStream.setFont(font, 12); + contentStream.newLineAtOffset(25, -15); + contentStream.showText("itp20.."); + contentStream.newLineAtOffset(25, 500); + contentStream.showText("Outline"); + contentStream.newLineAtOffset(0, -15); + contentStream.showText("First Unit"); + contentStream.close(); + continue; + } + contentStream.beginText(); + contentStream.setFont(font, 12); + contentStream.newLineAtOffset(25, 500); + String text = "This is the sample document"; + contentStream.showText(text); + contentStream.endText(); + contentStream.close(); + } + document.save(outputStream); + document.close(); + return new MockMultipartFile("file", "lectureFile.pdf", "application/json", outputStream.toByteArray()); + } + } + + private MockHttpServletRequestBuilder buildCreateAttachmentUnit(@NotNull AttachmentUnit attachmentUnit, @NotNull Attachment attachment) throws Exception { + var attachmentUnitPart = new MockMultipartFile("attachmentUnit", "", MediaType.APPLICATION_JSON_VALUE, mapper.writeValueAsString(attachmentUnit).getBytes()); + var attachmentPart = new MockMultipartFile("attachment", "", MediaType.APPLICATION_JSON_VALUE, mapper.writeValueAsString(attachment).getBytes()); + var filePart = createAttachmentUnitPdf(); + + return MockMvcRequestBuilders.multipart(HttpMethod.POST, "/api/lectures/" + lecture1.getId() + "/attachment-units").file(attachmentUnitPart).file(attachmentPart) + .file(filePart).contentType(MediaType.MULTIPART_FORM_DATA_VALUE); + } + } From babef53b091ee2cf14f0f5167402fc971eb2b5ef Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Fri, 19 Jul 2024 17:54:44 +0200 Subject: [PATCH 04/95] Linter --- .../connectors/pyris/PyrisWebhookService.java | 1 + src/main/webapp/app/entities/lecture.model.ts | 24 +++++++------------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java index d00fae8c7391..da5b89fef0d7 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java @@ -137,6 +137,7 @@ public String deleteLectureFromPyrisDB(List attachmentUnits) { * adds the lectures to the vector database on pyris * * @param attachmentUnit The attachmentUnit that got Updated / erased + * @return jobToken if the job was created */ public String addLectureUnitToPyrisDB(AttachmentUnit attachmentUnit) { if (lectureIngestionEnabled(attachmentUnit.getLecture().getCourse())) { diff --git a/src/main/webapp/app/entities/lecture.model.ts b/src/main/webapp/app/entities/lecture.model.ts index 989d79852bf2..dc62cff65ff3 100644 --- a/src/main/webapp/app/entities/lecture.model.ts +++ b/src/main/webapp/app/entities/lecture.model.ts @@ -25,10 +25,10 @@ export class Lecture implements BaseEntity { ingested?: IngestionState; constructor() { - this.ingested = this.checkIngestionState(); + this.ingested = this.checkIngestionState ? this.checkIngestionState() : undefined; } - private checkIngestionState(): IngestionState { + private checkIngestionState?(): IngestionState { if (!this.lectureUnits) { return IngestionState.NOT_STARTED; } @@ -38,21 +38,15 @@ export class Lecture implements BaseEntity { const allNotStarted = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.NOT_STARTED); const allFailed = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.ERROR); - if (allDone) { - return IngestionState.DONE; - } - if (allFailed) { - return IngestionState.ERROR; - } - - if (allNotStarted) { - return IngestionState.NOT_STARTED; - } + if (allDone) return IngestionState.DONE; + if (allFailed) return IngestionState.ERROR; + if (allNotStarted) return IngestionState.NOT_STARTED; return IngestionState.PARTIALLY_INGESTED; } - - public updateIngestionState(): void { - this.ingested = this.checkIngestionState(); + public updateIngestionState?(): void { + if (this.checkIngestionState) { + this.ingested = this.checkIngestionState(); + } } } From 69000696e7bf3dd524fd5ad0467f1aaa2d5d65e4 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Fri, 19 Jul 2024 18:08:37 +0200 Subject: [PATCH 05/95] Linter --- src/main/webapp/app/lecture/lecture.component.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/app/lecture/lecture.component.ts b/src/main/webapp/app/lecture/lecture.component.ts index cd35af947686..c8255267e139 100644 --- a/src/main/webapp/app/lecture/lecture.component.ts +++ b/src/main/webapp/app/lecture/lecture.component.ts @@ -160,7 +160,9 @@ export class LectureComponent implements OnInit, OnDestroy { this.lectures = res.map((lectureData) => { const lecture = new Lecture(); Object.assign(lecture, lectureData); - lecture.updateIngestionState(); + if (lecture.updateIngestionState) { + lecture.updateIngestionState(); + } return lecture; }); this.applyFilters(); From 53c3e6d063e8bbd62f34aca360f4ec3a7bc6182f Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Sat, 20 Jul 2024 02:07:11 +0200 Subject: [PATCH 06/95] Update tests to the new Lecture initialization method from findAllByCourseId to findAllByCourseIdWithSlides --- .../spec/component/lecture/lecture.component.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/javascript/spec/component/lecture/lecture.component.spec.ts b/src/test/javascript/spec/component/lecture/lecture.component.spec.ts index 62d315dd67e7..e29c9b109488 100644 --- a/src/test/javascript/spec/component/lecture/lecture.component.spec.ts +++ b/src/test/javascript/spec/component/lecture/lecture.component.spec.ts @@ -134,7 +134,7 @@ describe('Lecture', () => { }, }, MockProvider(LectureService, { - findAllByCourseId: () => { + findAllByCourseIdWithSlides: () => { return of( new HttpResponse({ body: [pastLecture, pastLecture2, currentLecture, currentLecture2, currentLecture3, futureLecture, futureLecture2, unspecifiedLecture], @@ -172,7 +172,7 @@ describe('Lecture', () => { }); it('should fetch lectures when initialized', () => { - const findAllSpy = jest.spyOn(lectureService, 'findAllByCourseId'); + const findAllSpy = jest.spyOn(lectureService, 'findAllByCourseIdWithSlides'); lectureComponentFixture.detectChanges(); From 6e1a67c1f9a0647803d38f91172d2ec1a6674046 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Sun, 21 Jul 2024 00:56:34 +0200 Subject: [PATCH 07/95] fix test initialisation, and edit class for ingestion button --- .../lecture-unit-management.component.html | 2 +- .../lecture-unit-management.component.ts | 107 ++++++++---------- .../lecture-unit-management.component.spec.ts | 29 ++++- 3 files changed, 74 insertions(+), 64 deletions(-) diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html index 5b5471279b35..bb802bd2aa49 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html @@ -74,7 +74,7 @@ }
+ } @else { + @if (lecture.course?.id && showCompetencies) { + + } + }
@if (this.emitEditEvents) { @if (editButtonAvailable(lectureUnit)) { diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts index 93a4d0f93b4c..41b64c4908d2 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts @@ -93,7 +93,6 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { // TODO: the lecture (without units) is already available through the lecture.route.ts resolver, it's not really good that we load it twice this.loadData(); } - this.initializeProfileInfo(); }); // debounceTime limits the amount of put requests sent for updating the lecture unit order @@ -119,6 +118,7 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { this.lecture = lecture; if (lecture?.lectureUnits) { this.lectureUnits = lecture?.lectureUnits; + this.initializeProfileInfo(); } else { this.lectureUnits = []; } @@ -144,8 +144,11 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { this.profileInfoSubscription = this.profileService.getProfileInfo().subscribe(async (profileInfo) => { this.irisEnabled = profileInfo.activeProfiles.includes(PROFILE_IRIS); if (this.irisEnabled && this.lecture.course && this.lecture.course.id) { + console.log('if (this.irisEnabled && this.lecture.course && this.lecture.course.id) {\n'); this.irisSettingsService.getCombinedCourseSettings(this.lecture.course.id).subscribe((settings) => { this.lectureIngestionEnabled = settings?.irisLectureIngestionSettings?.enabled || false; + console.log('updated this.lectureIngestionEnabled'); + console.log(this.lectureIngestionEnabled); }); } }); From 692ffd7d5dc6613c3a19d55490acedc854203d25 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Sun, 21 Jul 2024 18:17:54 +0200 Subject: [PATCH 10/95] Add client tests --- .../web/rest/lecture/LectureUnitResource.java | 4 +- .../lecture-unit-management.component.ts | 3 -- .../lectureUnit.service.ts | 12 +++-- .../lecture-unit-management.component.spec.ts | 52 ++++++++++++++++++- .../lecture/lecture.component.spec.ts | 42 +++++++++++++++ 5 files changed, 101 insertions(+), 12 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java index 1b503f4f82fb..7fa249c8bbec 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/lecture/LectureUnitResource.java @@ -216,9 +216,9 @@ public ResponseEntity getLectureUnitById(@PathVariable @Valid Long * @return the ResponseEntity with status 200 (OK) and a message success or null if the operation failed */ @PostMapping("lectures/{lectureId}/lecture-units/{lectureUnitId}/ingest") - public ResponseEntity ingestLectureUnit(@PathVariable Long lectureUnitId) { + public ResponseEntity ingestLectureUnit(@PathVariable Long lectureUnitId) { LectureUnit lectureUnit = this.lectureUnitRepository.findByIdWithCompetenciesAndSlidesElseThrow(lectureUnitId); lectureUnitService.ingestLectureUnitInPyris(lectureUnit); - return ResponseEntity.ok().body(true); + return ResponseEntity.ok().build(); } } diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts index 41b64c4908d2..7fcd4803c7fb 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts @@ -144,11 +144,8 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { this.profileInfoSubscription = this.profileService.getProfileInfo().subscribe(async (profileInfo) => { this.irisEnabled = profileInfo.activeProfiles.includes(PROFILE_IRIS); if (this.irisEnabled && this.lecture.course && this.lecture.course.id) { - console.log('if (this.irisEnabled && this.lecture.course && this.lecture.course.id) {\n'); this.irisSettingsService.getCombinedCourseSettings(this.lecture.course.id).subscribe((settings) => { this.lectureIngestionEnabled = settings?.irisLectureIngestionSettings?.enabled || false; - console.log('updated this.lectureIngestionEnabled'); - console.log(this.lectureIngestionEnabled); }); } }); diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts index df302a87b14c..87ac1851a6f2 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lectureUnit.service.ts @@ -168,12 +168,14 @@ export class LectureUnitService { return this.httpClient.get(`${this.resourceURL}/lecture-units/${lectureUnitId}`); } /** - * Triggers the ingestion of one lecture unit + * Triggers the ingestion of one lecture unit. + * + * @param lectureUnitId - The ID of the lecture unit to be ingested. + * @param lectureId - The ID of the lecture to which the unit belongs. + * @returns An Observable with an HttpResponse 200 if the request was successful . */ - ingestLectureUnitInPyris(lectureUnitId: number, lectureId: number): Observable> { - const params = new HttpParams(); - return this.httpClient.post(`${this.resourceURL}/lectures/${lectureId}/lecture-units/${lectureUnitId}/ingest`, null, { - params: params, + ingestLectureUnitInPyris(lectureUnitId: number, lectureId: number): Observable> { + return this.httpClient.post(`${this.resourceURL}/lectures/${lectureId}/lecture-units/${lectureUnitId}/ingest`, null, { observe: 'response', }); } diff --git a/src/test/javascript/spec/component/lecture-unit/lecture-unit-management.component.spec.ts b/src/test/javascript/spec/component/lecture-unit/lecture-unit-management.component.spec.ts index c85a6728e096..f13c7a117f3f 100644 --- a/src/test/javascript/spec/component/lecture-unit/lecture-unit-management.component.spec.ts +++ b/src/test/javascript/spec/component/lecture-unit/lecture-unit-management.component.spec.ts @@ -21,7 +21,7 @@ import { Lecture } from 'app/entities/lecture.model'; import { HttpResponse } from '@angular/common/http'; import { DeleteButtonDirective } from 'app/shared/delete-dialog/delete-button.directive'; import { HasAnyAuthorityDirective } from 'app/shared/auth/has-any-authority.directive'; -import { of } from 'rxjs'; +import { of, throwError } from 'rxjs'; import { By } from '@angular/platform-browser'; import { ActionType } from 'app/shared/delete-dialog/delete-dialog.model'; import { Competency } from 'app/entities/competency.model'; @@ -38,6 +38,7 @@ import { ProfileInfo } from 'app/shared/layouts/profiles/profile-info.model'; import { IrisCourseSettings } from 'app/entities/iris/settings/iris-settings.model'; import { IrisLectureIngestionSubSettings } from 'app/entities/iris/settings/iris-sub-settings.model'; import { PROFILE_IRIS } from 'app/app.constants'; +import { Course } from 'app/entities/course.model'; @Component({ selector: 'jhi-competencies-popover', template: '' }) class CompetenciesPopoverStubComponent { @@ -147,8 +148,8 @@ describe('LectureUnitManagementComponent', () => { getProfileInfo.mockReturnValue(of(profileInfo)); const irisCourseSettings = new IrisCourseSettings(); irisCourseSettings.irisLectureIngestionSettings = new IrisLectureIngestionSubSettings(); + irisCourseSettings.irisLectureIngestionSettings.enabled = true; getCombinedCourseSettings.mockReturnValue(of(irisCourseSettings)); - lectureUnitManagementComponentFixture.detectChanges(); }); }); @@ -201,4 +202,51 @@ describe('LectureUnitManagementComponent', () => { expect(lectureUnitManagementComponent.getActionType(new VideoUnit())).toEqual(ActionType.Delete); expect(lectureUnitManagementComponent.getActionType(new OnlineUnit())).toEqual(ActionType.Delete); }); + it('should call onIngestButtonClicked when button is clicked', () => { + const ingestLectureUnitInPyris = jest.spyOn(lectureUnitService, 'ingestLectureUnitInPyris'); + const returnValue = of(new HttpResponse({ status: 200 })); + ingestLectureUnitInPyris.mockReturnValue(returnValue); + const lectureUnitId = 1; + lectureUnitManagementComponent.lecture = { id: 2 } as any; + lectureUnitManagementComponent.onIngestButtonClicked(lectureUnitId); + expect(lectureUnitService.ingestLectureUnitInPyris).toHaveBeenCalledWith(lectureUnitId, lectureUnitManagementComponent.lecture.id); + }); + + it('should handle error when ingestLectureUnitInPyris fails', () => { + const ingestLectureUnitInPyris = jest.spyOn(lectureUnitService, 'ingestLectureUnitInPyris'); + const lectureUnitId = 1; + lectureUnitManagementComponent.lecture = { id: 2 } as any; + const error = new Error('Failed to send Ingestion request'); + ingestLectureUnitInPyris.mockReturnValue(throwError(() => error)); + + jest.spyOn(console, 'error').mockImplementation(() => {}); + + lectureUnitManagementComponent.onIngestButtonClicked(lectureUnitId); + + expect(lectureUnitService.ingestLectureUnitInPyris).toHaveBeenCalledWith(lectureUnitId, lectureUnitManagementComponent.lecture.id); + expect(console.error).toHaveBeenCalledWith('Failed to send Ingestion request', error); + }); + + it('should update the icon based on ingestion state', () => { + const attachmentUnit = { pyrisIngestionState: 'NOT_STARTED' } as any; + expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.sendToIris); + + attachmentUnit.pyrisIngestionState = 'IN_PROGRESS'; + expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.loading); + + attachmentUnit.pyrisIngestionState = 'DONE'; + expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.done); + + attachmentUnit.pyrisIngestionState = 'ERROR'; + expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.resendToIris); + }); + it('should initialize profile info and check for Iris settings', () => { + const mockCourse: Course = { id: 1, title: 'Test Course' }; + lectureUnitManagementComponent.lecture = { id: 2, course: mockCourse } as Lecture; + lectureUnitManagementComponent.initializeProfileInfo(); + expect(profileService.getProfileInfo).toHaveBeenCalled(); + expect(irisSettingsService.getCombinedCourseSettings).toHaveBeenCalledWith(lectureUnitManagementComponent.lecture!.course!.id); + expect(lectureUnitManagementComponent.irisEnabled).toBeTrue(); + expect(lectureUnitManagementComponent.lectureIngestionEnabled).toBeTrue(); + }); }); diff --git a/src/test/javascript/spec/component/lecture/lecture.component.spec.ts b/src/test/javascript/spec/component/lecture/lecture.component.spec.ts index e29c9b109488..740f47091aa2 100644 --- a/src/test/javascript/spec/component/lecture/lecture.component.spec.ts +++ b/src/test/javascript/spec/component/lecture/lecture.component.spec.ts @@ -28,6 +28,8 @@ import { IrisCourseSettings } from 'app/entities/iris/settings/iris-settings.mod import { PROFILE_IRIS } from 'app/app.constants'; import { ProfileInfo } from 'app/shared/layouts/profiles/profile-info.model'; import { ProfileService } from 'app/shared/layouts/profiles/profile.service'; +import { LectureUnit } from 'app/entities/lecture-unit/lectureUnit.model'; +import { AttachmentUnit, IngestionState } from 'app/entities/lecture-unit/attachmentUnit.model'; describe('Lecture', () => { let lectureComponentFixture: ComponentFixture; @@ -300,4 +302,44 @@ describe('Lecture', () => { expect(irisSettingsService.getCombinedCourseSettings).toHaveBeenCalledWith(lectureComponent.courseId); expect(lectureComponent.lectureIngestionEnabled).toBeTrue(); }); + it('should correctly update ingestion state based on lecture units', () => { + const mockLectureUnits: LectureUnit[] = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, + { id: 3, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, + ]; + + const lecture = new Lecture(); + lecture.id = 1; + lecture.title = 'Sample Lecture'; + lecture.lectureUnits = mockLectureUnits; + lecture.course = { id: 1, title: 'Sample Course' }; + expect(lecture.ingested).toBe(IngestionState.NOT_STARTED); + + if (lecture.updateIngestionState) { + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.PARTIALLY_INGESTED); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.DONE); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.NOT_STARTED); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.ERROR); + } + }); }); From 9969fc57d25b3696288f0e542bca0d6ddd56747c Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Sun, 21 Jul 2024 18:52:46 +0200 Subject: [PATCH 11/95] Fix lint and erase unused code --- .../lecture-unit-management.component.html | 40 ++++++++++--------- .../webapp/app/shared/sort/sort.directive.ts | 14 ------- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html index 7cb4713b0776..9ae028644ff5 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html @@ -68,24 +68,28 @@
- @if(this.lectureIngestionEnabled){ -
- @if (lecture.course?.id && showCompetencies) { - - } - -
+ @if (this.lectureIngestionEnabled) { +
+ @if (lecture.course?.id && showCompetencies) { + + } + +
} @else { @if (lecture.course?.id && showCompetencies) { diff --git a/src/main/webapp/app/shared/sort/sort.directive.ts b/src/main/webapp/app/shared/sort/sort.directive.ts index 8da860084700..7633048b9b69 100644 --- a/src/main/webapp/app/shared/sort/sort.directive.ts +++ b/src/main/webapp/app/shared/sort/sort.directive.ts @@ -36,18 +36,4 @@ export class SortDirective { this.ascendingChange.emit(this.ascending); this.sortChange.emit({ predicate: this.predicate, ascending: this.ascending }); } - - getSortedData(data: any[]): any[] { - if (typeof this.predicate === 'function') { - return [...data].sort((a, b) => { - const result = (this.predicate as any)(a) < (this.predicate as any)(b) ? -1 : 1; - return this.ascending ? result : -result; - }); - } else { - return [...data].sort((a, b) => { - const result = a[this.predicate] < b[this.predicate] ? -1 : 1; - return this.ascending ? result : -result; - }); - } - } } From 70d945afa03f841dc4192e3e7053e9712442bb36 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Mon, 22 Jul 2024 11:22:53 +0200 Subject: [PATCH 12/95] Fix Css --- .../lecture-unit-management.component.html | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html index 9ae028644ff5..50ff3700eb37 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html @@ -72,6 +72,7 @@
@if (lecture.course?.id && showCompetencies) { Date: Wed, 21 Aug 2024 00:23:48 +0200 Subject: [PATCH 13/95] Add feedback Server Side --- .../domain/lecture/AttachmentUnit.java | 2 +- .../pyris/PyrisConnectorService.java | 4 +-- .../pyris/PyrisPipelineService.java | 2 +- .../pyris/PyrisStatusUpdateService.java | 28 +++++++++---------- .../connectors/pyris/PyrisWebhookService.java | 4 +-- .../status/IngestionState.java | 2 +- .../status/PyrisStageState.java | 2 +- .../PyrisLectureIngestionStatusUpdateDTO.java | 2 +- .../PyrisLectureUnitsStatusUpdate.java | 13 --------- .../pyris/dto/status/PyrisStageDTO.java | 1 + .../artemis/web/rest/LectureResource.java | 6 ++-- .../resources/config/liquibase/master.xml | 1 - .../iris/IrisChatMessageIntegrationTest.java | 8 +++--- .../iris/PyrisLectureIngestionTest.java | 28 +++++++------------ 14 files changed, 41 insertions(+), 62 deletions(-) rename src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/{dto => domain}/status/IngestionState.java (70%) rename src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/{dto => domain}/status/PyrisStageState.java (85%) delete mode 100644 src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitsStatusUpdate.java diff --git a/src/main/java/de/tum/in/www1/artemis/domain/lecture/AttachmentUnit.java b/src/main/java/de/tum/in/www1/artemis/domain/lecture/AttachmentUnit.java index 79149530fa56..13e30b05653c 100644 --- a/src/main/java/de/tum/in/www1/artemis/domain/lecture/AttachmentUnit.java +++ b/src/main/java/de/tum/in/www1/artemis/domain/lecture/AttachmentUnit.java @@ -18,7 +18,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import de.tum.in.www1.artemis.domain.Attachment; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; @Entity @DiscriminatorValue("A") diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java index 36c4956a2745..cf30e5ac35bf 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java @@ -24,7 +24,7 @@ import de.tum.in.www1.artemis.service.connectors.pyris.dto.PyrisModelDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureDeletionExecutionDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.iris.exception.IrisException; import de.tum.in.www1.artemis.service.iris.exception.IrisForbiddenException; import de.tum.in.www1.artemis.service.iris.exception.IrisInternalPyrisErrorException; @@ -119,7 +119,7 @@ public void executeLectureAddtionWebhook(String variant, PyrisWebhookLectureInge } private void setIngestionStateToError(PyrisWebhookLectureIngestionExecutionDTO executionDTO, RuntimeException e) { - log.error("Failed to send lectures to Pyris", e); + log.error("Failed to send lecture unit {} to Pyris: {}", executionDTO.pyrisLectureUnit().lectureUnitId(), e.getMessage()); if (executionDTO != null) { Optional optionalUnit = lectureUnitRepository.findById(executionDTO.pyrisLectureUnit().lectureUnitId()); optionalUnit.ifPresent(unit -> { diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisPipelineService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisPipelineService.java index 54c8d2daf5f3..c4ba7d4b50dd 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisPipelineService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisPipelineService.java @@ -33,7 +33,7 @@ import de.tum.in.www1.artemis.service.connectors.pyris.dto.data.PyrisExtendedCourseDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.data.PyrisUserDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.iris.exception.IrisException; import de.tum.in.www1.artemis.service.iris.websocket.IrisChatWebsocketService; import de.tum.in.www1.artemis.service.metrics.LearningMetricsService; diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java index a0e6c629bd12..d5c315f35860 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java @@ -5,13 +5,12 @@ import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import de.tum.in.www1.artemis.domain.lecture.AttachmentUnit; import de.tum.in.www1.artemis.repository.AttachmentUnitRepository; import de.tum.in.www1.artemis.service.connectors.pyris.dto.chat.PyrisChatStatusUpdateDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureIngestionStatusUpdateDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.connectors.pyris.job.CourseChatJob; import de.tum.in.www1.artemis.service.connectors.pyris.job.ExerciseChatJob; import de.tum.in.www1.artemis.service.connectors.pyris.job.IngestionWebhookJob; @@ -91,18 +90,19 @@ private boolean removeJobIfTerminated(List stages, String job) { * @param statusUpdate the status update */ public void handleStatusUpdate(IngestionWebhookJob job, PyrisLectureIngestionStatusUpdateDTO statusUpdate) { - if (removeJobIfTerminated(statusUpdate.stages(), job.jobId()) && statusUpdate.id().isPresent()) { - if (attachmentUnitRepository.findById(statusUpdate.id().get()).isPresent()) { - AttachmentUnit unit = attachmentUnitRepository.findById(statusUpdate.id().get()).get(); - if (statusUpdate.stages().getLast().state() == PyrisStageState.DONE) { - unit.setPyrisIngestionState(IngestionState.DONE); - attachmentUnitRepository.save(unit); - } - else if (statusUpdate.stages().getLast().state() == PyrisStageState.ERROR || statusUpdate.stages().getLast().state() == PyrisStageState.SKIPPED) { - unit.setPyrisIngestionState(IngestionState.ERROR); + if (removeJobIfTerminated(statusUpdate.stages(), job.jobId())) { + attachmentUnitRepository.findById(statusUpdate.id()).ifPresent(unit -> { + PyrisStageState lastState = statusUpdate.stages().getLast().state(); + + if (lastState == PyrisStageState.DONE) { + unit.setPyrisIngestionState(IngestionState.DONE); + } else if (lastState == PyrisStageState.ERROR || lastState == PyrisStageState.SKIPPED) { + unit.setPyrisIngestionState(IngestionState.ERROR); + } + attachmentUnitRepository.save(unit); - } - } + }); } } + } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java index da5b89fef0d7..16b9c22f1de4 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java @@ -25,7 +25,7 @@ import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureUnitWebhookDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureDeletionExecutionDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.iris.exception.IrisInternalPyrisErrorException; import de.tum.in.www1.artemis.service.iris.settings.IrisSettingsService; @@ -147,7 +147,7 @@ public String addLectureUnitToPyrisDB(AttachmentUnit attachmentUnit) { } } catch (Exception e) { - log.error(e.getMessage()); + log.error("Error occurred while sending lecture unit {} to Pyris", attachmentUnit.getId(), e); if (attachmentUnit.getAttachment().getAttachmentType() == AttachmentType.FILE && attachmentUnit.getAttachment().getLink().endsWith(".pdf")) { attachmentUnit.setPyrisIngestionState(IngestionState.ERROR); lectureUnitRepository.save(attachmentUnit); diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/IngestionState.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/domain/status/IngestionState.java similarity index 70% rename from src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/IngestionState.java rename to src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/domain/status/IngestionState.java index 9090d887a78a..beb3839b8f49 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/IngestionState.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/domain/status/IngestionState.java @@ -1,4 +1,4 @@ -package de.tum.in.www1.artemis.service.connectors.pyris.dto.status; +package de.tum.in.www1.artemis.service.connectors.pyris.domain.status; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageState.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/domain/status/PyrisStageState.java similarity index 85% rename from src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageState.java rename to src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/domain/status/PyrisStageState.java index 10648687f7f2..19d6444191ce 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageState.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/domain/status/PyrisStageState.java @@ -1,4 +1,4 @@ -package de.tum.in.www1.artemis.service.connectors.pyris.dto.status; +package de.tum.in.www1.artemis.service.connectors.pyris.domain.status; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java index f871570fd6ea..3278ed7de921 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java @@ -8,5 +8,5 @@ import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record PyrisLectureIngestionStatusUpdateDTO(String result, List stages, Optional id) { +public record PyrisLectureIngestionStatusUpdateDTO(String result, List stages, Long id) { } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitsStatusUpdate.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitsStatusUpdate.java deleted file mode 100644 index 2c7312ec6415..000000000000 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureUnitsStatusUpdate.java +++ /dev/null @@ -1,13 +0,0 @@ -package de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook; - -import com.fasterxml.jackson.annotation.JsonInclude; - -/** - * Represents a webhook data transfer object for lecture units in the Pyris system. - * This DTO is used to encapsulate the information related to updates of lecture units, - * providing necessary details such as lecture and course identifiers, names, and descriptions. - */ -@JsonInclude(JsonInclude.Include.NON_EMPTY) - -public record PyrisLectureUnitsStatusUpdate(Long lectureUnitId, Long lectureId, Long courseId) { -} diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageDTO.java index 39728eceb4a1..0544b24eda13 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageDTO.java @@ -1,6 +1,7 @@ package de.tum.in.www1.artemis.service.connectors.pyris.dto.status; import com.fasterxml.jackson.annotation.JsonInclude; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; @JsonInclude(JsonInclude.Include.NON_EMPTY) public record PyrisStageDTO(String name, int weight, PyrisStageState state, String message) { diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/LectureResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/LectureResource.java index 249a2c2b31ec..8a4cebf28405 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/LectureResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/LectureResource.java @@ -269,7 +269,7 @@ public ResponseEntity importLecture(@PathVariable long sourceLectureId, * @return the ResponseEntity with status 200 (OK) and a message success or null if the operation failed */ @PostMapping("courses/{courseId}/ingest") - public ResponseEntity ingestLectures(@PathVariable Long courseId, @RequestParam(required = false) Optional lectureId) { + public ResponseEntity ingestLectures(@PathVariable Long courseId, @RequestParam(required = false) Optional lectureId) { log.debug("REST request to ingest lectures of course : {}", courseId); Course course = courseRepository.findByIdWithLecturesAndLectureUnitsElseThrow(courseId); if (lectureId.isPresent()) { @@ -278,14 +278,14 @@ public ResponseEntity ingestLectures(@PathVariable Long courseId, @Requ Set lecturesToIngest = new HashSet<>(); lecturesToIngest.add(lectureToIngest.get()); lectureService.ingestLecturesInPyris(lecturesToIngest); - return ResponseEntity.ok().body(true); + return ResponseEntity.ok().build(); } return ResponseEntity.badRequest() .headers(HeaderUtil.createAlert(applicationName, "Could not send lecture to Iris, no lecture found with the provided id.", "idExists")).body(null); } lectureService.ingestLecturesInPyris(course.getLectures()); - return ResponseEntity.ok().body(true); + return ResponseEntity.ok().build(); } /** diff --git a/src/main/resources/config/liquibase/master.xml b/src/main/resources/config/liquibase/master.xml index 3ca9adb7e2fa..6293b3a3f7e6 100644 --- a/src/main/resources/config/liquibase/master.xml +++ b/src/main/resources/config/liquibase/master.xml @@ -18,7 +18,6 @@ - diff --git a/src/test/java/de/tum/in/www1/artemis/iris/IrisChatMessageIntegrationTest.java b/src/test/java/de/tum/in/www1/artemis/iris/IrisChatMessageIntegrationTest.java index 2726328b19bf..fb6ba1e5d6d2 100644 --- a/src/test/java/de/tum/in/www1/artemis/iris/IrisChatMessageIntegrationTest.java +++ b/src/test/java/de/tum/in/www1/artemis/iris/IrisChatMessageIntegrationTest.java @@ -1,8 +1,8 @@ package de.tum.in.www1.artemis.iris; -import static de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState.DONE; -import static de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState.IN_PROGRESS; -import static de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState.NOT_STARTED; +import static de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState.DONE; +import static de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState.IN_PROGRESS; +import static de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState.NOT_STARTED; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatNoException; import static org.awaitility.Awaitility.await; @@ -46,7 +46,7 @@ import de.tum.in.www1.artemis.repository.iris.IrisSessionRepository; import de.tum.in.www1.artemis.service.connectors.pyris.dto.chat.PyrisChatStatusUpdateDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.iris.IrisMessageService; import de.tum.in.www1.artemis.service.iris.session.IrisExerciseChatSessionService; import de.tum.in.www1.artemis.service.iris.websocket.IrisWebsocketDTO; diff --git a/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java b/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java index 88d2c1dd6c70..4271ea5e0375 100644 --- a/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java +++ b/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java @@ -49,9 +49,9 @@ import de.tum.in.www1.artemis.service.connectors.pyris.PyrisStatusUpdateService; import de.tum.in.www1.artemis.service.connectors.pyris.PyrisWebhookService; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureIngestionStatusUpdateDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.IngestionState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.user.UserUtilService; class PyrisLectureIngestionTest extends AbstractIrisIntegrationTest { @@ -231,8 +231,7 @@ void testAllStagesDoneIngestionStateDone() throws Exception { if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), - Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); Optional optionalUnit = lectureUnitRepository.findById(unit.getId()); @@ -252,8 +251,7 @@ void testAllStagesDoneRemovesDeletionIngestionJob() throws Exception { if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), - Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); @@ -270,8 +268,7 @@ void testStageNotDoneKeepsAdditionIngestionJob() throws Exception { String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); PyrisStageDTO inProgressStage = new PyrisStageDTO("inProgressStage", 1, PyrisStageState.IN_PROGRESS, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), - Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNotNull(); @@ -288,8 +285,7 @@ void testStageNotDoneKeepsDeletionIngetionJob() throws Exception { String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); PyrisStageDTO inProgressStage = new PyrisStageDTO("inProgressStage", 1, PyrisStageState.IN_PROGRESS, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), - Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNotNull(); @@ -305,8 +301,7 @@ void testErrorStageRemovesDeletionIngetionJob() throws Exception { if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), - Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); @@ -326,8 +321,7 @@ void testErrorStageRemovesAdditionIngetionJob() throws Exception { if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), - Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); @@ -347,8 +341,7 @@ void testRunIdIngestionJob() throws Exception { String newJobToken = pyrisJobService.addIngestionWebhookJob(); String chatJobToken = pyrisJobService.addCourseChatJob(123L, 123L); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), - Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + chatJobToken)))); MockHttpServletResponse response = request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + newJobToken + "/status", statusUpdate, HttpStatus.CONFLICT, headers); @@ -366,8 +359,7 @@ void testIngestionJobDone() throws Exception { String newJobToken = pyrisJobService.addIngestionWebhookJob(); String chatJobToken = pyrisJobService.addCourseChatJob(123L, 123L); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), - Optional.ofNullable(lecture1.getLectureUnits().getFirst().getId())); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + chatJobToken)))); MockHttpServletResponse response = request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + newJobToken + "/status", statusUpdate, HttpStatus.CONFLICT, headers); From f735406fc980b6c67576948ff71ff3ddb8ca3ab6 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Wed, 21 Aug 2024 12:55:16 +0200 Subject: [PATCH 14/95] Client Side feedback --- .../pyris/PyrisConnectorService.java | 2 +- .../pyris/PyrisPipelineService.java | 2 +- .../pyris/PyrisStatusUpdateService.java | 27 ++++++----- .../connectors/pyris/PyrisWebhookService.java | 2 +- .../PyrisLectureIngestionStatusUpdateDTO.java | 1 - .../pyris/dto/status/PyrisStageDTO.java | 1 + src/main/webapp/app/entities/lecture.model.ts | 34 +++++-------- .../lecture-unit-management.component.html | 4 +- .../lecture-unit-management.component.ts | 23 ++++----- .../webapp/app/lecture/lecture.component.html | 14 ++++-- .../webapp/app/lecture/lecture.component.ts | 4 +- src/main/webapp/i18n/de/lecture.json | 2 +- .../iris/IrisChatMessageIntegrationTest.java | 2 +- .../iris/PyrisLectureIngestionTest.java | 22 +++++---- .../lecture-unit-management.component.spec.ts | 10 ++-- .../lecture/lecture.component.spec.ts | 48 +++++++++---------- 16 files changed, 100 insertions(+), 98 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java index cf30e5ac35bf..d15ee61ff30e 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisConnectorService.java @@ -21,10 +21,10 @@ import de.tum.in.www1.artemis.domain.lecture.AttachmentUnit; import de.tum.in.www1.artemis.domain.lecture.LectureUnit; import de.tum.in.www1.artemis.repository.LectureUnitRepository; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.PyrisModelDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureDeletionExecutionDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.iris.exception.IrisException; import de.tum.in.www1.artemis.service.iris.exception.IrisForbiddenException; import de.tum.in.www1.artemis.service.iris.exception.IrisInternalPyrisErrorException; diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisPipelineService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisPipelineService.java index c4ba7d4b50dd..cc6a873281f4 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisPipelineService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisPipelineService.java @@ -25,6 +25,7 @@ import de.tum.in.www1.artemis.domain.participation.StudentParticipation; import de.tum.in.www1.artemis.repository.CourseRepository; import de.tum.in.www1.artemis.repository.StudentParticipationRepository; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.PyrisPipelineExecutionSettingsDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.chat.PyrisChatPipelineExecutionBaseDataDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.chat.course.PyrisCourseChatPipelineExecutionDTO; @@ -33,7 +34,6 @@ import de.tum.in.www1.artemis.service.connectors.pyris.dto.data.PyrisExtendedCourseDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.data.PyrisUserDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.iris.exception.IrisException; import de.tum.in.www1.artemis.service.iris.websocket.IrisChatWebsocketService; import de.tum.in.www1.artemis.service.metrics.LearningMetricsService; diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java index d5c315f35860..f92e83ec7473 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisStatusUpdateService.java @@ -6,11 +6,11 @@ import org.springframework.stereotype.Service; import de.tum.in.www1.artemis.repository.AttachmentUnitRepository; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.chat.PyrisChatStatusUpdateDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureIngestionStatusUpdateDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.connectors.pyris.job.CourseChatJob; import de.tum.in.www1.artemis.service.connectors.pyris.job.ExerciseChatJob; import de.tum.in.www1.artemis.service.connectors.pyris.job.IngestionWebhookJob; @@ -91,17 +91,18 @@ private boolean removeJobIfTerminated(List stages, String job) { */ public void handleStatusUpdate(IngestionWebhookJob job, PyrisLectureIngestionStatusUpdateDTO statusUpdate) { if (removeJobIfTerminated(statusUpdate.stages(), job.jobId())) { - attachmentUnitRepository.findById(statusUpdate.id()).ifPresent(unit -> { - PyrisStageState lastState = statusUpdate.stages().getLast().state(); - - if (lastState == PyrisStageState.DONE) { - unit.setPyrisIngestionState(IngestionState.DONE); - } else if (lastState == PyrisStageState.ERROR || lastState == PyrisStageState.SKIPPED) { - unit.setPyrisIngestionState(IngestionState.ERROR); - } - - attachmentUnitRepository.save(unit); - }); + attachmentUnitRepository.findById(statusUpdate.id()).ifPresent(unit -> { + PyrisStageState lastState = statusUpdate.stages().getLast().state(); + + if (lastState == PyrisStageState.DONE) { + unit.setPyrisIngestionState(IngestionState.DONE); + } + else if (lastState == PyrisStageState.ERROR || lastState == PyrisStageState.SKIPPED) { + unit.setPyrisIngestionState(IngestionState.ERROR); + } + + attachmentUnitRepository.save(unit); + }); } } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java index 16b9c22f1de4..cd500620788e 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/PyrisWebhookService.java @@ -21,11 +21,11 @@ import de.tum.in.www1.artemis.repository.LectureUnitRepository; import de.tum.in.www1.artemis.repository.iris.IrisSettingsRepository; import de.tum.in.www1.artemis.service.FilePathService; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.PyrisPipelineExecutionSettingsDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureUnitWebhookDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureDeletionExecutionDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; import de.tum.in.www1.artemis.service.iris.exception.IrisInternalPyrisErrorException; import de.tum.in.www1.artemis.service.iris.settings.IrisSettingsService; diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java index 3278ed7de921..5a8c16476196 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/lectureingestionwebhook/PyrisLectureIngestionStatusUpdateDTO.java @@ -1,7 +1,6 @@ package de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook; import java.util.List; -import java.util.Optional; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageDTO.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageDTO.java index 0544b24eda13..8f04a0a2156b 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageDTO.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/dto/status/PyrisStageDTO.java @@ -1,6 +1,7 @@ package de.tum.in.www1.artemis.service.connectors.pyris.dto.status; import com.fasterxml.jackson.annotation.JsonInclude; + import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; @JsonInclude(JsonInclude.Include.NON_EMPTY) diff --git a/src/main/webapp/app/entities/lecture.model.ts b/src/main/webapp/app/entities/lecture.model.ts index dc62cff65ff3..3c0a4ac7651d 100644 --- a/src/main/webapp/app/entities/lecture.model.ts +++ b/src/main/webapp/app/entities/lecture.model.ts @@ -22,31 +22,23 @@ export class Lecture implements BaseEntity { channelName?: string; isAtLeastEditor?: boolean; isAtLeastInstructor?: boolean; - ingested?: IngestionState; + ingested: IngestionState; constructor() { - this.ingested = this.checkIngestionState ? this.checkIngestionState() : undefined; + this.updateIngestionState(); } - private checkIngestionState?(): IngestionState { - if (!this.lectureUnits) { - return IngestionState.NOT_STARTED; - } - - const attachmentUnits = this.lectureUnits.filter((unit) => unit.type === 'attachment') as AttachmentUnit[]; - const allDone = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.DONE); - const allNotStarted = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.NOT_STARTED); - const allFailed = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.ERROR); - - if (allDone) return IngestionState.DONE; - if (allFailed) return IngestionState.ERROR; - if (allNotStarted) return IngestionState.NOT_STARTED; - - return IngestionState.PARTIALLY_INGESTED; - } - public updateIngestionState?(): void { - if (this.checkIngestionState) { - this.ingested = this.checkIngestionState(); + public updateIngestionState(): void { + this.ingested = IngestionState.NOT_STARTED; + if (this.lectureUnits) { + const attachmentUnits = this.lectureUnits.filter((unit) => unit.type === 'attachment') as AttachmentUnit[]; + const allDone = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.DONE); + const allNotStarted = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.NOT_STARTED); + const allFailed = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.ERROR); + this.ingested = IngestionState.PARTIALLY_INGESTED; + if (allDone) this.ingested = IngestionState.DONE; + if (allFailed) this.ingested = IngestionState.ERROR; + if (allNotStarted) this.ingested = IngestionState.NOT_STARTED; } } } diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html index 50ff3700eb37..5d31480a08c1 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.html @@ -82,8 +82,8 @@ type="button" class="btn btn-primary btn-sm flex-grow-1" [ngClass]="{ - error: getIcon(lectureUnit) === resendToIris, - 'btn-success': getIcon(lectureUnit) === done, + error: getIcon(lectureUnit) === faRepeat, + 'btn-success': getIcon(lectureUnit) === faCheckCircle, }" (click)="onIngestButtonClicked(lectureUnit.id!)" [ngbTooltip]="'entity.action.sendToIris' | artemisTranslate" diff --git a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts index d740a15e859d..1b33b788b053 100644 --- a/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts +++ b/src/main/webapp/app/lecture/lecture-unit/lecture-unit-management/lecture-unit-management.component.ts @@ -56,12 +56,12 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { }; // Icons - faTrash = faTrash; - faPencilAlt = faPencilAlt; - sendToIris = faFileExport; - resendToIris = faRepeat; - done = faCheckCircle; - loading = faSpinner; + readonly faTrash = faTrash; + readonly faPencilAlt = faPencilAlt; + readonly faFileExport = faFileExport; + readonly faRepeat = faRepeat; + readonly faCheckCircle = faCheckCircle; + readonly faSpinner = faSpinner; constructor( private activatedRoute: ActivatedRoute, @@ -247,6 +247,7 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { return undefined; } } + onIngestButtonClicked(lectureUnitId: number) { this.lectureUnitService.ingestLectureUnitInPyris(lectureUnitId, this.lecture.id!).subscribe({ error: (error) => console.error('Failed to send Ingestion request', error), @@ -256,15 +257,15 @@ export class LectureUnitManagementComponent implements OnInit, OnDestroy { getIcon(attachmentUnit: AttachmentUnit): IconDefinition { switch (attachmentUnit.pyrisIngestionState) { case IngestionState.NOT_STARTED: - return this.sendToIris; + return this.faFileExport; case IngestionState.IN_PROGRESS: - return this.loading; + return this.faSpinner; case IngestionState.DONE: - return this.done; + return this.faCheckCircle; case IngestionState.ERROR: - return this.resendToIris; + return this.faRepeat; default: - return this.sendToIris; + return this.faFileExport; } } } diff --git a/src/main/webapp/app/lecture/lecture.component.html b/src/main/webapp/app/lecture/lecture.component.html index 6d69f5dbddab..14792de82dab 100644 --- a/src/main/webapp/app/lecture/lecture.component.html +++ b/src/main/webapp/app/lecture/lecture.component.html @@ -118,10 +118,12 @@

- - - - + @if (lectureIngestionEnabled) { + + + + + } @@ -138,7 +140,9 @@

{{ lecture.visibleDate | artemisDate }} {{ lecture.startDate | artemisDate }} {{ lecture.endDate | artemisDate }} - {{ lecture.ingested?.toString() }} + @if (lectureIngestionEnabled) { + {{ lecture.ingested }} + }
diff --git a/src/main/webapp/app/lecture/lecture.component.ts b/src/main/webapp/app/lecture/lecture.component.ts index c8255267e139..cd35af947686 100644 --- a/src/main/webapp/app/lecture/lecture.component.ts +++ b/src/main/webapp/app/lecture/lecture.component.ts @@ -160,9 +160,7 @@ export class LectureComponent implements OnInit, OnDestroy { this.lectures = res.map((lectureData) => { const lecture = new Lecture(); Object.assign(lecture, lectureData); - if (lecture.updateIngestionState) { - lecture.updateIngestionState(); - } + lecture.updateIngestionState(); return lecture; }); this.applyFilters(); diff --git a/src/main/webapp/i18n/de/lecture.json b/src/main/webapp/i18n/de/lecture.json index 9cf0c411de72..3b70ee0c8ab9 100644 --- a/src/main/webapp/i18n/de/lecture.json +++ b/src/main/webapp/i18n/de/lecture.json @@ -31,7 +31,7 @@ "endDate": "Ende", "visibleDate": "Sichtbar ab", "course": "Kurs", - "ingestionState": "Vorlesungsimport in Pyris", + "ingestionState": "Vorlesungsimport Zustand in Pyris", "detail": { "title": "Vorlesung", "sections": { diff --git a/src/test/java/de/tum/in/www1/artemis/iris/IrisChatMessageIntegrationTest.java b/src/test/java/de/tum/in/www1/artemis/iris/IrisChatMessageIntegrationTest.java index 5739dce75386..1b329e3d6b07 100644 --- a/src/test/java/de/tum/in/www1/artemis/iris/IrisChatMessageIntegrationTest.java +++ b/src/test/java/de/tum/in/www1/artemis/iris/IrisChatMessageIntegrationTest.java @@ -44,9 +44,9 @@ import de.tum.in.www1.artemis.participation.ParticipationUtilService; import de.tum.in.www1.artemis.repository.iris.IrisMessageRepository; import de.tum.in.www1.artemis.repository.iris.IrisSessionRepository; +import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.connectors.pyris.dto.chat.PyrisChatStatusUpdateDTO; import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; -import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; import de.tum.in.www1.artemis.service.iris.IrisMessageService; import de.tum.in.www1.artemis.service.iris.session.IrisExerciseChatSessionService; import de.tum.in.www1.artemis.service.iris.websocket.IrisWebsocketDTO; diff --git a/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java b/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java index 4271ea5e0375..99e9eb15dfef 100644 --- a/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java +++ b/src/test/java/de/tum/in/www1/artemis/iris/PyrisLectureIngestionTest.java @@ -48,10 +48,10 @@ import de.tum.in.www1.artemis.service.connectors.pyris.PyrisJobService; import de.tum.in.www1.artemis.service.connectors.pyris.PyrisStatusUpdateService; import de.tum.in.www1.artemis.service.connectors.pyris.PyrisWebhookService; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureIngestionStatusUpdateDTO; import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.IngestionState; -import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; import de.tum.in.www1.artemis.service.connectors.pyris.domain.status.PyrisStageState; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureIngestionStatusUpdateDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.status.PyrisStageDTO; import de.tum.in.www1.artemis.user.UserUtilService; class PyrisLectureIngestionTest extends AbstractIrisIntegrationTest { @@ -231,7 +231,8 @@ void testAllStagesDoneIngestionStateDone() throws Exception { if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), lecture1.getLectureUnits().getFirst().getId()); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), + lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); Optional optionalUnit = lectureUnitRepository.findById(unit.getId()); @@ -251,7 +252,8 @@ void testAllStagesDoneRemovesDeletionIngestionJob() throws Exception { if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), lecture1.getLectureUnits().getFirst().getId()); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage), + lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); @@ -268,7 +270,8 @@ void testStageNotDoneKeepsAdditionIngestionJob() throws Exception { String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); PyrisStageDTO inProgressStage = new PyrisStageDTO("inProgressStage", 1, PyrisStageState.IN_PROGRESS, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), lecture1.getLectureUnits().getFirst().getId()); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), + lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNotNull(); @@ -285,7 +288,8 @@ void testStageNotDoneKeepsDeletionIngetionJob() throws Exception { String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO doneStage = new PyrisStageDTO("done", 1, PyrisStageState.DONE, "Stage completed successfully."); PyrisStageDTO inProgressStage = new PyrisStageDTO("inProgressStage", 1, PyrisStageState.IN_PROGRESS, "Stage completed successfully."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), lecture1.getLectureUnits().getFirst().getId()); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(doneStage, inProgressStage), + lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNotNull(); @@ -301,7 +305,8 @@ void testErrorStageRemovesDeletionIngetionJob() throws Exception { if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { String jobToken = pyrisWebhookService.deleteLectureFromPyrisDB(List.of(unit)); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), lecture1.getLectureUnits().getFirst().getId()); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), + lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); @@ -321,7 +326,8 @@ void testErrorStageRemovesAdditionIngetionJob() throws Exception { if (lecture1.getLectureUnits().getFirst() instanceof AttachmentUnit unit) { String jobToken = pyrisWebhookService.addLectureUnitToPyrisDB(unit); PyrisStageDTO errorStage = new PyrisStageDTO("error", 1, PyrisStageState.ERROR, "Stage not broke due to error."); - PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), lecture1.getLectureUnits().getFirst().getId()); + PyrisLectureIngestionStatusUpdateDTO statusUpdate = new PyrisLectureIngestionStatusUpdateDTO("Success", List.of(errorStage), + lecture1.getLectureUnits().getFirst().getId()); var headers = new HttpHeaders(new LinkedMultiValueMap<>(Map.of("Authorization", List.of("Bearer " + jobToken)))); request.postWithoutResponseBody("/api/public/pyris/webhooks/ingestion/runs/" + jobToken + "/status", statusUpdate, HttpStatus.OK, headers); assertThat(pyrisJobService.getJob(jobToken)).isNull(); diff --git a/src/test/javascript/spec/component/lecture-unit/lecture-unit-management.component.spec.ts b/src/test/javascript/spec/component/lecture-unit/lecture-unit-management.component.spec.ts index f13c7a117f3f..7430ca838828 100644 --- a/src/test/javascript/spec/component/lecture-unit/lecture-unit-management.component.spec.ts +++ b/src/test/javascript/spec/component/lecture-unit/lecture-unit-management.component.spec.ts @@ -202,6 +202,7 @@ describe('LectureUnitManagementComponent', () => { expect(lectureUnitManagementComponent.getActionType(new VideoUnit())).toEqual(ActionType.Delete); expect(lectureUnitManagementComponent.getActionType(new OnlineUnit())).toEqual(ActionType.Delete); }); + it('should call onIngestButtonClicked when button is clicked', () => { const ingestLectureUnitInPyris = jest.spyOn(lectureUnitService, 'ingestLectureUnitInPyris'); const returnValue = of(new HttpResponse({ status: 200 })); @@ -229,17 +230,18 @@ describe('LectureUnitManagementComponent', () => { it('should update the icon based on ingestion state', () => { const attachmentUnit = { pyrisIngestionState: 'NOT_STARTED' } as any; - expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.sendToIris); + expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.faFileExport); attachmentUnit.pyrisIngestionState = 'IN_PROGRESS'; - expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.loading); + expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.faSpinner); attachmentUnit.pyrisIngestionState = 'DONE'; - expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.done); + expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.faCheckCircle); attachmentUnit.pyrisIngestionState = 'ERROR'; - expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.resendToIris); + expect(lectureUnitManagementComponent.getIcon(attachmentUnit)).toEqual(lectureUnitManagementComponent.faRepeat); }); + it('should initialize profile info and check for Iris settings', () => { const mockCourse: Course = { id: 1, title: 'Test Course' }; lectureUnitManagementComponent.lecture = { id: 2, course: mockCourse } as Lecture; diff --git a/src/test/javascript/spec/component/lecture/lecture.component.spec.ts b/src/test/javascript/spec/component/lecture/lecture.component.spec.ts index 740f47091aa2..eb2d780812be 100644 --- a/src/test/javascript/spec/component/lecture/lecture.component.spec.ts +++ b/src/test/javascript/spec/component/lecture/lecture.component.spec.ts @@ -316,30 +316,28 @@ describe('Lecture', () => { lecture.course = { id: 1, title: 'Sample Course' }; expect(lecture.ingested).toBe(IngestionState.NOT_STARTED); - if (lecture.updateIngestionState) { - lecture.updateIngestionState(); - expect(lecture.ingested).toBe(IngestionState.PARTIALLY_INGESTED); - - lecture.lectureUnits = [ - { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, - { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, - ]; - lecture.updateIngestionState(); - expect(lecture.ingested).toBe(IngestionState.DONE); - - lecture.lectureUnits = [ - { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, - { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, - ]; - lecture.updateIngestionState(); - expect(lecture.ingested).toBe(IngestionState.NOT_STARTED); - - lecture.lectureUnits = [ - { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, - { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, - ]; - lecture.updateIngestionState(); - expect(lecture.ingested).toBe(IngestionState.ERROR); - } + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.PARTIALLY_INGESTED); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.DONE); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.NOT_STARTED); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.ERROR); }); }); From 7b8f6a745c30caa43814856495134b50c789106d Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Wed, 21 Aug 2024 13:38:51 +0200 Subject: [PATCH 15/95] Fix optional updateIngestionState --- src/main/webapp/app/entities/lecture.model.ts | 34 ++++++++----- .../webapp/app/lecture/lecture.component.ts | 4 +- .../lecture/lecture.component.spec.ts | 48 ++++++++++--------- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/main/webapp/app/entities/lecture.model.ts b/src/main/webapp/app/entities/lecture.model.ts index 3c0a4ac7651d..e3ea750bb591 100644 --- a/src/main/webapp/app/entities/lecture.model.ts +++ b/src/main/webapp/app/entities/lecture.model.ts @@ -22,23 +22,31 @@ export class Lecture implements BaseEntity { channelName?: string; isAtLeastEditor?: boolean; isAtLeastInstructor?: boolean; - ingested: IngestionState; + ingested?: IngestionState; constructor() { - this.updateIngestionState(); + this.ingested = this.checkIngestionState(); } - public updateIngestionState(): void { - this.ingested = IngestionState.NOT_STARTED; - if (this.lectureUnits) { - const attachmentUnits = this.lectureUnits.filter((unit) => unit.type === 'attachment') as AttachmentUnit[]; - const allDone = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.DONE); - const allNotStarted = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.NOT_STARTED); - const allFailed = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.ERROR); - this.ingested = IngestionState.PARTIALLY_INGESTED; - if (allDone) this.ingested = IngestionState.DONE; - if (allFailed) this.ingested = IngestionState.ERROR; - if (allNotStarted) this.ingested = IngestionState.NOT_STARTED; + private checkIngestionState(): IngestionState { + if (!this.lectureUnits) { + return IngestionState.NOT_STARTED; + } + + const attachmentUnits = this.lectureUnits.filter((unit) => unit.type === 'attachment') as AttachmentUnit[]; + const allDone = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.DONE); + const allNotStarted = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.NOT_STARTED); + const allFailed = attachmentUnits.every((unit) => unit.pyrisIngestionState === IngestionState.ERROR); + + if (allDone) return IngestionState.DONE; + if (allFailed) return IngestionState.ERROR; + if (allNotStarted) return IngestionState.NOT_STARTED; + + return IngestionState.PARTIALLY_INGESTED; + } + public updateIngestionState?(): void { + if (this.checkIngestionState) { + this.ingested = this.checkIngestionState(); } } } diff --git a/src/main/webapp/app/lecture/lecture.component.ts b/src/main/webapp/app/lecture/lecture.component.ts index cd35af947686..c8255267e139 100644 --- a/src/main/webapp/app/lecture/lecture.component.ts +++ b/src/main/webapp/app/lecture/lecture.component.ts @@ -160,7 +160,9 @@ export class LectureComponent implements OnInit, OnDestroy { this.lectures = res.map((lectureData) => { const lecture = new Lecture(); Object.assign(lecture, lectureData); - lecture.updateIngestionState(); + if (lecture.updateIngestionState) { + lecture.updateIngestionState(); + } return lecture; }); this.applyFilters(); diff --git a/src/test/javascript/spec/component/lecture/lecture.component.spec.ts b/src/test/javascript/spec/component/lecture/lecture.component.spec.ts index eb2d780812be..740f47091aa2 100644 --- a/src/test/javascript/spec/component/lecture/lecture.component.spec.ts +++ b/src/test/javascript/spec/component/lecture/lecture.component.spec.ts @@ -316,28 +316,30 @@ describe('Lecture', () => { lecture.course = { id: 1, title: 'Sample Course' }; expect(lecture.ingested).toBe(IngestionState.NOT_STARTED); - lecture.updateIngestionState(); - expect(lecture.ingested).toBe(IngestionState.PARTIALLY_INGESTED); - - lecture.lectureUnits = [ - { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, - { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, - ]; - lecture.updateIngestionState(); - expect(lecture.ingested).toBe(IngestionState.DONE); - - lecture.lectureUnits = [ - { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, - { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, - ]; - lecture.updateIngestionState(); - expect(lecture.ingested).toBe(IngestionState.NOT_STARTED); - - lecture.lectureUnits = [ - { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, - { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, - ]; - lecture.updateIngestionState(); - expect(lecture.ingested).toBe(IngestionState.ERROR); + if (lecture.updateIngestionState) { + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.PARTIALLY_INGESTED); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.DONE } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.DONE); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.NOT_STARTED } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.NOT_STARTED); + + lecture.lectureUnits = [ + { id: 1, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, + { id: 2, type: 'attachment', pyrisIngestionState: IngestionState.ERROR } as AttachmentUnit, + ]; + lecture.updateIngestionState(); + expect(lecture.ingested).toBe(IngestionState.ERROR); + } }); }); From c76a4bcd4bd1404f15c8d3f77ec34db0612ea094 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Wed, 21 Aug 2024 13:43:04 +0200 Subject: [PATCH 16/95] Fix optional updateIngestionState --- src/main/webapp/app/entities/lecture.model.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/app/entities/lecture.model.ts b/src/main/webapp/app/entities/lecture.model.ts index e3ea750bb591..7c2e2d40a17a 100644 --- a/src/main/webapp/app/entities/lecture.model.ts +++ b/src/main/webapp/app/entities/lecture.model.ts @@ -25,10 +25,10 @@ export class Lecture implements BaseEntity { ingested?: IngestionState; constructor() { - this.ingested = this.checkIngestionState(); + this.ingested = this.checkIngestionState?.(); } - private checkIngestionState(): IngestionState { + private checkIngestionState?(): IngestionState { if (!this.lectureUnits) { return IngestionState.NOT_STARTED; } @@ -46,7 +46,7 @@ export class Lecture implements BaseEntity { } public updateIngestionState?(): void { if (this.checkIngestionState) { - this.ingested = this.checkIngestionState(); + this.ingested = this.checkIngestionState?.(); } } } From 6d0bebadda3b826d94ee15d5f52dc92ee711897f Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Wed, 21 Aug 2024 14:31:35 +0200 Subject: [PATCH 17/95] Fix server test --- .../in/www1/artemis/iris/PyrisConnectorServiceTest.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/java/de/tum/in/www1/artemis/iris/PyrisConnectorServiceTest.java b/src/test/java/de/tum/in/www1/artemis/iris/PyrisConnectorServiceTest.java index 42ad834cf2a9..a6b4e0be62b8 100644 --- a/src/test/java/de/tum/in/www1/artemis/iris/PyrisConnectorServiceTest.java +++ b/src/test/java/de/tum/in/www1/artemis/iris/PyrisConnectorServiceTest.java @@ -3,6 +3,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.List; import java.util.stream.Stream; import org.junit.jupiter.api.Test; @@ -13,6 +14,8 @@ import de.tum.in.www1.artemis.service.connectors.pyris.PyrisConnectorException; import de.tum.in.www1.artemis.service.connectors.pyris.PyrisConnectorService; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisLectureUnitWebhookDTO; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.lectureingestionwebhook.PyrisWebhookLectureIngestionExecutionDTO; import de.tum.in.www1.artemis.service.iris.exception.IrisForbiddenException; import de.tum.in.www1.artemis.service.iris.exception.IrisInternalPyrisErrorException; @@ -46,7 +49,10 @@ void testExceptionV2(int httpStatus, Class exceptionClass) { @MethodSource("irisExceptions") void testExceptionIngestionV2(int httpStatus, Class exceptionClass) { irisRequestMockProvider.mockIngestionWebhookRunError(httpStatus); - assertThatThrownBy(() -> pyrisConnectorService.executeLectureAddtionWebhook("fullIngestion", null)).isInstanceOf(exceptionClass); + PyrisLectureUnitWebhookDTO pyrisLectureUnitWebhookDTO = new PyrisLectureUnitWebhookDTO("example.pdf", 123L, "Lecture Unit Name", 456L, "Lecture Name", 789L, "Course Name", + "Course Description"); + PyrisWebhookLectureIngestionExecutionDTO executionDTO = new PyrisWebhookLectureIngestionExecutionDTO(pyrisLectureUnitWebhookDTO, null, List.of()); + assertThatThrownBy(() -> pyrisConnectorService.executeLectureAddtionWebhook("fullIngestion", executionDTO)).isInstanceOf(exceptionClass); } @ParameterizedTest From cf1e5c8b0c28bd2220a6f0da29b2e095f979c7f0 Mon Sep 17 00:00:00 2001 From: Yassine Souissi Date: Sun, 1 Sep 2024 20:00:39 +0200 Subject: [PATCH 18/95] merge2 --- CONTRIBUTING.md | 41 +++- docs/dev/guidelines/database.rst | 8 +- .../artemis/repository/CourseRepository.java | 2 - .../ci/ContinuousIntegrationService.java | 8 +- .../pyris/job/CompetencyExtractionJob.java | 22 ++ .../competency/CourseCompetencyResource.java | 31 +-- .../templates/aeolus/rust/default.sh | 26 ++ .../templates/aeolus/rust/default.yaml | 14 ++ .../templates/rust/exercise/src/context.rs | 1 + .../templates/rust/solution/src/context.rs | 36 +++ .../complaints-student-view.component.html | 10 +- .../complaints-for-tutor.component.html | 44 +--- .../form/complaints-form.component.html | 26 +- .../request/complaint-request.component.html | 15 +- ...-export-confirmation-dialog.component.html | 2 +- .../data-export/data-export.component.html | 6 +- .../competency-relation-graph.component.html | 10 +- .../competencies/course-competency.service.ts | 11 +- ...tency-recommendation-detail.component.html | 10 +- .../course-description-form.component.html | 12 +- .../course-description-form.component.ts | 7 +- .../import/competency-search.component.html | 24 +- .../course-scores.component.html | 12 +- .../exam-assessment-buttons.component.html | 2 +- .../course-lti-configuration.component.html | 34 +-- ...it-course-lti-configuration.component.html | 2 +- ...management-exercises-search.component.html | 28 +-- ...course-management-exercises.component.html | 4 +- .../manage/course-management.component.html | 8 +- .../course/manage/course-management.module.ts | 2 + .../manage/course-update.component.html | 38 ++- .../course-detail-line-chart.component.html | 21 +- .../course-management-card.component.html | 40 ++-- ...rse-management-exercise-row.component.html | 12 +- ...agement-overview-statistics.component.html | 2 +- ...-tutorial-group-free-period.component.html | 2 +- ...-tutorial-group-free-period.component.html | 2 +- ...eate-tutorial-group-session.component.html | 2 +- ...edit-tutorial-group-session.component.html | 2 +- ...torial-groups-configuration.component.html | 6 +- ...torial-groups-configuration.component.html | 2 +- .../create-tutorial-group.component.html | 2 +- .../edit-tutorial-group.component.html | 2 +- ...scores-average-scores-graph.component.html | 2 +- .../exam-scores/exam-scores.component.html | 145 +++++------- .../manage/exam-management.component.html | 30 +-- .../exam/manage/exam-status.component.html | 46 ++-- ...e-announcement-create-modal.component.html | 7 +- ...ive-announcement-create-modal.component.ts | 23 +- .../exam-checklist.component.html | 172 +++++++------- .../exam-edit-working-time.component.html | 2 +- .../manage/exams/exam-detail.component.html | 22 +- .../exam-import/exam-import.component.html | 8 +- .../exam-mode-picker.component.html | 17 +- .../manage/exams/exam-update.component.html | 22 +- .../students/exam-students.component.html | 2 +- ...m-students-attendance-check.component.html | 18 +- .../exam-participation-cover.component.html | 4 +- .../exam-navigation-bar.component.html | 25 +- .../exam-participation.component.html | 8 +- .../exam-general-information.component.html | 8 +- .../exam-result-summary.component.html | 8 +- .../events/exam-live-event.component.html | 17 +- ...drag-and-drop-question-edit.component.html | 27 +-- .../drag-and-drop-question-edit.component.ts | 36 ++- .../drag-and-drop-question.component.html | 20 +- .../difficulty-picker.component.html | 24 +- .../detailed-grading-system.component.html | 24 +- .../create-exercise-unit.component.html | 27 ++- ...course-competencies-details.component.html | 6 +- .../course-competencies.component.html | 10 +- ...nversations-code-of-conduct.component.html | 2 +- .../course-conversations.component.html | 4 +- .../course-wide-search.component.html | 8 +- ...conversation-add-users-form.component.html | 24 +- ...nversation-add-users-dialog.component.html | 4 +- .../conversation-detail-dialog.component.html | 41 +++- .../conversation-info.component.html | 32 ++- .../conversation-member-row.component.html | 27 ++- .../conversation-members.component.html | 18 +- .../conversation-settings.component.html | 27 ++- .../conversation-header.component.html | 10 +- .../conversation-messages.component.html | 17 +- .../course-dashboard.component.html | 2 +- .../course-exercise-row.component.html | 2 +- .../course-exercises.component.html | 2 +- .../course-lecture-details.component.html | 30 ++- .../course-lecture-row.component.html | 2 +- .../overview/course-overview.component.html | 4 +- ...course-prerequisites-button.component.html | 4 +- .../course-registration-button.component.html | 4 +- .../course-registration-detail.component.html | 8 +- .../course-prerequisites-modal.component.html | 2 +- .../course-registration.component.html | 8 +- .../course-statistics.component.html | 222 +++++++++--------- .../course-tutorial-group-card.component.html | 11 +- .../course-unenrollment-modal.component.html | 20 +- .../app/overview/courses.component.html | 12 +- .../discussion-section.component.html | 28 +-- .../course-exercise-details.component.html | 10 +- .../connection-warning.component.html | 4 +- .../consistency-check.component.html | 2 +- .../course-users-selector.component.html | 12 +- .../date-time-picker.component.html | 2 +- .../date-time-picker.component.ts | 13 +- ...tom-exercise-category-badge.component.html | 10 + ...tom-exercise-category-badge.component.scss | 18 ++ ...ustom-exercise-category-badge.component.ts | 23 ++ .../domainCommands/correctOptionCommand.ts | 32 --- .../domainCommands/credits.command.ts | 36 --- .../domain-multi-option-list.command.ts | 103 -------- .../domainCommands/domainCommand.ts | 62 ----- .../domainMultiOptionCommand.ts | 15 -- .../domainCommands/domainTag.command.ts | 6 - .../DataExportCreationServiceTest.java | 19 +- .../edit-competency.component.spec.ts | 8 +- .../edit-prerequisite.component.spec.ts | 8 +- ...cy-recommendation-detail.component.spec.ts | 4 +- .../course-description-form.component.spec.ts | 10 + .../complaints-for-tutor.component.spec.ts | 11 +- .../course/course-exercises.component.spec.ts | 3 +- .../course/course-update.component.spec.ts | 2 +- ...g-and-drop-question-edit.component.spec.ts | 33 ++- .../drag-and-drop-question.component.spec.ts | 2 - ...res-average-scores-graph.component.spec.ts | 5 +- .../exam/exam-update.component.spec.ts | 6 +- ...nnouncement-create-modal.component.spec.ts | 9 +- .../exam-navigation-bar.component.spec.ts | 5 +- .../events/exam-live-event.component.spec.ts | 12 +- .../detailed-grading-system.component.spec.ts | 2 + ...versation-add-users-form.component.spec.ts | 5 +- .../course-exams.component.spec.ts | 7 +- .../course-statistics.component.spec.ts | 18 +- .../course-users-selector.component.spec.ts | 119 +++++----- .../conversation-options.component.spec.ts | 5 +- ...urse-tutorial-group-card.component.spec.ts | 17 +- .../course-tutorial-groups.component.spec.ts | 5 +- .../course/CourseCommunicationPage.ts | 4 +- .../endpointanalysis/EndpointAnalyzer.java | 39 +-- .../endpointanalysis/EndpointInformation.java | 8 +- .../cit/endpointanalysis/EndpointParser.java | 10 +- 141 files changed, 1265 insertions(+), 1385 deletions(-) create mode 100644 src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/job/CompetencyExtractionJob.java create mode 100644 src/main/resources/templates/aeolus/rust/default.sh create mode 100644 src/main/resources/templates/aeolus/rust/default.yaml create mode 100644 src/main/resources/templates/rust/exercise/src/context.rs create mode 100644 src/main/resources/templates/rust/solution/src/context.rs create mode 100644 src/main/webapp/app/shared/exercise-categories/custom-exercise-category-badge/custom-exercise-category-badge.component.html create mode 100644 src/main/webapp/app/shared/exercise-categories/custom-exercise-category-badge/custom-exercise-category-badge.component.scss create mode 100644 src/main/webapp/app/shared/exercise-categories/custom-exercise-category-badge/custom-exercise-category-badge.component.ts delete mode 100644 src/main/webapp/app/shared/markdown-editor/domainCommands/correctOptionCommand.ts delete mode 100644 src/main/webapp/app/shared/markdown-editor/domainCommands/credits.command.ts delete mode 100644 src/main/webapp/app/shared/markdown-editor/domainCommands/domain-multi-option-list.command.ts delete mode 100644 src/main/webapp/app/shared/markdown-editor/domainCommands/domainCommand.ts delete mode 100644 src/main/webapp/app/shared/markdown-editor/domainCommands/domainMultiOptionCommand.ts delete mode 100644 src/main/webapp/app/shared/markdown-editor/domainCommands/domainTag.command.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea3d5c107479..482bdb00df6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,43 @@ -# Contributing Guide for Artemis +# Contribution Guidelines for Artemis Read the [setup guide](https://docs.artemis.cit.tum.de/dev/setup.html) on how to set up your local development environment. -Before creating a pull request, please read the [guidelines to the development process](https://docs.artemis.cit.tum.de/dev/development-process/development-process.html) as well as the [coding and design guidelines](https://docs.artemis.cit.tum.de/dev/guidelines.html). +## Identity and Transparency + +To ensure a transparent and trustworthy environment, we have established different guidelines for members of our organization and external contributors. + +### For Members of Our Organization + +1. **Real Names Required**: As a member of our organization, you must use your full real name in your GitHub profile. This is a prerequisite for joining our organization. Using a real name is crucial for building trust within the team and the broader community. It fosters accountability and transparency, which are essential for collaborative work. When members use their real identities, it encourages open communication and strengthens professional relationships. Furthermore, it aligns with best practices in open-source communities, where transparency is key to ensuring the integrity and reliability of contributions. + +2. **Profile Picture**: Members are required to upload an authentic profile picture. Use a clear, professional image and avoid comic-like pictures, memojis, or other non-authentic picture styles. Using a professional and authentic profile picture is essential for establishing credibility and fostering trust within the community. It helps others easily identify and connect with you, which is crucial for effective collaboration. By using a real photo, you present yourself as a serious and committed contributor, which in turn encourages others to take your work and interactions seriously. Avoiding non-authentic images ensures that the focus remains on the substance of your contributions rather than on distractions or misunderstandings that might arise from informal or unprofessional visuals. + +3. **Direct Branching and PR Creation**: As a member, you are encourages to create branches and pull requests (PRs) directly within the repository. Please follow the internal branching and code review processes outlined in [guidelines to the development process](https://docs.artemis.cit.tum.de/dev/development-process/development-process.html) and [coding and design guidelines](https://docs.artemis.cit.tum.de/dev/guidelines.html). + +### For External Contributors + +1. **Identity Verification**: External contributions will only be considered if the contributor uses their real name and an authentic profile picture (see above). This ensures accountability and trustworthiness in all external contributions. + +2. **Forking the Repository**: External contributors fork the repository and work on changes in their own branches. + +3. **Submit a Pull Request**: Once your work is complete, submit a pull request for review. Ensure that your branch is up to date with the main branch before submitting. + +4. **Compliance**: Contributions from external contributors that do not adhere to these guidelines may not be accepted. + +### References and Best Practices + +- We align our guidelines with the [GitHub Acceptable Use Policies](https://docs.github.com/en/site-policy/acceptable-use-policies) which stress the importance of authenticity and transparency in user profiles. +- For more insights on contributing to open-source projects, we recommend reviewing the [Open Source Guides by GitHub](https://opensource.guide/). + +By following these guidelines, we foster a collaborative environment built on mutual trust and respect, essential for the success of our project. + +## Contribution Process + +1. **External contributors only**: Fork the Repository and create a branch. +2. **Create a feature branch**: Work on your changes in a separate branch. +3. **Submit a pull request**: Once your work is complete, submit a pull request for review. + +Thank you for your contributions and for helping us maintain a high standard of quality and trust in this project. + + diff --git a/docs/dev/guidelines/database.rst b/docs/dev/guidelines/database.rst index 7898ab2caa33..0c7119159a9d 100644 --- a/docs/dev/guidelines/database.rst +++ b/docs/dev/guidelines/database.rst @@ -1,8 +1,6 @@ -********************** -Database Relationships -********************** - -WORK IN PROGRESS +******** +Database +******** 1. Retrieving and Building Objects ================================== diff --git a/src/main/java/de/tum/in/www1/artemis/repository/CourseRepository.java b/src/main/java/de/tum/in/www1/artemis/repository/CourseRepository.java index 470e789cdb27..1187d8c0e11d 100644 --- a/src/main/java/de/tum/in/www1/artemis/repository/CourseRepository.java +++ b/src/main/java/de/tum/in/www1/artemis/repository/CourseRepository.java @@ -222,8 +222,6 @@ SELECT COUNT(c) > 0 List findAllByShortName(String shortName); - Optional findById(long courseId); - /** * Returns the title of the course with the given id. * diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/ci/ContinuousIntegrationService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/ci/ContinuousIntegrationService.java index f22eebc954bf..2cb70653b3d4 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/ci/ContinuousIntegrationService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/ci/ContinuousIntegrationService.java @@ -219,8 +219,8 @@ enum RepositoryCheckoutPath implements CustomizableCheckoutPath { @Override public String forProgrammingLanguage(ProgrammingLanguage language) { return switch (language) { - case JAVA, PYTHON, C, HASKELL, KOTLIN, VHDL, ASSEMBLER, SWIFT, OCAML, EMPTY -> "assignment"; - case JAVASCRIPT, C_SHARP, C_PLUS_PLUS, SQL, R, TYPESCRIPT, RUST, GO, MATLAB, BASH, RUBY, POWERSHELL, ADA, DART, PHP -> + case JAVA, PYTHON, C, HASKELL, KOTLIN, VHDL, ASSEMBLER, SWIFT, OCAML, EMPTY, RUST -> "assignment"; + case JAVASCRIPT, C_SHARP, C_PLUS_PLUS, SQL, R, TYPESCRIPT, GO, MATLAB, BASH, RUBY, POWERSHELL, ADA, DART, PHP -> throw new UnsupportedOperationException("Unsupported programming language: " + language); }; } @@ -230,9 +230,9 @@ public String forProgrammingLanguage(ProgrammingLanguage language) { @Override public String forProgrammingLanguage(ProgrammingLanguage language) { return switch (language) { - case JAVA, PYTHON, HASKELL, KOTLIN, SWIFT, EMPTY -> ""; + case JAVA, PYTHON, HASKELL, KOTLIN, SWIFT, EMPTY, RUST -> ""; case C, VHDL, ASSEMBLER, OCAML -> "tests"; - case JAVASCRIPT, C_SHARP, C_PLUS_PLUS, SQL, R, TYPESCRIPT, RUST, GO, MATLAB, BASH, RUBY, POWERSHELL, ADA, DART, PHP -> + case JAVASCRIPT, C_SHARP, C_PLUS_PLUS, SQL, R, TYPESCRIPT, GO, MATLAB, BASH, RUBY, POWERSHELL, ADA, DART, PHP -> throw new UnsupportedOperationException("Unsupported programming language: " + language); }; } diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/job/CompetencyExtractionJob.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/job/CompetencyExtractionJob.java new file mode 100644 index 000000000000..2a76f5a3b072 --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/pyris/job/CompetencyExtractionJob.java @@ -0,0 +1,22 @@ +package de.tum.in.www1.artemis.service.connectors.pyris.job; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import de.tum.in.www1.artemis.domain.Course; + +/** + * A pyris job that extracts competencies from a course description. + * + * @param jobId the job id + * @param courseId the course in which the competencies are being extracted + * @param userLogin the user login of the user who started the job + */ +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public record CompetencyExtractionJob(String jobId, long courseId, String userLogin) implements PyrisJob { + + @Override + public boolean canAccess(Course course) { + return course.getId().equals(courseId); + } + +} diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/competency/CourseCompetencyResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/competency/CourseCompetencyResource.java index a3fdf0875c31..036b673d1674 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/competency/CourseCompetencyResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/competency/CourseCompetencyResource.java @@ -46,9 +46,10 @@ import de.tum.in.www1.artemis.service.competency.CompetencyProgressService; import de.tum.in.www1.artemis.service.competency.CompetencyRelationService; import de.tum.in.www1.artemis.service.competency.CourseCompetencyService; +import de.tum.in.www1.artemis.service.connectors.pyris.dto.competency.PyrisCompetencyExtractionInputDTO; import de.tum.in.www1.artemis.service.feature.Feature; import de.tum.in.www1.artemis.service.feature.FeatureToggle; -import de.tum.in.www1.artemis.service.iris.session.IrisCompetencyGenerationSessionService; +import de.tum.in.www1.artemis.service.iris.IrisCompetencyGenerationService; import de.tum.in.www1.artemis.web.rest.dto.CourseCompetencyProgressDTO; import de.tum.in.www1.artemis.web.rest.dto.SearchResultPageDTO; import de.tum.in.www1.artemis.web.rest.dto.competency.CompetencyJolPairDTO; @@ -80,7 +81,7 @@ public class CourseCompetencyResource { private final CompetencyRelationService competencyRelationService; - private final Optional irisCompetencyGenerationSessionService; + private final Optional irisCompetencyGenerationService; private final CompetencyJolService competencyJolService; @@ -91,7 +92,7 @@ public class CourseCompetencyResource { public CourseCompetencyResource(UserRepository userRepository, CourseCompetencyService courseCompetencyService, CourseCompetencyRepository courseCompetencyRepository, CourseRepository courseRepository, CompetencyProgressService competencyProgressService, CompetencyProgressRepository competencyProgressRepository, CompetencyRelationRepository competencyRelationRepository, CompetencyRelationService competencyRelationService, - Optional irisCompetencyGenerationSessionService, CompetencyJolService competencyJolService, + Optional irisCompetencyGenerationService, CompetencyJolService competencyJolService, AuthorizationCheckService authorizationCheckService) { this.userRepository = userRepository; this.courseCompetencyService = courseCompetencyService; @@ -101,7 +102,7 @@ public CourseCompetencyResource(UserRepository userRepository, CourseCompetencyS this.competencyProgressRepository = competencyProgressRepository; this.competencyRelationRepository = competencyRelationRepository; this.competencyRelationService = competencyRelationService; - this.irisCompetencyGenerationSessionService = irisCompetencyGenerationSessionService; + this.irisCompetencyGenerationService = irisCompetencyGenerationService; this.competencyJolService = competencyJolService; this.authorizationCheckService = authorizationCheckService; } @@ -332,25 +333,25 @@ public ResponseEntity removeCompetencyRelation(@PathVariable long courseId } /** - * POST courses/:courseId/course-competencies/generate-from-description : Generates a list of course competencies from a given course description by using - * IRIS. + * POST courses/:courseId/course-competencies/:competencyId/competencies/generate-from-description + * Generates a list of competencies from a given course description with IRIS. * - * @param courseId the id of the current course - * @param courseDescription the text description of the course - * @return the ResponseEntity with status 200 (OK) and body the generated competencies + * @param courseId the id of the current course + * @param input the course description and current competencies + * @return the ResponseEntity with status 202 (Accepted) */ @PostMapping("courses/{courseId}/course-competencies/generate-from-description") @EnforceAtLeastEditorInCourse - public ResponseEntity> generateCompetenciesFromCourseDescription(@PathVariable Long courseId, @RequestBody String courseDescription) { - var irisService = irisCompetencyGenerationSessionService.orElseThrow(); + public ResponseEntity generateCompetenciesFromCourseDescription(@PathVariable Long courseId, @RequestBody PyrisCompetencyExtractionInputDTO input) { + var competencyGenerationService = irisCompetencyGenerationService.orElseThrow(); var user = userRepository.getUserWithGroupsAndAuthorities(); var course = courseRepository.findByIdElseThrow(courseId); - var session = irisService.getOrCreateSession(course, user); - irisService.addUserTextMessageToSession(session, courseDescription); - var competencies = irisService.executeRequest(session); + // Start the Iris competency generation pipeline for the given course. + // The generated competencies will be sent async over the websocket on the topic /topic/iris/competencies/{courseId} + competencyGenerationService.executeCompetencyExtractionPipeline(user, course, input.courseDescription(), input.currentCompetencies()); - return ResponseEntity.ok(competencies); + return ResponseEntity.accepted().build(); } /** diff --git a/src/main/resources/templates/aeolus/rust/default.sh b/src/main/resources/templates/aeolus/rust/default.sh new file mode 100644 index 000000000000..d1ba23293a58 --- /dev/null +++ b/src/main/resources/templates/aeolus/rust/default.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -e +export AEOLUS_INITIAL_DIRECTORY=${PWD} +build () { + echo '⚙️ executing build' + cargo build --verbose +} + +run_all_tests () { + echo '⚙️ executing run_all_tests' + cargo nextest run --profile ci +} + +main () { + if [[ "${1}" == "aeolus_sourcing" ]]; then + return 0 # just source to use the methods in the subshell, no execution + fi + local _script_name + _script_name=${BASH_SOURCE[0]:-$0} + cd "${AEOLUS_INITIAL_DIRECTORY}" + bash -c "source ${_script_name} aeolus_sourcing; build" + cd "${AEOLUS_INITIAL_DIRECTORY}" + bash -c "source ${_script_name} aeolus_sourcing; run_all_tests" +} + +main "${@}" diff --git a/src/main/resources/templates/aeolus/rust/default.yaml b/src/main/resources/templates/aeolus/rust/default.yaml new file mode 100644 index 000000000000..4cd44c110048 --- /dev/null +++ b/src/main/resources/templates/aeolus/rust/default.yaml @@ -0,0 +1,14 @@ +api: v0.0.1 +metadata: + name: Rust + id: rust + description: Test crate using cargo +actions: + - name: build + script: cargo build --verbose + - name: run_all_tests + script: cargo nextest run --profile ci + results: + - name: junit_target/nextest/ci/junit.xml + path: target/nextest/ci/junit.xml + type: junit diff --git a/src/main/resources/templates/rust/exercise/src/context.rs b/src/main/resources/templates/rust/exercise/src/context.rs new file mode 100644 index 000000000000..30940f4db7a7 --- /dev/null +++ b/src/main/resources/templates/rust/exercise/src/context.rs @@ -0,0 +1 @@ +// TODO: Create and implement a Context struct according to the UML class diagram diff --git a/src/main/resources/templates/rust/solution/src/context.rs b/src/main/resources/templates/rust/solution/src/context.rs new file mode 100644 index 000000000000..71cd0d711753 --- /dev/null +++ b/src/main/resources/templates/rust/solution/src/context.rs @@ -0,0 +1,36 @@ +use crate::sort_strategy::SortStrategy; +use chrono::NaiveDate; +use std::ops::Deref; + +pub struct Context { + sort_algorithm: Option>>, +} + +impl Context { + pub fn new() -> Context { + Context { + sort_algorithm: None, + } + } + + /// Runs the configured sorting algorithm. + pub fn sort(&self, data: &mut [NaiveDate]) { + let sort_algorithm = self + .sort_algorithm + .as_ref() + .expect("sort_algorithm has to be set before sort() is called"); + + sort_algorithm.perform_sort(data); + } + + pub fn set_sort_algorithm(&mut self, sort_algorithm: Box>) { + self.sort_algorithm = Some(sort_algorithm); + } + + pub fn sort_algorithm(&self) -> &dyn SortStrategy { + self.sort_algorithm + .as_ref() + .expect("sort_algorithm has to be set") + .deref() + } +} diff --git a/src/main/webapp/app/complaints/complaints-for-students/complaints-student-view.component.html b/src/main/webapp/app/complaints/complaints-for-students/complaints-student-view.component.html index efd1d1c458d1..c59190b2dad7 100644 --- a/src/main/webapp/app/complaints/complaints-for-students/complaints-student-view.component.html +++ b/src/main/webapp/app/complaints/complaints-for-students/complaints-student-view.component.html @@ -30,9 +30,8 @@ ? ('artemisApp.complaint.complaintNotAllowedTooltip' | artemisTranslate) : '' }}" - > - {{ 'artemisApp.complaint.moreInfo' | artemisTranslate }} - + jhiTranslate="artemisApp.complaint.moreInfo" + > } @if (!isExamMode && course?.requestMoreFeedbackEnabled) { + jhiTranslate="artemisApp.moreFeedback.button" + > }
} diff --git a/src/main/webapp/app/complaints/complaints-for-tutor/complaints-for-tutor.component.html b/src/main/webapp/app/complaints/complaints-for-tutor/complaints-for-tutor.component.html index 0997b352dbbd..97bd1ae26257 100644 --- a/src/main/webapp/app/complaints/complaints-for-tutor/complaints-for-tutor.component.html +++ b/src/main/webapp/app/complaints/complaints-for-tutor/complaints-for-tutor.component.html @@ -1,25 +1,20 @@ @if (isLoading) {
- {{ 'loading' | artemisTranslate }} +
} @if (!isLoading && complaint) { -

- {{ complaint.complaintType === ComplaintType.MORE_FEEDBACK ? ('artemisApp.moreFeedback.review' | artemisTranslate) : ('artemisApp.complaint.review' | artemisTranslate) }} -

+

} @if (!isLoading && complaint) {
@if (handled) { -
- {{ - complaint.complaintType === ComplaintType.MORE_FEEDBACK - ? ('artemisApp.moreFeedback.alreadyHandled' | artemisTranslate) - : ('artemisApp.complaint.complaintAlreadyHandled' | artemisTranslate) - }} -
+
}
@if (showLockDuration) { @@ -35,21 +30,13 @@

} @if (lockedByCurrentUser) { - + }

- {{ - complaint.complaintType === ComplaintType.MORE_FEEDBACK - ? ('artemisApp.moreFeedback.title' | artemisTranslate) - : ('artemisApp.complaint.title' | artemisTranslate) - }} - + @if (handled) { @if (complaint?.accepted) { @@ -71,13 +58,9 @@

@if (handled || isAllowedToRespond) {
-

- {{ - complaint.complaintType === ComplaintType.MORE_FEEDBACK - ? ('artemisApp.moreFeedbackResponse.title' | artemisTranslate) - : ('artemisApp.complaintResponse.title' | artemisTranslate) - }} -

+

diff --git a/src/main/webapp/app/complaints/request/complaint-request.component.html b/src/main/webapp/app/complaints/request/complaint-request.component.html index 5c3b8f448f1c..2f86094c7345 100644 --- a/src/main/webapp/app/complaints/request/complaint-request.component.html +++ b/src/main/webapp/app/complaints/request/complaint-request.component.html @@ -7,18 +7,13 @@ }} {{ complaint.submittedTime | artemisTimeAgo }} @if (complaint.accepted === true) { - - {{ - complaint.complaintType === ComplaintType.COMPLAINT - ? ('artemisApp.complaint.acceptedLong' | artemisTranslate) - : ('artemisApp.moreFeedback.acceptedLong' | artemisTranslate) - }} - + } @if (complaint.accepted === false) { - - {{ 'artemisApp.complaint.rejectedLong' | artemisTranslate }} - + }

- +
- + {{ 'artemisApp.course.exercise.search.cardTitle' | artemisTrans
- + {{ 'artemisApp.course.exercise.search.cardTitle' | artemisTrans />
- + (focus)="tzFocus$.next($any($event).target.value)" (click)="tzClick$.next($any($event).target.value)" /> -
- {{ 'artemisApp.forms.configurationForm.timeZoneInput.explanation' | artemisTranslate }} -
+
}
diff --git a/src/main/webapp/app/course/manage/detail/course-detail-line-chart.component.html b/src/main/webapp/app/course/manage/detail/course-detail-line-chart.component.html index b499fda9b56b..9d379b38622e 100644 --- a/src/main/webapp/app/course/manage/detail/course-detail-line-chart.component.html +++ b/src/main/webapp/app/course/manage/detail/course-detail-line-chart.component.html @@ -1,7 +1,7 @@
@@ -16,25 +16,24 @@
{{ 'artemisApp.courseStatistics.activeStudents' | artemisTranslate }}
[disabled]="!showLifetimeOverview && displayedNumberOfWeeks === 4" (click)="displayPeriodOverview(4)" ngbTooltip="{{ 'artemisApp.courseStatistics.scopeButton.periodTooltip' | artemisTranslate: { amount: 4 } }}" - > - {{ 'artemisApp.courseStatistics.scopeButton.period' | artemisTranslate: { amount: 4 } }} - + jhiTranslate="artemisApp.courseStatistics.scopeButton.period" + [translateValues]="{ amount: 4 }" + > + jhiTranslate="artemisApp.courseStatistics.scopeButton.period" + [translateValues]="{ amount: 8 }" + > + jhiTranslate="artemisApp.courseStatistics.scopeButton.overview" + >
| } @@ -85,7 +84,7 @@

} @else { -

{{ 'artemisApp.course.notStartedYet' | artemisTranslate }}

+

{{ course.startDate | artemisDate }}

} diff --git a/src/main/webapp/app/course/manage/overview/course-management-card.component.html b/src/main/webapp/app/course/manage/overview/course-management-card.component.html index 3619d90a93d8..adf01ba9af77 100644 --- a/src/main/webapp/app/course/manage/overview/course-management-card.component.html +++ b/src/main/webapp/app/course/manage/overview/course-management-card.component.html @@ -40,9 +40,7 @@

{{ course.title }} ({{ } @if (courseWithUsers.numberOfStudents === undefined) { - - {{ 'artemisApp.course.students' | artemisTranslate }} - + }

@@ -57,9 +55,7 @@

{{ course.title }} ({{ } @if (courseWithUsers.numberOfTeachingAssistants === undefined) { - - {{ 'artemisApp.course.tutors' | artemisTranslate }} - + }

@@ -76,9 +72,7 @@

{{ course.title }} ({{ } @if (courseWithUsers.numberOfEditors === undefined) { - - {{ 'artemisApp.course.editors' | artemisTranslate }} - + }

@@ -93,9 +87,7 @@

{{ course.title }} ({{ } @if (courseWithUsers.numberOfInstructors === undefined) { - - {{ 'artemisApp.course.instructors' | artemisTranslate }} - + }

@@ -116,7 +108,7 @@

{{ course.title }} ({{
- {{ 'artemisApp.course.releasedSoon' | artemisTranslate }} +
@if (showFutureExercises) {
@@ -139,7 +131,7 @@

{{ course.title }} ({{
- {{ 'artemisApp.course.currentWorking' | artemisTranslate }} +
@if (showCurrentExercises) {
@@ -162,7 +154,7 @@

{{ course.title }} ({{
- {{ 'artemisApp.course.inAssessment' | artemisTranslate }} +
@if (showExercisesInAssessment) {
@@ -185,7 +177,7 @@

{{ course.title }} ({{
- {{ 'artemisApp.course.pastExercises' | artemisTranslate: { amount: pastExercises.length, total: pastExerciseCount } }} +
@if (showPastExercises) {
@@ -206,7 +198,7 @@

{{ course.title }} ({{ } @if ((futureExercises?.length || 0) + (currentExercises?.length || 0) + (exercisesInAssessment?.length || 0) + (pastExercises?.length || 0) === 0) {
-

{{ 'artemisApp.course.noExercises' | artemisTranslate }}

+

}

@@ -240,7 +232,7 @@

{{ 'artemisApp.course.noExer id="course-card-open-exams" > - {{ 'entity.action.exams' | artemisTranslate }} + } @if (course.isAtLeastTutor) { @@ -252,7 +244,7 @@

{{ 'artemisApp.course.noExer id="course-card-open-exercises" > - {{ 'entity.action.exercise' | artemisTranslate }} + } @if (course.isAtLeastEditor) { @@ -263,7 +255,7 @@

{{ 'artemisApp.course.noExer id="course-card-open-lectures" > - {{ 'entity.action.lecture' | artemisTranslate }} + } @if (course.isAtLeastTutor) { @@ -274,7 +266,7 @@

{{ 'artemisApp.course.noExer id="course-card-open-open-statistics" > - {{ 'artemisApp.courseStatistics.statistics' | artemisTranslate }} + } @if (isCommunicationEnabled(course) && course.isAtLeastTutor) { @@ -284,7 +276,7 @@

{{ 'artemisApp.course.noExer [ngbTooltip]="'artemisApp.courseOverview.menu.communication' | artemisTranslate" > - {{ 'artemisApp.metis.communication.label' | artemisTranslate }} + } @if (course.timeZone || course.isAtLeastInstructor) { @@ -332,7 +324,7 @@

{{ 'artemisApp.course.noExer id="course-card-open-assessment-dashboard" > - {{ 'entity.action.assessmentDashboard' | artemisTranslate }} + } @if (course.isAtLeastInstructor) { @@ -343,7 +335,7 @@

{{ 'artemisApp.course.noExer id="course-card-open-scores" > - {{ 'entity.action.scores' | artemisTranslate }} + }

diff --git a/src/main/webapp/app/course/manage/overview/course-management-exercise-row.component.html b/src/main/webapp/app/course/manage/overview/course-management-exercise-row.component.html index f4940c20c0c4..e5f8cdd9dee2 100644 --- a/src/main/webapp/app/course/manage/overview/course-management-exercise-row.component.html +++ b/src/main/webapp/app/course/manage/overview/course-management-exercise-row.component.html @@ -25,7 +25,7 @@
- {{ 'artemisApp.course.releaseDate' | artemisTranslate }} + @if (!details.releaseDate) {
@@ -51,7 +51,7 @@
- {{ 'artemisApp.course.dueDate' | artemisTranslate }} + @if (!details.dueDate) {
@@ -77,7 +77,7 @@
- {{ 'artemisApp.course.assessmentDueDate' | artemisTranslate }} + @if (!details.assessmentDueDate) {
@@ -105,7 +105,7 @@
@if (statistic && rowType === exerciseRowType.CURRENT) {
- {{ 'artemisApp.course.participations' | artemisTranslate }} + - {{ 'artemisApp.course.assessmentProgress' | artemisTranslate }} + - {{ 'artemisApp.course.averageScore' | artemisTranslate }} + diff --git a/src/main/webapp/app/course/tutorial-groups/tutorial-groups-management/tutorial-free-periods/crud/create-tutorial-group-free-period/create-tutorial-group-free-period.component.html b/src/main/webapp/app/course/tutorial-groups/tutorial-groups-management/tutorial-free-periods/crud/create-tutorial-group-free-period/create-tutorial-group-free-period.component.html index f0f12f3eaf09..a810c1c0c46d 100644 --- a/src/main/webapp/app/course/tutorial-groups/tutorial-groups-management/tutorial-free-periods/crud/create-tutorial-group-free-period/create-tutorial-group-free-period.component.html +++ b/src/main/webapp/app/course/tutorial-groups/tutorial-groups-management/tutorial-free-periods/crud/create-tutorial-group-free-period/create-tutorial-group-free-period.component.html @@ -2,7 +2,7 @@ @if (isInitialized) {