Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deployment tests data cleanup enhancements #119

Merged
merged 8 commits into from
Jul 18, 2024
Merged

Deployment tests data cleanup enhancements #119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
840b5fd
Add devnull transport
rpoet-jh Jul 8, 2024
76f3e0b
Add skip deposits prop
rpoet-jh Jul 8, 2024
f2826aa
Skip test deposit transport
rpoet-jh Jul 8, 2024
e385ecb
Update wiremock
rpoet-jh Jul 15, 2024
064a6aa
Add dependencies
rpoet-jh Jul 15, 2024
d60a2bb
Add option to delete dspace deposits
rpoet-jh Jul 15, 2024
d4591d7
Fix test bundle resp typo
rpoet-jh Jul 15, 2024
589d4ea
Make get header null safe
rpoet-jh Jul 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions pass-deposit-services/deposit-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,10 @@
<javax-json.version>1.1.4</javax-json.version>
<maven-model.version>3.9.6</maven-model.version>
<awsspring.version>3.1.0</awsspring.version>
<msal4j-version>1.15.1</msal4j-version>
<msal4j.version>1.15.1</msal4j.version>
<httpclient5.version>5.3.1</httpclient5.version>
<json-path.version>2.9.0</json-path.version>
<wiremock.version>3.3.1</wiremock.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -217,7 +220,19 @@
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>msal4j</artifactId>
<version>${msal4j-version}</version>
<version>${msal4j.version}</version>
</dependency>

<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
<version>${httpclient5.version}</version>
</dependency>

<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>${json-path.version}</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -276,6 +291,13 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.wiremock</groupId>
<artifactId>wiremock-standalone</artifactId>
<version>${wiremock.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
Expand Down
17 changes: 15 additions & 2 deletions ...sit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
import org.eclipse.pass.deposit.cri.CriticalRepositoryInteraction.CriticalResult;
import org.eclipse.pass.deposit.model.Packager;
import org.eclipse.pass.deposit.service.DepositUtil.DepositWorkerContext;
import org.eclipse.pass.deposit.support.deploymenttest.DeploymentTestDataService;
import org.eclipse.pass.deposit.transport.Transport;
import org.eclipse.pass.deposit.transport.TransportResponse;
import org.eclipse.pass.deposit.transport.TransportSession;
import org.eclipse.pass.deposit.transport.sword2.Sword2DepositReceiptResponse;
Expand Down Expand Up @@ -433,8 +435,9 @@ static Function<Deposit, TransportResponse> performDeposit(DepositWorkerContext
dc.deposit().getId(), e);
}

try (TransportSession transport = packager.getTransport().open(packagerConfig)) {
TransportResponse tr = transport.send(packageStream, packagerConfig);
Transport transport = getTransport(packager, dc);
try (TransportSession transportSession = transport.open(packagerConfig)) {
TransportResponse tr = transportSession.send(packageStream, packagerConfig);
deposit.setDepositStatus(DepositStatus.SUBMITTED);
deposit.setDepositStatusRef(packageStream.metadata().packageDepositStatusRef());
return tr;
Expand All @@ -445,6 +448,16 @@ static Function<Deposit, TransportResponse> performDeposit(DepositWorkerContext
};
}

static Transport getTransport(Packager packager, DepositWorkerContext dc) {
return skipDeploymentTestSubmission(dc) ? dc.getDevNullTransport() : packager.getTransport();
}

static boolean skipDeploymentTestSubmission(DepositWorkerContext dc) {
return dc.isSkipDeploymentTestDeposits() && dc.submission().getGrants().stream().anyMatch(
grant -> grant.getProjectName().contains(DeploymentTestDataService.PASS_E2E_TEST_GRANT)
);
}

/**
* Answers a {@code BiPredicate} that checks the TransportResponse for success and places the updated Deposit
* resource in the DepositWorkerContext. If the TransportResponse indicates an error, the exception is
Expand Down
16 changes: 12 additions & 4 deletions ...rvices/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTaskHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.eclipse.pass.deposit.model.DepositSubmission;
import org.eclipse.pass.deposit.model.Packager;
import org.eclipse.pass.deposit.status.DepositStatusProcessor;
import org.eclipse.pass.deposit.transport.devnull.DevNullTransport;
import org.eclipse.pass.support.client.PassClient;
import org.eclipse.pass.support.client.model.CopyStatus;
import org.eclipse.pass.support.client.model.Deposit;
Expand Down Expand Up @@ -79,6 +80,7 @@ public class DepositTaskHelper {

private final PassClient passClient;
private final CriticalRepositoryInteraction cri;
private final DevNullTransport devNullTransport;

@Value("${pass.deposit.transport.swordv2.sleep-time-ms}")
private long swordDepositSleepTimeMs;
Expand All @@ -89,15 +91,19 @@ public class DepositTaskHelper {
@Value("${jscholarship.hack.sword.statement.uri-replacement}")
private String statementUriReplacement;

@Value("${pass.test.skip.deposits}")
private Boolean skipDeploymentTestDeposits;

private final Repositories repositories;

@Autowired
public DepositTaskHelper(PassClient passClient,
CriticalRepositoryInteraction cri,
Repositories repositories) {
Repositories repositories, DevNullTransport devNullTransport) {
this.passClient = passClient;
this.cri = cri;
this.repositories = repositories;
this.devNullTransport = devNullTransport;
}

/**
Expand All @@ -122,11 +128,13 @@ public DepositTaskHelper(PassClient passClient,
* @param packager the Packager for the {@code repo}
*/
public void submitDeposit(Submission submission, DepositSubmission depositSubmission, Repository repo,
Deposit deposit,
Packager packager) {
Deposit deposit, Packager packager) {
try {
Submission includedSubmission = passClient.getObject(submission,
"publication", "repositories", "submitter", "preparers", "grants", "effectivePolicies");
DepositUtil.DepositWorkerContext dc = DepositUtil.toDepositWorkerContext(
deposit, submission, depositSubmission, repo, packager);
deposit, includedSubmission, depositSubmission, repo, packager, devNullTransport,
skipDeploymentTestDeposits);
DepositTask depositTask = new DepositTask(dc, passClient, cri);
depositTask.setSwordSleepTimeMs(swordDepositSleepTimeMs);
depositTask.setPrefixToMatch(statementUriPrefix);
Expand Down
17 changes: 16 additions & 1 deletion ...sit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.eclipse.pass.deposit.cri.CriticalRepositoryInteraction.CriticalResult;
import org.eclipse.pass.deposit.model.DepositSubmission;
import org.eclipse.pass.deposit.model.Packager;
import org.eclipse.pass.deposit.transport.devnull.DevNullTransport;
import org.eclipse.pass.support.client.model.AggregatedDepositStatus;
import org.eclipse.pass.support.client.model.Deposit;
import org.eclipse.pass.support.client.model.DepositStatus;
Expand Down Expand Up @@ -52,13 +53,17 @@ private DepositUtil() {
*/
public static DepositWorkerContext toDepositWorkerContext(Deposit depositResource, Submission submission,
DepositSubmission depositSubmission,
Repository repository, Packager packager) {
Repository repository, Packager packager,
DevNullTransport devNullTransport,
boolean skipDeploymentTestDeposits) {
DepositWorkerContext dc = new DepositWorkerContext();
dc.depositResource = depositResource;
dc.depositSubmission = depositSubmission;
dc.repository = repository;
dc.packager = packager;
dc.submission = submission;
dc.devNullTransport = devNullTransport;
dc.skipDeploymentTestDeposits = skipDeploymentTestDeposits;
return dc;
}

Expand Down Expand Up @@ -143,6 +148,8 @@ public static class DepositWorkerContext {
private Packager packager;
private RepositoryCopy repoCopy;
private String statusUri;
private DevNullTransport devNullTransport;
private boolean skipDeploymentTestDeposits;

/**
* the {@code Deposit} itself
Expand Down Expand Up @@ -227,6 +234,14 @@ public void statusUri(String statusUri) {
this.statusUri = statusUri;
}

public DevNullTransport getDevNullTransport() {
return devNullTransport;
}

public boolean isSkipDeploymentTestDeposits() {
return skipDeploymentTestDeposits;
}

@Override
public String toString() {
return "DepositWorkerContext{" +
Expand Down
61 changes: 56 additions & 5 deletions ...it/service/DeploymentTestDataService.java → ...oymenttest/DeploymentTestDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.eclipse.pass.deposit.service;
package org.eclipse.pass.deposit.support.deploymenttest;

import java.io.IOException;
import java.time.ZonedDateTime;
import java.util.List;
import java.util.Objects;

import org.eclipse.pass.support.client.ModelUtil;
import org.eclipse.pass.support.client.PassClient;
Expand All @@ -44,19 +45,27 @@
public class DeploymentTestDataService {
private static final Logger LOG = LoggerFactory.getLogger(DeploymentTestDataService.class);

static final String PASS_E2E_TEST_GRANT = "PASS_E2E_TEST_GRANT";
public static final String PASS_E2E_TEST_GRANT = "PASS_E2E_TEST_GRANT";

private final PassClient passClient;
private final DspaceDepositService dspaceDepositService;

@Value("${pass.test.data.policy.title}")
private String testPolicyTitle;

@Value("${pass.test.data.user.email}")
private String testUserEmail;

@Value("${pass.test.skip.deposits}")
private Boolean skipDeploymentTestDeposits;

@Value("${pass.test.dspace.repo.key}")
private String dspaceKey;

@Autowired
public DeploymentTestDataService(PassClient passClient) {
public DeploymentTestDataService(PassClient passClient, DspaceDepositService dspaceDepositService) {
this.passClient = passClient;
this.dspaceDepositService = dspaceDepositService;
}

public void processTestData() throws IOException {
Expand All @@ -65,6 +74,7 @@ public void processTestData() throws IOException {
grantSelector.setFilter(RSQL.equals("projectName", PASS_E2E_TEST_GRANT));
List<Grant> testGrants = passClient.streamObjects(grantSelector).toList();
Grant testGrant = testGrants.isEmpty() ? createTestGrantData() : testGrants.get(0);
deleteDepositsInRepoIfNeeded(testGrant);
deleteTestSubmissions(testGrant);
LOG.warn("Done running Deployment Test Data Service");
}
Expand Down Expand Up @@ -99,13 +109,54 @@ private void deleteTestSubmissions(Grant testGrant) throws IOException {
testSubmissionEvents.forEach(this::deleteObject);
deleteObject(testSubmission);
deleteObject(testSubmission.getPublication());
} catch (IOException e) {
throw new RuntimeException(e);
} catch (Exception e) {
LOG.error("Error deleting test submission ID=" + testSubmission.getId(), e);
}
});
LOG.warn("Deleted {} Test Submissions", testSubmissions.size());
}

private void deleteDepositsInRepoIfNeeded(Grant testGrant) throws IOException {
if (Boolean.FALSE.equals(skipDeploymentTestDeposits)) {
LOG.warn("Deleting Test Deposits In Repositories (skipDeploymentTestDeposits=" +
skipDeploymentTestDeposits + ")");
ZonedDateTime submissionToDate = ZonedDateTime.now().minusHours(1);
PassClientSelector<Submission> testSubmissionSelector = new PassClientSelector<>(Submission.class);
testSubmissionSelector.setFilter(RSQL.and(
RSQL.equals("grants.id", testGrant.getId()),
RSQL.lte("submittedDate", ModelUtil.dateTimeFormatter().format(submissionToDate))
));
List<Submission> testSubmissions = passClient.streamObjects(testSubmissionSelector).toList();
if (!testSubmissions.isEmpty()) {
DspaceDepositService.AuthContext authContext = dspaceDepositService.authenticate();
testSubmissions.forEach(testSubmission -> {
try {
PassClientSelector<Deposit> testDepositSelector = new PassClientSelector<>(Deposit.class);
testDepositSelector.setFilter(RSQL.equals("submission.id", testSubmission.getId()));
testDepositSelector.setInclude("submission", "repository", "repositoryCopy");
List<Deposit> testDeposits = passClient.streamObjects(testDepositSelector).toList();
testDeposits.forEach(testDeposit -> deleteDepositInRepoIfNeeded(testDeposit, authContext));
} catch (Exception e) {
LOG.error("Error deleting Deposits in Repository for Submission ID=" +
testSubmission.getId(), e);
}
});
}
LOG.warn("Finished Deleting Deposits In Repositories");
}
}

private void deleteDepositInRepoIfNeeded(Deposit deposit, DspaceDepositService.AuthContext authContext) {
if (isDspaceDeposit(deposit)) {
dspaceDepositService.deleteDeposit(deposit, authContext);
}
}

private boolean isDspaceDeposit(Deposit deposit) {
String repoKey = deposit.getRepository().getRepositoryKey();
return Objects.equals(repoKey, dspaceKey);
}

private void deleteObject(PassEntity entity) {
try {
passClient.deleteObject(entity);
Expand Down
Loading
Loading