Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add NIHMS ID to nihms loader logging #102

Merged
merged 1 commit into from
Mar 18, 2024
Merged

Add NIHMS ID to nihms loader logging #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions ...ms-data-transform-load/src/main/java/org/eclipse/pass/loader/nihms/NihmsCsvProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,15 +163,15 @@ private void consumeRow(CSVRecord row, Consumer<NihmsPublication> pubConsumer) {
row.get(TAGGINGCOMPLETE_COLNUM), row.get(FINALAPPROVAL_COLNUM),
row.get(ARTICLETITLE_COLNUM));

LOG.info("NIHMS record pmid={} is being processed", pub.getPmid());
LOG.info("NIHMS record PMID={} and NIHMS ID={} is being processed", pub.getPmid(), pub.getRawNihmsId());
pubConsumer.accept(pub);
LOG.info("NIHMS record pmid={} was processed successfully", pub.getPmid());
LOG.info("NIHMS record PMID={} and NIHMS ID={} was processed successfully", pub.getPmid(),
pub.getRawNihmsId());
} catch (Exception ex) {
failCount = failCount + 1;
LOG.error(
"A problem occurred while processing csv row {} with pmid {}. The record was not imported " +
"successfully.",
recCount + 1, pub.getPmid(), ex);
"A problem occurred while processing csv row {} with PMID {} and NIHMS ID {}. The record was not " +
"imported successfully.", recCount + 1, pub.getPmid(), pub.getRawNihmsId(), ex);
}
}

Expand Down
27 changes: 13 additions & 14 deletions ...nsform-load/src/main/java/org/eclipse/pass/loader/nihms/NihmsPublicationToSubmission.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@ public SubmissionDTO transform(NihmsPublication pub) throws IOException {
Grant grant = clientService.findMostRecentGrantByAwardNumber(pub.getGrantNumber());
if (grant == null) {
throw new RuntimeException(
String.format("No Grant matching award number \"%s\" was found. Cannot process submission with pmid %s",
pub.getGrantNumber(), pub.getPmid()));
String.format("No Grant matching award number \"%s\" was found. Cannot process submission with " +
"PMID %s and NIHMS ID %s", pub.getGrantNumber(), pub.getPmid(), pub.getRawNihmsId()));
}

//this stage will be all about building up the DTO
Expand Down Expand Up @@ -135,6 +135,8 @@ private Publication retrieveOrCreatePublication(NihmsPublication nihmsPub, Submi
Deposit deposit = clientService.findNihmsDepositbyNihmsId(nihmsPub.getRawNihmsId());

if (deposit != null) {
LOG.info("Deposit found for PMID {} using NIHMS ID {} lookup.", nihmsPub.getPmid(),
nihmsPub.getRawNihmsId());
submissionDTO.setUpdateDeposoit(true);
submissionDTO.setDeposit(deposit);

Expand Down Expand Up @@ -198,8 +200,8 @@ private Publication retrieveOrCreatePublication(NihmsPublication nihmsPub, Submi
* @return the Publication, never {@code null}
*/
private Publication initiateNewPublication(NihmsPublication nihmsPub, PubMedEntrezRecord pmr) throws IOException {
LOG.info("No existing publication found for PMID \"{}\", initiating new Publication record",
nihmsPub.getPmid());
LOG.info("No existing publication found for PMID {} and NIHMS ID {}, initiating new Publication record",
nihmsPub.getPmid(), nihmsPub.getRawNihmsId());
Publication publication = new Publication();

publication.setPmid(nihmsPub.getPmid());
Expand Down Expand Up @@ -265,8 +267,8 @@ private RepositoryCopy retrieveOrCreateRepositoryCopy(NihmsPublication pub, Stri

private RepositoryCopy initiateNewRepositoryCopy(NihmsPublication pub, String publicationId) throws IOException {
RepositoryCopy repositoryCopy = new RepositoryCopy();
LOG.info("NIHMS RepositoryCopy record needed for PMID \"{}\", initiating new RepositoryCopy record",
pub.getPmid());
LOG.info("NIHMS RepositoryCopy record needed for PMID {} and NIHMS ID {}, initiating new RepositoryCopy record",
pub.getPmid(), pub.getRawNihmsId());
repositoryCopy.setPublication((clientService.readPublication(publicationId)));
repositoryCopy.setCopyStatus(calcRepoCopyStatus(pub, null));
repositoryCopy.setRepository(clientService.readRepository(nihmsRepositoryId));
Expand Down Expand Up @@ -435,11 +437,9 @@ public static CopyStatus calcRepoCopyStatus(NihmsPublication pub, CopyStatus cur
// probably bad data
if (currCopyStatus != null && currCopyStatus.equals(CopyStatus.COMPLETE)) {
LOG.warn(
"A NIHMS record for a publication with PMID {} is attempting to change the status of a COMPLETED " +
"Repository Copy. "
+ "This may be due to two PMIDs being assigned to a single DOI, so the data should be checked. The " +
"status will not "
+ "be changed.", pub.getPmid());
"A NIHMS record for a publication with PMID {} and NIHMS ID {} is attempting to change the status " +
"of a COMPLETED Repository Copy. This may be due to two PMIDs being assigned to a single DOI, " +
"so the data should be checked. The status will not be changed.", pub.getPmid(), pub.getRawNihmsId());
return CopyStatus.COMPLETE;
}

Expand All @@ -460,9 +460,8 @@ public static CopyStatus calcRepoCopyStatus(NihmsPublication pub, CopyStatus cur
|| (currCopyStatus.equals(CopyStatus.IN_PROGRESS) && newStatus.equals(CopyStatus.ACCEPTED))) {
LOG.warn(
"The status of the RepositoryCopy in PASS was at a later stage than the current NIHMS status " +
"would imply. "
+ "Rolled back from \"{}\" to \"{}\" for pmid {}", currCopyStatus.toString(),
(newStatus == null ? "(null)" : newStatus.toString()), pub.getPmid());
"would imply. Rolled back from \"{}\" to \"{}\" for PMID {} and NIHMS ID {}", currCopyStatus,
(newStatus == null ? "(null)" : newStatus.toString()), pub.getPmid(), pub.getRawNihmsId());
}

}
Expand Down
27 changes: 13 additions & 14 deletions ...transform-load/src/main/java/org/eclipse/pass/loader/nihms/NihmsTransformLoadService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public void transformAndLoadFiles(Set<NihmsStatus> statusesToProcess) {
try {
transformAndLoadNihmsPub(pub);
} catch (IOException e) {
LOG.error("Error transforming and loading NIHMS publication for NihmsId {}", pub.getNihmsId(), e);
LOG.error("Error transforming and loading NIHMS publication for NIHMS ID {}", pub.getRawNihmsId(), e);
throw new RuntimeException(e);
}
};
Expand Down Expand Up @@ -111,9 +111,8 @@ void transformAndLoadNihmsPub(NihmsPublication pub) throws IOException {
// if the record is compliant, let's check the cache to see if it has been processed previously
if (pub.getNihmsStatus().equals(NihmsStatus.COMPLIANT)
&& completedPublicationsCache.contains(pub.getPmid(), pub.getGrantNumber())) {
LOG.info(
"Compliant NIHMS record with pmid {} and award number \"{}\" has been processed in a previous load",
pub.getPmid(), pub.getGrantNumber());
LOG.info("Compliant NIHMS record with PMID {}, NIHMS ID {}, and award number {} has been processed " +
"in a previous load", pub.getPmid(), pub.getRawNihmsId(), pub.getGrantNumber());
return;
}

Expand All @@ -124,24 +123,24 @@ void transformAndLoadNihmsPub(NihmsPublication pub) throws IOException {
if (transformedRecord.doUpdate()) {
submissionLoader.load(transformedRecord);
} else {
LOG.info("No update required for PMID {} with award number {}", pub.getPmid(),
pub.getGrantNumber());
LOG.info("No update required for PMID {}, NIHMS ID {}, and award number {}", pub.getPmid(),
pub.getRawNihmsId(), pub.getGrantNumber());
}

break;
} catch (IllegalStateException | ConcurrentModificationException | IllegalArgumentException ex) {
if (attempt < MAX_ATTEMPTS) {
LOG.warn("Update failed for PMID %s due to database conflict, attempting retry # %d", pub.getPmid(),
attempt);
LOG.warn("Update failed for PMID {} and NIHMS ID {} due to database conflict, " +
"attempting retry # {}", pub.getPmid(), pub.getRawNihmsId(), attempt);
LOG.warn("Error message: {}", ex.getMessage());
} else {
throw new RuntimeException(
String.format("Update could not be applied for PMID %s after %d attempts ", pub.getPmid(),
MAX_ATTEMPTS), ex);
String.format("Update could not be applied for PMID %s and NIHMS ID %s after %d attempts ",
pub.getPmid(), pub.getRawNihmsId(), MAX_ATTEMPTS), ex);
}
} catch (IOException e) {
LOG.error("Error transforming or loading record for PMID {} with award number {}", pub.getPmid(),
pub.getGrantNumber(), e);
LOG.error("Error transforming or loading record for PMID {} and NIHMS ID {} with award number {}",
pub.getPmid(), pub.getRawNihmsId(), pub.getGrantNumber(), e);
throw new IOException(e);
}
}
Expand All @@ -150,7 +149,7 @@ void transformAndLoadNihmsPub(NihmsPublication pub) throws IOException {
&& StringUtils.isNotEmpty(pub.getPmcId())) {
//add to cache so it doesn't check it again once it has been processed and has a pmcid assigned
completedPublicationsCache.add(pub.getPmid(), pub.getGrantNumber());
LOG.debug("Added pmid {} and grant \"{}\" to cache", pub.getPmid(), pub.getGrantNumber());
LOG.debug("Added PMID {} and grant \"{}\" to cache", pub.getPmid(), pub.getGrantNumber());
}
}

Expand All @@ -169,7 +168,7 @@ private List<Path> loadFiles(File downloadDirectory) {
}
if (CollectionUtils.isEmpty(filepaths)) {
throw new RuntimeException(
String.format("No file found to process at path %s", downloadDirectory.toString()));
String.format("No file found to process at path %s", downloadDirectory));
}
return filepaths;
}
Expand Down
Loading