From 9d177849d4a03cf6b5b1b4b445ff50296136cf46 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Thu, 14 Dec 2023 10:38:02 -0500 Subject: [PATCH 01/35] Add Grant class --- .../pass/deposit/model/DepositMetadata.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java b/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java index 4ea12928..6e0dbad7 100644 --- a/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java +++ b/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java @@ -536,6 +536,39 @@ public void setType(PERSON_TYPE type) { } } + public static class Grant { + //this is the grant awardNumber in PASS + String grantId; + String funder; + Person grantPi; + + public String getGrantId() { + return grantId; + } + + public void setGrantId(String grantId) { + this.grantId = grantId; + } + + public String getFunder() { + return funder; + } + + public void setFunder(String funder) { + this.funder = funder; + } + + public Person getGrantPi() { + return grantPi; + } + + public void setGrantPi(Person grantPi) { + this.grantPi = grantPi; + } + } + + + public DepositMetadata() { this.manuscriptMetadata = new Manuscript(); this.journalMetadata = new Journal(); From 109ed003ffcf0326ab1efc28fcfed9b32630ce91 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Thu, 14 Dec 2023 11:36:59 -0500 Subject: [PATCH 02/35] Update DepositSubmissionMapper --- .../deposit/builder/DepositSubmissionMapper.java | 10 ++++++++++ .../eclipse/pass/deposit/model/DepositMetadata.java | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java index 0ecdc8a1..3618fee6 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java @@ -100,6 +100,8 @@ public DepositSubmission createDepositSubmission(Submission submissionEntity, Li metadata.setJournalMetadata(journal); ArrayList persons = new ArrayList<>(); metadata.setPersons(persons); + ArrayList grants = new ArrayList<>(); + metadata.setGrantsMetadata(grants); // Data from the Submission resource submission.setId(submissionEntity.getId()); @@ -188,6 +190,14 @@ private DepositMetadata.Person createAuthor(String fullName) { return person; } + private DepositMetadata.Grant createGrant(User userEntity, Grant grantEntity) { + DepositMetadata.Grant grant = new DepositMetadata.Grant(); + grant.setGrantId(grantEntity.getAwardNumber()); + grant.setGrantPi(createPerson(userEntity, DepositMetadata.PERSON_TYPE.pi)); + grant.setFunder(grantEntity.getPrimaryFunder().getName()); + return grant; + } + /** * Convenience method for retrieving a string property. * diff --git a/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java b/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java index 6e0dbad7..30f13149 100644 --- a/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java +++ b/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java @@ -118,6 +118,11 @@ public enum PERSON_TYPE { */ private Article articleMetadata; + /** + * Metadata about the grants that funded the publication + */ + private List grants; + /** * Manuscript-related metadata fields */ @@ -608,6 +613,14 @@ public void setArticleMetadata(Article articleMetadata) { this.articleMetadata = articleMetadata; } + public List getGrantsMetadata() { + return grants; + } + + public void setGrantsMetadata(List grants) { + this.grants = grants; + } + @Override public String toString() { return "DepositMetadata{" + From 195b6149f772061b9840378491ec1e78508b2b3d Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Thu, 14 Dec 2023 14:31:49 -0500 Subject: [PATCH 03/35] Add createGrant --- .../eclipse/pass/deposit/builder/DepositSubmissionMapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java index 3618fee6..b27dc21b 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java @@ -131,6 +131,7 @@ public DepositSubmission createDepositSubmission(Submission submissionEntity, Li // Data from the User resources for the PI and CoPIs User piEntity = grantEntity.getPi(); persons.add(createPerson(piEntity, DepositMetadata.PERSON_TYPE.pi)); + grants.add(createGrant(piEntity, grantEntity)); for (User copiEntity : grantEntity.getCoPis()) { persons.add(createPerson(copiEntity, DepositMetadata.PERSON_TYPE.copi)); From c1deec8021e0505b02e24bcda0056b4cdcb75bdb Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Fri, 15 Dec 2023 10:48:13 -0500 Subject: [PATCH 04/35] Add writing grants to write_metadata --- .../nihms/NihmsMetadataSerializer.java | 37 +++++++++++++- .../provider/nihms/NihmsAssemblerIT.java | 10 ++-- .../nihms/NihmsMetadataSerializerTest.java | 51 ++++++++++++++++++- 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 67700fd6..afc77399 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -210,7 +210,40 @@ private void write_metadata(Document doc) { } } - // Could add grant information here if it was useful. - // Can only be used for a set list of funders + List grantsList = metadata.getGrantsMetadata(); + + if (grantsList != null && !grantsList.isEmpty()) { + Element grantsElement = doc.createElement("grants"); + root.appendChild(grantsElement); + + for (DepositMetadata.Grant grant : grantsList) { + Element grantElement = doc.createElement("grant"); + grantsElement.appendChild(grantElement); + + if (grant.getGrantId() != null) { + grantElement.setAttribute("id", grant.getGrantId()); + } + + if (grant.getFunder() != null) { + grantElement.setAttribute("funder", grant.getFunder()); + } + + DepositMetadata.Person pi = grant.getGrantPi(); + if (pi != null) { + Element piElement = doc.createElement("PI"); + grantElement.appendChild(piElement); + + if (pi.getFirstName() != null) { + piElement.setAttribute("fname", pi.getFirstName()); + } + if (pi.getLastName() != null) { + piElement.setAttribute("lname", pi.getLastName()); + } + if (pi.getEmail() != null) { + piElement.setAttribute("email", pi.getEmail()); + } + } + } + } } } diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index 6b96194e..722aaa5a 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -211,8 +211,8 @@ public void testPackageMetadata() throws Exception { Element title = asList(root.getElementsByTagName("manuscript-title")).get(0); assertEquals(submission.getMetadata().getManuscriptMetadata().getTitle(), title.getTextContent()); - // Insure that only one element is present in the submission metadata - // and insure that the is a PI or a Co-PI for the grant that funded the submission. + // Ensure that only one element is present in the submission metadata + // and Ensure that the is a PI or a Co-PI for the grant that funded the submission. List personElements = asList(root.getElementsByTagName("person")); // Assert that there is only one Person present in the metadata @@ -228,7 +228,7 @@ public void testPackageMetadata() throws Exception { return asPerson; }).toList(); - // Insure that the Person in the metadata matches a Person on the Submission, and that the person is a + // Ensure that the Person in the metadata matches a Person on the Submission, and that the person is a // corresponding pi asPersons.stream().forEach(person -> { assertTrue(submission.getMetadata().getPersons().stream().anyMatch(candidate -> @@ -241,6 +241,10 @@ public void testPackageMetadata() throws Exception { candidate.getType() == person.getType())); }); + //Ensure that the grants in the metadata matches a Grant on the submission, Check the attributes of a grant on + //submission against what is found in the metadata + //TODO: add test here for the grants + // Assert that the DOI is present in the metadata assertEquals(submission.getMetadata().getArticleMetadata().getDoi().toString(), root.getAttribute("doi")); diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java index 772d1860..27b63b01 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.StringWriter; import java.net.URI; import java.net.URL; import java.nio.file.Path; @@ -41,10 +42,15 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import com.github.jknack.handlebars.internal.Files; import org.apache.commons.io.IOUtils; +import org.eclipse.pass.deposit.assembler.MetadataBuilder; import org.eclipse.pass.deposit.assembler.SizedStream; import org.eclipse.pass.deposit.model.DepositMetadata; import org.eclipse.pass.deposit.model.JournalPublicationType; @@ -80,6 +86,7 @@ public void setup() throws Exception { DepositMetadata.Manuscript manuscript = new DepositMetadata.Manuscript(); DepositMetadata.Article article = new DepositMetadata.Article(); List personList = new ArrayList<>(); + List grantList = new ArrayList<>(); //populate journal metadata journal.setJournalId("FJ001"); @@ -143,10 +150,22 @@ public void setup() throws Exception { person4.setMiddleName("The"); personList.add(person4); + DepositMetadata.Grant grant1 = new DepositMetadata.Grant(); + grant1.setGrantId("R0123456789"); + grant1.setFunder("nih"); + grant1.setGrantPi(person1); + grantList.add(grant1); + + DepositMetadata.Grant grant2 = new DepositMetadata.Grant(); + grant2.setGrantId("R0123456000"); + grant2.setFunder("nih"); + grant2.setGrantPi(person2); + grantList.add(grant2); + metadata.setJournalMetadata(journal); metadata.setManuscriptMetadata(manuscript); metadata.setPersons(personList); - metadata.setArticleMetadata(article); + metadata.setGrantsMetadata(grantList); underTest = new NihmsMetadataSerializer(metadata); } @@ -212,6 +231,36 @@ public void testSerializedMetadataDoi() throws Exception { assertTrue(doi.contentEquals(path)); } + /** + * Test that the number of grants and their associated information is valid when serialized + */ + @Test + public void testSerializedMetaDataGrants() throws Exception { + DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + SizedStream sizedStream; + InputStream is; + List grants = new ArrayList<>(); + + sizedStream = underTest.serialize(); + is = sizedStream.getInputStream(); + Document document = builder.parse(is); + NodeList grantNodes = document.getElementsByTagName("grants"); + + // Convert Document to String + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + Transformer transformer = transformerFactory.newTransformer(); + StringWriter writer = new StringWriter(); + transformer.transform(new DOMSource(document), new StreamResult(writer)); + String documentString = writer.toString(); + + for (int i = 0; i < grantNodes.getLength(); i++) { + Node grantNode = grantNodes.item(i); + + //grant = node.getAttributes().getNamedItem("doi").getTextContent(); + } + + } + /** * A complete IssnPubType (having a non null publication type and issn value) should be serialized to the metadata. */ From a09dbc840e257df9c931a605245d06636963d66f Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Fri, 15 Dec 2023 15:06:45 -0500 Subject: [PATCH 05/35] Update testSerializedMetaDataGrants --- .../nihms/NihmsMetadataSerializerTest.java | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java index 27b63b01..ea75e8e5 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java @@ -29,6 +29,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; +import java.lang.reflect.Array; import java.net.URI; import java.net.URL; import java.nio.file.Path; @@ -239,7 +240,12 @@ public void testSerializedMetaDataGrants() throws Exception { DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); SizedStream sizedStream; InputStream is; - List grants = new ArrayList<>(); + + List validAwardNumbers = new ArrayList<>(List.of("R0123456789", "R0123456000")); + List validFunders = new ArrayList<>(List.of("nih")); + List validPiFname = new ArrayList<>(List.of("Bessie", "Elsie")); + List validPiLname = new ArrayList<>(List.of("Cow")); + List validPiEmail = new ArrayList<>(List.of("person@farm.com")); sizedStream = underTest.serialize(); is = sizedStream.getInputStream(); @@ -253,12 +259,36 @@ public void testSerializedMetaDataGrants() throws Exception { transformer.transform(new DOMSource(document), new StreamResult(writer)); String documentString = writer.toString(); - for (int i = 0; i < grantNodes.getLength(); i++) { - Node grantNode = grantNodes.item(i); + Node grantNode = grantNodes.item(0); + NodeList grantNodeChildList = grantNode.getChildNodes(); - //grant = node.getAttributes().getNamedItem("doi").getTextContent(); - } + for (int i = 0; i < grantNodeChildList.getLength(); i++) { + Node grantChildNode = grantNodeChildList.item(i); + + if (grantChildNode.getNodeType() == Node.ELEMENT_NODE) { + Element grantElement = (Element) grantChildNode; + + // Extracting grant attributes from the XML doc + String grantId = grantElement.getAttribute("id"); + String funder = grantElement.getAttribute("funder"); + + assertTrue(validAwardNumbers.contains(grantId)); + assertTrue(validFunders.contains(funder)); + // Extracting PI information from the XML doc + NodeList piNodes = grantElement.getElementsByTagName("PI"); + for (int k = 0; k < piNodes.getLength(); k++) { + Element piElement = (Element) piNodes.item(k); + String piFname = piElement.getAttribute("fname"); + String piLname = piElement.getAttribute("lname"); + String piEmail = piElement.getAttribute("email"); + + assertTrue(validPiFname.contains(piFname)); + assertTrue(validPiLname.contains(piLname)); + assertTrue(validPiEmail.contains(piEmail)); + } + } + } } /** From 5f6904a44442a92d9cf7ad68bdb3974d513e129b Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Fri, 15 Dec 2023 17:10:44 -0500 Subject: [PATCH 06/35] Add funder and grant to testPackageMetadata() --- .../provider/nihms/NihmsAssemblerIT.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index 722aaa5a..5c0bdf1f 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -56,6 +56,8 @@ import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; /** * Creates a package, then extracts it. Performs some basic tests on the extracted package. @@ -244,6 +246,36 @@ public void testPackageMetadata() throws Exception { //Ensure that the grants in the metadata matches a Grant on the submission, Check the attributes of a grant on //submission against what is found in the metadata //TODO: add test here for the grants + List grantElements = asList(root.getElementsByTagName("grant")); + // Assert that there is only two grants present in the metadata. If more grants are needed to test, + // add to the setup + assertEquals(2, grantElements.size()); + + List asGrants = grantElements.stream().map(element -> { + DepositMetadata.Grant asGrant = new DepositMetadata.Grant(); + Person pi = new Person(); + + NodeList grantPis = element.getElementsByTagName("PI"); + for (int i = 0; i < grantPis.getLength(); i++) { + Node grantPiNode = grantPis.item(i); + Element grantPiElement = (Element) grantPiNode; + pi.setFirstName(grantPiElement.getAttribute("fname")); + pi.setLastName(grantPiElement.getAttribute("lname")); + pi.setEmail(grantPiElement.getAttribute("email")); + } + asGrant.setGrantId(element.getAttribute("id")); + asGrant.setFunder(element.getAttribute("funder")); + asGrant.setGrantPi(pi); + return asGrant; + }).toList(); + + //TODO add assertions checking the serialized metadata is in the submission metadata + for (DepositMetadata.Grant grant : asGrants) { + //assert that the grant ID exists and is valid to the submission data + //assertTrue(submission.getMetadata().getGrantsMetadata().get()); + + } + // Assert that the DOI is present in the metadata assertEquals(submission.getMetadata().getArticleMetadata().getDoi().toString(), root.getAttribute("doi")); From 169c8cb4bb49d1879f332539e81cda13c2656652 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 18 Dec 2023 09:07:53 -0500 Subject: [PATCH 07/35] Remove convert doc to string --- .../pass/deposit/provider/nihms/NihmsAssemblerIT.java | 4 ++-- .../provider/nihms/NihmsMetadataSerializerTest.java | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index 5c0bdf1f..f1b021b9 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -246,10 +246,10 @@ public void testPackageMetadata() throws Exception { //Ensure that the grants in the metadata matches a Grant on the submission, Check the attributes of a grant on //submission against what is found in the metadata //TODO: add test here for the grants - List grantElements = asList(root.getElementsByTagName("grant")); + List grantElements = asList(root.getElementsByTagName("grants")); // Assert that there is only two grants present in the metadata. If more grants are needed to test, // add to the setup - assertEquals(2, grantElements.size()); + assertEquals(1, grantElements.size()); List asGrants = grantElements.stream().map(element -> { DepositMetadata.Grant asGrant = new DepositMetadata.Grant(); diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java index ea75e8e5..0416bf72 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java @@ -252,13 +252,6 @@ public void testSerializedMetaDataGrants() throws Exception { Document document = builder.parse(is); NodeList grantNodes = document.getElementsByTagName("grants"); - // Convert Document to String - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = transformerFactory.newTransformer(); - StringWriter writer = new StringWriter(); - transformer.transform(new DOMSource(document), new StreamResult(writer)); - String documentString = writer.toString(); - Node grantNode = grantNodes.item(0); NodeList grantNodeChildList = grantNode.getChildNodes(); From d09efe4d9dfbe037f1388a66c41df62c46739398 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 18 Dec 2023 10:18:43 -0500 Subject: [PATCH 08/35] Add new grants to sample1.json --- .../provider/nihms/NihmsAssemblerIT.java | 34 ++++++++++-- .../test/resources/submissions/sample1.json | 55 +++++++++++++------ 2 files changed, 66 insertions(+), 23 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index f1b021b9..3b3dc15f 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -27,16 +27,22 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -246,10 +252,19 @@ public void testPackageMetadata() throws Exception { //Ensure that the grants in the metadata matches a Grant on the submission, Check the attributes of a grant on //submission against what is found in the metadata //TODO: add test here for the grants - List grantElements = asList(root.getElementsByTagName("grants")); + List grantElements = asList(root.getElementsByTagName("grant")); + + // Convert root Document to String + //TODO remove + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + Transformer transformer = transformerFactory.newTransformer(); + StringWriter writer = new StringWriter(); + transformer.transform(new DOMSource(root), new StreamResult(writer)); + String documentString = writer.toString(); + // Assert that there is only two grants present in the metadata. If more grants are needed to test, // add to the setup - assertEquals(1, grantElements.size()); + assertEquals(2, grantElements.size()); List asGrants = grantElements.stream().map(element -> { DepositMetadata.Grant asGrant = new DepositMetadata.Grant(); @@ -270,13 +285,22 @@ public void testPackageMetadata() throws Exception { }).toList(); //TODO add assertions checking the serialized metadata is in the submission metadata - for (DepositMetadata.Grant grant : asGrants) { + List subGrantMetaData = submission.getMetadata().getGrantsMetadata(); + for (DepositMetadata.Grant subGrant : subGrantMetaData) { + String subGrantId = subGrant.getGrantId(); + + // Find the associated grant from asGrants that matches the ID + Optional metaDataGrant = asGrants.stream() + .filter(asGrant -> asGrant.getGrantId().equals(subGrantId)) + .findFirst(); + //assert that the grant ID exists and is valid to the submission data - //assertTrue(submission.getMetadata().getGrantsMetadata().get()); + assertEquals(subGrant.getGrantId(), metaDataGrant.get().getGrantId()); + assertEquals(subGrant.getGrantPi().getFirstName(), metaDataGrant.get().getGrantPi().getFirstName()); + assertEquals(subGrant.getGrantPi().getLastName(), metaDataGrant.get().getGrantPi().getLastName()); } - // Assert that the DOI is present in the metadata assertEquals(submission.getMetadata().getArticleMetadata().getDoi().toString(), root.getAttribute("doi")); diff --git a/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json b/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json index 5b1d56aa..70d649a3 100644 --- a/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json +++ b/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json @@ -167,6 +167,24 @@ "id": "13", "@type": "Grant" }, + { + "awardNumber": "R01EY026618", + "awardStatus": "ACTIVE", + "localKey": "112234", + "projectName": "Optimal magnification and oculomotor strategies in low vision patients", + "awardDate": "2017-08-01T00:00:00.000Z", + "startDate": "2017-09-01T00:00:00.000Z", + "endDate": "2020-06-01T00:00:00.000Z", + "primaryFunder": "11", + "directFunder": "12", + "pi": "2", + "coPis": [ + "3", + "1" + ], + "id": "14", + "@type": "Grant" + }, { "metadata": "{\n \"agreements\": {\n \"JScholarship\": \"NON-EXCLUSIVE LICENSE FOR USE OF MATERIALS This non-exclusive license defines the terms for the deposit of Materials in all formats into the digital repository of materials collected, preserved and made available through the Johns Hopkins Digital Repository, JScholarship. The Contributor hereby grants to Johns Hopkins a royalty free, non-exclusive worldwide license to use, re-use, display, distribute, transmit, publish, re-publish or copy the Materials, either digitally or in print, or in any other medium, now or hereafter known, for the purpose of including the Materials hereby licensed in the collection of materials in the Johns Hopkins Digital Repository for educational use worldwide. In some cases, access to content may be restricted according to provisions established in negotiation with the copyright holder. This license shall not authorize the commercial use of the Materials by Johns Hopkins or any other person or organization, but such Materials shall be restricted to non-profit educational use. Persons may apply for commercial use by contacting the copyright holder. Copyright and any other intellectual property right in or to the Materials shall not be transferred by this agreement and shall remain with the Contributor, or the Copyright holder if different from the Contributor. Other than this limited license, the Contributor or Copyright holder retains all rights, title, copyright and other interest in the images licensed. If the submission contains material for which the Contributor does not hold copyright, the Contributor represents that s/he has obtained the permission of the Copyright owner to grant Johns Hopkins the rights required by this license, and that such third-party owned material is clearly identified and acknowledged within the text or content of the submission. If the submission is based upon work that has been sponsored or supported by an agency or organization other than Johns Hopkins, the Contributor represents that s/he has fulfilled any right of review or other obligations required by such contract or agreement. Johns Hopkins will not make any alteration, other than as allowed by this license, to your submission. This agreement embodies the entire agreement of the parties. No modification of this agreement shall be of any effect unless it is made in writing and signed by all of the parties to the agreement.\"\n },\n \"abstract\": \"Differential enhancement of luminal growth factor bioactivity and targeted regional gut growth occurs dependent on dietary protein supplement.\",\n \"authors\": [\n {\n \"author\": \"Tania Marchbank\",\n \"orcid\": \"http://orcid.org/0000-0003-2076-9098\"\n },\n {\n \"author\": \"Nikki Mandir\"\n },\n {\n \"author\": \"Denis Calnan\"\n },\n {\n \"author\": \"Robert A. Goodlad\"\n },\n {\n \"author\": \"Theo Podas\"\n },\n {\n \"author\": \"Raymond J. Playford\",\n \"orcid\": \"http://orcid.org/0000-0003-1235-8504\"\n }\n ],\n \"doi\": \"10.1039/c7fo01251a\",\n \"Embargo-end-date\": \"2018-06-30\",\n \"journal-NLMTA-ID\": \"Food Funct\",\n \"journal-title\": \"Food & Function\",\n \"journal-title-short\": \"Food Funct.\",\n \"issue\": \"1\",\n \"issns\": [\n {\n \"issn\": \"2042-6496\",\n \"pubType\": \"Print\"\n },\n {\n \"issn\": \"2042-650X\",\n \"pubType\": \"Online\"\n }\n ],\n \"publisher\": \"Royal Society of Chemistry (RSC)\",\n \"publicationDate\": \"2018-09-12\",\n \"title\": \"Specific protein supplementation using soya, casein or whey differentially affects regional gut growth and luminal growth factor bioactivity in rats; implications for the treatment of gut injury and stimulating repair\",\n \"volume\": \"9\"\n}", "source": "PASS", @@ -183,9 +201,10 @@ ], "submitter": "2", "grants": [ - "13" + "13", + "14" ], - "id": "14", + "id": "15", "@type": "Submission" }, { @@ -194,8 +213,8 @@ "description": "Custodial content", "fileRole": "SUPPLEMENTAL", "mimeType": "image/jpg", - "submission": "14", - "id": "15", + "submission": "15", + "id": "16", "@type": "File" }, { @@ -204,8 +223,8 @@ "description": "Custodial content", "fileRole": "FIGURE", "mimeType": "image/tiff", - "submission": "14", - "id": "16", + "submission": "15", + "id": "17", "@type": "File" }, { @@ -214,8 +233,8 @@ "description": "Custodial content", "fileRole": "FIGURE", "mimeType": "image/png", - "submission": "14", - "id": "17", + "submission": "15", + "id": "18", "@type": "File" }, { @@ -224,8 +243,8 @@ "description": "Custodial content", "fileRole": "MANUSCRIPT", "mimeType": "application/msword", - "submission": "14", - "id": "18", + "submission": "15", + "id": "19", "@type": "File" }, { @@ -234,8 +253,8 @@ "description": "Custodial content", "fileRole": "TABLE", "mimeType": "application/vnd.oasis.opendocument.spreadsheet", - "submission": "14", - "id": "19", + "submission": "15", + "id": "20", "@type": "File" }, { @@ -244,8 +263,8 @@ "description": "Custodial content, meant to conflict with NIH package spec", "fileRole": "SUPPLEMENTAL", "mimeType": "application/xml", - "submission": "14", - "id": "20", + "submission": "15", + "id": "21", "@type": "File" }, { @@ -254,8 +273,8 @@ "description": "Custodial content, meant to conflict with NIH package spec", "fileRole": "SUPPLEMENTAL", "mimeType": "text/plain", - "submission": "14", - "id": "21", + "submission": "15", + "id": "22", "@type": "File" }, { @@ -264,8 +283,8 @@ "description": "Custodial content with a space in the filename", "fileRole": "SUPPLEMENTAL", "mimeType": "text/plain", - "submission": "14", - "id": "22", + "submission": "15", + "id": "23", "@type": "File" } ] From 46de878bfabbbe77338e94f2dca10f3ecebc4fdd Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 18 Dec 2023 11:27:20 -0500 Subject: [PATCH 09/35] Add assertions --- .../deposit/provider/nihms/NihmsAssemblerIT.java | 16 +++------------- .../pass/deposit/model/DepositMetadata.java | 2 -- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index 3b3dc15f..ed17bb42 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -251,19 +251,10 @@ public void testPackageMetadata() throws Exception { //Ensure that the grants in the metadata matches a Grant on the submission, Check the attributes of a grant on //submission against what is found in the metadata - //TODO: add test here for the grants List grantElements = asList(root.getElementsByTagName("grant")); - // Convert root Document to String - //TODO remove - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = transformerFactory.newTransformer(); - StringWriter writer = new StringWriter(); - transformer.transform(new DOMSource(root), new StreamResult(writer)); - String documentString = writer.toString(); - // Assert that there is only two grants present in the metadata. If more grants are needed to test, - // add to the setup + // add to sample1.json assertEquals(2, grantElements.size()); List asGrants = grantElements.stream().map(element -> { @@ -284,7 +275,6 @@ public void testPackageMetadata() throws Exception { return asGrant; }).toList(); - //TODO add assertions checking the serialized metadata is in the submission metadata List subGrantMetaData = submission.getMetadata().getGrantsMetadata(); for (DepositMetadata.Grant subGrant : subGrantMetaData) { String subGrantId = subGrant.getGrantId(); @@ -296,9 +286,10 @@ public void testPackageMetadata() throws Exception { //assert that the grant ID exists and is valid to the submission data assertEquals(subGrant.getGrantId(), metaDataGrant.get().getGrantId()); + assertEquals(subGrant.getFunder(), metaDataGrant.get().getFunder()); assertEquals(subGrant.getGrantPi().getFirstName(), metaDataGrant.get().getGrantPi().getFirstName()); assertEquals(subGrant.getGrantPi().getLastName(), metaDataGrant.get().getGrantPi().getLastName()); - + assertEquals(subGrant.getGrantPi().getEmail(), metaDataGrant.get().getGrantPi().getEmail()); } // Assert that the DOI is present in the metadata @@ -320,7 +311,6 @@ public void testPackageMetadata() throws Exception { assertEquals(issn.getAttribute("issn-type"), "print"); } }); - } /** diff --git a/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java b/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java index 30f13149..e3be938f 100644 --- a/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java +++ b/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java @@ -572,8 +572,6 @@ public void setGrantPi(Person grantPi) { } } - - public DepositMetadata() { this.manuscriptMetadata = new Manuscript(); this.journalMetadata = new Journal(); From 9a6a987470456a9a41e6fbe1bdbd1ccefe4f0a02 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 18 Dec 2023 17:26:46 -0500 Subject: [PATCH 10/35] Update DepositSubmissionModelBuilderIT - Add tests for the grants and changed test data --- .../builder/DepositSubmissionModelBuilderIT.java | 13 ++++++++++--- .../deposit/provider/nihms/NihmsAssemblerIT.java | 5 ----- .../provider/nihms/NihmsMetadataSerializerTest.java | 7 ------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/builder/DepositSubmissionModelBuilderIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/builder/DepositSubmissionModelBuilderIT.java index cec9207f..fec64145 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/builder/DepositSubmissionModelBuilderIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/builder/DepositSubmissionModelBuilderIT.java @@ -49,10 +49,12 @@ public class DepositSubmissionModelBuilderIT extends AbstractDepositSubmissionIT private static final String EXPECTED_DOI = "10.1039/c7fo01251a"; private static final String EXPECTED_EMBARGO_END_DATE = "2018-06-30"; private static final int EXPECTED_SUBMITER_COUNT = 1; - private static final int EXPECTED_PI_COUNT = 1; - private static final int EXPECTED_CO_PI_COUNT = 2; + private static final int EXPECTED_PI_COUNT = 2; + private static final int EXPECTED_CO_PI_COUNT = 4; private static final int EXPECTED_AUTHOR_COUNT = 6; private static final String EXPECTED_NLMTA = "Food Funct"; + private static final String EXPECTED_GRANT1 = "R01EY026617"; + private static final String EXPECTED_GRANT2 = "R01EY026618"; private static final Map EXPECTED_ISSNS = new HashMap<>() { { @@ -159,10 +161,15 @@ public void testBuildDepositSubmission() throws IOException { .anyMatch(author -> author.getName().equals("Raymond J. Playford"))); + //test the grants associated with the submission + List grants = submission.getMetadata().getGrantsMetadata(); + + assertNotNull(grants.stream().filter(matchGrant -> matchGrant.getGrantId() == EXPECTED_GRANT1)); + assertNotNull(grants.stream().filter(matchGrant -> matchGrant.getGrantId() == EXPECTED_GRANT2)); + // Read something out of the submission metadata assertTrue(submission.getSubmissionMeta().has("agreements")); JsonObject agreement = submission.getSubmissionMeta().getAsJsonObject("agreements"); assertTrue(agreement.has("JScholarship")); } - } \ No newline at end of file diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index ed17bb42..27fa1433 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -27,7 +27,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -39,10 +38,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java index 0416bf72..eb0bafd7 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java @@ -28,8 +28,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.StringWriter; -import java.lang.reflect.Array; import java.net.URI; import java.net.URL; import java.nio.file.Path; @@ -43,15 +41,10 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import com.github.jknack.handlebars.internal.Files; import org.apache.commons.io.IOUtils; -import org.eclipse.pass.deposit.assembler.MetadataBuilder; import org.eclipse.pass.deposit.assembler.SizedStream; import org.eclipse.pass.deposit.model.DepositMetadata; import org.eclipse.pass.deposit.model.JournalPublicationType; From dceff4e9f8c09dd95670b16f06413027c62832bb Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 19 Dec 2023 10:09:01 -0500 Subject: [PATCH 11/35] Update commonContributorsAndFewAuthors() - Should be 8 authors now, adding 1 more PI + 2 co-pis --- .../java/org/eclipse/pass/deposit/provider/j10p/MetadataIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/j10p/MetadataIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/j10p/MetadataIT.java index 88c4b35f..3d743240 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/j10p/MetadataIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/j10p/MetadataIT.java @@ -86,7 +86,7 @@ public void commonContributorsAndFewAuthors() throws Exception { assertEquals("Elsevier", qdc.getElementsByTagNameNS(DC_NS, DC_PUBLISHER).item(0).getTextContent()); // Contributor list does not include submitting user, only PIs (from Grant) and authors (from metadata) - assertEquals(5, qdc.getElementsByTagNameNS(DC_NS, DC_CONTRIBUTOR).getLength()); + assertEquals(8, qdc.getElementsByTagNameNS(DC_NS, DC_CONTRIBUTOR).getLength()); for (int i = 0; i < 5; i++) { // Contributors must have names, whether they come from // first/middle/last or only display name (in Submission), or from metadata. From 543fafa87bf72016d82d0f5d42cee56f05856bb3 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 20 Dec 2023 10:31:47 -0500 Subject: [PATCH 12/35] Refactor write_metadata --- .../nihms/NihmsMetadataSerializer.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index afc77399..970b127d 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -92,16 +92,26 @@ private void add_text_element(Document doc, Element parent, String name, String } private void write_metadata(Document doc) { - DepositMetadata.Manuscript manuscript = metadata.getManuscriptMetadata(); - DepositMetadata.Article article = metadata.getArticleMetadata(); - DepositMetadata.Journal journal = metadata.getJournalMetadata(); - Element root = doc.createElement("manuscript-submit"); doc.appendChild(root); + addManuscriptMetadata(doc, root); + addArticleMetadata(doc, root); + addManuscriptTitle(doc, root); + addContacts(doc, root); + addGrants(doc, root); + } + + private void addManuscriptMetadata(Document doc, Element root) { + DepositMetadata.Manuscript manuscript = metadata.getManuscriptMetadata(); + if (manuscript.getNihmsId() != null) { root.setAttribute("manuscript-id", manuscript.getNihmsId()); } + } + + private void addArticleMetadata(Document doc, Element root) { + DepositMetadata.Article article = metadata.getArticleMetadata(); if (article != null && metadata.getArticleMetadata().getEmbargoLiftDate() != null) { LocalDate lift = article.getEmbargoLiftDate().toLocalDate(); @@ -128,6 +138,9 @@ private void write_metadata(Document doc) { root.setAttribute("doi", path); } + } + private void addJournalMetadata(Document doc, Element root) { + DepositMetadata.Journal journal = metadata.getJournalMetadata(); // There is an optional agency attribute. // Should only be used for non-NIH funders when grant information is also given @@ -159,12 +172,17 @@ private void write_metadata(Document doc) { add_text_element(doc, journal_meta, "journal-title", journal.getJournalTitle()); } + } + private void addManuscriptTitle(Document doc, Element root) { + DepositMetadata.Manuscript manuscript = metadata.getManuscriptMetadata(); + if (manuscript != null && manuscript.title != null) { add_text_element(doc, root, "manuscript-title", manuscript.title); } - // Could add a citation element if we had all the information + } + private void addContacts(Document doc, Element root) { List persons = metadata.getPersons(); if (persons != null && persons.size() > 0) { @@ -209,7 +227,9 @@ private void write_metadata(Document doc) { } } } + } + private void addGrants(Document doc, Element root) { List grantsList = metadata.getGrantsMetadata(); if (grantsList != null && !grantsList.isEmpty()) { From 707dbc181686b4ee2eab87ff1236a8c037641618 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 20 Dec 2023 15:25:15 -0500 Subject: [PATCH 13/35] Refactor testPackageMetadata() - Extract out the grant and funder into a separate test --- .../nihms/NihmsMetadataSerializer.java | 71 ++++++++++--------- .../provider/nihms/NihmsAssemblerIT.java | 63 +++++++++------- 2 files changed, 75 insertions(+), 59 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 970b127d..0b0c6ba7 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -30,6 +30,7 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; +import org.apache.commons.lang3.StringUtils; import org.eclipse.pass.deposit.assembler.SizedStream; import org.eclipse.pass.deposit.model.DepositMetadata; import org.eclipse.pass.deposit.model.JournalPublicationType; @@ -95,22 +96,23 @@ private void write_metadata(Document doc) { Element root = doc.createElement("manuscript-submit"); doc.appendChild(root); - addManuscriptMetadata(doc, root); - addArticleMetadata(doc, root); + addManuscriptMetadata(root); + addArticleMetadata(root); + addJournalMetadata(doc, root); addManuscriptTitle(doc, root); addContacts(doc, root); addGrants(doc, root); } - private void addManuscriptMetadata(Document doc, Element root) { + private void addManuscriptMetadata(Element root) { DepositMetadata.Manuscript manuscript = metadata.getManuscriptMetadata(); - if (manuscript.getNihmsId() != null) { + if (StringUtils.isNotBlank(manuscript.getNihmsId())) { root.setAttribute("manuscript-id", manuscript.getNihmsId()); } } - private void addArticleMetadata(Document doc, Element root) { + private void addArticleMetadata(Element root) { DepositMetadata.Article article = metadata.getArticleMetadata(); if (article != null && metadata.getArticleMetadata().getEmbargoLiftDate() != null) { @@ -156,7 +158,7 @@ private void addJournalMetadata(Document doc, Element root) { // See https://github.com/OA-PASS/metadata-schemas/pull/28 and // https://github.com/OA-PASS/jhu-package-providers/issues/16 journal.getIssnPubTypes().values().forEach(issnPubType -> { - if (issnPubType.pubType == null || issnPubType.issn == null || issnPubType.issn.trim().isEmpty()) { + if (issnPubType.pubType == null || StringUtils.isBlank(issnPubType.issn)) { LOG.debug("Discarding incomplete ISSN: {}", issnPubType); return; } @@ -176,7 +178,7 @@ private void addJournalMetadata(Document doc, Element root) { private void addManuscriptTitle(Document doc, Element root) { DepositMetadata.Manuscript manuscript = metadata.getManuscriptMetadata(); - if (manuscript != null && manuscript.title != null) { + if (manuscript != null && StringUtils.isNotBlank(manuscript.title)) { add_text_element(doc, root, "manuscript-title", manuscript.title); } @@ -195,20 +197,20 @@ private void addContacts(Document doc, Element root) { Element p = doc.createElement("person"); contacts.appendChild(p); - if (person.getFirstName() != null) { + if (StringUtils.isNotBlank(person.getFirstName())) { p.setAttribute("fname", person.getFirstName()); } else { - if (person.getFullName() != null) { + if (StringUtils.isNotBlank(person.getFullName())) { p.setAttribute("fname", person.getFullName().split("\\s")[0]); } } - if (person.getMiddleName() != null) { + if (StringUtils.isNotBlank(person.getMiddleName())) { p.setAttribute("mname", person.getMiddleName()); } - if (person.getLastName() != null) { + if (StringUtils.isNotBlank(person.getLastName())) { p.setAttribute("lname", person.getLastName()); } else { - if (person.getFullName() != null) { + if (StringUtils.isNotBlank(person.getFullName())) { String[] split = person.getFullName().split("\\s"); if (split.length > 2) { // middle name is present @@ -219,7 +221,7 @@ private void addContacts(Document doc, Element root) { } } } - if (person.getEmail() != null) { + if (StringUtils.isNotBlank(person.getEmail())) { p.setAttribute("email", person.getEmail()); } @@ -233,34 +235,37 @@ private void addGrants(Document doc, Element root) { List grantsList = metadata.getGrantsMetadata(); if (grantsList != null && !grantsList.isEmpty()) { + //TODO: add check, if funder is not in the list of accepted funders then do not create grant element + //TODO: at least 1 accepted funder for a grant is required to create parent element grants Element grantsElement = doc.createElement("grants"); root.appendChild(grantsElement); for (DepositMetadata.Grant grant : grantsList) { - Element grantElement = doc.createElement("grant"); - grantsElement.appendChild(grantElement); + if (StringUtils.isNotBlank(grant.getFunder())) { + grantsElement.setAttribute("funder", grant.getFunder()); - if (grant.getGrantId() != null) { - grantElement.setAttribute("id", grant.getGrantId()); - } + Element grantElement = doc.createElement("grant"); + grantsElement.appendChild(grantElement); - if (grant.getFunder() != null) { - grantElement.setAttribute("funder", grant.getFunder()); - } + if (StringUtils.isNotBlank(grant.getGrantId())) { + grantElement.setAttribute("id", grant.getGrantId()); + } - DepositMetadata.Person pi = grant.getGrantPi(); - if (pi != null) { - Element piElement = doc.createElement("PI"); - grantElement.appendChild(piElement); - if (pi.getFirstName() != null) { - piElement.setAttribute("fname", pi.getFirstName()); - } - if (pi.getLastName() != null) { - piElement.setAttribute("lname", pi.getLastName()); - } - if (pi.getEmail() != null) { - piElement.setAttribute("email", pi.getEmail()); + DepositMetadata.Person pi = grant.getGrantPi(); + if (pi != null) { + Element piElement = doc.createElement("PI"); + grantElement.appendChild(piElement); + + if (StringUtils.isNotBlank(pi.getFirstName())) { + piElement.setAttribute("fname", pi.getFirstName()); + } + if (StringUtils.isNotBlank(pi.getLastName())) { + piElement.setAttribute("lname", pi.getLastName()); + } + if (StringUtils.isNotBlank(pi.getEmail())) { + piElement.setAttribute("email", pi.getEmail()); + } } } } diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index 27fa1433..514438d9 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -38,6 +38,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -203,12 +204,7 @@ public void testPackageManifest() throws Exception { @Test public void testPackageMetadata() throws Exception { - Document metaDom = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(metadata); - assertNotNull(metaDom); - - // root element is - Element root = metaDom.getDocumentElement(); - assertEquals("manuscript-submit", root.getTagName()); + Element root = initDom(); // required element is present with the manuscript title as the value Element title = asList(root.getElementsByTagName("manuscript-title")).get(0); @@ -244,6 +240,31 @@ public void testPackageMetadata() throws Exception { candidate.getType() == person.getType())); }); + + // Assert that the DOI is present in the metadata + assertEquals(submission.getMetadata().getArticleMetadata().getDoi().toString(), root.getAttribute("doi")); + + // Assert that the ISSNs are present in the metadata as the element + List issns = asList(root.getElementsByTagName("issn")); + Map issnPubTypes = + submission.getMetadata().getJournalMetadata().getIssnPubTypes(); + assertEquals(issnPubTypes.size(), issns.size()); + assertTrue(issnPubTypes.size() > 0); + + issns.forEach(issn -> assertTrue(issnPubTypes.containsKey(issn.getTextContent()))); + issns.forEach(issn -> { + DepositMetadata.IssnPubType pubType = issnPubTypes.get(issn.getTextContent()); + if (pubType.pubType == JournalPublicationType.OPUB || pubType.pubType == JournalPublicationType.EPUB) { + assertEquals(issn.getAttribute("issn-type"), "electronic"); + } else { + assertEquals(issn.getAttribute("issn-type"), "print"); + } + }); + } + + @Test + public void testPackageMetadataGrantFunder() throws Exception { + Element root = initDom(); //Ensure that the grants in the metadata matches a Grant on the submission, Check the attributes of a grant on //submission against what is found in the metadata List grantElements = asList(root.getElementsByTagName("grant")); @@ -286,26 +307,6 @@ public void testPackageMetadata() throws Exception { assertEquals(subGrant.getGrantPi().getLastName(), metaDataGrant.get().getGrantPi().getLastName()); assertEquals(subGrant.getGrantPi().getEmail(), metaDataGrant.get().getGrantPi().getEmail()); } - - // Assert that the DOI is present in the metadata - assertEquals(submission.getMetadata().getArticleMetadata().getDoi().toString(), root.getAttribute("doi")); - - // Assert that the ISSNs are present in the metadata as the element - List issns = asList(root.getElementsByTagName("issn")); - Map issnPubTypes = - submission.getMetadata().getJournalMetadata().getIssnPubTypes(); - assertEquals(issnPubTypes.size(), issns.size()); - assertTrue(issnPubTypes.size() > 0); - - issns.forEach(issn -> assertTrue(issnPubTypes.containsKey(issn.getTextContent()))); - issns.forEach(issn -> { - DepositMetadata.IssnPubType pubType = issnPubTypes.get(issn.getTextContent()); - if (pubType.pubType == JournalPublicationType.OPUB || pubType.pubType == JournalPublicationType.EPUB) { - assertEquals(issn.getAttribute("issn-type"), "electronic"); - } else { - assertEquals(issn.getAttribute("issn-type"), "print"); - } - }); } /** @@ -326,6 +327,16 @@ private static boolean isNullOrEmpty(String s) { return s == null || s.trim().length() == 0; } + private Element initDom() throws Exception { + Document metaDom = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(metadata); + assertNotNull(metaDom); + + // root element is + Element root = metaDom.getDocumentElement(); + assertEquals("manuscript-submit", root.getTagName()); + return root; + } + private static class ManifestLine { private static final String ERR = "File %s, line %s is missing %s"; private File manifestFile; From cc23c81c28cf85093cde8bedeec69dc095ad32e4 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 20 Dec 2023 15:37:41 -0500 Subject: [PATCH 14/35] Fix grantElement --- .../pass/deposit/provider/nihms/NihmsMetadataSerializer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 0b0c6ba7..d8a5111b 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -242,10 +242,9 @@ private void addGrants(Document doc, Element root) { for (DepositMetadata.Grant grant : grantsList) { if (StringUtils.isNotBlank(grant.getFunder())) { - grantsElement.setAttribute("funder", grant.getFunder()); - Element grantElement = doc.createElement("grant"); grantsElement.appendChild(grantElement); + grantElement.setAttribute("funder", grant.getFunder()); if (StringUtils.isNotBlank(grant.getGrantId())) { grantElement.setAttribute("id", grant.getGrantId()); From 2d7f8d360128c21d779c2325ac9813776f602873 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 20 Dec 2023 15:55:54 -0500 Subject: [PATCH 15/35] Cleanup --- .../pass/deposit/provider/nihms/NihmsMetadataSerializer.java | 3 ++- .../eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index d8a5111b..247100d3 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -141,6 +141,7 @@ private void addArticleMetadata(Element root) { root.setAttribute("doi", path); } } + private void addJournalMetadata(Document doc, Element root) { DepositMetadata.Journal journal = metadata.getJournalMetadata(); @@ -175,6 +176,7 @@ private void addJournalMetadata(Document doc, Element root) { } } + private void addManuscriptTitle(Document doc, Element root) { DepositMetadata.Manuscript manuscript = metadata.getManuscriptMetadata(); @@ -250,7 +252,6 @@ private void addGrants(Document doc, Element root) { grantElement.setAttribute("id", grant.getGrantId()); } - DepositMetadata.Person pi = grant.getGrantPi(); if (pi != null) { Element piElement = doc.createElement("PI"); diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index 514438d9..8ec9bf33 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -38,7 +38,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -240,7 +239,6 @@ public void testPackageMetadata() throws Exception { candidate.getType() == person.getType())); }); - // Assert that the DOI is present in the metadata assertEquals(submission.getMetadata().getArticleMetadata().getDoi().toString(), root.getAttribute("doi")); From 365d45372af86df3fa5d993c8d3e60794b7e9eb8 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 20 Dec 2023 17:18:00 -0500 Subject: [PATCH 16/35] Add funder-mapping to repositories.json and RepositoryDepositConfig --- .../repository/RepositoryDepositConfig.java | 16 +++++++++++++++- .../src/main/resources/repositories.json | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java index 2c659e60..d6dfb13b 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java @@ -17,6 +17,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; +import java.util.Map; + /** * @author Elliot Metsger (emetsger@jhu.edu) */ @@ -28,6 +31,9 @@ public class RepositoryDepositConfig { @JsonProperty("mapping") private StatusMapping statusMapping; + @JsonProperty("funder-mapping") + private Map funderMapping = new HashMap<>(); + public DepositProcessing getDepositProcessing() { return depositProcessing; } @@ -44,6 +50,14 @@ public void setStatusMapping(StatusMapping statusMapping) { this.statusMapping = statusMapping; } + public Map getFunderMapping() { + return funderMapping; + } + + public void setFunderMapping(Map funderMapping) { + this.funderMapping = funderMapping; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -72,6 +86,6 @@ public int hashCode() { @Override public String toString() { return "RepositoryDepositConfig{" + "depositProcessing=" + depositProcessing + - ", statusMapping=" + statusMapping + '}'; + ", statusMapping=" + statusMapping + " funderMapping=" + funderMapping + "}"; } } diff --git a/pass-deposit-services/deposit-core/src/main/resources/repositories.json b/pass-deposit-services/deposit-core/src/main/resources/repositories.json index c386d9e0..c53c0970 100644 --- a/pass-deposit-services/deposit-core/src/main/resources/repositories.json +++ b/pass-deposit-services/deposit-core/src/main/resources/repositories.json @@ -41,6 +41,20 @@ "createIfMissing": "true", "overwrite": "true" } + }, + "deposit-config": { + "funder-mapping": { + "nih": "", + "ahrq": "", + "aspr": "", + "cdc": "", + "epa": "", + "fda": "", + "hhmi": "", + "nasa": "", + "nist": "", + "va": "" + } } }, "BagIt": { From 7ea647df94a556610388db0582cd17dcfd5a7768 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 20 Dec 2023 17:19:18 -0500 Subject: [PATCH 17/35] Cleanup --- .../deposit/config/repository/RepositoryDepositConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java index d6dfb13b..97628ca5 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java @@ -15,11 +15,11 @@ */ package org.eclipse.pass.deposit.config.repository; -import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.HashMap; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonProperty; + /** * @author Elliot Metsger (emetsger@jhu.edu) */ From 30b755674f795e22619a43604d959eab66b79004 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 3 Jan 2024 15:34:44 -0500 Subject: [PATCH 18/35] Add funder-mapping to repositories.json --- .../src/main/resources/repositories.json | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/resources/repositories.json b/pass-deposit-services/deposit-core/src/main/resources/repositories.json index c53c0970..d5cabadb 100644 --- a/pass-deposit-services/deposit-core/src/main/resources/repositories.json +++ b/pass-deposit-services/deposit-core/src/main/resources/repositories.json @@ -44,16 +44,41 @@ }, "deposit-config": { "funder-mapping": { - "nih": "", - "ahrq": "", - "aspr": "", - "cdc": "", - "epa": "", - "fda": "", - "hhmi": "", - "nasa": "", - "nist": "", - "va": "" + "johnshopkins.edu:funder:300032": "ahqr", + "johnshopkins.edu:funder:300293": "cdc", + "johnshopkins.edu:funder:300859": "cdc", + "johnshopkins.edu:funder:301459": "va", + "johnshopkins.edu:funder:300453": "epa", + "johnshopkins.edu:funder:303444": "hhmi", + "johnshopkins.edu:funder:300484": "nih", + "johnshopkins.edu:funder:300865": "nih", + "johnshopkins.edu:funder:300869": "nih", + "johnshopkins.edu:funder:300866": "nih", + "johnshopkins.edu:funder:308302": "nih", + "johnshopkins.edu:funder:305950": "nih", + "johnshopkins.edu:funder:302727": "nih", + "johnshopkins.edu:funder:300863": "nih", + "johnshopkins.edu:funder:300874": "nih", + "johnshopkins.edu:funder:300861": "nih", + "johnshopkins.edu:funder:300842": "nih", + "johnshopkins.edu:funder:303587": "nih", + "johnshopkins.edu:funder:303586": "nih", + "johnshopkins.edu:funder:306099": "nih", + "johnshopkins.edu:funder:300858": "nih", + "johnshopkins.edu:funder:301479": "nih", + "johnshopkins.edu:funder:300870": "nih", + "johnshopkins.edu:funder:303589": "nih", + "johnshopkins.edu:funder:300860": "nih", + "johnshopkins.edu:funder:300852": "nih", + "johnshopkins.edu:funder:302822": "nih", + "johnshopkins.edu:funder:302592": "nih", + "johnshopkins.edu:funder:303585": "nih", + "johnshopkins.edu:funder:303574": "nih", + "johnshopkins.edu:funder:300867": "nih", + "johnshopkins.edu:funder:303580": "nih", + "johnshopkins.edu:funder:301978": "nih", + "johnshopkins.edu:funder:305204": "aspr", + "johnshopkins.edu:funder:303395": "fda" } } }, From 46ba7caa7d9fd7db7b5dccf658043ff8ef35791f Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Mon, 8 Jan 2024 16:15:26 -0500 Subject: [PATCH 19/35] Add localKey to metadata model --- .../deposit/builder/DepositSubmissionMapper.java | 2 ++ .../provider/nihms/NihmsMetadataSerializer.java | 7 +++++++ .../nihms/NihmsMetadataSerializerTest.java | 14 ++++++++++++-- .../pass/deposit/model/DepositMetadata.java | 9 +++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java index b27dc21b..58db8030 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java @@ -32,6 +32,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.eclipse.pass.deposit.DepositServiceRuntimeException; +import org.eclipse.pass.deposit.config.repository.RepositoryConfig; import org.eclipse.pass.deposit.model.DepositFile; import org.eclipse.pass.deposit.model.DepositFileType; import org.eclipse.pass.deposit.model.DepositManifest; @@ -196,6 +197,7 @@ private DepositMetadata.Grant createGrant(User userEntity, Grant grantEntity) { grant.setGrantId(grantEntity.getAwardNumber()); grant.setGrantPi(createPerson(userEntity, DepositMetadata.PERSON_TYPE.pi)); grant.setFunder(grantEntity.getPrimaryFunder().getName()); + grant.setFunderLocalKey(grantEntity.getLocalKey()); return grant; } diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 247100d3..8b937b7b 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -20,7 +20,9 @@ import java.time.LocalDate; import java.time.Period; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; @@ -49,10 +51,15 @@ public class NihmsMetadataSerializer implements StreamingSerializer { private static final Logger LOG = LoggerFactory.getLogger(NihmsMetadataSerializer.class); private DepositMetadata metadata; + private Map funderMapping = new HashMap<>(); public NihmsMetadataSerializer(DepositMetadata metadata) { this.metadata = metadata; } + public NihmsMetadataSerializer(DepositMetadata metadata, Map funderMapping) { + this.metadata = metadata; + this.funderMapping = funderMapping; + } @Override public SizedStream serialize() { diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java index eb0bafd7..f3f0bce2 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java @@ -146,16 +146,26 @@ public void setup() throws Exception { DepositMetadata.Grant grant1 = new DepositMetadata.Grant(); grant1.setGrantId("R0123456789"); - grant1.setFunder("nih"); + grant1.setFunder("FOGARTY INTERNATIONAL CENTER"); + grant1.setFunderLocalKey("johnshopkins.edu:funder:300484"); grant1.setGrantPi(person1); grantList.add(grant1); DepositMetadata.Grant grant2 = new DepositMetadata.Grant(); grant2.setGrantId("R0123456000"); - grant2.setFunder("nih"); + grant2.setFunder("CENTERS FOR DISEASE CONTROL"); + grant2.setFunderLocalKey("johnshopkins.edu:funder:300293"); grant2.setGrantPi(person2); grantList.add(grant2); + DepositMetadata.Grant grant3 = new DepositMetadata.Grant(); + grant2.setGrantId("R0123456897"); + grant2.setFunder("MYRIAD GENETICS INC"); + grant2.setFunderLocalKey("johnshopkins.edu:funder:301885"); + grant2.setGrantPi(person2); + grantList.add(grant2); + + metadata.setJournalMetadata(journal); metadata.setManuscriptMetadata(manuscript); metadata.setPersons(personList); diff --git a/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java b/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java index e3be938f..f95ff630 100644 --- a/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java +++ b/pass-deposit-services/deposit-model/src/main/java/org/eclipse/pass/deposit/model/DepositMetadata.java @@ -545,6 +545,7 @@ public static class Grant { //this is the grant awardNumber in PASS String grantId; String funder; + String funderLocalKey; Person grantPi; public String getGrantId() { @@ -563,6 +564,14 @@ public void setFunder(String funder) { this.funder = funder; } + public String getFunderLocalKey() { + return funderLocalKey; + } + + public void setFunderLocalKey(String funderLocalKey) { + this.funderLocalKey = funderLocalKey; + } + public Person getGrantPi() { return grantPi; } From fc63e58755099de92f96b36dc0d16cecab2fed98 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 9 Jan 2024 09:39:45 -0500 Subject: [PATCH 20/35] Add getRepositoryConfig to Packager --- .../main/java/org/eclipse/pass/deposit/model/Packager.java | 5 +++++ .../java/org/eclipse/pass/deposit/service/DepositTask.java | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java index ef535042..8d60b1f7 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java @@ -25,6 +25,7 @@ import org.eclipse.pass.deposit.assembler.PackageOptions; import org.eclipse.pass.deposit.config.repository.AssemblerOptions; import org.eclipse.pass.deposit.config.repository.RepositoryConfig; +import org.eclipse.pass.deposit.config.repository.RepositoryDepositConfig; import org.eclipse.pass.deposit.service.DepositTask; import org.eclipse.pass.deposit.status.DepositStatusProcessor; import org.eclipse.pass.deposit.transport.Transport; @@ -133,6 +134,10 @@ public Map getConfiguration() { return repositoryConfig.getTransportConfig().getProtocolBinding().asPropertiesMap(); } + public RepositoryDepositConfig getRepositoryConfig() { + return repositoryConfig.getRepositoryDepositConfig(); + } + /** * The {@link DepositStatusProcessor}, may be {@code null}. * diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java index a47a7565..eecf628b 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java @@ -424,8 +424,15 @@ static Function performDeposit(DepositWorkerContext try { packager = dc.packager(); + + packageStream = packager.getAssembler().assemble( dc.depositSubmission(), packager.getAssemblerOptions()); + //TODO add options for the packager.getRepoConfig + /* + packageStream = packager.getAssembler().assemble( + dc.depositSubmission(), packager.getAssemblerOptions(), packager.getRepositoryOptions()); + */ packagerConfig = packager.getConfiguration(); } catch (Exception e) { throw new RuntimeException("Error resolving a Packager or Packager configuration for " + From 22d0cde4e2acbe667548bd13d472012551dc7f61 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 9 Jan 2024 10:06:41 -0500 Subject: [PATCH 21/35] Remove getRepositoryConfig to Packager --- .../main/java/org/eclipse/pass/deposit/model/Packager.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java index 8d60b1f7..2e295e87 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java @@ -134,10 +134,6 @@ public Map getConfiguration() { return repositoryConfig.getTransportConfig().getProtocolBinding().asPropertiesMap(); } - public RepositoryDepositConfig getRepositoryConfig() { - return repositoryConfig.getRepositoryDepositConfig(); - } - /** * The {@link DepositStatusProcessor}, may be {@code null}. * From 4eadd005a1e8a63bb8cf245d274d36d9a7be2c17 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 9 Jan 2024 15:05:46 -0500 Subject: [PATCH 22/35] Move funder mapping to the assembler options --- .../src/main/resources/repositories.json | 78 +++++++++---------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/resources/repositories.json b/pass-deposit-services/deposit-core/src/main/resources/repositories.json index d5cabadb..7166232d 100644 --- a/pass-deposit-services/deposit-core/src/main/resources/repositories.json +++ b/pass-deposit-services/deposit-core/src/main/resources/repositories.json @@ -31,7 +31,44 @@ "algorithms": [ "sha512", "md5" - ] + ], + "funder-mapping": { + "johnshopkins.edu:funder:300032": "ahqr", + "johnshopkins.edu:funder:300293": "cdc", + "johnshopkins.edu:funder:300859": "cdc", + "johnshopkins.edu:funder:301459": "va", + "johnshopkins.edu:funder:300453": "epa", + "johnshopkins.edu:funder:303444": "hhmi", + "johnshopkins.edu:funder:300484": "nih", + "johnshopkins.edu:funder:300865": "nih", + "johnshopkins.edu:funder:300869": "nih", + "johnshopkins.edu:funder:300866": "nih", + "johnshopkins.edu:funder:308302": "nih", + "johnshopkins.edu:funder:305950": "nih", + "johnshopkins.edu:funder:302727": "nih", + "johnshopkins.edu:funder:300863": "nih", + "johnshopkins.edu:funder:300874": "nih", + "johnshopkins.edu:funder:300861": "nih", + "johnshopkins.edu:funder:300842": "nih", + "johnshopkins.edu:funder:303587": "nih", + "johnshopkins.edu:funder:303586": "nih", + "johnshopkins.edu:funder:306099": "nih", + "johnshopkins.edu:funder:300858": "nih", + "johnshopkins.edu:funder:301479": "nih", + "johnshopkins.edu:funder:300870": "nih", + "johnshopkins.edu:funder:303589": "nih", + "johnshopkins.edu:funder:300860": "nih", + "johnshopkins.edu:funder:300852": "nih", + "johnshopkins.edu:funder:302822": "nih", + "johnshopkins.edu:funder:302592": "nih", + "johnshopkins.edu:funder:303585": "nih", + "johnshopkins.edu:funder:303574": "nih", + "johnshopkins.edu:funder:300867": "nih", + "johnshopkins.edu:funder:303580": "nih", + "johnshopkins.edu:funder:301978": "nih", + "johnshopkins.edu:funder:305204": "aspr", + "johnshopkins.edu:funder:303395": "fda" + } } }, "transport-config": { @@ -41,45 +78,6 @@ "createIfMissing": "true", "overwrite": "true" } - }, - "deposit-config": { - "funder-mapping": { - "johnshopkins.edu:funder:300032": "ahqr", - "johnshopkins.edu:funder:300293": "cdc", - "johnshopkins.edu:funder:300859": "cdc", - "johnshopkins.edu:funder:301459": "va", - "johnshopkins.edu:funder:300453": "epa", - "johnshopkins.edu:funder:303444": "hhmi", - "johnshopkins.edu:funder:300484": "nih", - "johnshopkins.edu:funder:300865": "nih", - "johnshopkins.edu:funder:300869": "nih", - "johnshopkins.edu:funder:300866": "nih", - "johnshopkins.edu:funder:308302": "nih", - "johnshopkins.edu:funder:305950": "nih", - "johnshopkins.edu:funder:302727": "nih", - "johnshopkins.edu:funder:300863": "nih", - "johnshopkins.edu:funder:300874": "nih", - "johnshopkins.edu:funder:300861": "nih", - "johnshopkins.edu:funder:300842": "nih", - "johnshopkins.edu:funder:303587": "nih", - "johnshopkins.edu:funder:303586": "nih", - "johnshopkins.edu:funder:306099": "nih", - "johnshopkins.edu:funder:300858": "nih", - "johnshopkins.edu:funder:301479": "nih", - "johnshopkins.edu:funder:300870": "nih", - "johnshopkins.edu:funder:303589": "nih", - "johnshopkins.edu:funder:300860": "nih", - "johnshopkins.edu:funder:300852": "nih", - "johnshopkins.edu:funder:302822": "nih", - "johnshopkins.edu:funder:302592": "nih", - "johnshopkins.edu:funder:303585": "nih", - "johnshopkins.edu:funder:303574": "nih", - "johnshopkins.edu:funder:300867": "nih", - "johnshopkins.edu:funder:303580": "nih", - "johnshopkins.edu:funder:301978": "nih", - "johnshopkins.edu:funder:305204": "aspr", - "johnshopkins.edu:funder:303395": "fda" - } } }, "BagIt": { From 791fd0f90d9443e65e20a7d3ef9c862e7a7a7d3d Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 9 Jan 2024 16:40:45 -0500 Subject: [PATCH 23/35] Add check for funderKey in NihmsMetadataSerializer --- .../nihms/NihmsMetadataSerializer.java | 75 ++++++++++++------- .../provider/nihms/NihmsPackageProvider.java | 2 +- 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 8b937b7b..80465eed 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; @@ -52,13 +53,14 @@ public class NihmsMetadataSerializer implements StreamingSerializer { private DepositMetadata metadata; private Map funderMapping = new HashMap<>(); + private static final String FUNDER_MAPPING_KEY = "funder-mapping"; public NihmsMetadataSerializer(DepositMetadata metadata) { this.metadata = metadata; } - public NihmsMetadataSerializer(DepositMetadata metadata, Map funderMapping) { + public NihmsMetadataSerializer(DepositMetadata metadata, Map packageOptions) { this.metadata = metadata; - this.funderMapping = funderMapping; + this.funderMapping = (Map) packageOptions.get(FUNDER_MAPPING_KEY); } @Override @@ -244,38 +246,57 @@ private void addGrants(Document doc, Element root) { List grantsList = metadata.getGrantsMetadata(); if (grantsList != null && !grantsList.isEmpty()) { - //TODO: add check, if funder is not in the list of accepted funders then do not create grant element - //TODO: at least 1 accepted funder for a grant is required to create parent element grants - Element grantsElement = doc.createElement("grants"); - root.appendChild(grantsElement); - - for (DepositMetadata.Grant grant : grantsList) { - if (StringUtils.isNotBlank(grant.getFunder())) { - Element grantElement = doc.createElement("grant"); - grantsElement.appendChild(grantElement); - grantElement.setAttribute("funder", grant.getFunder()); - - if (StringUtils.isNotBlank(grant.getGrantId())) { - grantElement.setAttribute("id", grant.getGrantId()); - } + List funderKeys = grantsList.stream() + .map(DepositMetadata.Grant::getFunderLocalKey) // Map each Grant to its funderKey + .collect(Collectors.toList()); - DepositMetadata.Person pi = grant.getGrantPi(); - if (pi != null) { - Element piElement = doc.createElement("PI"); - grantElement.appendChild(piElement); + if(funderLocalKeyExists(funderKeys)) { - if (StringUtils.isNotBlank(pi.getFirstName())) { - piElement.setAttribute("fname", pi.getFirstName()); - } - if (StringUtils.isNotBlank(pi.getLastName())) { - piElement.setAttribute("lname", pi.getLastName()); + Element grantsElement = doc.createElement("grants"); + root.appendChild(grantsElement); + + for (DepositMetadata.Grant grant : grantsList) { + if (StringUtils.isNotBlank(grant.getFunder())) { + Element grantElement = doc.createElement("grant"); + grantsElement.appendChild(grantElement); + grantElement.setAttribute("funder", grant.getFunder()); + + if (StringUtils.isNotBlank(grant.getGrantId())) { + grantElement.setAttribute("id", grant.getGrantId()); } - if (StringUtils.isNotBlank(pi.getEmail())) { - piElement.setAttribute("email", pi.getEmail()); + + DepositMetadata.Person pi = grant.getGrantPi(); + if (pi != null) { + Element piElement = doc.createElement("PI"); + grantElement.appendChild(piElement); + + if (StringUtils.isNotBlank(pi.getFirstName())) { + piElement.setAttribute("fname", pi.getFirstName()); + } + if (StringUtils.isNotBlank(pi.getLastName())) { + piElement.setAttribute("lname", pi.getLastName()); + } + if (StringUtils.isNotBlank(pi.getEmail())) { + piElement.setAttribute("email", pi.getEmail()); + } } } } } } } + + /** + * Used to determine if one of the grants localKeys exists in the list of accepted nihms funders + * + * @return True if at least one nihms funderKey exists in the metadata grant list + */ + private boolean funderLocalKeyExists(List metaDataGrantFunderKeys){ + for (String key : metaDataGrantFunderKeys){ + if (funderMapping.containsKey(key)){ + return true; + } + } + return false; + } } diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageProvider.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageProvider.java index 008851a2..c32d0402 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageProvider.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageProvider.java @@ -74,7 +74,7 @@ public static String getNonCollidingFilename(String fileName, DepositFileType fi public void start(DepositSubmission submission, List custodialResources, Map packageOptions) { manifestSerializer = new NihmsManifestSerializer(submission.getManifest()); - metadataSerializer = new NihmsMetadataSerializer(submission.getMetadata()); + metadataSerializer = new NihmsMetadataSerializer(submission.getMetadata(), packageOptions); } /** From 03e8ec53afa82daf689ca2d1b58c34fa938365ab Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Tue, 9 Jan 2024 16:45:42 -0500 Subject: [PATCH 24/35] Cleanup --- .../nihms/NihmsMetadataSerializer.java | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 80465eed..7fb72271 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -198,7 +198,7 @@ private void addManuscriptTitle(Document doc, Element root) { private void addContacts(Document doc, Element root) { List persons = metadata.getPersons(); - if (persons != null && persons.size() > 0) { + if (persons != null && !persons.isEmpty()) { Element contacts = doc.createElement("contacts"); root.appendChild(contacts); @@ -247,37 +247,40 @@ private void addGrants(Document doc, Element root) { if (grantsList != null && !grantsList.isEmpty()) { List funderKeys = grantsList.stream() - .map(DepositMetadata.Grant::getFunderLocalKey) // Map each Grant to its funderKey - .collect(Collectors.toList()); + .map(DepositMetadata.Grant::getFunderLocalKey) + .toList(); + //Only create the top level grant element if at least one of the keys is associated with nihms if(funderLocalKeyExists(funderKeys)) { Element grantsElement = doc.createElement("grants"); root.appendChild(grantsElement); for (DepositMetadata.Grant grant : grantsList) { - if (StringUtils.isNotBlank(grant.getFunder())) { - Element grantElement = doc.createElement("grant"); - grantsElement.appendChild(grantElement); - grantElement.setAttribute("funder", grant.getFunder()); - - if (StringUtils.isNotBlank(grant.getGrantId())) { - grantElement.setAttribute("id", grant.getGrantId()); - } - - DepositMetadata.Person pi = grant.getGrantPi(); - if (pi != null) { - Element piElement = doc.createElement("PI"); - grantElement.appendChild(piElement); - - if (StringUtils.isNotBlank(pi.getFirstName())) { - piElement.setAttribute("fname", pi.getFirstName()); - } - if (StringUtils.isNotBlank(pi.getLastName())) { - piElement.setAttribute("lname", pi.getLastName()); + if(funderMapping.containsKey(grant.getFunderLocalKey())) { + if (StringUtils.isNotBlank(grant.getFunder())) { + Element grantElement = doc.createElement("grant"); + grantsElement.appendChild(grantElement); + grantElement.setAttribute("funder", funderMapping.get(grant.getFunderLocalKey())); + + if (StringUtils.isNotBlank(grant.getGrantId())) { + grantElement.setAttribute("id", grant.getGrantId()); } - if (StringUtils.isNotBlank(pi.getEmail())) { - piElement.setAttribute("email", pi.getEmail()); + + DepositMetadata.Person pi = grant.getGrantPi(); + if (pi != null) { + Element piElement = doc.createElement("PI"); + grantElement.appendChild(piElement); + + if (StringUtils.isNotBlank(pi.getFirstName())) { + piElement.setAttribute("fname", pi.getFirstName()); + } + if (StringUtils.isNotBlank(pi.getLastName())) { + piElement.setAttribute("lname", pi.getLastName()); + } + if (StringUtils.isNotBlank(pi.getEmail())) { + piElement.setAttribute("email", pi.getEmail()); + } } } } From ee45cfebf67d0110d1159f1e7c75ca2fb14604c1 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 09:41:31 -0500 Subject: [PATCH 25/35] Add package option and funderMapping check --- .../pass/deposit/assembler/PackageOptions.java | 7 +++++++ .../provider/nihms/NihmsMetadataSerializer.java | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pass-deposit-services/assembler-api/src/main/java/org/eclipse/pass/deposit/assembler/PackageOptions.java b/pass-deposit-services/assembler-api/src/main/java/org/eclipse/pass/deposit/assembler/PackageOptions.java index 95f7a019..052a8e55 100644 --- a/pass-deposit-services/assembler-api/src/main/java/org/eclipse/pass/deposit/assembler/PackageOptions.java +++ b/pass-deposit-services/assembler-api/src/main/java/org/eclipse/pass/deposit/assembler/PackageOptions.java @@ -98,4 +98,11 @@ enum OPTS { } + /** + * Funder Mapping for Nihms Packages + */ + interface FunderMapping { + String KEY = "FUNDER-MAPPING"; + } + } diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 7fb72271..479605a6 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -34,6 +34,7 @@ import javax.xml.transform.stream.StreamResult; import org.apache.commons.lang3.StringUtils; +import org.eclipse.pass.deposit.assembler.PackageOptions; import org.eclipse.pass.deposit.assembler.SizedStream; import org.eclipse.pass.deposit.model.DepositMetadata; import org.eclipse.pass.deposit.model.JournalPublicationType; @@ -53,14 +54,21 @@ public class NihmsMetadataSerializer implements StreamingSerializer { private DepositMetadata metadata; private Map funderMapping = new HashMap<>(); - private static final String FUNDER_MAPPING_KEY = "funder-mapping"; public NihmsMetadataSerializer(DepositMetadata metadata) { this.metadata = metadata; } public NihmsMetadataSerializer(DepositMetadata metadata, Map packageOptions) { this.metadata = metadata; - this.funderMapping = (Map) packageOptions.get(FUNDER_MAPPING_KEY); + Object funderMappingObj = packageOptions.get(PackageOptions.FunderMapping.KEY); + if (funderMappingObj instanceof Map) { + this.funderMapping = ((Map) funderMappingObj).entrySet().stream() + .collect(Collectors.toMap( + e -> (String) e.getKey(), + e -> (String) e.getValue())); + } else { + throw new ClassCastException("FunderMapping is not a Map"); + } } @Override @@ -261,6 +269,8 @@ private void addGrants(Document doc, Element root) { if (StringUtils.isNotBlank(grant.getFunder())) { Element grantElement = doc.createElement("grant"); grantsElement.appendChild(grantElement); + + //use the nihms abbreviations for funders, the accepted list is in the nihms DTD grantElement.setAttribute("funder", funderMapping.get(grant.getFunderLocalKey())); if (StringUtils.isNotBlank(grant.getGrantId())) { From bbd1a52f2947d344b4f705add2a64d362341d5b5 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 09:54:09 -0500 Subject: [PATCH 26/35] Update test data and IT setup --- .../pass/deposit/provider/nihms/NihmsAssemblerIT.java | 6 ++++++ .../src/test/resources/submissions/sample1.json | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index 8ec9bf33..e08e3a8f 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -44,9 +44,11 @@ import org.apache.commons.io.LineIterator; import org.eclipse.pass.deposit.assembler.AbstractAssembler; import org.eclipse.pass.deposit.assembler.AbstractAssemblerIT; +import org.eclipse.pass.deposit.assembler.PackageOptions; import org.eclipse.pass.deposit.assembler.PackageOptions.Archive; import org.eclipse.pass.deposit.assembler.PackageOptions.Compression; import org.eclipse.pass.deposit.assembler.PackageOptions.Spec; +import org.eclipse.pass.deposit.assembler.PackageOptions.FunderMapping; import org.eclipse.pass.deposit.assembler.PackageStream; import org.eclipse.pass.deposit.model.DepositFile; import org.eclipse.pass.deposit.model.DepositFileType; @@ -84,6 +86,10 @@ protected Map getOptions() { put(Spec.KEY, SPEC_NIHMS_NATIVE_2022_05); put(Archive.KEY, Archive.OPTS.TAR); put(Compression.KEY, Compression.OPTS.GZIP); + HashMap funderMapping = (HashMap) + Map.of("johnshopkins.edu:funder:300293", "cdc", + "johnshopkins.edu:funder:300484", "nih"); + put(FunderMapping.KEY, funderMapping); } }; } diff --git a/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json b/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json index 70d649a3..9380a16b 100644 --- a/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json +++ b/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json @@ -136,7 +136,7 @@ { "name": "National Eye Institute", "url": "http://example.com/eyeguys", - "localKey": "aabbcc", + "localKey": "johnshopkins.edu:funder:300484", "policy": "10", "id": "11", "@type": "Funder" @@ -144,7 +144,7 @@ { "name": "International Eye Institute", "url": "http://example.com/othereyeguys", - "localKey": "ddeeff", + "localKey": "johnshopkins.edu:funder:300293", "policy": "10", "id": "12", "@type": "Funder" From c5271b5fdd6dfea851741ec6e6194fbe88c7ed80 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 13:46:29 -0500 Subject: [PATCH 27/35] Fix funderKey --- .../eclipse/pass/deposit/builder/DepositSubmissionMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java index 58db8030..0daaca5b 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java @@ -197,7 +197,7 @@ private DepositMetadata.Grant createGrant(User userEntity, Grant grantEntity) { grant.setGrantId(grantEntity.getAwardNumber()); grant.setGrantPi(createPerson(userEntity, DepositMetadata.PERSON_TYPE.pi)); grant.setFunder(grantEntity.getPrimaryFunder().getName()); - grant.setFunderLocalKey(grantEntity.getLocalKey()); + grant.setFunderLocalKey(grantEntity.getPrimaryFunder().getLocalKey()); return grant; } From 59bf9aae616306185dbca882bae9f90e09c74032 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 15:30:04 -0500 Subject: [PATCH 28/35] Fix NihmsAssemblerIT - Update sample data --- .../provider/nihms/NihmsAssemblerIT.java | 21 +++++++++++++++---- .../test/resources/submissions/sample1.json | 4 ++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index e08e3a8f..067e498f 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -38,6 +39,10 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -81,14 +86,14 @@ public void setUp() throws Exception { @Override protected Map getOptions() { + Map funderMapping = + Map.of("johnshopkins.edu:funder:300293", "cdc", + "johnshopkins.edu:funder:300484", "nih"); return new HashMap<>() { { put(Spec.KEY, SPEC_NIHMS_NATIVE_2022_05); put(Archive.KEY, Archive.OPTS.TAR); put(Compression.KEY, Compression.OPTS.GZIP); - HashMap funderMapping = (HashMap) - Map.of("johnshopkins.edu:funder:300293", "cdc", - "johnshopkins.edu:funder:300484", "nih"); put(FunderMapping.KEY, funderMapping); } }; @@ -269,6 +274,14 @@ public void testPackageMetadata() throws Exception { @Test public void testPackageMetadataGrantFunder() throws Exception { Element root = initDom(); + //TODO - for testing, remove once done + TransformerFactory transformerFactory = TransformerFactory.newInstance(); + Transformer transformer = transformerFactory.newTransformer(); + StringWriter writer = new StringWriter(); + + transformer.transform(new DOMSource(root), new StreamResult(writer)); + + String rootAsString = writer.toString(); //Ensure that the grants in the metadata matches a Grant on the submission, Check the attributes of a grant on //submission against what is found in the metadata List grantElements = asList(root.getElementsByTagName("grant")); @@ -306,7 +319,7 @@ public void testPackageMetadataGrantFunder() throws Exception { //assert that the grant ID exists and is valid to the submission data assertEquals(subGrant.getGrantId(), metaDataGrant.get().getGrantId()); - assertEquals(subGrant.getFunder(), metaDataGrant.get().getFunder()); + assertTrue(List.of("nih", "cdc").contains(metaDataGrant.get().getFunder())); assertEquals(subGrant.getGrantPi().getFirstName(), metaDataGrant.get().getGrantPi().getFirstName()); assertEquals(subGrant.getGrantPi().getLastName(), metaDataGrant.get().getGrantPi().getLastName()); assertEquals(subGrant.getGrantPi().getEmail(), metaDataGrant.get().getGrantPi().getEmail()); diff --git a/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json b/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json index 9380a16b..fa0b43c9 100644 --- a/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json +++ b/pass-deposit-services/deposit-core/src/test/resources/submissions/sample1.json @@ -175,8 +175,8 @@ "awardDate": "2017-08-01T00:00:00.000Z", "startDate": "2017-09-01T00:00:00.000Z", "endDate": "2020-06-01T00:00:00.000Z", - "primaryFunder": "11", - "directFunder": "12", + "primaryFunder": "12", + "directFunder": "11", "pi": "2", "coPis": [ "3", From 2e1a53bbda3dc7c61926788e5b2456a3f3104591 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 15:40:52 -0500 Subject: [PATCH 29/35] Cleanup --- .../pass/deposit/builder/DepositSubmissionMapper.java | 1 - .../java/org/eclipse/pass/deposit/model/Packager.java | 1 - .../deposit/provider/nihms/NihmsMetadataSerializer.java | 9 +++++---- .../org/eclipse/pass/deposit/service/DepositTask.java | 6 ------ .../pass/deposit/provider/nihms/NihmsAssemblerIT.java | 3 +-- .../provider/nihms/NihmsMetadataSerializerTest.java | 1 - 6 files changed, 6 insertions(+), 15 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java index 0daaca5b..c59b7b69 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/builder/DepositSubmissionMapper.java @@ -32,7 +32,6 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.eclipse.pass.deposit.DepositServiceRuntimeException; -import org.eclipse.pass.deposit.config.repository.RepositoryConfig; import org.eclipse.pass.deposit.model.DepositFile; import org.eclipse.pass.deposit.model.DepositFileType; import org.eclipse.pass.deposit.model.DepositManifest; diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java index 2e295e87..ef535042 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/model/Packager.java @@ -25,7 +25,6 @@ import org.eclipse.pass.deposit.assembler.PackageOptions; import org.eclipse.pass.deposit.config.repository.AssemblerOptions; import org.eclipse.pass.deposit.config.repository.RepositoryConfig; -import org.eclipse.pass.deposit.config.repository.RepositoryDepositConfig; import org.eclipse.pass.deposit.service.DepositTask; import org.eclipse.pass.deposit.status.DepositStatusProcessor; import org.eclipse.pass.deposit.transport.Transport; diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 479605a6..a21e9fab 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -58,6 +58,7 @@ public class NihmsMetadataSerializer implements StreamingSerializer { public NihmsMetadataSerializer(DepositMetadata metadata) { this.metadata = metadata; } + public NihmsMetadataSerializer(DepositMetadata metadata, Map packageOptions) { this.metadata = metadata; Object funderMappingObj = packageOptions.get(PackageOptions.FunderMapping.KEY); @@ -259,7 +260,7 @@ private void addGrants(Document doc, Element root) { .toList(); //Only create the top level grant element if at least one of the keys is associated with nihms - if(funderLocalKeyExists(funderKeys)) { + if (funderLocalKeyExists(funderKeys)) { Element grantsElement = doc.createElement("grants"); root.appendChild(grantsElement); @@ -304,9 +305,9 @@ private void addGrants(Document doc, Element root) { * * @return True if at least one nihms funderKey exists in the metadata grant list */ - private boolean funderLocalKeyExists(List metaDataGrantFunderKeys){ - for (String key : metaDataGrantFunderKeys){ - if (funderMapping.containsKey(key)){ + private boolean funderLocalKeyExists(List metaDataGrantFunderKeys) { + for (String key : metaDataGrantFunderKeys) { + if (funderMapping.containsKey(key)) { return true; } } diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java index eecf628b..16ec48f4 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositTask.java @@ -425,14 +425,8 @@ static Function performDeposit(DepositWorkerContext try { packager = dc.packager(); - packageStream = packager.getAssembler().assemble( dc.depositSubmission(), packager.getAssemblerOptions()); - //TODO add options for the packager.getRepoConfig - /* - packageStream = packager.getAssembler().assemble( - dc.depositSubmission(), packager.getAssemblerOptions(), packager.getRepositoryOptions()); - */ packagerConfig = packager.getConfiguration(); } catch (Exception e) { throw new RuntimeException("Error resolving a Packager or Packager configuration for " + diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java index 067e498f..f56cfe77 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsAssemblerIT.java @@ -49,11 +49,10 @@ import org.apache.commons.io.LineIterator; import org.eclipse.pass.deposit.assembler.AbstractAssembler; import org.eclipse.pass.deposit.assembler.AbstractAssemblerIT; -import org.eclipse.pass.deposit.assembler.PackageOptions; import org.eclipse.pass.deposit.assembler.PackageOptions.Archive; import org.eclipse.pass.deposit.assembler.PackageOptions.Compression; -import org.eclipse.pass.deposit.assembler.PackageOptions.Spec; import org.eclipse.pass.deposit.assembler.PackageOptions.FunderMapping; +import org.eclipse.pass.deposit.assembler.PackageOptions.Spec; import org.eclipse.pass.deposit.assembler.PackageStream; import org.eclipse.pass.deposit.model.DepositFile; import org.eclipse.pass.deposit.model.DepositFileType; diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java index f3f0bce2..5e744b75 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java @@ -165,7 +165,6 @@ public void setup() throws Exception { grant2.setGrantPi(person2); grantList.add(grant2); - metadata.setJournalMetadata(journal); metadata.setManuscriptMetadata(manuscript); metadata.setPersons(personList); From f3ff1ad037c0693263e531ed2edd1a85f4387aef Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 15:44:08 -0500 Subject: [PATCH 30/35] Cleanup --- .../pass/deposit/provider/nihms/NihmsMetadataSerializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index a21e9fab..badaa247 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -266,7 +266,7 @@ private void addGrants(Document doc, Element root) { root.appendChild(grantsElement); for (DepositMetadata.Grant grant : grantsList) { - if(funderMapping.containsKey(grant.getFunderLocalKey())) { + if (funderMapping.containsKey(grant.getFunderLocalKey())) { if (StringUtils.isNotBlank(grant.getFunder())) { Element grantElement = doc.createElement("grant"); grantsElement.appendChild(grantElement); From 13ae074bacfa4868fc64d9fc3da01d460274b784 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 16:00:54 -0500 Subject: [PATCH 31/35] Update NihmsMetadataSerializer constructor --- .../nihms/NihmsMetadataSerializer.java | 4 ---- .../nihms/NihmsMetadataSerializerTest.java | 18 ++++++++++++++---- .../provider/nihms/NihmsPackageVerifier.java | 3 ++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index badaa247..3920207e 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -55,10 +55,6 @@ public class NihmsMetadataSerializer implements StreamingSerializer { private DepositMetadata metadata; private Map funderMapping = new HashMap<>(); - public NihmsMetadataSerializer(DepositMetadata metadata) { - this.metadata = metadata; - } - public NihmsMetadataSerializer(DepositMetadata metadata, Map packageOptions) { this.metadata = metadata; Object funderMappingObj = packageOptions.get(PackageOptions.FunderMapping.KEY); diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java index 5e744b75..0d0496fc 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java @@ -20,6 +20,7 @@ import static java.util.Spliterator.ORDERED; import static java.util.Spliterator.SIZED; import static java.util.stream.StreamSupport.stream; +import static org.eclipse.pass.deposit.provider.nihms.NihmsAssembler.SPEC_NIHMS_NATIVE_2022_05; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -35,6 +36,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Spliterators.AbstractSpliterator; import java.util.function.Consumer; import java.util.stream.Stream; @@ -45,6 +47,7 @@ import com.github.jknack.handlebars.internal.Files; import org.apache.commons.io.IOUtils; +import org.eclipse.pass.deposit.assembler.PackageOptions; import org.eclipse.pass.deposit.assembler.SizedStream; import org.eclipse.pass.deposit.model.DepositMetadata; import org.eclipse.pass.deposit.model.JournalPublicationType; @@ -71,6 +74,7 @@ public class NihmsMetadataSerializerTest { private Path tempDir; private static NihmsMetadataSerializer underTest; + private static HashMap packageOptions; private static final DepositMetadata metadata = new DepositMetadata(); @BeforeEach @@ -170,7 +174,13 @@ public void setup() throws Exception { metadata.setPersons(personList); metadata.setGrantsMetadata(grantList); - underTest = new NihmsMetadataSerializer(metadata); + Map funderMapping = + Map.of("johnshopkins.edu:funder:300293", "cdc", + "johnshopkins.edu:funder:300484", "nih"); + packageOptions = new HashMap<>(); + packageOptions.put(PackageOptions.FunderMapping.KEY, funderMapping); + + underTest = new NihmsMetadataSerializer(metadata, packageOptions); } @Test @@ -305,7 +315,7 @@ public void completeIssnPubtype() throws IOException, ParserConfigurationExcepti metadata.setJournalMetadata(journalMd); - underTest = new NihmsMetadataSerializer(metadata); + underTest = new NihmsMetadataSerializer(metadata, packageOptions); DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); Node doc = builder.parse(underTest.serialize().getInputStream()); @@ -340,7 +350,7 @@ public void incompleteIssnPubtypeNullIssn() throws IOException { metadata.setJournalMetadata(journalMd); - underTest = new NihmsMetadataSerializer(metadata); + underTest = new NihmsMetadataSerializer(metadata, packageOptions); assertFalse(IOUtils.toString(underTest.serialize().getInputStream(), UTF_8).contains("issn")); } @@ -362,7 +372,7 @@ public void incompleteIssnPubtypeNullPubType() throws IOException { metadata.setJournalMetadata(journalMd); - underTest = new NihmsMetadataSerializer(metadata); + underTest = new NihmsMetadataSerializer(metadata, packageOptions); assertFalse(IOUtils.toString(underTest.serialize().getInputStream(), UTF_8).contains("issn")); } diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageVerifier.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageVerifier.java index 249dfe62..b2584c30 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageVerifier.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageVerifier.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; +import java.util.HashMap; import java.util.Map; import org.apache.commons.io.IOUtils; @@ -65,7 +66,7 @@ public void verify(DepositSubmission submission, ExplodedPackage explodedPackage String expectedManifest = IOUtils.toString( new NihmsManifestSerializer(submission.getManifest()).serialize().getInputStream(), UTF_8); String expectedBulkMeta = IOUtils.toString( - new NihmsMetadataSerializer(submission.getMetadata()).serialize().getInputStream(), UTF_8); + new NihmsMetadataSerializer(submission.getMetadata(), map).serialize().getInputStream(), UTF_8); assertEquals(expectedManifest, IOUtils.toString(new FileInputStream(manifest), UTF_8)); assertEquals(expectedBulkMeta, IOUtils.toString(new FileInputStream(bulk_meta), UTF_8)); From 1a5d5b2378ff38f7da2651199963a0ba291c6467 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 16:02:22 -0500 Subject: [PATCH 32/35] Cleanup --- .../pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java | 1 - .../pass/deposit/provider/nihms/NihmsPackageVerifier.java | 1 - 2 files changed, 2 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java index 0d0496fc..7cd96f14 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializerTest.java @@ -20,7 +20,6 @@ import static java.util.Spliterator.ORDERED; import static java.util.Spliterator.SIZED; import static java.util.stream.StreamSupport.stream; -import static org.eclipse.pass.deposit.provider.nihms.NihmsAssembler.SPEC_NIHMS_NATIVE_2022_05; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageVerifier.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageVerifier.java index b2584c30..2f0a3687 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageVerifier.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsPackageVerifier.java @@ -27,7 +27,6 @@ import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; -import java.util.HashMap; import java.util.Map; import org.apache.commons.io.IOUtils; From d570c9bbf6326ba9b6e9f75d508579e5c3ae9613 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Wed, 10 Jan 2024 16:41:53 -0500 Subject: [PATCH 33/35] Update packageOptions in NihmsThreadedAssemblyIT --- .../pass/deposit/provider/nihms/NihmsThreadedAssemblyIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsThreadedAssemblyIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsThreadedAssemblyIT.java index 9ee2bbb5..1223dd2b 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsThreadedAssemblyIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/provider/nihms/NihmsThreadedAssemblyIT.java @@ -39,12 +39,16 @@ protected AbstractAssembler assemblerUnderTest() { @Override protected Map packageOptions() { + Map funderMapping = + Map.of("johnshopkins.edu:funder:300293", "cdc", + "johnshopkins.edu:funder:300484", "nih"); return new HashMap<>() { { put(PackageOptions.Spec.KEY, SPEC_NIHMS_NATIVE_2022_05); put(PackageOptions.Archive.KEY, PackageOptions.Archive.OPTS.TAR); put(PackageOptions.Compression.KEY, PackageOptions.Compression.OPTS.GZIP); put(PackageOptions.Checksum.KEY, singletonList(PackageOptions.Checksum.OPTS.SHA256)); + put(PackageOptions.FunderMapping.KEY, funderMapping); } }; } From b546743df5e42e6377f8f27161a02c69ffae8331 Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Fri, 12 Jan 2024 15:13:20 -0500 Subject: [PATCH 34/35] Cleanup --- .../repository/RepositoryDepositConfig.java | 16 +----------- .../nihms/NihmsMetadataSerializer.java | 25 ++++--------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java index 97628ca5..71176fca 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/repository/RepositoryDepositConfig.java @@ -15,9 +15,6 @@ */ package org.eclipse.pass.deposit.config.repository; -import java.util.HashMap; -import java.util.Map; - import com.fasterxml.jackson.annotation.JsonProperty; /** @@ -31,9 +28,6 @@ public class RepositoryDepositConfig { @JsonProperty("mapping") private StatusMapping statusMapping; - @JsonProperty("funder-mapping") - private Map funderMapping = new HashMap<>(); - public DepositProcessing getDepositProcessing() { return depositProcessing; } @@ -50,14 +44,6 @@ public void setStatusMapping(StatusMapping statusMapping) { this.statusMapping = statusMapping; } - public Map getFunderMapping() { - return funderMapping; - } - - public void setFunderMapping(Map funderMapping) { - this.funderMapping = funderMapping; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -86,6 +72,6 @@ public int hashCode() { @Override public String toString() { return "RepositoryDepositConfig{" + "depositProcessing=" + depositProcessing + - ", statusMapping=" + statusMapping + " funderMapping=" + funderMapping + "}"; + ", statusMapping=" + statusMapping + "}"; } } diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java index 3920207e..96fc2106 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/provider/nihms/NihmsMetadataSerializer.java @@ -20,7 +20,6 @@ import java.time.LocalDate; import java.time.Period; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -53,7 +52,7 @@ public class NihmsMetadataSerializer implements StreamingSerializer { private static final Logger LOG = LoggerFactory.getLogger(NihmsMetadataSerializer.class); private DepositMetadata metadata; - private Map funderMapping = new HashMap<>(); + private Map funderMapping; public NihmsMetadataSerializer(DepositMetadata metadata, Map packageOptions) { this.metadata = metadata; @@ -256,7 +255,7 @@ private void addGrants(Document doc, Element root) { .toList(); //Only create the top level grant element if at least one of the keys is associated with nihms - if (funderLocalKeyExists(funderKeys)) { + if (funderKeys.stream().anyMatch(funderMapping::containsKey)) { Element grantsElement = doc.createElement("grants"); root.appendChild(grantsElement); @@ -267,13 +266,13 @@ private void addGrants(Document doc, Element root) { Element grantElement = doc.createElement("grant"); grantsElement.appendChild(grantElement); - //use the nihms abbreviations for funders, the accepted list is in the nihms DTD - grantElement.setAttribute("funder", funderMapping.get(grant.getFunderLocalKey())); - if (StringUtils.isNotBlank(grant.getGrantId())) { grantElement.setAttribute("id", grant.getGrantId()); } + //use the nihms abbreviations for funders, the accepted list is in the nihms DTD + grantElement.setAttribute("funder", funderMapping.get(grant.getFunderLocalKey())); + DepositMetadata.Person pi = grant.getGrantPi(); if (pi != null) { Element piElement = doc.createElement("PI"); @@ -295,18 +294,4 @@ private void addGrants(Document doc, Element root) { } } } - - /** - * Used to determine if one of the grants localKeys exists in the list of accepted nihms funders - * - * @return True if at least one nihms funderKey exists in the metadata grant list - */ - private boolean funderLocalKeyExists(List metaDataGrantFunderKeys) { - for (String key : metaDataGrantFunderKeys) { - if (funderMapping.containsKey(key)) { - return true; - } - } - return false; - } } From 0c6a2e73375c201f924d7fdb1a0a4932d9bad6bd Mon Sep 17 00:00:00 2001 From: Timothy Sanders Date: Fri, 12 Jan 2024 15:56:50 -0500 Subject: [PATCH 35/35] Remove funder-mapping from respositories.json - This is institution specific. Has been moved to the test resources for testing JHU specific needs. --- .../src/main/resources/repositories.json | 39 +----- .../src/test/resources/repositories.json | 128 ++++++++++++++++++ 2 files changed, 129 insertions(+), 38 deletions(-) create mode 100644 pass-deposit-services/deposit-core/src/test/resources/repositories.json diff --git a/pass-deposit-services/deposit-core/src/main/resources/repositories.json b/pass-deposit-services/deposit-core/src/main/resources/repositories.json index 7166232d..c386d9e0 100644 --- a/pass-deposit-services/deposit-core/src/main/resources/repositories.json +++ b/pass-deposit-services/deposit-core/src/main/resources/repositories.json @@ -31,44 +31,7 @@ "algorithms": [ "sha512", "md5" - ], - "funder-mapping": { - "johnshopkins.edu:funder:300032": "ahqr", - "johnshopkins.edu:funder:300293": "cdc", - "johnshopkins.edu:funder:300859": "cdc", - "johnshopkins.edu:funder:301459": "va", - "johnshopkins.edu:funder:300453": "epa", - "johnshopkins.edu:funder:303444": "hhmi", - "johnshopkins.edu:funder:300484": "nih", - "johnshopkins.edu:funder:300865": "nih", - "johnshopkins.edu:funder:300869": "nih", - "johnshopkins.edu:funder:300866": "nih", - "johnshopkins.edu:funder:308302": "nih", - "johnshopkins.edu:funder:305950": "nih", - "johnshopkins.edu:funder:302727": "nih", - "johnshopkins.edu:funder:300863": "nih", - "johnshopkins.edu:funder:300874": "nih", - "johnshopkins.edu:funder:300861": "nih", - "johnshopkins.edu:funder:300842": "nih", - "johnshopkins.edu:funder:303587": "nih", - "johnshopkins.edu:funder:303586": "nih", - "johnshopkins.edu:funder:306099": "nih", - "johnshopkins.edu:funder:300858": "nih", - "johnshopkins.edu:funder:301479": "nih", - "johnshopkins.edu:funder:300870": "nih", - "johnshopkins.edu:funder:303589": "nih", - "johnshopkins.edu:funder:300860": "nih", - "johnshopkins.edu:funder:300852": "nih", - "johnshopkins.edu:funder:302822": "nih", - "johnshopkins.edu:funder:302592": "nih", - "johnshopkins.edu:funder:303585": "nih", - "johnshopkins.edu:funder:303574": "nih", - "johnshopkins.edu:funder:300867": "nih", - "johnshopkins.edu:funder:303580": "nih", - "johnshopkins.edu:funder:301978": "nih", - "johnshopkins.edu:funder:305204": "aspr", - "johnshopkins.edu:funder:303395": "fda" - } + ] } }, "transport-config": { diff --git a/pass-deposit-services/deposit-core/src/test/resources/repositories.json b/pass-deposit-services/deposit-core/src/test/resources/repositories.json new file mode 100644 index 00000000..7166232d --- /dev/null +++ b/pass-deposit-services/deposit-core/src/test/resources/repositories.json @@ -0,0 +1,128 @@ +{ + "JScholarship": { + "assembler": { + "specification": "simple", + "beanName": "simpleAssembler", + "options": { + "archive": "ZIP", + "compression": "NONE", + "algorithms": [ + "sha512", + "md5" + ] + } + }, + "transport-config": { + "protocol-binding": { + "protocol": "filesystem", + "baseDir": "target/packages", + "createIfMissing": "true", + "overwrite": "true" + } + } + }, + "PubMed Central": { + "assembler": { + "specification": "simple", + "beanName": "simpleAssembler", + "options": { + "archive": "ZIP", + "compression": "NONE", + "algorithms": [ + "sha512", + "md5" + ], + "funder-mapping": { + "johnshopkins.edu:funder:300032": "ahqr", + "johnshopkins.edu:funder:300293": "cdc", + "johnshopkins.edu:funder:300859": "cdc", + "johnshopkins.edu:funder:301459": "va", + "johnshopkins.edu:funder:300453": "epa", + "johnshopkins.edu:funder:303444": "hhmi", + "johnshopkins.edu:funder:300484": "nih", + "johnshopkins.edu:funder:300865": "nih", + "johnshopkins.edu:funder:300869": "nih", + "johnshopkins.edu:funder:300866": "nih", + "johnshopkins.edu:funder:308302": "nih", + "johnshopkins.edu:funder:305950": "nih", + "johnshopkins.edu:funder:302727": "nih", + "johnshopkins.edu:funder:300863": "nih", + "johnshopkins.edu:funder:300874": "nih", + "johnshopkins.edu:funder:300861": "nih", + "johnshopkins.edu:funder:300842": "nih", + "johnshopkins.edu:funder:303587": "nih", + "johnshopkins.edu:funder:303586": "nih", + "johnshopkins.edu:funder:306099": "nih", + "johnshopkins.edu:funder:300858": "nih", + "johnshopkins.edu:funder:301479": "nih", + "johnshopkins.edu:funder:300870": "nih", + "johnshopkins.edu:funder:303589": "nih", + "johnshopkins.edu:funder:300860": "nih", + "johnshopkins.edu:funder:300852": "nih", + "johnshopkins.edu:funder:302822": "nih", + "johnshopkins.edu:funder:302592": "nih", + "johnshopkins.edu:funder:303585": "nih", + "johnshopkins.edu:funder:303574": "nih", + "johnshopkins.edu:funder:300867": "nih", + "johnshopkins.edu:funder:303580": "nih", + "johnshopkins.edu:funder:301978": "nih", + "johnshopkins.edu:funder:305204": "aspr", + "johnshopkins.edu:funder:303395": "fda" + } + } + }, + "transport-config": { + "protocol-binding": { + "protocol": "filesystem", + "baseDir": "target/packages", + "createIfMissing": "true", + "overwrite": "true" + } + } + }, + "BagIt": { + "assembler": { + "specification": "simple", + "beanName": "simpleAssembler", + "options": { + "archive": "ZIP", + "compression": "NONE", + "algorithms": [ + "sha512", + "md5" + ], + "baginfo-template-resource": "/bag-info.hbm" + } + }, + "transport-config": { + "protocol-binding": { + "protocol": "filesystem", + "baseDir": "target/packages", + "createIfMissing": "true", + "overwrite": "true" + } + } + }, + "dash": { + "assembler": { + "specification": "simple", + "beanName": "simpleAssembler", + "options": { + "archive": "ZIP", + "compression": "NONE", + "algorithms": [ + "sha512", + "md5" + ] + } + }, + "transport-config": { + "protocol-binding": { + "protocol": "filesystem", + "baseDir": "target/packages", + "createIfMissing": "true", + "overwrite": "true" + } + } + } +} \ No newline at end of file