diff --git a/JHU-README.md b/JHU-README.md index 0915260..e18cee2 100644 --- a/JHU-README.md +++ b/JHU-README.md @@ -103,7 +103,7 @@ will therefore look like this: `8675309:policies/e7/3f/26/70/e73f2670-6ef6-4201-bbcd-04631a93d852` -The `policy.proerties` file will have to be kept up to date if additional funders are assigned policies. A different +The `policy.properties` file will have to be kept up to date if additional funders are assigned policies. A different mechanism will be needed to add additional policies, as we require the policies reference in the properties file to be already present in the system. @@ -165,4 +165,4 @@ must be filled out accordingly. We also have a command line option -m to pass in The processing of the ResultSet is straightforward - we simply construct a set of hash maps which represent the column names and the values for each record. We do not assume that the PASS objects in Fedora are updated only by this application, as there may be some fields on these objects which are not known to COEUS, but -may be populated by other applications eventually (for example ORCID on User, Submissions on Grant, or Policy on Funder). +may be populated by other applications eventually (for example ORCID on User, Submissions on Grant, or Policy on Funder). \ No newline at end of file diff --git a/pass-grant-cli/pom.xml b/pass-grant-cli/pom.xml index 76d24d5..8609c67 100644 --- a/pass-grant-cli/pom.xml +++ b/pass-grant-cli/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.3.0 + 1.4.0 4.0.0 diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/BaseGrantLoaderApp.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/BaseGrantLoaderApp.java index 35032f8..446af89 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/BaseGrantLoaderApp.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/BaseGrantLoaderApp.java @@ -34,6 +34,7 @@ import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.sql.SQLException; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; @@ -50,27 +51,28 @@ * use the PassLoader to take {@code List} representing the {@code ResultSet} to push this data into our PASS instance * via the java pass client. * + * * A large percentage of the code here is handling exceptional paths, as this is intended to be run in an automated * fashion, so care must be taken to log errors, report them to STDOUT, and also send email notifications. * * @author jrm@jhu.edu */ abstract class BaseGrantLoaderApp { - private static Logger LOG = LoggerFactory.getLogger(BaseGrantLoaderApp.class); + private static final Logger LOG = LoggerFactory.getLogger(BaseGrantLoaderApp.class); private EmailService emailService; - private File appHome; + private final File appHome; private String startDate; - private String awardEndDate; + private final String awardEndDate; private File updateTimestampsFile; - private boolean email; - private String mode; - private String action; - private String dataFileName; + private final boolean email; + private final String mode; + private final String action; + private final String dataFileName; private boolean local = false; - private boolean timestamp = true; + private boolean timestamp = false; - private String updateTimestampsFileName; + private final String updateTimestampsFileName; /** * Constructor for this class @@ -247,7 +249,7 @@ void run() throws PassCliException { try (FileInputStream fis = new FileInputStream(dataFile); ObjectInputStream in = new ObjectInputStream(fis) ) { - resultSet = (List>)in.readObject(); + resultSet = Collections.unmodifiableList((List>) in.readObject()); } catch (IOException | ClassNotFoundException ex) { ex.printStackTrace(); } diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/DataLoaderErrors.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/DataLoaderErrors.java index eaa7330..fd67083 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/DataLoaderErrors.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/DataLoaderErrors.java @@ -16,6 +16,9 @@ package org.dataconservancy.pass.grant.cli; +/** + * A class containing all error strings for errors caught by the Loader Apps + */ class DataLoaderErrors { static String ERR_HOME_DIRECTORY_NOT_FOUND = "No home directory found for the application. Please specify a valid absolute path."; static String ERR_HOME_DIRECTORY_NOT_READABLE_AND_WRITABLE = "Supplied home directory must be readable" + diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/EmailService.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/EmailService.java index a4f7093..aebdba6 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/EmailService.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/EmailService.java @@ -31,12 +31,12 @@ import org.slf4j.LoggerFactory; /** - * An email service for reporting errors or results of running the {@code CoeusGrantLoaderApp} + * An email service for reporting errors or results of running a GrantLoaderApp * @author jrm@jhu.edu */ class EmailService { - private static Logger LOG = LoggerFactory.getLogger(EmailService.class); - private Properties mailProperties; + private static final Logger LOG = LoggerFactory.getLogger(EmailService.class); + private final Properties mailProperties; /** * The constructor diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java index f8ae9f6..754ff3f 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java @@ -15,20 +15,18 @@ */ package org.dataconservancy.pass.grant.cli; - -import org.dataconservancy.pass.grant.data.CoeusConnector; -import org.dataconservancy.pass.grant.data.DateTimeUtil; -import org.dataconservancy.pass.grant.data.GrantConnector; -import org.dataconservancy.pass.grant.data.JhuPassUpdater; -import org.dataconservancy.pass.grant.data.PassUpdater; +import org.dataconservancy.pass.grant.data.*; import java.util.Properties; class JhuGrantLoaderApp extends BaseGrantLoaderApp { - JhuGrantLoaderApp(String startDate, String awardEndDate, boolean email, String mode, String action, String dataFileName) { + boolean init; + + JhuGrantLoaderApp(String startDate, String awardEndDate, boolean email, String mode, String action, String dataFileName, boolean init) { super(startDate, awardEndDate, email, mode, action, dataFileName); super.setTimestamp(true); + this.init = init; } @Override @@ -43,6 +41,9 @@ GrantConnector configureConnector(Properties connectionProperties, Properties po @Override PassUpdater configureUpdater() { + if ( init ) { + return new JhuPassInitUpdater(); + } return new JhuPassUpdater(); } diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderCLI.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderCLI.java index 501dc71..b7d65df 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderCLI.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderCLI.java @@ -64,6 +64,11 @@ public class JhuGrantLoaderCLI { "01/01/2011") private static String awardEndDate = "01/01/2011"; + /** Specifies whether this run is an "initializing run" which is allowed to overwrite normally non-writable fields on grants */ + @Option(name = "-i", aliases = {"-init", "--init", "-initialize", "--initialize" }, usage = "When set to true, changes the behavior of the loader to allow it" + + "to update all fields stored on grants with info coming in from the pull. This is useful when updating existing grant records due to a change in policy" + + "about what the semantics of the stored records are.") + private static boolean init = false; /** Specifies an optional action - either "pull" or "load" - to restrict the operation of the application to only pull data * from COEUS to store in a file, or to only load into PASS data taken from a stored file, respectively. In either case, the path to @@ -113,7 +118,7 @@ public static void main(String[] args) { } /* Run the package generation application proper */ - JhuGrantLoaderApp app = new JhuGrantLoaderApp(startDate, awardEndDate, email, mode, action, dataFileName); + JhuGrantLoaderApp app = new JhuGrantLoaderApp(startDate, awardEndDate, email, mode, action, dataFileName, init); app.run(); System.exit((0)); } catch (CmdLineException e) { diff --git a/pass-grant-cli/src/test/java/org/dataconservancy/pass/grant/cli/EmailServiceTest.java b/pass-grant-cli/src/test/java/org/dataconservancy/pass/grant/cli/EmailServiceTest.java index e5f3642..050a6df 100644 --- a/pass-grant-cli/src/test/java/org/dataconservancy/pass/grant/cli/EmailServiceTest.java +++ b/pass-grant-cli/src/test/java/org/dataconservancy/pass/grant/cli/EmailServiceTest.java @@ -19,7 +19,6 @@ import com.icegreen.greenmail.util.GreenMail; import com.icegreen.greenmail.util.ServerSetupTest; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -28,13 +27,16 @@ import java.io.IOException; import java.util.Properties; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test classs for email service * @author jrm@jhu.edu */ public class EmailServiceTest { - private Properties mailProperties = System.getProperties(); + private final Properties mailProperties = System.getProperties(); private EmailService underTest; private GreenMail testServer; @@ -63,7 +65,7 @@ public void setup() throws InterruptedException { if (!started) { // try one more time - Thread.sleep(5000l); + Thread.sleep(5000L); testServer.start(); } @@ -80,14 +82,14 @@ public void testSendMessage() throws MessagingException, IOException { String messageSubject = "TEST"; underTest.sendEmailMessage(messageSubject, messageBody); // Check that only one message was sent - Integer numMessages = testServer.getReceivedMessages().length; - Assert.assertTrue("Expected only one message, got " + numMessages, numMessages == 1); + int numMessages = testServer.getReceivedMessages().length; + assertEquals("Expected only one message, got " + numMessages, 1, numMessages); // Check that the message is just a plaintext message MimeMessage message = testServer.getReceivedMessages()[0]; - Assert.assertTrue("Subject of message was not correct", message.getSubject().equals(messageSubject)); - Assert.assertTrue("Content of message was not a string as expected", message.getContent() instanceof String); - Assert.assertTrue(message.getContent().toString().contains(messageBody)); + assertEquals("Subject of message was not correct", message.getSubject(), messageSubject); + assertTrue("Content of message was not a string as expected", message.getContent() instanceof String); + assertTrue(message.getContent().toString().contains(messageBody)); } @After diff --git a/pass-grant-data/pom.xml b/pass-grant-data/pom.xml index 163755a..40085e9 100644 --- a/pass-grant-data/pom.xml +++ b/pass-grant-data/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.3.0 + 1.4.0 4.0.0 diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java new file mode 100644 index 0000000..0617639 --- /dev/null +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java @@ -0,0 +1,197 @@ +/* + * Copyright 2019 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dataconservancy.pass.grant.data; + +import org.dataconservancy.pass.model.Funder; +import org.dataconservancy.pass.model.Grant; +import org.dataconservancy.pass.model.User; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * A utility class for handling Grants, Users or Funders. One function performed is comparison of two instances of + * these PASS entity classes. These comparisons are reduced to only those fields which are updatable by + * data from the source pull. Stored objects are evaluated and updated according to the PassEntityUtil implementation + * supplied to this updater + * + * + * This basic implementation deals with a smallish set of fields - other implementations will have to address more + * fields if they are to be stored on PASS objects. + * + * @author jrm@jhu.edu + */ + +public class BasicPassEntityUtil implements PassEntityUtil{ + + + /** + * This method takes a Funder, calculates whether it needs to be updated, and if so, returns the updated object + * to be be ingested into the repository. if not, returns null. + * @param stored the Funder as it is stored in the PASS backend + * @param system the version of the Funder from the Harvard pull + * @return the updated Funder - null if the Funder does not need to be updated + */ + public Funder update(Funder system, Funder stored) { + if (funderNeedsUpdate(system, stored)) { + return updateFunder(system, stored); + } + return null; + } + + /** + * This method takes a User, calculates whether it needs to be updated, and if so, returns the updated object + * to be be ingested into the repository. if not, returns null. + * @param stored the User as it is stored in the PASS backend + * @param system the version of the user from the pull + * @return the updated User - null if the User does not need to be updated + */ + public User update(User system, User stored) { + if (userNeedsUpdate(system, stored)) { + return updateUser(system, stored); + } + return null; + } + + /** + * This method takes a Harvard Grant, calculates whether it needs to be updated, and if so, returns the updated object + * to be be ingested into the repository. if not, returns null. + * @param stored the Grant as it is stored in the PASS backend + * @param system the version of the Grant from the pull + * @return the updated object - null if the Grant does not need to be updated + */ + public Grant update(Grant system, Grant stored) { + if (grantNeedsUpdate(system, stored)) { + return updateGrant(system, stored); + } + return null; + } + + /** + * Compare two Funder objects + * + * @param system the version of the Funder as seen in the system pull + * @param stored the version of the Funder as read from Pass + * @return a boolean which asserts whether the pulled object contains information which can + * update the stored object + */ + private boolean funderNeedsUpdate(Funder system, Funder stored) { + + //this adjustment handles the case where we take data from policy.properties file, which has no name info + if (system.getName() != null && !system.getName().equals(stored.getName())) return true; + if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return true; + return false; + } + + /** + * Update a Pass Funder object with new information from a system pull + * + * @param system the version of the Funder as seen in the system pull + * @param stored the version of the Funder as read from Pass + * @return the Funder object which represents the Pass object, with any new information from the pull merged in + */ + private Funder updateFunder (Funder system, Funder stored) { + stored.setLocalKey(system.getLocalKey()); + stored.setName(system.getName()); + stored.setPolicy(system.getPolicy()); + return stored; + } + + /** + * Compare two User objects. We only care about those fields for which is the authoritative source + * After recent changes. + * + * @param system the version of the User as seen in the Harvard system pull + * @param stored the version of the User as read from Pass + * @return a boolean which asserts whether the stored object needs updating + */ + private boolean userNeedsUpdate(User system, User stored) { + if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return true; + if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return true; + if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return true; + if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return true; + if (system.getEmail() != null && stored.getEmail() == null) return true; + if (system.getDisplayName() != null && stored.getDisplayName() == null) return true; + return false; + } + + /** + * Update a Pass User object with new information from the system. We check only those fields for which Harvard is + * authoritative. Other fields will be managed by other providers (Shibboleth for example). The exceptions are + * the localKey, which this application and Shibboleth both rely on; and email, which this application only populates + * if Shib hasn't done so already. + * + * @param system the version of the User as seen in the system pull + * @param stored the version of the User as read from Pass + * @return the User object which represents the Pass object, with any new information merged in + */ + private User updateUser (User system, User stored) { + stored.setFirstName(system.getFirstName()); + stored.setMiddleName(system.getMiddleName()); + stored.setLastName(system.getLastName()); + stored.setEmail(system.getEmail()); + Set idSet = new HashSet<>(); + idSet.addAll(stored.getLocatorIds()); + idSet.addAll(system.getLocatorIds()); + stored.setLocatorIds(idSet.stream().collect(Collectors.toList())); + stored.setDisplayName(system.getDisplayName()); + return stored; + } + + /** + * Compare two Grant objects. Note that the Lists of Co-Pis are compared as Sets + * @param system the version of the Grant as seen in the system pull + * @param stored the version of the Grant as read from Pass + * @return a boolean which asserts whether the stored ubject needs updating + */ + private boolean grantNeedsUpdate(Grant system, Grant stored) { + if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return true; + if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return true; + if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return true; + if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return true; + if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; + if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; + if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return true; + if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return true; + return false; + } + + /** + * Update a Pass Grant object with new information from Harvard + * + * @param system the version of the Grant as seen in the system pull + * @param stored the version of the Grant as read from Pass + * @return the Grant object which represents the Pass object, with any new information merged in + */ + private Grant updateGrant(Grant system, Grant stored) { + stored.setAwardNumber(system.getAwardNumber()); + stored.setLocalKey(system.getLocalKey()); + stored.setProjectName(system.getProjectName()); + stored.setPrimaryFunder(system.getPrimaryFunder()); + stored.setDirectFunder(system.getDirectFunder()); + stored.setPi(system.getPi()); + stored.setCoPis(system.getCoPis()); + stored.setStartDate(system.getStartDate()); + stored.setEndDate(system.getEndDate()); + return stored; + } + +} diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java new file mode 100644 index 0000000..7bc270f --- /dev/null +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java @@ -0,0 +1,528 @@ +/* + * Copyright 2018 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dataconservancy.pass.grant.data; + + import org.dataconservancy.pass.client.PassClient; + import org.dataconservancy.pass.client.PassClientFactory; + import org.dataconservancy.pass.model.Funder; + import org.dataconservancy.pass.model.Grant; + import org.dataconservancy.pass.model.User; + import org.dataconservancy.pass.model.support.Identifier; + import org.joda.time.DateTime; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + + import java.net.URI; + import java.util.ArrayList; + import java.util.Collection; + import java.util.HashMap; + import java.util.ListIterator; + import java.util.Map; + + import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; + import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; + +/** + * This class is responsible for taking the Set of Maps derived from the ResultSet from the database query and + * constructing a corresponding Collection of Grant or User objects, which it then sends to PASS to update. + * + * This represents the state of things for the default updater of version 1.3 + * + * @author jrm@jhu.edu + */ + +public class BasicPassUpdater implements PassUpdater{ + + private String DOMAIN = "default.domain"; + + private static final Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); + private String latestUpdateString = ""; + + private final PassClient passClient; + private final PassUpdateStatistics statistics = new PassUpdateStatistics(); + private final PassEntityUtil passEntityUtil; + + //used in test classes + private final Map grantUriMap = new HashMap<>(); + + //used in unit test + //some entities may be referenced many times during an update, but just need to be updated the first time + //they are encountered. these include Users and Funders. we save the overhead of redundant updates + //of these by looking them up here; if they are on the Map, they have already been processed + // + private final Map funderMap = new HashMap<>(); + private final Map userMap = new HashMap<>(); + + private String mode; + + public BasicPassUpdater(PassEntityUtil passEntityUtil) + { + this.passEntityUtil = passEntityUtil; + this.passClient = PassClientFactory.getPassClient(); + } + + //used in unit testing for injecting a mock client + public BasicPassUpdater(PassEntityUtil passEntityUtil, PassClient passClient) { + this.passEntityUtil = passEntityUtil; + this.passClient = passClient; + } + + public void updatePass(Collection> results, String mode) { + this.mode = mode; + userMap.clear(); + funderMap.clear(); + statistics.reset(); + statistics.setType(mode); + switch (mode) { + case "grant": + updateGrants(results); + break; + case "user": + updateUsers(results); + break; + case "funder": + updateFunders(results); + break; + } + } + + /** + * Build a Collection of Grants from a ResultSet, then update the grants in Pass + * Because we need to make sure we catch any updates to fields referenced by URIs, we construct + * these and update these as well + */ + private void updateGrants(Collection> results) { + + //a grant will have several rows in the ResultSet if there are co-pis. so we put the grant on this + //Map and add to it as additional rows add information. + Map grantMap = new HashMap<>(); + + LOG.info("Processing result set with " + results.size() + " rows"); + boolean modeChecked = false; + + for(Map rowMap : results){ + + String grantLocalKey; + + if (!modeChecked) { + if (!rowMap.containsKey(C_GRANT_LOCAL_KEY)) {//we always have this for grants + throw new RuntimeException("Mode of grant was supplied, but data does not seem to match."); + } else { + modeChecked = true; + } + } + + grantLocalKey = rowMap.get(C_GRANT_LOCAL_KEY); + + String directFunderLocalKey = rowMap.get(C_DIRECT_FUNDER_LOCAL_KEY); + String primaryFunderLocalKey = rowMap.get(C_PRIMARY_FUNDER_LOCAL_KEY); + primaryFunderLocalKey = (primaryFunderLocalKey == null? directFunderLocalKey: primaryFunderLocalKey); + Grant grant; + LOG.debug("Processing grant with localKey " + grantLocalKey); + //if this is the first record for this Grant, it will not be on the Map + //we process all data which is common to every record for this grant + //i.e., everything except the investigator(s) + if(!grantMap.containsKey(grantLocalKey)) { + grant = new Grant(); + grant.setAwardNumber(rowMap.get(C_GRANT_AWARD_NUMBER)); + + String status = rowMap.getOrDefault(C_GRANT_AWARD_STATUS, null); + + if (status != null) { + switch (status) { + case "Active": + grant.setAwardStatus(Grant.AwardStatus.ACTIVE); + break; + case "Pre-Award": + grant.setAwardStatus(Grant.AwardStatus.PRE_AWARD); + break; + case "Terminated": + grant.setAwardStatus(Grant.AwardStatus.TERMINATED); + } + } else { + grant.setAwardStatus(null); + } + + grant.setLocalKey(grantLocalKey); + grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); + grant.setAwardDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_AWARD_DATE, null))); + grant.setStartDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_START_DATE, null))); + grant.setEndDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_END_DATE, null))); + + //process direct funder, and primary funder if we have one + //update funder(s) in Pass as needed + if (funderMap.containsKey(directFunderLocalKey)) { + grant.setDirectFunder(funderMap.get(directFunderLocalKey)); + } else { + Funder updatedFunder = buildDirectFunder(rowMap); + URI passFunderURI = updateFunderInPass(updatedFunder); + funderMap.put(directFunderLocalKey, passFunderURI); + grant.setDirectFunder(passFunderURI); + } + + if(funderMap.containsKey(primaryFunderLocalKey)) { + grant.setPrimaryFunder(funderMap.get(primaryFunderLocalKey)); + } else { + Funder updatedFunder = buildPrimaryFunder(rowMap); + URI passFunderURI = updateFunderInPass(updatedFunder); + funderMap.put(primaryFunderLocalKey, passFunderURI); + grant.setPrimaryFunder(passFunderURI); + } + + grant.setCoPis(new ArrayList<>());//we will build this from scratch in either case + grantMap.put(grantLocalKey, grant);//save the state of this Grant + } + + //now we process the User (investigator) + grant = grantMap.get(grantLocalKey); + String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); + String abbreviatedRole = rowMap.get(C_ABBREVIATED_ROLE); + + if(abbreviatedRole.equals("C") || abbreviatedRole.equals("K") || grant.getPi() == null) { + if (!userMap.containsKey(employeeId)) { + User updatedUser = buildUser(rowMap); + URI passUserURI = updateUserInPass(updatedUser); + userMap.put(employeeId, passUserURI); + } + + //now our User URI is on the map - let's process: + if (abbreviatedRole.equals("P")) { + if (grant.getPi() == null) { + grant.setPi(userMap.get(employeeId)); + statistics.addPi(); + } else {// handle data with duplicate PIs + grant.getCoPis().add(userMap.get(employeeId)); + statistics.addCoPi(); + } + } else if ((abbreviatedRole.equals("C") || abbreviatedRole.equals("K")) && !grant.getCoPis().contains(userMap.get(employeeId))) { + grant.getCoPis().add(userMap.get(employeeId)); + statistics.addCoPi(); + } + } + //we are done with this record, let's save the state of this Grant + grantMap.put(grantLocalKey, grant); + //see if this is the latest grant updated + if (rowMap.containsKey(C_UPDATE_TIMESTAMP)) { + String grantUpdateString = rowMap.get(C_UPDATE_TIMESTAMP); + latestUpdateString = latestUpdateString.length() == 0 ? grantUpdateString : returnLaterUpdate(grantUpdateString, latestUpdateString); + } + } + + //now put updated grant objects in pass + for(Grant grant : grantMap.values()){ + grantUriMap.put(updateGrantInPass(grant), grant); + } + + //success - we capture some information to report + if (grantMap.size() > 0) { + statistics.setLatestUpdateString(latestUpdateString); + statistics.setReport(results.size(), grantMap.size()); + } else { + System.out.println("No records were processed in this update"); + } + } + + private void updateUsers(Collection> results) { + + boolean modeChecked = false; + + for(Map rowMap : results) { + + if (!modeChecked) { + if (!rowMap.containsKey(C_USER_EMPLOYEE_ID)) {//we always have this for users + throw new RuntimeException("Mode of user was supplied, but data does not seem to match."); + } else { + modeChecked = true; + } + } + + LOG.info("Processing result set with " + results.size() + " rows"); + User updatedUser = buildUser(rowMap); + updateUserInPass(updatedUser); + if (rowMap.containsKey(C_UPDATE_TIMESTAMP)) { + String userUpdateString = rowMap.get(C_UPDATE_TIMESTAMP); + latestUpdateString = latestUpdateString.length() == 0 ? userUpdateString : returnLaterUpdate(userUpdateString, latestUpdateString); + } + } + + if (results.size() > 0) { + statistics.setLatestUpdateString(latestUpdateString); + statistics.setReport(results.size(), results.size()); + } else { + System.out.println("No records were processed in this update"); + } + + } + + /** + * This method is called for the "funder" mode - the column names will have the values for primary funders + * @param results the data row map containing funder information + */ + private void updateFunders(Collection> results) { + + boolean modeChecked = false; + LOG.info("Processing result set with " + results.size() + " rows"); + for (Map rowMap : results) { + + if (!modeChecked) { + if (!rowMap.containsKey(C_PRIMARY_FUNDER_LOCAL_KEY) && !rowMap.containsKey(C_PRIMARY_FUNDER_NAME)) { + throw new RuntimeException("Mode of funder was supplied, but data does not seem to match."); + } else { + modeChecked = true; + } + } + + Funder updatedFunder = buildPrimaryFunder(rowMap); + updateFunderInPass(updatedFunder); + + } + statistics.setReport(results.size(), results.size()); + } + + User buildUser(Map rowMap) { + User user = new User(); + user.setFirstName(rowMap.get(C_USER_FIRST_NAME)); + user.setMiddleName(rowMap.getOrDefault(C_USER_MIDDLE_NAME, null)); + user.setLastName(rowMap.get(C_USER_LAST_NAME)); + user.setDisplayName(rowMap.get(C_USER_FIRST_NAME) + " " + rowMap.get(C_USER_LAST_NAME)); + user.setEmail(rowMap.get(C_USER_EMAIL)); + String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); + //Build the List of locatorIds - put the most reliable ids first + if (employeeId != null) { + String EMPLOYEE_ID_TYPE = "employeeid"; + user.getLocatorIds().add(new Identifier(DOMAIN, EMPLOYEE_ID_TYPE, employeeId).serialize()); + } + user.getRoles().add(User.Role.SUBMITTER); + LOG.debug("Built user with employee ID " + employeeId); + return user; + } + + /** + * this method gets called on a grant mode process if the primary funder is different from direct, and also + * any time the updater is called in funder mode + * @param rowMap the funder data map + * @return the funder + */ + Funder buildPrimaryFunder(Map rowMap) { + Funder funder = new Funder(); + funder.setName(rowMap.getOrDefault(C_PRIMARY_FUNDER_NAME, null)); + funder.setLocalKey(rowMap.get(C_PRIMARY_FUNDER_LOCAL_KEY)); + String policy = rowMap.get(C_PRIMARY_FUNDER_POLICY); + if (policy != null && policy.length()>0) { + String fedoraBaseUrl = System.getProperty("pass.fedora.baseurl"); + fedoraBaseUrl = fedoraBaseUrl.endsWith("/") ? fedoraBaseUrl : fedoraBaseUrl + "/"; + funder.setPolicy(URI.create(fedoraBaseUrl + policy)); + LOG.info("Processing Funder with localKey " + funder.getLocalKey() + + " and Policy " + policy); + } + LOG.debug("Built Funder with localKey " + funder.getLocalKey()); + + return funder; + } + + private Funder buildDirectFunder(Map rowMap) { + Funder funder = new Funder(); + if (rowMap.containsKey(C_DIRECT_FUNDER_NAME)) { + funder.setName(rowMap.get(C_DIRECT_FUNDER_NAME)); + } + funder.setLocalKey(rowMap.get(C_DIRECT_FUNDER_LOCAL_KEY)); + String policy = rowMap.get(C_DIRECT_FUNDER_POLICY); + if (policy != null ) { + String fedoraBaseUrl = System.getProperty("pass.fedora.baseurl"); + fedoraBaseUrl = fedoraBaseUrl.endsWith("/") ? fedoraBaseUrl : fedoraBaseUrl + "/"; + funder.setPolicy(URI.create(fedoraBaseUrl + policy)); + LOG.info("Processing Funder with localKey " + funder.getLocalKey() + + " and Policy " + policy); + } + LOG.debug("Built Funder with localKey " + funder.getLocalKey()); + + return funder; + } + + + /** + * Take a new Funder object populated as fully as possible from the COEUS pull, and use this + * new information to update an object for the same Funder in Pass (if it exists) + * + * @param systemFunder the new Funder object populated from COEUS + * @return the URI for the resource representing the updated Funder in Pass + */ + private URI updateFunderInPass(Funder systemFunder) { + String baseLocalKey = systemFunder.getLocalKey(); + String FUNDER_ID_TYPE = "funder"; + String fullLocalKey = new Identifier(DOMAIN, FUNDER_ID_TYPE, baseLocalKey).serialize(); + systemFunder.setLocalKey(fullLocalKey); + + URI passFunderURI = passClient.findByAttribute(Funder.class, "localKey", fullLocalKey); + if (passFunderURI != null ) { + Funder storedFunder = passClient.readResource(passFunderURI, Funder.class); + if (storedFunder == null) { + throw new RuntimeException("Could not read Funder object with URI " + passFunderURI); + } + Funder updatedFunder; + if ((updatedFunder = passEntityUtil.update(systemFunder, storedFunder)) != null) {//need to update + passClient.updateResource(updatedFunder); + statistics.addFundersUpdated(); + }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything + //this can happen if the Grant was updated in COEUS only with information we don't consume here + } else {//don't have a stored Funder for this URI - this one is new to Pass + if (systemFunder.getName() != null) {//only add if we have a name + passFunderURI = passClient.createResource(systemFunder); + statistics.addFundersCreated(); + } + } + return passFunderURI; + } + + /** + * Take a new User object populated as fully as possible from the COEUS pull, and use this + * new information to update an object for the same User in Pass (if it exists) + * + * @param systemUser the new User object populated from COEUS + * @return the URI for the resource representing the updated User in Pass + */ + private URI updateUserInPass(User systemUser) { + //we first check to see if the user is known by the Hopkins ID. If not, we check the employee ID. + //last attempt is the JHED ID. this order is specified by the order of the List as constructed on updatedUser + URI passUserUri = null; + ListIterator idIterator = systemUser.getLocatorIds().listIterator(); + + while (passUserUri == null && idIterator.hasNext()) { + String id = String.valueOf(idIterator.next()); + if (id != null) { + passUserUri = passClient.findByAttribute(User.class, "locatorIds", id); + } + } + + if (passUserUri != null ) { + User storedUser = passClient.readResource(passUserUri, User.class); + if (storedUser == null) { + throw new RuntimeException("Could not read User object with URI " + passUserUri); + } + User updatedUser; + if ((updatedUser = passEntityUtil.update(systemUser, storedUser)) != null){//need to update + //post COEUS processing goes here + if(!storedUser.getRoles().contains(User.Role.SUBMITTER)) { + storedUser.getRoles().add(User.Role.SUBMITTER); + } + passClient.updateResource(updatedUser); + statistics.addUsersUpdated(); + }//if the Pass version is COEUS-equal to our version from the update, and there are no null fields we care about, + //we don't have to do anything. this can happen if the User was updated in COEUS only with information we don't consume here + } else if (! mode.equals("user")) {//don't have a stored User for this URI - this one is new to Pass + //but don't update if we are in user mode - just update existing users + passUserUri = passClient.createResource(systemUser); + statistics.addUsersCreated(); + } + return passUserUri; + } + + /** + * Take a new Grant object populated as fully as possible from the COEUS pull, and use this + * new information to update an object for the same Grant in Pass (if it exists) + * + * @param systemGrant the new Grant object populated from COEUS + * @return the PASS identifier for the Grant object + */ + private URI updateGrantInPass(Grant systemGrant) { + String baseLocalKey = systemGrant.getLocalKey(); + String GRANT_ID_TYPE = "grant"; + String fullLocalKey = new Identifier(DOMAIN, GRANT_ID_TYPE, baseLocalKey).serialize(); + systemGrant.setLocalKey(fullLocalKey); + + LOG.debug("Looking for grant with localKey " + fullLocalKey); + URI passGrantURI = passClient.findByAttribute(Grant.class, "localKey", fullLocalKey); + if (passGrantURI != null ) { + LOG.debug("Found grant with localKey " + fullLocalKey); + Grant storedGrant = passClient.readResource(passGrantURI, Grant.class); + if (storedGrant == null) { + throw new RuntimeException("Could not read Funder object with URI " + passGrantURI); + } + Grant updatedGrant; + if ( (updatedGrant = passEntityUtil.update(systemGrant, storedGrant)) != null) {//need to update + LOG.debug("Updating grant with localKey " + storedGrant.getLocalKey() + " to localKey " + systemGrant.getLocalKey()); + passClient.updateResource(updatedGrant); + statistics.addGrantsUpdated(); + LOG.debug("Updating grant with award number " + systemGrant.getLocalKey()); + }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything + //this can happen if the Grant was updated in COEUS only with information we don't consume here + } else {//don't have a stored Grant for this URI - this one is new to Pass + passGrantURI = passClient.createResource(systemGrant); + statistics.addGrantsCreated(); + LOG.debug("Creating grant with award number " + systemGrant.getLocalKey()); + } + return passGrantURI; + } + + /** + * Compare two timestamps and return the later of them + * @param currentUpdateString the current latest timestamp string + * @param latestUpdateString the new timestamp to be compared against the current latest timestamp + * @return the later of the two parameters + */ + static String returnLaterUpdate(String currentUpdateString, String latestUpdateString) { + DateTime grantUpdateTime = createJodaDateTime(currentUpdateString); + DateTime previousLatestUpdateTime = createJodaDateTime(latestUpdateString); + return grantUpdateTime.isAfter(previousLatestUpdateTime)? currentUpdateString : latestUpdateString; + } + + /** + * This method provides the latest timestamp of all records processed. After processing, this timestamp + * will be used to be tha base timestamp for the next run of the app + * @return the latest update timestamp string + */ + public String getLatestUpdate(){ + return this.latestUpdateString; + } + + /** + * This returns the final statistics of the processing of the Grant or User Set + * @return the report + */ + public String getReport(){ + return statistics.getReport(); + } + + /** + * This returns the final statistics Object - useful in testing + * @return the statistics object + */ + public PassUpdateStatistics getStatistics() { + return statistics; + } + + + public Map getGrantUriMap() { + return grantUriMap; + } + + //this is used by an integration test + public PassClient getPassClient() { + return passClient; + } + + //used in unit test + Map getFunderMap() { return funderMap; } + + //used in unit test + Map getUserMap() { return userMap; } + + public void setDomain(String domain) { + this.DOMAIN = domain; + } + +} \ No newline at end of file diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java index 756b4cb..b0184e6 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java @@ -29,8 +29,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.Set; - import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; @@ -41,7 +39,7 @@ * @author jrm@jhu.edu */ public class CoeusConnector implements GrantConnector { - private static Logger LOG = LoggerFactory.getLogger(CoeusConnector.class); + private static final Logger LOG = LoggerFactory.getLogger(CoeusConnector.class); //property names private static final String COEUS_URL = "coeus.url"; private static final String COEUS_USER = "coeus.user"; @@ -51,7 +49,7 @@ public class CoeusConnector implements GrantConnector { private String coeusUser; private String coeusPassword; - private Properties funderPolicyProperties; + private final Properties funderPolicyProperties; private DirectoryServiceUtil directoryServiceUtil; @@ -285,11 +283,11 @@ private String buildGrantQueryString(String startDate, String awardEndDate){ sb.append(String.join(", ",viewFields)); sb.append(" FROM"); sb.append(" COEUS.JHU_FACULTY_FORCE_PROP A"); - sb.append(" INNER JOIN "); - sb.append(" (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP"); - sb.append(" FROM COEUS.JHU_FACULTY_FORCE_PROP GROUP BY GRANT_NUMBER) LATEST"); - sb.append(" ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP"); - sb.append(" AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER"); + // sb.append(" INNER JOIN "); + // sb.append(" (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP"); + // sb.append(" FROM COEUS.JHU_FACULTY_FORCE_PROP GROUP BY GRANT_NUMBER) LATEST"); + // sb.append(" ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP"); + // sb.append(" AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER"); sb.append(" INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B ON A.INST_PROPOSAL = B.INST_PROPOSAL"); sb.append(" INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C ON B.EMPLOYEE_ID = C.EMPLOYEE_ID"); sb.append(" LEFT JOIN COEUS.SWIFT_SPONSOR D ON A.PRIME_SPONSOR_CODE = D.SPONSOR_CODE"); @@ -309,7 +307,7 @@ private String buildGrantQueryString(String startDate, String awardEndDate){ } private String buildUserQueryString(String startDate) { - String viewFields [] = { + String[] viewFields = { C_USER_FIRST_NAME, C_USER_MIDDLE_NAME, C_USER_LAST_NAME, @@ -335,7 +333,7 @@ private String buildUserQueryString(String startDate) { private String buildFunderQueryString() { - String viewFields [] = {//doesn't matter whether the funder is primary or direct - these are the column names in the SWIFT_SPONSOR view + String[] viewFields = {//doesn't matter whether the funder is primary or direct - these are the column names in the SWIFT_SPONSOR view C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_LOCAL_KEY }; diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java index 5be4e4e..7900652 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java @@ -20,7 +20,7 @@ import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.User; - +import java.net.URI; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -45,7 +45,7 @@ public class CoeusPassEntityUtil implements PassEntityUtil{ * @return the updated Funder - null if the Funder does not need to be updated */ public Funder update(Funder system, Funder stored) { - if (!coeusFundersEqual(system, stored)) { + if (funderNeedsUpdate(system, stored)) { return updateFunder(system, stored); } return null; @@ -59,7 +59,7 @@ public Funder update(Funder system, Funder stored) { * @return the updated User - null if the User does not need to be updated */ public User update(User system, User stored) { - if (!coeusUsersEqual(system, stored)) { + if (userNeedsUpdate(system, stored)) { return updateUser(system, stored); } return null; @@ -73,26 +73,50 @@ public User update(User system, User stored) { * @return the updated object - null if the Grant does not need to be updated */ public Grant update(Grant system, Grant stored) { - if (!coeusGrantsEqual(system, stored)) { + //adjust the system view of co-pis by merging in the stored view of pi and co-pis + for( URI uri : stored.getCoPis() ) { + if ( !system.getCoPis().contains(uri) ) { + system.getCoPis().add(uri); + } + } + + //need to be careful, system pi might be null if there is no record for it + //this is to finalize the version of the co-pi list we want to compare between + //system and stored + URI storedPi = stored.getPi(); + if ( system.getPi() != null ) { + if (!system.getPi().equals(storedPi)) { + // stored.setPi( system.getPi() ); + if (!system.getCoPis().contains(storedPi)) { + system.getCoPis().add(storedPi); + } + system.getCoPis().remove(system.getPi()); + } + } else { //system view is null, do not trigger update based on this field + system.setPi( storedPi ); + } + + //now system view has all available info we want in this grant - look for update trigger + if (grantNeedsUpdate(system, stored)) { return updateGrant(system, stored); } return null; } - /** + /**0p * Compare two Funder objects * * @param system the version of the Funder as seen in the COEUS system pull * @param stored the version of the Funder as read from Pass * @return a boolean which asserts whether the two supplied Funders are "COEUS equal" */ - private boolean coeusFundersEqual(Funder system, Funder stored) { + private boolean funderNeedsUpdate(Funder system, Funder stored) { //this adjustment handles the case where we take data from policy.properties file, which has no name info - if (system.getName() != null && !system.getName().equals(stored.getName())) return false; - if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return false; - return true; + if (system.getName() != null && !system.getName().equals(stored.getName())) return true; + if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return true; + return false; } /** @@ -103,7 +127,7 @@ private boolean coeusFundersEqual(Funder system, Funder stored) { * @return the Funder object which represents the Pass object, with any new information from COEUS merged in */ private Funder updateFunder (Funder system, Funder stored) { - stored.setLocalKey(system.getLocalKey()); + //stored.setLocalKey(system.getLocalKey()); if (system.getName() != null) { stored.setName(system.getName()); } if (system.getPolicy() != null) { stored.setPolicy(system.getPolicy()); } return stored; @@ -117,16 +141,16 @@ private Funder updateFunder (Funder system, Funder stored) { * @param stored the version of the User as read from Pass * @return a boolean which asserts whether the two supplied Users are "COEUS equal" */ - private boolean coeusUsersEqual(User system, User stored) { + private boolean userNeedsUpdate(User system, User stored) { //first the fields for which COEUS is authoritative - if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return false; - if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return false; - if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return false; - if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return false; - //next, other fields which require some reasoning to decide whether an system is necessary - if (system.getEmail() != null && stored.getEmail() == null) return false; - if (system.getDisplayName() != null && stored.getDisplayName() == null) return false; - return true; + if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return true; + if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return true; + if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return true; + if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return true; + //next, other fields which require some reasoning to decide whether an update is necessary + if (system.getEmail() != null && stored.getEmail() == null) return true; + if (system.getDisplayName() != null && stored.getDisplayName() == null) return true; + return false; } /** @@ -160,46 +184,32 @@ private User updateUser (User system, User stored) { /** * Compare two Grant objects. Note that the Lists of Co-Pis are compared as Sets - * @param system the version of the Grant as seen in the COEUS system pull + * @param system the version of the Grant as seen in the COES system pull * @param stored the version of the Grant as read from Pass * @return a boolean which asserts whether the two supplied Grants are "COEUS equal" */ - private boolean coeusGrantsEqual(Grant system, Grant stored) { - if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return false; - if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return false; - if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return false; - if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return false; - if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return false; - if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return false; - if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return false; - if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return false; - if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return false; - if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return false; - return true; + private boolean grantNeedsUpdate(Grant system, Grant stored) { + if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return true; + if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; + if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; + if (system.getEndDate() != null? system.getEndDate().isAfter(stored.getEndDate()) : stored.getEndDate() != null) return true; + return false; } /** - * Update a Pass Grant object with new information from COEUS + * Update a Pass Grant object with new information from COEUS - only updatable fields are considered. + * the PASS version is authoritative for the rest * * @param system the version of the Grant as seen in the COEUS system pull * @param stored the version of the Grant as read from Pass * @return the Grant object which represents the Pass object, with any new information from COEUS merged in */ private Grant updateGrant(Grant system, Grant stored) { - stored.setAwardNumber(system.getAwardNumber()); stored.setAwardStatus(system.getAwardStatus()); - stored.setLocalKey(system.getLocalKey()); - stored.setProjectName(system.getProjectName()); - stored.setPrimaryFunder(system.getPrimaryFunder()); - stored.setDirectFunder(system.getDirectFunder()); stored.setPi(system.getPi()); - stored.setCoPis(system.getCoPis()); - stored.setAwardDate(system.getAwardDate()); - stored.setStartDate(system.getStartDate()); + stored.setCoPis( system.getCoPis() ); stored.setEndDate(system.getEndDate()); return stored; } - } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java new file mode 100644 index 0000000..e4ebba0 --- /dev/null +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java @@ -0,0 +1,94 @@ +package org.dataconservancy.pass.grant.data; + +import org.dataconservancy.pass.model.Grant; + + +import java.net.URI; +import java.util.HashSet; + +/** + * This subclass is for the special case where we need to correct information on existing PASS objects for which we + * normally consider the PASS information to be authoritative. + */ +public class CoeusPassInitEntityUtil extends CoeusPassEntityUtil { + + @Override + public Grant update(Grant system, Grant stored) { + //adjust the system view of co-pis by merging in the stored view of pi and co-pis + for( URI uri : stored.getCoPis() ) { + if ( !system.getCoPis().contains(uri) ) { + system.getCoPis().add(uri); + } + } + + //need to be careful, system pi might be null if there is no record for it + //this is to finalize the version of the co-pi list we want to compare between + //system and stored + URI storedPi = stored.getPi(); + if ( system.getPi() != null ) { + if (!system.getPi().equals(storedPi)) { + // stored.setPi( system.getPi() ); + if (!system.getCoPis().contains(storedPi)) { + system.getCoPis().add(storedPi); + } + system.getCoPis().remove(system.getPi()); + } + } else { //system view is null, do not trigger update based on this field + system.setPi( storedPi ); + } + + //now system view has all available info we want in this grant - look for update trigger + if (this.grantNeedsUpdate(system, stored)) { + return this.updateGrant(system, stored); + } + return null; + } + + + /** + * Compare two Grant objects. Note that the Lists of Co-Pis are compared as Sets + * @param system the version of the Grant as seen in the COEUS system pull + * @param stored the version of the Grant as read from Pass + * @return a boolean which asserts whether the stored grant needs to be updated + */ + + private boolean grantNeedsUpdate(Grant system, Grant stored) { + if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return true; + if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return true; + if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return true; + if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return true; + if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return true; + if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; + if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; + if (system.getAwardDate() != null? system.getAwardDate().isBefore(stored.getAwardDate()) : stored.getAwardDate() != null) return true; + if (system.getStartDate() != null? system.getStartDate().isBefore(stored.getStartDate()) : stored.getStartDate() != null) return true; + if (system.getEndDate() != null? system.getEndDate().isAfter(stored.getEndDate()) : stored.getEndDate() != null) return true; + return false; + } + + /** + * Update a Pass Grant object with new information from COEUS + * + * @param system the version of the Grant as seen in the COEUS system pull + * @param stored the version of the Grant as read from Pass + * @return the Grant object which represents the Pass object, with any new information from COEUS merged in + */ + private Grant updateGrant(Grant system, Grant stored) { + stored.setAwardNumber(system.getAwardNumber()); + stored.setAwardStatus(system.getAwardStatus()); + stored.setLocalKey(system.getLocalKey()); + stored.setProjectName(system.getProjectName()); + stored.setPrimaryFunder(system.getPrimaryFunder()); + stored.setDirectFunder(system.getDirectFunder()); + stored.setPi( system.getPi()); + stored.setCoPis( system.getCoPis() ); + + //since this is essentially an initial pull, we can just take the system values + stored.setAwardDate(system.getAwardDate()); + stored.setStartDate(system.getStartDate()); + stored.setEndDate(system.getEndDate()); + return stored; + } + +} diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index b419e7a..118a60a 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory; import java.net.URI; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.ListIterator; @@ -47,29 +46,27 @@ public class DefaultPassUpdater implements PassUpdater{ private String DOMAIN = "default.domain"; - private static Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); + private static final Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); private String latestUpdateString = ""; - private PassClient passClient = PassClientFactory.getPassClient(); - private PassUpdateStatistics statistics = new PassUpdateStatistics(); - private PassEntityUtil passEntityUtil; + private final PassClient passClient; + private final PassUpdateStatistics statistics = new PassUpdateStatistics(); + private final PassEntityUtil passEntityUtil; - //used in test classes - private Map grantUriMap = new HashMap<>(); + private final Map grantUriMap = new HashMap<>(); - //used in unit test //some entities may be referenced many times during an update, but just need to be updated the first time //they are encountered. these include Users and Funders. we save the overhead of redundant updates //of these by looking them up here; if they are on the Map, they have already been processed - // - private Map funderMap = new HashMap<>(); - private Map userMap = new HashMap<>(); + private final Map funderMap = new HashMap<>(); + private final Map userMap = new HashMap<>(); private String mode; DefaultPassUpdater(PassEntityUtil passEntityUtil) { this.passEntityUtil = passEntityUtil; + this.passClient = PassClientFactory.getPassClient(); } //used in unit testing for injecting a mock client @@ -111,9 +108,7 @@ private void updateGrants(Collection> results) { LOG.info("Processing result set with " + results.size() + " rows"); boolean modeChecked = false; - for(Map rowMap : results){ - - String grantLocalKey; + for(Map rowMap : results) { if (!modeChecked) { if (!rowMap.containsKey(C_GRANT_LOCAL_KEY)) {//we always have this for grants @@ -123,22 +118,87 @@ private void updateGrants(Collection> results) { } } - grantLocalKey = rowMap.get(C_GRANT_LOCAL_KEY); + String grantLocalKey = rowMap.get(C_GRANT_LOCAL_KEY); + //get funder local keys. if a primary funder is not specified, we set it to the direct funder String directFunderLocalKey = rowMap.get(C_DIRECT_FUNDER_LOCAL_KEY); String primaryFunderLocalKey = rowMap.get(C_PRIMARY_FUNDER_LOCAL_KEY); primaryFunderLocalKey = (primaryFunderLocalKey == null? directFunderLocalKey: primaryFunderLocalKey); - Grant grant; + + //we will need funder PASS URIs - retrieve or create them, + //updating the info on them if necessary + if ( !funderMap.containsKey(directFunderLocalKey)) { + Funder updatedFunder = buildDirectFunder(rowMap); + URI passFunderURI = updateFunderInPass(updatedFunder); + funderMap.put(directFunderLocalKey, passFunderURI); + } + + if( !funderMap.containsKey(primaryFunderLocalKey)) { + Funder updatedFunder = buildPrimaryFunder(rowMap); + URI passFunderURI = updateFunderInPass(updatedFunder); + funderMap.put(primaryFunderLocalKey, passFunderURI); + } + + //same for any users + String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); + String abbreviatedRole = rowMap.get(C_ABBREVIATED_ROLE); + if (!userMap.containsKey(employeeId)) { + User updatedUser = buildUser(rowMap); + URI passUserURI = updateUserInPass(updatedUser); + userMap.put(employeeId, passUserURI); + } + + //now we know all about our user and funders for this record + // let's get to the grant proper LOG.debug("Processing grant with localKey " + grantLocalKey); + //if this is the first record for this Grant, it will not be on the Map - //we process all data which is common to every record for this grant - //i.e., everything except the investigator(s) + Grant grant; if(!grantMap.containsKey(grantLocalKey)) { grant = new Grant(); + grant.setLocalKey(grantLocalKey); + grantMap.put(grantLocalKey, grant); + } + + grant = grantMap.get(grantLocalKey); + + //anybody who was ever a co-pi in an iteration will be in this list + if ( abbreviatedRole.equals("C") || abbreviatedRole.equals("K") ) { + URI userId = userMap.get( employeeId ); + if ( !grant.getCoPis().contains( userId ) ) { + grant.getCoPis().add( userId ); + statistics.addCoPi(); + } + } + + //now do things which may depend on the date - award date is the only one that changes + DateTime awardDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_AWARD_DATE, null)); + DateTime startDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_START_DATE, null)); + DateTime endDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_END_DATE, null)); + + //set values that should match earliest iteration of the grant. we wet these on the system record + //in case they are needed to update a stored grant record. + //these values will not override existing stored values unless the PassEntityUtil implementation + //allows it. + //we mostly have awardDate, but will use start date as a fallback if not + if ( (awardDate != null && (grant.getAwardDate() == null || awardDate.isBefore(grant.getAwardDate()))) || + awardDate == null && (startDate != null && (grant.getStartDate() == null || startDate.isBefore(grant.getStartDate())))) { + grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); grant.setAwardNumber(rowMap.get(C_GRANT_AWARD_NUMBER)); + grant.setDirectFunder(funderMap.get(directFunderLocalKey)); + grant.setPrimaryFunder(funderMap.get(primaryFunderLocalKey)); + grant.setStartDate(startDate); + grant.setAwardDate(awardDate); + } + //set values that should match the latest iteration of the grant + //use !isBefore in case more than one PI is specified, need to process more than one + //we mostly have awardDate, but will use end date as a fallback if not + if ( (awardDate != null && (grant.getAwardDate() == null || !awardDate.isBefore(grant.getAwardDate()))) || + awardDate == null && ( endDate != null && (grant.getEndDate() == null || !endDate.isBefore(grant.getEndDate()))) ){ + grant.setEndDate( endDate); + //status should be the latest one String status = rowMap.getOrDefault(C_GRANT_AWARD_STATUS, null); - if (status != null) { switch (status) { case "Active": @@ -154,62 +214,25 @@ private void updateGrants(Collection> results) { grant.setAwardStatus(null); } - grant.setLocalKey(grantLocalKey); - grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); - grant.setAwardDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_AWARD_DATE, null))); - grant.setStartDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_START_DATE, null))); - grant.setEndDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_END_DATE, null))); - - //process direct funder, and primary funder if we have one - //update funder(s) in Pass as needed - if (funderMap.containsKey(directFunderLocalKey)) { - grant.setDirectFunder(funderMap.get(directFunderLocalKey)); - } else { - Funder updatedFunder = buildDirectFunder(rowMap); - URI passFunderURI = updateFunderInPass(updatedFunder); - funderMap.put(directFunderLocalKey, passFunderURI); - grant.setDirectFunder(passFunderURI); - } - - if(funderMap.containsKey(primaryFunderLocalKey)) { - grant.setPrimaryFunder(funderMap.get(primaryFunderLocalKey)); - } else { - Funder updatedFunder = buildPrimaryFunder(rowMap); - URI passFunderURI = updateFunderInPass(updatedFunder); - funderMap.put(primaryFunderLocalKey, passFunderURI); - grant.setPrimaryFunder(passFunderURI); - } - - grant.setCoPis(new ArrayList<>());//we will build this from scratch in either case - grantMap.put(grantLocalKey, grant);//save the state of this Grant - } - - //now we process the User (investigator) - grant = grantMap.get(grantLocalKey); - String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); - String abbreviatedRole = rowMap.get(C_ABBREVIATED_ROLE); - - if(abbreviatedRole.equals("C") || abbreviatedRole.equals("K") || grant.getPi() == null) { - if (!userMap.containsKey(employeeId)) { - User updatedUser = buildUser(rowMap); - URI passUserURI = updateUserInPass(updatedUser); - userMap.put(employeeId, passUserURI); - } - - //now our User URI is on the map - let's process: - if (abbreviatedRole.equals("P")) { - if (grant.getPi() == null) { - grant.setPi(userMap.get(employeeId)); + //we want the PI to be the one listed on the most recent grant iteration + if ( abbreviatedRole.equals("P") ) { + URI userId=userMap.get(employeeId); + URI oldPiId=grant.getPi(); + grant.setPi(userId); + grant.getCoPis().remove(userId); + if ( oldPiId == null ) { statistics.addPi(); - } else {// handle data with duplicate PIs - grant.getCoPis().add(userMap.get(employeeId)); - statistics.addCoPi(); + } else { + if ( !oldPiId.equals(userId) ) { + if ( !grant.getCoPis().contains(oldPiId) ) { + grant.getCoPis().add(oldPiId); + statistics.addCoPi(); + } + } } - } else if ((abbreviatedRole.equals("C") || abbreviatedRole.equals("K")) && !grant.getCoPis().contains(userMap.get(employeeId))) { - grant.getCoPis().add(userMap.get(employeeId)); - statistics.addCoPi(); } } + //we are done with this record, let's save the state of this Grant grantMap.put(grantLocalKey, grant); //see if this is the latest grant updated @@ -374,8 +397,7 @@ private URI updateFunderInPass(Funder systemFunder) { if ((updatedFunder = passEntityUtil.update(systemFunder, storedFunder)) != null) {//need to update passClient.updateResource(updatedFunder); statistics.addFundersUpdated(); - }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything - //this can happen if the Grant was updated in COEUS only with information we don't consume here + } } else {//don't have a stored Funder for this URI - this one is new to Pass if (systemFunder.getName() != null) {//only add if we have a name passFunderURI = passClient.createResource(systemFunder); @@ -396,7 +418,7 @@ private URI updateUserInPass(User systemUser) { //we first check to see if the user is known by the Hopkins ID. If not, we check the employee ID. //last attempt is the JHED ID. this order is specified by the order of the List as constructed on updatedUser URI passUserUri = null; - ListIterator idIterator = systemUser.getLocatorIds().listIterator(); + ListIterator idIterator = systemUser.getLocatorIds().listIterator(); while (passUserUri == null && idIterator.hasNext()) { String id = String.valueOf(idIterator.next()); @@ -418,8 +440,7 @@ private URI updateUserInPass(User systemUser) { } passClient.updateResource(updatedUser); statistics.addUsersUpdated(); - }//if the Pass version is COEUS-equal to our version from the update, and there are no null fields we care about, - //we don't have to do anything. this can happen if the User was updated in COEUS only with information we don't consume here + } } else if (! mode.equals("user")) {//don't have a stored User for this URI - this one is new to Pass //but don't update if we are in user mode - just update existing users passUserUri = passClient.createResource(systemUser); @@ -451,16 +472,14 @@ private URI updateGrantInPass(Grant systemGrant) { } Grant updatedGrant; if ( (updatedGrant = passEntityUtil.update(systemGrant, storedGrant)) != null) {//need to update - LOG.debug("Updating grant with localKey " + storedGrant.getLocalKey() + " to localKey " + systemGrant.getLocalKey()); passClient.updateResource(updatedGrant); statistics.addGrantsUpdated(); - LOG.debug("Updating grant with award number " + systemGrant.getLocalKey()); - }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything - //this can happen if the Grant was updated in COEUS only with information we don't consume here + LOG.debug("Updating grant with local key " + systemGrant.getLocalKey()); + } } else {//don't have a stored Grant for this URI - this one is new to Pass passGrantURI = passClient.createResource(systemGrant); statistics.addGrantsCreated(); - LOG.debug("Creating grant with award number " + systemGrant.getLocalKey()); + LOG.debug("Creating grant with local key " + systemGrant.getLocalKey()); } return passGrantURI; } @@ -502,7 +521,6 @@ public PassUpdateStatistics getStatistics() { return statistics; } - public Map getGrantUriMap() { return grantUriMap; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java index 4c7771a..0efeaa1 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java @@ -25,6 +25,9 @@ import okhttp3.Response; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.Properties; import static java.util.concurrent.TimeUnit.SECONDS; @@ -33,7 +36,9 @@ * This util class is designed to hit a service which provides resolution of one identifier to another. The two endpoints * provide lookups between our Hopkins Id, which is a durable identifier for all members of the Hopkins community, and the * employee ID, which is a durable identifier for all Hopkins employees. This lookup service is necessary because we do not - * have access to the wider identifier in out grants data source. + * have access to the wider identifier in our grants data source. + * + * @author jrm */ class DirectoryServiceUtil { @@ -48,6 +53,10 @@ class DirectoryServiceUtil { private final OkHttpClient client; private final JsonFactory factory = new JsonFactory(); + //these are for caching results + private final Map hopkins2ee = new HashMap<>(); + private final Map ee2hopkins = new HashMap<>(); + DirectoryServiceUtil(Properties connectionProperties) { if (connectionProperties != null) { @@ -76,8 +85,8 @@ private enum Type { EMPLOYEE2HOPKINS("EmployeeID_to_HopkinsID", "employeeid"), HOPKINS2EMPLOYEE("HopkinsID_to_EmployeeID", "hopkinsid"); - private String serviceUrlEnding; - private String queryParameter; + private final String serviceUrlEnding; + private final String queryParameter; Type(String serviceUrlEnding, String queryParameter) { this.serviceUrlEnding = serviceUrlEnding; @@ -95,24 +104,40 @@ public String getQueryParameter() { /** - * Return Hopkins ID for a given employee ID + * Return Hopkins ID for a given employee ID. we cache lookups in a map so that we only need to perform + * a lookup once per session per user. * * @param employeeId the user's employeeId * @return the user's Hopkins ID - should never be null * @throws IOException if the service cannot be reached */ String getHopkinsIdForEmployeeId(String employeeId) throws IOException { - return askDirectoryForMappedValue(Type.EMPLOYEE2HOPKINS, employeeId); + String hopkinsId; + if ( !ee2hopkins.containsKey( employeeId) ) { + hopkinsId = askDirectoryForMappedValue(Type.EMPLOYEE2HOPKINS, employeeId); + ee2hopkins.put( employeeId, hopkinsId ); + } else { + hopkinsId = ee2hopkins.get ( employeeId ); + } + return hopkinsId; } /** - * + * Return employee ID for a given Hopkins ID. we cache lookups in a map so that we only need to perform + * a lookup once per session per user. * @param hopkinsId the user's Hopkins ID * @return the user's employee ID if it exists; null if it does not * @throws IOException if there is an IO exception */ String getEmployeeIdForHopkinsId(String hopkinsId) throws IOException { - return askDirectoryForMappedValue(Type.HOPKINS2EMPLOYEE, hopkinsId); + String employeeId; + if ( !hopkins2ee.containsKey( hopkinsId ) ) { + employeeId = askDirectoryForMappedValue(Type.HOPKINS2EMPLOYEE, hopkinsId); + hopkins2ee.put( hopkinsId, employeeId ); + } else { + employeeId = hopkins2ee.get ( hopkinsId ); + } + return employeeId; } private String askDirectoryForMappedValue(Type type, String sourceId) throws IOException { @@ -124,7 +149,7 @@ private String askDirectoryForMappedValue(Type type, String sourceId) throws IOE } serviceUrl = directoryBaseUrl + suffix; - HttpUrl.Builder urlBuilder = HttpUrl.parse(serviceUrl).newBuilder().addQueryParameter(name, sourceId); + HttpUrl.Builder urlBuilder = Objects.requireNonNull(HttpUrl.parse(serviceUrl)).newBuilder().addQueryParameter(name, sourceId); String url = urlBuilder.build().toString(); Request request = new Request.Builder().header("client_id", directoryClientId) @@ -132,7 +157,8 @@ private String askDirectoryForMappedValue(Type type, String sourceId) throws IOE Response response = client.newCall(request).execute(); - JsonParser parser = factory.createParser(response.body().string()); + assert response.body() != null; + JsonParser parser = factory.createParser(response.body().string()); String mappedValue = null; while (!parser.isClosed()) { JsonToken jsonToken = parser.nextToken(); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardFieldNames.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardFieldNames.java index 8579a46..451d98a 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardFieldNames.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardFieldNames.java @@ -23,7 +23,7 @@ public class HarvardFieldNames { public static final String H_GRANT_NAME = "Grant name"; public static final String H_INV_FIRST_NAME = "PI First Name"; public static final String H_INV_LAST_NAME ="PI Last Name"; - public static final String H_INV_ID = "PI Harvard ID";//guaranteed to be in grant data + //public static final String H_INV_ID = "PI Harvard ID";//guaranteed to be in grant data public static final String H_INV_EMAIL = "PI Email"; //public static final String H_INV_ROLE = public static final String H_GRANT_START_DATE = "Grant start date"; diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java index 9ba0f01..e510ee5 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java @@ -38,8 +38,8 @@ import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; /** - * This implementation of the Grant Connector interface processes date given to us in an Excel spreadsheet. We take in the information to produce - * an intermediate date object which is compatible with our PASS data loading setup. + * This implementation of the Grant Connector interface processes data given to us in an Excel spreadsheet. We take in the information to produce + * an intermediate data object which is compatible with our PASS data loading setup. * * @author jrm */ @@ -49,7 +49,7 @@ public class HarvardPilotConnector implements GrantConnector { protected static final String HARVARD_DATA_FILE_PATH_PROPERTY = "harvard.data.file.path"; private String xlsxDataFilePath; - private Properties funderPolicyProperties; + private final Properties funderPolicyProperties; private static final Logger LOG = LoggerFactory.getLogger(HarvardPilotConnector.class); @@ -99,7 +99,7 @@ public List> retrieveUpdates(String queryString, String mode Map rowMap = new HashMap<>(); rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, localKey.toString()); rowMap.put(C_PRIMARY_FUNDER_NAME, funderNameMap.get(localKey.toString())); - if (funderPolicyProperties.keySet().contains(localKey)) { + if (funderPolicyProperties.containsKey(localKey)) { rowMap.put(C_PRIMARY_FUNDER_POLICY, funderPolicyProperties.getProperty(localKey.toString())); } resultSet.add(rowMap); @@ -111,7 +111,8 @@ public List> retrieveUpdates(String queryString, String mode LOG.debug("Processing grant record ..."); //we only process rows with a Harvard ID - String employeeId = stringify(cells.getCell(6)); //G: user Harvard ID + String employeeId = stringify(cells.getCell(6)); + //String email = stringify(cells.getCell(7)); if(employeeId != null && employeeId.length()>0) { Map rowMap = new HashMap<>(); @@ -125,7 +126,7 @@ public List> retrieveUpdates(String queryString, String mode String role = stringify(cells.getCell(5)); //F: Role rowMap.put(C_ABBREVIATED_ROLE, sortRole(role)); - rowMap.put(C_USER_EMPLOYEE_ID, employeeId); + rowMap.put(C_USER_EMPLOYEE_ID, stringify(cells.getCell(6))); //row G used to be Harvard id, we hack it for now rowMap.put(C_USER_EMAIL, stringify(cells.getCell(7))); //H: PI Email String funderLocalKey = stringify(cells.getCell(8)); //I: Funder ID @@ -156,7 +157,7 @@ public List> retrieveUpdates(String queryString, String mode * Stringify a cell's contents. Since our numerical cells which are dates are all integers but are interpreted * by the POI framework as doubles, we correct these to integers * - * @param cell + * @param cell spreadsheet cell * @return a string representing a cell's contents */ private String stringify(Cell cell) { diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java index 6c27c75..931207a 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java @@ -21,6 +21,8 @@ import org.dataconservancy.pass.model.User; import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; /** @@ -45,7 +47,7 @@ public class HarvardPilotPassEntityUtil implements PassEntityUtil{ * @return the updated Funder - null if the Funder does not need to be updated */ public Funder update(Funder system, Funder stored) { - if (!harvardFundersEqual(system, stored)) { + if (funderNeedsUpdate(system, stored)) { return updateFunder(system, stored); } return null; @@ -59,7 +61,7 @@ public Funder update(Funder system, Funder stored) { * @return the updated User - null if the User does not need to be updated */ public User update(User system, User stored) { - if (!harvardUsersEqual(system, stored)) { + if (userNeedsUpdate(system, stored)) { return updateUser(system, stored); } return null; @@ -73,7 +75,7 @@ public User update(User system, User stored) { * @return the updated object - null if the Grant does not need to be updated */ public Grant update(Grant system, Grant stored) { - if (!harvardGrantsEqual(system, stored)) { + if (grantNeedsUpdate(system, stored)) { return updateGrant(system, stored); } return null; @@ -86,13 +88,13 @@ public Grant update(Grant system, Grant stored) { * @param stored the version of the Funder as read from Pass * @return a boolean which asserts whether the two supplied Funders are "Harvard equal" */ - private boolean harvardFundersEqual(Funder system, Funder stored) { + private boolean funderNeedsUpdate(Funder system, Funder stored) { //this adjustment handles the case where we take data from policy.properties file, which has no name info - if (system.getName() != null && !system.getName().equals(stored.getName())) return false; - if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return false; - return true; + if (system.getName() != null && !system.getName().equals(stored.getName())) return true; + if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return true; + return false; } /** @@ -117,16 +119,16 @@ private Funder updateFunder (Funder system, Funder stored) { * @param stored the version of the User as read from Pass * @return a boolean which asserts whether the two supplied Users are "Harvard equal" */ - private boolean harvardUsersEqual(User system, User stored) { + private boolean userNeedsUpdate(User system, User stored) { //first the fields for which Harvard is authoritative - if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return false; - //if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return false; - if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return false; - if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return false; - //next, other fields which require some reasoning to decide whether an system is necessary - if (system.getEmail() != null && stored.getEmail() == null) return false; - if (system.getDisplayName() != null && stored.getDisplayName() == null) return false; - return true; + if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return true; + //if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return true; + if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return true; + if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return true; + //next, other fields which require some reasoning to decide whether an update is necessary + if (system.getEmail() != null && stored.getEmail() == null) return true; + if (system.getDisplayName() != null && stored.getDisplayName() == null) return true; + return false; } /** @@ -143,7 +145,11 @@ private User updateUser (User system, User stored) { stored.setFirstName(system.getFirstName()); //stored.setMiddleName(system.getMiddleName()); stored.setLastName(system.getLastName()); - stored.setLocatorIds(system.getLocatorIds()); + Set idSet = new HashSet<>(); + idSet.addAll(stored.getLocatorIds()); + idSet.addAll(system.getLocatorIds()); + stored.setLocatorIds(idSet.stream().collect(Collectors.toList())); + //stored.setLocatorIds(system.getLocatorIds()); stored.setEmail(system.getEmail()); stored.setDisplayName(system.getDisplayName()); return stored; @@ -155,18 +161,18 @@ private User updateUser (User system, User stored) { * @param stored the version of the Grant as read from Pass * @return a boolean which asserts whether the two supplied Grants are "Harvard equal" */ - private boolean harvardGrantsEqual(Grant system, Grant stored) { - if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return false; - //if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return false; - if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return false; - if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return false; - if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return false; - if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return false; - if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return false; - //if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return false; - if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return false; - if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return false; + private boolean grantNeedsUpdate(Grant system, Grant stored) { + if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return true; + //if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return true; + if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return true; + if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return true; + if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return true; + if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; + if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; + //if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return true; + if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return true; + if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return true; return true; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java index e17fa81..74ba00f 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java @@ -16,11 +16,51 @@ package org.dataconservancy.pass.grant.data; -public class HarvardPilotPassUpdater extends DefaultPassUpdater { +import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.model.User; +import org.dataconservancy.pass.model.support.Identifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; + +public class HarvardPilotPassUpdater extends BasicPassUpdater { + + private static final Logger LOG = LoggerFactory.getLogger(HarvardPilotPassUpdater.class); + private static final String DOMAIN = "harvard.edu"; public HarvardPilotPassUpdater () { super(new HarvardPilotPassEntityUtil()); - super.setDomain("harvard.edu"); + super.setDomain(DOMAIN); + } + + public HarvardPilotPassUpdater (PassClient passClient) { + super(new HarvardPilotPassEntityUtil(), passClient ); + super.setDomain(DOMAIN); + } + + @Override + User buildUser(Map rowMap) { + User user = new User(); + user.setFirstName(rowMap.get(C_USER_FIRST_NAME)); + //user.setMiddleName(rowMap.getOrDefault(C_USER_MIDDLE_NAME, null)); + user.setLastName(rowMap.get(C_USER_LAST_NAME)); + user.setDisplayName(rowMap.get(C_USER_FIRST_NAME) + " " + rowMap.get(C_USER_LAST_NAME)); + String email =rowMap.get(C_USER_EMAIL); + user.setEmail(email); + // + //Build the List of locatorIds - put the most reliable ids first + //for the pilot, we construct the eppn locatorId from the email address + String eppn = email.split("@")[0]; + if (eppn != null) { + String INSTITUTIONAL_ID_TYPE = "jhed"; + user.getLocatorIds().add(new Identifier(DOMAIN, INSTITUTIONAL_ID_TYPE, eppn).serialize()); + } + user.getRoles().add(User.Role.SUBMITTER); + LOG.debug("Built user with institutional ID " + eppn); + return user; } } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java new file mode 100644 index 0000000..8c0e27c --- /dev/null +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java @@ -0,0 +1,67 @@ +package org.dataconservancy.pass.grant.data; + +import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.model.User; +import org.dataconservancy.pass.model.support.Identifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.C_USER_INSTITUTIONAL_ID; + +public class JhuPassInitUpdater extends DefaultPassUpdater { + + private static final Logger LOG = LoggerFactory.getLogger(JhuPassInitUpdater.class); + private static final String DOMAIN = "johnshopkins.edu"; + + public JhuPassInitUpdater(PassClient passClient) + { + super(new CoeusPassInitEntityUtil(), passClient); + super.setDomain(DOMAIN); + } + + public JhuPassInitUpdater() { + super(new CoeusPassInitEntityUtil()); + super.setDomain(DOMAIN); + } + + @Override + User buildUser(Map rowMap) { + User user = new User(); + user.setFirstName(rowMap.get(C_USER_FIRST_NAME)); + if (rowMap.containsKey(C_USER_MIDDLE_NAME)) { + user.setMiddleName(rowMap.get(C_USER_MIDDLE_NAME)); + } + user.setLastName(rowMap.get(C_USER_LAST_NAME)); + user.setDisplayName(rowMap.get(C_USER_FIRST_NAME) + " " + rowMap.get(C_USER_LAST_NAME)); + user.setEmail(rowMap.get(C_USER_EMAIL)); + String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); + String hopkinsId = null; + if (rowMap.containsKey(C_USER_HOPKINS_ID)) { + hopkinsId = rowMap.get(C_USER_HOPKINS_ID); + } + String jhedId = null; + if (rowMap.get(C_USER_INSTITUTIONAL_ID) != null) { + jhedId = rowMap.get(C_USER_INSTITUTIONAL_ID).toLowerCase(); + } + //Build the List of locatorIds - put the most reliable ids first + if (employeeId != null) { + String EMPLOYEE_ID_TYPE = "employeeid"; + user.getLocatorIds().add(new Identifier(DOMAIN, EMPLOYEE_ID_TYPE, employeeId).serialize()); + } + if (hopkinsId != null) { + String HOPKINS_ID_TYPE = "hopkinsid"; + user.getLocatorIds().add(new Identifier(DOMAIN, HOPKINS_ID_TYPE, hopkinsId).serialize()); + } + if (jhedId != null) { + String JHED_ID_TYPE = "jhed"; + user.getLocatorIds().add(new Identifier(DOMAIN, JHED_ID_TYPE, jhedId).serialize()); + } + user.getRoles().add(User.Role.SUBMITTER); + LOG.debug("Built user with employee ID " + employeeId); + return user; + } + +} diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassUpdater.java index 9bfca3b..8d2c678 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassUpdater.java @@ -37,17 +37,18 @@ public class JhuPassUpdater extends DefaultPassUpdater { - private static Logger LOG = LoggerFactory.getLogger(JhuPassUpdater.class); + private static final Logger LOG = LoggerFactory.getLogger(JhuPassUpdater.class); + private static final String DOMAIN = "johnshopkins.edu"; public JhuPassUpdater(PassClient passClient) { super(new CoeusPassEntityUtil(), passClient); - super.setDomain("johnshopkins.edu"); + super.setDomain(DOMAIN); } public JhuPassUpdater() { super(new CoeusPassEntityUtil()); - super.setDomain("johnshopkins.edu"); + super.setDomain(DOMAIN); } @Override @@ -70,7 +71,6 @@ User buildUser(Map rowMap) { jhedId = rowMap.get(C_USER_INSTITUTIONAL_ID).toLowerCase(); } //Build the List of locatorIds - put the most reliable ids first - String DOMAIN = "johnshopkins.edu"; if (employeeId != null) { String EMPLOYEE_ID_TYPE = "employeeid"; user.getLocatorIds().add(new Identifier(DOMAIN, EMPLOYEE_ID_TYPE, employeeId).serialize()); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/PassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/PassEntityUtil.java index a839d2b..7ed2f4a 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/PassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/PassEntityUtil.java @@ -20,6 +20,11 @@ import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.User; +/** + * This interface defines update methods for existing (stored) grants, users and funders. In practice, implementations will generally + * split this functionalite into two steps - the first step will reason over a stored object and the object developed in a system pull, + * and make th decision whether the object needs to be updated. The update methods eill act on that decision appropriately. + */ public interface PassEntityUtil { /** diff --git a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/CoeusConnectorTest.java b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/CoeusConnectorTest.java index 1e0d81d..1a48492 100644 --- a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/CoeusConnectorTest.java +++ b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/CoeusConnectorTest.java @@ -60,11 +60,7 @@ public void testBuildGrantString() { " A.AWARD_START, A.AWARD_END, A.SPONSOR, A.SPOSNOR_CODE, A.UPDATE_TIMESTAMP, B.ABBREVIATED_ROLE, B.EMPLOYEE_ID," + " C.FIRST_NAME, C.MIDDLE_NAME, C.LAST_NAME, C.EMAIL_ADDRESS, C.JHED_ID, D.SPONSOR_NAME, D.SPONSOR_CODE" + " FROM" + - " COEUS.JHU_FACULTY_FORCE_PROP A INNER JOIN " + - " (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP" + - " FROM COEUS.JHU_FACULTY_FORCE_PROP" + - " GROUP BY GRANT_NUMBER) LATEST" + - " ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER" + + " COEUS.JHU_FACULTY_FORCE_PROP A" + " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B" + " ON A.INST_PROPOSAL = B.INST_PROPOSAL" + " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C" + @@ -84,11 +80,7 @@ public void testBuildGrantString() { " A.AWARD_START, A.AWARD_END, A.SPONSOR, A.SPOSNOR_CODE, A.UPDATE_TIMESTAMP, B.ABBREVIATED_ROLE, B.EMPLOYEE_ID," + " C.FIRST_NAME, C.MIDDLE_NAME, C.LAST_NAME, C.EMAIL_ADDRESS, C.JHED_ID, D.SPONSOR_NAME, D.SPONSOR_CODE" + " FROM" + - " COEUS.JHU_FACULTY_FORCE_PROP A INNER JOIN " + - " (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP" + - " FROM COEUS.JHU_FACULTY_FORCE_PROP" + - " GROUP BY GRANT_NUMBER) LATEST" + - " ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER" + + " COEUS.JHU_FACULTY_FORCE_PROP A" + " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B" + " ON A.INST_PROPOSAL = B.INST_PROPOSAL" + " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C" + diff --git a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtilTest.java b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtilTest.java index e976fdc..86dd258 100644 --- a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtilTest.java +++ b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtilTest.java @@ -33,21 +33,29 @@ * This is a test class for a simple directory lookup service running at the endpoint specified by "serviceUrl" below * the service type completes the URL, and the client id and client secret are supplied as headers. * - * values for serviceUrl, clientId and slientSecret, must be supplied below. + * values for serviceUrl, clientId and clientSecret, must be supplied below. * * This test has been run against the running service with valid parameters and arguments supplied to the methods - this class has been * cleaned up after successful testing. because of the simplicity and isolation of this class, it does not need to be tested * every build - just when something about the service changes. so we ignore it for now. + * + * To test, provide real connection parameter values, and a real kopkins id / employee id pair + * + * @author jrm */ @Ignore public class DirectoryServiceUtilTest { private DirectoryServiceUtil underTest; + private final String validEeid = ""; //actual employee id + private final String validHopkinsId = ""; //actual matching hopkins id + @Before public void setup() { - String serviceUrl = "https://the.service/url"; + final String serviceUrl = "https://the.service/url"; final String clientId = "the-client-id"; final String clientSecret = "the-client-secret"; + Properties connectionProperties = new Properties(); connectionProperties.setProperty(DIRECTORY_SERVICE_BASE_URL, serviceUrl); connectionProperties.setProperty(DIRECTORY_SERVICE_CLIENT_ID, clientId); @@ -57,16 +65,14 @@ public void setup() { @Test public void testGetHopkinsId() throws java.io.IOException { - String result = underTest.getHopkinsIdForEmployeeId("supply valid argument here"); - Assert.assertEquals("expected value", result); - System.out.println(result); - + String result = underTest.getHopkinsIdForEmployeeId(validEeid); + Assert.assertEquals(validHopkinsId, result); } @Test public void testGetEmployeeId() throws java.io.IOException { - String result = underTest.getEmployeeIdForHopkinsId("A58756"); - Assert.assertEquals("expected value", result); + String result = underTest.getEmployeeIdForHopkinsId(validHopkinsId); + Assert.assertEquals(validEeid, result); } @Test diff --git a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java index 01bab86..b14bc99 100644 --- a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java +++ b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java @@ -101,16 +101,26 @@ public void testRetrieveGrantUpdates() throws IOException { "C", "P" }; - String[] harvardIds = {"86753091", - "86753092", - "86753093", - "86753094", - "86753095", + String[] harvardIds = {"wdrumstick", + "gflanksteak", + "ddbrisket", + "abacon", + "efarmer", "", - "86753097", - "86753098", - "86753099" + "csteer", + "dbovine", + "bcow" }; + /* String[] harvardIds = {"wdrumstick@harvard.edu", + "gflanksteak@harvard.edu", + "ddbrisket@harvard.edu", + "abacon@harvard.edu", + "efarmer@harvard.edu", + "", + "csteer@harvard.edu", + "dbovine@harvard.edu", + "bcow@harvard.edu" + }; */ String[] emails = {"wdrumstick@harvard.edu", "gflanksteak@harvard.edu", "ddbrisket@harvard.edu", diff --git a/pass-grant-data/src/test/resources/HarvardPASSTestData.xlsx b/pass-grant-data/src/test/resources/HarvardPASSTestData.xlsx index cbf3015..cdab19d 100644 Binary files a/pass-grant-data/src/test/resources/HarvardPASSTestData.xlsx and b/pass-grant-data/src/test/resources/HarvardPASSTestData.xlsx differ diff --git a/pass-grant-integration/pom.xml b/pass-grant-integration/pom.xml index 5729d06..5c92c8e 100644 --- a/pass-grant-integration/pom.xml +++ b/pass-grant-integration/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.3.0 + 1.4.0 4.0.0 diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java new file mode 100644 index 0000000..4216376 --- /dev/null +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java @@ -0,0 +1,526 @@ +/* + * Copyright 2020 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dataconservancy.pass.grant.integration; + +import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.client.PassClientFactory; +import org.dataconservancy.pass.grant.data.*; +import org.dataconservancy.pass.model.Funder; +import org.dataconservancy.pass.model.Grant; + +import org.dataconservancy.pass.model.Policy; +import org.dataconservancy.pass.model.User; +import org.dataconservancy.pass.model.support.Identifier; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static java.lang.Thread.sleep; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +/** + * An integration test class for the BasicPassUpdater. + */ +@RunWith(MockitoJUnitRunner.class) +public class BasicPassUpdaterIT { + + private final List> resultSet = new ArrayList<>(); + private static final String DOMAIN = "default.domain"; + private static final String employeeidPrefix = DOMAIN + ":employeeid:"; + private final CoeusPassEntityUtil passEntityUtil = new CoeusPassEntityUtil(); + private final Map funderPolicyUriMap = new HashMap<>(); + + private String directFunderPolicyUriString1; + private String primaryFunderPolicyUriString1; + + + private final PassClient passClient = PassClientFactory.getPassClient(); + PassUpdater passUpdater = new BasicPassUpdater(new BasicPassEntityUtil(), passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + @Rule + public TemporaryFolder folder= new TemporaryFolder(); + + @Before + public void setup() { + + for (int i = 0; i < 10; i++) { + + String prefix = System.getProperty("pass.fedora.baseurl"); + if (!prefix.endsWith("/")) { + prefix = prefix + "/"; + } + + Policy policy = new Policy(); + policy.setTitle("Primary Policy" + i); + policy.setDescription("MOO"); + URI policyURI = passClient.createResource(policy); + String primaryPolicyUriString = policyURI.toString().substring(prefix.length()); + funderPolicyUriMap.put("PrimaryFunderPolicy"+i, policyURI); + + policy = new Policy(); + policy.setTitle("Direct Policy" + i); + policy.setDescription("MOO"); + policyURI = passClient.createResource(policy); + String directPolicyUriString = policyURI.toString().substring(prefix.length()); + funderPolicyUriMap.put("DirectFunderPolicy"+i, policyURI); + + if(i==1) { + directFunderPolicyUriString1 = directPolicyUriString; + primaryFunderPolicyUriString1 = primaryPolicyUriString; + } + + Map rowMap = new HashMap<>(); + rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + i); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + i); + rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + i); + rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); + rowMap.put(C_GRANT_START_DATE, "01/01/2001"); + rowMap.put(C_GRANT_END_DATE, "01/01/2002"); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + i); + rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + i); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + i); + rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + i); + + rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); + rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); + rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); + rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); + rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); + + rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + i + ":00:00.0"); + rowMap.put(C_ABBREVIATED_ROLE, (i % 2 == 0 ? "P" : "C")); + rowMap.put(C_DIRECT_FUNDER_POLICY, directPolicyUriString ); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryPolicyUriString); + + resultSet.add(rowMap); + } + + } + + /** + * The behavior of PassUpdate's updateGrants() method is to compare the data coming in on the ResultSet with + * the existing data in Pass, and create objects if Pass does not yet have them, and update them if they exist in Pass but + * there are differences in the fields for which the pull source is the authoritative source, or COEUS has a clue about other fields which are null + * on the PASS object. + * + * @throws InterruptedException - the exception + */ + @Test + public void updateGrantsIT() throws InterruptedException { + + passUpdater.updatePass(resultSet, "grant"); + + assertEquals(5, statistics.getPisAdded()); + assertEquals(5, statistics.getCoPisAdded()); + assertEquals(20, statistics.getFundersCreated()); + assertEquals(0, statistics.getFundersUpdated()); + assertEquals(10, statistics.getGrantsCreated()); + assertEquals(0, statistics.getGrantsUpdated()); + assertEquals("2018-01-01 09:00:00.0", statistics.getLatestUpdateString()); + assertEquals(10, statistics.getUsersCreated()); + assertEquals(0, statistics.getUsersUpdated()); + + assertEquals(10, passUpdater.getGrantUriMap().size()); + + for (URI grantUri : passUpdater.getGrantUriMap().keySet()) { + Grant grant = passUpdater.getGrantUriMap().get(grantUri); + Grant passGrant = passUpdater.getPassClient().readResource(grantUri, Grant.class); + assertNull(passEntityUtil.update(grant, passGrant)); //this means grants are "coeus-equal" + + } + + sleep(20000); + //try depositing the exact same resultSet. nothing should happen in Pass + passUpdater.updatePass(resultSet, "grant"); + + assertEquals(0, statistics.getFundersCreated()); + assertEquals(0, statistics.getFundersUpdated()); + assertEquals(0, statistics.getGrantsCreated()); + assertEquals(0, statistics.getGrantsUpdated()); + assertEquals(0, statistics.getUsersCreated()); + assertEquals(0, statistics.getUsersUpdated()); + + //now let's monkey with a few things; we expect to update the changed objects + Map rowMap = new HashMap<>(); + rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + 1); + rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOO"); + rowMap.put(C_GRANT_AWARD_DATE, "01/01/1999"); + rowMap.put(C_GRANT_START_DATE, "01/01/1999"); + rowMap.put(C_GRANT_END_DATE, "01/01/2002"); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + 1); + rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + 1 + "MOO"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + 1); + rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + 1); + + rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + 1); + rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + 1 + "MOOO"); + rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + 1 + "MOOOOO"); + rowMap.put(C_USER_EMAIL, C_USER_EMAIL + 1); + //rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + 1); + rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + 1); + //rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + 1); + + rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); + rowMap.put(C_ABBREVIATED_ROLE, ("C")); + + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString1); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString1); + + + resultSet.clear(); + resultSet.add(rowMap); + + passUpdater.updatePass(resultSet, "grant"); + assertEquals(0, statistics.getFundersCreated()); + assertEquals(1, statistics.getFundersUpdated()); + assertEquals(0, statistics.getGrantsCreated()); + assertEquals(1, statistics.getGrantsUpdated()); + assertEquals(1, statistics.getUsersUpdated()); + + sleep(20000); + + for (int i = 0; i < 10; i++) { + Grant grant = new Grant(); + grant.setAwardNumber(C_GRANT_AWARD_NUMBER + i); + grant.setAwardStatus(Grant.AwardStatus.ACTIVE); + String grantIdPrefix = DOMAIN + ":grant:"; + grant.setLocalKey(grantIdPrefix + C_GRANT_LOCAL_KEY + i); + grant.setProjectName(C_GRANT_PROJECT_NAME + i); + grant.setAwardDate(createJodaDateTime("01/01/2000")); + grant.setStartDate(createJodaDateTime("01/01/2001")); + grant.setEndDate(createJodaDateTime("01/01/2002")); + + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grant.getLocalKey()); + Grant passGrant = passClient.readResource(passGrantUri, Grant.class); + + assertEquals(grant.getAwardNumber(), passGrant.getAwardNumber()); + assertEquals(grant.getAwardStatus(), passGrant.getAwardStatus()); + assertEquals(grant.getLocalKey(), passGrant.getLocalKey()); + if( i==1 ) { + assertEquals(grant.getProjectName() + "MOO", passGrant.getProjectName() ); + assertEquals( createJodaDateTime("01/01/1999"), passGrant.getStartDate()); + assertEquals( createJodaDateTime("01/01/1999"), passGrant.getStartDate()); + } else { + assertEquals(grant.getProjectName(), passGrant.getProjectName()); + assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); + assertEquals(grant.getStartDate(), passGrant.getStartDate()); + } + assertEquals(grant.getEndDate(), passGrant.getEndDate()); + + //let's check funder stuff + Funder directFunder = new Funder(); + String funderIdPrefix = DOMAIN + ":funder:"; + directFunder.setLocalKey(funderIdPrefix + C_DIRECT_FUNDER_LOCAL_KEY + i); + directFunder.setName(C_DIRECT_FUNDER_NAME + i); + directFunder.setPolicy(funderPolicyUriMap.get("DirectFunderPolicy" + i)); + + URI directFunderUri = passClient.findByAttribute(Funder.class, "localKey", directFunder.getLocalKey()); + Funder passDirectFunder = passClient.readResource(directFunderUri, Funder.class); + if (i == 1) { + assertEquals(directFunder.getName() + "MOO", passDirectFunder.getName()); + assertEquals(directFunder.getLocalKey(), passDirectFunder.getLocalKey()); + assertEquals(passDirectFunder.getId(), passGrant.getDirectFunder()); + } else { + assertEquals(directFunder.getName(), passDirectFunder.getName()); + } + + Funder primaryFunder = new Funder(); + primaryFunder.setLocalKey(funderIdPrefix + C_PRIMARY_FUNDER_LOCAL_KEY + i); + primaryFunder.setName(C_PRIMARY_FUNDER_NAME + i); + primaryFunder.setPolicy(funderPolicyUriMap.get("PrimaryFunderPolicy" + i)); + + URI primaryFunderUri = passClient.findByAttribute(Funder.class, "localKey", primaryFunder.getLocalKey()); + Funder passPrimaryFunder = passClient.readResource(primaryFunderUri, Funder.class); + assertEquals(primaryFunder.getName(), passPrimaryFunder.getName()); + assertEquals(passPrimaryFunder.getId(), passGrant.getPrimaryFunder()); + assertEquals(primaryFunder.getLocalKey(), passPrimaryFunder.getLocalKey()); + assertEquals(primaryFunder.getPolicy(), passPrimaryFunder.getPolicy()); + + User user = new User(); + + //employeeId and localKey were localized by the grant loader + user.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + i); + user.setFirstName(C_USER_FIRST_NAME + i); + user.setMiddleName(C_USER_MIDDLE_NAME + i); + user.setLastName(C_USER_LAST_NAME + i); + user.setEmail(C_USER_EMAIL + i); + + URI userUri = null; + ListIterator idIterator = user.getLocatorIds().listIterator(); + + while (userUri == null && idIterator.hasNext()) { + String id = String.valueOf(idIterator.next()); + if (id != null) { + userUri = passClient.findByAttribute(User.class, "locatorIds", id); + } + } + + User passUser = passClient.readResource(userUri, User.class); + assertEquals(user.getFirstName(), passUser.getFirstName()); + if (i == 1) { + assertEquals(user.getMiddleName() + "MOOO", passUser.getMiddleName()); + assertEquals(user.getLastName() + "MOOOOO", passUser.getLastName()); + } else { + assertEquals(user.getMiddleName(), passUser.getMiddleName()); + assertEquals(user.getLastName(), passUser.getLastName()); + } + + assertEquals(user.getEmail(), passUser.getEmail()); + assertTrue(user.getLocatorIds().containsAll(passUser.getLocatorIds())); + assertTrue(passUser.getLocatorIds().containsAll(user.getLocatorIds())); + assertEquals(passUser.getLocatorIds().size(), user.getLocatorIds().size()); + + if (i % 2 == 0) { + assertNotNull(passGrant.getPi()); + assertEquals(0, passGrant.getCoPis().size()); + } else { + assertNull(passGrant.getPi()); + assertEquals(1, passGrant.getCoPis().size()); + } + + } + } + + + @Test + public void updateUsersIT() throws InterruptedException { + + User user10 = new User(); + user10.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + 10); + user10.setFirstName(C_USER_FIRST_NAME + 10); + user10.setMiddleName(C_USER_MIDDLE_NAME + 10); + user10.setLastName(C_USER_LAST_NAME + 10); + + URI passUserURI = passUpdater.getPassClient().createResource(user10); + + User passUser = passClient.readResource(passUserURI, User.class); + assertNull(passUser.getDisplayName()); + assertEquals(1, passUser.getLocatorIds().size()); + assertNull(passUser.getEmail()); + + sleep(20000); + + List> userResultSet = new ArrayList<>(); + + for (int i = 10; i < 12; i++) { + Map rowMap = new HashMap<>(); + + rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); + rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); + rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); + rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); + rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); + //rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); + rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); + userResultSet.add(rowMap); + } + + + passUpdater.updatePass(userResultSet, "user"); + + //now update from the set of two users - the second one is not in PASS, but is not created + //the first (user10) should be updated, with new fields added + assertEquals(0, statistics.getUsersCreated()); + assertEquals(1, statistics.getUsersUpdated()); + + assertNotNull(passUserURI); + User updatedUser = passUpdater.getPassClient().readResource(passUserURI, User.class); + + assertNotNull(updatedUser.getEmail()); + assertNotNull(updatedUser.getDisplayName()); + assertNotNull(updatedUser.getLocatorIds()); + assertTrue(updatedUser.getLocatorIds().contains(employeeidPrefix + C_USER_EMPLOYEE_ID + 10)); + assertEquals(C_USER_EMAIL + 10, updatedUser.getEmail()); + } + + /** + * Create some policies, deposit them into Fedora + * Then create a java data object linking funders to them + * this basically tests what happens when pulling in data from a policy properties file first, + * or from a coeus pull second + */ + @Test + public void updateFundersIT() throws InterruptedException { + Policy policy1 = new Policy(); + policy1. setTitle("Policy One"); + policy1. setDescription("Policy one Description"); + Policy policy2 = new Policy(); + policy2. setTitle("Policy Two"); + policy2. setDescription("Policy Two Description"); + + URI policy1Uri = passClient.createResource(policy1); + URI policy2Uri = passClient.createResource(policy2); + + assertNotNull(passClient.readResource(policy1Uri, Policy.class)); + assertNotNull(passClient.readResource(policy2Uri, Policy.class)); + + Funder funder1 = new Funder(); + String DOMAIN = "default.domain"; + String fullLocalKey = new Identifier(DOMAIN, "funder", "22229999").serialize(); + funder1.setLocalKey(fullLocalKey); + funder1.setName("Funder One"); + Funder funder2 = new Funder(); + fullLocalKey = new Identifier(DOMAIN, "funder", "33330000").serialize(); + funder2.setLocalKey(fullLocalKey);//use full localKey + funder2.setName("Funder Two"); + + URI funder1Uri = passClient.createResource(funder1); + URI funder2Uri = passClient.createResource(funder2); + + assertNotNull(passClient.readResource(funder1Uri, Funder.class)); + assertNotNull(passClient.readResource(funder2Uri, Funder.class)); + + + String policyString1 = policy1Uri.getPath().substring("/fcrepo/rest/".length()); + assertTrue(policyString1.startsWith("policies")); + String policyString2 = policy2Uri.getPath().substring("/fcrepo/rest/".length()); + assertTrue(policyString2.startsWith("policies")); + + List> funderResultSet = new ArrayList<>(); + + Map rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "22229999"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString1); + funderResultSet.add(rowMap); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "33330000"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); + funderResultSet.add(rowMap); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "88888888"); // this one does not exist in pass + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); + funderResultSet.add(rowMap); + + + sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch + + passUpdater.updatePass(funderResultSet, "funder"); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + assertNotNull(passClient.readResource(funder1Uri, Funder.class)); + assertNotNull(passClient.readResource(funder1Uri, Funder.class).getPolicy()); + assertNotNull(passClient.readResource(funder2Uri, Funder.class)); + assertNotNull(passClient.readResource(funder2Uri, Funder.class).getPolicy()); + assertEquals(policy1Uri, passClient.readResource(funder1Uri, Funder.class).getPolicy()); + assertEquals(policy2Uri, passClient.readResource(funder2Uri, Funder.class).getPolicy()); + + assertEquals(0, statistics.getFundersCreated()); + assertEquals(2, statistics.getFundersUpdated()); + + //coeus pulls will have the funder names, we should be able to add one we don't know about + + funderResultSet = new ArrayList<>(); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "22229999"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 1"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); //let's change policies for this one + funderResultSet.add(rowMap); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "33330000"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 2"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); + funderResultSet.add(rowMap); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "88888888"); // this one does not exist in pass + rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 3"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); + funderResultSet.add(rowMap); + + sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch + + passUpdater.updatePass(funderResultSet, "funder"); + statistics = passUpdater.getStatistics(); + + assertNotNull(passClient.readResource(funder1Uri, Funder.class)); + assertNotNull(passClient.readResource(funder1Uri, Funder.class).getPolicy()); + assertNotNull(passClient.readResource(funder2Uri, Funder.class)); + assertNotNull(passClient.readResource(funder2Uri, Funder.class).getPolicy()); + assertEquals(policy2Uri, passClient.readResource(funder1Uri, Funder.class).getPolicy()); + assertEquals(policy2Uri, passClient.readResource(funder2Uri, Funder.class).getPolicy()); + + assertEquals(1, statistics.getFundersCreated()); + assertEquals(2, statistics.getFundersUpdated()); + + + //DO AGAIN!! DO AGAIN!! + + sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch + + passUpdater.updatePass(funderResultSet, "funder"); + statistics = passUpdater.getStatistics(); + + assertEquals(0, statistics.getFundersCreated()); + assertEquals(0, statistics.getFundersUpdated()); + + } + + @Test + public void testSerializeAndDeserialize() throws IOException { + File serialized= folder.newFile("serializedData"); + + try (FileOutputStream fos = new FileOutputStream(serialized); + ObjectOutputStream out = new ObjectOutputStream(fos) + ){ + out.writeObject(resultSet); + } catch (IOException e) { + e.printStackTrace(); + } + + List> input = null; + try (FileInputStream fis = new FileInputStream(serialized); + ObjectInputStream in = new ObjectInputStream(fis) + ){ + input = (List>)in.readObject(); + } catch (IOException | ClassNotFoundException ex) { + ex.printStackTrace(); + } + + assertEquals(resultSet, input); + } + +} \ No newline at end of file diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java index 0f6663a..c30e138 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java @@ -18,13 +18,7 @@ import org.dataconservancy.pass.client.PassClient; import org.dataconservancy.pass.client.PassClientFactory; -import org.dataconservancy.pass.grant.data.DateTimeUtil; -import org.dataconservancy.pass.grant.data.HarvardPilotPassEntityUtil; -import org.dataconservancy.pass.grant.data.HarvardPilotPassUpdater; -import org.dataconservancy.pass.grant.data.PassEntityUtil; -import org.dataconservancy.pass.grant.data.PassUpdateStatistics; -import org.dataconservancy.pass.grant.data.PassUpdater; -import org.dataconservancy.pass.model.Funder; +import org.dataconservancy.pass.grant.data.*; import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.Policy; import org.dataconservancy.pass.model.User; @@ -36,283 +30,184 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.HashMap; import static java.lang.Thread.sleep; import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; import static org.junit.Assert.*; + public class HarvardPilotPassUpdaterIT { - private static final String DOMAIN = "harvard.edu"; + PassClient passClient = PassClientFactory.getPassClient(); + PassUpdater passUpdater = new HarvardPilotPassUpdater(passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + String[] grantAwardNumber = { "C10000000", "C10000001", "C10000002" }; + String[] grantLocalKey = { "10000002", "10000002","10000002" }; //all the same, different from other ITs tho + String[] grantProjectName = {"Amazing Research Project I", "Amazing Research Project II", "Amazing Research Project III" }; + String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; + String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; + // String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; + String[] userEmployeeId= { "A0000001", "A0000002", "A0000003"}; + String[] userFirstName = {"John", "Simon", "Rocket"}; + String[] userLastName = { "Public", "Sinister", "Squirrel" }; + String[] userEmail = { "jpubli1@harvard.edu", "ssinis11@harvard.edu", "rsquir1@harvard.edu" }; - private List> resultSet = new ArrayList<>(); - private static String employeeidPrefix = DOMAIN + ":employeeid:"; - private PassEntityUtil passEntityUtil = new HarvardPilotPassEntityUtil(); - private Map funderPolicyUriMap = new HashMap<>(); + String primaryFunderPolicyUriString; + String directFunderPolicyUriString; - private String directFunderPolicyUriString1; - private String primaryFunderPolicyUriString1; + String grantIdPrefix = "harvard.edu:grant:"; + String institutionalIdPrefix = "harvard.edu:jhed:"; - private PassClient passClient = PassClientFactory.getPassClient(); - @Rule public TemporaryFolder folder= new TemporaryFolder(); @Before public void setup() { - - for (int i = 0; i < 10; i++) { - - String prefix = System.getProperty("pass.fedora.baseurl"); - if (!prefix.endsWith("/")) { - prefix = prefix + "/"; - } - - Policy policy = new Policy(); - policy.setTitle("Primary Policy" + i); - policy.setDescription("MOO"); - URI policyURI = passClient.createResource(policy); - String primaryPolicyUriString = policyURI.toString().substring(prefix.length()); - funderPolicyUriMap.put("PrimaryFunderPolicy" + i, policyURI); - - policy = new Policy(); - policy.setTitle("Direct Policy" + i); - policy.setDescription("MOO"); - policyURI = passClient.createResource(policy); - String directPolicyUriString = policyURI.toString().substring(prefix.length()); - funderPolicyUriMap.put("DirectFunderPolicy" + i, policyURI); - - if (i == 1) { - directFunderPolicyUriString1 = directPolicyUriString; - primaryFunderPolicyUriString1 = primaryPolicyUriString; - } - - - - - Map rowMap = new HashMap<>(); - - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + i); - //rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + i); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + i); - //rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); - rowMap.put(C_GRANT_START_DATE, "01/01/2001"); - rowMap.put(C_GRANT_END_DATE, "01/01/2002"); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + i); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + i); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + i); - rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + i); - - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); - //rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - //rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - // rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); - - //rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + i + ":00:00.0"); - rowMap.put(C_ABBREVIATED_ROLE, (i % 2 == 0 ? "P" : "C")); - rowMap.put(C_DIRECT_FUNDER_POLICY, directPolicyUriString); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryPolicyUriString); - - resultSet.add(rowMap); + String prefix = System.getProperty("pass.fedora.baseurl"); + if ( !prefix.endsWith("/") ) { + prefix = prefix + "/"; } + + Policy policy = new Policy(); + policy.setTitle("Primary Policy 2"); + policy.setDescription("BAA"); + URI policyURI = passClient.createResource(policy); + primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + policy =new Policy(); + policy.setTitle("Direct Policy 2"); + policy.setDescription("BAA"); + policyURI =passClient.createResource(policy); + directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + } + /** * The behavior of PassUpdate's updatePass() method is to compare the data coming in on the ResultSet with * the existing data in Pass, and create objects if Pass does not yet have them, and update them if they exist in Pass but - * there are differences in the fields for which COEUS is the authoritative source, or COEUS has a clue about other fields which are null + * there are differences in the fields for whichthe pull is the authoritative source, or else has a clue about other fields which are null * on the PASS object. * + * For the Harvard Pilot, we use the user name on the email address to stand for the user's eppn fpr the purposes of populating that + * locator id field + * * @throws InterruptedException - the exception */ @Test public void updateGrantsIT() throws InterruptedException { - PassUpdater passUpdater = new HarvardPilotPassUpdater(); - passUpdater.updatePass(resultSet, "grant"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); - - assertEquals(5, statistics.getPisAdded()); - assertEquals(5, statistics.getCoPisAdded()); - assertEquals(20, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - assertEquals(10, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - //assertEquals("2018-01-01 09:00:00.0", statistics.getLatestUpdateString()); - assertEquals(10, statistics.getUsersCreated()); - assertEquals(0, statistics.getUsersUpdated()); - - assertEquals(10, passUpdater.getGrantUriMap().size()); + List> resultSet = new ArrayList<>(); - for (URI grantUri : passUpdater.getGrantUriMap().keySet()) { - Grant grant = passUpdater.getGrantUriMap().get(grantUri); - Grant passGrant = passUpdater.getPassClient().readResource(grantUri, Grant.class); - assertNull(passEntityUtil.update(grant, passGrant)); //this means grants are "harvard-equal" + //put in initial iteration as a correct existing record - PI is Public, Co-pi is Sinister + Map piRecord0 = makeRowMap(0, 0, "P"); + Map coPiRecord0 = makeRowMap(0, 1, "C"); - } + resultSet.add(piRecord0); + resultSet.add(coPiRecord0); - sleep(20000); - //try depositing the exact same resultSet. nothing should happen in Pass passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", institutionalIdPrefix + userEmail[0].split("@")[0] ); + assertNotNull( passUser0Uri ); + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); + assertNotNull( passGrantUri ); + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", institutionalIdPrefix + userEmail[1].split("@")[0] ); + assertNotNull( passUser1Uri ); + + Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); + + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() ); + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate[0]), passGrant.getEndDate() ); + assertEquals( passUser0Uri, passGrant.getPi() ); //Pblic + assertEquals( 1, passGrant.getCoPis().size() ); + assertEquals( passUser1Uri, passGrant.getCoPis().get(0)); + + //check statistics + assertEquals(1, statistics.getGrantsCreated()); + assertEquals(2, statistics.getUsersCreated()); + assertEquals(1, statistics.getPisAdded()); + assertEquals(1, statistics.getCoPisAdded()); + + //now simulate an incremental pull since the initial, adjust the stored grant + //we add a new co-pi Squirrel in the "1" iteration, and change the pi to Einstein in the "2" iteration + //we drop co-pi Squirrel in the last iteration + + Map piRecord1 = makeRowMap(1, 0, "P"); + Map coPiRecord1 = makeRowMap(1, 1, "C"); + Map newCoPiRecord1 = makeRowMap(1, 2, "C"); + Map piRecord2 = makeRowMap (2, 1, "P"); + + //add in everything since the initial pull + resultSet.clear(); + resultSet.add(piRecord1); + resultSet.add(coPiRecord1); + resultSet.add(newCoPiRecord1); + resultSet.add(piRecord2); - assertEquals(0, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - assertEquals(0, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); - assertEquals(0, statistics.getUsersUpdated()); + passUpdater.updatePass(resultSet, "grant"); + sleep(12000); + + passGrant = passClient.readResource( passGrantUri, Grant.class ); + + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", institutionalIdPrefix + userEmail[2].split("@")[0] ); + assertNotNull( passUser2Uri ); + + assertEquals( grantAwardNumber[1], passGrant.getAwardNumber() );//earliest of the additions + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[1], passGrant.getLocalKey() );//earliest of the additions + assertEquals( grantProjectName[1], passGrant.getProjectName() );//earliest of the additions + assertEquals( createJodaDateTime(grantStartDate[1]), passGrant.getStartDate() );//earliest of the additions + assertEquals( createJodaDateTime(grantEndDate[1]), passGrant.getEndDate() );//earliest of the additions + assertEquals( passUser0Uri, passGrant.getPi() );//first one in the pull + assertEquals( 2, passGrant.getCoPis().size() ); + assertTrue( passGrant.getCoPis().contains(passUser1Uri) );//Public + assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Sinister + } - //now let's monkey with a few things; we expect to update the changed objects + /** + * utility method to produce data as it would look coming from the Harvard spreadsheet + * @param iteration the iteration of the (multi-award) grant + * @param user the user supplied in the record + * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") + * @return the row map for the pull record + */ + private Map makeRowMap( int iteration, int user, String abbrRole) { Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1); - //rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + 1); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOOO"); - //rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); - rowMap.put(C_GRANT_START_DATE, "01/01/2001"); - rowMap.put(C_GRANT_END_DATE, "01/01/2002"); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + 1 + "MOOOO"); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + 1); + rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber[iteration]); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey[iteration]); + rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName[iteration]); + rowMap.put(C_GRANT_START_DATE, grantStartDate[iteration]); + rowMap.put(C_GRANT_END_DATE, grantEndDate[iteration]); - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + 1 + "MOOOOO"); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + 1); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + 1); - //rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + 1); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + 1); - //rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + 1); + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000002"); + rowMap.put(C_DIRECT_FUNDER_NAME, "Gargantuan State University"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000003"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "D Warbucks Foundation"); - //rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); - rowMap.put(C_ABBREVIATED_ROLE, ("C")); + rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); + rowMap.put(C_USER_LAST_NAME, userLastName[user]); + rowMap.put(C_USER_EMAIL, userEmail[user]); + rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); - rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString1); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString1); + rowMap.put(C_ABBREVIATED_ROLE, abbrRole); + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); - resultSet.clear(); - resultSet.add(rowMap); - - passUpdater.updatePass(resultSet, "grant"); - assertEquals(0, statistics.getFundersCreated()); - assertEquals(1, statistics.getFundersUpdated()); - assertEquals(0, statistics.getGrantsCreated()); - assertEquals(1, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); - assertEquals(1, statistics.getUsersUpdated()); - - sleep(20000); - - for (int i = 0; i < 10; i++) { - Grant grant = new Grant(); - grant.setAwardNumber(C_GRANT_AWARD_NUMBER + i); - //grant.setAwardStatus(Grant.AwardStatus.ACTIVE); - String grantIdPrefix = DOMAIN + ":grant:"; - grant.setLocalKey(grantIdPrefix + C_GRANT_LOCAL_KEY + i); - grant.setProjectName(C_GRANT_PROJECT_NAME + i); - //grant.setAwardDate(DateTimeUtil.createJodaDateTime("01/01/2000")); - grant.setStartDate(DateTimeUtil.createJodaDateTime("01/01/2001")); - grant.setEndDate(DateTimeUtil.createJodaDateTime("01/01/2002")); - - URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grant.getLocalKey()); - Grant passGrant = passClient.readResource(passGrantUri, Grant.class); - - assertEquals(grant.getAwardNumber(), passGrant.getAwardNumber()); - //assertEquals(grant.getAwardStatus(), passGrant.getAwardStatus()); - assertEquals(grant.getLocalKey(), passGrant.getLocalKey()); - if (i == 1) { - assertEquals(grant.getProjectName() + "MOOO", passGrant.getProjectName()); - } else { - assertEquals(grant.getProjectName(), passGrant.getProjectName()); - } - //assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); - assertEquals(grant.getStartDate(), passGrant.getStartDate()); - assertEquals(grant.getEndDate(), passGrant.getEndDate()); - - //let's check funder stuff - Funder directFunder = new Funder(); - String funderIdPrefix = DOMAIN + ":funder:"; - directFunder.setLocalKey(funderIdPrefix + C_DIRECT_FUNDER_LOCAL_KEY + i); - directFunder.setName(C_DIRECT_FUNDER_NAME + i); - directFunder.setPolicy(funderPolicyUriMap.get("DirectFunderPolicy" + i)); - - URI directFunderUri = passClient.findByAttribute(Funder.class, "localKey", directFunder.getLocalKey()); - Funder passDirectFunder = passClient.readResource(directFunderUri, Funder.class); - if (i == 1) { - assertEquals(directFunder.getName() + "MOOOO", passDirectFunder.getName()); - assertEquals(directFunder.getLocalKey(), passDirectFunder.getLocalKey()); - assertEquals(passDirectFunder.getId(), passGrant.getDirectFunder()); - } else { - assertEquals(directFunder.getName(), passDirectFunder.getName()); - } - - Funder primaryFunder = new Funder(); - primaryFunder.setLocalKey(funderIdPrefix + C_PRIMARY_FUNDER_LOCAL_KEY + i); - primaryFunder.setName(C_PRIMARY_FUNDER_NAME + i); - primaryFunder.setPolicy(funderPolicyUriMap.get("PrimaryFunderPolicy" + i)); - - URI primaryFunderUri = passClient.findByAttribute(Funder.class, "localKey", primaryFunder.getLocalKey()); - Funder passPrimaryFunder = passClient.readResource(primaryFunderUri, Funder.class); - assertEquals(primaryFunder.getName(), passPrimaryFunder.getName()); - assertEquals(passPrimaryFunder.getId(), passGrant.getPrimaryFunder()); - assertEquals(primaryFunder.getLocalKey(), passPrimaryFunder.getLocalKey()); - assertEquals(primaryFunder.getPolicy(), passPrimaryFunder.getPolicy()); - - User user = new User(); - - //institutionalId and localKey were localized by the grant loader - user.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + i); - //String idPrefix = "johnshopkins.edu:hopkinsid:"; - //user.getLocatorIds().add(hopkinsidPrefix + C_USER_HOPKINS_ID + i); - //user.getLocatorIds().add(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + i); - user.setFirstName(C_USER_FIRST_NAME + i); - //user.setMiddleName(C_USER_MIDDLE_NAME + i); - user.setLastName(C_USER_LAST_NAME + i); - user.setEmail(C_USER_EMAIL + i); - - URI userUri = null; - ListIterator idIterator = user.getLocatorIds().listIterator(); - - while (userUri == null && idIterator.hasNext()) { - String id = String.valueOf(idIterator.next()); - if (id != null) { - userUri = passClient.findByAttribute(User.class, "locatorIds", id); - } - } - - User passUser = passClient.readResource(userUri, User.class); - if (i == 1) { - assertEquals(user.getFirstName() + "MOOOOO", passUser.getFirstName()); - } else { - assertEquals(user.getFirstName(), passUser.getFirstName()); - } - assertEquals(user.getLastName(), passUser.getLastName()); - assertEquals(user.getEmail(), passUser.getEmail()); - assertTrue(user.getLocatorIds().containsAll(passUser.getLocatorIds())); - assertTrue(passUser.getLocatorIds().containsAll(user.getLocatorIds())); - assertEquals(passUser.getLocatorIds().size(), user.getLocatorIds().size()); - - if (i % 2 == 0) { - assertNotNull(passGrant.getPi()); - assertEquals(0, passGrant.getCoPis().size()); - } else { - assertNull(passGrant.getPi()); - assertEquals(1, passGrant.getCoPis().size()); - } - - } + return rowMap; } - } diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java new file mode 100644 index 0000000..cd8cb40 --- /dev/null +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java @@ -0,0 +1,216 @@ +/* + * Copyright 2020 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dataconservancy.pass.grant.integration; + +import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.client.PassClientFactory; +import org.dataconservancy.pass.grant.data.JhuPassInitUpdater; +import org.dataconservancy.pass.grant.data.PassUpdateStatistics; +import org.dataconservancy.pass.model.Grant; +import org.dataconservancy.pass.model.Policy; +import org.dataconservancy.pass.model.User; +import org.junit.Before; +import org.junit.Test; + +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.lang.Thread.sleep; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; +import static org.junit.Assert.*; + +public class JhuPassInitUpdaterIT { + + String[] grantAwardNumber = { "A10000000", "A10000001", "A10000002" }; + String[] grantLocalKey = { "10000000", "10000000","10000000" }; //all the same + String[] grantProjectName = {"Awesome Research Project I", "Awesome Research Project II", "Awesome Research Project III" }; + String[] grantAwardDate = { "01/01/1999", "01/01/2001", "01/01/2003" }; + String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; + String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; + String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; + String[] userEmployeeId= { "30000000", "30000001", "30000002"}; + String[] userInstitutionalId = {"amelon1", "aeinst1", "jjones1" }; + String[] userHopkinsId = {"RANDOM", "OMRNDA", "DRMONA" }; + String[] userFirstName = {"Andrew", "Albert", "Junie"}; + String[] userMiddleName = {"Smith", "Carnegie", "Beatrice" }; + String[] userLastName = { "Melon", "Einstein", "Jones" }; + String[] userEmail = { "amelon1@jhu.edu", "aeinst1@jhu.edu", "jjones1@jhu.edu" }; + + + String primaryFunderPolicyUriString; + String directFunderPolicyUriString; + + String grantIdPrefix = "johnshopkins.edu:grant:"; + //String funderIdPrefix = "johnshopkins.edu:funder:"; + //String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; + String employeeidPrefix = "johnshopkins.edu:employeeid:"; + //String jhedidPrefis = "johnshopkins.edu:jhed:"; + + PassClient passClient = PassClientFactory.getPassClient(); + JhuPassInitUpdater passUpdater = new JhuPassInitUpdater(passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + @Before + public void setup() { + String prefix = System.getProperty("pass.fedora.baseurl"); + if ( !prefix.endsWith("/") ) { + prefix = prefix + "/"; + } + + Policy policy = new Policy(); + policy.setTitle("Primary Policy"); + policy.setDescription("MOO"); + URI policyURI = passClient.createResource(policy); + primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + policy =new Policy(); + policy.setTitle("Direct Policy"); + policy.setDescription("MOO"); + policyURI =passClient.createResource(policy); + directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + } + + /** + * we put an initial award for a grant into fedora, then simulate a pull of all records related + * to this grant from the Beginning of Time (including records which created the initial object) + * + * We expect to see some fields retained from the initial award, and others updated. The most + * interesting fields are the investigator fields: all CO-PIs ever on the grant should stay on the + * co-pi field throughout iterations. If a PI is changed, they should appear on the CO-PI field + * + * @throws InterruptedException from joda data time creation + */ + @Test + public void processInitGrantIT() throws InterruptedException { + List> resultSet = new ArrayList<>(); + + //put in last iteration as existing record - PI is Einstein + Map piRecord2 = makeRowMap (2, 1, "P"); + resultSet.add(piRecord2); + + passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); + assertNotNull( passGrantUri ); + + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[1] ); + assertNotNull( passUser1Uri ); + + Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); + + assertEquals( grantAwardNumber[2], passGrant.getAwardNumber() ); + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[2], passGrant.getLocalKey() ); + assertEquals( grantProjectName[2], passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantAwardDate[2]), passGrant.getAwardDate() ); + assertEquals( createJodaDateTime(grantStartDate[2]), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() ); + assertEquals( passUser1Uri, passGrant.getPi() ); //Einstein + assertEquals( 0, passGrant.getCoPis().size() ); + + //check statistics + assertEquals(1, statistics.getGrantsCreated()); + assertEquals(1, statistics.getUsersCreated()); + assertEquals(1, statistics.getPisAdded()); + assertEquals(0, statistics.getCoPisAdded()); + + //now simulate a complete pull from the Beginning of Time and adjust the stored grant + //we add a new co-pi Jones in the "1" iteration, and change the pi to Einstein in the "2" iteration + //we drop co-pi jones in the last iteration + Map piRecord0 = makeRowMap(0, 0, "P"); + Map coPiRecord0 = makeRowMap(0, 1, "C"); + Map piRecord1 = makeRowMap(1, 0, "P"); + Map coPiRecord1 = makeRowMap(1, 1, "C"); + Map newCoPiRecord1 = makeRowMap(1, 2, "C"); + + //in the initial pull, we will find all of the records (check?) + resultSet.clear(); + resultSet.add(piRecord0); + resultSet.add(coPiRecord0); + resultSet.add(piRecord1); + resultSet.add(coPiRecord1); + resultSet.add(newCoPiRecord1); + resultSet.add(piRecord2); + + passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + + passGrant = passClient.readResource( passGrantUri, Grant.class ); + URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[0] ); + assertNotNull( passUser0Uri ); + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[2] ); + assertNotNull( passUser2Uri ); + + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() );//initial + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() );//initial + assertEquals( createJodaDateTime(grantAwardDate[0]), passGrant.getAwardDate() );//initial + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() );//initial + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() );//latest + assertEquals( passUser1Uri, passGrant.getPi() );//Einstein + assertEquals( 2, passGrant.getCoPis().size() ); + assertTrue( passGrant.getCoPis().contains(passUser0Uri) );//Melon + assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Jones + } + + /** + * utility method to produce data as it would look coming from COEUS + * @param iteration the iteration of the (multi-award) grant + * @param user the user supplied in the record + * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") + * @return the row map for the record + */ + private Map makeRowMap( int iteration, int user, String abbrRole) { + Map rowMap = new HashMap<>(); + rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber[iteration]); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey[iteration]); + rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName[iteration]); + rowMap.put(C_GRANT_AWARD_DATE, grantAwardDate[iteration]); + rowMap.put(C_GRANT_START_DATE, grantStartDate[iteration]); + rowMap.put(C_GRANT_END_DATE, grantEndDate[iteration]); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000000"); + rowMap.put(C_DIRECT_FUNDER_NAME, "Enormous State University"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000001"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "J L Gotrocks Foundation"); + + rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); + rowMap.put(C_USER_MIDDLE_NAME, userMiddleName[user]); + rowMap.put(C_USER_LAST_NAME, userLastName[user]); + rowMap.put(C_USER_EMAIL, userEmail[user]); + rowMap.put(C_USER_INSTITUTIONAL_ID, userInstitutionalId[user]); + rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); + rowMap.put(C_USER_HOPKINS_ID, userHopkinsId[user]); + + rowMap.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp[iteration]); + rowMap.put(C_ABBREVIATED_ROLE, abbrRole); + + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); + + return rowMap; + } + +} diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java index feb9b43..a26ba7a 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Johns Hopkins University + * Copyright 2020 Johns Hopkins University * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,530 +13,206 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.dataconservancy.pass.grant.integration; import org.dataconservancy.pass.client.PassClient; import org.dataconservancy.pass.client.PassClientFactory; -import org.dataconservancy.pass.grant.data.DateTimeUtil; -import org.dataconservancy.pass.grant.data.PassEntityUtil; -import org.dataconservancy.pass.grant.data.PassUpdateStatistics; import org.dataconservancy.pass.grant.data.JhuPassUpdater; -import org.dataconservancy.pass.grant.data.CoeusPassEntityUtil; -import org.dataconservancy.pass.model.Funder; +import org.dataconservancy.pass.grant.data.PassUpdateStatistics; import org.dataconservancy.pass.model.Grant; - import org.dataconservancy.pass.model.Policy; import org.dataconservancy.pass.model.User; -import org.dataconservancy.pass.model.support.Identifier; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; -import static java.lang.Thread.currentThread; -import static java.lang.Thread.sleep; -import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertNotNull; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.ListIterator; import java.util.Map; -/** - * An integration test class for the JhuPassUpdater. - */ -@RunWith(MockitoJUnitRunner.class) -public class JhuPassUpdaterIT { - - private final String DOMAIN = "johnshopkins.edu"; - - - private List> resultSet = new ArrayList<>(); - private static String employeeidPrefix = "johnshopkins.edu:employeeid:"; - private static String jhedPrefix = "johnshopkins.edu:jhed:"; - private CoeusPassEntityUtil passEntityUtil = new CoeusPassEntityUtil(); - private Map funderPolicyUriMap = new HashMap<>(); - private String prefix; - - private String directFunderPolicyUriString1; - private String primaryFunderPolicyUriString1; +import static java.lang.Thread.sleep; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; +import static org.junit.Assert.*; - private PassClient passClient = PassClientFactory.getPassClient(); +public class JhuPassUpdaterIT { - @Rule - public TemporaryFolder folder= new TemporaryFolder(); + String[] grantAwardNumber = { "B10000000", "B10000001", "B10000002" }; + String[] grantLocalKey = { "10000001", "10000001","10000001" }; //all the same, different from other ITs tho + String[] grantProjectName = {"Stupendous Research Project I", "Stupendous Research Project II", "Stupendous Research Project III" }; + String[] grantAwardDate = { "01/01/1999", "01/01/2001", "01/01/2003" }; + String[] grantStartDate = { "07/01/2000", "07/01/2000", "07/01/2000" }; //these appear to ge the same for all awards + String[] grantEndDate = { "06/30/2004", "06/30/2004", "06/30/2004"};//these seem to be the same for all awards + String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; + String[] userEmployeeId= { "31000000", "31000001", "31000002"}; + String[] userInstitutionalId = {"arecko1", "sclass1", "jgunn1" }; + String[] userHopkinsId = {"DOMNAR", "NROAD", "ROMAND" }; + String[] userFirstName = {"Amanda", "Skip", "Janie"}; + String[] userMiddleName = {"Bea", "Avery", "Gotta" }; + String[] userLastName = { "Reckondwith", "Class", "Gunn" }; + String[] userEmail = { "arecko1@jhu.edu", "sclass1@jhu.edu", "jgunn1@jhu.edu" }; + + + String primaryFunderPolicyUriString; + String directFunderPolicyUriString; + + String grantIdPrefix = "johnshopkins.edu:grant:"; + //String funderIdPrefix = "johnshopkins.edu:funder:"; + //String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; + String employeeidPrefix = "johnshopkins.edu:employeeid:"; + //String jhedidPrefis = "johnshopkins.edu:jhed:"; + + PassClient passClient = PassClientFactory.getPassClient(); + JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); @Before public void setup() { - - for (int i = 0; i < 10; i++) { - - String prefix = System.getProperty("pass.fedora.baseurl"); - if (!prefix.endsWith("/")) { - prefix = prefix + "/"; - } - - Policy policy = new Policy(); - policy.setTitle("Primary Policy" + i); - policy.setDescription("MOO"); - URI policyURI = passClient.createResource(policy); - String primaryPolicyUriString = policyURI.toString().substring(prefix.length()); - funderPolicyUriMap.put("PrimaryFunderPolicy"+i, policyURI); - - policy = new Policy(); - policy.setTitle("Direct Policy" + i); - policy.setDescription("MOO"); - policyURI = passClient.createResource(policy); - String directPolicyUriString = policyURI.toString().substring(prefix.length()); - funderPolicyUriMap.put("DirectFunderPolicy"+i, policyURI); - - if(i==1) { - directFunderPolicyUriString1 = directPolicyUriString; - primaryFunderPolicyUriString1 = primaryPolicyUriString; - } - - Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + i); - rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + i); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + i); - rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); - rowMap.put(C_GRANT_START_DATE, "01/01/2001"); - rowMap.put(C_GRANT_END_DATE, "01/01/2002"); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + i); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + i); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + i); - rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + i); - - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); - rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); - - rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + i + ":00:00.0"); - rowMap.put(C_ABBREVIATED_ROLE, (i % 2 == 0 ? "P" : "C")); - rowMap.put(C_DIRECT_FUNDER_POLICY, directPolicyUriString ); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryPolicyUriString); - - resultSet.add(rowMap); + String prefix = System.getProperty("pass.fedora.baseurl"); + if ( !prefix.endsWith("/") ) { + prefix = prefix + "/"; } + Policy policy = new Policy(); + policy.setTitle("Primary Policy 2"); + policy.setDescription("BAA"); + URI policyURI = passClient.createResource(policy); + primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + policy =new Policy(); + policy.setTitle("Direct Policy 2"); + policy.setDescription("BAA"); + policyURI =passClient.createResource(policy); + directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + } /** - * The behavior of PassUpdate's updatePass() method is to compare the data coming in on the ResultSet with - * the existing data in Pass, and create objects if Pass does not yet have them, and update them if they exist in Pass but - * there are differences in the fields for which COEUS is the authoritative source, or COEUS has a clue about other fields which are null - * on the PASS object. + * we put an initial award for a grant into fedora, then simulate a pull of all subsequent records * - * @throws InterruptedException - the exception + * We expect to see some fields retained from the initial award, and others updated. The most + * interesting fields are the investigator fields: all CO-PIs ever on the grant should stay on the + * co-pi field throughout iterations. If a PI is changed, they should appear on the CO-PI field + * + * @throws InterruptedException from joda date time creation */ @Test - public void updateGrantsIT() throws InterruptedException { - - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); - passUpdater.updatePass(resultSet, "grant"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); + public void processGrantIT() throws InterruptedException { + List> resultSet = new ArrayList<>(); - assertEquals(5, statistics.getPisAdded()); - assertEquals(5, statistics.getCoPisAdded()); - assertEquals(20, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - assertEquals(10, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - assertEquals("2018-01-01 09:00:00.0", statistics.getLatestUpdateString()); - assertEquals(10, statistics.getUsersCreated()); - assertEquals(0, statistics.getUsersUpdated()); + //put in initial iteration as a correct existing record - PI is Reckondwith, Co-pi is Class + Map piRecord0 = makeRowMap(0, 0, "P"); + Map coPiRecord0 = makeRowMap(0, 1, "C"); - assertEquals(10, passUpdater.getGrantUriMap().size()); - - for (URI grantUri : passUpdater.getGrantUriMap().keySet()) { - Grant grant = passUpdater.getGrantUriMap().get(grantUri); - Grant passGrant = passUpdater.getPassClient().readResource(grantUri, Grant.class); - assertNull(passEntityUtil.update(grant, passGrant)); //this means grants are "coeus-equal" - - } + resultSet.add(piRecord0); + resultSet.add(coPiRecord0); - sleep(20000); - //try depositing the exact same resultSet. nothing should happen in Pass passUpdater.updatePass(resultSet, "grant"); - - assertEquals(0, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - assertEquals(0, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); - assertEquals(0, statistics.getUsersUpdated()); - - //now let's monkey with a few things; we expect to update the changed objects - Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1); - rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + 1); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOOO"); - rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); - rowMap.put(C_GRANT_START_DATE, "01/01/2001"); - rowMap.put(C_GRANT_END_DATE, "01/01/2002"); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + 1 + "MOOOOO"); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + 1); - - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + 1); - rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + 1 + "MOOOO"); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + 1); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + 1); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + 1); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + 1); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + 1); - - rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); - rowMap.put(C_ABBREVIATED_ROLE, ("C")); - - rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString1); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString1); - - + sleep(10000); + URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[0] ); + assertNotNull( passUser0Uri ); + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); + assertNotNull( passGrantUri ); + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[1] ); + assertNotNull( passUser1Uri ); + + Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); + + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() ); + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantAwardDate[0]), passGrant.getAwardDate() ); + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate[0]), passGrant.getEndDate() ); + assertEquals( passUser0Uri, passGrant.getPi() ); //Reckondwith + assertEquals( 1, passGrant.getCoPis().size() ); + assertEquals( passUser1Uri, passGrant.getCoPis().get(0)); + + //check statistics + assertEquals(1, statistics.getGrantsCreated()); + assertEquals(2, statistics.getUsersCreated()); + assertEquals(1, statistics.getPisAdded()); + assertEquals(1, statistics.getCoPisAdded()); + + //now simulate an incremental pull since the initial, adjust the stored grant + //we add a new co-pi Jones in the "1" iteration, and change the pi to Einstein in the "2" iteration + //we drop co-pi jones in the last iteration + + Map piRecord1 = makeRowMap(1, 0, "P"); + Map coPiRecord1 = makeRowMap(1, 1, "C"); + Map newCoPiRecord1 = makeRowMap(1, 2, "C"); + Map piRecord2 = makeRowMap (2, 1, "P"); + + //add in everything since the initial pull resultSet.clear(); - resultSet.add(rowMap); + resultSet.add(piRecord1); + resultSet.add(coPiRecord1); + resultSet.add(newCoPiRecord1); + resultSet.add(piRecord2); passUpdater.updatePass(resultSet, "grant"); - assertEquals(0, statistics.getFundersCreated()); - assertEquals(1, statistics.getFundersUpdated()); - assertEquals(0, statistics.getGrantsCreated()); - assertEquals(1, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); - assertEquals(1, statistics.getUsersUpdated()); - - sleep(20000); - - for (int i = 0; i < 10; i++) { - Grant grant = new Grant(); - grant.setAwardNumber(C_GRANT_AWARD_NUMBER + i); - grant.setAwardStatus(Grant.AwardStatus.ACTIVE); - String grantIdPrefix = "johnshopkins.edu:grant:"; - grant.setLocalKey(grantIdPrefix + C_GRANT_LOCAL_KEY + i); - grant.setProjectName(C_GRANT_PROJECT_NAME + i); - grant.setAwardDate(DateTimeUtil.createJodaDateTime("01/01/2000")); - grant.setStartDate(DateTimeUtil.createJodaDateTime("01/01/2001")); - grant.setEndDate(DateTimeUtil.createJodaDateTime("01/01/2002")); - - URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grant.getLocalKey()); - Grant passGrant = passClient.readResource(passGrantUri, Grant.class); - - assertEquals(grant.getAwardNumber(), passGrant.getAwardNumber()); - assertEquals(grant.getAwardStatus(), passGrant.getAwardStatus()); - assertEquals(grant.getLocalKey(), passGrant.getLocalKey()); - if (i == 1) { - assertEquals(grant.getProjectName() + "MOOO", passGrant.getProjectName()); - } else { - assertEquals(grant.getProjectName(), passGrant.getProjectName()); - } - assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); - assertEquals(grant.getStartDate(), passGrant.getStartDate()); - assertEquals(grant.getEndDate(), passGrant.getEndDate()); - - //let's check funder stuff - Funder directFunder = new Funder(); - String funderIdPrefix = "johnshopkins.edu:funder:"; - directFunder.setLocalKey(funderIdPrefix + C_DIRECT_FUNDER_LOCAL_KEY + i); - directFunder.setName(C_DIRECT_FUNDER_NAME + i); - directFunder.setPolicy(funderPolicyUriMap.get("DirectFunderPolicy" + i)); - - URI directFunderUri = passClient.findByAttribute(Funder.class, "localKey", directFunder.getLocalKey()); - Funder passDirectFunder = passClient.readResource(directFunderUri, Funder.class); - if (i == 1) { - assertEquals(directFunder.getName() + "MOOOOO", passDirectFunder.getName()); - assertEquals(directFunder.getLocalKey(), passDirectFunder.getLocalKey()); - assertEquals(passDirectFunder.getId(), passGrant.getDirectFunder()); - } else { - assertEquals(directFunder.getName(), passDirectFunder.getName()); - } - - Funder primaryFunder = new Funder(); - primaryFunder.setLocalKey(funderIdPrefix + C_PRIMARY_FUNDER_LOCAL_KEY + i); - primaryFunder.setName(C_PRIMARY_FUNDER_NAME + i); - primaryFunder.setPolicy(funderPolicyUriMap.get("PrimaryFunderPolicy" + i)); - - URI primaryFunderUri = passClient.findByAttribute(Funder.class, "localKey", primaryFunder.getLocalKey()); - Funder passPrimaryFunder = passClient.readResource(primaryFunderUri, Funder.class); - assertEquals(primaryFunder.getName(), passPrimaryFunder.getName()); - assertEquals(passPrimaryFunder.getId(), passGrant.getPrimaryFunder()); - assertEquals(primaryFunder.getLocalKey(), passPrimaryFunder.getLocalKey()); - assertEquals(primaryFunder.getPolicy(), passPrimaryFunder.getPolicy()); - - User user = new User(); - - //institutionalId and localKey were localized by the grant loader - user.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + i); - String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; - user.getLocatorIds().add(hopkinsidPrefix + C_USER_HOPKINS_ID + i); - user.getLocatorIds().add(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + i); - user.setFirstName(C_USER_FIRST_NAME + i); - user.setMiddleName(C_USER_MIDDLE_NAME + i); - user.setLastName(C_USER_LAST_NAME + i); - user.setEmail(C_USER_EMAIL + i); - - URI userUri = null; - ListIterator idIterator = user.getLocatorIds().listIterator(); - - while (userUri == null && idIterator.hasNext()) { - String id = String.valueOf(idIterator.next()); - if (id != null) { - userUri = passClient.findByAttribute(User.class, "locatorIds", id); - } - } - - User passUser = passClient.readResource(userUri, User.class); - assertEquals(user.getFirstName(), passUser.getFirstName()); - if (i == 1) { - assertEquals(user.getMiddleName() + "MOOOO", passUser.getMiddleName()); - } else { - assertEquals(user.getMiddleName(), passUser.getMiddleName()); - } - assertEquals(user.getLastName(), passUser.getLastName()); - assertEquals(user.getEmail(), passUser.getEmail()); - assertTrue(user.getLocatorIds().containsAll(passUser.getLocatorIds())); - assertTrue(passUser.getLocatorIds().containsAll(user.getLocatorIds())); - assertEquals(passUser.getLocatorIds().size(), user.getLocatorIds().size()); - - if (i % 2 == 0) { - assertNotNull(passGrant.getPi()); - assertEquals(0, passGrant.getCoPis().size()); - } else { - assertNull(passGrant.getPi()); - assertEquals(1, passGrant.getCoPis().size()); - } - - } - } - - - @Test - public void updateUsersIT() throws InterruptedException { - - User user10 = new User(); - user10.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + 10); - user10.setFirstName(C_USER_FIRST_NAME + 10); - user10.setMiddleName(C_USER_MIDDLE_NAME + 10); - user10.setLastName(C_USER_LAST_NAME + 10); - - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); - - URI passUserURI = passUpdater.getPassClient().createResource(user10); - - User passUser = passClient.readResource(passUserURI, User.class); - assertNull(passUser.getDisplayName()); - assertEquals(1, passUser.getLocatorIds().size()); - assertNull(passUser.getEmail()); - - sleep(20000); - - List> userResultSet = new ArrayList<>(); - - for (int i = 10; i < 12; i++) { - Map rowMap = new HashMap<>(); - - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); - rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); - rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); - userResultSet.add(rowMap); - } - - - passUpdater.updatePass(userResultSet, "user"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); - - //now update from the set of two users - the second one is not in PASS, but is not created - //the first (user10) should be updated, with new fields added - assertEquals(0, statistics.getUsersCreated()); - assertEquals(1, statistics.getUsersUpdated()); - - assertNotNull(passUserURI); - User updatedUser = passUpdater.getPassClient().readResource(passUserURI, User.class); - - assertNotNull(updatedUser.getEmail()); - assertNotNull(updatedUser.getDisplayName()); - assertNotNull(updatedUser.getLocatorIds()); - assertTrue(updatedUser.getLocatorIds().contains(employeeidPrefix + C_USER_EMPLOYEE_ID + 10)); - assertTrue(updatedUser.getLocatorIds().contains(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + 10)); - - assertEquals(C_USER_EMAIL + 10, updatedUser.getEmail()); + sleep(10000); + + passGrant = passClient.readResource( passGrantUri, Grant.class ); + + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[2] ); + assertNotNull( passUser2Uri ); + + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() );//initial + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() );//initial + assertEquals( createJodaDateTime(grantAwardDate[0]), passGrant.getAwardDate() );//initial + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() );//initial + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() );//latest + assertEquals( passUser1Uri, passGrant.getPi() );//Class + assertEquals( 2, passGrant.getCoPis().size() ); + assertTrue( passGrant.getCoPis().contains(passUser0Uri) );//Reckondwith + assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Gunn } /** - * Create some policies, deposit them into Fedora - * Then create a java data object linking funders to them - * this basically tests what happens when pulling in data from a policy properties file first, - * or from a coeus pull second + * utility method to produce data as it would look coming from COEUS + * @param iteration the iteration of the (multi-award) grant + * @param user the user supplied in the record + * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") + * @return row map for pull record */ - @Test - public void updateFundersIT() throws InterruptedException { - Policy policy1 = new Policy(); - policy1. setTitle("Policy One"); - policy1. setDescription("Policy one Description"); - Policy policy2 = new Policy(); - policy2. setTitle("Policy Two"); - policy2. setDescription("Policy Two Description"); - - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); - URI policy1Uri = passClient.createResource(policy1); - URI policy2Uri = passClient.createResource(policy2); - - assertNotNull(passClient.readResource(policy1Uri, Policy.class)); - assertNotNull(passClient.readResource(policy2Uri, Policy.class)); - - Funder funder1 = new Funder(); - String fullLocalKey = new Identifier(DOMAIN, "funder", "22229999").serialize(); - funder1.setLocalKey(fullLocalKey); - funder1.setName("Funder One"); - Funder funder2 = new Funder(); - fullLocalKey = new Identifier(DOMAIN, "funder", "33330000").serialize(); - funder2.setLocalKey(fullLocalKey);//use full localKey - funder2.setName("Funder Two"); - - URI funder1Uri = passClient.createResource(funder1); - URI funder2Uri = passClient.createResource(funder2); - - assertNotNull(passClient.readResource(funder1Uri, Funder.class)); - assertNotNull(passClient.readResource(funder2Uri, Funder.class)); - - - String policyString1 = policy1Uri.getPath().substring("/fcrepo/rest/".length()); - assertTrue(policyString1.startsWith("policies")); - String policyString2 = policy2Uri.getPath().substring("/fcrepo/rest/".length()); - assertTrue(policyString2.startsWith("policies")); - - List> funderResultSet = new ArrayList<>(); - + private Map makeRowMap( int iteration, int user, String abbrRole) { Map rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "22229999"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString1); - funderResultSet.add(rowMap); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "33330000"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); - funderResultSet.add(rowMap); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "88888888"); // this one does not exist in pass - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); - funderResultSet.add(rowMap); - - - sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch - - passUpdater.updatePass(funderResultSet, "funder"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); - - assertNotNull(passClient.readResource(funder1Uri, Funder.class)); - assertNotNull(passClient.readResource(funder1Uri, Funder.class).getPolicy()); - assertNotNull(passClient.readResource(funder2Uri, Funder.class)); - assertNotNull(passClient.readResource(funder2Uri, Funder.class).getPolicy()); - assertEquals(policy1Uri, passClient.readResource(funder1Uri, Funder.class).getPolicy()); - assertEquals(policy2Uri, passClient.readResource(funder2Uri, Funder.class).getPolicy()); - - assertEquals(0, statistics.getFundersCreated()); - assertEquals(2, statistics.getFundersUpdated()); - - //coeus pulls will have the funder names, we should be able to add one we don't know about - - funderResultSet = new ArrayList<>(); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "22229999"); - rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 1"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); //let's change policies for this one - funderResultSet.add(rowMap); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "33330000"); - rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 2"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); - funderResultSet.add(rowMap); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "88888888"); // this one does not exist in pass - rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 3"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); - funderResultSet.add(rowMap); - - sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch - - passUpdater.updatePass(funderResultSet, "funder"); - statistics = passUpdater.getStatistics(); - - assertNotNull(passClient.readResource(funder1Uri, Funder.class)); - assertNotNull(passClient.readResource(funder1Uri, Funder.class).getPolicy()); - assertNotNull(passClient.readResource(funder2Uri, Funder.class)); - assertNotNull(passClient.readResource(funder2Uri, Funder.class).getPolicy()); - assertEquals(policy2Uri, passClient.readResource(funder1Uri, Funder.class).getPolicy()); - assertEquals(policy2Uri, passClient.readResource(funder2Uri, Funder.class).getPolicy()); - - assertEquals(1, statistics.getFundersCreated()); - assertEquals(2, statistics.getFundersUpdated()); - - - //DO AGAIN!! DO AGAIN!! - - sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch - - passUpdater.updatePass(funderResultSet, "funder"); - statistics = passUpdater.getStatistics(); - - assertEquals(0, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - + rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber[iteration]); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey[iteration]); + rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName[iteration]); + rowMap.put(C_GRANT_AWARD_DATE, grantAwardDate[iteration]); + rowMap.put(C_GRANT_START_DATE, grantStartDate[iteration]); + rowMap.put(C_GRANT_END_DATE, grantEndDate[iteration]); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000000"); + rowMap.put(C_DIRECT_FUNDER_NAME, "Enormous State University"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000001"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "J L Gotrocks Foundation"); + + rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); + rowMap.put(C_USER_MIDDLE_NAME, userMiddleName[user]); + rowMap.put(C_USER_LAST_NAME, userLastName[user]); + rowMap.put(C_USER_EMAIL, userEmail[user]); + rowMap.put(C_USER_INSTITUTIONAL_ID, userInstitutionalId[user]); + rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); + rowMap.put(C_USER_HOPKINS_ID, userHopkinsId[user]); + + rowMap.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp[iteration]); + rowMap.put(C_ABBREVIATED_ROLE, abbrRole); + + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); + + return rowMap; } - @Test - public void testSerializeAndDeserialize() throws IOException { - File serialized= folder.newFile("serializedData"); - - try (FileOutputStream fos = new FileOutputStream(serialized); - ObjectOutputStream out = new ObjectOutputStream(fos) - ){ - out.writeObject(resultSet); - } catch (IOException e) { - e.printStackTrace(); - } - - List input = null; - try (FileInputStream fis = new FileInputStream(serialized); - ObjectInputStream in = new ObjectInputStream(fis) - ){ - input = (List>)in.readObject(); - } catch (IOException | ClassNotFoundException ex) { - ex.printStackTrace(); - } - - assertEquals(resultSet, input); - } -} \ No newline at end of file +} diff --git a/pass-grant-integration/src/test/resources/docker/mnt/context.jsonld b/pass-grant-integration/src/test/resources/docker/mnt/context.jsonld index ea7e7b0..3ead31c 100644 --- a/pass-grant-integration/src/test/resources/docker/mnt/context.jsonld +++ b/pass-grant-integration/src/test/resources/docker/mnt/context.jsonld @@ -20,7 +20,7 @@ "abstract": {"@id": "pass:abstract"}, "accessUrl": {"@id": "pass:accessUrl", "@type": "@id"}, - "affiliation": {"@id": "pass:affiliation"}, + "affiliation": {"@id": "pass:affiliation", "@container": "@set"}, "agreementText": {"@id": "pass:agreementText"}, "aggregatedDepositStatus": {"@id": "pass:aggregatedDepositStatus"}, "awardDate": {"@id": "pass:awardDate", "@type": "xsd:dateTime"}, @@ -35,6 +35,7 @@ "directFunder": {"@id": "pass:directFunder", "@type": "@id"}, "displayName": {"@id": "pass:displayName"}, "doi": {"@id": "pass:doi"}, + "effectivePolicies": {"@id": "pass:effectivePolicies", "@container": "@set", "@type": "@id"}, "email": {"@id": "pass:email"}, "endDate": {"@id": "pass:endDate", "@type": "xsd:dateTime"}, "eventType": {"@id": "pass:eventType"}, @@ -77,6 +78,7 @@ "repositoryKey": {"@id": "pass:repositoryKey"}, "repositories": {"@id": "pass:repositories", "@container": "@set", "@type": "@id"}, "roles": {"@id": "pass:roles", "@container": "@set"}, + "schemas": {"@id": "pass:schemas", "@container": "@set", "@type": "@id"}, "source": {"@id": "pass:source"}, "startDate": {"@id": "pass:startDate", "@type": "xsd:dateTime"}, "submission": {"@id": "pass:submission", "@type": "@id"}, @@ -84,6 +86,8 @@ "submitted": {"@id": "pass:submitted", "@type:": "xsd:boolean"}, "submittedDate": {"@id": "pass:submittedDate", "@type": "xsd:dateTime"}, "submitter": {"@id": "pass:submitter", "@type": "@id"}, + "submitterEmail": {"@id": "pass:submitterEmail", "@type": "@id"}, + "submitterName": {"@id": "pass:submitterName"}, "title": {"@id": "pass:title"}, "uri": {"@id": "pass:uri", "@type": "@id"}, "url": {"@id": "pass:url", "@type": "@id"}, diff --git a/pom.xml b/pom.xml index b46cb69..e295172 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ org.dataconservancy.pass pass-grant-loader pom - 1.3.0 + 1.4.0 pass-grant-cli @@ -51,29 +51,29 @@ UTF-8 22 2.33 - 3.0.0 - 1.9.3 - 1.12 - 1.6 - 4.1.0 + 3.1.0 + 1.9.4 + 1.14 + 1.8 + 4.1.2 0.30.0 - 1.5.10 + 1.5.13 1.6.2 - 4.12 + 4.13 1.2.3 3.8.1 2.22.2 - 3.2.1 + 3.2.3 2.22.2 - 3.3.2 + 3.4.0 2.8.2 - 2.27.0 - 0.6.0 - 0.6.0 - 0.6.0 + 3.3.3 + 0.7.0 + 0.7.0 + 0.7.0 12.2.0.1 - 0.8.12 - 1.7.26 + 0.8.13 + 1.7.30 3.4.6