From 57f7ca0bbfa480563b0943ddef72cbf73fe3593f Mon Sep 17 00:00:00 2001 From: jrmartino Date: Mon, 27 Apr 2020 09:41:32 -0400 Subject: [PATCH 01/14] remove LATEST join from the coeus query string update the grant updater to select values for an update by start and end dates update the utility class to revise the notions of equality and updating for grants update tests to verify new behavior --- .../pass/grant/data/CoeusConnector.java | 10 +- .../pass/grant/data/CoeusPassEntityUtil.java | 38 +++-- .../pass/grant/data/DefaultPassUpdater.java | 149 ++++++++++-------- .../pass/grant/data/CoeusConnectorTest.java | 12 +- .../grant/integration/JhuPassUpdaterIT.java | 40 ++--- 5 files changed, 131 insertions(+), 118 deletions(-) diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java index 756b4cb..2eb98bb 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java @@ -285,11 +285,11 @@ private String buildGrantQueryString(String startDate, String awardEndDate){ sb.append(String.join(", ",viewFields)); sb.append(" FROM"); sb.append(" COEUS.JHU_FACULTY_FORCE_PROP A"); - sb.append(" INNER JOIN "); - sb.append(" (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP"); - sb.append(" FROM COEUS.JHU_FACULTY_FORCE_PROP GROUP BY GRANT_NUMBER) LATEST"); - sb.append(" ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP"); - sb.append(" AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER"); + // sb.append(" INNER JOIN "); + // sb.append(" (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP"); + // sb.append(" FROM COEUS.JHU_FACULTY_FORCE_PROP GROUP BY GRANT_NUMBER) LATEST"); + // sb.append(" ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP"); + // sb.append(" AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER"); sb.append(" INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B ON A.INST_PROPOSAL = B.INST_PROPOSAL"); sb.append(" INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C ON B.EMPLOYEE_ID = C.EMPLOYEE_ID"); sb.append(" LEFT JOIN COEUS.SWIFT_SPONSOR D ON A.PRIME_SPONSOR_CODE = D.SPONSOR_CODE"); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java index 5be4e4e..c80cd54 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java @@ -20,7 +20,7 @@ import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.User; - +import java.net.URI; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -165,16 +165,16 @@ private User updateUser (User system, User stored) { * @return a boolean which asserts whether the two supplied Grants are "COEUS equal" */ private boolean coeusGrantsEqual(Grant system, Grant stored) { - if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return false; + // if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return false; if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return false; - if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return false; - if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return false; - if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return false; + // if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; + // if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return false; + // if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return false; + // if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return false; if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return false; if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return false; - if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return false; - if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return false; + // if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return false; + // if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return false; if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return false; return true; } @@ -187,16 +187,20 @@ private boolean coeusGrantsEqual(Grant system, Grant stored) { * @return the Grant object which represents the Pass object, with any new information from COEUS merged in */ private Grant updateGrant(Grant system, Grant stored) { - stored.setAwardNumber(system.getAwardNumber()); + //stored.setAwardNumber(system.getAwardNumber()); stored.setAwardStatus(system.getAwardStatus()); - stored.setLocalKey(system.getLocalKey()); - stored.setProjectName(system.getProjectName()); - stored.setPrimaryFunder(system.getPrimaryFunder()); - stored.setDirectFunder(system.getDirectFunder()); - stored.setPi(system.getPi()); - stored.setCoPis(system.getCoPis()); - stored.setAwardDate(system.getAwardDate()); - stored.setStartDate(system.getStartDate()); + //stored.setLocalKey(system.getLocalKey()); + //stored.setProjectName(system.getProjectName()); + //stored.setPrimaryFunder(system.getPrimaryFunder()); + //stored.setDirectFunder(system.getDirectFunder()); + for( URI uri : stored.getCoPis() ) { + if ( !system.getCoPis().contains(uri) ) { + system.getCoPis().add(uri); + } + } + stored.setCoPis( system.getCoPis() ); + //stored.setAwardDate(system.getAwardDate()); + //stored.setStartDate(system.getStartDate()); stored.setEndDate(system.getEndDate()); return stored; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index b419e7a..4b08c7c 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -111,9 +111,7 @@ private void updateGrants(Collection> results) { LOG.info("Processing result set with " + results.size() + " rows"); boolean modeChecked = false; - for(Map rowMap : results){ - - String grantLocalKey; + for(Map rowMap : results) { if (!modeChecked) { if (!rowMap.containsKey(C_GRANT_LOCAL_KEY)) {//we always have this for grants @@ -123,19 +121,101 @@ private void updateGrants(Collection> results) { } } - grantLocalKey = rowMap.get(C_GRANT_LOCAL_KEY); + String grantLocalKey = rowMap.get(C_GRANT_LOCAL_KEY); + //get funder local keys. if a primary funder is not specified, we set it to the direct funder String directFunderLocalKey = rowMap.get(C_DIRECT_FUNDER_LOCAL_KEY); String primaryFunderLocalKey = rowMap.get(C_PRIMARY_FUNDER_LOCAL_KEY); primaryFunderLocalKey = (primaryFunderLocalKey == null? directFunderLocalKey: primaryFunderLocalKey); - Grant grant; + + //we will need funder PASS URIs - retrieve or create them, + //updating the info on them if necessary + if ( !funderMap.containsKey(directFunderLocalKey)) { + Funder updatedFunder = buildDirectFunder(rowMap); + URI passFunderURI = updateFunderInPass(updatedFunder); + funderMap.put(directFunderLocalKey, passFunderURI); + } + + if( !funderMap.containsKey(primaryFunderLocalKey)) { + Funder updatedFunder = buildPrimaryFunder(rowMap); + URI passFunderURI = updateFunderInPass(updatedFunder); + funderMap.put(primaryFunderLocalKey, passFunderURI); + } + + //same for any users + String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); + String abbreviatedRole = rowMap.get(C_ABBREVIATED_ROLE); + if (!userMap.containsKey(employeeId)) { + User updatedUser = buildUser(rowMap); + URI passUserURI = updateUserInPass(updatedUser); + userMap.put(employeeId, passUserURI); + } + + //now we know all about our user and funders for this record + + LOG.debug("Processing grant with localKey " + grantLocalKey); + //if this is the first record for this Grant, it will not be on the Map - //we process all data which is common to every record for this grant - //i.e., everything except the investigator(s) + Grant grant; if(!grantMap.containsKey(grantLocalKey)) { grant = new Grant(); + grant.setLocalKey(grantLocalKey); + grant.setCoPis(new ArrayList<>()); + grantMap.put(grantLocalKey, grant); + } + + grant = grantMap.get(grantLocalKey); + + //anybody who was ever a co-pi in an iteration will be in this list + if ( abbreviatedRole.equals("C") || abbreviatedRole.equals("K") ) { + URI userId=userMap.get( employeeId ); + if ( !grant.getCoPis().contains( userId ) ) { + grant.getCoPis().add( userId ); + statistics.addCoPi(); + } + } + + //now do things which may depend on the date + DateTime awardDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_AWARD_DATE, null)); + DateTime startDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_START_DATE, null)); + DateTime endDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_END_DATE, null)); + + //set values that should match earliest iteration of the grant + //these are used only for the initial grant load + //we test for both award date and start date in case one is missing - belt and suspenders + if (startDate != null && (grant.getStartDate() == null || startDate.isBefore(grant.getStartDate())) || + awardDate != null && (grant.getAwardDate() == null || awardDate.isBefore(grant.getAwardDate()))) { + grant.setAwardDate(awardDate); + grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); grant.setAwardNumber(rowMap.get(C_GRANT_AWARD_NUMBER)); + grant.setDirectFunder(funderMap.get(directFunderLocalKey)); + grant.setPrimaryFunder(funderMap.get(primaryFunderLocalKey)); + grant.setAwardDate(awardDate); + grant.setStartDate(startDate); + } + + //set values that should match the latest iteration of the grant + //use !isBefore in case more than one PI is specified, need to process more than one + if (endDate != null && (grant.getEndDate() == null || !endDate.isBefore(grant.getEndDate()))) { + grant.setEndDate(endDate); + + //we want the PI to be the one listed on the most recent grant iteration + if ( abbreviatedRole.equals("P") ) { + URI userId=userMap.get(employeeId); + URI oldPiId=grant.getPi(); + if ( oldPiId == null ) { + grant.setPi(userId); + statistics.addPi(); + } else { + if ( !oldPiId.equals(userId) ) { + if ( !grant.getCoPis().contains(oldPiId) ) { + grant.getCoPis().add(oldPiId); + statistics.addCoPi(); + } + } + } + } String status = rowMap.getOrDefault(C_GRANT_AWARD_STATUS, null); @@ -153,63 +233,8 @@ private void updateGrants(Collection> results) { } else { grant.setAwardStatus(null); } - - grant.setLocalKey(grantLocalKey); - grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); - grant.setAwardDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_AWARD_DATE, null))); - grant.setStartDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_START_DATE, null))); - grant.setEndDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_END_DATE, null))); - - //process direct funder, and primary funder if we have one - //update funder(s) in Pass as needed - if (funderMap.containsKey(directFunderLocalKey)) { - grant.setDirectFunder(funderMap.get(directFunderLocalKey)); - } else { - Funder updatedFunder = buildDirectFunder(rowMap); - URI passFunderURI = updateFunderInPass(updatedFunder); - funderMap.put(directFunderLocalKey, passFunderURI); - grant.setDirectFunder(passFunderURI); - } - - if(funderMap.containsKey(primaryFunderLocalKey)) { - grant.setPrimaryFunder(funderMap.get(primaryFunderLocalKey)); - } else { - Funder updatedFunder = buildPrimaryFunder(rowMap); - URI passFunderURI = updateFunderInPass(updatedFunder); - funderMap.put(primaryFunderLocalKey, passFunderURI); - grant.setPrimaryFunder(passFunderURI); - } - - grant.setCoPis(new ArrayList<>());//we will build this from scratch in either case - grantMap.put(grantLocalKey, grant);//save the state of this Grant } - //now we process the User (investigator) - grant = grantMap.get(grantLocalKey); - String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); - String abbreviatedRole = rowMap.get(C_ABBREVIATED_ROLE); - - if(abbreviatedRole.equals("C") || abbreviatedRole.equals("K") || grant.getPi() == null) { - if (!userMap.containsKey(employeeId)) { - User updatedUser = buildUser(rowMap); - URI passUserURI = updateUserInPass(updatedUser); - userMap.put(employeeId, passUserURI); - } - - //now our User URI is on the map - let's process: - if (abbreviatedRole.equals("P")) { - if (grant.getPi() == null) { - grant.setPi(userMap.get(employeeId)); - statistics.addPi(); - } else {// handle data with duplicate PIs - grant.getCoPis().add(userMap.get(employeeId)); - statistics.addCoPi(); - } - } else if ((abbreviatedRole.equals("C") || abbreviatedRole.equals("K")) && !grant.getCoPis().contains(userMap.get(employeeId))) { - grant.getCoPis().add(userMap.get(employeeId)); - statistics.addCoPi(); - } - } //we are done with this record, let's save the state of this Grant grantMap.put(grantLocalKey, grant); //see if this is the latest grant updated diff --git a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/CoeusConnectorTest.java b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/CoeusConnectorTest.java index 1e0d81d..1a48492 100644 --- a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/CoeusConnectorTest.java +++ b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/CoeusConnectorTest.java @@ -60,11 +60,7 @@ public void testBuildGrantString() { " A.AWARD_START, A.AWARD_END, A.SPONSOR, A.SPOSNOR_CODE, A.UPDATE_TIMESTAMP, B.ABBREVIATED_ROLE, B.EMPLOYEE_ID," + " C.FIRST_NAME, C.MIDDLE_NAME, C.LAST_NAME, C.EMAIL_ADDRESS, C.JHED_ID, D.SPONSOR_NAME, D.SPONSOR_CODE" + " FROM" + - " COEUS.JHU_FACULTY_FORCE_PROP A INNER JOIN " + - " (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP" + - " FROM COEUS.JHU_FACULTY_FORCE_PROP" + - " GROUP BY GRANT_NUMBER) LATEST" + - " ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER" + + " COEUS.JHU_FACULTY_FORCE_PROP A" + " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B" + " ON A.INST_PROPOSAL = B.INST_PROPOSAL" + " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C" + @@ -84,11 +80,7 @@ public void testBuildGrantString() { " A.AWARD_START, A.AWARD_END, A.SPONSOR, A.SPOSNOR_CODE, A.UPDATE_TIMESTAMP, B.ABBREVIATED_ROLE, B.EMPLOYEE_ID," + " C.FIRST_NAME, C.MIDDLE_NAME, C.LAST_NAME, C.EMAIL_ADDRESS, C.JHED_ID, D.SPONSOR_NAME, D.SPONSOR_CODE" + " FROM" + - " COEUS.JHU_FACULTY_FORCE_PROP A INNER JOIN " + - " (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP" + - " FROM COEUS.JHU_FACULTY_FORCE_PROP" + - " GROUP BY GRANT_NUMBER) LATEST" + - " ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER" + + " COEUS.JHU_FACULTY_FORCE_PROP A" + " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B" + " ON A.INST_PROPOSAL = B.INST_PROPOSAL" + " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C" + diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java index feb9b43..9a6caf3 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java @@ -18,7 +18,6 @@ import org.dataconservancy.pass.client.PassClient; import org.dataconservancy.pass.client.PassClientFactory; import org.dataconservancy.pass.grant.data.DateTimeUtil; -import org.dataconservancy.pass.grant.data.PassEntityUtil; import org.dataconservancy.pass.grant.data.PassUpdateStatistics; import org.dataconservancy.pass.grant.data.JhuPassUpdater; import org.dataconservancy.pass.grant.data.CoeusPassEntityUtil; @@ -35,7 +34,6 @@ import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; -import static java.lang.Thread.currentThread; import static java.lang.Thread.sleep; import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; import static org.junit.Assert.assertTrue; @@ -62,21 +60,17 @@ @RunWith(MockitoJUnitRunner.class) public class JhuPassUpdaterIT { - private final String DOMAIN = "johnshopkins.edu"; - - - private List> resultSet = new ArrayList<>(); - private static String employeeidPrefix = "johnshopkins.edu:employeeid:"; - private static String jhedPrefix = "johnshopkins.edu:jhed:"; - private CoeusPassEntityUtil passEntityUtil = new CoeusPassEntityUtil(); - private Map funderPolicyUriMap = new HashMap<>(); - private String prefix; + private final List> resultSet = new ArrayList<>(); + private static final String employeeidPrefix = "johnshopkins.edu:employeeid:"; + private static final String jhedPrefix = "johnshopkins.edu:jhed:"; + private final CoeusPassEntityUtil passEntityUtil = new CoeusPassEntityUtil(); + private final Map funderPolicyUriMap = new HashMap<>(); private String directFunderPolicyUriString1; private String primaryFunderPolicyUriString1; - private PassClient passClient = PassClientFactory.getPassClient(); + private final PassClient passClient = PassClientFactory.getPassClient(); @Rule public TemporaryFolder folder= new TemporaryFolder(); @@ -188,13 +182,14 @@ public void updateGrantsIT() throws InterruptedException { assertEquals(0, statistics.getUsersUpdated()); //now let's monkey with a few things; we expect to update the changed objects + //first change all tehthings that shouldn't matter Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1); + rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1 + "MOO"); rowMap.put(C_GRANT_AWARD_STATUS, "Active"); rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + 1); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOOO"); - rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); - rowMap.put(C_GRANT_START_DATE, "01/01/2001"); + rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOO"); + rowMap.put(C_GRANT_AWARD_DATE, "01/01/1999"); + rowMap.put(C_GRANT_START_DATE, "01/01/1999"); rowMap.put(C_GRANT_END_DATE, "01/01/2002"); rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + 1); @@ -224,7 +219,7 @@ public void updateGrantsIT() throws InterruptedException { assertEquals(0, statistics.getFundersCreated()); assertEquals(1, statistics.getFundersUpdated()); assertEquals(0, statistics.getGrantsCreated()); - assertEquals(1, statistics.getGrantsUpdated()); + assertEquals(0, statistics.getGrantsUpdated()); assertEquals(0, statistics.getUsersCreated()); assertEquals(1, statistics.getUsersUpdated()); @@ -247,11 +242,7 @@ public void updateGrantsIT() throws InterruptedException { assertEquals(grant.getAwardNumber(), passGrant.getAwardNumber()); assertEquals(grant.getAwardStatus(), passGrant.getAwardStatus()); assertEquals(grant.getLocalKey(), passGrant.getLocalKey()); - if (i == 1) { - assertEquals(grant.getProjectName() + "MOOO", passGrant.getProjectName()); - } else { - assertEquals(grant.getProjectName(), passGrant.getProjectName()); - } + assertEquals(grant.getProjectName(), passGrant.getProjectName()); assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); assertEquals(grant.getStartDate(), passGrant.getStartDate()); assertEquals(grant.getEndDate(), passGrant.getEndDate()); @@ -298,7 +289,7 @@ public void updateGrantsIT() throws InterruptedException { user.setEmail(C_USER_EMAIL + i); URI userUri = null; - ListIterator idIterator = user.getLocatorIds().listIterator(); + ListIterator idIterator = user.getLocatorIds().listIterator(); while (userUri == null && idIterator.hasNext()) { String id = String.valueOf(idIterator.next()); @@ -412,6 +403,7 @@ public void updateFundersIT() throws InterruptedException { assertNotNull(passClient.readResource(policy2Uri, Policy.class)); Funder funder1 = new Funder(); + String DOMAIN = "johnshopkins.edu"; String fullLocalKey = new Identifier(DOMAIN, "funder", "22229999").serialize(); funder1.setLocalKey(fullLocalKey); funder1.setName("Funder One"); @@ -527,7 +519,7 @@ public void testSerializeAndDeserialize() throws IOException { e.printStackTrace(); } - List input = null; + List> input = null; try (FileInputStream fis = new FileInputStream(serialized); ObjectInputStream in = new ObjectInputStream(fis) ){ From 583cf87b1846124bc94ffd4fa8942881de9871e0 Mon Sep 17 00:00:00 2001 From: jrmartino Date: Mon, 27 Apr 2020 09:42:37 -0400 Subject: [PATCH 02/14] update version in pom to 1.4.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b46cb69..3912ce5 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ org.dataconservancy.pass pass-grant-loader pom - 1.3.0 + 1.4.0-SNAPSHOT pass-grant-cli From a84944fc6432bb5b6af9ba2eab300b4d70c80cef Mon Sep 17 00:00:00 2001 From: jrmartino Date: Sun, 3 May 2020 22:25:34 -0400 Subject: [PATCH 03/14] add a new implementation JhuPassInitUpdater supported by a new option for the CLI and a new CoeusPassInitEntityUtil class Clarify method names and loginc in JHU utility classes --- pass-grant-cli/pom.xml | 2 +- .../pass/grant/cli/JhuGrantLoaderApp.java | 14 ++-- .../pass/grant/cli/JhuGrantLoaderCLI.java | 7 +- pass-grant-data/pom.xml | 2 +- .../pass/grant/data/CoeusPassEntityUtil.java | 72 ++++++++--------- .../grant/data/CoeusPassInitEntityUtil.java | 78 +++++++++++++++++++ .../pass/grant/data/DefaultPassUpdater.java | 5 +- .../data/HarvardPilotPassEntityUtil.java | 2 +- .../pass/grant/data/JhuPassInitUpdater.java | 67 ++++++++++++++++ pass-grant-integration/pom.xml | 2 +- .../grant/integration/JhuPassUpdaterIT.java | 29 +++++++ 11 files changed, 228 insertions(+), 52 deletions(-) create mode 100644 pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java create mode 100644 pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java diff --git a/pass-grant-cli/pom.xml b/pass-grant-cli/pom.xml index 76d24d5..159fce1 100644 --- a/pass-grant-cli/pom.xml +++ b/pass-grant-cli/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.3.0 + 1.4.0-SNAPSHOT 4.0.0 diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java index f8ae9f6..6e19756 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java @@ -16,19 +16,18 @@ package org.dataconservancy.pass.grant.cli; -import org.dataconservancy.pass.grant.data.CoeusConnector; -import org.dataconservancy.pass.grant.data.DateTimeUtil; -import org.dataconservancy.pass.grant.data.GrantConnector; -import org.dataconservancy.pass.grant.data.JhuPassUpdater; -import org.dataconservancy.pass.grant.data.PassUpdater; +import org.dataconservancy.pass.grant.data.*; import java.util.Properties; class JhuGrantLoaderApp extends BaseGrantLoaderApp { - JhuGrantLoaderApp(String startDate, String awardEndDate, boolean email, String mode, String action, String dataFileName) { + boolean init = false; + + JhuGrantLoaderApp(String startDate, String awardEndDate, boolean email, String mode, String action, String dataFileName, boolean init) { super(startDate, awardEndDate, email, mode, action, dataFileName); super.setTimestamp(true); + this.init = init; } @Override @@ -43,6 +42,9 @@ GrantConnector configureConnector(Properties connectionProperties, Properties po @Override PassUpdater configureUpdater() { + if ( init ) { + return new JhuPassInitUpdater(); + } return new JhuPassUpdater(); } diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderCLI.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderCLI.java index 501dc71..b7d65df 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderCLI.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderCLI.java @@ -64,6 +64,11 @@ public class JhuGrantLoaderCLI { "01/01/2011") private static String awardEndDate = "01/01/2011"; + /** Specifies whether this run is an "initializing run" which is allowed to overwrite normally non-writable fields on grants */ + @Option(name = "-i", aliases = {"-init", "--init", "-initialize", "--initialize" }, usage = "When set to true, changes the behavior of the loader to allow it" + + "to update all fields stored on grants with info coming in from the pull. This is useful when updating existing grant records due to a change in policy" + + "about what the semantics of the stored records are.") + private static boolean init = false; /** Specifies an optional action - either "pull" or "load" - to restrict the operation of the application to only pull data * from COEUS to store in a file, or to only load into PASS data taken from a stored file, respectively. In either case, the path to @@ -113,7 +118,7 @@ public static void main(String[] args) { } /* Run the package generation application proper */ - JhuGrantLoaderApp app = new JhuGrantLoaderApp(startDate, awardEndDate, email, mode, action, dataFileName); + JhuGrantLoaderApp app = new JhuGrantLoaderApp(startDate, awardEndDate, email, mode, action, dataFileName, init); app.run(); System.exit((0)); } catch (CmdLineException e) { diff --git a/pass-grant-data/pom.xml b/pass-grant-data/pom.xml index 163755a..544f460 100644 --- a/pass-grant-data/pom.xml +++ b/pass-grant-data/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.3.0 + 1.4.0-SNAPSHOT 4.0.0 diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java index c80cd54..832b961 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java @@ -45,7 +45,7 @@ public class CoeusPassEntityUtil implements PassEntityUtil{ * @return the updated Funder - null if the Funder does not need to be updated */ public Funder update(Funder system, Funder stored) { - if (!coeusFundersEqual(system, stored)) { + if (funderNeedsUpdate(system, stored)) { return updateFunder(system, stored); } return null; @@ -59,7 +59,7 @@ public Funder update(Funder system, Funder stored) { * @return the updated User - null if the User does not need to be updated */ public User update(User system, User stored) { - if (!coeusUsersEqual(system, stored)) { + if (userNeedsUpdate(system, stored)) { return updateUser(system, stored); } return null; @@ -73,7 +73,7 @@ public User update(User system, User stored) { * @return the updated object - null if the Grant does not need to be updated */ public Grant update(Grant system, Grant stored) { - if (!coeusGrantsEqual(system, stored)) { + if (grantNeedsUpdate(system, stored)) { return updateGrant(system, stored); } return null; @@ -86,13 +86,13 @@ public Grant update(Grant system, Grant stored) { * @param stored the version of the Funder as read from Pass * @return a boolean which asserts whether the two supplied Funders are "COEUS equal" */ - private boolean coeusFundersEqual(Funder system, Funder stored) { + private boolean funderNeedsUpdate(Funder system, Funder stored) { //this adjustment handles the case where we take data from policy.properties file, which has no name info - if (system.getName() != null && !system.getName().equals(stored.getName())) return false; - if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return false; - return true; + if (system.getName() != null && !system.getName().equals(stored.getName())) return true; + if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return true; + return false; } /** @@ -117,16 +117,16 @@ private Funder updateFunder (Funder system, Funder stored) { * @param stored the version of the User as read from Pass * @return a boolean which asserts whether the two supplied Users are "COEUS equal" */ - private boolean coeusUsersEqual(User system, User stored) { + private boolean userNeedsUpdate(User system, User stored) { //first the fields for which COEUS is authoritative - if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return false; - if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return false; - if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return false; - if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return false; - //next, other fields which require some reasoning to decide whether an system is necessary - if (system.getEmail() != null && stored.getEmail() == null) return false; - if (system.getDisplayName() != null && stored.getDisplayName() == null) return false; - return true; + if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return true; + if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return true; + if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return true; + if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return true; + //next, other fields which require some reasoning to decide whether an update is necessary + if (system.getEmail() != null && stored.getEmail() == null) return true; + if (system.getDisplayName() != null && stored.getDisplayName() == null) return true; + return false; } /** @@ -160,23 +160,16 @@ private User updateUser (User system, User stored) { /** * Compare two Grant objects. Note that the Lists of Co-Pis are compared as Sets - * @param system the version of the Grant as seen in the COEUS system pull + * @param system the version of the Grant as seen in the COES system pull * @param stored the version of the Grant as read from Pass * @return a boolean which asserts whether the two supplied Grants are "COEUS equal" */ - private boolean coeusGrantsEqual(Grant system, Grant stored) { - // if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return false; - if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return false; - // if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - // if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return false; - // if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return false; - // if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return false; - if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return false; - if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return false; - // if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return false; - // if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return false; - if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return false; - return true; + private boolean grantNeedsUpdate(Grant system, Grant stored) { + if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return true; + if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; + if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; + if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return true; + return false; } /** @@ -187,20 +180,23 @@ private boolean coeusGrantsEqual(Grant system, Grant stored) { * @return the Grant object which represents the Pass object, with any new information from COEUS merged in */ private Grant updateGrant(Grant system, Grant stored) { - //stored.setAwardNumber(system.getAwardNumber()); stored.setAwardStatus(system.getAwardStatus()); - //stored.setLocalKey(system.getLocalKey()); - //stored.setProjectName(system.getProjectName()); - //stored.setPrimaryFunder(system.getPrimaryFunder()); - //stored.setDirectFunder(system.getDirectFunder()); + + //adjust the system view of co-pis by merging in the stored view for( URI uri : stored.getCoPis() ) { if ( !system.getCoPis().contains(uri) ) { system.getCoPis().add(uri); } } + URI storedPi = stored.getPi(); + if ( !system.getPi().equals( storedPi )) { + if ( !system.getCoPis().contains( storedPi )) { + system.getCoPis().add ( storedPi ); + } + } + + stored.setPi( system.getPi() ); stored.setCoPis( system.getCoPis() ); - //stored.setAwardDate(system.getAwardDate()); - //stored.setStartDate(system.getStartDate()); stored.setEndDate(system.getEndDate()); return stored; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java new file mode 100644 index 0000000..8fe77bf --- /dev/null +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java @@ -0,0 +1,78 @@ +package org.dataconservancy.pass.grant.data; + +import org.dataconservancy.pass.model.Grant; + + +import java.net.URI; +import java.util.HashSet; + +public class CoeusPassInitEntityUtil extends CoeusPassEntityUtil { + + @Override + public Grant update(Grant system, Grant stored) { + if (this.grantNeedsUpdate(system, stored)) { + return this.updateGrant(system, stored); + } + return null; + } + + + /** + * Compare two Grant objects. Note that the Lists of Co-Pis are compared as Sets + * @param system the version of the Grant as seen in the COEUS system pull + * @param stored the version of the Grant as read from Pass + * @return a boolean which asserts whether the stored grant needs to be updated + */ + + private boolean grantNeedsUpdate(Grant system, Grant stored) { + if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return true; + if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return true; + if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return true; + if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return true; + if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return true; + if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; + if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; + if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return true; + if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return true; + if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return true; + return false; + } + + /** + * Update a Pass Grant object with new information from COEUS + * + * @param system the version of the Grant as seen in the COEUS system pull + * @param stored the version of the Grant as read from Pass + * @return the Grant object which represents the Pass object, with any new information from COEUS merged in + */ + private Grant updateGrant(Grant system, Grant stored) { + stored.setAwardNumber(system.getAwardNumber()); + stored.setAwardStatus(system.getAwardStatus()); + stored.setLocalKey(system.getLocalKey()); + stored.setProjectName(system.getProjectName()); + stored.setPrimaryFunder(system.getPrimaryFunder()); + stored.setDirectFunder(system.getDirectFunder()); + + //adjust the system view of co-pis by merging in the stored view + for( URI uri : stored.getCoPis() ) { + if ( !system.getCoPis().contains(uri) ) { + system.getCoPis().add(uri); + } + } + URI storedPi = stored.getPi(); + if ( !system.getPi().equals( storedPi )) { + if ( !system.getCoPis().contains( storedPi )) { + system.getCoPis().add ( storedPi ); + } + } + + stored.setPi( system.getPi() ); + stored.setCoPis( system.getCoPis() ); + stored.setAwardDate(system.getAwardDate()); + stored.setStartDate(system.getStartDate()); + stored.setEndDate(system.getEndDate()); + return stored; + } + +} diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index 4b08c7c..2fb6be8 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -476,16 +476,15 @@ private URI updateGrantInPass(Grant systemGrant) { } Grant updatedGrant; if ( (updatedGrant = passEntityUtil.update(systemGrant, storedGrant)) != null) {//need to update - LOG.debug("Updating grant with localKey " + storedGrant.getLocalKey() + " to localKey " + systemGrant.getLocalKey()); passClient.updateResource(updatedGrant); statistics.addGrantsUpdated(); - LOG.debug("Updating grant with award number " + systemGrant.getLocalKey()); + LOG.debug("Updating grant with local key " + systemGrant.getLocalKey()); }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything //this can happen if the Grant was updated in COEUS only with information we don't consume here } else {//don't have a stored Grant for this URI - this one is new to Pass passGrantURI = passClient.createResource(systemGrant); statistics.addGrantsCreated(); - LOG.debug("Creating grant with award number " + systemGrant.getLocalKey()); + LOG.debug("Creating grant with local key " + systemGrant.getLocalKey()); } return passGrantURI; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java index 6c27c75..bbce610 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java @@ -123,7 +123,7 @@ private boolean harvardUsersEqual(User system, User stored) { //if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return false; if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return false; if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return false; - //next, other fields which require some reasoning to decide whether an system is necessary + //next, other fields which require some reasoning to decide whether an update is necessary if (system.getEmail() != null && stored.getEmail() == null) return false; if (system.getDisplayName() != null && stored.getDisplayName() == null) return false; return true; diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java new file mode 100644 index 0000000..63283e2 --- /dev/null +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java @@ -0,0 +1,67 @@ +package org.dataconservancy.pass.grant.data; + +import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.model.User; +import org.dataconservancy.pass.model.support.Identifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.C_USER_INSTITUTIONAL_ID; + +public class JhuPassInitUpdater extends DefaultPassUpdater { + + private static Logger LOG = LoggerFactory.getLogger(JhuPassInitUpdater.class); + + public JhuPassInitUpdater(PassClient passClient) + { + super(new CoeusPassInitEntityUtil(), passClient); + super.setDomain("johnshopkins.edu"); + } + + public JhuPassInitUpdater() { + super(new CoeusPassInitEntityUtil()); + super.setDomain("johnshopkins.edu"); + } + + @Override + User buildUser(Map rowMap) { + User user = new User(); + user.setFirstName(rowMap.get(C_USER_FIRST_NAME)); + if (rowMap.containsKey(C_USER_MIDDLE_NAME)) { + user.setMiddleName(rowMap.get(C_USER_MIDDLE_NAME)); + } + user.setLastName(rowMap.get(C_USER_LAST_NAME)); + user.setDisplayName(rowMap.get(C_USER_FIRST_NAME) + " " + rowMap.get(C_USER_LAST_NAME)); + user.setEmail(rowMap.get(C_USER_EMAIL)); + String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); + String hopkinsId = null; + if (rowMap.containsKey(C_USER_HOPKINS_ID)) { + hopkinsId = rowMap.get(C_USER_HOPKINS_ID); + } + String jhedId = null; + if (rowMap.get(C_USER_INSTITUTIONAL_ID) != null) { + jhedId = rowMap.get(C_USER_INSTITUTIONAL_ID).toLowerCase(); + } + //Build the List of locatorIds - put the most reliable ids first + String DOMAIN = "johnshopkins.edu"; + if (employeeId != null) { + String EMPLOYEE_ID_TYPE = "employeeid"; + user.getLocatorIds().add(new Identifier(DOMAIN, EMPLOYEE_ID_TYPE, employeeId).serialize()); + } + if (hopkinsId != null) { + String HOPKINS_ID_TYPE = "hopkinsid"; + user.getLocatorIds().add(new Identifier(DOMAIN, HOPKINS_ID_TYPE, hopkinsId).serialize()); + } + if (jhedId != null) { + String JHED_ID_TYPE = "jhed"; + user.getLocatorIds().add(new Identifier(DOMAIN, JHED_ID_TYPE, jhedId).serialize()); + } + user.getRoles().add(User.Role.SUBMITTER); + LOG.debug("Built user with employee ID " + employeeId); + return user; + } + +} diff --git a/pass-grant-integration/pom.xml b/pass-grant-integration/pom.xml index 5729d06..76e6bfd 100644 --- a/pass-grant-integration/pom.xml +++ b/pass-grant-integration/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.3.0 + 1.4.0-SNAPSHOT 4.0.0 diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java index 9a6caf3..1c81411 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java @@ -531,4 +531,33 @@ public void testSerializeAndDeserialize() throws IOException { assertEquals(resultSet, input); } + @Test + public void testCreateAndUpdateGrantsWithSeveralDates() { + String firstAwardDate="01/01/2000"; + String secondAwardDate="06/01/2002"; + String thirdAwardDate="08/15/2004"; + + String[] awardDates = { firstAwardDate, secondAwardDate, thirdAwardDate }; + + String firstStartDate="01/01/2001"; + String secondStartDate="01/01/2003"; + String thirdStartDate="01/01/2005"; + + String[] startDates= { firstStartDate, secondStartDate, thirdStartDate}; + + String firstEndDate="12/31/2001"; + String secondEndDate="12/31/2004"; + String thirdEndDate="12/31/2007"; + + String[] endDates = { firstEndDate, secondEndDate, thirdEndDate }; + + String firstAwardNumber="777979"; + String secondAwardNumber="77111111"; + String thirdAwardNumber="109111111"; + + String[] awardNumbers= {firstAwardNumber, secondAwardNumber, thirdAwardNumber }; + + String grantNumber = "M00-8675309"; + } + } \ No newline at end of file From 181a76b9bf99016124ee13824d4d755d6afbf810 Mon Sep 17 00:00:00 2001 From: jrmartino Date: Tue, 5 May 2020 23:17:46 -0400 Subject: [PATCH 04/14] update integration tests update default pass updater amd entity util classesto handle co-pi / pi shanges correctly --- .../pass/grant/data/CoeusPassEntityUtil.java | 10 +- .../grant/data/CoeusPassInitEntityUtil.java | 11 +- .../pass/grant/data/DefaultPassUpdater.java | 19 +-- .../HarvardPilotPassUpdaterIT.java | 2 +- .../integration/JhuPassInitUpdaterIT.java | 126 +++++++++++++++ .../grant/integration/JhuPassUpdaterIT.java | 2 +- .../pass/grant/integration/PassUpdaterIT.java | 150 ++++++++++++++++++ 7 files changed, 300 insertions(+), 20 deletions(-) create mode 100644 pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java create mode 100644 pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/PassUpdaterIT.java diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java index 832b961..4629d4e 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java @@ -168,7 +168,7 @@ private boolean grantNeedsUpdate(Grant system, Grant stored) { if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return true; if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; - if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return true; + if (system.getEndDate() != null? system.getEndDate().isAfter(stored.getEndDate()) : stored.getEndDate() != null) return true; return false; } @@ -195,9 +195,15 @@ private Grant updateGrant(Grant system, Grant stored) { } } + if (system.getCoPis().contains(system.getPi())) { + system.getCoPis().remove(system.getPi()); + } + stored.setPi( system.getPi() ); stored.setCoPis( system.getCoPis() ); - stored.setEndDate(system.getEndDate()); + if ( system.getEndDate().isAfter(stored.getEndDate())) { + stored.setEndDate(system.getEndDate()); + } return stored; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java index 8fe77bf..b7adb0f 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java @@ -33,9 +33,9 @@ private boolean grantNeedsUpdate(Grant system, Grant stored) { if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return true; if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; - if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return true; - if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return true; - if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return true; + if (system.getAwardDate() != null? system.getAwardDate().isBefore(stored.getAwardDate()) : stored.getAwardDate() != null) return true; + if (system.getStartDate() != null? system.getStartDate().isBefore(stored.getStartDate()) : stored.getStartDate() != null) return true; + if (system.getEndDate() != null? system.getEndDate().isAfter(stored.getEndDate()) : stored.getEndDate() != null) return true; return false; } @@ -67,8 +67,13 @@ private Grant updateGrant(Grant system, Grant stored) { } } + if (system.getCoPis().contains(system.getPi())) { + system.getCoPis().remove(system.getPi()); + } + stored.setPi( system.getPi() ); stored.setCoPis( system.getCoPis() ); + //since this is essentially an initial pull, we can just take the system values stored.setAwardDate(system.getAwardDate()); stored.setStartDate(system.getStartDate()); stored.setEndDate(system.getEndDate()); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index 2fb6be8..c0637ee 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -54,14 +54,11 @@ public class DefaultPassUpdater implements PassUpdater{ private PassUpdateStatistics statistics = new PassUpdateStatistics(); private PassEntityUtil passEntityUtil; - //used in test classes private Map grantUriMap = new HashMap<>(); - //used in unit test //some entities may be referenced many times during an update, but just need to be updated the first time //they are encountered. these include Users and Funders. we save the overhead of redundant updates //of these by looking them up here; if they are on the Map, they have already been processed - // private Map funderMap = new HashMap<>(); private Map userMap = new HashMap<>(); @@ -152,8 +149,7 @@ private void updateGrants(Collection> results) { } //now we know all about our user and funders for this record - - + // let's get to the grant proper LOG.debug("Processing grant with localKey " + grantLocalKey); //if this is the first record for this Grant, it will not be on the Map @@ -161,7 +157,6 @@ private void updateGrants(Collection> results) { if(!grantMap.containsKey(grantLocalKey)) { grant = new Grant(); grant.setLocalKey(grantLocalKey); - grant.setCoPis(new ArrayList<>()); grantMap.put(grantLocalKey, grant); } @@ -169,7 +164,7 @@ private void updateGrants(Collection> results) { //anybody who was ever a co-pi in an iteration will be in this list if ( abbreviatedRole.equals("C") || abbreviatedRole.equals("K") ) { - URI userId=userMap.get( employeeId ); + URI userId = userMap.get( employeeId ); if ( !grant.getCoPis().contains( userId ) ) { grant.getCoPis().add( userId ); statistics.addCoPi(); @@ -209,6 +204,7 @@ private void updateGrants(Collection> results) { statistics.addPi(); } else { if ( !oldPiId.equals(userId) ) { + grant.setPi(userId); if ( !grant.getCoPis().contains(oldPiId) ) { grant.getCoPis().add(oldPiId); statistics.addCoPi(); @@ -399,8 +395,7 @@ private URI updateFunderInPass(Funder systemFunder) { if ((updatedFunder = passEntityUtil.update(systemFunder, storedFunder)) != null) {//need to update passClient.updateResource(updatedFunder); statistics.addFundersUpdated(); - }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything - //this can happen if the Grant was updated in COEUS only with information we don't consume here + } } else {//don't have a stored Funder for this URI - this one is new to Pass if (systemFunder.getName() != null) {//only add if we have a name passFunderURI = passClient.createResource(systemFunder); @@ -443,8 +438,7 @@ private URI updateUserInPass(User systemUser) { } passClient.updateResource(updatedUser); statistics.addUsersUpdated(); - }//if the Pass version is COEUS-equal to our version from the update, and there are no null fields we care about, - //we don't have to do anything. this can happen if the User was updated in COEUS only with information we don't consume here + } } else if (! mode.equals("user")) {//don't have a stored User for this URI - this one is new to Pass //but don't update if we are in user mode - just update existing users passUserUri = passClient.createResource(systemUser); @@ -479,8 +473,7 @@ private URI updateGrantInPass(Grant systemGrant) { passClient.updateResource(updatedGrant); statistics.addGrantsUpdated(); LOG.debug("Updating grant with local key " + systemGrant.getLocalKey()); - }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything - //this can happen if the Grant was updated in COEUS only with information we don't consume here + } } else {//don't have a stored Grant for this URI - this one is new to Pass passGrantURI = passClient.createResource(systemGrant); statistics.addGrantsCreated(); diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java index 0f6663a..42281fb 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java @@ -28,7 +28,7 @@ import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.Policy; import org.dataconservancy.pass.model.User; -import org.junit.Before; +import org.junit.Before;; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java new file mode 100644 index 0000000..b2dba32 --- /dev/null +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java @@ -0,0 +1,126 @@ + package org.dataconservancy.pass.grant.integration; + + + import org.dataconservancy.pass.grant.data.JhuPassInitUpdater; + import org.dataconservancy.pass.grant.data.PassUpdateStatistics; + import org.dataconservancy.pass.model.Grant; + import org.dataconservancy.pass.model.User; + import org.junit.Test; + + import java.net.URI; + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + + import static java.lang.Thread.sleep; + import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; + import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; + import static org.junit.Assert.*; + + public class JhuPassInitUpdaterIT extends PassUpdaterIT { + + JhuPassInitUpdater passUpdater = new JhuPassInitUpdater(passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + @Test + public void processInitGrantIT() throws InterruptedException { + List> resultSet = new ArrayList<>(); + + Map piRecord1 = makeBaseRowMap(); + resultSet.add(piRecord1); + + //Add a different user as a co-pi + //(Map startMap, String firstName, String middleName, String lastName, + // String email, String instId, String employId, String hopkinsId, String abbrRole) + Map coPiRecord1 = changeUserInMap(makeBaseRowMap(), "Albert", "Jones", + "Einstein", "aeinst1@jhu.edu", "aeinst1", userEmployeeId2, "WWTTRE", "C" ); + + resultSet.add(coPiRecord1); + + passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey1); + assertNotNull( passGrantUri ); + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId1 ); + assertNotNull( passUser1Uri ); + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId2 ); + assertNotNull( passUser2Uri ); + + Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); + + assertEquals( grantAwardNumber1, passGrant.getAwardNumber() ); + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey1, passGrant.getLocalKey() ); + assertEquals( grantProjectName1, passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantAwardDate1), passGrant.getAwardDate() ); + assertEquals( createJodaDateTime(grantStartDate1), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate1), passGrant.getEndDate() ); + assertEquals( passUser1Uri, passGrant.getPi() ); + assertEquals( 1, passGrant.getCoPis().size() ); + assertEquals( passUser2Uri, passGrant.getCoPis().get(0) ); + + //now do a multi-iteration pull, and check the result + + resultSet.clear(); + + //add another CoPi at the second iteration + Map adjustments = new HashMap<>(); + adjustments.put(C_GRANT_AWARD_DATE, grantAwardDate2); + adjustments.put(C_GRANT_START_DATE, grantStartDate2); + adjustments.put(C_GRANT_END_DATE, grantEndDate2); + adjustments.put(C_GRANT_PROJECT_NAME, grantProjectName2); + adjustments.put(C_GRANT_AWARD_NUMBER, grantAwardNumber2); + adjustments.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp2); + + Map piRecord2 = makeAdjustedBaseRowMap( adjustments ); + Map coPiRecord2 = makeAdjustedRowMap( coPiRecord1, adjustments ); + Map newCoPiRecord2 = changeUserInMap( coPiRecord2, "Junie", "Beatrice", + "Jones", "jjones1@jhu.edu", "ajjones1", userEmployeeId3, "THIHKX", "C" ); + + //change PI at the third iteration + adjustments.clear(); + adjustments.put(C_GRANT_AWARD_DATE, grantAwardDate3); + adjustments.put(C_GRANT_START_DATE, grantStartDate3); + adjustments.put(C_GRANT_END_DATE, grantEndDate3); + adjustments.put(C_GRANT_PROJECT_NAME, grantProjectName3); + adjustments.put(C_GRANT_AWARD_NUMBER, grantAwardNumber3); + adjustments.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp3); + + //user 2 is the new PI, drop user 1; user 1 should show up as a co-pi + Map piRecord3 = makeAdjustedBaseRowMap( adjustments ); + Map newPiRecord3 = changeUserInMap(piRecord3, "Albert", "Jones", + "Einstein", "aeinst1@jhu.edu", "aeinst1", userEmployeeId2, + "WWTTRE", "P" ); + Map newCoPiRecord3 = makeAdjustedRowMap(newCoPiRecord2, adjustments); + + //in the initial pull, we will findall of the records (check?) + resultSet.add(piRecord1); + resultSet.add(coPiRecord1); + resultSet.add(coPiRecord2); + resultSet.add(newCoPiRecord2); + resultSet.add(newPiRecord3); + resultSet.add(newCoPiRecord3); + + passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + + passGrant = passClient.readResource( passGrantUri, Grant.class ); + + URI passUser3Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId3 ); + assertNotNull( passUser3Uri ); + + assertEquals( grantAwardNumber1, passGrant.getAwardNumber() ); + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey1, passGrant.getLocalKey() ); + assertEquals( grantProjectName1, passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantAwardDate1), passGrant.getAwardDate() ); + assertEquals( createJodaDateTime(grantStartDate1), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate3), passGrant.getEndDate() ); + assertEquals( passUser2Uri, passGrant.getPi() ); + assertEquals( 2, passGrant.getCoPis().size() ); + assertTrue( passGrant.getCoPis().contains(passUser1Uri) ); + assertTrue( passGrant.getCoPis().contains(passUser3Uri) ); + } + } diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java index 1c81411..5a79d0e 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java @@ -182,7 +182,7 @@ public void updateGrantsIT() throws InterruptedException { assertEquals(0, statistics.getUsersUpdated()); //now let's monkey with a few things; we expect to update the changed objects - //first change all tehthings that shouldn't matter + //first change all the things that shouldn't matter Map rowMap = new HashMap<>(); rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1 + "MOO"); rowMap.put(C_GRANT_AWARD_STATUS, "Active"); diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/PassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/PassUpdaterIT.java new file mode 100644 index 0000000..953c75d --- /dev/null +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/PassUpdaterIT.java @@ -0,0 +1,150 @@ +package org.dataconservancy.pass.grant.integration; + +import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.client.PassClientFactory; +import org.dataconservancy.pass.model.Policy; +import org.junit.Before; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; + +/** + * This class is a base class mainly for setting up test data to be processed by descendants of the DefaultPassUpdater + */ +public class PassUpdaterIT { + + String grantAwardNumber1 = "A10000000"; + String grantAwardNumber2 = "A10000001"; + String grantAwardNumber3 = "A10000002"; + String grantLocalKey1 = "10000000"; + String grantLocalKey2 = "100000001"; + String grantLocalKey3 = "100000002"; + String grantProjectName1 = "Awesome Research Project I"; + String grantProjectName2 = "Awesome Research Project II"; + String grantProjectName3 = "Awesome Research Project III"; + String grantAwardDate1 = "01/01/1999"; + String grantAwardDate2 = "01/01/2001"; + String grantAwardDate3 = "01/01/2003"; + String grantStartDate1 = "07/01/2000"; + String grantStartDate2 = "07/01/2002"; + String grantStartDate3 = "07/01/2004"; + String grantEndDate1 = "06/30/2002"; + String grantEndDate2 = "06/30/2004"; + String grantEndDate3 = "06/30/2006"; + + String grantUpdateTimestamp1 = "2006-03-11 00:00:00.0"; + String grantUpdateTimestamp2 = "2010-04-05 00:00:00.0"; + String grantUpdateTimestamp3 = "2015-11-11 00:00:00.0"; + + String userEmployeeId1 = "30000000"; + String userEmployeeId2 = "30000001"; + String userEmployeeId3 = "30000002"; + + String primaryFunderPolicyUriString; + String directFunderPolicyUriString; + + String grantIdPrefix = "johnshopkins.edu:grant:"; + String funderIdPrefix = "johnshopkins.edu:funder:"; + String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; + String employeeidPrefix = "johnshopkins.edu:employeeid:"; + String jhedidPrefis = "johnshopkins.edu:jhed:"; + + PassClient passClient = PassClientFactory.getPassClient(); + + @Before + public void setup() { + String prefix = System.getProperty("pass.fedora.baseurl"); + if ( !prefix.endsWith("/") ) { + prefix = prefix + "/"; + } + + Policy policy = new Policy(); + policy.setTitle("Primary Policy"); + policy.setDescription("MOO"); + URI policyURI = passClient.createResource(policy); + primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + // funderPolicyUriMap.put("PrimaryFunderPolicy"+i,policyURI); + + policy =new Policy(); + policy.setTitle("Direct Policy"); + policy.setDescription("MOO"); + policyURI =passClient.createResource(policy); + directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + // funderPolicyUriMap.put("DirectFunderPolicy"+i,policyURI); + + } + + Map makeBaseRowMap() { + + Map rowMap = new HashMap<>(); + rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber1); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey1); + rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName1); + rowMap.put(C_GRANT_AWARD_DATE, grantAwardDate1); + rowMap.put(C_GRANT_START_DATE, grantStartDate1); + rowMap.put(C_GRANT_END_DATE, grantEndDate1); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000000"); + rowMap.put(C_DIRECT_FUNDER_NAME, "Enormous State University"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000001"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "J L Gotrocks Foundation"); + + rowMap.put(C_USER_FIRST_NAME, "Andrew"); + rowMap.put(C_USER_MIDDLE_NAME, "Carnegie"); + rowMap.put(C_USER_LAST_NAME, "Melon"); + rowMap.put(C_USER_EMAIL, "amelon1@jhu.edu"); + rowMap.put(C_USER_INSTITUTIONAL_ID, "amelon1"); + rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId1); + rowMap.put(C_USER_HOPKINS_ID, "OEJSNR"); + + rowMap.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp1); + rowMap.put(C_ABBREVIATED_ROLE, ("P")); + + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); + + return rowMap; + } + + Map makeAdjustedBaseRowMap( Map replacements) { + Map adjustedMap = new HashMap<>(); + adjustedMap.putAll(makeBaseRowMap()); + for (String key : replacements.keySet()) { + adjustedMap.replace(key, replacements.get(key)); + } + return adjustedMap; + } + + Map makeAdjustedRowMap( Map startMap, Map replacements) { + Map adjustedMap = new HashMap<>(); + adjustedMap.putAll( startMap ); + for (String key : replacements.keySet()) { + adjustedMap.replace(key, replacements.get(key)); + } + return adjustedMap; + } + + + Map changeUserInMap(Map startMap, String firstName, String middleName, String lastName, + String email, String instId, String employId, + String hopkinsId, String abbrRole) { + Map adjustedMap = new HashMap<>(); + adjustedMap.putAll( startMap ); + adjustedMap.replace(C_USER_FIRST_NAME, firstName); + adjustedMap.replace(C_USER_MIDDLE_NAME, middleName); + adjustedMap.replace(C_USER_LAST_NAME, lastName); + adjustedMap.replace(C_USER_EMAIL, email); + adjustedMap.replace(C_USER_INSTITUTIONAL_ID, instId); + adjustedMap.replace(C_USER_EMPLOYEE_ID, employId); + adjustedMap.replace(C_USER_HOPKINS_ID, hopkinsId); + adjustedMap.replace(C_ABBREVIATED_ROLE, abbrRole); + + return adjustedMap; + } + + + +} \ No newline at end of file From e41b8b16584bb741f8728adf063ffa357a5e61ea Mon Sep 17 00:00:00 2001 From: jrmartino Date: Wed, 6 May 2020 13:19:27 -0400 Subject: [PATCH 05/14] update harvard pilot code to not use a harvard id; substitute smail address for employeeid instead refactor integration tests to make them easier to maintain and understand --- .../pass/grant/data/DefaultPassUpdater.java | 1 - .../pass/grant/data/HarvardFieldNames.java | 2 +- .../grant/data/HarvardPilotConnector.java | 4 +- .../grant/data/HarvardPilotPassUpdater.java | 7 + .../grant/data/HarvardPilotConnectorTest.java | 12 +- .../grant/integration/BasicPassUpdaterIT.java | 565 +++++++++++++++ .../HarvardPilotPassUpdaterIT.java | 370 ++++------ .../integration/JhuPassInitUpdaterIT.java | 244 +++++-- .../grant/integration/JhuPassUpdaterIT.java | 663 +++++------------- .../pass/grant/integration/PassUpdaterIT.java | 150 ---- 10 files changed, 1044 insertions(+), 974 deletions(-) create mode 100644 pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java delete mode 100644 pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/PassUpdaterIT.java diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index c0637ee..d268b42 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory; import java.net.URI; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.ListIterator; diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardFieldNames.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardFieldNames.java index 8579a46..451d98a 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardFieldNames.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardFieldNames.java @@ -23,7 +23,7 @@ public class HarvardFieldNames { public static final String H_GRANT_NAME = "Grant name"; public static final String H_INV_FIRST_NAME = "PI First Name"; public static final String H_INV_LAST_NAME ="PI Last Name"; - public static final String H_INV_ID = "PI Harvard ID";//guaranteed to be in grant data + //public static final String H_INV_ID = "PI Harvard ID";//guaranteed to be in grant data public static final String H_INV_EMAIL = "PI Email"; //public static final String H_INV_ROLE = public static final String H_GRANT_START_DATE = "Grant start date"; diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java index 9ba0f01..84bbf31 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java @@ -111,7 +111,7 @@ public List> retrieveUpdates(String queryString, String mode LOG.debug("Processing grant record ..."); //we only process rows with a Harvard ID - String employeeId = stringify(cells.getCell(6)); //G: user Harvard ID + String employeeId = stringify(cells.getCell(7)); //we use email as employee id if(employeeId != null && employeeId.length()>0) { Map rowMap = new HashMap<>(); @@ -125,7 +125,7 @@ public List> retrieveUpdates(String queryString, String mode String role = stringify(cells.getCell(5)); //F: Role rowMap.put(C_ABBREVIATED_ROLE, sortRole(role)); - rowMap.put(C_USER_EMPLOYEE_ID, employeeId); + rowMap.put(C_USER_EMPLOYEE_ID, employeeId); //row G used to be Harvard id, is missing now rowMap.put(C_USER_EMAIL, stringify(cells.getCell(7))); //H: PI Email String funderLocalKey = stringify(cells.getCell(8)); //I: Funder ID diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java index e17fa81..1da2007 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java @@ -16,6 +16,8 @@ package org.dataconservancy.pass.grant.data; +import org.dataconservancy.pass.client.PassClient; + public class HarvardPilotPassUpdater extends DefaultPassUpdater { public HarvardPilotPassUpdater () { @@ -23,4 +25,9 @@ public HarvardPilotPassUpdater () { super.setDomain("harvard.edu"); } + public HarvardPilotPassUpdater (PassClient passClient) { + super(new HarvardPilotPassEntityUtil(), passClient ); + super.setDomain("harvard.edu"); + } + } diff --git a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java index 01bab86..e27c561 100644 --- a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java +++ b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java @@ -101,7 +101,7 @@ public void testRetrieveGrantUpdates() throws IOException { "C", "P" }; - String[] harvardIds = {"86753091", + /* String[] harvardIds = {"86753091", "86753092", "86753093", "86753094", @@ -110,6 +110,16 @@ public void testRetrieveGrantUpdates() throws IOException { "86753097", "86753098", "86753099" + }; commented out because we aren't using these (yet) - emails instead*/ + String[] harvardIds = {"wdrumstick@harvard.edu", + "gflanksteak@harvard.edu", + "ddbrisket@harvard.edu", + "abacon@harvard.edu", + "efarmer@harvard.edu", + "", + "csteer@harvard.edu", + "dbovine@harvard.edu", + "bcow@harvard.edu" }; String[] emails = {"wdrumstick@harvard.edu", "gflanksteak@harvard.edu", diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java new file mode 100644 index 0000000..4e70083 --- /dev/null +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java @@ -0,0 +1,565 @@ +/* + * Copyright 2020 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.dataconservancy.pass.grant.integration; + +import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.client.PassClientFactory; +import org.dataconservancy.pass.grant.data.DateTimeUtil; +import org.dataconservancy.pass.grant.data.PassUpdateStatistics; +import org.dataconservancy.pass.grant.data.JhuPassUpdater; +import org.dataconservancy.pass.grant.data.CoeusPassEntityUtil; +import org.dataconservancy.pass.model.Funder; +import org.dataconservancy.pass.model.Grant; + +import org.dataconservancy.pass.model.Policy; +import org.dataconservancy.pass.model.User; +import org.dataconservancy.pass.model.support.Identifier; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static java.lang.Thread.sleep; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +/** + * An integration test class for the JhuPassUpdater. + */ +@RunWith(MockitoJUnitRunner.class) +@Ignore +public class BasicPassUpdaterIT { + + private final List> resultSet = new ArrayList<>(); + private static final String employeeidPrefix = "johnshopkins.edu:employeeid:"; + private static final String jhedPrefix = "johnshopkins.edu:jhed:"; + private final CoeusPassEntityUtil passEntityUtil = new CoeusPassEntityUtil(); + private final Map funderPolicyUriMap = new HashMap<>(); + + private String directFunderPolicyUriString1; + private String primaryFunderPolicyUriString1; + + + private final PassClient passClient = PassClientFactory.getPassClient(); + + @Rule + public TemporaryFolder folder= new TemporaryFolder(); + + @Before + public void setup() { + + for (int i = 0; i < 10; i++) { + + String prefix = System.getProperty("pass.fedora.baseurl"); + if (!prefix.endsWith("/")) { + prefix = prefix + "/"; + } + + Policy policy = new Policy(); + policy.setTitle("Primary Policy" + i); + policy.setDescription("MOO"); + URI policyURI = passClient.createResource(policy); + String primaryPolicyUriString = policyURI.toString().substring(prefix.length()); + funderPolicyUriMap.put("PrimaryFunderPolicy"+i, policyURI); + + policy = new Policy(); + policy.setTitle("Direct Policy" + i); + policy.setDescription("MOO"); + policyURI = passClient.createResource(policy); + String directPolicyUriString = policyURI.toString().substring(prefix.length()); + funderPolicyUriMap.put("DirectFunderPolicy"+i, policyURI); + + if(i==1) { + directFunderPolicyUriString1 = directPolicyUriString; + primaryFunderPolicyUriString1 = primaryPolicyUriString; + } + + Map rowMap = new HashMap<>(); + rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + i); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + i); + rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + i); + rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); + rowMap.put(C_GRANT_START_DATE, "01/01/2001"); + rowMap.put(C_GRANT_END_DATE, "01/01/2002"); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + i); + rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + i); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + i); + rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + i); + + rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); + rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); + rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); + rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); + rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); + rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); + rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); + + rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + i + ":00:00.0"); + rowMap.put(C_ABBREVIATED_ROLE, (i % 2 == 0 ? "P" : "C")); + rowMap.put(C_DIRECT_FUNDER_POLICY, directPolicyUriString ); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryPolicyUriString); + + resultSet.add(rowMap); + } + + } + + /** + * The behavior of PassUpdate's updateGrants() method is to compare the data coming in on the ResultSet with + * the existing data in Pass, and create objects if Pass does not yet have them, and update them if they exist in Pass but + * there are differences in the fields for which COEUS is the authoritative source, or COEUS has a clue about other fields which are null + * on the PASS object. + * + * @throws InterruptedException - the exception + */ + @Test + public void updateGrantsIT() throws InterruptedException { + + JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); + passUpdater.updatePass(resultSet, "grant"); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + assertEquals(5, statistics.getPisAdded()); + assertEquals(5, statistics.getCoPisAdded()); + assertEquals(20, statistics.getFundersCreated()); + assertEquals(0, statistics.getFundersUpdated()); + assertEquals(10, statistics.getGrantsCreated()); + assertEquals(0, statistics.getGrantsUpdated()); + assertEquals("2018-01-01 09:00:00.0", statistics.getLatestUpdateString()); + assertEquals(10, statistics.getUsersCreated()); + assertEquals(0, statistics.getUsersUpdated()); + + assertEquals(10, passUpdater.getGrantUriMap().size()); + + for (URI grantUri : passUpdater.getGrantUriMap().keySet()) { + Grant grant = passUpdater.getGrantUriMap().get(grantUri); + Grant passGrant = passUpdater.getPassClient().readResource(grantUri, Grant.class); + assertNull(passEntityUtil.update(grant, passGrant)); //this means grants are "coeus-equal" + + } + + sleep(20000); + //try depositing the exact same resultSet. nothing should happen in Pass + passUpdater.updatePass(resultSet, "grant"); + + assertEquals(0, statistics.getFundersCreated()); + assertEquals(0, statistics.getFundersUpdated()); + assertEquals(0, statistics.getGrantsCreated()); + assertEquals(0, statistics.getGrantsUpdated()); + assertEquals(0, statistics.getUsersCreated()); + assertEquals(0, statistics.getUsersUpdated()); + + //now let's monkey with a few things; we expect to update the changed objects + //first change all the things that shouldn't matter + Map rowMap = new HashMap<>(); + rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1 + "MOO"); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + 1); + rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOO"); + rowMap.put(C_GRANT_AWARD_DATE, "01/01/1999"); + rowMap.put(C_GRANT_START_DATE, "01/01/1999"); + rowMap.put(C_GRANT_END_DATE, "01/01/2002"); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + 1); + rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + 1 + "MOOOOO"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + 1); + rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + 1); + + rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + 1); + rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + 1 + "MOOOO"); + rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + 1); + rowMap.put(C_USER_EMAIL, C_USER_EMAIL + 1); + rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + 1); + rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + 1); + rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + 1); + + rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); + rowMap.put(C_ABBREVIATED_ROLE, ("C")); + + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString1); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString1); + + + resultSet.clear(); + resultSet.add(rowMap); + + passUpdater.updatePass(resultSet, "grant"); + assertEquals(0, statistics.getFundersCreated()); + assertEquals(1, statistics.getFundersUpdated()); + assertEquals(0, statistics.getGrantsCreated()); + assertEquals(0, statistics.getGrantsUpdated()); + assertEquals(0, statistics.getUsersCreated()); + assertEquals(1, statistics.getUsersUpdated()); + + sleep(20000); + + for (int i = 0; i < 10; i++) { + Grant grant = new Grant(); + grant.setAwardNumber(C_GRANT_AWARD_NUMBER + i); + grant.setAwardStatus(Grant.AwardStatus.ACTIVE); + String grantIdPrefix = "johnshopkins.edu:grant:"; + grant.setLocalKey(grantIdPrefix + C_GRANT_LOCAL_KEY + i); + grant.setProjectName(C_GRANT_PROJECT_NAME + i); + grant.setAwardDate(DateTimeUtil.createJodaDateTime("01/01/2000")); + grant.setStartDate(DateTimeUtil.createJodaDateTime("01/01/2001")); + grant.setEndDate(DateTimeUtil.createJodaDateTime("01/01/2002")); + + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grant.getLocalKey()); + Grant passGrant = passClient.readResource(passGrantUri, Grant.class); + + assertEquals(grant.getAwardNumber(), passGrant.getAwardNumber()); + assertEquals(grant.getAwardStatus(), passGrant.getAwardStatus()); + assertEquals(grant.getLocalKey(), passGrant.getLocalKey()); + assertEquals(grant.getProjectName(), passGrant.getProjectName()); + assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); + assertEquals(grant.getStartDate(), passGrant.getStartDate()); + assertEquals(grant.getEndDate(), passGrant.getEndDate()); + + //let's check funder stuff + Funder directFunder = new Funder(); + String funderIdPrefix = "johnshopkins.edu:funder:"; + directFunder.setLocalKey(funderIdPrefix + C_DIRECT_FUNDER_LOCAL_KEY + i); + directFunder.setName(C_DIRECT_FUNDER_NAME + i); + directFunder.setPolicy(funderPolicyUriMap.get("DirectFunderPolicy" + i)); + + URI directFunderUri = passClient.findByAttribute(Funder.class, "localKey", directFunder.getLocalKey()); + Funder passDirectFunder = passClient.readResource(directFunderUri, Funder.class); + if (i == 1) { + assertEquals(directFunder.getName() + "MOOOOO", passDirectFunder.getName()); + assertEquals(directFunder.getLocalKey(), passDirectFunder.getLocalKey()); + assertEquals(passDirectFunder.getId(), passGrant.getDirectFunder()); + } else { + assertEquals(directFunder.getName(), passDirectFunder.getName()); + } + + Funder primaryFunder = new Funder(); + primaryFunder.setLocalKey(funderIdPrefix + C_PRIMARY_FUNDER_LOCAL_KEY + i); + primaryFunder.setName(C_PRIMARY_FUNDER_NAME + i); + primaryFunder.setPolicy(funderPolicyUriMap.get("PrimaryFunderPolicy" + i)); + + URI primaryFunderUri = passClient.findByAttribute(Funder.class, "localKey", primaryFunder.getLocalKey()); + Funder passPrimaryFunder = passClient.readResource(primaryFunderUri, Funder.class); + assertEquals(primaryFunder.getName(), passPrimaryFunder.getName()); + assertEquals(passPrimaryFunder.getId(), passGrant.getPrimaryFunder()); + assertEquals(primaryFunder.getLocalKey(), passPrimaryFunder.getLocalKey()); + assertEquals(primaryFunder.getPolicy(), passPrimaryFunder.getPolicy()); + + User user = new User(); + + //institutionalId and localKey were localized by the grant loader + user.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + i); + String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; + user.getLocatorIds().add(hopkinsidPrefix + C_USER_HOPKINS_ID + i); + user.getLocatorIds().add(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + i); + user.setFirstName(C_USER_FIRST_NAME + i); + user.setMiddleName(C_USER_MIDDLE_NAME + i); + user.setLastName(C_USER_LAST_NAME + i); + user.setEmail(C_USER_EMAIL + i); + + URI userUri = null; + ListIterator idIterator = user.getLocatorIds().listIterator(); + + while (userUri == null && idIterator.hasNext()) { + String id = String.valueOf(idIterator.next()); + if (id != null) { + userUri = passClient.findByAttribute(User.class, "locatorIds", id); + } + } + + User passUser = passClient.readResource(userUri, User.class); + assertEquals(user.getFirstName(), passUser.getFirstName()); + if (i == 1) { + assertEquals(user.getMiddleName() + "MOOOO", passUser.getMiddleName()); + } else { + assertEquals(user.getMiddleName(), passUser.getMiddleName()); + } + assertEquals(user.getLastName(), passUser.getLastName()); + assertEquals(user.getEmail(), passUser.getEmail()); + assertTrue(user.getLocatorIds().containsAll(passUser.getLocatorIds())); + assertTrue(passUser.getLocatorIds().containsAll(user.getLocatorIds())); + assertEquals(passUser.getLocatorIds().size(), user.getLocatorIds().size()); + + if (i % 2 == 0) { + assertNotNull(passGrant.getPi()); + assertEquals(0, passGrant.getCoPis().size()); + } else { + assertNull(passGrant.getPi()); + assertEquals(1, passGrant.getCoPis().size()); + } + + } + } + + + @Test + public void updateUsersIT() throws InterruptedException { + + User user10 = new User(); + user10.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + 10); + user10.setFirstName(C_USER_FIRST_NAME + 10); + user10.setMiddleName(C_USER_MIDDLE_NAME + 10); + user10.setLastName(C_USER_LAST_NAME + 10); + + JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); + + URI passUserURI = passUpdater.getPassClient().createResource(user10); + + User passUser = passClient.readResource(passUserURI, User.class); + assertNull(passUser.getDisplayName()); + assertEquals(1, passUser.getLocatorIds().size()); + assertNull(passUser.getEmail()); + + sleep(20000); + + List> userResultSet = new ArrayList<>(); + + for (int i = 10; i < 12; i++) { + Map rowMap = new HashMap<>(); + + rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); + rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); + rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); + rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); + rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); + rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); + rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); + rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); + userResultSet.add(rowMap); + } + + + passUpdater.updatePass(userResultSet, "user"); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + //now update from the set of two users - the second one is not in PASS, but is not created + //the first (user10) should be updated, with new fields added + assertEquals(0, statistics.getUsersCreated()); + assertEquals(1, statistics.getUsersUpdated()); + + assertNotNull(passUserURI); + User updatedUser = passUpdater.getPassClient().readResource(passUserURI, User.class); + + assertNotNull(updatedUser.getEmail()); + assertNotNull(updatedUser.getDisplayName()); + assertNotNull(updatedUser.getLocatorIds()); + assertTrue(updatedUser.getLocatorIds().contains(employeeidPrefix + C_USER_EMPLOYEE_ID + 10)); + assertTrue(updatedUser.getLocatorIds().contains(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + 10)); + + assertEquals(C_USER_EMAIL + 10, updatedUser.getEmail()); + } + + /** + * Create some policies, deposit them into Fedora + * Then create a java data object linking funders to them + * this basically tests what happens when pulling in data from a policy properties file first, + * or from a coeus pull second + */ + @Test + public void updateFundersIT() throws InterruptedException { + Policy policy1 = new Policy(); + policy1. setTitle("Policy One"); + policy1. setDescription("Policy one Description"); + Policy policy2 = new Policy(); + policy2. setTitle("Policy Two"); + policy2. setDescription("Policy Two Description"); + + JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); + URI policy1Uri = passClient.createResource(policy1); + URI policy2Uri = passClient.createResource(policy2); + + assertNotNull(passClient.readResource(policy1Uri, Policy.class)); + assertNotNull(passClient.readResource(policy2Uri, Policy.class)); + + Funder funder1 = new Funder(); + String DOMAIN = "johnshopkins.edu"; + String fullLocalKey = new Identifier(DOMAIN, "funder", "22229999").serialize(); + funder1.setLocalKey(fullLocalKey); + funder1.setName("Funder One"); + Funder funder2 = new Funder(); + fullLocalKey = new Identifier(DOMAIN, "funder", "33330000").serialize(); + funder2.setLocalKey(fullLocalKey);//use full localKey + funder2.setName("Funder Two"); + + URI funder1Uri = passClient.createResource(funder1); + URI funder2Uri = passClient.createResource(funder2); + + assertNotNull(passClient.readResource(funder1Uri, Funder.class)); + assertNotNull(passClient.readResource(funder2Uri, Funder.class)); + + + String policyString1 = policy1Uri.getPath().substring("/fcrepo/rest/".length()); + assertTrue(policyString1.startsWith("policies")); + String policyString2 = policy2Uri.getPath().substring("/fcrepo/rest/".length()); + assertTrue(policyString2.startsWith("policies")); + + List> funderResultSet = new ArrayList<>(); + + Map rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "22229999"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString1); + funderResultSet.add(rowMap); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "33330000"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); + funderResultSet.add(rowMap); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "88888888"); // this one does not exist in pass + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); + funderResultSet.add(rowMap); + + + sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch + + passUpdater.updatePass(funderResultSet, "funder"); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + assertNotNull(passClient.readResource(funder1Uri, Funder.class)); + assertNotNull(passClient.readResource(funder1Uri, Funder.class).getPolicy()); + assertNotNull(passClient.readResource(funder2Uri, Funder.class)); + assertNotNull(passClient.readResource(funder2Uri, Funder.class).getPolicy()); + assertEquals(policy1Uri, passClient.readResource(funder1Uri, Funder.class).getPolicy()); + assertEquals(policy2Uri, passClient.readResource(funder2Uri, Funder.class).getPolicy()); + + assertEquals(0, statistics.getFundersCreated()); + assertEquals(2, statistics.getFundersUpdated()); + + //coeus pulls will have the funder names, we should be able to add one we don't know about + + funderResultSet = new ArrayList<>(); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "22229999"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 1"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); //let's change policies for this one + funderResultSet.add(rowMap); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "33330000"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 2"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); + funderResultSet.add(rowMap); + + rowMap = new HashMap<>(); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "88888888"); // this one does not exist in pass + rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 3"); + rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); + funderResultSet.add(rowMap); + + sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch + + passUpdater.updatePass(funderResultSet, "funder"); + statistics = passUpdater.getStatistics(); + + assertNotNull(passClient.readResource(funder1Uri, Funder.class)); + assertNotNull(passClient.readResource(funder1Uri, Funder.class).getPolicy()); + assertNotNull(passClient.readResource(funder2Uri, Funder.class)); + assertNotNull(passClient.readResource(funder2Uri, Funder.class).getPolicy()); + assertEquals(policy2Uri, passClient.readResource(funder1Uri, Funder.class).getPolicy()); + assertEquals(policy2Uri, passClient.readResource(funder2Uri, Funder.class).getPolicy()); + + assertEquals(1, statistics.getFundersCreated()); + assertEquals(2, statistics.getFundersUpdated()); + + + //DO AGAIN!! DO AGAIN!! + + sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch + + passUpdater.updatePass(funderResultSet, "funder"); + statistics = passUpdater.getStatistics(); + + assertEquals(0, statistics.getFundersCreated()); + assertEquals(0, statistics.getFundersUpdated()); + + } + + @Test + public void testSerializeAndDeserialize() throws IOException { + File serialized= folder.newFile("serializedData"); + + try (FileOutputStream fos = new FileOutputStream(serialized); + ObjectOutputStream out = new ObjectOutputStream(fos) + ){ + out.writeObject(resultSet); + } catch (IOException e) { + e.printStackTrace(); + } + + List> input = null; + try (FileInputStream fis = new FileInputStream(serialized); + ObjectInputStream in = new ObjectInputStream(fis) + ){ + input = (List>)in.readObject(); + } catch (IOException | ClassNotFoundException ex) { + ex.printStackTrace(); + } + + assertEquals(resultSet, input); + } + + @Test + public void testCreateAndUpdateGrantsWithSeveralDates() { + String firstAwardDate="01/01/2000"; + String secondAwardDate="06/01/2002"; + String thirdAwardDate="08/15/2004"; + + String[] awardDates = { firstAwardDate, secondAwardDate, thirdAwardDate }; + + String firstStartDate="01/01/2001"; + String secondStartDate="01/01/2003"; + String thirdStartDate="01/01/2005"; + + String[] startDates= { firstStartDate, secondStartDate, thirdStartDate}; + + String firstEndDate="12/31/2001"; + String secondEndDate="12/31/2004"; + String thirdEndDate="12/31/2007"; + + String[] endDates = { firstEndDate, secondEndDate, thirdEndDate }; + + String firstAwardNumber="777979"; + String secondAwardNumber="77111111"; + String thirdAwardNumber="109111111"; + + String[] awardNumbers= {firstAwardNumber, secondAwardNumber, thirdAwardNumber }; + + String grantNumber = "M00-8675309"; + } + +} \ No newline at end of file diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java index 42281fb..1e2e307 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java @@ -18,12 +18,7 @@ import org.dataconservancy.pass.client.PassClient; import org.dataconservancy.pass.client.PassClientFactory; -import org.dataconservancy.pass.grant.data.DateTimeUtil; -import org.dataconservancy.pass.grant.data.HarvardPilotPassEntityUtil; -import org.dataconservancy.pass.grant.data.HarvardPilotPassUpdater; -import org.dataconservancy.pass.grant.data.PassEntityUtil; -import org.dataconservancy.pass.grant.data.PassUpdateStatistics; -import org.dataconservancy.pass.grant.data.PassUpdater; +import org.dataconservancy.pass.grant.data.*; import org.dataconservancy.pass.model.Funder; import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.Policy; @@ -36,97 +31,66 @@ import java.net.URI; import java.util.ArrayList; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.HashMap; import static java.lang.Thread.sleep; import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; import static org.junit.Assert.*; + public class HarvardPilotPassUpdaterIT { - private static final String DOMAIN = "harvard.edu"; + PassClient passClient = PassClientFactory.getPassClient(); + PassUpdater passUpdater = new HarvardPilotPassUpdater(passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + String[] grantAwardNumber = { "C10000000", "C10000001", "C10000002" }; + String[] grantLocalKey = { "10000002", "10000002","10000002" }; //all the same, different from other ITs tho + String[] grantProjectName = {"Amazing Research Project I", "Amazing Research Project II", "Amazing Research Project III" }; + String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; + String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; + // String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; + String[] userEmployeeId= { "jpubli1@harvard.edu", "ssinis11@harvard.edu", "rsquir1@harvard.edu"}; + String[] userFirstName = {"John", "Simon", "Rocket"}; + String[] userLastName = { "Public", "Sinister", "Squirrel" }; + String[] userEmail = { "jpubli1@harvard.edu", "ssinis11@harvard.edu", "rsquir1@harvard.edu" }; - private List> resultSet = new ArrayList<>(); - private static String employeeidPrefix = DOMAIN + ":employeeid:"; - private PassEntityUtil passEntityUtil = new HarvardPilotPassEntityUtil(); - private Map funderPolicyUriMap = new HashMap<>(); + String primaryFunderPolicyUriString; + String directFunderPolicyUriString; - private String directFunderPolicyUriString1; - private String primaryFunderPolicyUriString1; + String grantIdPrefix = "harvard.edu:grant:"; + String employeeidPrefix = "harvard.edu:employeeid:"; - private PassClient passClient = PassClientFactory.getPassClient(); - @Rule public TemporaryFolder folder= new TemporaryFolder(); @Before public void setup() { - - for (int i = 0; i < 10; i++) { - - String prefix = System.getProperty("pass.fedora.baseurl"); - if (!prefix.endsWith("/")) { - prefix = prefix + "/"; - } - - Policy policy = new Policy(); - policy.setTitle("Primary Policy" + i); - policy.setDescription("MOO"); - URI policyURI = passClient.createResource(policy); - String primaryPolicyUriString = policyURI.toString().substring(prefix.length()); - funderPolicyUriMap.put("PrimaryFunderPolicy" + i, policyURI); - - policy = new Policy(); - policy.setTitle("Direct Policy" + i); - policy.setDescription("MOO"); - policyURI = passClient.createResource(policy); - String directPolicyUriString = policyURI.toString().substring(prefix.length()); - funderPolicyUriMap.put("DirectFunderPolicy" + i, policyURI); - - if (i == 1) { - directFunderPolicyUriString1 = directPolicyUriString; - primaryFunderPolicyUriString1 = primaryPolicyUriString; - } - - - - - Map rowMap = new HashMap<>(); - - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + i); - //rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + i); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + i); - //rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); - rowMap.put(C_GRANT_START_DATE, "01/01/2001"); - rowMap.put(C_GRANT_END_DATE, "01/01/2002"); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + i); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + i); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + i); - rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + i); - - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); - //rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - //rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - // rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); - - //rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + i + ":00:00.0"); - rowMap.put(C_ABBREVIATED_ROLE, (i % 2 == 0 ? "P" : "C")); - rowMap.put(C_DIRECT_FUNDER_POLICY, directPolicyUriString); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryPolicyUriString); - - resultSet.add(rowMap); + String prefix = System.getProperty("pass.fedora.baseurl"); + if ( !prefix.endsWith("/") ) { + prefix = prefix + "/"; } + + Policy policy = new Policy(); + policy.setTitle("Primary Policy 2"); + policy.setDescription("BAA"); + URI policyURI = passClient.createResource(policy); + primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + policy =new Policy(); + policy.setTitle("Direct Policy 2"); + policy.setDescription("BAA"); + policyURI =passClient.createResource(policy); + directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + } + /** * The behavior of PassUpdate's updatePass() method is to compare the data coming in on the ResultSet with * the existing data in Pass, and create objects if Pass does not yet have them, and update them if they exist in Pass but @@ -138,181 +102,111 @@ public void setup() { @Test public void updateGrantsIT() throws InterruptedException { - PassUpdater passUpdater = new HarvardPilotPassUpdater(); - passUpdater.updatePass(resultSet, "grant"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); - - assertEquals(5, statistics.getPisAdded()); - assertEquals(5, statistics.getCoPisAdded()); - assertEquals(20, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - assertEquals(10, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - //assertEquals("2018-01-01 09:00:00.0", statistics.getLatestUpdateString()); - assertEquals(10, statistics.getUsersCreated()); - assertEquals(0, statistics.getUsersUpdated()); - - assertEquals(10, passUpdater.getGrantUriMap().size()); + List> resultSet = new ArrayList<>(); - for (URI grantUri : passUpdater.getGrantUriMap().keySet()) { - Grant grant = passUpdater.getGrantUriMap().get(grantUri); - Grant passGrant = passUpdater.getPassClient().readResource(grantUri, Grant.class); - assertNull(passEntityUtil.update(grant, passGrant)); //this means grants are "harvard-equal" + //put in initial iteration as a correct existing record - PI is Public, Co-pi is Sinister + Map piRecord0 = makeRowMap(0, 0, "P"); + Map coPiRecord0 = makeRowMap(0, 1, "C"); - } + resultSet.add(piRecord0); + resultSet.add(coPiRecord0); - sleep(20000); - //try depositing the exact same resultSet. nothing should happen in Pass passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[0] ); + assertNotNull( passUser0Uri ); + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); + assertNotNull( passGrantUri ); + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[1] ); + assertNotNull( passUser1Uri ); + + Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); + + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() ); + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate[0]), passGrant.getEndDate() ); + assertEquals( passUser0Uri, passGrant.getPi() ); //Reckondwith + assertEquals( 1, passGrant.getCoPis().size() ); + assertEquals( passUser1Uri, passGrant.getCoPis().get(0)); + + //check statistics + assertEquals(1, statistics.getGrantsCreated()); + assertEquals(2, statistics.getUsersCreated()); + assertEquals(1, statistics.getPisAdded()); + assertEquals(1, statistics.getCoPisAdded()); + + //now simulate an incremental pull since the initial, adjust the stored grant + //we add a new co-pi Jones in the "1" iteration, and change the pi to Einstein in the "2" iteration + //we drop co-pi jones in the last iteration + + Map piRecord1 = makeRowMap(1, 0, "P"); + Map coPiRecord1 = makeRowMap(1, 1, "C"); + Map newCoPiRecord1 = makeRowMap(1, 2, "C"); + Map piRecord2 = makeRowMap (2, 1, "P"); + + //add in everything since the initial pull + resultSet.clear(); + resultSet.add(piRecord1); + resultSet.add(coPiRecord1); + resultSet.add(newCoPiRecord1); + resultSet.add(piRecord2); - assertEquals(0, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - assertEquals(0, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); - assertEquals(0, statistics.getUsersUpdated()); + passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + + passGrant = passClient.readResource( passGrantUri, Grant.class ); + + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[2] ); + assertNotNull( passUser2Uri ); + + assertEquals( grantAwardNumber[1], passGrant.getAwardNumber() );//earliest of the additions + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[1], passGrant.getLocalKey() );//earliest of the additions + assertEquals( grantProjectName[1], passGrant.getProjectName() );//earliest of the additions + assertEquals( createJodaDateTime(grantStartDate[1]), passGrant.getStartDate() );//earliest of the additions + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() );//latest of the additions + assertEquals( passUser1Uri, passGrant.getPi() );//Class + assertEquals( 3, passGrant.getCoPis().size() ); + assertTrue( passGrant.getCoPis().contains(passUser0Uri) );//Public + assertTrue( passGrant.getCoPis().contains(passUser1Uri) );//Sinister + assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Squirrel + } - //now let's monkey with a few things; we expect to update the changed objects + /** + * utility method to produce data as it would look coming from the Harvard spreadsheet + * @param iteration the iteration of the (multi-award) grant + * @param user the user supplied in the record + * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") + * @return + */ + private Map makeRowMap( int iteration, int user, String abbrRole) { Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1); - //rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + 1); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOOO"); - //rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); - rowMap.put(C_GRANT_START_DATE, "01/01/2001"); - rowMap.put(C_GRANT_END_DATE, "01/01/2002"); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + 1 + "MOOOO"); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + 1); + rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber[iteration]); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey[iteration]); + rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName[iteration]); + rowMap.put(C_GRANT_START_DATE, grantStartDate[iteration]); + rowMap.put(C_GRANT_END_DATE, grantEndDate[iteration]); - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + 1 + "MOOOOO"); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + 1); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + 1); - //rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + 1); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + 1); - //rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + 1); + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000002"); + rowMap.put(C_DIRECT_FUNDER_NAME, "Gargantuan State University"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000003"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "D Warbucks Foundation"); - //rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); - rowMap.put(C_ABBREVIATED_ROLE, ("C")); + rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); + rowMap.put(C_USER_LAST_NAME, userLastName[user]); + rowMap.put(C_USER_EMAIL, userEmail[user]);; + rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); - rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString1); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString1); + rowMap.put(C_ABBREVIATED_ROLE, abbrRole); + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); - resultSet.clear(); - resultSet.add(rowMap); - - passUpdater.updatePass(resultSet, "grant"); - assertEquals(0, statistics.getFundersCreated()); - assertEquals(1, statistics.getFundersUpdated()); - assertEquals(0, statistics.getGrantsCreated()); - assertEquals(1, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); - assertEquals(1, statistics.getUsersUpdated()); - - sleep(20000); - - for (int i = 0; i < 10; i++) { - Grant grant = new Grant(); - grant.setAwardNumber(C_GRANT_AWARD_NUMBER + i); - //grant.setAwardStatus(Grant.AwardStatus.ACTIVE); - String grantIdPrefix = DOMAIN + ":grant:"; - grant.setLocalKey(grantIdPrefix + C_GRANT_LOCAL_KEY + i); - grant.setProjectName(C_GRANT_PROJECT_NAME + i); - //grant.setAwardDate(DateTimeUtil.createJodaDateTime("01/01/2000")); - grant.setStartDate(DateTimeUtil.createJodaDateTime("01/01/2001")); - grant.setEndDate(DateTimeUtil.createJodaDateTime("01/01/2002")); - - URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grant.getLocalKey()); - Grant passGrant = passClient.readResource(passGrantUri, Grant.class); - - assertEquals(grant.getAwardNumber(), passGrant.getAwardNumber()); - //assertEquals(grant.getAwardStatus(), passGrant.getAwardStatus()); - assertEquals(grant.getLocalKey(), passGrant.getLocalKey()); - if (i == 1) { - assertEquals(grant.getProjectName() + "MOOO", passGrant.getProjectName()); - } else { - assertEquals(grant.getProjectName(), passGrant.getProjectName()); - } - //assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); - assertEquals(grant.getStartDate(), passGrant.getStartDate()); - assertEquals(grant.getEndDate(), passGrant.getEndDate()); - - //let's check funder stuff - Funder directFunder = new Funder(); - String funderIdPrefix = DOMAIN + ":funder:"; - directFunder.setLocalKey(funderIdPrefix + C_DIRECT_FUNDER_LOCAL_KEY + i); - directFunder.setName(C_DIRECT_FUNDER_NAME + i); - directFunder.setPolicy(funderPolicyUriMap.get("DirectFunderPolicy" + i)); - - URI directFunderUri = passClient.findByAttribute(Funder.class, "localKey", directFunder.getLocalKey()); - Funder passDirectFunder = passClient.readResource(directFunderUri, Funder.class); - if (i == 1) { - assertEquals(directFunder.getName() + "MOOOO", passDirectFunder.getName()); - assertEquals(directFunder.getLocalKey(), passDirectFunder.getLocalKey()); - assertEquals(passDirectFunder.getId(), passGrant.getDirectFunder()); - } else { - assertEquals(directFunder.getName(), passDirectFunder.getName()); - } - - Funder primaryFunder = new Funder(); - primaryFunder.setLocalKey(funderIdPrefix + C_PRIMARY_FUNDER_LOCAL_KEY + i); - primaryFunder.setName(C_PRIMARY_FUNDER_NAME + i); - primaryFunder.setPolicy(funderPolicyUriMap.get("PrimaryFunderPolicy" + i)); - - URI primaryFunderUri = passClient.findByAttribute(Funder.class, "localKey", primaryFunder.getLocalKey()); - Funder passPrimaryFunder = passClient.readResource(primaryFunderUri, Funder.class); - assertEquals(primaryFunder.getName(), passPrimaryFunder.getName()); - assertEquals(passPrimaryFunder.getId(), passGrant.getPrimaryFunder()); - assertEquals(primaryFunder.getLocalKey(), passPrimaryFunder.getLocalKey()); - assertEquals(primaryFunder.getPolicy(), passPrimaryFunder.getPolicy()); - - User user = new User(); - - //institutionalId and localKey were localized by the grant loader - user.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + i); - //String idPrefix = "johnshopkins.edu:hopkinsid:"; - //user.getLocatorIds().add(hopkinsidPrefix + C_USER_HOPKINS_ID + i); - //user.getLocatorIds().add(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + i); - user.setFirstName(C_USER_FIRST_NAME + i); - //user.setMiddleName(C_USER_MIDDLE_NAME + i); - user.setLastName(C_USER_LAST_NAME + i); - user.setEmail(C_USER_EMAIL + i); - - URI userUri = null; - ListIterator idIterator = user.getLocatorIds().listIterator(); - - while (userUri == null && idIterator.hasNext()) { - String id = String.valueOf(idIterator.next()); - if (id != null) { - userUri = passClient.findByAttribute(User.class, "locatorIds", id); - } - } - - User passUser = passClient.readResource(userUri, User.class); - if (i == 1) { - assertEquals(user.getFirstName() + "MOOOOO", passUser.getFirstName()); - } else { - assertEquals(user.getFirstName(), passUser.getFirstName()); - } - assertEquals(user.getLastName(), passUser.getLastName()); - assertEquals(user.getEmail(), passUser.getEmail()); - assertTrue(user.getLocatorIds().containsAll(passUser.getLocatorIds())); - assertTrue(passUser.getLocatorIds().containsAll(user.getLocatorIds())); - assertEquals(passUser.getLocatorIds().size(), user.getLocatorIds().size()); - - if (i % 2 == 0) { - assertNotNull(passGrant.getPi()); - assertEquals(0, passGrant.getCoPis().size()); - } else { - assertNull(passGrant.getPi()); - assertEquals(1, passGrant.getCoPis().size()); - } - - } + return rowMap; } - } diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java index b2dba32..ebf0a2b 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java @@ -1,10 +1,29 @@ - package org.dataconservancy.pass.grant.integration; +/* + * Copyright 2020 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.dataconservancy.pass.grant.integration; + import org.dataconservancy.pass.client.PassClient; + import org.dataconservancy.pass.client.PassClientFactory; import org.dataconservancy.pass.grant.data.JhuPassInitUpdater; import org.dataconservancy.pass.grant.data.PassUpdateStatistics; import org.dataconservancy.pass.model.Grant; + import org.dataconservancy.pass.model.Policy; import org.dataconservancy.pass.model.User; + import org.junit.Before; import org.junit.Test; import java.net.URI; @@ -18,109 +37,180 @@ import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; import static org.junit.Assert.*; - public class JhuPassInitUpdaterIT extends PassUpdaterIT { - + public class JhuPassInitUpdaterIT { + + String[] grantAwardNumber = { "A10000000", "A10000001", "A10000002" }; + String[] grantLocalKey = { "10000000", "10000000","10000000" }; //all the same + String[] grantProjectName = {"Awesome Research Project I", "Awesome Research Project II", "Awesome Research Project III" }; + String[] grantAwardDate = { "01/01/1999", "01/01/2001", "01/01/2003" }; + String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; + String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; + String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; + String[] userEmployeeId= { "30000000", "30000001", "30000002"}; + String[] userInstitutionalId = {"amelon1", "aeinst1", "jjones1" }; + String[] userHopkinsId = {"RANDOM", "OMRNDA", "DRMONA" }; + String[] userFirstName = {"Andrew", "Albert", "Junie"}; + String[] userMiddleName = {"Smith", "Carnegie", "Beatrice" }; + String[] userLastName = { "Melon", "Einstein", "Jones" }; + String[] userEmail = { "amelon1@jhu.edu", "aeinst1@jhu.edu", "jjones1@jhu.edu" }; + + + String primaryFunderPolicyUriString; + String directFunderPolicyUriString; + + String grantIdPrefix = "johnshopkins.edu:grant:"; + //String funderIdPrefix = "johnshopkins.edu:funder:"; + //String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; + String employeeidPrefix = "johnshopkins.edu:employeeid:"; + //String jhedidPrefis = "johnshopkins.edu:jhed:"; + + PassClient passClient = PassClientFactory.getPassClient(); JhuPassInitUpdater passUpdater = new JhuPassInitUpdater(passClient); PassUpdateStatistics statistics = passUpdater.getStatistics(); + @Before + public void setup() { + String prefix = System.getProperty("pass.fedora.baseurl"); + if ( !prefix.endsWith("/") ) { + prefix = prefix + "/"; + } + + Policy policy = new Policy(); + policy.setTitle("Primary Policy"); + policy.setDescription("MOO"); + URI policyURI = passClient.createResource(policy); + primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + policy =new Policy(); + policy.setTitle("Direct Policy"); + policy.setDescription("MOO"); + policyURI =passClient.createResource(policy); + directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + } + + /** + * we put an initial award for a grant into fedora, then simulate a pull of all records related + * to this grant from the Beginning of Time (including records which created the initial object) + * + * We expect to see some fields retained from the initial award, and others updated. The most + * interesting fields are the investigator fields: all CO-PIs ever on the grant should stay on the + * co-pi field throughout iterations. If a PI is changed, they should appear on the CO-PI field + * + * @throws InterruptedException + */ @Test public void processInitGrantIT() throws InterruptedException { List> resultSet = new ArrayList<>(); - Map piRecord1 = makeBaseRowMap(); - resultSet.add(piRecord1); - - //Add a different user as a co-pi - //(Map startMap, String firstName, String middleName, String lastName, - // String email, String instId, String employId, String hopkinsId, String abbrRole) - Map coPiRecord1 = changeUserInMap(makeBaseRowMap(), "Albert", "Jones", - "Einstein", "aeinst1@jhu.edu", "aeinst1", userEmployeeId2, "WWTTRE", "C" ); - - resultSet.add(coPiRecord1); + //put in last iteration as existing record - PI is Einstein + Map piRecord2 = makeRowMap (2, 1, "P"); + resultSet.add(piRecord2); passUpdater.updatePass(resultSet, "grant"); sleep(10000); - URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey1); + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); assertNotNull( passGrantUri ); - URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId1 ); + + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[1] ); assertNotNull( passUser1Uri ); - URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId2 ); - assertNotNull( passUser2Uri ); Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); - assertEquals( grantAwardNumber1, passGrant.getAwardNumber() ); + assertEquals( grantAwardNumber[2], passGrant.getAwardNumber() ); assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); - assertEquals( grantIdPrefix + grantLocalKey1, passGrant.getLocalKey() ); - assertEquals( grantProjectName1, passGrant.getProjectName() ); - assertEquals( createJodaDateTime(grantAwardDate1), passGrant.getAwardDate() ); - assertEquals( createJodaDateTime(grantStartDate1), passGrant.getStartDate() ); - assertEquals( createJodaDateTime(grantEndDate1), passGrant.getEndDate() ); - assertEquals( passUser1Uri, passGrant.getPi() ); - assertEquals( 1, passGrant.getCoPis().size() ); - assertEquals( passUser2Uri, passGrant.getCoPis().get(0) ); - - //now do a multi-iteration pull, and check the result - + assertEquals( grantIdPrefix + grantLocalKey[2], passGrant.getLocalKey() ); + assertEquals( grantProjectName[2], passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantAwardDate[2]), passGrant.getAwardDate() ); + assertEquals( createJodaDateTime(grantStartDate[2]), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() ); + assertEquals( passUser1Uri, passGrant.getPi() ); //Einstein + assertEquals( 0, passGrant.getCoPis().size() ); + + //check statistics + assertEquals(1, statistics.getGrantsCreated()); + assertEquals(1, statistics.getUsersCreated()); + assertEquals(1, statistics.getPisAdded()); + assertEquals(0, statistics.getCoPisAdded()); + + //now simulate a complete pull from the Beginning of Time and adjust the stored grant + //we add a new co-pi Jones in the "1" iteration, and change the pi to Einstein in the "2" iteration + //we drop co-pi jones in the last iteration + Map piRecord0 = makeRowMap(0, 0, "P"); + Map coPiRecord0 = makeRowMap(0, 1, "C"); + Map piRecord1 = makeRowMap(1, 0, "P"); + Map coPiRecord1 = makeRowMap(1, 1, "C"); + Map newCoPiRecord1 = makeRowMap(1, 2, "C"); + + //in the initial pull, we will find all of the records (check?) resultSet.clear(); - - //add another CoPi at the second iteration - Map adjustments = new HashMap<>(); - adjustments.put(C_GRANT_AWARD_DATE, grantAwardDate2); - adjustments.put(C_GRANT_START_DATE, grantStartDate2); - adjustments.put(C_GRANT_END_DATE, grantEndDate2); - adjustments.put(C_GRANT_PROJECT_NAME, grantProjectName2); - adjustments.put(C_GRANT_AWARD_NUMBER, grantAwardNumber2); - adjustments.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp2); - - Map piRecord2 = makeAdjustedBaseRowMap( adjustments ); - Map coPiRecord2 = makeAdjustedRowMap( coPiRecord1, adjustments ); - Map newCoPiRecord2 = changeUserInMap( coPiRecord2, "Junie", "Beatrice", - "Jones", "jjones1@jhu.edu", "ajjones1", userEmployeeId3, "THIHKX", "C" ); - - //change PI at the third iteration - adjustments.clear(); - adjustments.put(C_GRANT_AWARD_DATE, grantAwardDate3); - adjustments.put(C_GRANT_START_DATE, grantStartDate3); - adjustments.put(C_GRANT_END_DATE, grantEndDate3); - adjustments.put(C_GRANT_PROJECT_NAME, grantProjectName3); - adjustments.put(C_GRANT_AWARD_NUMBER, grantAwardNumber3); - adjustments.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp3); - - //user 2 is the new PI, drop user 1; user 1 should show up as a co-pi - Map piRecord3 = makeAdjustedBaseRowMap( adjustments ); - Map newPiRecord3 = changeUserInMap(piRecord3, "Albert", "Jones", - "Einstein", "aeinst1@jhu.edu", "aeinst1", userEmployeeId2, - "WWTTRE", "P" ); - Map newCoPiRecord3 = makeAdjustedRowMap(newCoPiRecord2, adjustments); - - //in the initial pull, we will findall of the records (check?) + resultSet.add(piRecord0); + resultSet.add(coPiRecord0); resultSet.add(piRecord1); resultSet.add(coPiRecord1); - resultSet.add(coPiRecord2); - resultSet.add(newCoPiRecord2); - resultSet.add(newPiRecord3); - resultSet.add(newCoPiRecord3); + resultSet.add(newCoPiRecord1); + resultSet.add(piRecord2); passUpdater.updatePass(resultSet, "grant"); sleep(10000); passGrant = passClient.readResource( passGrantUri, Grant.class ); + URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[0] ); + assertNotNull( passUser0Uri ); + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[2] ); + assertNotNull( passUser2Uri ); - URI passUser3Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId3 ); - assertNotNull( passUser3Uri ); - - assertEquals( grantAwardNumber1, passGrant.getAwardNumber() ); + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() );//initial assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); - assertEquals( grantIdPrefix + grantLocalKey1, passGrant.getLocalKey() ); - assertEquals( grantProjectName1, passGrant.getProjectName() ); - assertEquals( createJodaDateTime(grantAwardDate1), passGrant.getAwardDate() ); - assertEquals( createJodaDateTime(grantStartDate1), passGrant.getStartDate() ); - assertEquals( createJodaDateTime(grantEndDate3), passGrant.getEndDate() ); - assertEquals( passUser2Uri, passGrant.getPi() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() );//initial + assertEquals( createJodaDateTime(grantAwardDate[0]), passGrant.getAwardDate() );//initial + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() );//initial + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() );//latest + assertEquals( passUser1Uri, passGrant.getPi() );//Einstein assertEquals( 2, passGrant.getCoPis().size() ); - assertTrue( passGrant.getCoPis().contains(passUser1Uri) ); - assertTrue( passGrant.getCoPis().contains(passUser3Uri) ); + assertTrue( passGrant.getCoPis().contains(passUser0Uri) );//Melon + assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Jones } + + /** + * utility method to produce data as it would look coming from COEUS + * @param iteration the iteration of the (multi-award) grant + * @param user the user supplied in the record + * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") + * @return + */ + private Map makeRowMap( int iteration, int user, String abbrRole) { + Map rowMap = new HashMap<>(); + rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber[iteration]); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey[iteration]); + rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName[iteration]); + rowMap.put(C_GRANT_AWARD_DATE, grantAwardDate[iteration]); + rowMap.put(C_GRANT_START_DATE, grantStartDate[iteration]); + rowMap.put(C_GRANT_END_DATE, grantEndDate[iteration]); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000000"); + rowMap.put(C_DIRECT_FUNDER_NAME, "Enormous State University"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000001"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "J L Gotrocks Foundation"); + + rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); + rowMap.put(C_USER_MIDDLE_NAME, userMiddleName[user]); + rowMap.put(C_USER_LAST_NAME, userLastName[user]); + rowMap.put(C_USER_EMAIL, userEmail[user]); + rowMap.put(C_USER_INSTITUTIONAL_ID, userInstitutionalId[user]); + rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); + rowMap.put(C_USER_HOPKINS_ID, userHopkinsId[user]); + + rowMap.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp[iteration]); + rowMap.put(C_ABBREVIATED_ROLE, abbrRole); + + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); + + return rowMap; + } + } diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java index 5a79d0e..9434e68 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Johns Hopkins University + * Copyright 2020 Johns Hopkins University * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,551 +13,206 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.dataconservancy.pass.grant.integration; import org.dataconservancy.pass.client.PassClient; import org.dataconservancy.pass.client.PassClientFactory; -import org.dataconservancy.pass.grant.data.DateTimeUtil; -import org.dataconservancy.pass.grant.data.PassUpdateStatistics; import org.dataconservancy.pass.grant.data.JhuPassUpdater; -import org.dataconservancy.pass.grant.data.CoeusPassEntityUtil; -import org.dataconservancy.pass.model.Funder; +import org.dataconservancy.pass.grant.data.PassUpdateStatistics; import org.dataconservancy.pass.model.Grant; - import org.dataconservancy.pass.model.Policy; import org.dataconservancy.pass.model.User; -import org.dataconservancy.pass.model.support.Identifier; -import org.junit.Before; -import org.junit.Rule; +import org.junit.Before;; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; -import static java.lang.Thread.sleep; -import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertNotNull; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.ListIterator; import java.util.Map; -/** - * An integration test class for the JhuPassUpdater. - */ -@RunWith(MockitoJUnitRunner.class) -public class JhuPassUpdaterIT { - - private final List> resultSet = new ArrayList<>(); - private static final String employeeidPrefix = "johnshopkins.edu:employeeid:"; - private static final String jhedPrefix = "johnshopkins.edu:jhed:"; - private final CoeusPassEntityUtil passEntityUtil = new CoeusPassEntityUtil(); - private final Map funderPolicyUriMap = new HashMap<>(); - - private String directFunderPolicyUriString1; - private String primaryFunderPolicyUriString1; +import static java.lang.Thread.sleep; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; +import static org.junit.Assert.*; - private final PassClient passClient = PassClientFactory.getPassClient(); +public class JhuPassUpdaterIT { - @Rule - public TemporaryFolder folder= new TemporaryFolder(); + String[] grantAwardNumber = { "B10000000", "B10000001", "B10000002" }; + String[] grantLocalKey = { "10000001", "10000001","10000001" }; //all the same, different from other ITs tho + String[] grantProjectName = {"Stupendous Research Project I", "Stupendous Research Project II", "Stupendous Research Project III" }; + String[] grantAwardDate = { "01/01/1999", "01/01/2001", "01/01/2003" }; + String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; + String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; + String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; + String[] userEmployeeId= { "31000000", "31000001", "31000002"}; + String[] userInstitutionalId = {"arecko1", "sclass1", "jgunn1" }; + String[] userHopkinsId = {"DOMNAR", "NROAD", "ROMAND" }; + String[] userFirstName = {"Amanda", "Skip", "Janie"}; + String[] userMiddleName = {"Bea", "Avery", "Gotta" }; + String[] userLastName = { "Reckondwith", "Class", "Gunn" }; + String[] userEmail = { "arecko1@jhu.edu", "sclass1@jhu.edu", "jgunn1@jhu.edu" }; + + + String primaryFunderPolicyUriString; + String directFunderPolicyUriString; + + String grantIdPrefix = "johnshopkins.edu:grant:"; + //String funderIdPrefix = "johnshopkins.edu:funder:"; + //String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; + String employeeidPrefix = "johnshopkins.edu:employeeid:"; + //String jhedidPrefis = "johnshopkins.edu:jhed:"; + + PassClient passClient = PassClientFactory.getPassClient(); + JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); @Before public void setup() { - - for (int i = 0; i < 10; i++) { - - String prefix = System.getProperty("pass.fedora.baseurl"); - if (!prefix.endsWith("/")) { - prefix = prefix + "/"; - } - - Policy policy = new Policy(); - policy.setTitle("Primary Policy" + i); - policy.setDescription("MOO"); - URI policyURI = passClient.createResource(policy); - String primaryPolicyUriString = policyURI.toString().substring(prefix.length()); - funderPolicyUriMap.put("PrimaryFunderPolicy"+i, policyURI); - - policy = new Policy(); - policy.setTitle("Direct Policy" + i); - policy.setDescription("MOO"); - policyURI = passClient.createResource(policy); - String directPolicyUriString = policyURI.toString().substring(prefix.length()); - funderPolicyUriMap.put("DirectFunderPolicy"+i, policyURI); - - if(i==1) { - directFunderPolicyUriString1 = directPolicyUriString; - primaryFunderPolicyUriString1 = primaryPolicyUriString; - } - - Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + i); - rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + i); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + i); - rowMap.put(C_GRANT_AWARD_DATE, "01/01/2000"); - rowMap.put(C_GRANT_START_DATE, "01/01/2001"); - rowMap.put(C_GRANT_END_DATE, "01/01/2002"); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + i); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + i); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + i); - rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + i); - - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); - rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); - - rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + i + ":00:00.0"); - rowMap.put(C_ABBREVIATED_ROLE, (i % 2 == 0 ? "P" : "C")); - rowMap.put(C_DIRECT_FUNDER_POLICY, directPolicyUriString ); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryPolicyUriString); - - resultSet.add(rowMap); + String prefix = System.getProperty("pass.fedora.baseurl"); + if ( !prefix.endsWith("/") ) { + prefix = prefix + "/"; } + Policy policy = new Policy(); + policy.setTitle("Primary Policy 2"); + policy.setDescription("BAA"); + URI policyURI = passClient.createResource(policy); + primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + policy =new Policy(); + policy.setTitle("Direct Policy 2"); + policy.setDescription("BAA"); + policyURI =passClient.createResource(policy); + directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + } /** - * The behavior of PassUpdate's updatePass() method is to compare the data coming in on the ResultSet with - * the existing data in Pass, and create objects if Pass does not yet have them, and update them if they exist in Pass but - * there are differences in the fields for which COEUS is the authoritative source, or COEUS has a clue about other fields which are null - * on the PASS object. + * we put an initial award for a grant into fedora, then simulate a pull of all subsequent records + * + * We expect to see some fields retained from the initial award, and others updated. The most + * interesting fields are the investigator fields: all CO-PIs ever on the grant should stay on the + * co-pi field throughout iterations. If a PI is changed, they should appear on the CO-PI field * - * @throws InterruptedException - the exception + * @throws InterruptedException */ @Test - public void updateGrantsIT() throws InterruptedException { - - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); - passUpdater.updatePass(resultSet, "grant"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); - - assertEquals(5, statistics.getPisAdded()); - assertEquals(5, statistics.getCoPisAdded()); - assertEquals(20, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - assertEquals(10, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - assertEquals("2018-01-01 09:00:00.0", statistics.getLatestUpdateString()); - assertEquals(10, statistics.getUsersCreated()); - assertEquals(0, statistics.getUsersUpdated()); + public void processGrantIT() throws InterruptedException { + List> resultSet = new ArrayList<>(); - assertEquals(10, passUpdater.getGrantUriMap().size()); + //put in initial iteration as a correct existing record - PI is Reckondwith, Co-pi is Class + Map piRecord0 = makeRowMap(0, 0, "P"); + Map coPiRecord0 = makeRowMap(0, 1, "C"); - for (URI grantUri : passUpdater.getGrantUriMap().keySet()) { - Grant grant = passUpdater.getGrantUriMap().get(grantUri); - Grant passGrant = passUpdater.getPassClient().readResource(grantUri, Grant.class); - assertNull(passEntityUtil.update(grant, passGrant)); //this means grants are "coeus-equal" + resultSet.add(piRecord0); + resultSet.add(coPiRecord0); - } - - sleep(20000); - //try depositing the exact same resultSet. nothing should happen in Pass passUpdater.updatePass(resultSet, "grant"); - - assertEquals(0, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - assertEquals(0, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); - assertEquals(0, statistics.getUsersUpdated()); - - //now let's monkey with a few things; we expect to update the changed objects - //first change all the things that shouldn't matter - Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1 + "MOO"); - rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + 1); - rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOO"); - rowMap.put(C_GRANT_AWARD_DATE, "01/01/1999"); - rowMap.put(C_GRANT_START_DATE, "01/01/1999"); - rowMap.put(C_GRANT_END_DATE, "01/01/2002"); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + 1 + "MOOOOO"); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + 1); - - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + 1); - rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + 1 + "MOOOO"); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + 1); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + 1); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + 1); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + 1); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + 1); - - rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); - rowMap.put(C_ABBREVIATED_ROLE, ("C")); - - rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString1); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString1); - - + sleep(10000); + URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[0] ); + assertNotNull( passUser0Uri ); + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); + assertNotNull( passGrantUri ); + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[1] ); + assertNotNull( passUser1Uri ); + + Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); + + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() ); + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantAwardDate[0]), passGrant.getAwardDate() ); + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate[0]), passGrant.getEndDate() ); + assertEquals( passUser0Uri, passGrant.getPi() ); //Reckondwith + assertEquals( 1, passGrant.getCoPis().size() ); + assertEquals( passUser1Uri, passGrant.getCoPis().get(0)); + + //check statistics + assertEquals(1, statistics.getGrantsCreated()); + assertEquals(2, statistics.getUsersCreated()); + assertEquals(1, statistics.getPisAdded()); + assertEquals(1, statistics.getCoPisAdded()); + + //now simulate an incremental pull since the initial, adjust the stored grant + //we add a new co-pi Jones in the "1" iteration, and change the pi to Einstein in the "2" iteration + //we drop co-pi jones in the last iteration + + Map piRecord1 = makeRowMap(1, 0, "P"); + Map coPiRecord1 = makeRowMap(1, 1, "C"); + Map newCoPiRecord1 = makeRowMap(1, 2, "C"); + Map piRecord2 = makeRowMap (2, 1, "P"); + + //add in everything since the initial pull resultSet.clear(); - resultSet.add(rowMap); + resultSet.add(piRecord1); + resultSet.add(coPiRecord1); + resultSet.add(newCoPiRecord1); + resultSet.add(piRecord2); passUpdater.updatePass(resultSet, "grant"); - assertEquals(0, statistics.getFundersCreated()); - assertEquals(1, statistics.getFundersUpdated()); - assertEquals(0, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); - assertEquals(1, statistics.getUsersUpdated()); - - sleep(20000); - - for (int i = 0; i < 10; i++) { - Grant grant = new Grant(); - grant.setAwardNumber(C_GRANT_AWARD_NUMBER + i); - grant.setAwardStatus(Grant.AwardStatus.ACTIVE); - String grantIdPrefix = "johnshopkins.edu:grant:"; - grant.setLocalKey(grantIdPrefix + C_GRANT_LOCAL_KEY + i); - grant.setProjectName(C_GRANT_PROJECT_NAME + i); - grant.setAwardDate(DateTimeUtil.createJodaDateTime("01/01/2000")); - grant.setStartDate(DateTimeUtil.createJodaDateTime("01/01/2001")); - grant.setEndDate(DateTimeUtil.createJodaDateTime("01/01/2002")); - - URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grant.getLocalKey()); - Grant passGrant = passClient.readResource(passGrantUri, Grant.class); - - assertEquals(grant.getAwardNumber(), passGrant.getAwardNumber()); - assertEquals(grant.getAwardStatus(), passGrant.getAwardStatus()); - assertEquals(grant.getLocalKey(), passGrant.getLocalKey()); - assertEquals(grant.getProjectName(), passGrant.getProjectName()); - assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); - assertEquals(grant.getStartDate(), passGrant.getStartDate()); - assertEquals(grant.getEndDate(), passGrant.getEndDate()); - - //let's check funder stuff - Funder directFunder = new Funder(); - String funderIdPrefix = "johnshopkins.edu:funder:"; - directFunder.setLocalKey(funderIdPrefix + C_DIRECT_FUNDER_LOCAL_KEY + i); - directFunder.setName(C_DIRECT_FUNDER_NAME + i); - directFunder.setPolicy(funderPolicyUriMap.get("DirectFunderPolicy" + i)); - - URI directFunderUri = passClient.findByAttribute(Funder.class, "localKey", directFunder.getLocalKey()); - Funder passDirectFunder = passClient.readResource(directFunderUri, Funder.class); - if (i == 1) { - assertEquals(directFunder.getName() + "MOOOOO", passDirectFunder.getName()); - assertEquals(directFunder.getLocalKey(), passDirectFunder.getLocalKey()); - assertEquals(passDirectFunder.getId(), passGrant.getDirectFunder()); - } else { - assertEquals(directFunder.getName(), passDirectFunder.getName()); - } - - Funder primaryFunder = new Funder(); - primaryFunder.setLocalKey(funderIdPrefix + C_PRIMARY_FUNDER_LOCAL_KEY + i); - primaryFunder.setName(C_PRIMARY_FUNDER_NAME + i); - primaryFunder.setPolicy(funderPolicyUriMap.get("PrimaryFunderPolicy" + i)); - - URI primaryFunderUri = passClient.findByAttribute(Funder.class, "localKey", primaryFunder.getLocalKey()); - Funder passPrimaryFunder = passClient.readResource(primaryFunderUri, Funder.class); - assertEquals(primaryFunder.getName(), passPrimaryFunder.getName()); - assertEquals(passPrimaryFunder.getId(), passGrant.getPrimaryFunder()); - assertEquals(primaryFunder.getLocalKey(), passPrimaryFunder.getLocalKey()); - assertEquals(primaryFunder.getPolicy(), passPrimaryFunder.getPolicy()); - - User user = new User(); - - //institutionalId and localKey were localized by the grant loader - user.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + i); - String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; - user.getLocatorIds().add(hopkinsidPrefix + C_USER_HOPKINS_ID + i); - user.getLocatorIds().add(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + i); - user.setFirstName(C_USER_FIRST_NAME + i); - user.setMiddleName(C_USER_MIDDLE_NAME + i); - user.setLastName(C_USER_LAST_NAME + i); - user.setEmail(C_USER_EMAIL + i); - - URI userUri = null; - ListIterator idIterator = user.getLocatorIds().listIterator(); - - while (userUri == null && idIterator.hasNext()) { - String id = String.valueOf(idIterator.next()); - if (id != null) { - userUri = passClient.findByAttribute(User.class, "locatorIds", id); - } - } - - User passUser = passClient.readResource(userUri, User.class); - assertEquals(user.getFirstName(), passUser.getFirstName()); - if (i == 1) { - assertEquals(user.getMiddleName() + "MOOOO", passUser.getMiddleName()); - } else { - assertEquals(user.getMiddleName(), passUser.getMiddleName()); - } - assertEquals(user.getLastName(), passUser.getLastName()); - assertEquals(user.getEmail(), passUser.getEmail()); - assertTrue(user.getLocatorIds().containsAll(passUser.getLocatorIds())); - assertTrue(passUser.getLocatorIds().containsAll(user.getLocatorIds())); - assertEquals(passUser.getLocatorIds().size(), user.getLocatorIds().size()); - - if (i % 2 == 0) { - assertNotNull(passGrant.getPi()); - assertEquals(0, passGrant.getCoPis().size()); - } else { - assertNull(passGrant.getPi()); - assertEquals(1, passGrant.getCoPis().size()); - } - - } - } - - - @Test - public void updateUsersIT() throws InterruptedException { - - User user10 = new User(); - user10.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + 10); - user10.setFirstName(C_USER_FIRST_NAME + 10); - user10.setMiddleName(C_USER_MIDDLE_NAME + 10); - user10.setLastName(C_USER_LAST_NAME + 10); - - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); - - URI passUserURI = passUpdater.getPassClient().createResource(user10); - - User passUser = passClient.readResource(passUserURI, User.class); - assertNull(passUser.getDisplayName()); - assertEquals(1, passUser.getLocatorIds().size()); - assertNull(passUser.getEmail()); - - sleep(20000); - - List> userResultSet = new ArrayList<>(); - - for (int i = 10; i < 12; i++) { - Map rowMap = new HashMap<>(); - - rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + i); - rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); - rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); - rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); - rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); - userResultSet.add(rowMap); - } - - - passUpdater.updatePass(userResultSet, "user"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); - - //now update from the set of two users - the second one is not in PASS, but is not created - //the first (user10) should be updated, with new fields added - assertEquals(0, statistics.getUsersCreated()); - assertEquals(1, statistics.getUsersUpdated()); - - assertNotNull(passUserURI); - User updatedUser = passUpdater.getPassClient().readResource(passUserURI, User.class); - - assertNotNull(updatedUser.getEmail()); - assertNotNull(updatedUser.getDisplayName()); - assertNotNull(updatedUser.getLocatorIds()); - assertTrue(updatedUser.getLocatorIds().contains(employeeidPrefix + C_USER_EMPLOYEE_ID + 10)); - assertTrue(updatedUser.getLocatorIds().contains(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + 10)); - - assertEquals(C_USER_EMAIL + 10, updatedUser.getEmail()); + sleep(10000); + + passGrant = passClient.readResource( passGrantUri, Grant.class ); + + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[2] ); + assertNotNull( passUser2Uri ); + + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() );//initial + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() );//initial + assertEquals( createJodaDateTime(grantAwardDate[0]), passGrant.getAwardDate() );//initial + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() );//initial + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() );//latest + assertEquals( passUser1Uri, passGrant.getPi() );//Class + assertEquals( 2, passGrant.getCoPis().size() ); + assertTrue( passGrant.getCoPis().contains(passUser0Uri) );//Reckondwith + assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Gunn } /** - * Create some policies, deposit them into Fedora - * Then create a java data object linking funders to them - * this basically tests what happens when pulling in data from a policy properties file first, - * or from a coeus pull second + * utility method to produce data as it would look coming from COEUS + * @param iteration the iteration of the (multi-award) grant + * @param user the user supplied in the record + * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") + * @return */ - @Test - public void updateFundersIT() throws InterruptedException { - Policy policy1 = new Policy(); - policy1. setTitle("Policy One"); - policy1. setDescription("Policy one Description"); - Policy policy2 = new Policy(); - policy2. setTitle("Policy Two"); - policy2. setDescription("Policy Two Description"); - - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); - URI policy1Uri = passClient.createResource(policy1); - URI policy2Uri = passClient.createResource(policy2); - - assertNotNull(passClient.readResource(policy1Uri, Policy.class)); - assertNotNull(passClient.readResource(policy2Uri, Policy.class)); - - Funder funder1 = new Funder(); - String DOMAIN = "johnshopkins.edu"; - String fullLocalKey = new Identifier(DOMAIN, "funder", "22229999").serialize(); - funder1.setLocalKey(fullLocalKey); - funder1.setName("Funder One"); - Funder funder2 = new Funder(); - fullLocalKey = new Identifier(DOMAIN, "funder", "33330000").serialize(); - funder2.setLocalKey(fullLocalKey);//use full localKey - funder2.setName("Funder Two"); - - URI funder1Uri = passClient.createResource(funder1); - URI funder2Uri = passClient.createResource(funder2); - - assertNotNull(passClient.readResource(funder1Uri, Funder.class)); - assertNotNull(passClient.readResource(funder2Uri, Funder.class)); - - - String policyString1 = policy1Uri.getPath().substring("/fcrepo/rest/".length()); - assertTrue(policyString1.startsWith("policies")); - String policyString2 = policy2Uri.getPath().substring("/fcrepo/rest/".length()); - assertTrue(policyString2.startsWith("policies")); - - List> funderResultSet = new ArrayList<>(); - + private Map makeRowMap( int iteration, int user, String abbrRole) { Map rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "22229999"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString1); - funderResultSet.add(rowMap); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "33330000"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); - funderResultSet.add(rowMap); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "88888888"); // this one does not exist in pass - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); - funderResultSet.add(rowMap); - - - sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch - - passUpdater.updatePass(funderResultSet, "funder"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); - - assertNotNull(passClient.readResource(funder1Uri, Funder.class)); - assertNotNull(passClient.readResource(funder1Uri, Funder.class).getPolicy()); - assertNotNull(passClient.readResource(funder2Uri, Funder.class)); - assertNotNull(passClient.readResource(funder2Uri, Funder.class).getPolicy()); - assertEquals(policy1Uri, passClient.readResource(funder1Uri, Funder.class).getPolicy()); - assertEquals(policy2Uri, passClient.readResource(funder2Uri, Funder.class).getPolicy()); - - assertEquals(0, statistics.getFundersCreated()); - assertEquals(2, statistics.getFundersUpdated()); - - //coeus pulls will have the funder names, we should be able to add one we don't know about - - funderResultSet = new ArrayList<>(); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "22229999"); - rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 1"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); //let's change policies for this one - funderResultSet.add(rowMap); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "33330000"); - rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 2"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); - funderResultSet.add(rowMap); - - rowMap = new HashMap<>(); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "88888888"); // this one does not exist in pass - rowMap.put(C_PRIMARY_FUNDER_NAME, "Funder Name 3"); - rowMap.put(C_PRIMARY_FUNDER_POLICY, policyString2); - funderResultSet.add(rowMap); - - sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch - - passUpdater.updatePass(funderResultSet, "funder"); - statistics = passUpdater.getStatistics(); - - assertNotNull(passClient.readResource(funder1Uri, Funder.class)); - assertNotNull(passClient.readResource(funder1Uri, Funder.class).getPolicy()); - assertNotNull(passClient.readResource(funder2Uri, Funder.class)); - assertNotNull(passClient.readResource(funder2Uri, Funder.class).getPolicy()); - assertEquals(policy2Uri, passClient.readResource(funder1Uri, Funder.class).getPolicy()); - assertEquals(policy2Uri, passClient.readResource(funder2Uri, Funder.class).getPolicy()); - - assertEquals(1, statistics.getFundersCreated()); - assertEquals(2, statistics.getFundersUpdated()); - - - //DO AGAIN!! DO AGAIN!! - - sleep(20000); //allow indexer to index stuff - java client has to use elasticsearch - - passUpdater.updatePass(funderResultSet, "funder"); - statistics = passUpdater.getStatistics(); - - assertEquals(0, statistics.getFundersCreated()); - assertEquals(0, statistics.getFundersUpdated()); - - } - - @Test - public void testSerializeAndDeserialize() throws IOException { - File serialized= folder.newFile("serializedData"); - - try (FileOutputStream fos = new FileOutputStream(serialized); - ObjectOutputStream out = new ObjectOutputStream(fos) - ){ - out.writeObject(resultSet); - } catch (IOException e) { - e.printStackTrace(); - } - - List> input = null; - try (FileInputStream fis = new FileInputStream(serialized); - ObjectInputStream in = new ObjectInputStream(fis) - ){ - input = (List>)in.readObject(); - } catch (IOException | ClassNotFoundException ex) { - ex.printStackTrace(); - } - - assertEquals(resultSet, input); + rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber[iteration]); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey[iteration]); + rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName[iteration]); + rowMap.put(C_GRANT_AWARD_DATE, grantAwardDate[iteration]); + rowMap.put(C_GRANT_START_DATE, grantStartDate[iteration]); + rowMap.put(C_GRANT_END_DATE, grantEndDate[iteration]); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000000"); + rowMap.put(C_DIRECT_FUNDER_NAME, "Enormous State University"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000001"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "J L Gotrocks Foundation"); + + rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); + rowMap.put(C_USER_MIDDLE_NAME, userMiddleName[user]); + rowMap.put(C_USER_LAST_NAME, userLastName[user]); + rowMap.put(C_USER_EMAIL, userEmail[user]); + rowMap.put(C_USER_INSTITUTIONAL_ID, userInstitutionalId[user]); + rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); + rowMap.put(C_USER_HOPKINS_ID, userHopkinsId[user]); + + rowMap.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp[iteration]); + rowMap.put(C_ABBREVIATED_ROLE, abbrRole); + + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); + + return rowMap; } - @Test - public void testCreateAndUpdateGrantsWithSeveralDates() { - String firstAwardDate="01/01/2000"; - String secondAwardDate="06/01/2002"; - String thirdAwardDate="08/15/2004"; - - String[] awardDates = { firstAwardDate, secondAwardDate, thirdAwardDate }; - - String firstStartDate="01/01/2001"; - String secondStartDate="01/01/2003"; - String thirdStartDate="01/01/2005"; - - String[] startDates= { firstStartDate, secondStartDate, thirdStartDate}; - - String firstEndDate="12/31/2001"; - String secondEndDate="12/31/2004"; - String thirdEndDate="12/31/2007"; - - String[] endDates = { firstEndDate, secondEndDate, thirdEndDate }; - - String firstAwardNumber="777979"; - String secondAwardNumber="77111111"; - String thirdAwardNumber="109111111"; - - String[] awardNumbers= {firstAwardNumber, secondAwardNumber, thirdAwardNumber }; - - String grantNumber = "M00-8675309"; - } -} \ No newline at end of file +} diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/PassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/PassUpdaterIT.java deleted file mode 100644 index 953c75d..0000000 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/PassUpdaterIT.java +++ /dev/null @@ -1,150 +0,0 @@ -package org.dataconservancy.pass.grant.integration; - -import org.dataconservancy.pass.client.PassClient; -import org.dataconservancy.pass.client.PassClientFactory; -import org.dataconservancy.pass.model.Policy; -import org.junit.Before; - -import java.net.URI; -import java.util.HashMap; -import java.util.Map; -import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; - -/** - * This class is a base class mainly for setting up test data to be processed by descendants of the DefaultPassUpdater - */ -public class PassUpdaterIT { - - String grantAwardNumber1 = "A10000000"; - String grantAwardNumber2 = "A10000001"; - String grantAwardNumber3 = "A10000002"; - String grantLocalKey1 = "10000000"; - String grantLocalKey2 = "100000001"; - String grantLocalKey3 = "100000002"; - String grantProjectName1 = "Awesome Research Project I"; - String grantProjectName2 = "Awesome Research Project II"; - String grantProjectName3 = "Awesome Research Project III"; - String grantAwardDate1 = "01/01/1999"; - String grantAwardDate2 = "01/01/2001"; - String grantAwardDate3 = "01/01/2003"; - String grantStartDate1 = "07/01/2000"; - String grantStartDate2 = "07/01/2002"; - String grantStartDate3 = "07/01/2004"; - String grantEndDate1 = "06/30/2002"; - String grantEndDate2 = "06/30/2004"; - String grantEndDate3 = "06/30/2006"; - - String grantUpdateTimestamp1 = "2006-03-11 00:00:00.0"; - String grantUpdateTimestamp2 = "2010-04-05 00:00:00.0"; - String grantUpdateTimestamp3 = "2015-11-11 00:00:00.0"; - - String userEmployeeId1 = "30000000"; - String userEmployeeId2 = "30000001"; - String userEmployeeId3 = "30000002"; - - String primaryFunderPolicyUriString; - String directFunderPolicyUriString; - - String grantIdPrefix = "johnshopkins.edu:grant:"; - String funderIdPrefix = "johnshopkins.edu:funder:"; - String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; - String employeeidPrefix = "johnshopkins.edu:employeeid:"; - String jhedidPrefis = "johnshopkins.edu:jhed:"; - - PassClient passClient = PassClientFactory.getPassClient(); - - @Before - public void setup() { - String prefix = System.getProperty("pass.fedora.baseurl"); - if ( !prefix.endsWith("/") ) { - prefix = prefix + "/"; - } - - Policy policy = new Policy(); - policy.setTitle("Primary Policy"); - policy.setDescription("MOO"); - URI policyURI = passClient.createResource(policy); - primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); - // funderPolicyUriMap.put("PrimaryFunderPolicy"+i,policyURI); - - policy =new Policy(); - policy.setTitle("Direct Policy"); - policy.setDescription("MOO"); - policyURI =passClient.createResource(policy); - directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); - // funderPolicyUriMap.put("DirectFunderPolicy"+i,policyURI); - - } - - Map makeBaseRowMap() { - - Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber1); - rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey1); - rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName1); - rowMap.put(C_GRANT_AWARD_DATE, grantAwardDate1); - rowMap.put(C_GRANT_START_DATE, grantStartDate1); - rowMap.put(C_GRANT_END_DATE, grantEndDate1); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000000"); - rowMap.put(C_DIRECT_FUNDER_NAME, "Enormous State University"); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000001"); - rowMap.put(C_PRIMARY_FUNDER_NAME, "J L Gotrocks Foundation"); - - rowMap.put(C_USER_FIRST_NAME, "Andrew"); - rowMap.put(C_USER_MIDDLE_NAME, "Carnegie"); - rowMap.put(C_USER_LAST_NAME, "Melon"); - rowMap.put(C_USER_EMAIL, "amelon1@jhu.edu"); - rowMap.put(C_USER_INSTITUTIONAL_ID, "amelon1"); - rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId1); - rowMap.put(C_USER_HOPKINS_ID, "OEJSNR"); - - rowMap.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp1); - rowMap.put(C_ABBREVIATED_ROLE, ("P")); - - rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); - - return rowMap; - } - - Map makeAdjustedBaseRowMap( Map replacements) { - Map adjustedMap = new HashMap<>(); - adjustedMap.putAll(makeBaseRowMap()); - for (String key : replacements.keySet()) { - adjustedMap.replace(key, replacements.get(key)); - } - return adjustedMap; - } - - Map makeAdjustedRowMap( Map startMap, Map replacements) { - Map adjustedMap = new HashMap<>(); - adjustedMap.putAll( startMap ); - for (String key : replacements.keySet()) { - adjustedMap.replace(key, replacements.get(key)); - } - return adjustedMap; - } - - - Map changeUserInMap(Map startMap, String firstName, String middleName, String lastName, - String email, String instId, String employId, - String hopkinsId, String abbrRole) { - Map adjustedMap = new HashMap<>(); - adjustedMap.putAll( startMap ); - adjustedMap.replace(C_USER_FIRST_NAME, firstName); - adjustedMap.replace(C_USER_MIDDLE_NAME, middleName); - adjustedMap.replace(C_USER_LAST_NAME, lastName); - adjustedMap.replace(C_USER_EMAIL, email); - adjustedMap.replace(C_USER_INSTITUTIONAL_ID, instId); - adjustedMap.replace(C_USER_EMPLOYEE_ID, employId); - adjustedMap.replace(C_USER_HOPKINS_ID, hopkinsId); - adjustedMap.replace(C_ABBREVIATED_ROLE, abbrRole); - - return adjustedMap; - } - - - -} \ No newline at end of file From 09edc95bdb72e06815fcc13d4ef77bcfc612972c Mon Sep 17 00:00:00 2001 From: jrmartino Date: Thu, 7 May 2020 17:17:38 -0400 Subject: [PATCH 06/14] add base loader implementation to breserve basic v 1.3 operation minor code cleanup in EntityUtil classes --- .../pass/grant/data/BasicPassEntityUtil.java | 194 +++++++ .../pass/grant/data/BasicPassUpdater.java | 528 ++++++++++++++++++ .../pass/grant/data/DefaultPassUpdater.java | 3 +- .../data/HarvardPilotPassEntityUtil.java | 64 ++- .../grant/data/HarvardPilotPassUpdater.java | 2 +- .../grant/integration/BasicPassUpdaterIT.java | 119 ++-- .../HarvardPilotPassUpdaterIT.java | 18 +- .../integration/JhuPassInitUpdaterIT.java | 390 ++++++------- 8 files changed, 1006 insertions(+), 312 deletions(-) create mode 100644 pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java create mode 100644 pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java new file mode 100644 index 0000000..43a3fc4 --- /dev/null +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java @@ -0,0 +1,194 @@ +/* + * Copyright 2019 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dataconservancy.pass.grant.data; + +import org.dataconservancy.pass.model.Funder; +import org.dataconservancy.pass.model.Grant; +import org.dataconservancy.pass.model.User; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * A utility class for handling Grants, Users or Funders. One function performed is comparison of two instances of + * these PASS entity classes. These comparisons are reduced to only those fields which are updatable by + * data from the Harvard data, so that two objects are considered "Harvard equal" iff they agree on these fields. + * + * Another function performed by this utility class is to construct an updated version of an instance of one of these classes + * by merging a (possibly) existing Pass object with new information obtained from a Harvard data pull. + * + * @author jrm@jhu.edu + */ + +public class BasicPassEntityUtil implements PassEntityUtil{ + + + /** + * This method takes a Harvard Funder, calculates whether it needs to be updated, and if so, returns the updated object + * to be be ingested into the repository. if not, returns null. + * @param stored the Funder as it is stored in the PASS backend + * @param system the version of the Funder from the Harvard pull + * @return the updated Funder - null if the Funder does not need to be updated + */ + public Funder update(Funder system, Funder stored) { + if (funderNeedsUpdate(system, stored)) { + return updateFunder(system, stored); + } + return null; + } + + /** + * This method takes a Harvard User, calculates whether it needs to be updated, and if so, returns the updated object + * to be be ingested into the repository. if not, returns null. + * @param stored the User as it is stored in the PASS backend + * @param system the version of the user from the Harvard pull + * @return the updated User - null if the User does not need to be updated + */ + public User update(User system, User stored) { + if (userNeedsUpdate(system, stored)) { + return updateUser(system, stored); + } + return null; + } + + /** + * This method takes a Harvard Grant, calculates whether it needs to be updated, and if so, returns the updated object + * to be be ingested into the repository. if not, returns null. + * @param stored the Grant as it is stored in the PASS backend + * @param system the version of the Grant from the Harvard pull + * @return the updated object - null if the Grant does not need to be updated + */ + public Grant update(Grant system, Grant stored) { + if (grantNeedsUpdate(system, stored)) { + return updateGrant(system, stored); + } + return null; + } + + /** + * Compare two Funder objects + * + * @param system the version of the Funder as seen in the Harvard system pull + * @param stored the version of the Funder as read from Pass + * @return a boolean which asserts whether the two supplied Funders are "Harvard equal" + */ + private boolean funderNeedsUpdate(Funder system, Funder stored) { + + //this adjustment handles the case where we take data from policy.properties file, which has no name info + if (system.getName() != null && !system.getName().equals(stored.getName())) return true; + if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return true; + return false; + } + + /** + * Update a Pass Funder object with new information from Harvard + * + * @param system the version of the Funder as seen in the Harvard system pull + * @param stored the version of the Funder as read from Pass + * @return the Funder object which represents the Pass object, with any new information from Harvard merged in + */ + private Funder updateFunder (Funder system, Funder stored) { + stored.setLocalKey(system.getLocalKey()); + stored.setName(system.getName()); + stored.setPolicy(system.getPolicy()); + return stored; + } + + /** + * Compare two User objects. We only care about those fields for which is the authoritative source + * After recent changes. this method would be more accurately named "storedUserDoesNotNeedToBeUpdated" + * + * @param system the version of the User as seen in the Harvard system pull + * @param stored the version of the User as read from Pass + * @return a boolean which asserts whether the two supplied Users are "Harvard equal" + */ + private boolean userNeedsUpdate(User system, User stored) { + if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return true; + if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return true; + if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return true; + if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return true; + if (system.getEmail() != null && stored.getEmail() == null) return true; + if (system.getDisplayName() != null && stored.getDisplayName() == null) return true; + return false; + } + + /** + * Update a Pass User object with new information from the system. We check only those fields for which Harvard is + * authoritative. Other fields will be managed by other providers (Shibboleth for example). The exceptions are + * the localKey, which this application and Shibboleth both rely on; and email, which this application only populates + * if Shib hasn't done so already. + * + * @param system the version of the User as seen in the Harvard system pull + * @param stored the version of the User as read from Pass + * @return the User object which represents the Pass object, with any new information from Harvard merged in + */ + private User updateUser (User system, User stored) { + stored.setFirstName(system.getFirstName()); + stored.setMiddleName(system.getMiddleName()); + stored.setLastName(system.getLastName()); + stored.setEmail(system.getEmail()); + Set idSet = new HashSet<>(); + idSet.addAll(stored.getLocatorIds()); + idSet.addAll(system.getLocatorIds()); + stored.setLocatorIds(idSet.stream().collect(Collectors.toList())); + stored.setDisplayName(system.getDisplayName()); + return stored; + } + + /** + * Compare two Grant objects. Note that the Lists of Co-Pis are compared as Sets + * @param system the version of the Grant as seen in the system pull + * @param stored the version of the Grant as read from Pass + * @return a boolean which asserts whether the two supplied Grants are "Harvard equal" + */ + private boolean grantNeedsUpdate(Grant system, Grant stored) { + if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return true; + if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return true; + if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return true; + if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return true; + if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; + if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; + if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return true; + if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return true; + return false; + } + + /** + * Update a Pass Grant object with new information from Harvard + * + * @param system the version of the Grant as seen in the system pull + * @param stored the version of the Grant as read from Pass + * @return the Grant object which represents the Pass object, with any new information from Harvard merged in + */ + private Grant updateGrant(Grant system, Grant stored) { + stored.setAwardNumber(system.getAwardNumber()); + stored.setLocalKey(system.getLocalKey()); + stored.setProjectName(system.getProjectName()); + stored.setPrimaryFunder(system.getPrimaryFunder()); + stored.setDirectFunder(system.getDirectFunder()); + stored.setPi(system.getPi()); + stored.setCoPis(system.getCoPis()); + stored.setStartDate(system.getStartDate()); + stored.setEndDate(system.getEndDate()); + return stored; + } + +} diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java new file mode 100644 index 0000000..55515b7 --- /dev/null +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java @@ -0,0 +1,528 @@ +/* + * Copyright 2018 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.dataconservancy.pass.grant.data; + + import org.dataconservancy.pass.client.PassClient; + import org.dataconservancy.pass.client.PassClientFactory; + import org.dataconservancy.pass.model.Funder; + import org.dataconservancy.pass.model.Grant; + import org.dataconservancy.pass.model.User; + import org.dataconservancy.pass.model.support.Identifier; + import org.joda.time.DateTime; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + + import java.net.URI; + import java.util.ArrayList; + import java.util.Collection; + import java.util.HashMap; + import java.util.ListIterator; + import java.util.Map; + + import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; + import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; + +/** + * This class is responsible for taking the Set of Maps derived from the ResultSet from the database query and + * constructing a corresponding Collection of Grant or User objects, which it then sends to PASS to update. + * + * This represents the state of things for the default updater of version 1.3 + * + * @author jrm@jhu.edu + */ + +public class BasicPassUpdater implements PassUpdater{ + + private String DOMAIN = "default.domain"; + + private static Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); + private String latestUpdateString = ""; + + private PassClient passClient; + private PassUpdateStatistics statistics = new PassUpdateStatistics(); + private PassEntityUtil passEntityUtil; + + //used in test classes + private Map grantUriMap = new HashMap<>(); + + //used in unit test + //some entities may be referenced many times during an update, but just need to be updated the first time + //they are encountered. these include Users and Funders. we save the overhead of redundant updates + //of these by looking them up here; if they are on the Map, they have already been processed + // + private Map funderMap = new HashMap<>(); + private Map userMap = new HashMap<>(); + + private String mode; + + public BasicPassUpdater(PassEntityUtil passEntityUtil) + { + this.passEntityUtil = passEntityUtil; + this.passClient = PassClientFactory.getPassClient(); + } + + //used in unit testing for injecting a mock client + public BasicPassUpdater(PassEntityUtil passEntityUtil, PassClient passClient) { + this.passEntityUtil = passEntityUtil; + this.passClient = passClient; + } + + public void updatePass(Collection> results, String mode) { + this.mode = mode; + userMap.clear(); + funderMap.clear(); + statistics.reset(); + statistics.setType(mode); + switch (mode) { + case "grant": + updateGrants(results); + break; + case "user": + updateUsers(results); + break; + case "funder": + updateFunders(results); + break; + } + } + + /** + * Build a Collection of Grants from a ResultSet, then update the grants in Pass + * Because we need to make sure we catch any updates to fields referenced by URIs, we construct + * these and update these as well + */ + private void updateGrants(Collection> results) { + + //a grant will have several rows in the ResultSet if there are co-pis. so we put the grant on this + //Map and add to it as additional rows add information. + Map grantMap = new HashMap<>(); + + LOG.info("Processing result set with " + results.size() + " rows"); + boolean modeChecked = false; + + for(Map rowMap : results){ + + String grantLocalKey; + + if (!modeChecked) { + if (!rowMap.containsKey(C_GRANT_LOCAL_KEY)) {//we always have this for grants + throw new RuntimeException("Mode of grant was supplied, but data does not seem to match."); + } else { + modeChecked = true; + } + } + + grantLocalKey = rowMap.get(C_GRANT_LOCAL_KEY); + + String directFunderLocalKey = rowMap.get(C_DIRECT_FUNDER_LOCAL_KEY); + String primaryFunderLocalKey = rowMap.get(C_PRIMARY_FUNDER_LOCAL_KEY); + primaryFunderLocalKey = (primaryFunderLocalKey == null? directFunderLocalKey: primaryFunderLocalKey); + Grant grant; + LOG.debug("Processing grant with localKey " + grantLocalKey); + //if this is the first record for this Grant, it will not be on the Map + //we process all data which is common to every record for this grant + //i.e., everything except the investigator(s) + if(!grantMap.containsKey(grantLocalKey)) { + grant = new Grant(); + grant.setAwardNumber(rowMap.get(C_GRANT_AWARD_NUMBER)); + + String status = rowMap.getOrDefault(C_GRANT_AWARD_STATUS, null); + + if (status != null) { + switch (status) { + case "Active": + grant.setAwardStatus(Grant.AwardStatus.ACTIVE); + break; + case "Pre-Award": + grant.setAwardStatus(Grant.AwardStatus.PRE_AWARD); + break; + case "Terminated": + grant.setAwardStatus(Grant.AwardStatus.TERMINATED); + } + } else { + grant.setAwardStatus(null); + } + + grant.setLocalKey(grantLocalKey); + grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); + grant.setAwardDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_AWARD_DATE, null))); + grant.setStartDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_START_DATE, null))); + grant.setEndDate(createJodaDateTime(rowMap.getOrDefault(C_GRANT_END_DATE, null))); + + //process direct funder, and primary funder if we have one + //update funder(s) in Pass as needed + if (funderMap.containsKey(directFunderLocalKey)) { + grant.setDirectFunder(funderMap.get(directFunderLocalKey)); + } else { + Funder updatedFunder = buildDirectFunder(rowMap); + URI passFunderURI = updateFunderInPass(updatedFunder); + funderMap.put(directFunderLocalKey, passFunderURI); + grant.setDirectFunder(passFunderURI); + } + + if(funderMap.containsKey(primaryFunderLocalKey)) { + grant.setPrimaryFunder(funderMap.get(primaryFunderLocalKey)); + } else { + Funder updatedFunder = buildPrimaryFunder(rowMap); + URI passFunderURI = updateFunderInPass(updatedFunder); + funderMap.put(primaryFunderLocalKey, passFunderURI); + grant.setPrimaryFunder(passFunderURI); + } + + grant.setCoPis(new ArrayList<>());//we will build this from scratch in either case + grantMap.put(grantLocalKey, grant);//save the state of this Grant + } + + //now we process the User (investigator) + grant = grantMap.get(grantLocalKey); + String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); + String abbreviatedRole = rowMap.get(C_ABBREVIATED_ROLE); + + if(abbreviatedRole.equals("C") || abbreviatedRole.equals("K") || grant.getPi() == null) { + if (!userMap.containsKey(employeeId)) { + User updatedUser = buildUser(rowMap); + URI passUserURI = updateUserInPass(updatedUser); + userMap.put(employeeId, passUserURI); + } + + //now our User URI is on the map - let's process: + if (abbreviatedRole.equals("P")) { + if (grant.getPi() == null) { + grant.setPi(userMap.get(employeeId)); + statistics.addPi(); + } else {// handle data with duplicate PIs + grant.getCoPis().add(userMap.get(employeeId)); + statistics.addCoPi(); + } + } else if ((abbreviatedRole.equals("C") || abbreviatedRole.equals("K")) && !grant.getCoPis().contains(userMap.get(employeeId))) { + grant.getCoPis().add(userMap.get(employeeId)); + statistics.addCoPi(); + } + } + //we are done with this record, let's save the state of this Grant + grantMap.put(grantLocalKey, grant); + //see if this is the latest grant updated + if (rowMap.containsKey(C_UPDATE_TIMESTAMP)) { + String grantUpdateString = rowMap.get(C_UPDATE_TIMESTAMP); + latestUpdateString = latestUpdateString.length() == 0 ? grantUpdateString : returnLaterUpdate(grantUpdateString, latestUpdateString); + } + } + + //now put updated grant objects in pass + for(Grant grant : grantMap.values()){ + grantUriMap.put(updateGrantInPass(grant), grant); + } + + //success - we capture some information to report + if (grantMap.size() > 0) { + statistics.setLatestUpdateString(latestUpdateString); + statistics.setReport(results.size(), grantMap.size()); + } else { + System.out.println("No records were processed in this update"); + } + } + + private void updateUsers(Collection> results) { + + boolean modeChecked = false; + + for(Map rowMap : results) { + + if (!modeChecked) { + if (!rowMap.containsKey(C_USER_EMPLOYEE_ID)) {//we always have this for users + throw new RuntimeException("Mode of user was supplied, but data does not seem to match."); + } else { + modeChecked = true; + } + } + + LOG.info("Processing result set with " + results.size() + " rows"); + User updatedUser = buildUser(rowMap); + updateUserInPass(updatedUser); + if (rowMap.containsKey(C_UPDATE_TIMESTAMP)) { + String userUpdateString = rowMap.get(C_UPDATE_TIMESTAMP); + latestUpdateString = latestUpdateString.length() == 0 ? userUpdateString : returnLaterUpdate(userUpdateString, latestUpdateString); + } + } + + if (results.size() > 0) { + statistics.setLatestUpdateString(latestUpdateString); + statistics.setReport(results.size(), results.size()); + } else { + System.out.println("No records were processed in this update"); + } + + } + + /** + * This method is called for the "funder" mode - the column names will have the values for primary funders + * @param results the data row map containing funder information + */ + private void updateFunders(Collection> results) { + + boolean modeChecked = false; + LOG.info("Processing result set with " + results.size() + " rows"); + for (Map rowMap : results) { + + if (!modeChecked) { + if (!rowMap.containsKey(C_PRIMARY_FUNDER_LOCAL_KEY) && !rowMap.containsKey(C_PRIMARY_FUNDER_NAME)) { + throw new RuntimeException("Mode of funder was supplied, but data does not seem to match."); + } else { + modeChecked = true; + } + } + + Funder updatedFunder = buildPrimaryFunder(rowMap); + updateFunderInPass(updatedFunder); + + } + statistics.setReport(results.size(), results.size()); + } + + User buildUser(Map rowMap) { + User user = new User(); + user.setFirstName(rowMap.get(C_USER_FIRST_NAME)); + user.setMiddleName(rowMap.getOrDefault(C_USER_MIDDLE_NAME, null)); + user.setLastName(rowMap.get(C_USER_LAST_NAME)); + user.setDisplayName(rowMap.get(C_USER_FIRST_NAME) + " " + rowMap.get(C_USER_LAST_NAME)); + user.setEmail(rowMap.get(C_USER_EMAIL)); + String employeeId = rowMap.get(C_USER_EMPLOYEE_ID); + //Build the List of locatorIds - put the most reliable ids first + if (employeeId != null) { + String EMPLOYEE_ID_TYPE = "employeeid"; + user.getLocatorIds().add(new Identifier(DOMAIN, EMPLOYEE_ID_TYPE, employeeId).serialize()); + } + user.getRoles().add(User.Role.SUBMITTER); + LOG.debug("Built user with employee ID " + employeeId); + return user; + } + + /** + * this method gets called on a grant mode process if the primary funder is different from direct, and also + * any time the updater is called in funder mode + * @param rowMap the funder data map + * @return the funder + */ + Funder buildPrimaryFunder(Map rowMap) { + Funder funder = new Funder(); + funder.setName(rowMap.getOrDefault(C_PRIMARY_FUNDER_NAME, null)); + funder.setLocalKey(rowMap.get(C_PRIMARY_FUNDER_LOCAL_KEY)); + String policy = rowMap.get(C_PRIMARY_FUNDER_POLICY); + if (policy != null && policy.length()>0) { + String fedoraBaseUrl = System.getProperty("pass.fedora.baseurl"); + fedoraBaseUrl = fedoraBaseUrl.endsWith("/") ? fedoraBaseUrl : fedoraBaseUrl + "/"; + funder.setPolicy(URI.create(fedoraBaseUrl + policy)); + LOG.info("Processing Funder with localKey " + funder.getLocalKey() + + " and Policy " + policy); + } + LOG.debug("Built Funder with localKey " + funder.getLocalKey()); + + return funder; + } + + private Funder buildDirectFunder(Map rowMap) { + Funder funder = new Funder(); + if (rowMap.containsKey(C_DIRECT_FUNDER_NAME)) { + funder.setName(rowMap.get(C_DIRECT_FUNDER_NAME)); + } + funder.setLocalKey(rowMap.get(C_DIRECT_FUNDER_LOCAL_KEY)); + String policy = rowMap.get(C_DIRECT_FUNDER_POLICY); + if (policy != null ) { + String fedoraBaseUrl = System.getProperty("pass.fedora.baseurl"); + fedoraBaseUrl = fedoraBaseUrl.endsWith("/") ? fedoraBaseUrl : fedoraBaseUrl + "/"; + funder.setPolicy(URI.create(fedoraBaseUrl + policy)); + LOG.info("Processing Funder with localKey " + funder.getLocalKey() + + " and Policy " + policy); + } + LOG.debug("Built Funder with localKey " + funder.getLocalKey()); + + return funder; + } + + + /** + * Take a new Funder object populated as fully as possible from the COEUS pull, and use this + * new information to update an object for the same Funder in Pass (if it exists) + * + * @param systemFunder the new Funder object populated from COEUS + * @return the URI for the resource representing the updated Funder in Pass + */ + private URI updateFunderInPass(Funder systemFunder) { + String baseLocalKey = systemFunder.getLocalKey(); + String FUNDER_ID_TYPE = "funder"; + String fullLocalKey = new Identifier(DOMAIN, FUNDER_ID_TYPE, baseLocalKey).serialize(); + systemFunder.setLocalKey(fullLocalKey); + + URI passFunderURI = passClient.findByAttribute(Funder.class, "localKey", fullLocalKey); + if (passFunderURI != null ) { + Funder storedFunder = passClient.readResource(passFunderURI, Funder.class); + if (storedFunder == null) { + throw new RuntimeException("Could not read Funder object with URI " + passFunderURI); + } + Funder updatedFunder; + if ((updatedFunder = passEntityUtil.update(systemFunder, storedFunder)) != null) {//need to update + passClient.updateResource(updatedFunder); + statistics.addFundersUpdated(); + }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything + //this can happen if the Grant was updated in COEUS only with information we don't consume here + } else {//don't have a stored Funder for this URI - this one is new to Pass + if (systemFunder.getName() != null) {//only add if we have a name + passFunderURI = passClient.createResource(systemFunder); + statistics.addFundersCreated(); + } + } + return passFunderURI; + } + + /** + * Take a new User object populated as fully as possible from the COEUS pull, and use this + * new information to update an object for the same User in Pass (if it exists) + * + * @param systemUser the new User object populated from COEUS + * @return the URI for the resource representing the updated User in Pass + */ + private URI updateUserInPass(User systemUser) { + //we first check to see if the user is known by the Hopkins ID. If not, we check the employee ID. + //last attempt is the JHED ID. this order is specified by the order of the List as constructed on updatedUser + URI passUserUri = null; + ListIterator idIterator = systemUser.getLocatorIds().listIterator(); + + while (passUserUri == null && idIterator.hasNext()) { + String id = String.valueOf(idIterator.next()); + if (id != null) { + passUserUri = passClient.findByAttribute(User.class, "locatorIds", id); + } + } + + if (passUserUri != null ) { + User storedUser = passClient.readResource(passUserUri, User.class); + if (storedUser == null) { + throw new RuntimeException("Could not read User object with URI " + passUserUri); + } + User updatedUser; + if ((updatedUser = passEntityUtil.update(systemUser, storedUser)) != null){//need to update + //post COEUS processing goes here + if(!storedUser.getRoles().contains(User.Role.SUBMITTER)) { + storedUser.getRoles().add(User.Role.SUBMITTER); + } + passClient.updateResource(updatedUser); + statistics.addUsersUpdated(); + }//if the Pass version is COEUS-equal to our version from the update, and there are no null fields we care about, + //we don't have to do anything. this can happen if the User was updated in COEUS only with information we don't consume here + } else if (! mode.equals("user")) {//don't have a stored User for this URI - this one is new to Pass + //but don't update if we are in user mode - just update existing users + passUserUri = passClient.createResource(systemUser); + statistics.addUsersCreated(); + } + return passUserUri; + } + + /** + * Take a new Grant object populated as fully as possible from the COEUS pull, and use this + * new information to update an object for the same Grant in Pass (if it exists) + * + * @param systemGrant the new Grant object populated from COEUS + * @return the PASS identifier for the Grant object + */ + private URI updateGrantInPass(Grant systemGrant) { + String baseLocalKey = systemGrant.getLocalKey(); + String GRANT_ID_TYPE = "grant"; + String fullLocalKey = new Identifier(DOMAIN, GRANT_ID_TYPE, baseLocalKey).serialize(); + systemGrant.setLocalKey(fullLocalKey); + + LOG.debug("Looking for grant with localKey " + fullLocalKey); + URI passGrantURI = passClient.findByAttribute(Grant.class, "localKey", fullLocalKey); + if (passGrantURI != null ) { + LOG.debug("Found grant with localKey " + fullLocalKey); + Grant storedGrant = passClient.readResource(passGrantURI, Grant.class); + if (storedGrant == null) { + throw new RuntimeException("Could not read Funder object with URI " + passGrantURI); + } + Grant updatedGrant; + if ( (updatedGrant = passEntityUtil.update(systemGrant, storedGrant)) != null) {//need to update + LOG.debug("Updating grant with localKey " + storedGrant.getLocalKey() + " to localKey " + systemGrant.getLocalKey()); + passClient.updateResource(updatedGrant); + statistics.addGrantsUpdated(); + LOG.debug("Updating grant with award number " + systemGrant.getLocalKey()); + }//if the Pass version is COEUS-equal to our version from the update, we don't have to do anything + //this can happen if the Grant was updated in COEUS only with information we don't consume here + } else {//don't have a stored Grant for this URI - this one is new to Pass + passGrantURI = passClient.createResource(systemGrant); + statistics.addGrantsCreated(); + LOG.debug("Creating grant with award number " + systemGrant.getLocalKey()); + } + return passGrantURI; + } + + /** + * Compare two timestamps and return the later of them + * @param currentUpdateString the current latest timestamp string + * @param latestUpdateString the new timestamp to be compared against the current latest timestamp + * @return the later of the two parameters + */ + static String returnLaterUpdate(String currentUpdateString, String latestUpdateString) { + DateTime grantUpdateTime = createJodaDateTime(currentUpdateString); + DateTime previousLatestUpdateTime = createJodaDateTime(latestUpdateString); + return grantUpdateTime.isAfter(previousLatestUpdateTime)? currentUpdateString : latestUpdateString; + } + + /** + * This method provides the latest timestamp of all records processed. After processing, this timestamp + * will be used to be tha base timestamp for the next run of the app + * @return the latest update timestamp string + */ + public String getLatestUpdate(){ + return this.latestUpdateString; + } + + /** + * This returns the final statistics of the processing of the Grant or User Set + * @return the report + */ + public String getReport(){ + return statistics.getReport(); + } + + /** + * This returns the final statistics Object - useful in testing + * @return the statistics object + */ + public PassUpdateStatistics getStatistics() { + return statistics; + } + + + public Map getGrantUriMap() { + return grantUriMap; + } + + //this is used by an integration test + public PassClient getPassClient() { + return passClient; + } + + //used in unit test + Map getFunderMap() { return funderMap; } + + //used in unit test + Map getUserMap() { return userMap; } + + void setDomain(String domain) { + this.DOMAIN = domain; + } + +} \ No newline at end of file diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index d268b42..7f25dce 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -49,7 +49,7 @@ public class DefaultPassUpdater implements PassUpdater{ private static Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); private String latestUpdateString = ""; - private PassClient passClient = PassClientFactory.getPassClient(); + private PassClient passClient; private PassUpdateStatistics statistics = new PassUpdateStatistics(); private PassEntityUtil passEntityUtil; @@ -66,6 +66,7 @@ public class DefaultPassUpdater implements PassUpdater{ DefaultPassUpdater(PassEntityUtil passEntityUtil) { this.passEntityUtil = passEntityUtil; + this.passClient = PassClientFactory.getPassClient(); } //used in unit testing for injecting a mock client diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java index bbce610..931207a 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassEntityUtil.java @@ -21,6 +21,8 @@ import org.dataconservancy.pass.model.User; import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; /** @@ -45,7 +47,7 @@ public class HarvardPilotPassEntityUtil implements PassEntityUtil{ * @return the updated Funder - null if the Funder does not need to be updated */ public Funder update(Funder system, Funder stored) { - if (!harvardFundersEqual(system, stored)) { + if (funderNeedsUpdate(system, stored)) { return updateFunder(system, stored); } return null; @@ -59,7 +61,7 @@ public Funder update(Funder system, Funder stored) { * @return the updated User - null if the User does not need to be updated */ public User update(User system, User stored) { - if (!harvardUsersEqual(system, stored)) { + if (userNeedsUpdate(system, stored)) { return updateUser(system, stored); } return null; @@ -73,7 +75,7 @@ public User update(User system, User stored) { * @return the updated object - null if the Grant does not need to be updated */ public Grant update(Grant system, Grant stored) { - if (!harvardGrantsEqual(system, stored)) { + if (grantNeedsUpdate(system, stored)) { return updateGrant(system, stored); } return null; @@ -86,13 +88,13 @@ public Grant update(Grant system, Grant stored) { * @param stored the version of the Funder as read from Pass * @return a boolean which asserts whether the two supplied Funders are "Harvard equal" */ - private boolean harvardFundersEqual(Funder system, Funder stored) { + private boolean funderNeedsUpdate(Funder system, Funder stored) { //this adjustment handles the case where we take data from policy.properties file, which has no name info - if (system.getName() != null && !system.getName().equals(stored.getName())) return false; - if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return false; - return true; + if (system.getName() != null && !system.getName().equals(stored.getName())) return true; + if (system.getLocalKey() != null ? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getPolicy() != null ? !system.getPolicy().equals(stored.getPolicy()) : stored.getPolicy() != null) return true; + return false; } /** @@ -117,16 +119,16 @@ private Funder updateFunder (Funder system, Funder stored) { * @param stored the version of the User as read from Pass * @return a boolean which asserts whether the two supplied Users are "Harvard equal" */ - private boolean harvardUsersEqual(User system, User stored) { + private boolean userNeedsUpdate(User system, User stored) { //first the fields for which Harvard is authoritative - if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return false; - //if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return false; - if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return false; - if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return false; + if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return true; + //if (system.getMiddleName() != null ? !system.getMiddleName().equals(stored.getMiddleName()) : stored.getMiddleName() != null) return true; + if (system.getLastName() != null ? !system.getLastName().equals(stored.getLastName()) : stored.getLastName() != null) return true; + if (system.getLocatorIds() != null? !stored.getLocatorIds().containsAll(system.getLocatorIds()): stored.getLocatorIds() != null) return true; //next, other fields which require some reasoning to decide whether an update is necessary - if (system.getEmail() != null && stored.getEmail() == null) return false; - if (system.getDisplayName() != null && stored.getDisplayName() == null) return false; - return true; + if (system.getEmail() != null && stored.getEmail() == null) return true; + if (system.getDisplayName() != null && stored.getDisplayName() == null) return true; + return false; } /** @@ -143,7 +145,11 @@ private User updateUser (User system, User stored) { stored.setFirstName(system.getFirstName()); //stored.setMiddleName(system.getMiddleName()); stored.setLastName(system.getLastName()); - stored.setLocatorIds(system.getLocatorIds()); + Set idSet = new HashSet<>(); + idSet.addAll(stored.getLocatorIds()); + idSet.addAll(system.getLocatorIds()); + stored.setLocatorIds(idSet.stream().collect(Collectors.toList())); + //stored.setLocatorIds(system.getLocatorIds()); stored.setEmail(system.getEmail()); stored.setDisplayName(system.getDisplayName()); return stored; @@ -155,18 +161,18 @@ private User updateUser (User system, User stored) { * @param stored the version of the Grant as read from Pass * @return a boolean which asserts whether the two supplied Grants are "Harvard equal" */ - private boolean harvardGrantsEqual(Grant system, Grant stored) { - if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return false; - //if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return false; - if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return false; - if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return false; - if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return false; - if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return false; - if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return false; - if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return false; - //if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return false; - if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return false; - if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return false; + private boolean grantNeedsUpdate(Grant system, Grant stored) { + if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return true; + //if (system.getAwardStatus() != null? !system.getAwardStatus().equals(stored.getAwardStatus()) : stored.getAwardStatus() != null) return true; + if (system.getLocalKey() != null? !system.getLocalKey().equals(stored.getLocalKey()) : stored.getLocalKey() != null) return true; + if (system.getProjectName() != null? !system.getProjectName().equals(stored.getProjectName()) : stored.getProjectName() != null) return true; + if (system.getPrimaryFunder() != null? !system.getPrimaryFunder().equals(stored.getPrimaryFunder()) : stored.getPrimaryFunder() != null) return true; + if (system.getDirectFunder() != null? !system.getDirectFunder().equals(stored.getDirectFunder()) : stored.getDirectFunder() != null) return true; + if (system.getPi() != null? !system.getPi().equals(stored.getPi()) : stored.getPi() != null) return true; + if (system.getCoPis() != null? !new HashSet(system.getCoPis()).equals(new HashSet(stored.getCoPis())): stored.getCoPis() != null) return true; + //if (system.getAwardDate() != null? !system.getAwardDate().equals(stored.getAwardDate()) : stored.getAwardDate() != null) return true; + if (system.getStartDate() != null? !system.getStartDate().equals(stored.getStartDate()) : stored.getStartDate() != null) return true; + if (system.getEndDate() != null? !system.getEndDate().equals(stored.getEndDate()) : stored.getEndDate() != null) return true; return true; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java index 1da2007..de33356 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java @@ -18,7 +18,7 @@ import org.dataconservancy.pass.client.PassClient; -public class HarvardPilotPassUpdater extends DefaultPassUpdater { +public class HarvardPilotPassUpdater extends BasicPassUpdater { public HarvardPilotPassUpdater () { super(new HarvardPilotPassEntityUtil()); diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java index 4e70083..ed4ecbb 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java @@ -17,10 +17,7 @@ import org.dataconservancy.pass.client.PassClient; import org.dataconservancy.pass.client.PassClientFactory; -import org.dataconservancy.pass.grant.data.DateTimeUtil; -import org.dataconservancy.pass.grant.data.PassUpdateStatistics; -import org.dataconservancy.pass.grant.data.JhuPassUpdater; -import org.dataconservancy.pass.grant.data.CoeusPassEntityUtil; +import org.dataconservancy.pass.grant.data.*; import org.dataconservancy.pass.model.Funder; import org.dataconservancy.pass.model.Grant; @@ -28,7 +25,6 @@ import org.dataconservancy.pass.model.User; import org.dataconservancy.pass.model.support.Identifier; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -37,6 +33,7 @@ import static java.lang.Thread.sleep; import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -56,15 +53,15 @@ import java.util.Map; /** - * An integration test class for the JhuPassUpdater. + * An integration test class for the BasicPassUpdater. */ @RunWith(MockitoJUnitRunner.class) -@Ignore public class BasicPassUpdaterIT { private final List> resultSet = new ArrayList<>(); - private static final String employeeidPrefix = "johnshopkins.edu:employeeid:"; - private static final String jhedPrefix = "johnshopkins.edu:jhed:"; + private static final String DOMAIN = "default.domain"; + private static final String employeeidPrefix = DOMAIN + ":employeeid:"; + private static final String jhedPrefix = DOMAIN + ":jhed:"; private final CoeusPassEntityUtil passEntityUtil = new CoeusPassEntityUtil(); private final Map funderPolicyUriMap = new HashMap<>(); @@ -73,6 +70,8 @@ public class BasicPassUpdaterIT { private final PassClient passClient = PassClientFactory.getPassClient(); + PassUpdater passUpdater = new BasicPassUpdater(new BasicPassEntityUtil(), passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); @Rule public TemporaryFolder folder= new TemporaryFolder(); @@ -124,9 +123,9 @@ public void setup() { rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); + //rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); + //rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + i + ":00:00.0"); rowMap.put(C_ABBREVIATED_ROLE, (i % 2 == 0 ? "P" : "C")); @@ -141,7 +140,7 @@ public void setup() { /** * The behavior of PassUpdate's updateGrants() method is to compare the data coming in on the ResultSet with * the existing data in Pass, and create objects if Pass does not yet have them, and update them if they exist in Pass but - * there are differences in the fields for which COEUS is the authoritative source, or COEUS has a clue about other fields which are null + * there are differences in the fields for which the pull source is the authoritative source, or COEUS has a clue about other fields which are null * on the PASS object. * * @throws InterruptedException - the exception @@ -149,9 +148,7 @@ public void setup() { @Test public void updateGrantsIT() throws InterruptedException { - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); passUpdater.updatePass(resultSet, "grant"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); assertEquals(5, statistics.getPisAdded()); assertEquals(5, statistics.getCoPisAdded()); @@ -184,9 +181,8 @@ public void updateGrantsIT() throws InterruptedException { assertEquals(0, statistics.getUsersUpdated()); //now let's monkey with a few things; we expect to update the changed objects - //first change all the things that shouldn't matter Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1 + "MOO"); + rowMap.put(C_GRANT_AWARD_NUMBER, C_GRANT_AWARD_NUMBER + 1); rowMap.put(C_GRANT_AWARD_STATUS, "Active"); rowMap.put(C_GRANT_LOCAL_KEY, C_GRANT_LOCAL_KEY + 1); rowMap.put(C_GRANT_PROJECT_NAME, C_GRANT_PROJECT_NAME + 1 + "MOO"); @@ -195,17 +191,17 @@ public void updateGrantsIT() throws InterruptedException { rowMap.put(C_GRANT_END_DATE, "01/01/2002"); rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, C_DIRECT_FUNDER_LOCAL_KEY + 1); - rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + 1 + "MOOOOO"); + rowMap.put(C_DIRECT_FUNDER_NAME, C_DIRECT_FUNDER_NAME + 1 + "MOO"); rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, C_PRIMARY_FUNDER_LOCAL_KEY + 1); rowMap.put(C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_NAME + 1); rowMap.put(C_USER_FIRST_NAME, C_USER_FIRST_NAME + 1); - rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + 1 + "MOOOO"); - rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + 1); + rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + 1 + "MOOO"); + rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + 1 + "MOOOOO"); rowMap.put(C_USER_EMAIL, C_USER_EMAIL + 1); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + 1); + //rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + 1); rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + 1); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + 1); + //rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + 1); rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); rowMap.put(C_ABBREVIATED_ROLE, ("C")); @@ -221,8 +217,7 @@ public void updateGrantsIT() throws InterruptedException { assertEquals(0, statistics.getFundersCreated()); assertEquals(1, statistics.getFundersUpdated()); assertEquals(0, statistics.getGrantsCreated()); - assertEquals(0, statistics.getGrantsUpdated()); - assertEquals(0, statistics.getUsersCreated()); + assertEquals(1, statistics.getGrantsUpdated()); assertEquals(1, statistics.getUsersUpdated()); sleep(20000); @@ -231,12 +226,12 @@ public void updateGrantsIT() throws InterruptedException { Grant grant = new Grant(); grant.setAwardNumber(C_GRANT_AWARD_NUMBER + i); grant.setAwardStatus(Grant.AwardStatus.ACTIVE); - String grantIdPrefix = "johnshopkins.edu:grant:"; + String grantIdPrefix = DOMAIN + ":grant:"; grant.setLocalKey(grantIdPrefix + C_GRANT_LOCAL_KEY + i); grant.setProjectName(C_GRANT_PROJECT_NAME + i); - grant.setAwardDate(DateTimeUtil.createJodaDateTime("01/01/2000")); - grant.setStartDate(DateTimeUtil.createJodaDateTime("01/01/2001")); - grant.setEndDate(DateTimeUtil.createJodaDateTime("01/01/2002")); + grant.setAwardDate(createJodaDateTime("01/01/2000")); + grant.setStartDate(createJodaDateTime("01/01/2001")); + grant.setEndDate(createJodaDateTime("01/01/2002")); URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grant.getLocalKey()); Grant passGrant = passClient.readResource(passGrantUri, Grant.class); @@ -244,14 +239,20 @@ public void updateGrantsIT() throws InterruptedException { assertEquals(grant.getAwardNumber(), passGrant.getAwardNumber()); assertEquals(grant.getAwardStatus(), passGrant.getAwardStatus()); assertEquals(grant.getLocalKey(), passGrant.getLocalKey()); - assertEquals(grant.getProjectName(), passGrant.getProjectName()); - assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); - assertEquals(grant.getStartDate(), passGrant.getStartDate()); + if( i==1 ) { + assertEquals(grant.getProjectName() + "MOO", passGrant.getProjectName() ); + assertEquals( createJodaDateTime("01/01/1999"), passGrant.getStartDate()); + assertEquals( createJodaDateTime("01/01/1999"), passGrant.getStartDate()); + } else { + assertEquals(grant.getProjectName(), passGrant.getProjectName()); + assertEquals(grant.getAwardDate(), passGrant.getAwardDate()); + assertEquals(grant.getStartDate(), passGrant.getStartDate()); + } assertEquals(grant.getEndDate(), passGrant.getEndDate()); //let's check funder stuff Funder directFunder = new Funder(); - String funderIdPrefix = "johnshopkins.edu:funder:"; + String funderIdPrefix = DOMAIN + ":funder:"; directFunder.setLocalKey(funderIdPrefix + C_DIRECT_FUNDER_LOCAL_KEY + i); directFunder.setName(C_DIRECT_FUNDER_NAME + i); directFunder.setPolicy(funderPolicyUriMap.get("DirectFunderPolicy" + i)); @@ -259,7 +260,7 @@ public void updateGrantsIT() throws InterruptedException { URI directFunderUri = passClient.findByAttribute(Funder.class, "localKey", directFunder.getLocalKey()); Funder passDirectFunder = passClient.readResource(directFunderUri, Funder.class); if (i == 1) { - assertEquals(directFunder.getName() + "MOOOOO", passDirectFunder.getName()); + assertEquals(directFunder.getName() + "MOO", passDirectFunder.getName()); assertEquals(directFunder.getLocalKey(), passDirectFunder.getLocalKey()); assertEquals(passDirectFunder.getId(), passGrant.getDirectFunder()); } else { @@ -280,11 +281,11 @@ public void updateGrantsIT() throws InterruptedException { User user = new User(); - //institutionalId and localKey were localized by the grant loader + //employeeId and localKey were localized by the grant loader user.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + i); - String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; - user.getLocatorIds().add(hopkinsidPrefix + C_USER_HOPKINS_ID + i); - user.getLocatorIds().add(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + i); + //String hopkinsidPrefix = DOMAIN + ":hopkinsid:"; + //user.getLocatorIds().add(hopkinsidPrefix + C_USER_HOPKINS_ID + i); + //user.getLocatorIds().add(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + i); user.setFirstName(C_USER_FIRST_NAME + i); user.setMiddleName(C_USER_MIDDLE_NAME + i); user.setLastName(C_USER_LAST_NAME + i); @@ -303,11 +304,13 @@ public void updateGrantsIT() throws InterruptedException { User passUser = passClient.readResource(userUri, User.class); assertEquals(user.getFirstName(), passUser.getFirstName()); if (i == 1) { - assertEquals(user.getMiddleName() + "MOOOO", passUser.getMiddleName()); + assertEquals(user.getMiddleName() + "MOOO", passUser.getMiddleName()); + assertEquals(user.getLastName() + "MOOOOO", passUser.getLastName()); } else { assertEquals(user.getMiddleName(), passUser.getMiddleName()); + assertEquals(user.getLastName(), passUser.getLastName()); } - assertEquals(user.getLastName(), passUser.getLastName()); + assertEquals(user.getEmail(), passUser.getEmail()); assertTrue(user.getLocatorIds().containsAll(passUser.getLocatorIds())); assertTrue(passUser.getLocatorIds().containsAll(user.getLocatorIds())); @@ -334,8 +337,6 @@ public void updateUsersIT() throws InterruptedException { user10.setMiddleName(C_USER_MIDDLE_NAME + 10); user10.setLastName(C_USER_LAST_NAME + 10); - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); - URI passUserURI = passUpdater.getPassClient().createResource(user10); User passUser = passClient.readResource(passUserURI, User.class); @@ -354,16 +355,14 @@ public void updateUsersIT() throws InterruptedException { rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); + //rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + 1 + ":00:00.0"); userResultSet.add(rowMap); } passUpdater.updatePass(userResultSet, "user"); - PassUpdateStatistics statistics = passUpdater.getStatistics(); //now update from the set of two users - the second one is not in PASS, but is not created //the first (user10) should be updated, with new fields added @@ -377,8 +376,6 @@ public void updateUsersIT() throws InterruptedException { assertNotNull(updatedUser.getDisplayName()); assertNotNull(updatedUser.getLocatorIds()); assertTrue(updatedUser.getLocatorIds().contains(employeeidPrefix + C_USER_EMPLOYEE_ID + 10)); - assertTrue(updatedUser.getLocatorIds().contains(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + 10)); - assertEquals(C_USER_EMAIL + 10, updatedUser.getEmail()); } @@ -397,7 +394,6 @@ public void updateFundersIT() throws InterruptedException { policy2. setTitle("Policy Two"); policy2. setDescription("Policy Two Description"); - JhuPassUpdater passUpdater = new JhuPassUpdater(passClient); URI policy1Uri = passClient.createResource(policy1); URI policy2Uri = passClient.createResource(policy2); @@ -405,7 +401,7 @@ public void updateFundersIT() throws InterruptedException { assertNotNull(passClient.readResource(policy2Uri, Policy.class)); Funder funder1 = new Funder(); - String DOMAIN = "johnshopkins.edu"; + String DOMAIN = "default.domain"; String fullLocalKey = new Identifier(DOMAIN, "funder", "22229999").serialize(); funder1.setLocalKey(fullLocalKey); funder1.setName("Funder One"); @@ -533,33 +529,4 @@ public void testSerializeAndDeserialize() throws IOException { assertEquals(resultSet, input); } - @Test - public void testCreateAndUpdateGrantsWithSeveralDates() { - String firstAwardDate="01/01/2000"; - String secondAwardDate="06/01/2002"; - String thirdAwardDate="08/15/2004"; - - String[] awardDates = { firstAwardDate, secondAwardDate, thirdAwardDate }; - - String firstStartDate="01/01/2001"; - String secondStartDate="01/01/2003"; - String thirdStartDate="01/01/2005"; - - String[] startDates= { firstStartDate, secondStartDate, thirdStartDate}; - - String firstEndDate="12/31/2001"; - String secondEndDate="12/31/2004"; - String thirdEndDate="12/31/2007"; - - String[] endDates = { firstEndDate, secondEndDate, thirdEndDate }; - - String firstAwardNumber="777979"; - String secondAwardNumber="77111111"; - String thirdAwardNumber="109111111"; - - String[] awardNumbers= {firstAwardNumber, secondAwardNumber, thirdAwardNumber }; - - String grantNumber = "M00-8675309"; - } - } \ No newline at end of file diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java index 1e2e307..7fb6fde 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java @@ -19,7 +19,6 @@ import org.dataconservancy.pass.client.PassClient; import org.dataconservancy.pass.client.PassClientFactory; import org.dataconservancy.pass.grant.data.*; -import org.dataconservancy.pass.model.Funder; import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.Policy; import org.dataconservancy.pass.model.User; @@ -128,7 +127,7 @@ public void updateGrantsIT() throws InterruptedException { assertEquals( grantProjectName[0], passGrant.getProjectName() ); assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() ); assertEquals( createJodaDateTime(grantEndDate[0]), passGrant.getEndDate() ); - assertEquals( passUser0Uri, passGrant.getPi() ); //Reckondwith + assertEquals( passUser0Uri, passGrant.getPi() ); //Pblic assertEquals( 1, passGrant.getCoPis().size() ); assertEquals( passUser1Uri, passGrant.getCoPis().get(0)); @@ -139,8 +138,8 @@ public void updateGrantsIT() throws InterruptedException { assertEquals(1, statistics.getCoPisAdded()); //now simulate an incremental pull since the initial, adjust the stored grant - //we add a new co-pi Jones in the "1" iteration, and change the pi to Einstein in the "2" iteration - //we drop co-pi jones in the last iteration + //we add a new co-pi Squirrel in the "1" iteration, and change the pi to Einstein in the "2" iteration + //we drop co-pi Squirrel in the last iteration Map piRecord1 = makeRowMap(1, 0, "P"); Map coPiRecord1 = makeRowMap(1, 1, "C"); @@ -167,12 +166,11 @@ public void updateGrantsIT() throws InterruptedException { assertEquals( grantIdPrefix + grantLocalKey[1], passGrant.getLocalKey() );//earliest of the additions assertEquals( grantProjectName[1], passGrant.getProjectName() );//earliest of the additions assertEquals( createJodaDateTime(grantStartDate[1]), passGrant.getStartDate() );//earliest of the additions - assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() );//latest of the additions - assertEquals( passUser1Uri, passGrant.getPi() );//Class - assertEquals( 3, passGrant.getCoPis().size() ); - assertTrue( passGrant.getCoPis().contains(passUser0Uri) );//Public - assertTrue( passGrant.getCoPis().contains(passUser1Uri) );//Sinister - assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Squirrel + assertEquals( createJodaDateTime(grantEndDate[1]), passGrant.getEndDate() );//earliest of the additions + assertEquals( passUser0Uri, passGrant.getPi() );//first one in the pull + assertEquals( 2, passGrant.getCoPis().size() ); + assertTrue( passGrant.getCoPis().contains(passUser1Uri) );//Public + assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Sinister } /** diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java index ebf0a2b..c662f3c 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java @@ -14,203 +14,203 @@ * limitations under the License. */ - package org.dataconservancy.pass.grant.integration; - - import org.dataconservancy.pass.client.PassClient; - import org.dataconservancy.pass.client.PassClientFactory; - import org.dataconservancy.pass.grant.data.JhuPassInitUpdater; - import org.dataconservancy.pass.grant.data.PassUpdateStatistics; - import org.dataconservancy.pass.model.Grant; - import org.dataconservancy.pass.model.Policy; - import org.dataconservancy.pass.model.User; - import org.junit.Before; - import org.junit.Test; - - import java.net.URI; - import java.util.ArrayList; - import java.util.HashMap; - import java.util.List; - import java.util.Map; - - import static java.lang.Thread.sleep; - import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; - import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; - import static org.junit.Assert.*; - - public class JhuPassInitUpdaterIT { - - String[] grantAwardNumber = { "A10000000", "A10000001", "A10000002" }; - String[] grantLocalKey = { "10000000", "10000000","10000000" }; //all the same - String[] grantProjectName = {"Awesome Research Project I", "Awesome Research Project II", "Awesome Research Project III" }; - String[] grantAwardDate = { "01/01/1999", "01/01/2001", "01/01/2003" }; - String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; - String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; - String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; - String[] userEmployeeId= { "30000000", "30000001", "30000002"}; - String[] userInstitutionalId = {"amelon1", "aeinst1", "jjones1" }; - String[] userHopkinsId = {"RANDOM", "OMRNDA", "DRMONA" }; - String[] userFirstName = {"Andrew", "Albert", "Junie"}; - String[] userMiddleName = {"Smith", "Carnegie", "Beatrice" }; - String[] userLastName = { "Melon", "Einstein", "Jones" }; - String[] userEmail = { "amelon1@jhu.edu", "aeinst1@jhu.edu", "jjones1@jhu.edu" }; - - - String primaryFunderPolicyUriString; - String directFunderPolicyUriString; - - String grantIdPrefix = "johnshopkins.edu:grant:"; - //String funderIdPrefix = "johnshopkins.edu:funder:"; - //String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; - String employeeidPrefix = "johnshopkins.edu:employeeid:"; - //String jhedidPrefis = "johnshopkins.edu:jhed:"; - - PassClient passClient = PassClientFactory.getPassClient(); - JhuPassInitUpdater passUpdater = new JhuPassInitUpdater(passClient); - PassUpdateStatistics statistics = passUpdater.getStatistics(); - - @Before - public void setup() { - String prefix = System.getProperty("pass.fedora.baseurl"); - if ( !prefix.endsWith("/") ) { - prefix = prefix + "/"; - } - - Policy policy = new Policy(); - policy.setTitle("Primary Policy"); - policy.setDescription("MOO"); - URI policyURI = passClient.createResource(policy); - primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); - - policy =new Policy(); - policy.setTitle("Direct Policy"); - policy.setDescription("MOO"); - policyURI =passClient.createResource(policy); - directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); - +package org.dataconservancy.pass.grant.integration; + +import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.client.PassClientFactory; +import org.dataconservancy.pass.grant.data.JhuPassInitUpdater; +import org.dataconservancy.pass.grant.data.PassUpdateStatistics; +import org.dataconservancy.pass.model.Grant; +import org.dataconservancy.pass.model.Policy; +import org.dataconservancy.pass.model.User; +import org.junit.Before; +import org.junit.Test; + +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.lang.Thread.sleep; +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; +import static org.dataconservancy.pass.grant.data.DateTimeUtil.createJodaDateTime; +import static org.junit.Assert.*; + +public class JhuPassInitUpdaterIT { + + String[] grantAwardNumber = { "A10000000", "A10000001", "A10000002" }; + String[] grantLocalKey = { "10000000", "10000000","10000000" }; //all the same + String[] grantProjectName = {"Awesome Research Project I", "Awesome Research Project II", "Awesome Research Project III" }; + String[] grantAwardDate = { "01/01/1999", "01/01/2001", "01/01/2003" }; + String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; + String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; + String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; + String[] userEmployeeId= { "30000000", "30000001", "30000002"}; + String[] userInstitutionalId = {"amelon1", "aeinst1", "jjones1" }; + String[] userHopkinsId = {"RANDOM", "OMRNDA", "DRMONA" }; + String[] userFirstName = {"Andrew", "Albert", "Junie"}; + String[] userMiddleName = {"Smith", "Carnegie", "Beatrice" }; + String[] userLastName = { "Melon", "Einstein", "Jones" }; + String[] userEmail = { "amelon1@jhu.edu", "aeinst1@jhu.edu", "jjones1@jhu.edu" }; + + + String primaryFunderPolicyUriString; + String directFunderPolicyUriString; + + String grantIdPrefix = "johnshopkins.edu:grant:"; + //String funderIdPrefix = "johnshopkins.edu:funder:"; + //String hopkinsidPrefix = "johnshopkins.edu:hopkinsid:"; + String employeeidPrefix = "johnshopkins.edu:employeeid:"; + //String jhedidPrefis = "johnshopkins.edu:jhed:"; + + PassClient passClient = PassClientFactory.getPassClient(); + JhuPassInitUpdater passUpdater = new JhuPassInitUpdater(passClient); + PassUpdateStatistics statistics = passUpdater.getStatistics(); + + @Before + public void setup() { + String prefix = System.getProperty("pass.fedora.baseurl"); + if ( !prefix.endsWith("/") ) { + prefix = prefix + "/"; } - /** - * we put an initial award for a grant into fedora, then simulate a pull of all records related - * to this grant from the Beginning of Time (including records which created the initial object) - * - * We expect to see some fields retained from the initial award, and others updated. The most - * interesting fields are the investigator fields: all CO-PIs ever on the grant should stay on the - * co-pi field throughout iterations. If a PI is changed, they should appear on the CO-PI field - * - * @throws InterruptedException - */ - @Test - public void processInitGrantIT() throws InterruptedException { - List> resultSet = new ArrayList<>(); - - //put in last iteration as existing record - PI is Einstein - Map piRecord2 = makeRowMap (2, 1, "P"); - resultSet.add(piRecord2); - - passUpdater.updatePass(resultSet, "grant"); - sleep(10000); - - URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); - assertNotNull( passGrantUri ); - - URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[1] ); - assertNotNull( passUser1Uri ); - - Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); - - assertEquals( grantAwardNumber[2], passGrant.getAwardNumber() ); - assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); - assertEquals( grantIdPrefix + grantLocalKey[2], passGrant.getLocalKey() ); - assertEquals( grantProjectName[2], passGrant.getProjectName() ); - assertEquals( createJodaDateTime(grantAwardDate[2]), passGrant.getAwardDate() ); - assertEquals( createJodaDateTime(grantStartDate[2]), passGrant.getStartDate() ); - assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() ); - assertEquals( passUser1Uri, passGrant.getPi() ); //Einstein - assertEquals( 0, passGrant.getCoPis().size() ); - - //check statistics - assertEquals(1, statistics.getGrantsCreated()); - assertEquals(1, statistics.getUsersCreated()); - assertEquals(1, statistics.getPisAdded()); - assertEquals(0, statistics.getCoPisAdded()); - - //now simulate a complete pull from the Beginning of Time and adjust the stored grant - //we add a new co-pi Jones in the "1" iteration, and change the pi to Einstein in the "2" iteration - //we drop co-pi jones in the last iteration - Map piRecord0 = makeRowMap(0, 0, "P"); - Map coPiRecord0 = makeRowMap(0, 1, "C"); - Map piRecord1 = makeRowMap(1, 0, "P"); - Map coPiRecord1 = makeRowMap(1, 1, "C"); - Map newCoPiRecord1 = makeRowMap(1, 2, "C"); - - //in the initial pull, we will find all of the records (check?) - resultSet.clear(); - resultSet.add(piRecord0); - resultSet.add(coPiRecord0); - resultSet.add(piRecord1); - resultSet.add(coPiRecord1); - resultSet.add(newCoPiRecord1); - resultSet.add(piRecord2); - - passUpdater.updatePass(resultSet, "grant"); - sleep(10000); - - passGrant = passClient.readResource( passGrantUri, Grant.class ); - URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[0] ); - assertNotNull( passUser0Uri ); - URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[2] ); - assertNotNull( passUser2Uri ); - - assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() );//initial - assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); - assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); - assertEquals( grantProjectName[0], passGrant.getProjectName() );//initial - assertEquals( createJodaDateTime(grantAwardDate[0]), passGrant.getAwardDate() );//initial - assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() );//initial - assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() );//latest - assertEquals( passUser1Uri, passGrant.getPi() );//Einstein - assertEquals( 2, passGrant.getCoPis().size() ); - assertTrue( passGrant.getCoPis().contains(passUser0Uri) );//Melon - assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Jones - } + Policy policy = new Policy(); + policy.setTitle("Primary Policy"); + policy.setDescription("MOO"); + URI policyURI = passClient.createResource(policy); + primaryFunderPolicyUriString = policyURI.toString().substring(prefix.length()); - /** - * utility method to produce data as it would look coming from COEUS - * @param iteration the iteration of the (multi-award) grant - * @param user the user supplied in the record - * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") - * @return - */ - private Map makeRowMap( int iteration, int user, String abbrRole) { - Map rowMap = new HashMap<>(); - rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber[iteration]); - rowMap.put(C_GRANT_AWARD_STATUS, "Active"); - rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey[iteration]); - rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName[iteration]); - rowMap.put(C_GRANT_AWARD_DATE, grantAwardDate[iteration]); - rowMap.put(C_GRANT_START_DATE, grantStartDate[iteration]); - rowMap.put(C_GRANT_END_DATE, grantEndDate[iteration]); - - rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000000"); - rowMap.put(C_DIRECT_FUNDER_NAME, "Enormous State University"); - rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000001"); - rowMap.put(C_PRIMARY_FUNDER_NAME, "J L Gotrocks Foundation"); - - rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); - rowMap.put(C_USER_MIDDLE_NAME, userMiddleName[user]); - rowMap.put(C_USER_LAST_NAME, userLastName[user]); - rowMap.put(C_USER_EMAIL, userEmail[user]); - rowMap.put(C_USER_INSTITUTIONAL_ID, userInstitutionalId[user]); - rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); - rowMap.put(C_USER_HOPKINS_ID, userHopkinsId[user]); - - rowMap.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp[iteration]); - rowMap.put(C_ABBREVIATED_ROLE, abbrRole); - - rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); - rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); - - return rowMap; - } + policy =new Policy(); + policy.setTitle("Direct Policy"); + policy.setDescription("MOO"); + policyURI =passClient.createResource(policy); + directFunderPolicyUriString = policyURI.toString().substring(prefix.length()); + + } + + /** + * we put an initial award for a grant into fedora, then simulate a pull of all records related + * to this grant from the Beginning of Time (including records which created the initial object) + * + * We expect to see some fields retained from the initial award, and others updated. The most + * interesting fields are the investigator fields: all CO-PIs ever on the grant should stay on the + * co-pi field throughout iterations. If a PI is changed, they should appear on the CO-PI field + * + * @throws InterruptedException + */ + @Test + public void processInitGrantIT() throws InterruptedException { + List> resultSet = new ArrayList<>(); + + //put in last iteration as existing record - PI is Einstein + Map piRecord2 = makeRowMap (2, 1, "P"); + resultSet.add(piRecord2); + + passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + + URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); + assertNotNull( passGrantUri ); + + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[1] ); + assertNotNull( passUser1Uri ); + + Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); + + assertEquals( grantAwardNumber[2], passGrant.getAwardNumber() ); + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[2], passGrant.getLocalKey() ); + assertEquals( grantProjectName[2], passGrant.getProjectName() ); + assertEquals( createJodaDateTime(grantAwardDate[2]), passGrant.getAwardDate() ); + assertEquals( createJodaDateTime(grantStartDate[2]), passGrant.getStartDate() ); + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() ); + assertEquals( passUser1Uri, passGrant.getPi() ); //Einstein + assertEquals( 0, passGrant.getCoPis().size() ); + + //check statistics + assertEquals(1, statistics.getGrantsCreated()); + assertEquals(1, statistics.getUsersCreated()); + assertEquals(1, statistics.getPisAdded()); + assertEquals(0, statistics.getCoPisAdded()); + + //now simulate a complete pull from the Beginning of Time and adjust the stored grant + //we add a new co-pi Jones in the "1" iteration, and change the pi to Einstein in the "2" iteration + //we drop co-pi jones in the last iteration + Map piRecord0 = makeRowMap(0, 0, "P"); + Map coPiRecord0 = makeRowMap(0, 1, "C"); + Map piRecord1 = makeRowMap(1, 0, "P"); + Map coPiRecord1 = makeRowMap(1, 1, "C"); + Map newCoPiRecord1 = makeRowMap(1, 2, "C"); + + //in the initial pull, we will find all of the records (check?) + resultSet.clear(); + resultSet.add(piRecord0); + resultSet.add(coPiRecord0); + resultSet.add(piRecord1); + resultSet.add(coPiRecord1); + resultSet.add(newCoPiRecord1); + resultSet.add(piRecord2); + + passUpdater.updatePass(resultSet, "grant"); + sleep(10000); + + passGrant = passClient.readResource( passGrantUri, Grant.class ); + URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[0] ); + assertNotNull( passUser0Uri ); + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[2] ); + assertNotNull( passUser2Uri ); + + assertEquals( grantAwardNumber[0], passGrant.getAwardNumber() );//initial + assertEquals( Grant.AwardStatus.ACTIVE, passGrant.getAwardStatus() ); + assertEquals( grantIdPrefix + grantLocalKey[0], passGrant.getLocalKey() ); + assertEquals( grantProjectName[0], passGrant.getProjectName() );//initial + assertEquals( createJodaDateTime(grantAwardDate[0]), passGrant.getAwardDate() );//initial + assertEquals( createJodaDateTime(grantStartDate[0]), passGrant.getStartDate() );//initial + assertEquals( createJodaDateTime(grantEndDate[2]), passGrant.getEndDate() );//latest + assertEquals( passUser1Uri, passGrant.getPi() );//Einstein + assertEquals( 2, passGrant.getCoPis().size() ); + assertTrue( passGrant.getCoPis().contains(passUser0Uri) );//Melon + assertTrue( passGrant.getCoPis().contains(passUser2Uri) );//Jones + } + /** + * utility method to produce data as it would look coming from COEUS + * @param iteration the iteration of the (multi-award) grant + * @param user the user supplied in the record + * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") + * @return + */ + private Map makeRowMap( int iteration, int user, String abbrRole) { + Map rowMap = new HashMap<>(); + rowMap.put(C_GRANT_AWARD_NUMBER, grantAwardNumber[iteration]); + rowMap.put(C_GRANT_AWARD_STATUS, "Active"); + rowMap.put(C_GRANT_LOCAL_KEY, grantLocalKey[iteration]); + rowMap.put(C_GRANT_PROJECT_NAME, grantProjectName[iteration]); + rowMap.put(C_GRANT_AWARD_DATE, grantAwardDate[iteration]); + rowMap.put(C_GRANT_START_DATE, grantStartDate[iteration]); + rowMap.put(C_GRANT_END_DATE, grantEndDate[iteration]); + + rowMap.put(C_DIRECT_FUNDER_LOCAL_KEY, "20000000"); + rowMap.put(C_DIRECT_FUNDER_NAME, "Enormous State University"); + rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, "20000001"); + rowMap.put(C_PRIMARY_FUNDER_NAME, "J L Gotrocks Foundation"); + + rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); + rowMap.put(C_USER_MIDDLE_NAME, userMiddleName[user]); + rowMap.put(C_USER_LAST_NAME, userLastName[user]); + rowMap.put(C_USER_EMAIL, userEmail[user]); + rowMap.put(C_USER_INSTITUTIONAL_ID, userInstitutionalId[user]); + rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); + rowMap.put(C_USER_HOPKINS_ID, userHopkinsId[user]); + + rowMap.put(C_UPDATE_TIMESTAMP, grantUpdateTimestamp[iteration]); + rowMap.put(C_ABBREVIATED_ROLE, abbrRole); + + rowMap.put(C_DIRECT_FUNDER_POLICY, directFunderPolicyUriString); + rowMap.put(C_PRIMARY_FUNDER_POLICY, primaryFunderPolicyUriString); + + return rowMap; } + +} From 89f60cc398e0e98613d6c7ef636378ecfc140c6d Mon Sep 17 00:00:00 2001 From: jrmartino Date: Sat, 9 May 2020 21:09:48 -0400 Subject: [PATCH 07/14] updating harvard pilot machinery to work with new data scheme which does not have an employee if - we use the e[[ part of email addy to serve as out local key for users some minor refactoring with constatnts in updater impls many minor updates, removing warnings, typos, clarification of comments --- .../pass/grant/data/BasicPassEntityUtil.java | 37 +++++++++--------- .../pass/grant/data/BasicPassUpdater.java | 18 ++++----- .../pass/grant/data/CoeusConnector.java | 6 +-- .../pass/grant/data/CoeusPassEntityUtil.java | 1 - .../pass/grant/data/DefaultPassUpdater.java | 27 ++++++------- .../pass/grant/data/DirectoryServiceUtil.java | 4 +- .../grant/data/HarvardPilotConnector.java | 9 +++-- .../grant/data/HarvardPilotPassUpdater.java | 37 +++++++++++++++++- .../pass/grant/data/JhuPassInitUpdater.java | 8 ++-- .../pass/grant/data/JhuPassUpdater.java | 8 ++-- .../grant/data/HarvardPilotConnectorTest.java | 22 +++++------ .../test/resources/HarvardPASSTestData.xlsx | Bin 6389 -> 7455 bytes .../grant/integration/BasicPassUpdaterIT.java | 6 --- .../HarvardPilotPassUpdaterIT.java | 23 ++++++----- .../integration/JhuPassInitUpdaterIT.java | 4 +- .../grant/integration/JhuPassUpdaterIT.java | 6 +-- 16 files changed, 121 insertions(+), 95 deletions(-) diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java index 43a3fc4..793f366 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java @@ -28,10 +28,8 @@ /** * A utility class for handling Grants, Users or Funders. One function performed is comparison of two instances of * these PASS entity classes. These comparisons are reduced to only those fields which are updatable by - * data from the Harvard data, so that two objects are considered "Harvard equal" iff they agree on these fields. - * - * Another function performed by this utility class is to construct an updated version of an instance of one of these classes - * by merging a (possibly) existing Pass object with new information obtained from a Harvard data pull. + * data from the source pull. Stored objects are evaluated and updated according to the PassEntityUtil implementation + * supplied to this updater * * @author jrm@jhu.edu */ @@ -40,7 +38,7 @@ public class BasicPassEntityUtil implements PassEntityUtil{ /** - * This method takes a Harvard Funder, calculates whether it needs to be updated, and if so, returns the updated object + * This method takes a Funder, calculates whether it needs to be updated, and if so, returns the updated object * to be be ingested into the repository. if not, returns null. * @param stored the Funder as it is stored in the PASS backend * @param system the version of the Funder from the Harvard pull @@ -54,10 +52,10 @@ public Funder update(Funder system, Funder stored) { } /** - * This method takes a Harvard User, calculates whether it needs to be updated, and if so, returns the updated object + * This method takes a User, calculates whether it needs to be updated, and if so, returns the updated object * to be be ingested into the repository. if not, returns null. * @param stored the User as it is stored in the PASS backend - * @param system the version of the user from the Harvard pull + * @param system the version of the user from the pull * @return the updated User - null if the User does not need to be updated */ public User update(User system, User stored) { @@ -71,7 +69,7 @@ public User update(User system, User stored) { * This method takes a Harvard Grant, calculates whether it needs to be updated, and if so, returns the updated object * to be be ingested into the repository. if not, returns null. * @param stored the Grant as it is stored in the PASS backend - * @param system the version of the Grant from the Harvard pull + * @param system the version of the Grant from the pull * @return the updated object - null if the Grant does not need to be updated */ public Grant update(Grant system, Grant stored) { @@ -84,9 +82,10 @@ public Grant update(Grant system, Grant stored) { /** * Compare two Funder objects * - * @param system the version of the Funder as seen in the Harvard system pull + * @param system the version of the Funder as seen in the system pull * @param stored the version of the Funder as read from Pass - * @return a boolean which asserts whether the two supplied Funders are "Harvard equal" + * @return a boolean which asserts whether the pulled object contains information which can + * update the stored object */ private boolean funderNeedsUpdate(Funder system, Funder stored) { @@ -98,11 +97,11 @@ private boolean funderNeedsUpdate(Funder system, Funder stored) { } /** - * Update a Pass Funder object with new information from Harvard + * Update a Pass Funder object with new information from a system pull * - * @param system the version of the Funder as seen in the Harvard system pull + * @param system the version of the Funder as seen in the system pull * @param stored the version of the Funder as read from Pass - * @return the Funder object which represents the Pass object, with any new information from Harvard merged in + * @return the Funder object which represents the Pass object, with any new information from the pull merged in */ private Funder updateFunder (Funder system, Funder stored) { stored.setLocalKey(system.getLocalKey()); @@ -113,11 +112,11 @@ private Funder updateFunder (Funder system, Funder stored) { /** * Compare two User objects. We only care about those fields for which is the authoritative source - * After recent changes. this method would be more accurately named "storedUserDoesNotNeedToBeUpdated" + * After recent changes. * * @param system the version of the User as seen in the Harvard system pull * @param stored the version of the User as read from Pass - * @return a boolean which asserts whether the two supplied Users are "Harvard equal" + * @return a boolean which asserts whether the stored object needs updating */ private boolean userNeedsUpdate(User system, User stored) { if (system.getFirstName() != null ? !system.getFirstName().equals(stored.getFirstName()) : stored.getFirstName() != null) return true; @@ -135,9 +134,9 @@ private boolean userNeedsUpdate(User system, User stored) { * the localKey, which this application and Shibboleth both rely on; and email, which this application only populates * if Shib hasn't done so already. * - * @param system the version of the User as seen in the Harvard system pull + * @param system the version of the User as seen in the system pull * @param stored the version of the User as read from Pass - * @return the User object which represents the Pass object, with any new information from Harvard merged in + * @return the User object which represents the Pass object, with any new information merged in */ private User updateUser (User system, User stored) { stored.setFirstName(system.getFirstName()); @@ -156,7 +155,7 @@ private User updateUser (User system, User stored) { * Compare two Grant objects. Note that the Lists of Co-Pis are compared as Sets * @param system the version of the Grant as seen in the system pull * @param stored the version of the Grant as read from Pass - * @return a boolean which asserts whether the two supplied Grants are "Harvard equal" + * @return a boolean which asserts whether the stored ubject needs updating */ private boolean grantNeedsUpdate(Grant system, Grant stored) { if (system.getAwardNumber() != null ? !system.getAwardNumber().equals(stored.getAwardNumber()) : stored.getAwardNumber() != null) return true; @@ -176,7 +175,7 @@ private boolean grantNeedsUpdate(Grant system, Grant stored) { * * @param system the version of the Grant as seen in the system pull * @param stored the version of the Grant as read from Pass - * @return the Grant object which represents the Pass object, with any new information from Harvard merged in + * @return the Grant object which represents the Pass object, with any new information merged in */ private Grant updateGrant(Grant system, Grant stored) { stored.setAwardNumber(system.getAwardNumber()); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java index 55515b7..7bc270f 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassUpdater.java @@ -49,23 +49,23 @@ public class BasicPassUpdater implements PassUpdater{ private String DOMAIN = "default.domain"; - private static Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); + private static final Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); private String latestUpdateString = ""; - private PassClient passClient; - private PassUpdateStatistics statistics = new PassUpdateStatistics(); - private PassEntityUtil passEntityUtil; + private final PassClient passClient; + private final PassUpdateStatistics statistics = new PassUpdateStatistics(); + private final PassEntityUtil passEntityUtil; //used in test classes - private Map grantUriMap = new HashMap<>(); + private final Map grantUriMap = new HashMap<>(); //used in unit test //some entities may be referenced many times during an update, but just need to be updated the first time //they are encountered. these include Users and Funders. we save the overhead of redundant updates //of these by looking them up here; if they are on the Map, they have already been processed // - private Map funderMap = new HashMap<>(); - private Map userMap = new HashMap<>(); + private final Map funderMap = new HashMap<>(); + private final Map userMap = new HashMap<>(); private String mode; @@ -399,7 +399,7 @@ private URI updateUserInPass(User systemUser) { //we first check to see if the user is known by the Hopkins ID. If not, we check the employee ID. //last attempt is the JHED ID. this order is specified by the order of the List as constructed on updatedUser URI passUserUri = null; - ListIterator idIterator = systemUser.getLocatorIds().listIterator(); + ListIterator idIterator = systemUser.getLocatorIds().listIterator(); while (passUserUri == null && idIterator.hasNext()) { String id = String.valueOf(idIterator.next()); @@ -521,7 +521,7 @@ public PassClient getPassClient() { //used in unit test Map getUserMap() { return userMap; } - void setDomain(String domain) { + public void setDomain(String domain) { this.DOMAIN = domain; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java index 2eb98bb..49930f6 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java @@ -29,8 +29,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.Set; - import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; @@ -41,7 +39,7 @@ * @author jrm@jhu.edu */ public class CoeusConnector implements GrantConnector { - private static Logger LOG = LoggerFactory.getLogger(CoeusConnector.class); + private static final Logger LOG = LoggerFactory.getLogger(CoeusConnector.class); //property names private static final String COEUS_URL = "coeus.url"; private static final String COEUS_USER = "coeus.user"; @@ -51,7 +49,7 @@ public class CoeusConnector implements GrantConnector { private String coeusUser; private String coeusPassword; - private Properties funderPolicyProperties; + private final Properties funderPolicyProperties; private DirectoryServiceUtil directoryServiceUtil; diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java index 4629d4e..8107f70 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java @@ -207,5 +207,4 @@ private Grant updateGrant(Grant system, Grant stored) { return stored; } - } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index 7f25dce..5ba6651 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -46,20 +46,20 @@ public class DefaultPassUpdater implements PassUpdater{ private String DOMAIN = "default.domain"; - private static Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); + private static final Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class); private String latestUpdateString = ""; - private PassClient passClient; - private PassUpdateStatistics statistics = new PassUpdateStatistics(); - private PassEntityUtil passEntityUtil; + private final PassClient passClient; + private final PassUpdateStatistics statistics = new PassUpdateStatistics(); + private final PassEntityUtil passEntityUtil; - private Map grantUriMap = new HashMap<>(); + private final Map grantUriMap = new HashMap<>(); //some entities may be referenced many times during an update, but just need to be updated the first time //they are encountered. these include Users and Funders. we save the overhead of redundant updates //of these by looking them up here; if they are on the Map, they have already been processed - private Map funderMap = new HashMap<>(); - private Map userMap = new HashMap<>(); + private final Map funderMap = new HashMap<>(); + private final Map userMap = new HashMap<>(); private String mode; @@ -176,11 +176,11 @@ private void updateGrants(Collection> results) { DateTime startDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_START_DATE, null)); DateTime endDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_END_DATE, null)); - //set values that should match earliest iteration of the grant - //these are used only for the initial grant load - //we test for both award date and start date in case one is missing - belt and suspenders - if (startDate != null && (grant.getStartDate() == null || startDate.isBefore(grant.getStartDate())) || - awardDate != null && (grant.getAwardDate() == null || awardDate.isBefore(grant.getAwardDate()))) { + //set values that should match earliest iteration of the grant. we wet these on the system record + //in case they are needed to update a stored grant record. + //these values will not override existing stored values unless the PassEntityUtil implementation + //allows it. + if (startDate != null && (grant.getStartDate() == null || startDate.isBefore(grant.getStartDate()))) { grant.setAwardDate(awardDate); grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); grant.setAwardNumber(rowMap.get(C_GRANT_AWARD_NUMBER)); @@ -416,7 +416,7 @@ private URI updateUserInPass(User systemUser) { //we first check to see if the user is known by the Hopkins ID. If not, we check the employee ID. //last attempt is the JHED ID. this order is specified by the order of the List as constructed on updatedUser URI passUserUri = null; - ListIterator idIterator = systemUser.getLocatorIds().listIterator(); + ListIterator idIterator = systemUser.getLocatorIds().listIterator(); while (passUserUri == null && idIterator.hasNext()) { String id = String.valueOf(idIterator.next()); @@ -519,7 +519,6 @@ public PassUpdateStatistics getStatistics() { return statistics; } - public Map getGrantUriMap() { return grantUriMap; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java index 4c7771a..1336df1 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java @@ -76,8 +76,8 @@ private enum Type { EMPLOYEE2HOPKINS("EmployeeID_to_HopkinsID", "employeeid"), HOPKINS2EMPLOYEE("HopkinsID_to_EmployeeID", "hopkinsid"); - private String serviceUrlEnding; - private String queryParameter; + private final String serviceUrlEnding; + private final String queryParameter; Type(String serviceUrlEnding, String queryParameter) { this.serviceUrlEnding = serviceUrlEnding; diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java index 84bbf31..b44b909 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java @@ -38,8 +38,8 @@ import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; /** - * This implementation of the Grant Connector interface processes date given to us in an Excel spreadsheet. We take in the information to produce - * an intermediate date object which is compatible with our PASS data loading setup. + * This implementation of the Grant Connector interface processes data given to us in an Excel spreadsheet. We take in the information to produce + * an intermediate data object which is compatible with our PASS data loading setup. * * @author jrm */ @@ -111,7 +111,8 @@ public List> retrieveUpdates(String queryString, String mode LOG.debug("Processing grant record ..."); //we only process rows with a Harvard ID - String employeeId = stringify(cells.getCell(7)); //we use email as employee id + String employeeId = stringify(cells.getCell(6)); + //String email = stringify(cells.getCell(7)); if(employeeId != null && employeeId.length()>0) { Map rowMap = new HashMap<>(); @@ -125,7 +126,7 @@ public List> retrieveUpdates(String queryString, String mode String role = stringify(cells.getCell(5)); //F: Role rowMap.put(C_ABBREVIATED_ROLE, sortRole(role)); - rowMap.put(C_USER_EMPLOYEE_ID, employeeId); //row G used to be Harvard id, is missing now + rowMap.put(C_USER_EMPLOYEE_ID, stringify(cells.getCell(6))); //row G used to be Harvard id, we hack it for now rowMap.put(C_USER_EMAIL, stringify(cells.getCell(7))); //H: PI Email String funderLocalKey = stringify(cells.getCell(8)); //I: Funder ID diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java index de33356..74ba00f 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotPassUpdater.java @@ -17,17 +17,50 @@ package org.dataconservancy.pass.grant.data; import org.dataconservancy.pass.client.PassClient; +import org.dataconservancy.pass.model.User; +import org.dataconservancy.pass.model.support.Identifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.dataconservancy.pass.grant.data.CoeusFieldNames.*; public class HarvardPilotPassUpdater extends BasicPassUpdater { + private static final Logger LOG = LoggerFactory.getLogger(HarvardPilotPassUpdater.class); + private static final String DOMAIN = "harvard.edu"; + public HarvardPilotPassUpdater () { super(new HarvardPilotPassEntityUtil()); - super.setDomain("harvard.edu"); + super.setDomain(DOMAIN); } public HarvardPilotPassUpdater (PassClient passClient) { super(new HarvardPilotPassEntityUtil(), passClient ); - super.setDomain("harvard.edu"); + super.setDomain(DOMAIN); + } + + @Override + User buildUser(Map rowMap) { + User user = new User(); + user.setFirstName(rowMap.get(C_USER_FIRST_NAME)); + //user.setMiddleName(rowMap.getOrDefault(C_USER_MIDDLE_NAME, null)); + user.setLastName(rowMap.get(C_USER_LAST_NAME)); + user.setDisplayName(rowMap.get(C_USER_FIRST_NAME) + " " + rowMap.get(C_USER_LAST_NAME)); + String email =rowMap.get(C_USER_EMAIL); + user.setEmail(email); + // + //Build the List of locatorIds - put the most reliable ids first + //for the pilot, we construct the eppn locatorId from the email address + String eppn = email.split("@")[0]; + if (eppn != null) { + String INSTITUTIONAL_ID_TYPE = "jhed"; + user.getLocatorIds().add(new Identifier(DOMAIN, INSTITUTIONAL_ID_TYPE, eppn).serialize()); + } + user.getRoles().add(User.Role.SUBMITTER); + LOG.debug("Built user with institutional ID " + eppn); + return user; } } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java index 63283e2..8c0e27c 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassInitUpdater.java @@ -13,17 +13,18 @@ public class JhuPassInitUpdater extends DefaultPassUpdater { - private static Logger LOG = LoggerFactory.getLogger(JhuPassInitUpdater.class); + private static final Logger LOG = LoggerFactory.getLogger(JhuPassInitUpdater.class); + private static final String DOMAIN = "johnshopkins.edu"; public JhuPassInitUpdater(PassClient passClient) { super(new CoeusPassInitEntityUtil(), passClient); - super.setDomain("johnshopkins.edu"); + super.setDomain(DOMAIN); } public JhuPassInitUpdater() { super(new CoeusPassInitEntityUtil()); - super.setDomain("johnshopkins.edu"); + super.setDomain(DOMAIN); } @Override @@ -46,7 +47,6 @@ User buildUser(Map rowMap) { jhedId = rowMap.get(C_USER_INSTITUTIONAL_ID).toLowerCase(); } //Build the List of locatorIds - put the most reliable ids first - String DOMAIN = "johnshopkins.edu"; if (employeeId != null) { String EMPLOYEE_ID_TYPE = "employeeid"; user.getLocatorIds().add(new Identifier(DOMAIN, EMPLOYEE_ID_TYPE, employeeId).serialize()); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassUpdater.java index 9bfca3b..8d2c678 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/JhuPassUpdater.java @@ -37,17 +37,18 @@ public class JhuPassUpdater extends DefaultPassUpdater { - private static Logger LOG = LoggerFactory.getLogger(JhuPassUpdater.class); + private static final Logger LOG = LoggerFactory.getLogger(JhuPassUpdater.class); + private static final String DOMAIN = "johnshopkins.edu"; public JhuPassUpdater(PassClient passClient) { super(new CoeusPassEntityUtil(), passClient); - super.setDomain("johnshopkins.edu"); + super.setDomain(DOMAIN); } public JhuPassUpdater() { super(new CoeusPassEntityUtil()); - super.setDomain("johnshopkins.edu"); + super.setDomain(DOMAIN); } @Override @@ -70,7 +71,6 @@ User buildUser(Map rowMap) { jhedId = rowMap.get(C_USER_INSTITUTIONAL_ID).toLowerCase(); } //Build the List of locatorIds - put the most reliable ids first - String DOMAIN = "johnshopkins.edu"; if (employeeId != null) { String EMPLOYEE_ID_TYPE = "employeeid"; user.getLocatorIds().add(new Identifier(DOMAIN, EMPLOYEE_ID_TYPE, employeeId).serialize()); diff --git a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java index e27c561..b14bc99 100644 --- a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java +++ b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/HarvardPilotConnectorTest.java @@ -101,17 +101,17 @@ public void testRetrieveGrantUpdates() throws IOException { "C", "P" }; - /* String[] harvardIds = {"86753091", - "86753092", - "86753093", - "86753094", - "86753095", + String[] harvardIds = {"wdrumstick", + "gflanksteak", + "ddbrisket", + "abacon", + "efarmer", "", - "86753097", - "86753098", - "86753099" - }; commented out because we aren't using these (yet) - emails instead*/ - String[] harvardIds = {"wdrumstick@harvard.edu", + "csteer", + "dbovine", + "bcow" + }; + /* String[] harvardIds = {"wdrumstick@harvard.edu", "gflanksteak@harvard.edu", "ddbrisket@harvard.edu", "abacon@harvard.edu", @@ -120,7 +120,7 @@ public void testRetrieveGrantUpdates() throws IOException { "csteer@harvard.edu", "dbovine@harvard.edu", "bcow@harvard.edu" - }; + }; */ String[] emails = {"wdrumstick@harvard.edu", "gflanksteak@harvard.edu", "ddbrisket@harvard.edu", diff --git a/pass-grant-data/src/test/resources/HarvardPASSTestData.xlsx b/pass-grant-data/src/test/resources/HarvardPASSTestData.xlsx index cbf3015ea9fce9a002e1aacba1a4c3f1ac161556..cdab19df5e54eae812c6a0d079d9b3a09ea82504 100644 GIT binary patch literal 7455 zcmbVxWl&wsvNkRo2p-%Wg1fuBy9Rf63lb!_1b2tv?(XiggF^^T65R4_a?V%D%{}M+ zaeG&-T2r;UXU{V|-91k$$wES5fWg7RflU;ZDS`bV2%vXEXER$DMuwMXWx})!Br{6b zxnE?Qk4F=xm}z}y_)LbBUtsFQvoU8H6ONmoPcWQLSrKfkwEvGuPSNyrH1n&-Ovy+I zhjJadsF+7=pMsoMH^+FLHky-UQyMbl$ZyBW7Y1xEi4*L>>+qOX^pyLw40v{oitt;* zOxI0)LhSSnA%B{Fy$&zC-mu(wh}H?>QC!vsmbA2 z6E}7mB7=5@%emAlma0yLm8*A696c_&5JnVF)Nn)ogP|t$eG`w!?->B`y*d_Q0E{ML z7;;Gy2U%s^rc^AEK2^GrUAL!rO);NAv!Ta9y3^?)S+6F+1$eXJPg`>k@bp*TRPQla z$i?+!zU~q3JCrK~Q7`t4MZ&?NmW>kuHEK2y!16n_uOUUA|vW1P?iD8Ie zH;*(~Es!bNd3buird=LRoeiJTJTA}`_<%2qyXDD*<8IMR>`pOci>5&Kg+!4kU1HD{ zLrI$?O57ttbh~g5`fHcDWY8m>xV`!MvT9YHY|42jr(T{Nw(3^~Du^itDICVWNHsJk zR&)#|53dC+{AH1NJ>*fcNqT(x};v`b`nY9`21TR=Jlk2BbF5GI$-BK+2eqlyL z&Vr5hUdIJl4r!4m!%EHhJ=oUyys59+jgR!W_ltzNE7_wYB}--FjxJ+D33BYEu zuO81jk5DfTg%6mp!vHx`2jo9F6w<#O%Ei^o*39L_oldp&9Wt0O{5Gp=9=jl{1l9nh zZ3=1EEb2D2@@v;yf#4O-LPJ)f0Z+F0a1B<=6zMV)-yyoX4nFtyx+C5m*husee1t~j z3*3lD0RIvn4zCF5S)Urf_byi{d^j8s1F5wGksU_#D3Xj}Xn{?ai{pvfBUkn~;S@>T zkBBKrI;?j?F2Yjis#>fNFB`Sf7l`d9QN`;YgL_UYrge;vbXLmoV)HX*-2wR#oz9 z8GUn3%`0G3fr7Skwh+4XRoj`$h!!K8n;G&F2?}}=Am*z)VGqsE#*S ztQEYp8$JGXp7L>a>@4*1!X!CM7E{WKiFfw@>8Psy8tsn zZp45Q3Q)c1kr!t+n|?+yhQ+gzD&?t%x#TBK3E!H6r(ON#c$(OS121;zc)%Q;CA5;7iO~k~ zcG5Q}mwx^uoe1W3(>@=x9$-i*>s+2EaMuEStK^{_T%I@gP+>+!W}=|3cY-JNn{28= zW#TLQCIpXAv%+&>NbXailMg$UX>GMn4H(bhFXtyW)@cqJbarxZ|LOdY{SL@3mS$$I zE{y-mTfYX(F9+&Z2AfQ5e+5B;BJR^2Bv<`u8oT~ebh_3G_ETVd);dL<5Loncr>nJz zu{3wJiz4g({j7~U6;Tp9#83eQHjb5znGX2DynRcE8#{GuA#-u^3=Ni68?&$wR)MY) zp{XSQ`*=6gxx(76;Z&JT=dkaQ^xveF-?aQR6}lQUeb5QqTz+d*Qc@7G(qxrX7@gdl zE%%7p+b;H9C(R}(-+>0u^PzKaQGB=EtHx~%C_w5JjC<=)D_vzWuQzDk8ae5LscTBi zL4CJet;zTT%+bVv^9X3!q)`6^4EFy5<~JmrY4V9X94PP4?=TrQS+O5SF=|FHn3*Qd zY|Y&3)m!0pjh3;;bHH(7=D&0%iodv{tfugya>Cz-3i zv5|`kHaa>Vwo=X8u}Zg(2xEfvV(&XCy~Ovmq&Y5=BhStF8Ma75b#B7gyhMaB^zy@{=k*8=M@x)x;$?_BEs0-wb z@&XsFxMRvqw3)Aq<;JQD)XH;2_M6GeYhsjFUdJ!-+vR(mT)T?0E|))7Q78vjnvOL- zKa)>=VjwKLX2xPc7OWF1WMMgq+jI?;o2jIt1c!a?HV)lxx9KGvsLX?nLt(Ev90M%r>VhV537w7fqa;!EKt zwTMiBpvEzcQ-!xSx3*pK(+H01HOs=S6rWMPk8~9CY&$t+sv~0r4P}R{$$R*p37!LD zj?xp%s0L^Va$+e1=#=*<19Ne;E-Jh#hZ<)amHTIlc~_S+f$7LFP$S$N#6r^%;NU?D zNN`H7=0bsmu}C{da5JtO#Mv=m%2;hFeQJA(NZU3c$Cp0D=@i9$oUWwx_h*%h9!@;M zrvgTWu(*9Ifq|y-h;XHOW#HiKY^)&&iyra!{7U97SY?}#W=)Rq_rz;dSQSJeBqI}O zs@ZtZ`t|o4m5VoyL&7{nh;S=%Zy_N#<_7`;Su#upphq)}p~}uluJn~6T#(DoA?&aS zRfsTl62lge+ZjspDT@8;Zk}{2&Ud4NhkbkCR5k;Ju8-$dAnm5 z!?HHJ%ex$u1f9Ve2a*mvvSvQ{JK9`9W-6@QG4K5U&uf0*p@7Cg*ZzSEHz55tP`m+w z3$O~=5TE?_`i2cm#(vMS;HfwP7t%f|{k_o6S1qzuV!F^tafjYB^A~qvysP=2A zCR@vzmvC}A`L-@ymwltqC1o5R5PH4@=;0w9Pd zA4k5*6N9arJ8ci^!AkDcp?F`w9?u1)b#3yly)(EsP+IThSe?(-jBVi{&r1hg;|7_^P0@OJ);Er9}+A=J!y`Q z!Rd;`)@?GU4>%9%y)3|ccOYS7{QefokG<~fpH$rKGFLQhh#lGj!9>3en2-IPvoDj{r zRxs7np@>iOnG~?}%ht9kc|MNP6v;XbShdq^q-t*!Lb-|M#LJ;Xd zLvLW(Wjel_Ca39V#@Zk|1WlC|r}#1P_VkAIF5W!9Gocnup#d11`sFUwZgE|)h=B=U z1ypy1Dd4@@(P))U8(qrH-XmXKMXP5ixI(R>?c|B5=P|6CTC$ve>C?2|8a&J;Pl+@88+K66BH#pa#K&2a%(O_+CN@3~LiwwZc(a(iA zuPGMKByIg5CV5ooOzeSW=!%A%D!s7kv2h&ETmjk074B_ZCe3)rlOEp+y$!zcYV5Cb z!fk-HlzB|EmEQ6lnvvz_u*0X}fhtFYE>^C?-p&#PuQg(e$%wPD)-9sQSt?4f0*OVt z3q~*RxJUHzcYVm{0l^7=LbamrLKNdNx(Z1i2pmTcXAIYDZTf4+&l*3=ZfNGlC8pzz z2R;=Z?MR9lJoFun)fRuUjfLI%>|b(j=6;&9LLA|HE@AE*LJuKCQI3{CzgMc()yJ+|B1ew-Q6 zKpw7R4S2O}*n(oADoZqq?R8(&8z%pRc~98_&Ay0H3*h#loa%pb@Ka9VrMQc0R>vW4U$9J`lnB+ zYmI{`r^x%M44S>D%JuGp!`~vwk5ZsU*88XJkpx{VUBW2w*DfKtw3+exhL^Vwn&0Qd zZdqVm{9yKVCJ<<2_hk#pG3=3idpqcMd%JY;4NjmfFn0K2{P<$Dayt>#*B3k|YWtjF zE4Wusu_%=ZIM!OEnfo($GUw^Y@w$8*&fDCAFjVbC9w2%Psl!zI4t<-uuf{eL4tMuS zZsDEr+cSLn%s2Ui>uiTr;>VtvtY2So_%}!Ot>(91Sk|S$KzF6nW_GTr@ntdew{-h5 zrp)OnLO-GARn4T*D)C;!oG~gR%Q0EV;Muqt9yy=W-c35?yQEy#-k@*Rn>CN2Dtn<# zLx*W6(&f$F*LVIcv=!(BuK_W3?(%p(zSlg_De ztg~~0CR-H@EU=e{xAzI(9OAD}Q1_R&N|N^lVvjN>t^~&y4ovMCc(5Jd+o|$8%*OBD z05@4GB#2H89)5*9$NieaHPHI&{*evB_!mMhmd4IzrmC*aR`wRZ?(kVDz4kyR#P{bu zfZm${`{qXR_(I*CBKkNC(RmXK9TO9hx;^Q-`m4iXay)k&n6W^{kAjhNiZYZ`9)h@J z-j~9>yps$=;*>KoB@OHH^~)*yUn$W^g4B&N2d%ErMcp}lIN?bEERDwLi_A^UUWBI6 zNFlX0vvVnxKaj+{uiE(<2!%s&nK>xC-$Yh)fZMT>=v(ZvwUba~EpGDaWx`f{u;qaF z=HTXTMWUDbE?*CI$!}f{d%^EX;ZA6a5xY+wTQBX;b{jieR}b(@pL~LEnjxMJy)GO_ z$1)TKJe?pSKNh@li8Z8tO)1_>eoF1-ivb{YIJF>ah;;bwrKVTYtgc+3$NpyNL*7mxcA#Aho4j|(_*^DnF^{c2 zj5KU!!sYo`QQZ?6${Q|!k|bfkb{20>{N<3{=!S&!f^E$X_J^pLUk){GtfBi*IOf>O zEJ{(!3{`|=-oAYWR7Jc;z(nUOpQtw)W!7-~Vynn}GfKCfK&V5{y}sxKMHaP_!|AUm z(gKu}c#3_EE+&vV5n8*<}dOG|&qcqs?DMZw5|a1rAnZ%fM3Pbx~9j^BgkU5Qo>yQULiN9#?PvRpUa zZ(D6b`pLNlGMTbzXg+LJ{16=~Lo`@FTf;NgYFB?<1YILZ65oOxbOJ;o{ddX|738F* z4sVs59UNU4-#R#(y(DBil}*JyCKUghYOVV&H$16^Rf+ufRMK*RmcA9I%;mbv4Y0vK zH|%>tMu2{T_xk5+wr$lFMzBj|5(nfFCTM}Lqt)6bMyFM_uSUFq^(+|&~F~xz1baD?SSHmZw1yR5sHa_=yG@dfYJRDSJ8#dMODDT zz#{)KYM}hNA;yl5FZ+=dr)UF;rh_0$y_e}+<0909POI3wm`$P_?g&+t4TR5Ol}VSt zX<+WD7bidex5Ud8_u67EVY#*T?9ZC)G(FPQ4ZN3oex5YTg;La+XhXbTM{&?rzw=(d z_6rQhRf&gZxY`TH=18LSf}&GNwoAzxm84@D_ssQAZwRqqv_N=erMgTI3f)ViWYz*U zO626y_fsS}MewF5b{n51wjSJ;3E~~AH$7}@AL1H>gYyIF%3C-I*>~KiUGooc)!lTK znvIL1i=W)rCdY4E&xWe#?O}!<`>>=6?K-{F8R|6`X76gK=H+PSqW=Qbp!&E%KNCvF4IQbW@<|hHh|(OPCw?BZCHZhh{V|F|KkJOqrCwq#@@D=V$BUZME7EmFfMYk`)6jE5YxR1TpmGKp`>VHbfV z3o+qtDDVT5W@}Q?MuzN=o`%RI!6FQXgtMK^nUfWQxo6WBbj>$M&ncElK5C)|rC_-P zn~}S^@3WJ*&CMNfRI`g5j7mizc&ejOK8W=wW|dqB$;V(-Nn9#e=Sj+99Nbk&Ls@>4 zMruxC#K&82PMXOK5F13(!A#C-*EIm)=AZ9*4c;}x$^1tuL^U4o)q-Z84>==|@e$eOpv|a@HjkO}WAIEC@hIGRZ*44HK zMT@aLAc&vr<8!?YoT_5r?F7G+)(=y;jbELSfEbw$JJ3!UgRTHdvfvOHV85G2e|Ak? znnwRq{%9fnt@tN9{?b(WEjXa5{%W!Ot^6l7_(Gun7Hd#d15*A?r~j?`Cwcb59sZVb z(DMF0?>{k#e;fSsa`eKv{g!^%zYYFo;QqGqXGQiut+2iNk5+zNtNym~XVvmjz5Nzw zv|m>KOD*@eoj()HOO5kes6dDE1>e6bp1&>p83|t!&~KT*{155qZ}mR|#7nsRE&5o0 qsQ){7{;mC|!@dNO-x3PK_CJIaC0S_D=7WJDfPPD$*vLCsD?@ zSKkdG>&F=Mb|0h;H@cC|6$kPNG7soV2`RQaMcI}ylxV}g^aY!U-w@otP0=W#1$k6P zEm-jJT@KUwW0RZ`63@`)J2&(s#HRQNmIlfQ=NPi1#9ra`s65o`$y-tQCJjn>{Jtuc zWoD%jGXZmU`^U|pfC@qUeVODd-$fV$E+f8U@86rse}!A9C82qY(dzBT9|8;#ieax> zy*8pOdm@e}@`Gk~qFuYL^OI@&1bG_1U9^Sbqp!Y|R+x0YAKI%# z4rV8c**k>F**I*AO(qMvN|`E6HC1szxCm>1YzzPZ7ZU)`{NK8g@Bdx-e|6;}dq*25 zD+>!}C!X`m&!~*+aGX&FKS_m8`u0v(y<+FW>V??~b!TOm#gyJ5SG>z5ui|sEcT1Te zmXAyrCJavXx_W5w3OsAS;U*-aqMKy_D;`fbBQ%eEK33+MtnYnV?v7s|qubUZ)YB=e zo=+G8@yOph&yh9VQD81UZRIdQ;bKyl5-;$}?GCNi%8ZCb4^%dRIuOX+gb7B>%er)u!CgLV~NzN*VX0YVQ)nK)@n{&j1 zbg^w>s+SJs2w$5OCZ58g4ct=n-5uWGdS$;k`tI0}62AWR-9n4hEAiW{tBn?+-Y@3I z=DXd;u6LN<4s-DAp40zi@q=khwdymRX!D2ABa(qLI$$h^^=wkia7&p0+lp^Xz5EDiwLV^yBLT^;Ms&y zpeyG7yS4#!k3ZsU*gEX&QYj~sfT%J=eSIh@7{CsthNPglPzuEeLiJ&IJC2eN6;*LL z_4^1ymr6t;I zWfAt3l86Nl$6-+xl8u*6KN9lT*%cXY#KS6cS7(54qW6sBt-=5|xlJWw)68lMGWV~# z9M^=NERyg`83|wb!6P3+|{X+>_ElcS4T0)?&5D06O0empa_o3mO(e>vi*xK_cX=jhlp3Z zdQ$A{)ix0XT ziO}v%bMQe)ytQeuF|5GsSchQ;C|_e^GwSi^K6;Zdzrt&AXvl zE}tjIIc&g($IFbXW}6oDRGQ#RbTDHzZO{uw+~^tqJspT0ciS2V?+ufruoq8Z`{egd z5FfvPoRLeKh#ed2A{;l$lzM-7s`3D9_8_D)qMmZUO?FsiO|mt){V-%O>*^PC;}EU= z=h4E{xsmbRQj(D-Y81Sb`{G;bmB}fraAVmaOWgWv@q>sfT|)YZMh0p>^Nz>c-1cIogT%YR)cg;Ztsw+etyD+wqk%{QbGyS#dzo>n^Cp zMqw!XY66bMy?qy*rdm|MM+yxug&Fp=tt$y40>xY?E+Nwc3Rm|;?go)su55KRO2iE6 z6%P{mG@h(9FwPyZfKqWb>`IQ~dEZ6mWP02tS(sd95?sw%=(*(%nVwmqMX_v06Pn9( z*G9j&Y5OMS=*?$ujI+4udCm;~hE`FD*nf(fKW5!|^qgm1*hf|TCSEYk6q#!f@8jr- zOC;8#VW5c{5)Bb*kYN@^)U(M+Tk=cXMm@(}!p9aTNvsc9A&NH4bK;naiUtx1CQVrq zcKZ#o@)-|Go-lKgrxgm#$Fqbkvk+ul;$&n|rb0O5GWyY2wKfz$qoaDhQ=8Fv@?3e% zvBw<3Ifi#?Ctc2Gka6`uaj{{vrBAA2y+*6A@Aa4(%MVw^vNk35808%a1-zvCN2x_q zC~U142;E}$b0-91>{67WlZPGch`&!B@}Jb>WOdik!u*!Al4y&YO)I;~9Jneji z)y%%_rTYo;1cJh{D+v19I`innU{T^bK72HbefG|M>odkHNtV8^C&bxn-$=C;&tBD= z%Fb4#i%2e}IM}@ZbokR{nJUu0+w3V2T5kL`@HBU0`kLRq-c$jfkUNMkz&=# zCrmbF(jVwQTdz@Ck7H{zNDN=jhX-P>Ki>3y&zkE1pg~O1&DK0)v&((E(WyiQNC{&q z7G4(Et(u{PvQZ3ZDVci25V(*E8DZ|2JJ8(gFVlv8 z(7fj4kd;4ih=OQe!~5jsYM0pQ*0I0X@Q6b&)&;ziW=Nofzu8LqtTJ(JUi`pW2NGw< z-8?s=tb)50bE@jL(m<{tCy$^DE#%d||8Q+mfO`#iiMGMSjwqp^SixfDE5JH{r1cpH zR1imaq?UH8?uBSkfqj5HVw>*C`b)Xj(i_Q23__G|PkZAqWfqR^#$$ETWDP?dzv!=J z`*vh3q4FLx=gX)vlhu{=uSXtSllKC0=blI-*vW510dlwu;yZ@hWhNuJGfzcqFKbo6 zp4iFYFQ~2=F+l)VPo=}_n7RE6B9@vbU>-74g1f9rs%s{`qq{{?gt1B-N{nv$t(MN9 zm}quAIN|b-&(MOD!KYZRe~kxI;%x@Ov!^%XJ&ZLvT)wz z?x_?I0MN$$QveeFGXS04Y%R_s?~~pHGLe_u%dggd{-{+Y#>ECp=M!`GgfnG#<^&CI z<7@8-JF%sK4XSAA_pdE<4egAuQX$@p*W22H@uHLhi-<86D5dePeuSP(#Fgz9$BCqr3V6PU`n?2wvo)}W#`?#mG} zLt4d4tHLTWpFhlBjeO2=1zfRyUv8T^e6G4rFBTbTqvCAvEvY-Qu$?0P5JRQsu9uvW zw$qnEN`T?5Hh%7^Rd+oRln(UPGH0ht#+AYOT-oW}a=`pNP6pt>fKroI1;4-1|8>Fh zIA#)RcvSA-efc9FePQe12Q_g#Fr5hR^4tlSac$&-ps4(VG=J zJi$rUx1<-8*fc}yW{AwjAxKY5Tm!Cnz#~#(c_QOfp>zwXNn4t9h=xFl-{VCdre@5GxRzTR>+nhJd(Gy5NUaBW=;QP|< zh1iRiulgC3RJoR{!Qc9fdMTTED;rWxo=B3QddwNGzYMI?NG;>ZbIH>k^x5uD2xT6K zlZRJP^sn)M+pUi0RG!1+_q%4p09D#FPnOEBE4aRvFCiZ%ED5y$>E3v?c-3klrvm72 zioYswNyO#x3J0#*0V#u(g5`r_HNog7NqDbln~M|}Mc>^qvEiL2c#x(q{n)g#)6}Y_ zw%NKPdA9;|3+bXEi>aPI-;2kE3=n{rWyW70{Dc)rac3FpdRU&TsuOj({dd zE#xRCOfR;@jt{cv`6y1c=3TLUbk1pHR2)kO&tTF-YKtpZD{d5h5H&LMjk4t6dUK!0 zr!CJ@@p67RjCOfm9Ro!$AmTYKBLuD$y$7s#X;?!s>_A$rGt*$#0!{ap6DlV;urX?} zyuEdJO`V4ih;J?Z&a8L#aS!>1`Lcq<>G_w*`LwfSj4nd9e>~Hm`T1pj3g3(0=e!8l zMm64V<|RP4;u~;h)1*wZP_3L;*i!ii`i9{fR}zL>`$y;37hx4ynw>-ztc=v2r$L+U z+mc~3KBy;C6gmocz*@YAo3(yJdxv`%SjBsK&l)rQsPTllyWN8_7T(DrgPHl>M+4ig zK(d%;9r|?wf+g#n8E+;yCAw3WkF3&L7{~1MaZ&ds@wR+sS2i^CwOS1Jz8h3XOCX88 zEtI~klidZ&ln|BwL2*d! z`FcUyII8-{GXA@vVv0xY0_GWGrZVME&7l3zgpT9CFWHySZ~S!AKlO09?|#!^K*{25$kllwac!>4-zhN>w$GBK&c`nTta}9bm2*kViS4X01j`q6?@8s&|rX4(wPl|U^XJL#sFuPgY zsyzBOD(oPFVkz_XDA$cWQE%$;Qa@4=?AGg?22bDL8%XSUN2siTe5g|*mbJ-%)j`7r z-p4RjWQTBNCbb5F7-Ea0Ve~{B0Xy93s{Vryik9c_IP~Ln#F6{gSE6JH1CXQcU3A%brNjP@Ue&nc3n=sYb_;56DV!8wA-l08nx)XPJwE)`T zx##^929O-^>!|kP$>Z5k?O*Nh4sJD7|EDOb=z0GG%DK+|9pz%z{MWU_pfdxF@_Vm$ zeq8rE%Ej*SOzZzLTC^hhE6RWDw|@t`*jD^?SqA8$^H;!ME$Htk7j^kxmlcWsJIaM+ z{c1>l2fV1W&gA?ri$PNbdRhNP-2aYnv38#6v0p|>^b_IV>g@NS7s>ssxc;(0@V|HS zuTuN_;EQ~B#{a(zLV7;quXI2Xy}j2(ZO3YtMKaOLj)9)cv!{OmyO>}U diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java index ed4ecbb..4216376 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/BasicPassUpdaterIT.java @@ -61,7 +61,6 @@ public class BasicPassUpdaterIT { private final List> resultSet = new ArrayList<>(); private static final String DOMAIN = "default.domain"; private static final String employeeidPrefix = DOMAIN + ":employeeid:"; - private static final String jhedPrefix = DOMAIN + ":jhed:"; private final CoeusPassEntityUtil passEntityUtil = new CoeusPassEntityUtil(); private final Map funderPolicyUriMap = new HashMap<>(); @@ -123,9 +122,7 @@ public void setup() { rowMap.put(C_USER_MIDDLE_NAME, C_USER_MIDDLE_NAME + i); rowMap.put(C_USER_LAST_NAME, C_USER_LAST_NAME + i); rowMap.put(C_USER_EMAIL, C_USER_EMAIL + i); - //rowMap.put(C_USER_INSTITUTIONAL_ID, C_USER_INSTITUTIONAL_ID + i); rowMap.put(C_USER_EMPLOYEE_ID, C_USER_EMPLOYEE_ID + i); - //rowMap.put(C_USER_HOPKINS_ID, C_USER_HOPKINS_ID + i); rowMap.put(C_UPDATE_TIMESTAMP, "2018-01-01 0" + i + ":00:00.0"); rowMap.put(C_ABBREVIATED_ROLE, (i % 2 == 0 ? "P" : "C")); @@ -283,9 +280,6 @@ public void updateGrantsIT() throws InterruptedException { //employeeId and localKey were localized by the grant loader user.getLocatorIds().add(employeeidPrefix + C_USER_EMPLOYEE_ID + i); - //String hopkinsidPrefix = DOMAIN + ":hopkinsid:"; - //user.getLocatorIds().add(hopkinsidPrefix + C_USER_HOPKINS_ID + i); - //user.getLocatorIds().add(jhedPrefix + C_USER_INSTITUTIONAL_ID.toLowerCase() + i); user.setFirstName(C_USER_FIRST_NAME + i); user.setMiddleName(C_USER_MIDDLE_NAME + i); user.setLastName(C_USER_LAST_NAME + i); diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java index 7fb6fde..c30e138 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/HarvardPilotPassUpdaterIT.java @@ -22,7 +22,7 @@ import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.Policy; import org.dataconservancy.pass.model.User; -import org.junit.Before;; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -51,7 +51,7 @@ public class HarvardPilotPassUpdaterIT { String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; // String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; - String[] userEmployeeId= { "jpubli1@harvard.edu", "ssinis11@harvard.edu", "rsquir1@harvard.edu"}; + String[] userEmployeeId= { "A0000001", "A0000002", "A0000003"}; String[] userFirstName = {"John", "Simon", "Rocket"}; String[] userLastName = { "Public", "Sinister", "Squirrel" }; String[] userEmail = { "jpubli1@harvard.edu", "ssinis11@harvard.edu", "rsquir1@harvard.edu" }; @@ -60,7 +60,7 @@ public class HarvardPilotPassUpdaterIT { String directFunderPolicyUriString; String grantIdPrefix = "harvard.edu:grant:"; - String employeeidPrefix = "harvard.edu:employeeid:"; + String institutionalIdPrefix = "harvard.edu:jhed:"; @@ -93,9 +93,12 @@ public void setup() { /** * The behavior of PassUpdate's updatePass() method is to compare the data coming in on the ResultSet with * the existing data in Pass, and create objects if Pass does not yet have them, and update them if they exist in Pass but - * there are differences in the fields for which COEUS is the authoritative source, or COEUS has a clue about other fields which are null + * there are differences in the fields for whichthe pull is the authoritative source, or else has a clue about other fields which are null * on the PASS object. * + * For the Harvard Pilot, we use the user name on the email address to stand for the user's eppn fpr the purposes of populating that + * locator id field + * * @throws InterruptedException - the exception */ @Test @@ -112,11 +115,11 @@ public void updateGrantsIT() throws InterruptedException { passUpdater.updatePass(resultSet, "grant"); sleep(10000); - URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[0] ); + URI passUser0Uri = passClient.findByAttribute(User.class, "locatorIds", institutionalIdPrefix + userEmail[0].split("@")[0] ); assertNotNull( passUser0Uri ); URI passGrantUri = passClient.findByAttribute(Grant.class, "localKey", grantIdPrefix + grantLocalKey[2]); assertNotNull( passGrantUri ); - URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[1] ); + URI passUser1Uri = passClient.findByAttribute(User.class, "locatorIds", institutionalIdPrefix + userEmail[1].split("@")[0] ); assertNotNull( passUser1Uri ); Grant passGrant = passClient.readResource( passGrantUri, Grant.class ); @@ -154,11 +157,11 @@ public void updateGrantsIT() throws InterruptedException { resultSet.add(piRecord2); passUpdater.updatePass(resultSet, "grant"); - sleep(10000); + sleep(12000); passGrant = passClient.readResource( passGrantUri, Grant.class ); - URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", employeeidPrefix + userEmployeeId[2] ); + URI passUser2Uri = passClient.findByAttribute(User.class, "locatorIds", institutionalIdPrefix + userEmail[2].split("@")[0] ); assertNotNull( passUser2Uri ); assertEquals( grantAwardNumber[1], passGrant.getAwardNumber() );//earliest of the additions @@ -178,7 +181,7 @@ public void updateGrantsIT() throws InterruptedException { * @param iteration the iteration of the (multi-award) grant * @param user the user supplied in the record * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") - * @return + * @return the row map for the pull record */ private Map makeRowMap( int iteration, int user, String abbrRole) { Map rowMap = new HashMap<>(); @@ -196,7 +199,7 @@ private Map makeRowMap( int iteration, int user, String abbrRole rowMap.put(C_USER_FIRST_NAME, userFirstName[user]); rowMap.put(C_USER_LAST_NAME, userLastName[user]); - rowMap.put(C_USER_EMAIL, userEmail[user]);; + rowMap.put(C_USER_EMAIL, userEmail[user]); rowMap.put(C_USER_EMPLOYEE_ID, userEmployeeId[user]); rowMap.put(C_ABBREVIATED_ROLE, abbrRole); diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java index c662f3c..cd8cb40 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassInitUpdaterIT.java @@ -97,7 +97,7 @@ public void setup() { * interesting fields are the investigator fields: all CO-PIs ever on the grant should stay on the * co-pi field throughout iterations. If a PI is changed, they should appear on the CO-PI field * - * @throws InterruptedException + * @throws InterruptedException from joda data time creation */ @Test public void processInitGrantIT() throws InterruptedException { @@ -179,7 +179,7 @@ public void processInitGrantIT() throws InterruptedException { * @param iteration the iteration of the (multi-award) grant * @param user the user supplied in the record * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") - * @return + * @return the row map for the record */ private Map makeRowMap( int iteration, int user, String abbrRole) { Map rowMap = new HashMap<>(); diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java index 9434e68..de0650e 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java @@ -23,7 +23,7 @@ import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.Policy; import org.dataconservancy.pass.model.User; -import org.junit.Before;; +import org.junit.Before; import org.junit.Test; import java.net.URI; @@ -97,7 +97,7 @@ public void setup() { * interesting fields are the investigator fields: all CO-PIs ever on the grant should stay on the * co-pi field throughout iterations. If a PI is changed, they should appear on the CO-PI field * - * @throws InterruptedException + * @throws InterruptedException from joda date time creation */ @Test public void processGrantIT() throws InterruptedException { @@ -180,7 +180,7 @@ public void processGrantIT() throws InterruptedException { * @param iteration the iteration of the (multi-award) grant * @param user the user supplied in the record * @param abbrRole the role: Pi ("P") or co-pi (C" or "K") - * @return + * @return row map for pull record */ private Map makeRowMap( int iteration, int user, String abbrRole) { Map rowMap = new HashMap<>(); From cd1bc460195c463da11690eb3b9cd0c227061ae5 Mon Sep 17 00:00:00 2001 From: jrmartino Date: Sun, 10 May 2020 13:32:32 -0400 Subject: [PATCH 08/14] clean up some warnings, add some comments --- .../pass/grant/cli/BaseGrantLoaderApp.java | 22 ++++++++++--------- .../pass/grant/cli/DataLoaderErrors.java | 3 +++ .../pass/grant/cli/EmailService.java | 6 ++--- .../pass/grant/cli/JhuGrantLoaderApp.java | 3 +-- .../pass/grant/cli/EmailServiceTest.java | 18 ++++++++------- .../pass/grant/data/BasicPassEntityUtil.java | 4 ++++ .../pass/grant/data/CoeusConnector.java | 4 ++-- .../pass/grant/data/CoeusPassEntityUtil.java | 5 +---- .../grant/data/CoeusPassInitEntityUtil.java | 9 ++++---- .../grant/data/HarvardPilotConnector.java | 6 ++--- .../pass/grant/data/PassEntityUtil.java | 5 +++++ 11 files changed, 49 insertions(+), 36 deletions(-) diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/BaseGrantLoaderApp.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/BaseGrantLoaderApp.java index 35032f8..446af89 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/BaseGrantLoaderApp.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/BaseGrantLoaderApp.java @@ -34,6 +34,7 @@ import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.sql.SQLException; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; @@ -50,27 +51,28 @@ * use the PassLoader to take {@code List} representing the {@code ResultSet} to push this data into our PASS instance * via the java pass client. * + * * A large percentage of the code here is handling exceptional paths, as this is intended to be run in an automated * fashion, so care must be taken to log errors, report them to STDOUT, and also send email notifications. * * @author jrm@jhu.edu */ abstract class BaseGrantLoaderApp { - private static Logger LOG = LoggerFactory.getLogger(BaseGrantLoaderApp.class); + private static final Logger LOG = LoggerFactory.getLogger(BaseGrantLoaderApp.class); private EmailService emailService; - private File appHome; + private final File appHome; private String startDate; - private String awardEndDate; + private final String awardEndDate; private File updateTimestampsFile; - private boolean email; - private String mode; - private String action; - private String dataFileName; + private final boolean email; + private final String mode; + private final String action; + private final String dataFileName; private boolean local = false; - private boolean timestamp = true; + private boolean timestamp = false; - private String updateTimestampsFileName; + private final String updateTimestampsFileName; /** * Constructor for this class @@ -247,7 +249,7 @@ void run() throws PassCliException { try (FileInputStream fis = new FileInputStream(dataFile); ObjectInputStream in = new ObjectInputStream(fis) ) { - resultSet = (List>)in.readObject(); + resultSet = Collections.unmodifiableList((List>) in.readObject()); } catch (IOException | ClassNotFoundException ex) { ex.printStackTrace(); } diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/DataLoaderErrors.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/DataLoaderErrors.java index eaa7330..fd67083 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/DataLoaderErrors.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/DataLoaderErrors.java @@ -16,6 +16,9 @@ package org.dataconservancy.pass.grant.cli; +/** + * A class containing all error strings for errors caught by the Loader Apps + */ class DataLoaderErrors { static String ERR_HOME_DIRECTORY_NOT_FOUND = "No home directory found for the application. Please specify a valid absolute path."; static String ERR_HOME_DIRECTORY_NOT_READABLE_AND_WRITABLE = "Supplied home directory must be readable" + diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/EmailService.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/EmailService.java index a4f7093..aebdba6 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/EmailService.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/EmailService.java @@ -31,12 +31,12 @@ import org.slf4j.LoggerFactory; /** - * An email service for reporting errors or results of running the {@code CoeusGrantLoaderApp} + * An email service for reporting errors or results of running a GrantLoaderApp * @author jrm@jhu.edu */ class EmailService { - private static Logger LOG = LoggerFactory.getLogger(EmailService.class); - private Properties mailProperties; + private static final Logger LOG = LoggerFactory.getLogger(EmailService.class); + private final Properties mailProperties; /** * The constructor diff --git a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java index 6e19756..754ff3f 100644 --- a/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java +++ b/pass-grant-cli/src/main/java/org/dataconservancy/pass/grant/cli/JhuGrantLoaderApp.java @@ -15,14 +15,13 @@ */ package org.dataconservancy.pass.grant.cli; - import org.dataconservancy.pass.grant.data.*; import java.util.Properties; class JhuGrantLoaderApp extends BaseGrantLoaderApp { - boolean init = false; + boolean init; JhuGrantLoaderApp(String startDate, String awardEndDate, boolean email, String mode, String action, String dataFileName, boolean init) { super(startDate, awardEndDate, email, mode, action, dataFileName); diff --git a/pass-grant-cli/src/test/java/org/dataconservancy/pass/grant/cli/EmailServiceTest.java b/pass-grant-cli/src/test/java/org/dataconservancy/pass/grant/cli/EmailServiceTest.java index e5f3642..050a6df 100644 --- a/pass-grant-cli/src/test/java/org/dataconservancy/pass/grant/cli/EmailServiceTest.java +++ b/pass-grant-cli/src/test/java/org/dataconservancy/pass/grant/cli/EmailServiceTest.java @@ -19,7 +19,6 @@ import com.icegreen.greenmail.util.GreenMail; import com.icegreen.greenmail.util.ServerSetupTest; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -28,13 +27,16 @@ import java.io.IOException; import java.util.Properties; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * Test classs for email service * @author jrm@jhu.edu */ public class EmailServiceTest { - private Properties mailProperties = System.getProperties(); + private final Properties mailProperties = System.getProperties(); private EmailService underTest; private GreenMail testServer; @@ -63,7 +65,7 @@ public void setup() throws InterruptedException { if (!started) { // try one more time - Thread.sleep(5000l); + Thread.sleep(5000L); testServer.start(); } @@ -80,14 +82,14 @@ public void testSendMessage() throws MessagingException, IOException { String messageSubject = "TEST"; underTest.sendEmailMessage(messageSubject, messageBody); // Check that only one message was sent - Integer numMessages = testServer.getReceivedMessages().length; - Assert.assertTrue("Expected only one message, got " + numMessages, numMessages == 1); + int numMessages = testServer.getReceivedMessages().length; + assertEquals("Expected only one message, got " + numMessages, 1, numMessages); // Check that the message is just a plaintext message MimeMessage message = testServer.getReceivedMessages()[0]; - Assert.assertTrue("Subject of message was not correct", message.getSubject().equals(messageSubject)); - Assert.assertTrue("Content of message was not a string as expected", message.getContent() instanceof String); - Assert.assertTrue(message.getContent().toString().contains(messageBody)); + assertEquals("Subject of message was not correct", message.getSubject(), messageSubject); + assertTrue("Content of message was not a string as expected", message.getContent() instanceof String); + assertTrue(message.getContent().toString().contains(messageBody)); } @After diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java index 793f366..0617639 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/BasicPassEntityUtil.java @@ -31,6 +31,10 @@ * data from the source pull. Stored objects are evaluated and updated according to the PassEntityUtil implementation * supplied to this updater * + * + * This basic implementation deals with a smallish set of fields - other implementations will have to address more + * fields if they are to be stored on PASS objects. + * * @author jrm@jhu.edu */ diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java index 49930f6..b0184e6 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusConnector.java @@ -307,7 +307,7 @@ private String buildGrantQueryString(String startDate, String awardEndDate){ } private String buildUserQueryString(String startDate) { - String viewFields [] = { + String[] viewFields = { C_USER_FIRST_NAME, C_USER_MIDDLE_NAME, C_USER_LAST_NAME, @@ -333,7 +333,7 @@ private String buildUserQueryString(String startDate) { private String buildFunderQueryString() { - String viewFields [] = {//doesn't matter whether the funder is primary or direct - these are the column names in the SWIFT_SPONSOR view + String[] viewFields = {//doesn't matter whether the funder is primary or direct - these are the column names in the SWIFT_SPONSOR view C_PRIMARY_FUNDER_NAME, C_PRIMARY_FUNDER_LOCAL_KEY }; diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java index 8107f70..28554ec 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java @@ -194,10 +194,7 @@ private Grant updateGrant(Grant system, Grant stored) { system.getCoPis().add ( storedPi ); } } - - if (system.getCoPis().contains(system.getPi())) { - system.getCoPis().remove(system.getPi()); - } + system.getCoPis().remove(system.getPi()); stored.setPi( system.getPi() ); stored.setCoPis( system.getCoPis() ); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java index b7adb0f..c5feef2 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java @@ -6,6 +6,10 @@ import java.net.URI; import java.util.HashSet; +/** + * This subclass is for the special case where we need to correct information on existing PASS objects for which we + * normally consider the PASS information to be authoritative. + */ public class CoeusPassInitEntityUtil extends CoeusPassEntityUtil { @Override @@ -66,10 +70,7 @@ private Grant updateGrant(Grant system, Grant stored) { system.getCoPis().add ( storedPi ); } } - - if (system.getCoPis().contains(system.getPi())) { - system.getCoPis().remove(system.getPi()); - } + system.getCoPis().remove(system.getPi()); stored.setPi( system.getPi() ); stored.setCoPis( system.getCoPis() ); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java index b44b909..e510ee5 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/HarvardPilotConnector.java @@ -49,7 +49,7 @@ public class HarvardPilotConnector implements GrantConnector { protected static final String HARVARD_DATA_FILE_PATH_PROPERTY = "harvard.data.file.path"; private String xlsxDataFilePath; - private Properties funderPolicyProperties; + private final Properties funderPolicyProperties; private static final Logger LOG = LoggerFactory.getLogger(HarvardPilotConnector.class); @@ -99,7 +99,7 @@ public List> retrieveUpdates(String queryString, String mode Map rowMap = new HashMap<>(); rowMap.put(C_PRIMARY_FUNDER_LOCAL_KEY, localKey.toString()); rowMap.put(C_PRIMARY_FUNDER_NAME, funderNameMap.get(localKey.toString())); - if (funderPolicyProperties.keySet().contains(localKey)) { + if (funderPolicyProperties.containsKey(localKey)) { rowMap.put(C_PRIMARY_FUNDER_POLICY, funderPolicyProperties.getProperty(localKey.toString())); } resultSet.add(rowMap); @@ -157,7 +157,7 @@ public List> retrieveUpdates(String queryString, String mode * Stringify a cell's contents. Since our numerical cells which are dates are all integers but are interpreted * by the POI framework as doubles, we correct these to integers * - * @param cell + * @param cell spreadsheet cell * @return a string representing a cell's contents */ private String stringify(Cell cell) { diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/PassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/PassEntityUtil.java index a839d2b..7ed2f4a 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/PassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/PassEntityUtil.java @@ -20,6 +20,11 @@ import org.dataconservancy.pass.model.Grant; import org.dataconservancy.pass.model.User; +/** + * This interface defines update methods for existing (stored) grants, users and funders. In practice, implementations will generally + * split this functionalite into two steps - the first step will reason over a stored object and the object developed in a system pull, + * and make th decision whether the object needs to be updated. The update methods eill act on that decision appropriately. + */ public interface PassEntityUtil { /** From 13a8d6a12ca80da4be9bc2ad0bd80133d407bc11 Mon Sep 17 00:00:00 2001 From: jrmartino Date: Sun, 10 May 2020 15:40:01 -0400 Subject: [PATCH 09/14] typo --- JHU-README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JHU-README.md b/JHU-README.md index 0915260..e18cee2 100644 --- a/JHU-README.md +++ b/JHU-README.md @@ -103,7 +103,7 @@ will therefore look like this: `8675309:policies/e7/3f/26/70/e73f2670-6ef6-4201-bbcd-04631a93d852` -The `policy.proerties` file will have to be kept up to date if additional funders are assigned policies. A different +The `policy.properties` file will have to be kept up to date if additional funders are assigned policies. A different mechanism will be needed to add additional policies, as we require the policies reference in the properties file to be already present in the system. @@ -165,4 +165,4 @@ must be filled out accordingly. We also have a command line option -m to pass in The processing of the ResultSet is straightforward - we simply construct a set of hash maps which represent the column names and the values for each record. We do not assume that the PASS objects in Fedora are updated only by this application, as there may be some fields on these objects which are not known to COEUS, but -may be populated by other applications eventually (for example ORCID on User, Submissions on Grant, or Policy on Funder). +may be populated by other applications eventually (for example ORCID on User, Submissions on Grant, or Policy on Funder). \ No newline at end of file From 76380c8b63dfe92cdf80bb13904761d09581f853 Mon Sep 17 00:00:00 2001 From: jrmartino Date: Thu, 14 May 2020 14:03:00 -0400 Subject: [PATCH 10/14] update dependency versions and context.jsonld --- .../test/resources/docker/mnt/context.jsonld | 6 +++- pom.xml | 30 +++++++++---------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/pass-grant-integration/src/test/resources/docker/mnt/context.jsonld b/pass-grant-integration/src/test/resources/docker/mnt/context.jsonld index ea7e7b0..3ead31c 100644 --- a/pass-grant-integration/src/test/resources/docker/mnt/context.jsonld +++ b/pass-grant-integration/src/test/resources/docker/mnt/context.jsonld @@ -20,7 +20,7 @@ "abstract": {"@id": "pass:abstract"}, "accessUrl": {"@id": "pass:accessUrl", "@type": "@id"}, - "affiliation": {"@id": "pass:affiliation"}, + "affiliation": {"@id": "pass:affiliation", "@container": "@set"}, "agreementText": {"@id": "pass:agreementText"}, "aggregatedDepositStatus": {"@id": "pass:aggregatedDepositStatus"}, "awardDate": {"@id": "pass:awardDate", "@type": "xsd:dateTime"}, @@ -35,6 +35,7 @@ "directFunder": {"@id": "pass:directFunder", "@type": "@id"}, "displayName": {"@id": "pass:displayName"}, "doi": {"@id": "pass:doi"}, + "effectivePolicies": {"@id": "pass:effectivePolicies", "@container": "@set", "@type": "@id"}, "email": {"@id": "pass:email"}, "endDate": {"@id": "pass:endDate", "@type": "xsd:dateTime"}, "eventType": {"@id": "pass:eventType"}, @@ -77,6 +78,7 @@ "repositoryKey": {"@id": "pass:repositoryKey"}, "repositories": {"@id": "pass:repositories", "@container": "@set", "@type": "@id"}, "roles": {"@id": "pass:roles", "@container": "@set"}, + "schemas": {"@id": "pass:schemas", "@container": "@set", "@type": "@id"}, "source": {"@id": "pass:source"}, "startDate": {"@id": "pass:startDate", "@type": "xsd:dateTime"}, "submission": {"@id": "pass:submission", "@type": "@id"}, @@ -84,6 +86,8 @@ "submitted": {"@id": "pass:submitted", "@type:": "xsd:boolean"}, "submittedDate": {"@id": "pass:submittedDate", "@type": "xsd:dateTime"}, "submitter": {"@id": "pass:submitter", "@type": "@id"}, + "submitterEmail": {"@id": "pass:submitterEmail", "@type": "@id"}, + "submitterName": {"@id": "pass:submitterName"}, "title": {"@id": "pass:title"}, "uri": {"@id": "pass:uri", "@type": "@id"}, "url": {"@id": "pass:url", "@type": "@id"}, diff --git a/pom.xml b/pom.xml index 3912ce5..628f492 100644 --- a/pom.xml +++ b/pom.xml @@ -51,29 +51,29 @@ UTF-8 22 2.33 - 3.0.0 - 1.9.3 - 1.12 - 1.6 - 4.1.0 + 3.1.0 + 1.9.4 + 1.14 + 1.8 + 4.1.2 0.30.0 - 1.5.10 + 1.5.13 1.6.2 - 4.12 + 4.13 1.2.3 3.8.1 2.22.2 - 3.2.1 + 3.2.3 2.22.2 - 3.3.2 + 3.4.0 2.8.2 - 2.27.0 - 0.6.0 - 0.6.0 - 0.6.0 + 3.3.3 + 0.7.0 + 0.7.0 + 0.7.0 12.2.0.1 - 0.8.12 - 1.7.26 + 0.8.13 + 1.7.30 3.4.6 From 2d28b721298e2cf62fbfc39cfccaba8e6bb010c5 Mon Sep 17 00:00:00 2001 From: jrmartino Date: Wed, 20 May 2020 20:28:26 -0400 Subject: [PATCH 11/14] update version to 1.4.0 --- pass-grant-cli/pom.xml | 2 +- pass-grant-data/pom.xml | 2 +- pass-grant-integration/pom.xml | 2 +- pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pass-grant-cli/pom.xml b/pass-grant-cli/pom.xml index 159fce1..8609c67 100644 --- a/pass-grant-cli/pom.xml +++ b/pass-grant-cli/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.4.0-SNAPSHOT + 1.4.0 4.0.0 diff --git a/pass-grant-data/pom.xml b/pass-grant-data/pom.xml index 544f460..40085e9 100644 --- a/pass-grant-data/pom.xml +++ b/pass-grant-data/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.4.0-SNAPSHOT + 1.4.0 4.0.0 diff --git a/pass-grant-integration/pom.xml b/pass-grant-integration/pom.xml index 76e6bfd..5c92c8e 100644 --- a/pass-grant-integration/pom.xml +++ b/pass-grant-integration/pom.xml @@ -21,7 +21,7 @@ pass-grant-loader org.dataconservancy.pass - 1.4.0-SNAPSHOT + 1.4.0 4.0.0 diff --git a/pom.xml b/pom.xml index 628f492..e295172 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ org.dataconservancy.pass pass-grant-loader pom - 1.4.0-SNAPSHOT + 1.4.0 pass-grant-cli From acd475ce8d62c5077c3c9166f901416971b8896f Mon Sep 17 00:00:00 2001 From: jrmartino Date: Thu, 21 May 2020 14:24:21 -0400 Subject: [PATCH 12/14] optimise directory lookup service to use caching --- .../pass/grant/data/DirectoryServiceUtil.java | 40 +++++++++++++++---- .../grant/data/DirectoryServiceUtilTest.java | 22 ++++++---- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java index 1336df1..0efeaa1 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtil.java @@ -25,6 +25,9 @@ import okhttp3.Response; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.Properties; import static java.util.concurrent.TimeUnit.SECONDS; @@ -33,7 +36,9 @@ * This util class is designed to hit a service which provides resolution of one identifier to another. The two endpoints * provide lookups between our Hopkins Id, which is a durable identifier for all members of the Hopkins community, and the * employee ID, which is a durable identifier for all Hopkins employees. This lookup service is necessary because we do not - * have access to the wider identifier in out grants data source. + * have access to the wider identifier in our grants data source. + * + * @author jrm */ class DirectoryServiceUtil { @@ -48,6 +53,10 @@ class DirectoryServiceUtil { private final OkHttpClient client; private final JsonFactory factory = new JsonFactory(); + //these are for caching results + private final Map hopkins2ee = new HashMap<>(); + private final Map ee2hopkins = new HashMap<>(); + DirectoryServiceUtil(Properties connectionProperties) { if (connectionProperties != null) { @@ -95,24 +104,40 @@ public String getQueryParameter() { /** - * Return Hopkins ID for a given employee ID + * Return Hopkins ID for a given employee ID. we cache lookups in a map so that we only need to perform + * a lookup once per session per user. * * @param employeeId the user's employeeId * @return the user's Hopkins ID - should never be null * @throws IOException if the service cannot be reached */ String getHopkinsIdForEmployeeId(String employeeId) throws IOException { - return askDirectoryForMappedValue(Type.EMPLOYEE2HOPKINS, employeeId); + String hopkinsId; + if ( !ee2hopkins.containsKey( employeeId) ) { + hopkinsId = askDirectoryForMappedValue(Type.EMPLOYEE2HOPKINS, employeeId); + ee2hopkins.put( employeeId, hopkinsId ); + } else { + hopkinsId = ee2hopkins.get ( employeeId ); + } + return hopkinsId; } /** - * + * Return employee ID for a given Hopkins ID. we cache lookups in a map so that we only need to perform + * a lookup once per session per user. * @param hopkinsId the user's Hopkins ID * @return the user's employee ID if it exists; null if it does not * @throws IOException if there is an IO exception */ String getEmployeeIdForHopkinsId(String hopkinsId) throws IOException { - return askDirectoryForMappedValue(Type.HOPKINS2EMPLOYEE, hopkinsId); + String employeeId; + if ( !hopkins2ee.containsKey( hopkinsId ) ) { + employeeId = askDirectoryForMappedValue(Type.HOPKINS2EMPLOYEE, hopkinsId); + hopkins2ee.put( hopkinsId, employeeId ); + } else { + employeeId = hopkins2ee.get ( hopkinsId ); + } + return employeeId; } private String askDirectoryForMappedValue(Type type, String sourceId) throws IOException { @@ -124,7 +149,7 @@ private String askDirectoryForMappedValue(Type type, String sourceId) throws IOE } serviceUrl = directoryBaseUrl + suffix; - HttpUrl.Builder urlBuilder = HttpUrl.parse(serviceUrl).newBuilder().addQueryParameter(name, sourceId); + HttpUrl.Builder urlBuilder = Objects.requireNonNull(HttpUrl.parse(serviceUrl)).newBuilder().addQueryParameter(name, sourceId); String url = urlBuilder.build().toString(); Request request = new Request.Builder().header("client_id", directoryClientId) @@ -132,7 +157,8 @@ private String askDirectoryForMappedValue(Type type, String sourceId) throws IOE Response response = client.newCall(request).execute(); - JsonParser parser = factory.createParser(response.body().string()); + assert response.body() != null; + JsonParser parser = factory.createParser(response.body().string()); String mappedValue = null; while (!parser.isClosed()) { JsonToken jsonToken = parser.nextToken(); diff --git a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtilTest.java b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtilTest.java index e976fdc..86dd258 100644 --- a/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtilTest.java +++ b/pass-grant-data/src/test/java/org/dataconservancy/pass/grant/data/DirectoryServiceUtilTest.java @@ -33,21 +33,29 @@ * This is a test class for a simple directory lookup service running at the endpoint specified by "serviceUrl" below * the service type completes the URL, and the client id and client secret are supplied as headers. * - * values for serviceUrl, clientId and slientSecret, must be supplied below. + * values for serviceUrl, clientId and clientSecret, must be supplied below. * * This test has been run against the running service with valid parameters and arguments supplied to the methods - this class has been * cleaned up after successful testing. because of the simplicity and isolation of this class, it does not need to be tested * every build - just when something about the service changes. so we ignore it for now. + * + * To test, provide real connection parameter values, and a real kopkins id / employee id pair + * + * @author jrm */ @Ignore public class DirectoryServiceUtilTest { private DirectoryServiceUtil underTest; + private final String validEeid = ""; //actual employee id + private final String validHopkinsId = ""; //actual matching hopkins id + @Before public void setup() { - String serviceUrl = "https://the.service/url"; + final String serviceUrl = "https://the.service/url"; final String clientId = "the-client-id"; final String clientSecret = "the-client-secret"; + Properties connectionProperties = new Properties(); connectionProperties.setProperty(DIRECTORY_SERVICE_BASE_URL, serviceUrl); connectionProperties.setProperty(DIRECTORY_SERVICE_CLIENT_ID, clientId); @@ -57,16 +65,14 @@ public void setup() { @Test public void testGetHopkinsId() throws java.io.IOException { - String result = underTest.getHopkinsIdForEmployeeId("supply valid argument here"); - Assert.assertEquals("expected value", result); - System.out.println(result); - + String result = underTest.getHopkinsIdForEmployeeId(validEeid); + Assert.assertEquals(validHopkinsId, result); } @Test public void testGetEmployeeId() throws java.io.IOException { - String result = underTest.getEmployeeIdForHopkinsId("A58756"); - Assert.assertEquals("expected value", result); + String result = underTest.getEmployeeIdForHopkinsId(validHopkinsId); + Assert.assertEquals(validEeid, result); } @Test From d01de2fab8ba6b8efddf3864e004432595a76fc4 Mon Sep 17 00:00:00 2001 From: jrmartino Date: Fri, 22 May 2020 22:05:09 -0400 Subject: [PATCH 13/14] remove duplicate assignment; make sure award status is the latest in the pull --- .../pass/grant/data/DefaultPassUpdater.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index 5ba6651..2fade3e 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -181,7 +181,6 @@ private void updateGrants(Collection> results) { //these values will not override existing stored values unless the PassEntityUtil implementation //allows it. if (startDate != null && (grant.getStartDate() == null || startDate.isBefore(grant.getStartDate()))) { - grant.setAwardDate(awardDate); grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); grant.setAwardNumber(rowMap.get(C_GRANT_AWARD_NUMBER)); grant.setDirectFunder(funderMap.get(directFunderLocalKey)); @@ -194,6 +193,22 @@ private void updateGrants(Collection> results) { //use !isBefore in case more than one PI is specified, need to process more than one if (endDate != null && (grant.getEndDate() == null || !endDate.isBefore(grant.getEndDate()))) { grant.setEndDate(endDate); + //status should be the latest one + String status = rowMap.getOrDefault(C_GRANT_AWARD_STATUS, null); + if (status != null) { + switch (status) { + case "Active": + grant.setAwardStatus(Grant.AwardStatus.ACTIVE); + break; + case "Pre-Award": + grant.setAwardStatus(Grant.AwardStatus.PRE_AWARD); + break; + case "Terminated": + grant.setAwardStatus(Grant.AwardStatus.TERMINATED); + } + } else { + grant.setAwardStatus(null); + } //we want the PI to be the one listed on the most recent grant iteration if ( abbreviatedRole.equals("P") ) { @@ -212,23 +227,6 @@ private void updateGrants(Collection> results) { } } } - - String status = rowMap.getOrDefault(C_GRANT_AWARD_STATUS, null); - - if (status != null) { - switch (status) { - case "Active": - grant.setAwardStatus(Grant.AwardStatus.ACTIVE); - break; - case "Pre-Award": - grant.setAwardStatus(Grant.AwardStatus.PRE_AWARD); - break; - case "Terminated": - grant.setAwardStatus(Grant.AwardStatus.TERMINATED); - } - } else { - grant.setAwardStatus(null); - } } //we are done with this record, let's save the state of this Grant From 0ffd3c42fd6a115ad5bc63b5d099bec7c6525789 Mon Sep 17 00:00:00 2001 From: jrmartino Date: Thu, 28 May 2020 21:09:39 -0400 Subject: [PATCH 14/14] adjust behavior to accommodate data peculiarities --- .../pass/grant/data/CoeusPassEntityUtil.java | 52 +++++++++++-------- .../grant/data/CoeusPassInitEntityUtil.java | 42 +++++++++------ .../pass/grant/data/DefaultPassUpdater.java | 18 ++++--- .../grant/integration/JhuPassUpdaterIT.java | 4 +- 4 files changed, 69 insertions(+), 47 deletions(-) diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java index 28554ec..7900652 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassEntityUtil.java @@ -73,13 +73,37 @@ public User update(User system, User stored) { * @return the updated object - null if the Grant does not need to be updated */ public Grant update(Grant system, Grant stored) { + //adjust the system view of co-pis by merging in the stored view of pi and co-pis + for( URI uri : stored.getCoPis() ) { + if ( !system.getCoPis().contains(uri) ) { + system.getCoPis().add(uri); + } + } + + //need to be careful, system pi might be null if there is no record for it + //this is to finalize the version of the co-pi list we want to compare between + //system and stored + URI storedPi = stored.getPi(); + if ( system.getPi() != null ) { + if (!system.getPi().equals(storedPi)) { + // stored.setPi( system.getPi() ); + if (!system.getCoPis().contains(storedPi)) { + system.getCoPis().add(storedPi); + } + system.getCoPis().remove(system.getPi()); + } + } else { //system view is null, do not trigger update based on this field + system.setPi( storedPi ); + } + + //now system view has all available info we want in this grant - look for update trigger if (grantNeedsUpdate(system, stored)) { return updateGrant(system, stored); } return null; } - /** + /**0p * Compare two Funder objects * * @param system the version of the Funder as seen in the COEUS system pull @@ -103,7 +127,7 @@ private boolean funderNeedsUpdate(Funder system, Funder stored) { * @return the Funder object which represents the Pass object, with any new information from COEUS merged in */ private Funder updateFunder (Funder system, Funder stored) { - stored.setLocalKey(system.getLocalKey()); + //stored.setLocalKey(system.getLocalKey()); if (system.getName() != null) { stored.setName(system.getName()); } if (system.getPolicy() != null) { stored.setPolicy(system.getPolicy()); } return stored; @@ -173,7 +197,8 @@ private boolean grantNeedsUpdate(Grant system, Grant stored) { } /** - * Update a Pass Grant object with new information from COEUS + * Update a Pass Grant object with new information from COEUS - only updatable fields are considered. + * the PASS version is authoritative for the rest * * @param system the version of the Grant as seen in the COEUS system pull * @param stored the version of the Grant as read from Pass @@ -181,26 +206,9 @@ private boolean grantNeedsUpdate(Grant system, Grant stored) { */ private Grant updateGrant(Grant system, Grant stored) { stored.setAwardStatus(system.getAwardStatus()); - - //adjust the system view of co-pis by merging in the stored view - for( URI uri : stored.getCoPis() ) { - if ( !system.getCoPis().contains(uri) ) { - system.getCoPis().add(uri); - } - } - URI storedPi = stored.getPi(); - if ( !system.getPi().equals( storedPi )) { - if ( !system.getCoPis().contains( storedPi )) { - system.getCoPis().add ( storedPi ); - } - } - system.getCoPis().remove(system.getPi()); - - stored.setPi( system.getPi() ); + stored.setPi(system.getPi()); stored.setCoPis( system.getCoPis() ); - if ( system.getEndDate().isAfter(stored.getEndDate())) { - stored.setEndDate(system.getEndDate()); - } + stored.setEndDate(system.getEndDate()); return stored; } diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java index c5feef2..e4ebba0 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/CoeusPassInitEntityUtil.java @@ -14,6 +14,30 @@ public class CoeusPassInitEntityUtil extends CoeusPassEntityUtil { @Override public Grant update(Grant system, Grant stored) { + //adjust the system view of co-pis by merging in the stored view of pi and co-pis + for( URI uri : stored.getCoPis() ) { + if ( !system.getCoPis().contains(uri) ) { + system.getCoPis().add(uri); + } + } + + //need to be careful, system pi might be null if there is no record for it + //this is to finalize the version of the co-pi list we want to compare between + //system and stored + URI storedPi = stored.getPi(); + if ( system.getPi() != null ) { + if (!system.getPi().equals(storedPi)) { + // stored.setPi( system.getPi() ); + if (!system.getCoPis().contains(storedPi)) { + system.getCoPis().add(storedPi); + } + system.getCoPis().remove(system.getPi()); + } + } else { //system view is null, do not trigger update based on this field + system.setPi( storedPi ); + } + + //now system view has all available info we want in this grant - look for update trigger if (this.grantNeedsUpdate(system, stored)) { return this.updateGrant(system, stored); } @@ -57,23 +81,9 @@ private Grant updateGrant(Grant system, Grant stored) { stored.setProjectName(system.getProjectName()); stored.setPrimaryFunder(system.getPrimaryFunder()); stored.setDirectFunder(system.getDirectFunder()); - - //adjust the system view of co-pis by merging in the stored view - for( URI uri : stored.getCoPis() ) { - if ( !system.getCoPis().contains(uri) ) { - system.getCoPis().add(uri); - } - } - URI storedPi = stored.getPi(); - if ( !system.getPi().equals( storedPi )) { - if ( !system.getCoPis().contains( storedPi )) { - system.getCoPis().add ( storedPi ); - } - } - system.getCoPis().remove(system.getPi()); - - stored.setPi( system.getPi() ); + stored.setPi( system.getPi()); stored.setCoPis( system.getCoPis() ); + //since this is essentially an initial pull, we can just take the system values stored.setAwardDate(system.getAwardDate()); stored.setStartDate(system.getStartDate()); diff --git a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java index 2fade3e..118a60a 100644 --- a/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java +++ b/pass-grant-data/src/main/java/org/dataconservancy/pass/grant/data/DefaultPassUpdater.java @@ -171,7 +171,7 @@ private void updateGrants(Collection> results) { } } - //now do things which may depend on the date + //now do things which may depend on the date - award date is the only one that changes DateTime awardDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_AWARD_DATE, null)); DateTime startDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_START_DATE, null)); DateTime endDate = createJodaDateTime(rowMap.getOrDefault(C_GRANT_END_DATE, null)); @@ -180,19 +180,23 @@ private void updateGrants(Collection> results) { //in case they are needed to update a stored grant record. //these values will not override existing stored values unless the PassEntityUtil implementation //allows it. - if (startDate != null && (grant.getStartDate() == null || startDate.isBefore(grant.getStartDate()))) { + //we mostly have awardDate, but will use start date as a fallback if not + if ( (awardDate != null && (grant.getAwardDate() == null || awardDate.isBefore(grant.getAwardDate()))) || + awardDate == null && (startDate != null && (grant.getStartDate() == null || startDate.isBefore(grant.getStartDate())))) { grant.setProjectName(rowMap.get(C_GRANT_PROJECT_NAME)); grant.setAwardNumber(rowMap.get(C_GRANT_AWARD_NUMBER)); grant.setDirectFunder(funderMap.get(directFunderLocalKey)); grant.setPrimaryFunder(funderMap.get(primaryFunderLocalKey)); - grant.setAwardDate(awardDate); grant.setStartDate(startDate); + grant.setAwardDate(awardDate); } //set values that should match the latest iteration of the grant //use !isBefore in case more than one PI is specified, need to process more than one - if (endDate != null && (grant.getEndDate() == null || !endDate.isBefore(grant.getEndDate()))) { - grant.setEndDate(endDate); + //we mostly have awardDate, but will use end date as a fallback if not + if ( (awardDate != null && (grant.getAwardDate() == null || !awardDate.isBefore(grant.getAwardDate()))) || + awardDate == null && ( endDate != null && (grant.getEndDate() == null || !endDate.isBefore(grant.getEndDate()))) ){ + grant.setEndDate( endDate); //status should be the latest one String status = rowMap.getOrDefault(C_GRANT_AWARD_STATUS, null); if (status != null) { @@ -214,12 +218,12 @@ private void updateGrants(Collection> results) { if ( abbreviatedRole.equals("P") ) { URI userId=userMap.get(employeeId); URI oldPiId=grant.getPi(); + grant.setPi(userId); + grant.getCoPis().remove(userId); if ( oldPiId == null ) { - grant.setPi(userId); statistics.addPi(); } else { if ( !oldPiId.equals(userId) ) { - grant.setPi(userId); if ( !grant.getCoPis().contains(oldPiId) ) { grant.getCoPis().add(oldPiId); statistics.addCoPi(); diff --git a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java index de0650e..a26ba7a 100644 --- a/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java +++ b/pass-grant-integration/src/test/java/org/dataconservancy/pass/grant/integration/JhuPassUpdaterIT.java @@ -44,8 +44,8 @@ public class JhuPassUpdaterIT { String[] grantLocalKey = { "10000001", "10000001","10000001" }; //all the same, different from other ITs tho String[] grantProjectName = {"Stupendous Research Project I", "Stupendous Research Project II", "Stupendous Research Project III" }; String[] grantAwardDate = { "01/01/1999", "01/01/2001", "01/01/2003" }; - String[] grantStartDate = { "07/01/2000", "07/01/2002", "07/01/2004" }; - String[] grantEndDate = { "06/30/2002", "06/30/2004", "06/30/2006"}; + String[] grantStartDate = { "07/01/2000", "07/01/2000", "07/01/2000" }; //these appear to ge the same for all awards + String[] grantEndDate = { "06/30/2004", "06/30/2004", "06/30/2004"};//these seem to be the same for all awards String[] grantUpdateTimestamp = { "2006-03-11 00:00:00.0","2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0" }; String[] userEmployeeId= { "31000000", "31000001", "31000002"}; String[] userInstitutionalId = {"arecko1", "sclass1", "jgunn1" };