From 6a59e7c6108cc891a4d24fbf25d0d5e7656ac9e2 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Thu, 23 May 2024 16:27:09 -0400 Subject: [PATCH 01/19] Change grant db from coeus to fibi --- ...onnector.java => JhuGrantDbConnector.java} | 111 ++++++++---------- ...ava => JhuGrantDbConnectorManualTest.java} | 6 +- 2 files changed, 55 insertions(+), 62 deletions(-) rename pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/{CoeusConnector.java => JhuGrantDbConnector.java} (74%) rename pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/{CoeusConnectorManualTest.java => JhuGrantDbConnectorManualTest.java} (91%) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/CoeusConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java similarity index 74% rename from pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/CoeusConnector.java rename to pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index 68a9aa092..8450daab0 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/CoeusConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -41,31 +41,28 @@ import org.springframework.stereotype.Component; /** - * This class connects to a COEUS database via the Oracle JDBC driver. The query string reflects local JHU + * This class connects to a JHU Grant database. The query string reflects local JHU * database views * * @author jrm@jhu.edu */ @Component @Profile("jhu") -public class CoeusConnector implements GrantConnector { - private static final Logger LOG = LoggerFactory.getLogger(CoeusConnector.class); +public class JhuGrantDbConnector implements GrantConnector { + private static final Logger LOG = LoggerFactory.getLogger(JhuGrantDbConnector.class); static final String C_GRANT_AWARD_NUMBER = "AWARD_ID"; static final String C_GRANT_AWARD_STATUS = "AWARD_STATUS"; - static final String C_GRANT_LOCAL_KEY = "GRANT_NUMBER"; + static final String C_GRANT_LOCAL_KEY = "SAP_GRANT_NUMBER"; static final String C_GRANT_PROJECT_NAME = "TITLE"; static final String C_GRANT_AWARD_DATE = "AWARD_DATE"; - static final String C_GRANT_START_DATE = "AWARD_START"; - static final String C_GRANT_END_DATE = "AWARD_END"; + static final String C_GRANT_START_DATE = "AWARD_START_DATE"; + static final String C_GRANT_END_DATE = "AWARD_END_DATE"; - static final String C_DIRECT_FUNDER_LOCAL_KEY = "SPOSNOR_CODE";// misspelling in COEUS view - if this gets - // corrected - //it will collide with C_PRIMARY_SPONSOR_CODE below - this field will then have to be aliased in order to - //access it in the ResultSet - static final String C_DIRECT_FUNDER_NAME = "SPONSOR"; - static final String C_PRIMARY_FUNDER_LOCAL_KEY = "SPONSOR_CODE"; - static final String C_PRIMARY_FUNDER_NAME = "SPONSOR_NAME"; + static final String C_DIRECT_FUNDER_LOCAL_KEY = "SPONSOR_CODE"; + static final String C_DIRECT_FUNDER_NAME = "SPONSOR_NAME"; + static final String C_PRIMARY_FUNDER_LOCAL_KEY = "PRIME_SPONSOR_CODE"; + static final String C_PRIMARY_FUNDER_NAME = "PRIME_SPONSOR_NAME"; static final String C_USER_FIRST_NAME = "FIRST_NAME"; static final String C_USER_MIDDLE_NAME = "MIDDLE_NAME"; @@ -76,7 +73,7 @@ public class CoeusConnector implements GrantConnector { //these fields are accessed for processing, but are not mapped to PASS objects static final String C_UPDATE_TIMESTAMP = "UPDATE_TIMESTAMP"; - static final String C_ABBREVIATED_ROLE = "ABBREVIATED_ROLE"; + static final String C_ABBREVIATED_ROLE = "ROLE"; private static final String SELECT_GRANT_SQL = "SELECT " + @@ -88,7 +85,9 @@ public class CoeusConnector implements GrantConnector { "A." + C_GRANT_START_DATE + ", " + "A." + C_GRANT_END_DATE + ", " + "A." + C_DIRECT_FUNDER_NAME + ", " + - "A." + C_DIRECT_FUNDER_LOCAL_KEY + ", " + //"SPOSNOR_CODE" + "A." + C_DIRECT_FUNDER_LOCAL_KEY + ", " + + "A." + C_PRIMARY_FUNDER_NAME + ", " + + "A." + C_PRIMARY_FUNDER_LOCAL_KEY + ", " + "A." + C_UPDATE_TIMESTAMP + ", " + "B." + C_ABBREVIATED_ROLE + ", " + "B." + C_USER_EMPLOYEE_ID + ", " + @@ -96,60 +95,54 @@ public class CoeusConnector implements GrantConnector { "C." + C_USER_MIDDLE_NAME + ", " + "C." + C_USER_LAST_NAME + ", " + "C." + C_USER_EMAIL + ", " + - "C." + C_USER_INSTITUTIONAL_ID + ", " + - "D." + C_PRIMARY_FUNDER_NAME + ", " + - "D." + C_PRIMARY_FUNDER_LOCAL_KEY + " " + - "FROM " + - "COEUS.JHU_FACULTY_FORCE_PROP A " + - "INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B ON A.INST_PROPOSAL = B.INST_PROPOSAL " + - "INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C ON B.EMPLOYEE_ID = C.EMPLOYEE_ID " + - "LEFT JOIN COEUS.SWIFT_SPONSOR D ON A.PRIME_SPONSOR_CODE = D.SPONSOR_CODE " + - "WHERE (B.ABBREVIATED_ROLE = 'P' OR B.ABBREVIATED_ROLE = 'C' " + - "OR REGEXP_LIKE (UPPER(B.ROLE), '^CO ?-?INVESTIGATOR$')) " + - "AND TO_DATE(A.AWARD_END, 'MM/DD/YYYY') >= TO_DATE('01/01/2011', 'MM/DD/YYYY') " + - "AND A.PROPOSAL_STATUS = 'Funded' " + - "AND A.GRANT_NUMBER IS NOT NULL " + + "C." + C_USER_INSTITUTIONAL_ID + " " + + "FROM JHU_PASS_AWD_VIEW A, " + + "JHU_FIBI_IP_INV_VIEW B, " + + "JHU_PERSON_VIEW C " + + "WHERE A.inst_proposal = B.inst_proposal " + + "AND B.employee_id = C.employee_id " + "AND EXISTS (" + - " select * from COEUS.JHU_FACULTY_FORCE_PROP EA where" + - " EA.UPDATE_TIMESTAMP > ?" + - " AND EA.GRANT_NUMBER = A.GRANT_NUMBER" + - " AND TO_DATE(EA.AWARD_END, 'MM/DD/YYYY') >= TO_DATE(?, 'MM/DD/YYYY')" + - " AND EA.PROPOSAL_STATUS = 'Funded' "; + " select * from JHU_PASS_AWD_VIEW EA where" + + " EA.UPDATE_TIMESTAMP > ? " + + " AND STR_TO_DATE(EA.AWARD_END_DATE, '%m/%d/%Y') >= STR_TO_DATE(?, '%m/%d/%Y') " + + " and EA.SAP_GRANT_NUMBER = A.SAP_GRANT_NUMBER "; private static final String SELECT_USER_SQL = "SELECT " + - C_USER_FIRST_NAME + ", " + - C_USER_MIDDLE_NAME + ", " + - C_USER_LAST_NAME + ", " + - C_USER_EMAIL + ", " + - C_USER_INSTITUTIONAL_ID + ", " + - C_USER_EMPLOYEE_ID + ", " + - C_UPDATE_TIMESTAMP + " " + - "FROM COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL " + - "WHERE UPDATE_TIMESTAMP > ?"; + "A." + C_USER_FIRST_NAME + ", " + + "A." + C_USER_MIDDLE_NAME + ", " + + "A." + C_USER_LAST_NAME + ", " + + "A." + C_USER_EMAIL + ", " + + "A." + C_USER_INSTITUTIONAL_ID + ", " + + "B." + C_USER_EMPLOYEE_ID + ", " + + "A." + C_UPDATE_TIMESTAMP + " " + + "FROM JHU_PERSON_VIEW A, " + + "JHU_FIBI_IP_INV_VIEW B " + + "WHERE A.employee_id = B.employee_id " + + "and A.UPDATE_TIMESTAMP > ?"; private static final String SELECT_FUNDER_SQL = "SELECT " + - C_PRIMARY_FUNDER_NAME + ", " + - C_PRIMARY_FUNDER_LOCAL_KEY + " " + - "FROM COEUS.SWIFT_SPONSOR " + + C_DIRECT_FUNDER_NAME + ", " + + C_DIRECT_FUNDER_LOCAL_KEY + " " + + "FROM JHU_SPONSOR_VIEW " + "WHERE SPONSOR_CODE IN (%s)"; @Value("${grant.db.url}") - private String coeusUrl; + private String grantDbUrl; @Value("${grant.db.username}") - private String coeusUser; + private String grantDbUser; @Value("${grant.db.password}") - private String coeusPassword; + private String grantDbPassword; private final Set funderIds; /** * Class constructor. */ - public CoeusConnector(@Qualifier("policyProperties") Properties policyProperties) { + public JhuGrantDbConnector(@Qualifier("policyProperties") Properties policyProperties) { this.funderIds = policyProperties.stringPropertyNames(); } @@ -171,7 +164,7 @@ private List retrieveGrantUpdates(String startDate, String aw List grantIngestRecords = new ArrayList<>(); try ( - Connection con = DriverManager.getConnection(coeusUrl, coeusUser, coeusPassword); + Connection con = DriverManager.getConnection(grantDbUrl, grantDbUser, grantDbPassword); PreparedStatement ps = con.prepareStatement(sql); ) { LocalDateTime startLd = LocalDateTime.from(DateTimeUtil.DATE_TIME_FORMATTER.parse(startDate)); @@ -192,7 +185,9 @@ private List retrieveGrantUpdates(String startDate, String aw grantIngestRecord.setAwardStart(rs.getString(C_GRANT_START_DATE)); grantIngestRecord.setAwardEnd(rs.getString(C_GRANT_END_DATE)); grantIngestRecord.setDirectFunderName(rs.getString(C_DIRECT_FUNDER_NAME)); + grantIngestRecord.setDirectFunderCode(rs.getString(C_DIRECT_FUNDER_LOCAL_KEY)); grantIngestRecord.setPrimaryFunderName(rs.getString(C_PRIMARY_FUNDER_NAME)); + grantIngestRecord.setPrimaryFunderCode(rs.getString(C_PRIMARY_FUNDER_LOCAL_KEY)); grantIngestRecord.setPiFirstName(rs.getString(C_USER_FIRST_NAME)); grantIngestRecord.setPiMiddleName(rs.getString(C_USER_MIDDLE_NAME)); grantIngestRecord.setPiLastName(rs.getString(C_USER_LAST_NAME)); @@ -201,8 +196,6 @@ private List retrieveGrantUpdates(String startDate, String aw grantIngestRecord.setPiInstitutionalId(rs.getString(C_USER_INSTITUTIONAL_ID)); grantIngestRecord.setUpdateTimeStamp(rs.getString(C_UPDATE_TIMESTAMP)); grantIngestRecord.setPiRole(rs.getString(C_ABBREVIATED_ROLE)); - grantIngestRecord.setPrimaryFunderCode(rs.getString(C_PRIMARY_FUNDER_LOCAL_KEY)); - grantIngestRecord.setDirectFunderCode(rs.getString(C_DIRECT_FUNDER_LOCAL_KEY)); LOG.debug("Record processed: {}", grantIngestRecord); if (!grantIngestRecords.contains(grantIngestRecord)) { grantIngestRecords.add(grantIngestRecord); @@ -210,14 +203,14 @@ private List retrieveGrantUpdates(String startDate, String aw } } } - LOG.info("Retrieved result set from COEUS: {} records processed", grantIngestRecords.size()); + LOG.info("Retrieved result set from JHU Grant DB: {} records processed", grantIngestRecords.size()); return grantIngestRecords; } private String buildGrantQueryString(String grant) { return StringUtils.isEmpty(grant) - ? SELECT_GRANT_SQL + "AND EA.GRANT_NUMBER IS NOT NULL)" - : SELECT_GRANT_SQL + "AND EA.GRANT_NUMBER = ?)"; + ? SELECT_GRANT_SQL + "AND A.SAP_GRANT_NUMBER IS NOT NULL)" + : SELECT_GRANT_SQL + "AND A.SAP_GRANT_NUMBER = ?)"; } private List retrieveFunderUpdates(Set funderIds) throws SQLException { @@ -225,7 +218,7 @@ private List retrieveFunderUpdates(Set funderIds) thr String funderSql = String.format(SELECT_FUNDER_SQL, funderIds.stream().map(v -> "?").collect(Collectors.joining(", "))); try ( - Connection con = DriverManager.getConnection(coeusUrl, coeusUser, coeusPassword); + Connection con = DriverManager.getConnection(grantDbUrl, grantDbUser, grantDbPassword); PreparedStatement ps = con.prepareStatement(funderSql); ) { int index = 1; @@ -235,8 +228,8 @@ private List retrieveFunderUpdates(Set funderIds) thr try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { //these are the field names in the swift sponsor view GrantIngestRecord grantIngestRecord = new GrantIngestRecord(); - grantIngestRecord.setPrimaryFunderCode(rs.getString(C_PRIMARY_FUNDER_LOCAL_KEY)); - grantIngestRecord.setPrimaryFunderName(rs.getString(C_PRIMARY_FUNDER_NAME)); + grantIngestRecord.setPrimaryFunderCode(rs.getString(C_DIRECT_FUNDER_LOCAL_KEY)); + grantIngestRecord.setPrimaryFunderName(rs.getString(C_DIRECT_FUNDER_NAME)); grantIngestRecords.add(grantIngestRecord); } } @@ -247,7 +240,7 @@ private List retrieveFunderUpdates(Set funderIds) thr private List retrieveUserUpdates(String startDate) throws SQLException { List grantIngestRecords = new ArrayList<>(); try ( - Connection con = DriverManager.getConnection(coeusUrl, coeusUser, coeusPassword); + Connection con = DriverManager.getConnection(grantDbUrl, grantDbUser, grantDbPassword); PreparedStatement ps = con.prepareStatement(SELECT_USER_SQL); ) { LocalDateTime startLd = LocalDateTime.from(DateTimeUtil.DATE_TIME_FORMATTER.parse(startDate)); diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/CoeusConnectorManualTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorManualTest.java similarity index 91% rename from pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/CoeusConnectorManualTest.java rename to pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorManualTest.java index 98ffec747..8bacb21bc 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/CoeusConnectorManualTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorManualTest.java @@ -28,7 +28,7 @@ import org.springframework.test.context.TestPropertySource; /** - * Test class for the COEUS connector. This is strictly a manual test for querying the Coeus database. + * Test class for the JHU Grant DB connector. This is strictly a manual test for querying the database. * This test is Disabled, you can enable it and run each query test if needed for validation. *

* In order to run the tests, you must put a connection.properties file with valid url and creds in the @@ -39,9 +39,9 @@ @SpringBootTest @TestPropertySource("classpath:test-application.properties") @Disabled -public class CoeusConnectorManualTest { +public class JhuGrantDbConnectorManualTest { - @Autowired private CoeusConnector connector; + @Autowired private JhuGrantDbConnector connector; @Disabled @Test From 8fe635dfa8669364b1233d38a09895ae7170b177 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Thu, 23 May 2024 17:09:53 -0400 Subject: [PATCH 02/19] Update alias --- .../pass/support/grant/data/jhu/JhuGrantDbConnector.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index 8450daab0..ff6c34684 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -209,8 +209,8 @@ private List retrieveGrantUpdates(String startDate, String aw private String buildGrantQueryString(String grant) { return StringUtils.isEmpty(grant) - ? SELECT_GRANT_SQL + "AND A.SAP_GRANT_NUMBER IS NOT NULL)" - : SELECT_GRANT_SQL + "AND A.SAP_GRANT_NUMBER = ?)"; + ? SELECT_GRANT_SQL + "AND EA.SAP_GRANT_NUMBER IS NOT NULL)" + : SELECT_GRANT_SQL + "AND EA.SAP_GRANT_NUMBER = ?)"; } private List retrieveFunderUpdates(Set funderIds) throws SQLException { From dd075d4e0705ab021541cf2fdb74cf4986be603e Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Tue, 28 May 2024 10:48:36 -0400 Subject: [PATCH 03/19] Update grant sql with filters --- .../support/grant/data/jhu/JhuGrantDbConnector.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index ff6c34684..62c87e387 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -101,11 +101,15 @@ public class JhuGrantDbConnector implements GrantConnector { "JHU_PERSON_VIEW C " + "WHERE A.inst_proposal = B.inst_proposal " + "AND B.employee_id = C.employee_id " + + "AND STR_TO_DATE(A.AWARD_END, '%m/%d/%Y') >= STR_TO_DATE('01/01/2011', '%m/%d/%Y') " + + // TODO confirm with Bob this is in view sql + "AND A.PROPOSAL_STATUS = 'Funded' " + + "AND A.SAP_GRANT_NUMBER IS NOT NULL " + "AND EXISTS (" + " select * from JHU_PASS_AWD_VIEW EA where" + - " EA.UPDATE_TIMESTAMP > ? " + - " AND STR_TO_DATE(EA.AWARD_END_DATE, '%m/%d/%Y') >= STR_TO_DATE(?, '%m/%d/%Y') " + - " and EA.SAP_GRANT_NUMBER = A.SAP_GRANT_NUMBER "; + " EA.UPDATE_TIMESTAMP > ?" + + " and EA.SAP_GRANT_NUMBER = A.SAP_GRANT_NUMBER" + + " AND STR_TO_DATE(EA.AWARD_END_DATE, '%m/%d/%Y') >= STR_TO_DATE(?, '%m/%d/%Y') "; private static final String SELECT_USER_SQL = "SELECT " + From 3e873306b0b1963040ae8ccd574e19e2c3650af8 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Tue, 28 May 2024 14:28:26 -0400 Subject: [PATCH 04/19] Remove refs to COEUS name --- pass-grant-loader/README.md | 4 ++-- .../grant/data/AbstractDefaultPassUpdater.java | 2 +- .../grant/data/jhu/JhuGrantDbConnector.java | 2 +- .../support/grant/data/jhu/JhuPassUpdater.java | 15 +++++++-------- .../pass/support/grant/GrantLoaderManualIT.java | 9 ++++----- .../pass/support/grant/GrantLoaderManualTest.java | 8 +++----- .../support/grant/data/jhu/JhuPassUpdaterIT.java | 2 +- .../grant/data/jhu/JhuPassUpdaterOrderedIT.java | 2 +- 8 files changed, 20 insertions(+), 24 deletions(-) diff --git a/pass-grant-loader/README.md b/pass-grant-loader/README.md index eb860b957..d91181bd3 100644 --- a/pass-grant-loader/README.md +++ b/pass-grant-loader/README.md @@ -36,10 +36,10 @@ policies require. ### JHU (jhu) -The JHU implementation is used to pull data from the COEUS Oracle database views for the purpose of performing regular +The JHU implementation is used to pull data from the Grant database views for the purpose of performing regular updates. We look at grants which have been updated since a particular time (typically the time of the previous update), join this with user and funder information associated with the grant, and then use this information to update the data -in the PASS backend. The JHU implementation also treats the COEUS database as authoritative for all fields in the +in the PASS backend. The JHU implementation also treats the Grant database as authoritative for all fields in the data. If a grant is being passed in for update, it is assumed that all records for that grant are included in the input. diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/AbstractDefaultPassUpdater.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/AbstractDefaultPassUpdater.java index 08f51485d..f0fe9ce68 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/AbstractDefaultPassUpdater.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/AbstractDefaultPassUpdater.java @@ -611,7 +611,7 @@ private User updateUserInPass(User systemUser) throws IOException { if (Objects.nonNull(passUser)) { User updatedUser = updateUserIfNeeded(systemUser, passUser); if (Objects.nonNull(updatedUser)) { //need to update - //post COEUS processing goes here + //post Grant database processing goes here if (!updatedUser.getRoles().contains(UserRole.SUBMITTER)) { updatedUser.getRoles().add(UserRole.SUBMITTER); } diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index 62c87e387..1e8e185e9 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -266,7 +266,7 @@ private List retrieveUserUpdates(String startDate) throws SQL } } } - LOG.info("Retrieved Users result set from COEUS: {} records processed", grantIngestRecords.size()); + LOG.info("Retrieved Users result set from JHU Grant DB: {} records processed", grantIngestRecords.size()); return grantIngestRecords; } diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdater.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdater.java index fbc8edda5..3f5b901e8 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdater.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdater.java @@ -158,7 +158,7 @@ private Funder updateFunder(Funder system, Funder stored) { } private boolean userNeedsUpdate(User system, User stored) { - //first the fields for which COEUS is authoritative + //first the fields for which Grant database is authoritative if (!Objects.equals(system.getFirstName(), stored.getFirstName())) { return true; } @@ -190,15 +190,14 @@ private String findLocatorId(User user, String locatorIdPrefix) { } /** - * Update a Pass User object with new information from COEUS. We check only those fields for which COEUS is - * authoritative. Other fields will be managed by other providers (Shibboleth for example). The exceptions are - * the localKey, which this application and Shibboleth both rely on; and email, which this application only - * populates - * if Shib hasn't done so already. + * Update a Pass User object with new information from Grant database. + * We check only those fields for which the Grant database is authoritative. Other fields will be managed by + * other providers (Shibboleth for example). The exceptions are the localKey, which this application and + * Shibboleth both rely on; and email, which this application only populates if Shib hasn't done so already. * - * @param system the version of the User as seen in the COEUS system pull + * @param system the version of the User as seen in the Grant database system pull * @param stored the version of the User as read from Pass - * @return the User object which represents the Pass object, with any new information from COEUS merged in + * @return the User object which represents the Pass object, with any new information from Grant database merged in */ private User updateUser(User system, User stored) { stored.setFirstName(system.getFirstName()); diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualIT.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualIT.java index f9a181317..4ae6f99b3 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualIT.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualIT.java @@ -28,19 +28,18 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +// todo need to add spring support for grant db file public class GrantLoaderManualIT { /** - * This is a manual test that can run locally to test loading the COEUS data. - * You need to set the system prop APP_HOME to the path of the test/resouces dir. - * You also need the connection.properties file in test/resources which contains connection props - * for COEUS and the Directory Service. + * This is a manual test that can run locally to test loading the grant data. + * You also need the grant db connection props in the test-application.properties file. * Be careful with the startDateTime to no pull too much data. Know what the impact is on pulling * data before running this test. */ @Disabled @Test - public void testLoadCoeusFile() { + public void testLoadGrantFile() { System.setProperty( "APP_HOME", "full_path_to/pass-support/pass-grant-loader/pass-grant-cli/src/test/resources" diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java index 196a03237..038233eed 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java @@ -21,16 +21,14 @@ public class GrantLoaderManualTest { /** - * This is a manual test that can run locally to test pulling the COEUS data into a file. - * You need to set the system prop APP_HOME to the path of the test/resouces dir. - * You also need the connection.properties file in test/resources which contains connection props - * for COEUS and the Directory Service. + * This is a manual test that can run locally to test pulling the grant data into a file. + * You also need the grant db connection props in the test-application.properties file. * Be careful with the startDateTime to no pull too much data. Know what the impact is on pulling * data before running this test. */ @Disabled @Test - public void testPullCoeusFile() { + public void testPullGrantFile() { System.setProperty( "APP_HOME", "full_path_to/pass-support/pass-grant-loader/pass-grant-cli/src/test/resources" diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterIT.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterIT.java index 8997f8e22..b241a209d 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterIT.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterIT.java @@ -753,7 +753,7 @@ public void testUpdateGrant_UpdateUserLocatorsJhed() throws IOException, GrantDa assertEquals(0, jhuPassUpdater.getStatistics().getCoPisAdded()); // WHEN - // JHED ID and Hopkins ID update from coeus + // JHED ID and Hopkins ID update from Grant database GrantIngestRecord piRecordUpdate = TestUtil.makeGrantIngestRecord(4, 4, "P"); piRecordUpdate.setPiInstitutionalId("newjdoe1jhed"); diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterOrderedIT.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterOrderedIT.java index 136c8253b..cccf4290b 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterOrderedIT.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterOrderedIT.java @@ -226,7 +226,7 @@ private User getVerifiedUser(int userIndex) throws IOException { } /** - * utility method to produce data as it would look coming from COEUS + * utility method to produce data as it would look coming from Grant database * * @param iteration the iteration of the (multi-award) grant * @param user the user supplied in the record From 0a25ce9877177203904797fd6e486f6705fdcba6 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Wed, 29 May 2024 07:37:55 -0400 Subject: [PATCH 05/19] Remove filter no longer needed Because it is in the view sql --- .../pass/support/grant/data/jhu/JhuGrantDbConnector.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index 1e8e185e9..2f5964e5f 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -102,8 +102,6 @@ public class JhuGrantDbConnector implements GrantConnector { "WHERE A.inst_proposal = B.inst_proposal " + "AND B.employee_id = C.employee_id " + "AND STR_TO_DATE(A.AWARD_END, '%m/%d/%Y') >= STR_TO_DATE('01/01/2011', '%m/%d/%Y') " + - // TODO confirm with Bob this is in view sql - "AND A.PROPOSAL_STATUS = 'Funded' " + "AND A.SAP_GRANT_NUMBER IS NOT NULL " + "AND EXISTS (" + " select * from JHU_PASS_AWD_VIEW EA where" + @@ -123,6 +121,7 @@ public class JhuGrantDbConnector implements GrantConnector { "FROM JHU_PERSON_VIEW A, " + "JHU_FIBI_IP_INV_VIEW B " + "WHERE A.employee_id = B.employee_id " + + // todo need reply from Bob on this "and A.UPDATE_TIMESTAMP > ?"; private static final String SELECT_FUNDER_SQL = From b6f9b7ce81e322a40c4de3861268fb4b1951db54 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Wed, 29 May 2024 07:55:49 -0400 Subject: [PATCH 06/19] Clean manual tests --- .../support/grant/GrantLoaderManualIT.java | 68 ------------------- .../support/grant/GrantLoaderManualTest.java | 61 ++++++++++++++--- 2 files changed, 53 insertions(+), 76 deletions(-) delete mode 100644 pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualIT.java diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualIT.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualIT.java deleted file mode 100644 index 4ae6f99b3..000000000 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualIT.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2023 Johns Hopkins University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.eclipse.pass.support.grant; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -import java.io.IOException; - -import org.eclipse.pass.support.client.PassClient; -import org.eclipse.pass.support.client.PassClientResult; -import org.eclipse.pass.support.client.PassClientSelector; -import org.eclipse.pass.support.client.RSQL; -import org.eclipse.pass.support.client.model.Grant; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -// todo need to add spring support for grant db file -public class GrantLoaderManualIT { - - /** - * This is a manual test that can run locally to test loading the grant data. - * You also need the grant db connection props in the test-application.properties file. - * Be careful with the startDateTime to no pull too much data. Know what the impact is on pulling - * data before running this test. - */ - @Disabled - @Test - public void testLoadGrantFile() { - System.setProperty( - "APP_HOME", - "full_path_to/pass-support/pass-grant-loader/pass-grant-cli/src/test/resources" - ); - String[] args = {"-a", "load", "full_path_to/testresults"}; - GrantLoaderCLI.main(args); - } - - @Disabled - @Test - void testCheckGrant() throws IOException { - System.setProperty("pass.core.url","http://localhost:8080"); - System.setProperty("pass.core.user",""); - System.setProperty("pass.core.password",""); - PassClient passClient = PassClient.newInstance(); - - PassClientSelector grantSelector = new PassClientSelector<>(Grant.class); - grantSelector.setFilter(RSQL.equals("localKey", "johnshopkins.edu:grant:143377")); - grantSelector.setInclude("primaryFunder", "directFunder", "pi", "coPis"); - PassClientResult resultGrant = passClient.selectObjects(grantSelector); - assertEquals(1, resultGrant.getTotal()); - Grant passGrant = resultGrant.getObjects().get(0); - assertNotNull(passGrant); - } - -} diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java index 038233eed..55eecf513 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java @@ -15,11 +15,33 @@ */ package org.eclipse.pass.support.grant; +import org.eclipse.pass.support.client.PassClient; +import org.eclipse.pass.support.client.PassClientResult; +import org.eclipse.pass.support.client.PassClientSelector; +import org.eclipse.pass.support.client.RSQL; +import org.eclipse.pass.support.client.model.Grant; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * @author Russ Poetker (rpoetke1@jh.edu) + */ +@SpringBootTest +@TestPropertySource("classpath:test-application.properties") public class GrantLoaderManualTest { + @Autowired private GrantLoaderApp grantLoaderApp; + @MockBean private GrantLoaderCLIRunner grantLoaderCLIRunner; + /** * This is a manual test that can run locally to test pulling the grant data into a file. * You also need the grant db connection props in the test-application.properties file. @@ -28,14 +50,37 @@ public class GrantLoaderManualTest { */ @Disabled @Test - public void testPullGrantFile() { - System.setProperty( - "APP_HOME", - "full_path_to/pass-support/pass-grant-loader/pass-grant-cli/src/test/resources" - ); - String[] args = {"-a", "pull", "-s", "2023-04-01 00:00:00.000", "-z", "04/01/2023", - "full_path_to/testresults"}; - GrantLoaderCLI.main(args); + public void testPullGrantFile() throws PassCliException { + grantLoaderApp.run("2023-04-01 00:00:00", "04/01/2023", + "grant", "pull", "file:./src/test/resources/your-file.csv", null); + } + + /** + * This is a manual test that can run locally to test loading the grant data. + * You also need to set the test pass.core props in the test-application.properties file. + */ + @Disabled + @Test + public void testLoadGrantFile() throws PassCliException { + grantLoaderApp.run("2023-04-01 00:00:00", "04/01/2023", + "grant", "load", "file:./src/test/resources/your-file.csv", null); + } + + @Disabled + @Test + void testCheckGrant() throws IOException { + System.setProperty("pass.core.url","http://localhost:8080"); + System.setProperty("pass.core.user",""); + System.setProperty("pass.core.password",""); + PassClient passClient = PassClient.newInstance(); + + PassClientSelector grantSelector = new PassClientSelector<>(Grant.class); + grantSelector.setFilter(RSQL.equals("localKey", "johnshopkins.edu:grant:143377")); + grantSelector.setInclude("primaryFunder", "directFunder", "pi", "coPis"); + PassClientResult resultGrant = passClient.selectObjects(grantSelector); + assertEquals(1, resultGrant.getTotal()); + Grant passGrant = resultGrant.getObjects().get(0); + assertNotNull(passGrant); } } From 99cbbcdafd09e820c53636ac53a04df458788bfa Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Wed, 29 May 2024 08:31:29 -0400 Subject: [PATCH 07/19] Fix checkstyle issue --- .../pass/support/grant/GrantLoaderManualTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java index 55eecf513..55918e0c7 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java @@ -15,6 +15,11 @@ */ package org.eclipse.pass.support.grant; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.IOException; + import org.eclipse.pass.support.client.PassClient; import org.eclipse.pass.support.client.PassClientResult; import org.eclipse.pass.support.client.PassClientSelector; @@ -27,11 +32,6 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.TestPropertySource; -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - /** * @author Russ Poetker (rpoetke1@jh.edu) */ From 25ef9956e04fe62b7f6aaaa424dc968355791c29 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Wed, 29 May 2024 08:41:17 -0400 Subject: [PATCH 08/19] Update cli arg descriptions --- .../support/grant/GrantLoaderCLIRunner.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java index 0649fbfe8..6372e1891 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java @@ -57,20 +57,18 @@ public class GrantLoaderCLIRunner implements CommandLineRunner { */ @Option(name = "-s", aliases = {"-startDateTime", "--startDateTime"}, usage = "DateTime to start the query against Grant source system. This will cause " + - "a return of all records updated since this DateTime. Syntax must be yyyy-mm-dd hh:mm:ss.m{mm}. " + - "This value will override the most recent " + - "dateTime listed in the updates file.") + "a return of all grants having a record updated since this DateTime. " + + "Syntax must be yyyy-mm-dd hh:mm:ss.m{mm}. This value will override the most recent " + + "dateTime listed in the updates file.") protected static String startDate = ""; - /** * Specifies an award end date for basing the database query */ @Option(name = "-z", aliases = {"-awardEndDate", "--awardEndDate"}, usage = "Date for the AWARD_END to start the query against Grant source system. This will cause " + - "a return of all records having an AWARD_END after the supplied date. Syntax must be MM/dd/yyyy. " + - "If not specified, the default will be " + - "01/01/2011") + "a return of all grants having a record with an AWARD_END after the supplied date. " + + "Syntax must be MM/dd/yyyy. If not specified, the default will be 01/01/2011") protected static String awardEndDate = "01/01/2011"; /** @@ -81,11 +79,10 @@ public class GrantLoaderCLIRunner implements CommandLineRunner { */ @Option(name = "-a", aliases = {"-action", "--action"}, usage = "Action to be taken - 'pull' is for Grant source system pull only," + - "'load' is for PASS load only. Either option requires a file path specified as an argument " + - "after all options - an" + - "output file in the case of 'pull', and an input file in the case of 'load'. If no action is " + - "specified, " + - "the data will be pulled from Grant source system and loaded directly into PASS") + "'load' is for PASS load only. Either option requires a file path specified as an argument " + + "after all options - an output file in the case of 'pull', and an input file in the case of " + + "'load'. If no action is specified, the data will be pulled from Grant source system and loaded " + + "directly into PASS") protected static String action = ""; /** From 3e44ff814eeb8c21aef85485552b905f9d3b532e Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Wed, 29 May 2024 09:39:45 -0400 Subject: [PATCH 09/19] Add mysql jdbc jar --- pass-grant-loader/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pass-grant-loader/pom.xml b/pass-grant-loader/pom.xml index 9b24c8e3f..83a3d5db3 100644 --- a/pass-grant-loader/pom.xml +++ b/pass-grant-loader/pom.xml @@ -32,7 +32,7 @@ 2.33 1.10.0 3.14.0 - 19.18.0.0 + 8.0.33 1.18.30 2.0.9 1.4.14 @@ -107,9 +107,9 @@ ${commons-lang.version} - com.oracle.database.jdbc - ojdbc10 - ${ojdbc10.version} + mysql + mysql-connector-java + ${mysql.jdbc.version} org.slf4j @@ -298,7 +298,7 @@ org.projectlombok:lombok: org.mockito:mockito-inline: - com.oracle.database.jdbc:: + mysql:mysql-connector-java:: From 1212c893bae6b337525ed2208bf0352b7c6f1d4d Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Thu, 30 May 2024 14:15:37 -0400 Subject: [PATCH 10/19] Update grant sql --- .../pass/support/grant/data/jhu/JhuGrantDbConnector.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index 2f5964e5f..75b6f0f75 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -101,13 +101,13 @@ public class JhuGrantDbConnector implements GrantConnector { "JHU_PERSON_VIEW C " + "WHERE A.inst_proposal = B.inst_proposal " + "AND B.employee_id = C.employee_id " + - "AND STR_TO_DATE(A.AWARD_END, '%m/%d/%Y') >= STR_TO_DATE('01/01/2011', '%m/%d/%Y') " + - "AND A.SAP_GRANT_NUMBER IS NOT NULL " + + "AND A.AWARD_END_DATE >= STR_TO_DATE('01/01/2011', '%m/%d/%Y') " + + "AND B.ROLE != 'KP' " + "AND EXISTS (" + " select * from JHU_PASS_AWD_VIEW EA where" + " EA.UPDATE_TIMESTAMP > ?" + " and EA.SAP_GRANT_NUMBER = A.SAP_GRANT_NUMBER" + - " AND STR_TO_DATE(EA.AWARD_END_DATE, '%m/%d/%Y') >= STR_TO_DATE(?, '%m/%d/%Y') "; + " AND EA.AWARD_END_DATE >= STR_TO_DATE(?, '%m/%d/%Y') "; private static final String SELECT_USER_SQL = "SELECT " + @@ -212,7 +212,7 @@ private List retrieveGrantUpdates(String startDate, String aw private String buildGrantQueryString(String grant) { return StringUtils.isEmpty(grant) - ? SELECT_GRANT_SQL + "AND EA.SAP_GRANT_NUMBER IS NOT NULL)" + ? SELECT_GRANT_SQL + ")" : SELECT_GRANT_SQL + "AND EA.SAP_GRANT_NUMBER = ?)"; } From 7157c0da7799e269d812379fa50199613283a8d6 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Mon, 3 Jun 2024 15:46:46 -0400 Subject: [PATCH 11/19] Enable h2 for tests --- pass-grant-loader/pom.xml | 12 ++++++++++++ .../src/test/resources/test-application.properties | 11 ++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pass-grant-loader/pom.xml b/pass-grant-loader/pom.xml index 83a3d5db3..e9c3e086c 100644 --- a/pass-grant-loader/pom.xml +++ b/pass-grant-loader/pom.xml @@ -91,6 +91,16 @@ spring-boot-starter-test test + + org.springframework.boot + spring-boot-starter-data-jpa + test + + + com.h2database + h2 + test + args4j args4j @@ -299,6 +309,8 @@ org.projectlombok:lombok: org.mockito:mockito-inline: mysql:mysql-connector-java:: + + com.h2database:h2:: diff --git a/pass-grant-loader/src/test/resources/test-application.properties b/pass-grant-loader/src/test/resources/test-application.properties index a46e66756..75ee9d63d 100644 --- a/pass-grant-loader/src/test/resources/test-application.properties +++ b/pass-grant-loader/src/test/resources/test-application.properties @@ -3,4 +3,13 @@ pass.grant.update.ts.path=file:./src/test/resources/grant_update_timestamps # pass-core properties pass.client.url=http://localhost:8080/ pass.client.user=backend -pass.client.password=backend \ No newline at end of file +pass.client.password=backend + +spring.datasource.driver-class-name=org.h2.Driver +spring.datasource.url=jdbc:h2:mem:db +spring.datasource.username=sa +spring.datasource.password=sa + +grant.db.url=${spring.datasource.url} +grant.db.username=sa +grant.db.password=sa \ No newline at end of file From f29f9b2b55f960d695caec39eb10f14a786eb1a2 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Mon, 3 Jun 2024 15:47:01 -0400 Subject: [PATCH 12/19] Add test data files --- pass-grant-loader/src/test/resources/data.sql | 27 ++++++++++++++++ .../src/test/resources/schema.sql | 31 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 pass-grant-loader/src/test/resources/data.sql create mode 100644 pass-grant-loader/src/test/resources/schema.sql diff --git a/pass-grant-loader/src/test/resources/data.sql b/pass-grant-loader/src/test/resources/data.sql new file mode 100644 index 000000000..ed077ec0c --- /dev/null +++ b/pass-grant-loader/src/test/resources/data.sql @@ -0,0 +1,27 @@ + +INSERT INTO JHU_PASS_AWD_VIEW (INST_PROPOSAL, AWARD_ID, AWARD_STATUS, SAP_GRANT_NUMBER, TITLE, AWARD_DATE, AWARD_START_DATE, + AWARD_END_DATE, SPONSOR_CODE, SPONSOR_NAME, PRIME_SPONSOR_CODE, PRIME_SPONSOR_NAME, + UPDATE_TIMESTAMP) +VALUES('1', 'B10000000', 'Active', '10000001', 'Stupendous "Research Project" I', '2021-01-01','2021-07-01', + '2024-06-30', '20000000', 'Enormous State University', '20000001', 'J L Gotrocks Foundation', '2023-03-11 00:00:00.0'); + +INSERT INTO JHU_FIBI_IP_INV_VIEW (INST_PROPOSAL, EMPLOYEE_ID, ROLE) +VALUES ('1', '31000000', 'PI'); +INSERT INTO JHU_FIBI_IP_INV_VIEW (INST_PROPOSAL, EMPLOYEE_ID, ROLE) +VALUES ('1', '31000001', 'Co-I'); + +INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID) +VALUES ('31000000', 'Amanda', 'Bea', 'Reckondwith', 'arecko1@jhu.edu', 'arecko1'); +INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID) +VALUES ('31000001', 'Skip', 'Avery', 'Class', 'sclass1@jhu.edu', 'sclass1'); + +INSERT INTO JHU_PASS_AWD_VIEW (INST_PROPOSAL, AWARD_ID, AWARD_STATUS, SAP_GRANT_NUMBER, TITLE, AWARD_DATE, AWARD_START_DATE, + AWARD_END_DATE, SPONSOR_CODE, SPONSOR_NAME, PRIME_SPONSOR_CODE, PRIME_SPONSOR_NAME, + UPDATE_TIMESTAMP) +VALUES('2', 'B10000003', 'Active', '10000002', 'Stupendous Research ProjectIV', '2014-01-01','2014-07-01', + '2017-06-30', '20000000', 'Enormous State University', '20000001', 'J L Gotrocks Foundation', '2016-11-11'); + +INSERT INTO JHU_FIBI_IP_INV_VIEW (INST_PROPOSAL, EMPLOYEE_ID, ROLE) +VALUES ('2', '31000001', 'PI'); + + diff --git a/pass-grant-loader/src/test/resources/schema.sql b/pass-grant-loader/src/test/resources/schema.sql new file mode 100644 index 000000000..a518370d7 --- /dev/null +++ b/pass-grant-loader/src/test/resources/schema.sql @@ -0,0 +1,31 @@ + +CREATE TABLE IF NOT EXISTS JHU_PASS_AWD_VIEW ( + INST_PROPOSAL varchar(100) NOT NULL, + AWARD_ID varchar(100) NOT NULL, + AWARD_STATUS varchar(100), + SAP_GRANT_NUMBER varchar(100), + TITLE varchar(255), + AWARD_DATE date, + AWARD_START_DATE date, + AWARD_END_DATE date, + SPONSOR_CODE varchar(100), + SPONSOR_NAME varchar(100), + PRIME_SPONSOR_CODE varchar(100), + PRIME_SPONSOR_NAME varchar(100), + UPDATE_TIMESTAMP timestamp +); + +CREATE TABLE IF NOT EXISTS JHU_FIBI_IP_INV_VIEW ( + INST_PROPOSAL varchar(100) NOT NULL, + EMPLOYEE_ID varchar(100), + ROLE varchar(100) +); + +CREATE TABLE IF NOT EXISTS JHU_PERSON_VIEW ( + EMPLOYEE_ID varchar(100), + FIRST_NAME varchar(100) NOT NULL, + MIDDLE_NAME varchar(100), + LAST_NAME varchar(100), + EMAIL_ADDRESS varchar(100), + JHED_ID varchar(100) +); \ No newline at end of file From 68d550cf9696dd08ee0d2f9db95ca7e7ebcc1840 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Mon, 3 Jun 2024 15:48:05 -0400 Subject: [PATCH 13/19] Update award date format --- .../pass/support/grant/GrantLoaderApp.java | 3 +- .../support/grant/GrantLoaderCLIRunner.java | 4 +-- .../pass/support/grant/data/DateTimeUtil.java | 28 ++----------------- .../support/grant/data/DateTimeUtilTest.java | 16 +---------- 4 files changed, 7 insertions(+), 44 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderApp.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderApp.java index eedcd0887..fb288f01d 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderApp.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderApp.java @@ -26,7 +26,6 @@ import static org.eclipse.pass.support.grant.DataLoaderErrors.ERR_REQUIRED_CONFIGURATION_FILE_MISSING; import static org.eclipse.pass.support.grant.DataLoaderErrors.ERR_RESULT_SET_NULL; import static org.eclipse.pass.support.grant.DataLoaderErrors.ERR_SQL_EXCEPTION; -import static org.eclipse.pass.support.grant.data.DateTimeUtil.verifyDate; import static org.eclipse.pass.support.grant.data.DateTimeUtil.verifyDateTimeFormat; import java.io.BufferedReader; @@ -132,7 +131,7 @@ public void run(String startDate, String awardEndDate, String mode, String actio } } if (awardEndDate != null) { - if (!verifyDate(awardEndDate)) { + if (!verifyDateTimeFormat(awardEndDate)) { throw processException(format(ERR_INVALID_COMMAND_LINE_DATE, awardEndDate), null); } } diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java index 6372e1891..5149a210d 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java @@ -68,8 +68,8 @@ public class GrantLoaderCLIRunner implements CommandLineRunner { @Option(name = "-z", aliases = {"-awardEndDate", "--awardEndDate"}, usage = "Date for the AWARD_END to start the query against Grant source system. This will cause " + "a return of all grants having a record with an AWARD_END after the supplied date. " + - "Syntax must be MM/dd/yyyy. If not specified, the default will be 01/01/2011") - protected static String awardEndDate = "01/01/2011"; + "Syntax must be yyyy-mm-dd. If not specified, the default will be 2011-01-01") + protected static String awardEndDate = "2011-01-01"; /** * Specifies an optional action - either "pull" or "load" - to restrict the operation of the application to only diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java index 28de175ce..cdfd9671d 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java @@ -15,14 +15,12 @@ */ package org.eclipse.pass.support.grant.data; -import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; -import java.time.format.ResolverStyle; import java.time.temporal.ChronoField; import org.apache.commons.lang3.StringUtils; @@ -38,9 +36,6 @@ public class DateTimeUtil { .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) .toFormatter(); - static final DateTimeFormatter DATE_FORMATTER = - DateTimeFormatter.ofPattern("M/d/uuuu") - .withResolverStyle(ResolverStyle.STRICT); private DateTimeUtil () { //never called @@ -57,10 +52,6 @@ public static ZonedDateTime createZonedDateTime(String dateString) throws GrantD LocalDateTime localDateTime = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER); return localDateTime.atZone(ZoneOffset.UTC); } - if (verifyDate(dateString)) { //we may have just a date - date format is mm/day/year - LocalDate localDate = LocalDate.parse(dateString, DATE_FORMATTER); - return localDate.atStartOfDay(ZoneOffset.UTC); - } if (StringUtils.isNotBlank(dateString)) { throw new GrantDataException("Invalid Format for " + dateString + ". Valid Format is " + DATE_TIME_PATTERN); @@ -73,28 +64,15 @@ public static ZonedDateTime createZonedDateTime(String dateString) throws GrantD * validity * (for example, "2018-02-31 ... " passes) * - * @param dateStr the date string to be checked + * @param dateTimeStr the datetime string to be checked * @return a boolean indicating whether the date matches the required format */ - public static boolean verifyDateTimeFormat(String dateStr) { - return checkDateTimeFormat(dateStr, DATE_TIME_FORMATTER); - } - - /** - * Date must be in the form "mm/dd/yyyy" - * @param date the date to verify - * @return true if date format is valid, false if not - */ - public static boolean verifyDate(String date) { - return checkDateTimeFormat(date, DATE_FORMATTER); - } - - private static boolean checkDateTimeFormat(String dateTimeStr, DateTimeFormatter formatter) { + public static boolean verifyDateTimeFormat(String dateTimeStr) { if (dateTimeStr == null) { return false; } try { - formatter.parse(dateTimeStr); + DATE_TIME_FORMATTER.parse(dateTimeStr); } catch (DateTimeParseException e) { return false; } diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/DateTimeUtilTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/DateTimeUtilTest.java index 8c9f060e0..53a16aa5d 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/DateTimeUtilTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/DateTimeUtilTest.java @@ -153,7 +153,7 @@ void testCreateZonedDateTime() throws GrantDataException { assertEquals(0, dateTime3.getNano()); assertEquals(ZoneOffset.UTC, dateTime3.getZone()); - String date = "01/30/2018"; + String date = "2018-01-30"; dateTime = DateTimeUtil.createZonedDateTime(date); assertEquals(2018, dateTime.getYear()); assertEquals(1, dateTime.getMonthValue()); @@ -184,20 +184,6 @@ void testCreateZonedDateTime_Invalid_timezoneNotSupported() { "Valid Format is uuuu-MM-dd[ [HH][:mm][:ss][[.SSS][.SS][.S]]]", grantDataException2.getMessage()); } - /** - * Test that verifyDate works - */ - @Test - void testDateVerify() { - String date = "01/01/2011"; - assertTrue(DateTimeUtil.verifyDate(date)); - - date = "02/30/1999"; - assertFalse(DateTimeUtil.verifyDate(date)); - - assertFalse(DateTimeUtil.verifyDate(null)); - } - /** * Test static timestamp utility method to verify it returns the later of two supplied timestamps */ From abbd0c87e6c0532cd1f0293dfacfe14c876ccfa3 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Mon, 3 Jun 2024 15:49:28 -0400 Subject: [PATCH 14/19] Update grant db rs mappings --- .../grant/data/jhu/JhuGrantDbConnector.java | 21 +++++++++++++----- .../grant/AbstractIntegrationTest.java | 2 -- .../support/grant/AbstractRoundTripTest.java | 20 +++++------------ .../support/grant/GrantLoaderCLITest.java | 4 ++-- .../grant/GrantLoaderFileRoundTripTest.java | 16 +++----------- .../support/grant/GrantLoaderLoadFileIT.java | 2 +- .../grant/GrantLoaderS3RoundTripTest.java | 16 +++----------- .../eclipse/pass/support/grant/TestUtil.java | 4 ++-- .../data/jhu/JhuPassUpdaterOrderedIT.java | 6 ++--- .../grant/data/jhu/JhuPassUpdaterTest.java | 8 +++---- .../src/test/resources/expected-csv.csv | 6 ++--- .../src/test/resources/logback-test.xml | 2 +- .../src/test/resources/test-load.csv | 22 +++++++++---------- 13 files changed, 55 insertions(+), 74 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index 75b6f0f75..dc84ac280 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -16,14 +16,17 @@ package org.eclipse.pass.support.grant.data.jhu; import java.sql.Connection; +import java.sql.Date; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; +import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; @@ -101,13 +104,12 @@ public class JhuGrantDbConnector implements GrantConnector { "JHU_PERSON_VIEW C " + "WHERE A.inst_proposal = B.inst_proposal " + "AND B.employee_id = C.employee_id " + - "AND A.AWARD_END_DATE >= STR_TO_DATE('01/01/2011', '%m/%d/%Y') " + - "AND B.ROLE != 'KP' " + + "AND A.AWARD_END_DATE >= '2011-01-01' " + "AND EXISTS (" + " select * from JHU_PASS_AWD_VIEW EA where" + " EA.UPDATE_TIMESTAMP > ?" + " and EA.SAP_GRANT_NUMBER = A.SAP_GRANT_NUMBER" + - " AND EA.AWARD_END_DATE >= STR_TO_DATE(?, '%m/%d/%Y') "; + " AND EA.AWARD_END_DATE >= ? "; private static final String SELECT_USER_SQL = "SELECT " + @@ -131,6 +133,12 @@ public class JhuGrantDbConnector implements GrantConnector { "FROM JHU_SPONSOR_VIEW " + "WHERE SPONSOR_CODE IN (%s)"; + private final Map jhuGrantDbRoleMapping = Map.of( + "PI", "P", + "Co-PI", "C", + "Co-I", "C" + ); + @Value("${grant.db.url}") private String grantDbUrl; @@ -172,7 +180,8 @@ private List retrieveGrantUpdates(String startDate, String aw ) { LocalDateTime startLd = LocalDateTime.from(DateTimeUtil.DATE_TIME_FORMATTER.parse(startDate)); ps.setTimestamp(1, Timestamp.valueOf(startLd)); - ps.setString(2, awardEndDate); + LocalDate awardEndFilter = LocalDate.from(DateTimeUtil.DATE_TIME_FORMATTER.parse(awardEndDate)); + ps.setDate(2, Date.valueOf(awardEndFilter)); if (StringUtils.isNotEmpty(grant)) { ps.setString(3, grant); } @@ -198,7 +207,9 @@ private List retrieveGrantUpdates(String startDate, String aw grantIngestRecord.setPiEmployeeId(rs.getString(C_USER_EMPLOYEE_ID)); grantIngestRecord.setPiInstitutionalId(rs.getString(C_USER_INSTITUTIONAL_ID)); grantIngestRecord.setUpdateTimeStamp(rs.getString(C_UPDATE_TIMESTAMP)); - grantIngestRecord.setPiRole(rs.getString(C_ABBREVIATED_ROLE)); + grantIngestRecord.setPiRole( + jhuGrantDbRoleMapping.get(rs.getString(C_ABBREVIATED_ROLE)) + ); LOG.debug("Record processed: {}", grantIngestRecord); if (!grantIngestRecords.contains(grantIngestRecord)) { grantIngestRecords.add(grantIngestRecord); diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/AbstractIntegrationTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/AbstractIntegrationTest.java index 183f4dbe2..4db9084f1 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/AbstractIntegrationTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/AbstractIntegrationTest.java @@ -20,7 +20,6 @@ import org.apache.maven.model.Model; import org.apache.maven.model.io.xpp3.MavenXpp3Reader; import org.eclipse.pass.support.client.PassClient; -import org.eclipse.pass.support.grant.data.GrantConnector; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -67,7 +66,6 @@ public abstract class AbstractIntegrationTest { @SpyBean protected PassClient passClient; @Autowired protected GrantLoaderApp grantLoaderApp; - @MockBean protected GrantConnector grantConnector; @MockBean protected GrantLoaderCLIRunner grantLoaderCLIRunner; @DynamicPropertySource diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/AbstractRoundTripTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/AbstractRoundTripTest.java index 03b04b89d..63943540e 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/AbstractRoundTripTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/AbstractRoundTripTest.java @@ -27,20 +27,12 @@ import org.eclipse.pass.support.client.model.AwardStatus; import org.eclipse.pass.support.client.model.Funder; import org.eclipse.pass.support.client.model.Grant; -import org.eclipse.pass.support.grant.data.GrantIngestRecord; /** * @author Russ Poetker (rpoetke1@jh.edu) */ public class AbstractRoundTripTest extends AbstractIntegrationTest { - protected List getTestIngestRecords() { - GrantIngestRecord piRecord1 = TestUtil.makeGrantIngestRecord(0, 0, "P"); - GrantIngestRecord coPiRecord1 = TestUtil.makeGrantIngestRecord(0, 1, "C"); - GrantIngestRecord piRecord2 = TestUtil.makeGrantIngestRecord(3, 1, "P"); - return List.of(piRecord1, coPiRecord1, piRecord2); - } - protected void verifyGrantOne() throws IOException { PassClientSelector grantSelector = new PassClientSelector<>(Grant.class); grantSelector.setFilter(RSQL.equals("localKey", "johnshopkins.edu:grant:10000001")); @@ -54,9 +46,9 @@ protected void verifyGrantOne() throws IOException { assertEquals("Stupendous \"Research Project\" I", passGrant.getProjectName()); assertEquals(AwardStatus.ACTIVE, passGrant.getAwardStatus()); - assertEquals("1999-01-01T00:00Z", passGrant.getAwardDate().toString()); - assertEquals("2000-07-01T00:00Z", passGrant.getStartDate().toString()); - assertEquals("2004-06-30T00:00Z", passGrant.getEndDate().toString()); + assertEquals("2021-01-01T00:00Z", passGrant.getAwardDate().toString()); + assertEquals("2021-07-01T00:00Z", passGrant.getStartDate().toString()); + assertEquals("2024-06-30T00:00Z", passGrant.getEndDate().toString()); Funder primaryFunder = passClient.getObject(passGrant.getPrimaryFunder(), "policy"); assertEquals("johnshopkins.edu:funder:20000001", primaryFunder.getLocalKey()); @@ -96,9 +88,9 @@ protected void verifyGrantTwo() throws IOException { assertEquals("B10000003", passGrant.getAwardNumber()); assertEquals("Stupendous Research ProjectIV", passGrant.getProjectName()); assertEquals(AwardStatus.ACTIVE, passGrant.getAwardStatus()); - assertEquals("2004-01-01T00:00Z", passGrant.getAwardDate().toString()); - assertEquals("2004-07-01T00:00Z", passGrant.getStartDate().toString()); - assertEquals("2007-06-30T00:00Z", passGrant.getEndDate().toString()); + assertEquals("2014-01-01T00:00Z", passGrant.getAwardDate().toString()); + assertEquals("2014-07-01T00:00Z", passGrant.getStartDate().toString()); + assertEquals("2017-06-30T00:00Z", passGrant.getEndDate().toString()); Funder primaryFunder = passClient.getObject(passGrant.getPrimaryFunder(), "policy"); assertEquals("johnshopkins.edu:funder:20000001", primaryFunder.getLocalKey()); diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderCLITest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderCLITest.java index b85cc5628..e9e8ab327 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderCLITest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderCLITest.java @@ -29,7 +29,7 @@ * @author Russ Poetker (rpoetke1@jh.edu) */ @SpringBootTest(classes = GrantLoaderCLI.class, - args = {"--startDateTime=2024-01-01T00:00:00", "-awardEndDate=01/01/2025", "-action=pull", "test-pull-file.csv"}) + args = {"--startDateTime=2024-01-01T00:00:00", "-awardEndDate=2025-01-01", "-action=pull", "test-pull-file.csv"}) @TestPropertySource("classpath:test-application.properties") public class GrantLoaderCLITest { @@ -39,7 +39,7 @@ public class GrantLoaderCLITest { public void testHarvesterCLI() throws PassCliException { // GIVEN/WHEN // THEN - verify(grantLoaderApp).run(eq("2024-01-01T00:00:00"), eq("01/01/2025"), eq("grant"), + verify(grantLoaderApp).run(eq("2024-01-01T00:00:00"), eq("2025-01-01"), eq("grant"), eq("pull"), eq("test-pull-file.csv"), eq(null)); } diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderFileRoundTripTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderFileRoundTripTest.java index bec1a1c7b..7911ffa62 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderFileRoundTripTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderFileRoundTripTest.java @@ -16,18 +16,12 @@ package org.eclipse.pass.support.grant; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.SQLException; -import java.util.List; import org.eclipse.pass.support.client.model.Policy; -import org.eclipse.pass.support.grant.data.GrantIngestRecord; import org.eclipse.pass.support.grant.data.PassUpdater; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -50,7 +44,7 @@ void cleanUp() throws IOException { } @Test - public void testRoundTripCvsFile() throws PassCliException, SQLException, IOException { + public void testRoundTripCvsFile() throws PassCliException, IOException { // GIVEN Policy policy = new Policy(); policy.setTitle("test policy"); @@ -60,12 +54,8 @@ public void testRoundTripCvsFile() throws PassCliException, SQLException, IOExce Files.createFile(TEST_CSV_PATH); Files.createFile(GRANT_UPTS_PATH); - List grantIngestRecordList = getTestIngestRecords(); - doReturn(grantIngestRecordList).when(grantConnector).retrieveUpdates(anyString(), anyString(), anyString(), - any()); - // WHEN - grantLoaderApp.run("2011-01-01 00:00:00", "01/01/2011", + grantLoaderApp.run("2011-01-01 00:00:00", "2011-01-01", "grant", "pull", "file:./src/test/resources/test-pull.csv", null); // THEN @@ -75,7 +65,7 @@ public void testRoundTripCvsFile() throws PassCliException, SQLException, IOExce // WHEN // Use CSV file create above and load into PASS - grantLoaderApp.run("", "01/01/2011", "grant", + grantLoaderApp.run("", "2011-01-01", "grant", "load", "file:./src/test/resources/test-pull.csv", null); // THEN diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderLoadFileIT.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderLoadFileIT.java index 69c66e709..075b4c2c7 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderLoadFileIT.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderLoadFileIT.java @@ -61,7 +61,7 @@ public void testLoadCvsFile() throws IOException { // WHEN PassCliException passCliException = assertThrows(PassCliException.class, () -> { - grantLoaderApp.run("", "01/01/2011", "grant", + grantLoaderApp.run("", "2011-01-01", "grant", "load", "file:./src/test/resources/test-load.csv", null); }); diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderS3RoundTripTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderS3RoundTripTest.java index 0f00c3a9a..e142420b1 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderS3RoundTripTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderS3RoundTripTest.java @@ -16,9 +16,6 @@ package org.eclipse.pass.support.grant; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doReturn; import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; import java.io.IOException; @@ -26,13 +23,10 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.SQLException; -import java.util.List; import io.awspring.cloud.s3.S3Resource; import io.awspring.cloud.s3.S3Template; import org.eclipse.pass.support.client.model.Policy; -import org.eclipse.pass.support.grant.data.GrantIngestRecord; import org.eclipse.pass.support.grant.data.PassUpdater; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -88,18 +82,14 @@ static void beforeAll() throws IOException, InterruptedException { } @Test - public void testRoundTripCvsFileS3() throws PassCliException, SQLException, IOException { + public void testRoundTripCvsFileS3() throws PassCliException, IOException { // GIVEN Policy policy = new Policy(); policy.setTitle("test policy"); passClient.createObject(policy); - List grantIngestRecordList = getTestIngestRecords(); - doReturn(grantIngestRecordList).when(grantConnector).retrieveUpdates(anyString(), anyString(), anyString(), - any()); - // WHEN - grantLoaderApp.run("2011-01-01 00:00:00", "01/01/2011", + grantLoaderApp.run("2011-01-01 00:00:00", "2011-01-01", "grant", "pull", "s3://test-bucket/test-pull.csv", null); // THEN @@ -113,7 +103,7 @@ public void testRoundTripCvsFileS3() throws PassCliException, SQLException, IOEx // WHEN // Use CSV file create above and load into PASS - grantLoaderApp.run("", "01/01/2011", "grant", + grantLoaderApp.run("", "2011-01-01", "grant", "load", "s3://test-bucket/test-pull.csv", null); // THEN diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/TestUtil.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/TestUtil.java index 1b7eadab9..fb47f19aa 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/TestUtil.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/TestUtil.java @@ -25,8 +25,8 @@ public class TestUtil { public final static String[] grantProjectName = {"Stupendous \"Research Project\" I", "Stupendous Research Project II", "Stupendous Research ProjectIII", "Stupendous Research ProjectIV", "Stupendous Research ProjectV"}; - public final static String[] grantAwardDate = {"01/01/1999", "01/01/2001", "01/01/2003", "01/01/2004", - "01/01/2005"}; + public final static String[] grantAwardDate = {"1999-01-01", "2001-01-01", "2003-01-01", "2004-01-01", + "2005-01-01"}; //these appear to ge the same for all awards public final static String[] grantStartDate = {"2000-07-01 00:00:00", "2001-07-01 00:00:00", "2003-07-01 00:00:00", "2004-07-01 00:00:00", diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterOrderedIT.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterOrderedIT.java index cccf4290b..bf8b018f3 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterOrderedIT.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterOrderedIT.java @@ -49,9 +49,9 @@ public class JhuPassUpdaterOrderedIT extends AbstractIntegrationTest { private final String[] grantLocalKey = {"10000000", "10000000", "10000000"}; //all the same private final String[] grantProjectName = {"Awesome Research Project I", "Awesome Research Project II", "Awesome Research Project III"}; - private final String[] grantAwardDate = {"01/01/1999", "01/01/2001", "01/01/2003"}; - private final String[] grantStartDate = {"07/01/2000", "07/01/2002", "07/01/2004"}; - private final String[] grantEndDate = {"06/30/2002", "06/30/2004", "06/30/2006"}; + private final String[] grantAwardDate = {"1999-01-01", "2001-01-01", "2003-01-01"}; + private final String[] grantStartDate = {"2000-07-01", "2002-07-01", "2004-07-01"}; + private final String[] grantEndDate = {"2002-06-30", "2004-06-30", "2006-06-30"}; private final String[] grantUpdateTimestamp = {"2006-03-11 00:00:00.0", "2010-04-05 00:00:00.0", "2015-11-11 00:00:00.0"}; private final String[] userEmployeeId = {"30000000", "30000001", "30000002"}; diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterTest.java index 54924f97b..b7eb541c2 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuPassUpdaterTest.java @@ -96,7 +96,7 @@ public void testUpdatePassGrant_Success_NewGrant() throws IOException, IllegalAc assertEquals("12345678", grant.getAwardNumber()); assertEquals(AwardStatus.ACTIVE, grant.getAwardStatus()); assertEquals("johnshopkins.edu:grant:8675309", grant.getLocalKey()); - Assertions.assertEquals(DateTimeUtil.createZonedDateTime("01/01/2000"), grant.getAwardDate()); + Assertions.assertEquals(DateTimeUtil.createZonedDateTime("2000-01-01"), grant.getAwardDate()); assertEquals(DateTimeUtil.createZonedDateTime("2001-01-01"), grant.getStartDate()); assertEquals(DateTimeUtil.createZonedDateTime("2002-01-01"), grant.getEndDate()); assertEquals("2018-01-01 00:00:00.0", jhuPassUpdater.getLatestUpdate());//latest @@ -133,9 +133,9 @@ private List buildTestInputResultSet() { String awardStatus = "Active"; String localKey = "8675309"; String projectName = "Moo Project"; - String awardDate = "01/01/2000"; - String startDate = "01/01/2001"; - String endDate = "01/01/2002"; + String awardDate = "2000-01-01"; + String startDate = "2001-01-01"; + String endDate = "2002-01-01"; String directFunderId = "000029282"; String directFunderName = "JHU Department of Synergy"; String primaryFunderId = "8675309"; diff --git a/pass-grant-loader/src/test/resources/expected-csv.csv b/pass-grant-loader/src/test/resources/expected-csv.csv index 9ca1e3120..75155ab78 100644 --- a/pass-grant-loader/src/test/resources/expected-csv.csv +++ b/pass-grant-loader/src/test/resources/expected-csv.csv @@ -1,4 +1,4 @@ GRANT_NUMBER,GRANT_TITLE,AWARD_NUMBER,AWARD_STATUS,AWARD_DATE,AWARD_START,AWARD_END,PRIMARY_FUNDER_NAME,PRIMARY_FUNDER_CODE,DIRECT_FUNDER_NAME,DIRECT_FUNDER_CODE,PI_FIRST_NAME,PI_MIDDLE_NAME,PI_LAST_NAME,PI_EMAIL,PI_INSTITUTIONAL_ID,PI_EMPLOYEE_ID,PI_ROLE,UPDATE_TIMESTAMP -10000001,"Stupendous ""Research Project"" I",B10000000,Active,01/01/1999,2000-07-01 00:00:00,2004-06-30 00:00:00,J L Gotrocks Foundation,20000001,Enormous State University,20000000,Amanda,Bea,Reckondwith,arecko1@jhu.edu,arecko1,31000000,P,2006-03-11 00:00:00.0 -10000001,"Stupendous ""Research Project"" I",B10000000,Active,01/01/1999,2000-07-01 00:00:00,2004-06-30 00:00:00,J L Gotrocks Foundation,20000001,Enormous State University,20000000,Skip,Avery,Class,sclass1@jhu.edu,sclass1,31000001,C,2006-03-11 00:00:00.0 -10000002,Stupendous Research ProjectIV,B10000003,Active,01/01/2004,2004-07-01 00:00:00,2007-06-30 00:00:00,J L Gotrocks Foundation,20000001,Enormous State University,20000000,Skip,Avery,Class,sclass1@jhu.edu,sclass1,31000001,P,2016-11-11 00:00:00.0 +10000001,"Stupendous ""Research Project"" I",B10000000,Active,2021-01-01,2021-07-01,2024-06-30,J L Gotrocks Foundation,20000001,Enormous State University,20000000,Amanda,Bea,Reckondwith,arecko1@jhu.edu,arecko1,31000000,P,2023-03-11 00:00:00 +10000001,"Stupendous ""Research Project"" I",B10000000,Active,2021-01-01,2021-07-01,2024-06-30,J L Gotrocks Foundation,20000001,Enormous State University,20000000,Skip,Avery,Class,sclass1@jhu.edu,sclass1,31000001,C,2023-03-11 00:00:00 +10000002,Stupendous Research ProjectIV,B10000003,Active,2014-01-01,2014-07-01,2017-06-30,J L Gotrocks Foundation,20000001,Enormous State University,20000000,Skip,Avery,Class,sclass1@jhu.edu,sclass1,31000001,P,2016-11-11 00:00:00 diff --git a/pass-grant-loader/src/test/resources/logback-test.xml b/pass-grant-loader/src/test/resources/logback-test.xml index 0979af53b..af19b804a 100644 --- a/pass-grant-loader/src/test/resources/logback-test.xml +++ b/pass-grant-loader/src/test/resources/logback-test.xml @@ -27,7 +27,7 @@ - + \ No newline at end of file diff --git a/pass-grant-loader/src/test/resources/test-load.csv b/pass-grant-loader/src/test/resources/test-load.csv index af7942bd1..9d6037313 100644 --- a/pass-grant-loader/src/test/resources/test-load.csv +++ b/pass-grant-loader/src/test/resources/test-load.csv @@ -1,15 +1,15 @@ GRANT_NUMBER,GRANT_TITLE,AWARD_NUMBER,AWARD_STATUS,AWARD_DATE,AWARD_START,AWARD_END,PRIMARY_FUNDER_NAME,PRIMARY_FUNDER_CODE,DIRECT_FUNDER_NAME,DIRECT_FUNDER_CODE,PI_FIRST_NAME,PI_MIDDLE_NAME,PI_LAST_NAME,PI_EMAIL,PI_INSTITUTIONAL_ID,PI_EMPLOYEE_ID,PI_ROLE,UPDATE_TIMESTAMP -138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,9/29/2021,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,userone,123456,P,2023-02-01 13:10:00 -138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,9/29/2021,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,userone,123456,P,2023-02-01 13:10:00 -138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,9/29/2021,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,userone,123456,P,2023-02-01 13:10:00 -138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,9/29/2021,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserThreeFn,,UserThreeLn,,,789123,C,2023-02-01 13:10:00 -138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,9/29/2021,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserThreeFn,,UserThreeLn,,,789123,C,2023-02-01 13:10:00 -130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,12/14/2018,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserThreeFn,,UserThreeLn,,,789123,C,2023-03-10 15:15:20.234 -130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,12/14/2018,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserTwoFn,UserTwoMn,UserTwoLn,,usertwo,,P,2023-03-10 15:15:20.234 -130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,12/14/2018,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,userone,123456,C,2023-03-10 15:15:20.234 -130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,12/14/2018,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserFourFn,UserFourMn,UserFourLn,userfour@jhu.edu,userfour,,C,2023-03-10 15:15:20.234 -130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,12/14/2018,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserFiveFn,,UserFiveLn,userfive@jhu.edu,,,C,2023-03-10 15:15:20.234 -130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,12/14/2018,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserSixFn,,UserSixLn,usersix@jhu.edu,,,C,2023-03-10 15:15:20.234 +138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,2021-09-29,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,userone,123456,P,2023-02-01 13:10:00 +138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,2021-09-29,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,userone,123456,P,2023-02-01 13:10:00 +138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,2021-09-29,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,userone,123456,P,2023-02-01 13:10:00 +138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,2021-09-29,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserThreeFn,,UserThreeLn,,,789123,C,2023-02-01 13:10:00 +138058,NPTX2: Preserving memory circuits in normative aging and Alzheimer's disease,625628,active,2021-09-29,2021-05-01 09:00:00,2026-04-30 14:00:00,,,UNIV OF ARIZONA,301313,UserThreeFn,,UserThreeLn,,,789123,C,2023-02-01 13:10:00 +130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,2018-12-14,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserThreeFn,,UserThreeLn,,,789123,C,2023-03-10 15:15:20.234 +130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,2018-12-14,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserTwoFn,UserTwoMn,UserTwoLn,,usertwo,,P,2023-03-10 15:15:20.234 +130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,2018-12-14,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,userone,123456,C,2023-03-10 15:15:20.234 +130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,2018-12-14,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserFourFn,UserFourMn,UserFourLn,userfour@jhu.edu,userfour,,C,2023-03-10 15:15:20.234 +130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,2018-12-14,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserFiveFn,,UserFiveLn,userfive@jhu.edu,,,C,2023-03-10 15:15:20.234 +130823,The Subclinical Vascular Contributions to Alzheimer's Disease: The Multi-Ethnic Study of Atherosclerosis (MESA) ,107-33664-1000055108,active,2018-12-14,2018-08-15 10:00:00,2025-05-31 00:00:00,NATIONAL INSTITUTES OF HEALTH,300865,WAKE FOREST UNIV,304106,UserSixFn,,UserSixLn,usersix@jhu.edu,,,C,2023-03-10 15:15:20.234 333333,Test Grant 3,323453,,2021-10-01,2021-05-01,2026-04-30,NATIONAL INSTITUTES OF HEALTH,300865,NATIONAL INSTITUTES OF HEALTH,300865,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,P,2023-02-01 444444,Test Grant 4 - invalid award date,423454,active,10-01-2021,2021-05-01,2026-04-30,NATIONAL INSTITUTES OF HEALTH,300865,NATIONAL INSTITUTES OF HEALTH,300865,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,P,2023-02-01 555555,Test Grant 5 - missing required funder,423454,active,2021-10-01,2021-05-01,2026-04-30,,,,,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,P,2023-02-01 From 5cae4f3462a9b9b1dbe15a1a24a7f5426f6d5909 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Mon, 3 Jun 2024 16:13:55 -0400 Subject: [PATCH 15/19] Map KP to co pi --- .../pass/support/grant/data/jhu/JhuGrantDbConnector.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index dc84ac280..bd7b00825 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -136,7 +136,8 @@ public class JhuGrantDbConnector implements GrantConnector { private final Map jhuGrantDbRoleMapping = Map.of( "PI", "P", "Co-PI", "C", - "Co-I", "C" + "Co-I", "C", + "KP", "C" ); @Value("${grant.db.url}") From 2d9ce1c36a8b1bee863282cd52498e1ba995718c Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Tue, 4 Jun 2024 08:20:20 -0400 Subject: [PATCH 16/19] Clean date format in tests --- .../org/eclipse/pass/support/grant/GrantLoaderManualTest.java | 4 ++-- .../pass/support/grant/GrantLoaderS3RoundTripTest.java | 2 +- .../org/eclipse/pass/support/grant/data/DateTimeUtilTest.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java index 55918e0c7..cb269c6f6 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderManualTest.java @@ -51,7 +51,7 @@ public class GrantLoaderManualTest { @Disabled @Test public void testPullGrantFile() throws PassCliException { - grantLoaderApp.run("2023-04-01 00:00:00", "04/01/2023", + grantLoaderApp.run("2023-04-01 00:00:00", "2023-04-01", "grant", "pull", "file:./src/test/resources/your-file.csv", null); } @@ -62,7 +62,7 @@ public void testPullGrantFile() throws PassCliException { @Disabled @Test public void testLoadGrantFile() throws PassCliException { - grantLoaderApp.run("2023-04-01 00:00:00", "04/01/2023", + grantLoaderApp.run("2023-04-01 00:00:00", "2023-04-01", "grant", "load", "file:./src/test/resources/your-file.csv", null); } diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderS3RoundTripTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderS3RoundTripTest.java index e142420b1..04ecc6553 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderS3RoundTripTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderS3RoundTripTest.java @@ -118,7 +118,7 @@ public void testRoundTripCvsFileS3() throws PassCliException, IOException { // WHEN - run again to verify grant update timestamps String firstLastUpdate = passUpdater.getLatestUpdate(); - grantLoaderApp.run("", "01/01/2011", "grant", + grantLoaderApp.run("", "2011-01-01", "grant", "load", "s3://test-bucket/test-pull.csv", null); S3Resource actualTestGrantUpTs2 = s3Template.download("test-bucket", "s3-testgrantupdatets"); diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/DateTimeUtilTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/DateTimeUtilTest.java index 53a16aa5d..424a469b2 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/DateTimeUtilTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/DateTimeUtilTest.java @@ -176,11 +176,11 @@ void testCreateZonedDateTime_Invalid_timezoneNotSupported() { assertEquals("Invalid Format for 2018-01-30 23:59:58.0+05:00. " + "Valid Format is uuuu-MM-dd[ [HH][:mm][:ss][[.SSS][.SS][.S]]]", grantDataException.getMessage()); - String date = "01/30/2018-03:00"; + String date = "2018-03-01-03:00"; GrantDataException grantDataException2 = assertThrows(GrantDataException.class, () -> { DateTimeUtil.createZonedDateTime(date); }); - assertEquals("Invalid Format for 01/30/2018-03:00. " + + assertEquals("Invalid Format for 2018-03-01-03:00. " + "Valid Format is uuuu-MM-dd[ [HH][:mm][:ss][[.SSS][.SS][.S]]]", grantDataException2.getMessage()); } From b79326d116614a37a2c60a4e304474cc66f3a07c Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Tue, 4 Jun 2024 08:38:53 -0400 Subject: [PATCH 17/19] Add more grant db tests --- .../grant/data/jhu/JhuGrantDbConnector.java | 5 +- .../jhu/JhuGrantDbConnectorManualTest.java | 72 --------------- .../data/jhu/JhuGrantDbConnectorTest.java | 88 +++++++++++++++++++ pass-grant-loader/src/test/resources/data.sql | 16 +++- .../src/test/resources/schema.sql | 15 +++- 5 files changed, 115 insertions(+), 81 deletions(-) delete mode 100644 pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorManualTest.java create mode 100644 pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index bd7b00825..c5eb39cba 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -175,6 +175,8 @@ private List retrieveGrantUpdates(String startDate, String aw String sql = buildGrantQueryString(grant); List grantIngestRecords = new ArrayList<>(); + LOG.info("Retrieving Grants from JHU Grant DB using start date={}, award end date={}", startDate, awardEndDate); + try ( Connection con = DriverManager.getConnection(grantDbUrl, grantDbUser, grantDbPassword); PreparedStatement ps = con.prepareStatement(sql); @@ -218,7 +220,7 @@ private List retrieveGrantUpdates(String startDate, String aw } } } - LOG.info("Retrieved result set from JHU Grant DB: {} records processed", grantIngestRecords.size()); + LOG.info("Retrieved Grants result set from JHU Grant DB: {} records processed", grantIngestRecords.size()); return grantIngestRecords; } @@ -249,6 +251,7 @@ private List retrieveFunderUpdates(Set funderIds) thr } } } + LOG.info("Retrieved Funders result set from JHU Grant DB: {} records processed", grantIngestRecords.size()); return grantIngestRecords; } diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorManualTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorManualTest.java deleted file mode 100644 index 8bacb21bc..000000000 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorManualTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2023 Johns Hopkins University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.eclipse.pass.support.grant.data.jhu; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -import java.sql.SQLException; -import java.util.List; - -import org.eclipse.pass.support.grant.data.GrantIngestRecord; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.TestPropertySource; - -/** - * Test class for the JHU Grant DB connector. This is strictly a manual test for querying the database. - * This test is Disabled, you can enable it and run each query test if needed for validation. - *

- * In order to run the tests, you must put a connection.properties file with valid url and creds in the - * test/resources dir. - * - * @author jrm@jhu.edu - */ -@SpringBootTest -@TestPropertySource("classpath:test-application.properties") -@Disabled -public class JhuGrantDbConnectorManualTest { - - @Autowired private JhuGrantDbConnector connector; - - @Disabled - @Test - public void testGrantQuery() throws SQLException { - List results = - connector.retrieveUpdates("2023-10-20 00:00:00", "01/01/2011", "grant", null); - assertNotNull(results); - } - - @Disabled - @Test - public void testUserQuery() throws SQLException { - List results = - connector.retrieveUpdates("2023-10-20 00:00:00", null, "user", null); - assertNotNull(results); - } - - @Disabled - @Test - public void testFunderQuery() throws SQLException { - List results = - connector.retrieveUpdates(null, null, "funder", null); - assertNotNull(results); - } - -} - - diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java new file mode 100644 index 000000000..30f354c37 --- /dev/null +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2023 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.eclipse.pass.support.grant.data.jhu; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.SQLException; +import java.util.List; + +import org.eclipse.pass.support.grant.GrantLoaderCLIRunner; +import org.eclipse.pass.support.grant.data.GrantIngestRecord; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; + +/** + * @author jrm@jhu.edu + */ +@SpringBootTest +@TestPropertySource("classpath:test-application.properties") +public class JhuGrantDbConnectorTest { + + @MockBean protected GrantLoaderCLIRunner grantLoaderCLIRunner; + @Autowired private JhuGrantDbConnector connector; + + @Test + public void testGrantQuery() throws SQLException { + List results = + connector.retrieveUpdates("2011-01-01 00:00:00", "2011-01-01", "grant", null); + List actualAwardNums = results.stream().map(GrantIngestRecord::getAwardNumber).distinct().toList(); + assertEquals(2, actualAwardNums.size()); + assertTrue(actualAwardNums.contains("B10000000")); + assertTrue(actualAwardNums.contains("B10000003")); + } + + @Test + public void testGrantQueryStartDate() throws SQLException { + List results = + connector.retrieveUpdates("2023-03-01 00:00:00", "2011-01-01", "grant", null); + List actualAwardNums = results.stream().map(GrantIngestRecord::getAwardNumber).distinct().toList(); + assertEquals(1, actualAwardNums.size()); + assertTrue(actualAwardNums.contains("B10000000")); + } + + @Test + public void testGrantQueryAwardEndDate() throws SQLException { + List results = + connector.retrieveUpdates("2011-01-01 00:00:00", "2024-01-01", "grant", null); + List actualAwardNums = results.stream().map(GrantIngestRecord::getAwardNumber).distinct().toList(); + assertEquals(1, actualAwardNums.size()); + assertTrue(actualAwardNums.contains("B10000000")); + } + + @Test + public void testUserQuery() throws SQLException { + List results = + connector.retrieveUpdates("2023-03-01 00:00:00", null, "user", null); + assertEquals(1, results.size()); + assertEquals("sclass1@jhu.edu", results.get(0).getPiEmail()); + } + + @Test + public void testFunderQuery() throws SQLException { + List results = + connector.retrieveUpdates(null, null, "funder", null); + assertEquals(2, results.size()); + List actualNames = results.stream().map(GrantIngestRecord::getPrimaryFunderName).toList(); + assertTrue(actualNames.contains("Sponsor 1")); + assertTrue(actualNames.contains("Sponsor 2")); + } + +} diff --git a/pass-grant-loader/src/test/resources/data.sql b/pass-grant-loader/src/test/resources/data.sql index ed077ec0c..0096583c6 100644 --- a/pass-grant-loader/src/test/resources/data.sql +++ b/pass-grant-loader/src/test/resources/data.sql @@ -1,3 +1,7 @@ +DELETE FROM JHU_PASS_AWD_VIEW; +DELETE FROM JHU_FIBI_IP_INV_VIEW; +DELETE FROM JHU_PERSON_VIEW; +DELETE FROM JHU_SPONSOR_VIEW; INSERT INTO JHU_PASS_AWD_VIEW (INST_PROPOSAL, AWARD_ID, AWARD_STATUS, SAP_GRANT_NUMBER, TITLE, AWARD_DATE, AWARD_START_DATE, AWARD_END_DATE, SPONSOR_CODE, SPONSOR_NAME, PRIME_SPONSOR_CODE, PRIME_SPONSOR_NAME, @@ -10,10 +14,10 @@ VALUES ('1', '31000000', 'PI'); INSERT INTO JHU_FIBI_IP_INV_VIEW (INST_PROPOSAL, EMPLOYEE_ID, ROLE) VALUES ('1', '31000001', 'Co-I'); -INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID) -VALUES ('31000000', 'Amanda', 'Bea', 'Reckondwith', 'arecko1@jhu.edu', 'arecko1'); -INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID) -VALUES ('31000001', 'Skip', 'Avery', 'Class', 'sclass1@jhu.edu', 'sclass1'); +INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID, UPDATE_TIMESTAMP) +VALUES ('31000000', 'Amanda', 'Bea', 'Reckondwith', 'arecko1@jhu.edu', 'arecko1', '2022-03-11 00:00:00.0'); +INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID, UPDATE_TIMESTAMP) +VALUES ('31000001', 'Skip', 'Avery', 'Class', 'sclass1@jhu.edu', 'sclass1', '2023-03-11 00:00:00.0'); INSERT INTO JHU_PASS_AWD_VIEW (INST_PROPOSAL, AWARD_ID, AWARD_STATUS, SAP_GRANT_NUMBER, TITLE, AWARD_DATE, AWARD_START_DATE, AWARD_END_DATE, SPONSOR_CODE, SPONSOR_NAME, PRIME_SPONSOR_CODE, PRIME_SPONSOR_NAME, @@ -24,4 +28,8 @@ VALUES('2', 'B10000003', 'Active', '10000002', 'Stupendous Research ProjectIV', INSERT INTO JHU_FIBI_IP_INV_VIEW (INST_PROPOSAL, EMPLOYEE_ID, ROLE) VALUES ('2', '31000001', 'PI'); +INSERT INTO JHU_SPONSOR_VIEW (SPONSOR_CODE, SPONSOR_NAME) +VALUES ('20000000', 'Sponsor 1'); +INSERT INTO JHU_SPONSOR_VIEW (SPONSOR_CODE, SPONSOR_NAME) +VALUES ('20000001', 'Sponsor 2'); diff --git a/pass-grant-loader/src/test/resources/schema.sql b/pass-grant-loader/src/test/resources/schema.sql index a518370d7..e5ea53fbc 100644 --- a/pass-grant-loader/src/test/resources/schema.sql +++ b/pass-grant-loader/src/test/resources/schema.sql @@ -1,5 +1,6 @@ +DROP ALL OBJECTS; -CREATE TABLE IF NOT EXISTS JHU_PASS_AWD_VIEW ( +CREATE TABLE JHU_PASS_AWD_VIEW ( INST_PROPOSAL varchar(100) NOT NULL, AWARD_ID varchar(100) NOT NULL, AWARD_STATUS varchar(100), @@ -15,17 +16,23 @@ CREATE TABLE IF NOT EXISTS JHU_PASS_AWD_VIEW ( UPDATE_TIMESTAMP timestamp ); -CREATE TABLE IF NOT EXISTS JHU_FIBI_IP_INV_VIEW ( +CREATE TABLE JHU_FIBI_IP_INV_VIEW ( INST_PROPOSAL varchar(100) NOT NULL, EMPLOYEE_ID varchar(100), ROLE varchar(100) ); -CREATE TABLE IF NOT EXISTS JHU_PERSON_VIEW ( +CREATE TABLE JHU_PERSON_VIEW ( EMPLOYEE_ID varchar(100), FIRST_NAME varchar(100) NOT NULL, MIDDLE_NAME varchar(100), LAST_NAME varchar(100), EMAIL_ADDRESS varchar(100), - JHED_ID varchar(100) + JHED_ID varchar(100), + UPDATE_TIMESTAMP timestamp +); + +CREATE TABLE IF NOT EXISTS JHU_SPONSOR_VIEW ( + SPONSOR_CODE varchar(100) NOT NULL, + SPONSOR_NAME varchar(100) ); \ No newline at end of file From fe05542338ffe31e4d2708c7589e6cf245a70765 Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Wed, 26 Jun 2024 11:48:38 -0400 Subject: [PATCH 18/19] Add KP role filter in grant sql --- .../pass/support/grant/data/jhu/JhuGrantDbConnector.java | 5 ++--- .../pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java | 5 +++++ pass-grant-loader/src/test/resources/data.sql | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java index c5eb39cba..10995b506 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnector.java @@ -105,6 +105,7 @@ public class JhuGrantDbConnector implements GrantConnector { "WHERE A.inst_proposal = B.inst_proposal " + "AND B.employee_id = C.employee_id " + "AND A.AWARD_END_DATE >= '2011-01-01' " + + "AND B.ROLE != 'KP' " + "AND EXISTS (" + " select * from JHU_PASS_AWD_VIEW EA where" + " EA.UPDATE_TIMESTAMP > ?" + @@ -123,7 +124,6 @@ public class JhuGrantDbConnector implements GrantConnector { "FROM JHU_PERSON_VIEW A, " + "JHU_FIBI_IP_INV_VIEW B " + "WHERE A.employee_id = B.employee_id " + - // todo need reply from Bob on this "and A.UPDATE_TIMESTAMP > ?"; private static final String SELECT_FUNDER_SQL = @@ -136,8 +136,7 @@ public class JhuGrantDbConnector implements GrantConnector { private final Map jhuGrantDbRoleMapping = Map.of( "PI", "P", "Co-PI", "C", - "Co-I", "C", - "KP", "C" + "Co-I", "C" ); @Value("${grant.db.url}") diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java index 30f354c37..eb3f7c492 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/data/jhu/JhuGrantDbConnectorTest.java @@ -47,6 +47,11 @@ public void testGrantQuery() throws SQLException { assertEquals(2, actualAwardNums.size()); assertTrue(actualAwardNums.contains("B10000000")); assertTrue(actualAwardNums.contains("B10000003")); + // Verify KP user is not in result set + List actualPiEmails = results.stream().map(GrantIngestRecord::getPiEmail).distinct().toList(); + assertEquals(2, actualPiEmails.size()); + assertTrue(actualPiEmails.contains("arecko1@jhu.edu")); + assertTrue(actualPiEmails.contains("sclass1@jhu.edu")); } @Test diff --git a/pass-grant-loader/src/test/resources/data.sql b/pass-grant-loader/src/test/resources/data.sql index 0096583c6..86f1bd7ab 100644 --- a/pass-grant-loader/src/test/resources/data.sql +++ b/pass-grant-loader/src/test/resources/data.sql @@ -13,11 +13,15 @@ INSERT INTO JHU_FIBI_IP_INV_VIEW (INST_PROPOSAL, EMPLOYEE_ID, ROLE) VALUES ('1', '31000000', 'PI'); INSERT INTO JHU_FIBI_IP_INV_VIEW (INST_PROPOSAL, EMPLOYEE_ID, ROLE) VALUES ('1', '31000001', 'Co-I'); +INSERT INTO JHU_FIBI_IP_INV_VIEW (INST_PROPOSAL, EMPLOYEE_ID, ROLE) +VALUES ('1', '31000002', 'KP'); INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID, UPDATE_TIMESTAMP) VALUES ('31000000', 'Amanda', 'Bea', 'Reckondwith', 'arecko1@jhu.edu', 'arecko1', '2022-03-11 00:00:00.0'); INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID, UPDATE_TIMESTAMP) VALUES ('31000001', 'Skip', 'Avery', 'Class', 'sclass1@jhu.edu', 'sclass1', '2023-03-11 00:00:00.0'); +INSERT INTO JHU_PERSON_VIEW (EMPLOYEE_ID, FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID, UPDATE_TIMESTAMP) +VALUES ('31000002', 'John', null, 'Doe', 'jdoe1@jhu.edu', 'jdoe1', '2022-03-11 00:00:00.0'); INSERT INTO JHU_PASS_AWD_VIEW (INST_PROPOSAL, AWARD_ID, AWARD_STATUS, SAP_GRANT_NUMBER, TITLE, AWARD_DATE, AWARD_START_DATE, AWARD_END_DATE, SPONSOR_CODE, SPONSOR_NAME, PRIME_SPONSOR_CODE, PRIME_SPONSOR_NAME, From 3e88f955302de0ae5c38bfbb0633f87e30a1d8ad Mon Sep 17 00:00:00 2001 From: Russ Poetker Date: Wed, 26 Jun 2024 13:24:13 -0400 Subject: [PATCH 19/19] Remove K from user role enum And add null test to ingest validation --- .../pass/support/grant/data/GrantIngestUserRole.java | 6 +----- .../pass/support/grant/GrantLoaderLoadFileIT.java | 10 +++++++++- pass-grant-loader/src/test/resources/test-load.csv | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/GrantIngestUserRole.java b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/GrantIngestUserRole.java index 8b747fc5b..7b91d4eb3 100644 --- a/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/GrantIngestUserRole.java +++ b/pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/GrantIngestUserRole.java @@ -26,9 +26,5 @@ public enum GrantIngestUserRole { /** * Co-PI */ - C, - /** - * K - */ - K + C } diff --git a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderLoadFileIT.java b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderLoadFileIT.java index 075b4c2c7..d60233d20 100644 --- a/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderLoadFileIT.java +++ b/pass-grant-loader/src/test/java/org/eclipse/pass/support/grant/GrantLoaderLoadFileIT.java @@ -249,7 +249,15 @@ private void verifyInvalidGrants(List ingestRecordErrors) throws IOExcep assertEquals(0, resultGrant4.getTotal()); assertTrue(ingestRecordErrors.stream().anyMatch(message -> message.matches(".*GrantIngestRecord.*grantNumber=777777.*\\sError Message: " + - "Invalid Pi Role: PI. Valid \\[P, C, K]"))); + "Invalid Pi Role: PI. Valid \\[P, C]"))); + + grantSelector.setFilter(RSQL.equals("localKey", "johnshopkins.edu:grant:888888")); + PassClientResult resultGrant8 = passClient.selectObjects(grantSelector); + // null pi role, skips row + assertEquals(0, resultGrant8.getTotal()); + assertTrue(ingestRecordErrors.stream().anyMatch(message -> + message.matches(".*GrantIngestRecord.*grantNumber=888888.*\\sError Message: " + + "Required value missing for piRole"))); } } diff --git a/pass-grant-loader/src/test/resources/test-load.csv b/pass-grant-loader/src/test/resources/test-load.csv index 9d6037313..37a63b674 100644 --- a/pass-grant-loader/src/test/resources/test-load.csv +++ b/pass-grant-loader/src/test/resources/test-load.csv @@ -14,4 +14,5 @@ GRANT_NUMBER,GRANT_TITLE,AWARD_NUMBER,AWARD_STATUS,AWARD_DATE,AWARD_START,AWARD_ 444444,Test Grant 4 - invalid award date,423454,active,10-01-2021,2021-05-01,2026-04-30,NATIONAL INSTITUTES OF HEALTH,300865,NATIONAL INSTITUTES OF HEALTH,300865,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,P,2023-02-01 555555,Test Grant 5 - missing required funder,423454,active,2021-10-01,2021-05-01,2026-04-30,,,,,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,P,2023-02-01 666666,Test Grant 6 - invalid award status,423454,foobar,2021-10-01,2021-05-01,2026-04-30,NATIONAL INSTITUTES OF HEALTH,300865,NATIONAL INSTITUTES OF HEALTH,300865,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,P,2023-02-01 -777777,Test Grant 7 - invalid pi role,423454,active,2021-10-01,2021-05-01,2026-04-30,NATIONAL INSTITUTES OF HEALTH,300865,NATIONAL INSTITUTES OF HEALTH,300865,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,PI,2023-02-01 \ No newline at end of file +777777,Test Grant 7 - invalid pi role,423454,active,2021-10-01,2021-05-01,2026-04-30,NATIONAL INSTITUTES OF HEALTH,300865,NATIONAL INSTITUTES OF HEALTH,300865,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,PI,2023-02-01 +888888,Test Grant 8 - null pi role,423454,active,2021-10-01,2021-05-01,2026-04-30,NATIONAL INSTITUTES OF HEALTH,300865,NATIONAL INSTITUTES OF HEALTH,300865,UserOneFn,UserOneMn,UserOneLn,userone@jhu.edu,,123456,,2023-02-01 \ No newline at end of file