diff --git a/pass-grant-loader/pass-grant-cli/src/main/java/org/eclipse/pass/support/grant/cli/BaseGrantLoaderApp.java b/pass-grant-loader/pass-grant-cli/src/main/java/org/eclipse/pass/support/grant/cli/BaseGrantLoaderApp.java index c160dcaa..14f92c81 100644 --- a/pass-grant-loader/pass-grant-cli/src/main/java/org/eclipse/pass/support/grant/cli/BaseGrantLoaderApp.java +++ b/pass-grant-loader/pass-grant-cli/src/main/java/org/eclipse/pass/support/grant/cli/BaseGrantLoaderApp.java @@ -20,14 +20,12 @@ import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_COULD_NOT_APPEND_UPDATE_TIMESTAMP; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_COULD_NOT_OPEN_CONFIGURATION_FILE; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_DATA_FILE_CANNOT_READ; -import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_DIRECTORY_LOOKUP_ERROR; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_HOME_DIRECTORY_NOT_FOUND; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_HOME_DIRECTORY_NOT_READABLE_AND_WRITABLE; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_INVALID_COMMAND_LINE_DATE; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_INVALID_COMMAND_LINE_TIMESTAMP; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_INVALID_TIMESTAMP; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_MODE_NOT_VALID; -import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_ORACLE_DRIVER_NOT_FOUND; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_REQUIRED_CONFIGURATION_FILE_MISSING; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_REQUIRED_DATA_FILE_MISSING; import static org.eclipse.pass.support.grant.cli.DataLoaderErrors.ERR_RESULT_SET_NULL; @@ -238,24 +236,12 @@ void run() throws PassCliException { } GrantConnector connector = configureConnector(connectionProperties, policyProperties); - String queryString = connector.buildQueryString(startDate, awardEndDate, mode, grant); - - //special case for when we process funders, but do not want to consult COEUS - - //just use local properties file to map funders to policies - if (mode.equals("funder") && local) { - queryString = null; - } - try { - resultSet = connector.retrieveUpdates(queryString, mode); - } catch (ClassNotFoundException e) { - throw processException(ERR_ORACLE_DRIVER_NOT_FOUND, e); + resultSet = connector.retrieveUpdates(startDate, awardEndDate, mode, grant); } catch (SQLException e) { throw processException(ERR_SQL_EXCEPTION, e); } catch (RuntimeException e) { throw processException("Runtime Exception", e); - } catch (IOException e) { - throw processException(ERR_DIRECTORY_LOOKUP_ERROR, e); } } else { //just doing a PASS load, must have results set in the data file try (FileInputStream fis = new FileInputStream(dataFile); diff --git a/pass-grant-loader/pass-grant-cli/src/main/java/org/eclipse/pass/support/grant/cli/DataLoaderErrors.java b/pass-grant-loader/pass-grant-cli/src/main/java/org/eclipse/pass/support/grant/cli/DataLoaderErrors.java index dccb1c5d..6391ec2e 100644 --- a/pass-grant-loader/pass-grant-cli/src/main/java/org/eclipse/pass/support/grant/cli/DataLoaderErrors.java +++ b/pass-grant-loader/pass-grant-cli/src/main/java/org/eclipse/pass/support/grant/cli/DataLoaderErrors.java @@ -43,10 +43,8 @@ private DataLoaderErrors() { static String ERR_COULD_NOT_APPEND_UPDATE_TIMESTAMP = "The updated succeeded, but could not append last modified " + "date %s to update timestamp file"; static String ERR_SQL_EXCEPTION = "An SQL error occurred querying the grant data source"; - static String ERR_ORACLE_DRIVER_NOT_FOUND = "Could not find the oracle db driver on classpath."; static String ERR_MODE_NOT_VALID = "%s is not a valid mode - must be either \"grant\" or \"user\""; static String ERR_ACTION_NOT_VALID = "%s is not a valid action - must be either \"pull\" or \"load\""; - static String ERR_DIRECTORY_LOOKUP_ERROR = "Error looking up Hopkins ID from employee ID"; static String ERR_RESULT_SET_NULL = "The result set was null - either the data pull failed, or there was an error" + " reading the result set from the data file"; } diff --git a/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/CoeusConnector.java b/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/CoeusConnector.java index 585ba340..4ead9a74 100644 --- a/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/CoeusConnector.java +++ b/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/CoeusConnector.java @@ -15,18 +15,21 @@ */ package org.eclipse.pass.support.grant.data; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; +import java.sql.Timestamp; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.eclipse.pass.support.client.ModelUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +47,58 @@ public class CoeusConnector implements GrantConnector { private static final String COEUS_USER = "coeus.user"; private static final String COEUS_PASS = "coeus.pass"; + private static final String SELECT_GRANT_SQL = + "SELECT " + + "A." + CoeusFieldNames.C_GRANT_AWARD_NUMBER + ", " + + "A." + CoeusFieldNames.C_GRANT_AWARD_STATUS + ", " + + "A." + CoeusFieldNames.C_GRANT_LOCAL_KEY + ", " + + "A." + CoeusFieldNames.C_GRANT_PROJECT_NAME + ", " + + "A." + CoeusFieldNames.C_GRANT_AWARD_DATE + ", " + + "A." + CoeusFieldNames.C_GRANT_START_DATE + ", " + + "A." + CoeusFieldNames.C_GRANT_END_DATE + ", " + + "A." + CoeusFieldNames.C_DIRECT_FUNDER_NAME + ", " + + "A." + CoeusFieldNames.C_DIRECT_FUNDER_LOCAL_KEY + ", " + //"SPOSNOR_CODE" + "A." + CoeusFieldNames.C_UPDATE_TIMESTAMP + ", " + + "B." + CoeusFieldNames.C_ABBREVIATED_ROLE + ", " + + "B." + CoeusFieldNames.C_USER_EMPLOYEE_ID + ", " + + "C." + CoeusFieldNames.C_USER_FIRST_NAME + ", " + + "C." + CoeusFieldNames.C_USER_MIDDLE_NAME + ", " + + "C." + CoeusFieldNames.C_USER_LAST_NAME + ", " + + "C." + CoeusFieldNames.C_USER_EMAIL + ", " + + "C." + CoeusFieldNames.C_USER_INSTITUTIONAL_ID + ", " + + "D." + CoeusFieldNames.C_PRIMARY_FUNDER_NAME + ", " + + "D." + CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY + " " + + "FROM " + + "COEUS.JHU_FACULTY_FORCE_PROP A " + + "INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B ON A.INST_PROPOSAL = B.INST_PROPOSAL " + + "INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C ON B.EMPLOYEE_ID = C.EMPLOYEE_ID " + + "LEFT JOIN COEUS.SWIFT_SPONSOR D ON A.PRIME_SPONSOR_CODE = D.SPONSOR_CODE " + + "WHERE A.UPDATE_TIMESTAMP > ? " + + "AND TO_DATE(A.AWARD_END, 'MM/DD/YYYY') >= TO_DATE(?, 'MM/DD/YYYY') " + + "AND A.PROPOSAL_STATUS = 'Funded' " + + "AND (B.ABBREVIATED_ROLE = 'P' OR B.ABBREVIATED_ROLE = 'C' " + + "OR REGEXP_LIKE (UPPER(B.ROLE), '^CO ?-?INVESTIGATOR$')) "; + + private static final String SELECT_USER_SQL = + "SELECT " + + CoeusFieldNames.C_USER_FIRST_NAME + ", " + + CoeusFieldNames.C_USER_MIDDLE_NAME + ", " + + CoeusFieldNames.C_USER_LAST_NAME + ", " + + CoeusFieldNames.C_USER_EMAIL + ", " + + CoeusFieldNames.C_USER_INSTITUTIONAL_ID + ", " + + CoeusFieldNames.C_USER_EMPLOYEE_ID + ", " + + CoeusFieldNames.C_UPDATE_TIMESTAMP + " " + + "FROM COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL " + + "WHERE UPDATE_TIMESTAMP > ?"; + + private static final String SELECT_FUNDER_SQL = + "SELECT " + + CoeusFieldNames.C_PRIMARY_FUNDER_NAME + ", " + + CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY + " " + + "FROM COEUS.SWIFT_SPONSOR " + + "WHERE SPONSOR_CODE IN (%s)"; + + private String coeusUrl; private String coeusUser; private String coeusPassword; @@ -73,78 +128,77 @@ public CoeusConnector(Properties connectionProperties, Properties funderPolicyPr } - public List> retrieveUpdates(String queryString, String mode) - throws ClassNotFoundException, SQLException, IOException { + public List> retrieveUpdates(String startDate, String awardEndDate, String mode, String grant) + throws SQLException { if (mode.equals("user")) { - return retrieveUserUpdates(queryString); + return retrieveUserUpdates(startDate); } else if (mode.equals("funder")) { - return retrieveFunderUpdates(queryString); + return retrieveFunderUpdates(); } else { - return retrieveGrantUpdates(queryString); + return retrieveGrantUpdates(startDate, awardEndDate, grant); } } - /** - * This method returns a {@code ResultSet} for a query for a specific set of fields in several views in COEUS. - * - * @param queryString the query string to the COEUS database needed to update the information - * @return the {@code ResultSet} from the query - */ - private List> retrieveGrantUpdates(String queryString) - throws ClassNotFoundException, SQLException { + private List> retrieveGrantUpdates(String startDate, String awardEndDate, String grant) + throws SQLException { + String sql = buildGrantQueryString(grant); List> mapList = new ArrayList<>(); - Class.forName("oracle.jdbc.driver.OracleDriver"); - try ( Connection con = DriverManager.getConnection(coeusUrl, coeusUser, coeusPassword); - Statement stmt = con.createStatement(); - ResultSet rs = stmt.executeQuery(queryString) + PreparedStatement ps = con.prepareStatement(sql); ) { - while (rs.next()) { - Map rowMap = new HashMap<>(); - - rowMap.put(CoeusFieldNames.C_GRANT_AWARD_NUMBER, + LocalDateTime startLd = LocalDateTime.from(DateTimeUtil.DATE_TIME_FORMATTER.parse(startDate)); + ps.setTimestamp(1, Timestamp.valueOf(startLd)); + ps.setString(2, awardEndDate); + if (StringUtils.isNotEmpty(grant)) { + ps.setString(3, grant); + } + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map rowMap = new HashMap<>(); + rowMap.put(CoeusFieldNames.C_GRANT_AWARD_NUMBER, ModelUtil.normalizeAwardNumber(rs.getString(CoeusFieldNames.C_GRANT_AWARD_NUMBER))); - rowMap.put(CoeusFieldNames.C_GRANT_AWARD_STATUS, rs.getString(CoeusFieldNames.C_GRANT_AWARD_STATUS)); - rowMap.put(CoeusFieldNames.C_GRANT_LOCAL_KEY, rs.getString(CoeusFieldNames.C_GRANT_LOCAL_KEY)); - rowMap.put(CoeusFieldNames.C_GRANT_PROJECT_NAME, rs.getString(CoeusFieldNames.C_GRANT_PROJECT_NAME)); - rowMap.put(CoeusFieldNames.C_GRANT_AWARD_DATE, rs.getString(CoeusFieldNames.C_GRANT_AWARD_DATE)); - rowMap.put(CoeusFieldNames.C_GRANT_START_DATE, rs.getString(CoeusFieldNames.C_GRANT_START_DATE)); - rowMap.put(CoeusFieldNames.C_GRANT_END_DATE, rs.getString(CoeusFieldNames.C_GRANT_END_DATE)); - - rowMap.put(CoeusFieldNames.C_DIRECT_FUNDER_NAME, rs.getString(CoeusFieldNames.C_DIRECT_FUNDER_NAME)); - - rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_NAME, rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_NAME)); - rowMap.put(CoeusFieldNames.C_USER_FIRST_NAME, rs.getString(CoeusFieldNames.C_USER_FIRST_NAME)); - rowMap.put(CoeusFieldNames.C_USER_MIDDLE_NAME, rs.getString(CoeusFieldNames.C_USER_MIDDLE_NAME)); - rowMap.put(CoeusFieldNames.C_USER_LAST_NAME, rs.getString(CoeusFieldNames.C_USER_LAST_NAME)); - rowMap.put(CoeusFieldNames.C_USER_EMAIL, rs.getString(CoeusFieldNames.C_USER_EMAIL)); - rowMap.put(CoeusFieldNames.C_USER_EMPLOYEE_ID, rs.getString(CoeusFieldNames.C_USER_EMPLOYEE_ID)); - rowMap.put(CoeusFieldNames.C_USER_INSTITUTIONAL_ID, + rowMap.put(CoeusFieldNames.C_GRANT_AWARD_STATUS, + rs.getString(CoeusFieldNames.C_GRANT_AWARD_STATUS)); + rowMap.put(CoeusFieldNames.C_GRANT_LOCAL_KEY, rs.getString(CoeusFieldNames.C_GRANT_LOCAL_KEY)); + rowMap.put(CoeusFieldNames.C_GRANT_PROJECT_NAME, + rs.getString(CoeusFieldNames.C_GRANT_PROJECT_NAME)); + rowMap.put(CoeusFieldNames.C_GRANT_AWARD_DATE, rs.getString(CoeusFieldNames.C_GRANT_AWARD_DATE)); + rowMap.put(CoeusFieldNames.C_GRANT_START_DATE, rs.getString(CoeusFieldNames.C_GRANT_START_DATE)); + rowMap.put(CoeusFieldNames.C_GRANT_END_DATE, rs.getString(CoeusFieldNames.C_GRANT_END_DATE)); + rowMap.put(CoeusFieldNames.C_DIRECT_FUNDER_NAME, + rs.getString(CoeusFieldNames.C_DIRECT_FUNDER_NAME)); + rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_NAME, + rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_NAME)); + rowMap.put(CoeusFieldNames.C_USER_FIRST_NAME, rs.getString(CoeusFieldNames.C_USER_FIRST_NAME)); + rowMap.put(CoeusFieldNames.C_USER_MIDDLE_NAME, rs.getString(CoeusFieldNames.C_USER_MIDDLE_NAME)); + rowMap.put(CoeusFieldNames.C_USER_LAST_NAME, rs.getString(CoeusFieldNames.C_USER_LAST_NAME)); + rowMap.put(CoeusFieldNames.C_USER_EMAIL, rs.getString(CoeusFieldNames.C_USER_EMAIL)); + rowMap.put(CoeusFieldNames.C_USER_EMPLOYEE_ID, rs.getString(CoeusFieldNames.C_USER_EMPLOYEE_ID)); + rowMap.put(CoeusFieldNames.C_USER_INSTITUTIONAL_ID, rs.getString(CoeusFieldNames.C_USER_INSTITUTIONAL_ID)); - rowMap.put(CoeusFieldNames.C_UPDATE_TIMESTAMP, rs.getString(CoeusFieldNames.C_UPDATE_TIMESTAMP)); - rowMap.put(CoeusFieldNames.C_ABBREVIATED_ROLE, rs.getString(CoeusFieldNames.C_ABBREVIATED_ROLE)); - - String primaryFunderLocalKey = rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY); - rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY, primaryFunderLocalKey); - if (primaryFunderLocalKey != null && - funderPolicyProperties.stringPropertyNames().contains(primaryFunderLocalKey)) { - rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_POLICY, + rowMap.put(CoeusFieldNames.C_UPDATE_TIMESTAMP, rs.getString(CoeusFieldNames.C_UPDATE_TIMESTAMP)); + rowMap.put(CoeusFieldNames.C_ABBREVIATED_ROLE, rs.getString(CoeusFieldNames.C_ABBREVIATED_ROLE)); + String primaryFunderLocalKey = rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY); + rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY, primaryFunderLocalKey); + if (primaryFunderLocalKey != null && + funderPolicyProperties.stringPropertyNames().contains(primaryFunderLocalKey)) { + rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_POLICY, funderPolicyProperties.getProperty(primaryFunderLocalKey)); - } - - String directFunderLocalKey = rs.getString(CoeusFieldNames.C_DIRECT_FUNDER_LOCAL_KEY); - rowMap.put(CoeusFieldNames.C_DIRECT_FUNDER_LOCAL_KEY, directFunderLocalKey); - if (directFunderLocalKey != null && - funderPolicyProperties.stringPropertyNames().contains(directFunderLocalKey)) { - rowMap.put(CoeusFieldNames.C_DIRECT_FUNDER_POLICY, + } + String directFunderLocalKey = rs.getString(CoeusFieldNames.C_DIRECT_FUNDER_LOCAL_KEY); + rowMap.put(CoeusFieldNames.C_DIRECT_FUNDER_LOCAL_KEY, directFunderLocalKey); + if (directFunderLocalKey != null && + funderPolicyProperties.stringPropertyNames().contains(directFunderLocalKey)) { + rowMap.put(CoeusFieldNames.C_DIRECT_FUNDER_POLICY, funderPolicyProperties.getProperty(directFunderLocalKey)); - } - LOG.debug("Record processed: {}", rowMap); - if (!mapList.contains(rowMap)) { - mapList.add(rowMap); + } + LOG.debug("Record processed: {}", rowMap); + if (!mapList.contains(rowMap)) { + mapList.add(rowMap); + } } } } @@ -152,216 +206,71 @@ private List> retrieveGrantUpdates(String queryString) return mapList; } - private List> retrieveFunderUpdates(String queryString) - throws ClassNotFoundException, SQLException { + private String buildGrantQueryString(String grant) { + return StringUtils.isEmpty(grant) + ? SELECT_GRANT_SQL + "AND A.GRANT_NUMBER IS NOT NULL" + : SELECT_GRANT_SQL + "AND A.GRANT_NUMBER = ?"; + } + private List> retrieveFunderUpdates() throws SQLException { List> mapList = new ArrayList<>(); - - if (queryString != null) { //we will go to COEUS for the info - - Class.forName("oracle.jdbc.driver.OracleDriver"); - - try ( - Connection con = DriverManager.getConnection(coeusUrl, coeusUser, coeusPassword); - Statement stmt = con.createStatement(); - ResultSet rs = stmt.executeQuery(queryString) - ) { + String funderSql = String.format(SELECT_FUNDER_SQL, + funderPolicyProperties.stringPropertyNames().stream() + .map(v -> "?") + .collect(Collectors.joining(", "))); + try ( + Connection con = DriverManager.getConnection(coeusUrl, coeusUser, coeusPassword); + PreparedStatement ps = con.prepareStatement(funderSql); + ) { + int index = 1; + for ( String funderKey : funderPolicyProperties.stringPropertyNames() ) { + ps.setString(index++, funderKey); + } + try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { //these are the field names in the swift sponsor view Map rowMap = new HashMap<>(); rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY, - rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY)); + rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY)); rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_NAME, - rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_NAME)); + rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_NAME)); rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_POLICY, - funderPolicyProperties.getProperty( - rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY))); + funderPolicyProperties.getProperty( + rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY))); mapList.add(rowMap); } - - } - - } else { //we will prepare partial Funder from the properties file - - for (Object localKey : funderPolicyProperties.keySet()) { - Map rowMap = new HashMap<>(); - rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY, localKey.toString()); - rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_POLICY, - funderPolicyProperties.getProperty(localKey.toString())); - mapList.add(rowMap); } } - return mapList; } - private List> retrieveUserUpdates(String queryString) - throws ClassNotFoundException, SQLException { - + private List> retrieveUserUpdates(String startDate) throws SQLException { List> mapList = new ArrayList<>(); - - Class.forName("oracle.jdbc.driver.OracleDriver"); - try ( Connection con = DriverManager.getConnection(coeusUrl, coeusUser, coeusPassword); - Statement stmt = con.createStatement(); - ResultSet rs = stmt.executeQuery(queryString) + PreparedStatement ps = con.prepareStatement(SELECT_USER_SQL); ) { - while (rs.next()) { - Map rowMap = new HashMap<>(); - rowMap.put(CoeusFieldNames.C_USER_FIRST_NAME, rs.getString(CoeusFieldNames.C_USER_FIRST_NAME)); - rowMap.put(CoeusFieldNames.C_USER_MIDDLE_NAME, rs.getString(CoeusFieldNames.C_USER_MIDDLE_NAME)); - rowMap.put(CoeusFieldNames.C_USER_LAST_NAME, rs.getString(CoeusFieldNames.C_USER_LAST_NAME)); - rowMap.put(CoeusFieldNames.C_USER_EMAIL, rs.getString(CoeusFieldNames.C_USER_EMAIL)); - rowMap.put(CoeusFieldNames.C_USER_INSTITUTIONAL_ID, + LocalDateTime startLd = LocalDateTime.from(DateTimeUtil.DATE_TIME_FORMATTER.parse(startDate)); + ps.setTimestamp(1, Timestamp.valueOf(startLd)); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + Map rowMap = new HashMap<>(); + rowMap.put(CoeusFieldNames.C_USER_FIRST_NAME, rs.getString(CoeusFieldNames.C_USER_FIRST_NAME)); + rowMap.put(CoeusFieldNames.C_USER_MIDDLE_NAME, rs.getString(CoeusFieldNames.C_USER_MIDDLE_NAME)); + rowMap.put(CoeusFieldNames.C_USER_LAST_NAME, rs.getString(CoeusFieldNames.C_USER_LAST_NAME)); + rowMap.put(CoeusFieldNames.C_USER_EMAIL, rs.getString(CoeusFieldNames.C_USER_EMAIL)); + rowMap.put(CoeusFieldNames.C_USER_INSTITUTIONAL_ID, rs.getString(CoeusFieldNames.C_USER_INSTITUTIONAL_ID)); - rowMap.put(CoeusFieldNames.C_USER_EMPLOYEE_ID, rs.getString(CoeusFieldNames.C_USER_EMPLOYEE_ID)); - rowMap.put(CoeusFieldNames.C_UPDATE_TIMESTAMP, rs.getString(CoeusFieldNames.C_UPDATE_TIMESTAMP)); - LOG.debug("Record processed: {}", rowMap); - if (!mapList.contains(rowMap)) { - mapList.add(rowMap); + rowMap.put(CoeusFieldNames.C_USER_EMPLOYEE_ID, rs.getString(CoeusFieldNames.C_USER_EMPLOYEE_ID)); + rowMap.put(CoeusFieldNames.C_UPDATE_TIMESTAMP, rs.getString(CoeusFieldNames.C_UPDATE_TIMESTAMP)); + LOG.debug("Record processed: {}", rowMap); + if (!mapList.contains(rowMap)) { + mapList.add(rowMap); + } } } } - LOG.info("Retrieved result set from COEUS: {} records processed", mapList.size()); + LOG.info("Retrieved Users result set from COEUS: {} records processed", mapList.size()); return mapList; } - public String buildQueryString(String startDate, String awardEndDate, String mode, String grant) { - if (mode.equals("user")) { - return buildUserQueryString(startDate); - } else if (mode.equals("funder")) { - return buildFunderQueryString(); - } else { - return buildGrantQueryString(startDate, awardEndDate, grant); - } - } - - /** - * Method for building the query string against the COEUS database. We draw from four views. - * Dates are stored in the views as strings, except for the UPDATE_TIMESTAMP, which is a timestamp. - * We will pull all records which have been updated since the last update timestamp - this value becomes out - * startDate. - * - * Because we are only interested in the latest update for any grant number, we restrict the search to the latest - * update timestamp for grants, even if these correspond to different institutional proposal numbers. This is - * because we - * only need the granularity of grant number for the purposes of publication submission. - * - * NB: the join of the PROP view with the PRSN view will result in one row in the ResultSet for each investigator - * on the grant. if there are co-pis in addition to a pi, there will be multiple rows. - * - * COEUS.JHU_FACULTY_FORCE_PROP aliased to A - * COEUS.JHU_FACULTY_FORCE_PRSN aliased to B - * COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL aliased to C - * COEUS.SWIFT_SPONSOR aliased to D - * - * @param startDate - the date we want to start the query against UPDATE_TIMESTAMP - * @return the SQL query string - */ - private String buildGrantQueryString(String startDate, String awardEndDate, String grant) { - - String[] viewFields = { - "A." + CoeusFieldNames.C_GRANT_AWARD_NUMBER, - "A." + CoeusFieldNames.C_GRANT_AWARD_STATUS, - "A." + CoeusFieldNames.C_GRANT_LOCAL_KEY, - "A." + CoeusFieldNames.C_GRANT_PROJECT_NAME, - "A." + CoeusFieldNames.C_GRANT_AWARD_DATE, - "A." + CoeusFieldNames.C_GRANT_START_DATE, - "A." + CoeusFieldNames.C_GRANT_END_DATE, - "A." + CoeusFieldNames.C_DIRECT_FUNDER_NAME, - "A." + CoeusFieldNames.C_DIRECT_FUNDER_LOCAL_KEY, //"SPOSNOR_CODE" - "A." + CoeusFieldNames.C_UPDATE_TIMESTAMP, - - "B." + CoeusFieldNames.C_ABBREVIATED_ROLE, - "B." + CoeusFieldNames.C_USER_EMPLOYEE_ID, - - "C." + CoeusFieldNames.C_USER_FIRST_NAME, - "C." + CoeusFieldNames.C_USER_MIDDLE_NAME, - "C." + CoeusFieldNames.C_USER_LAST_NAME, - "C." + CoeusFieldNames.C_USER_EMAIL, - "C." + CoeusFieldNames.C_USER_INSTITUTIONAL_ID, - - "D." + CoeusFieldNames.C_PRIMARY_FUNDER_NAME, - "D." + CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY}; - - StringBuilder sb = new StringBuilder(); - sb.append("SELECT "); - sb.append(String.join(", ", viewFields)); - sb.append(" FROM"); - sb.append(" COEUS.JHU_FACULTY_FORCE_PROP A"); - // sb.append(" INNER JOIN "); - // sb.append(" (SELECT GRANT_NUMBER, MAX(UPDATE_TIMESTAMP) AS MAX_UPDATE_TIMESTAMP"); - // sb.append(" FROM COEUS.JHU_FACULTY_FORCE_PROP GROUP BY GRANT_NUMBER) LATEST"); - // sb.append(" ON A.UPDATE_TIMESTAMP = LATEST.MAX_UPDATE_TIMESTAMP"); - // sb.append(" AND A.GRANT_NUMBER = LATEST.GRANT_NUMBER"); - sb.append(" INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B ON A.INST_PROPOSAL = B.INST_PROPOSAL"); - sb.append(" INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C ON B.EMPLOYEE_ID = C.EMPLOYEE_ID"); - sb.append(" LEFT JOIN COEUS.SWIFT_SPONSOR D ON A.PRIME_SPONSOR_CODE = D.SPONSOR_CODE"); - sb.append(" WHERE A.UPDATE_TIMESTAMP > TIMESTAMP '"); - sb.append(startDate); - sb.append("' "); - sb.append("AND TO_DATE(A.AWARD_END, 'MM/DD/YYYY') >= TO_DATE('" + awardEndDate + "', 'MM/DD/YYYY') "); - sb.append("AND A.PROPOSAL_STATUS = 'Funded' "); - sb.append( - "AND (B.ABBREVIATED_ROLE = 'P' OR B.ABBREVIATED_ROLE = 'C' OR REGEXP_LIKE (UPPER(B.ROLE), '^CO " + - "?-?INVESTIGATOR$')) "); - if (grant == null || grant.isEmpty()) { - sb.append("AND A.GRANT_NUMBER IS NOT NULL"); - } else { // have a specifig grant to process - sb.append("AND A.GRANT_NUMBER = '" + grant + "'"); - } - String queryString = sb.toString(); - - LOG.debug("Query string is: {}", queryString); - return queryString; - } - - private String buildUserQueryString(String startDate) { - String[] viewFields = { - CoeusFieldNames.C_USER_FIRST_NAME, - CoeusFieldNames.C_USER_MIDDLE_NAME, - CoeusFieldNames.C_USER_LAST_NAME, - CoeusFieldNames.C_USER_EMAIL, - CoeusFieldNames.C_USER_INSTITUTIONAL_ID, - CoeusFieldNames.C_USER_EMPLOYEE_ID, - CoeusFieldNames.C_UPDATE_TIMESTAMP}; - - StringBuilder sb = new StringBuilder(); - sb.append("SELECT "); - sb.append(String.join(", ", viewFields)); - sb.append(" FROM"); - sb.append(" COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL"); - sb.append(" WHERE UPDATE_TIMESTAMP > TIMESTAMP '"); - sb.append(startDate); - sb.append("'"); - - String queryString = sb.toString(); - - LOG.debug("Query string is: {}", queryString); - return queryString; - } - - private String buildFunderQueryString() { - - String[] viewFields = { //doesn't matter whether the funder is primary or direct - these are the column names - // in the SWIFT_SPONSOR view - CoeusFieldNames.C_PRIMARY_FUNDER_NAME, - CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY}; - - StringBuilder sb = new StringBuilder(); - sb.append("SELECT "); - sb.append(String.join(", ", viewFields)); - sb.append(" FROM"); - sb.append(" COEUS.SWIFT_SPONSOR"); - sb.append(" WHERE"); - sb.append(" SPONSOR_CODE IN ("); - List keyList = new ArrayList<>(); - sb.append(String.join(", ", (funderPolicyProperties.stringPropertyNames()))); - sb.append(")"); - String queryString = sb.toString(); - - LOG.debug("Query string is: {} ", queryString); - return queryString; - - } } diff --git a/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java b/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java index 9b5173e4..6191101a 100644 --- a/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java +++ b/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java @@ -28,10 +28,10 @@ */ public class DateTimeUtil { - private static final DateTimeFormatter DATE_TIME_FORMATTER = + static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss[[.SSS][.SS][.S]]") .withResolverStyle(ResolverStyle.STRICT); - private static final DateTimeFormatter DATE_FORMATTER = + static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("MM/dd/uuuu") .withResolverStyle(ResolverStyle.STRICT); diff --git a/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/GrantConnector.java b/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/GrantConnector.java index bef7fe6b..c24fd523 100644 --- a/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/GrantConnector.java +++ b/pass-grant-loader/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/GrantConnector.java @@ -15,7 +15,6 @@ */ package org.eclipse.pass.support.grant.data; -import java.io.IOException; import java.sql.SQLException; import java.util.List; import java.util.Map; @@ -26,28 +25,16 @@ public interface GrantConnector { /** - * If the grant data source is a database, we will need a query string + * This method retrieves the data from a data source. The format is a List of Maps - one List element for each + * grant or user record. * * @param startDate - the date of the earliest record we wish to get on this pull * @param awardEndDate - the end date of the award * @param mode - indicates whether the data pull is for grants, or users * @param grant - a grant number - * @return the query string - */ - String buildQueryString(String startDate, String awardEndDate, String mode, String grant); - - /** - * This method retrieves the data from a data source. The format is a List of Maps - one List element for each - * grant or user record. - * - * @param queryString - a query string, if required - * @param mode - indicates whether the data pull is for grants, or users - * @return the query string - * @throws ClassNotFoundException if the driver is not found * @throws SQLException if there is an SQL exception - * @throws IOException if there is an IO exception */ - List> retrieveUpdates(String queryString, String mode) throws - ClassNotFoundException, SQLException, IOException; + List> retrieveUpdates(String startDate, String awardEndDate, String mode, String grant) throws + SQLException; } diff --git a/pass-grant-loader/pass-grant-data/src/test/java/org/eclipse/pass/support/grant/data/CoeusConnectorManualTest.java b/pass-grant-loader/pass-grant-data/src/test/java/org/eclipse/pass/support/grant/data/CoeusConnectorManualTest.java new file mode 100644 index 00000000..8205326e --- /dev/null +++ b/pass-grant-loader/pass-grant-data/src/test/java/org/eclipse/pass/support/grant/data/CoeusConnectorManualTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2023 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.eclipse.pass.support.grant.data; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +/** + * Test class for the COEUS connector. This is strictly a manual test for querying the Coeus database. + * This test is Disabled, you can enable it and run each query test if needed for validation. + *

+ * In order to run the tests, you must put a connection.properties file with valid url and creds in the + * test/resources dir. + * + * @author jrm@jhu.edu + */ +@Disabled +public class CoeusConnectorManualTest { + + private CoeusConnector connector; + + private final File policyPropertiesFile = new File( + getClass().getClassLoader().getResource("policy.properties").getFile()); + + private final File connectionPropertiesFile = new File( + getClass().getClassLoader().getResource("connection.properties").getFile()); + + private final Properties policyProperties = new Properties(); + + @BeforeEach + public void setup() throws Exception { + + try (InputStream resourceStream = new FileInputStream(policyPropertiesFile)) { + policyProperties.load(resourceStream); + } + Properties connectionProperties = new Properties(); + try (InputStream resourceStream = new FileInputStream(connectionPropertiesFile)) { + connectionProperties.load(resourceStream); + } + connector = new CoeusConnector( + connectionProperties, policyProperties); + } + + @Disabled + @Test + public void testGrantQuery() throws SQLException { + List> results = + connector.retrieveUpdates("2023-10-20 00:00:00", "01/01/2011", "grant", null); + assertNotNull(results); + } + + @Disabled + @Test + public void testUserQuery() throws SQLException { + List> results = + connector.retrieveUpdates("2023-10-20 00:00:00", null, "user", null); + assertNotNull(results); + } + + @Disabled + @Test + public void testFunderQuery() throws SQLException { + List> results = + connector.retrieveUpdates(null, null, "funder", null); + assertNotNull(results); + } + +} + + diff --git a/pass-grant-loader/pass-grant-data/src/test/java/org/eclipse/pass/support/grant/data/CoeusConnectorTest.java b/pass-grant-loader/pass-grant-data/src/test/java/org/eclipse/pass/support/grant/data/CoeusConnectorTest.java deleted file mode 100644 index 690bc50a..00000000 --- a/pass-grant-loader/pass-grant-data/src/test/java/org/eclipse/pass/support/grant/data/CoeusConnectorTest.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2023 Johns Hopkins University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.eclipse.pass.support.grant.data; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * Test class for the COEUS connector - * - * @author jrm@jhu.edu - */ -public class CoeusConnectorTest { - - private CoeusConnector connector; - - private File policyPropertiesFile = new File( - getClass().getClassLoader().getResource("policy.properties").getFile()); - - private Properties policyProperties = new Properties(); - - @BeforeEach - public void setup() throws Exception { - - try (InputStream resourceStream = new FileInputStream(policyPropertiesFile)) { - policyProperties.load(resourceStream); - } - connector = new CoeusConnector( - null, policyProperties); - } - - - /** - * Test that the query string produced is as expected - */ - @Test - public void testBuildGrantString() { - - String expectedQueryString = "SELECT A.AWARD_ID, A.AWARD_STATUS, A.GRANT_NUMBER, A.TITLE, A.AWARD_DATE," + - " A.AWARD_START, A.AWARD_END, A.SPONSOR, A.SPOSNOR_CODE, A.UPDATE_TIMESTAMP, B" + - ".ABBREVIATED_ROLE, B.EMPLOYEE_ID," + - " C.FIRST_NAME, C.MIDDLE_NAME, C.LAST_NAME, C.EMAIL_ADDRESS, C.JHED_ID, D" + - ".SPONSOR_NAME, D.SPONSOR_CODE" + - " FROM" + - " COEUS.JHU_FACULTY_FORCE_PROP A" + - " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B" + - " ON A.INST_PROPOSAL = B.INST_PROPOSAL" + - " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C" + - " ON B.EMPLOYEE_ID = C.EMPLOYEE_ID" + - " LEFT JOIN COEUS.SWIFT_SPONSOR D" + - " ON A.PRIME_SPONSOR_CODE = D.SPONSOR_CODE" + - " WHERE A.UPDATE_TIMESTAMP > TIMESTAMP '2018-06-01 06:00:00.0'" + - " AND TO_DATE(A.AWARD_END, 'MM/DD/YYYY') >= TO_DATE('01/01/2011', 'MM/DD/YYYY')" + - " AND A.PROPOSAL_STATUS = 'Funded'" + - " AND (B.ABBREVIATED_ROLE = 'P' OR B.ABBREVIATED_ROLE = 'C' OR REGEXP_LIKE " + - "(UPPER(B.ROLE), '^CO ?-?INVESTIGATOR$'))" + - " AND A.GRANT_NUMBER IS NOT NULL"; - - assertEquals(expectedQueryString, - connector.buildQueryString("2018-06-01 06:00:00.0", "01/01/2011", "grant", null)); - - expectedQueryString = "SELECT A.AWARD_ID, A.AWARD_STATUS, A.GRANT_NUMBER, A.TITLE, A.AWARD_DATE," + - " A.AWARD_START, A.AWARD_END, A.SPONSOR, A.SPOSNOR_CODE, A.UPDATE_TIMESTAMP, B" + - ".ABBREVIATED_ROLE, B.EMPLOYEE_ID," + - " C.FIRST_NAME, C.MIDDLE_NAME, C.LAST_NAME, C.EMAIL_ADDRESS, C.JHED_ID, D.SPONSOR_NAME," + - " D.SPONSOR_CODE" + - " FROM" + - " COEUS.JHU_FACULTY_FORCE_PROP A" + - " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B" + - " ON A.INST_PROPOSAL = B.INST_PROPOSAL" + - " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C" + - " ON B.EMPLOYEE_ID = C.EMPLOYEE_ID" + - " LEFT JOIN COEUS.SWIFT_SPONSOR D" + - " ON A.PRIME_SPONSOR_CODE = D.SPONSOR_CODE" + - " WHERE A.UPDATE_TIMESTAMP > TIMESTAMP '2018-06-01 06:00:00.0'" + - " AND TO_DATE(A.AWARD_END, 'MM/DD/YYYY') >= TO_DATE('02/03/1999', 'MM/DD/YYYY')" + - " AND A.PROPOSAL_STATUS = 'Funded'" + - " AND (B.ABBREVIATED_ROLE = 'P' OR B.ABBREVIATED_ROLE = 'C' OR REGEXP_LIKE (UPPER(B" + - ".ROLE), '^CO ?-?INVESTIGATOR$'))" + - " AND A.GRANT_NUMBER IS NOT NULL"; - - assertEquals(expectedQueryString, - connector.buildQueryString("2018-06-01 06:00:00.0", "02/03/1999", "grant", null)); - - expectedQueryString = "SELECT A.AWARD_ID, A.AWARD_STATUS, A.GRANT_NUMBER, A.TITLE, A.AWARD_DATE," + - " A.AWARD_START, A.AWARD_END, A.SPONSOR, A.SPOSNOR_CODE, A.UPDATE_TIMESTAMP, B" + - ".ABBREVIATED_ROLE, B.EMPLOYEE_ID," + - " C.FIRST_NAME, C.MIDDLE_NAME, C.LAST_NAME, C.EMAIL_ADDRESS, C.JHED_ID, D.SPONSOR_NAME," + - " D.SPONSOR_CODE" + - " FROM" + - " COEUS.JHU_FACULTY_FORCE_PROP A" + - " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN B" + - " ON A.INST_PROPOSAL = B.INST_PROPOSAL" + - " INNER JOIN COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL C" + - " ON B.EMPLOYEE_ID = C.EMPLOYEE_ID" + - " LEFT JOIN COEUS.SWIFT_SPONSOR D" + - " ON A.PRIME_SPONSOR_CODE = D.SPONSOR_CODE" + - " WHERE A.UPDATE_TIMESTAMP > TIMESTAMP '2018-06-01 06:00:00.0'" + - " AND TO_DATE(A.AWARD_END, 'MM/DD/YYYY') >= TO_DATE('02/03/1999', 'MM/DD/YYYY')" + - " AND A.PROPOSAL_STATUS = 'Funded'" + - " AND (B.ABBREVIATED_ROLE = 'P' OR B.ABBREVIATED_ROLE = 'C' OR REGEXP_LIKE (UPPER(B" + - ".ROLE), '^CO ?-?INVESTIGATOR$'))" + - " AND A.GRANT_NUMBER = '12345678'"; - - assertEquals(expectedQueryString, - connector.buildQueryString("2018-06-01 06:00:00.0", "02/03/1999", "grant", "12345678")); - } - - @Test - public void testBuildUserQueryString() { - - String expectedQueryString = "SELECT FIRST_NAME, MIDDLE_NAME, LAST_NAME, EMAIL_ADDRESS, JHED_ID, EMPLOYEE_ID," + - " " + - "UPDATE_TIMESTAMP FROM COEUS.JHU_FACULTY_FORCE_PRSN_DETAIL " + - "WHERE UPDATE_TIMESTAMP > TIMESTAMP '2018-13-14 06:00:00.0'"; - assertEquals(expectedQueryString, - connector.buildQueryString("2018-13-14 06:00:00.0", "01/01/2011", "user", null)); - - } - - @Test - public void testBuildFunderQueryString() { - - String expectedQueryString1 = - "SELECT SPONSOR_NAME, SPONSOR_CODE FROM COEUS.SWIFT_SPONSOR WHERE SPONSOR_CODE IN (moo, baa)"; - String expectedQueryString2 = - "SELECT SPONSOR_NAME, SPONSOR_CODE FROM COEUS.SWIFT_SPONSOR WHERE SPONSOR_CODE IN (baa, moo)"; - String actualQueryString = connector.buildQueryString(null, null, "funder", null); - assertTrue(expectedQueryString1.equals(actualQueryString) || - expectedQueryString2.equals(actualQueryString)); - } - -} - -