Grant Loader Fibi Migration

pass-grant-loader/README.md

@@ -36,10 +36,10 @@ policies require.
 
 ### JHU (jhu)
 
-The JHU implementation is used to pull data from the COEUS Oracle database views for the purpose of performing regular
+The JHU implementation is used to pull data from the Grant database views for the purpose of performing regular
 updates. We look at grants which have been updated since a particular time (typically the time of the previous update),
 join this with user and funder information associated with the grant, and then use this information to update the data
-in the PASS backend. The JHU implementation also treats the COEUS database as authoritative for all fields in the
+in the PASS backend. The JHU implementation also treats the Grant database as authoritative for all fields in the
 data. If a grant is being passed in for update, it is assumed that all records for that grant are included in the
 input.
 

pass-grant-loader/pom.xml

@@ -32,7 +32,7 @@
     <args4j.version>2.33</args4j.version>
     <commons.csv.version>1.10.0</commons.csv.version>
     <commons-lang.version>3.14.0</commons-lang.version>
-    <ojdbc10.version>19.18.0.0</ojdbc10.version>
+    <mysql.jdbc.version>8.0.33</mysql.jdbc.version>
     <lombok.version>1.18.30</lombok.version>
     <slf4j.version>2.0.9</slf4j.version>
     <logback.version>1.4.14</logback.version>
@@ -91,6 +91,16 @@
         <artifactId>spring-boot-starter-test</artifactId>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-data-jpa</artifactId>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>com.h2database</groupId>
+        <artifactId>h2</artifactId>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>args4j</groupId>
         <artifactId>args4j</artifactId>
@@ -107,9 +117,9 @@
         <version>${commons-lang.version}</version>
       </dependency>
       <dependency>
-        <groupId>com.oracle.database.jdbc</groupId>
-        <artifactId>ojdbc10</artifactId>
-        <version>${ojdbc10.version}</version>
+        <groupId>mysql</groupId>
+        <artifactId>mysql-connector-java</artifactId>
+        <version>${mysql.jdbc.version}</version>
       </dependency>
       <dependency>
         <groupId>org.slf4j</groupId>
@@ -298,7 +308,9 @@
                   <!-- Deficiency of analyzer bytecode -->
                   <ignoredUnusedDeclaredDependency>org.projectlombok:lombok:</ignoredUnusedDeclaredDependency>
                   <ignoredUnusedDeclaredDependency>org.mockito:mockito-inline:</ignoredUnusedDeclaredDependency>
-                  <ignoredUnusedDeclaredDependency>com.oracle.database.jdbc::</ignoredUnusedDeclaredDependency>
+                  <ignoredUnusedDeclaredDependency>mysql:mysql-connector-java::</ignoredUnusedDeclaredDependency>
+                  <!-- Used in tests by spring boot -->
+                  <ignoredUnusedDeclaredDependency>com.h2database:h2::</ignoredUnusedDeclaredDependency>
                 </ignoredUnusedDeclaredDependencies>
               </configuration>
             </execution>

pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderApp.java

@@ -26,7 +26,6 @@
 import static org.eclipse.pass.support.grant.DataLoaderErrors.ERR_REQUIRED_CONFIGURATION_FILE_MISSING;
 import static org.eclipse.pass.support.grant.DataLoaderErrors.ERR_RESULT_SET_NULL;
 import static org.eclipse.pass.support.grant.DataLoaderErrors.ERR_SQL_EXCEPTION;
-import static org.eclipse.pass.support.grant.data.DateTimeUtil.verifyDate;
 import static org.eclipse.pass.support.grant.data.DateTimeUtil.verifyDateTimeFormat;
 
 import java.io.BufferedReader;
@@ -132,7 +131,7 @@ public void run(String startDate, String awardEndDate, String mode, String actio
                     }
                 }
                 if (awardEndDate != null) {
-                    if (!verifyDate(awardEndDate)) {
+                    if (!verifyDateTimeFormat(awardEndDate)) {
                         throw processException(format(ERR_INVALID_COMMAND_LINE_DATE, awardEndDate), null);
                     }
                 }

pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/GrantLoaderCLIRunner.java

@@ -57,21 +57,19 @@ public class GrantLoaderCLIRunner implements CommandLineRunner {
      */
     @Option(name = "-s", aliases = {"-startDateTime", "--startDateTime"},
             usage = "DateTime to start the query against Grant source system. This will cause " +
-                    "a return of all records updated since this DateTime. Syntax must be yyyy-mm-dd hh:mm:ss.m{mm}. " +
-                    "This value will override the most recent " +
-                    "dateTime listed in the updates file.")
+                "a return of all grants having a record updated since this DateTime. " +
+                "Syntax must be yyyy-mm-dd hh:mm:ss.m{mm}. This value will override the most recent " +
+                "dateTime listed in the updates file.")
     protected static String startDate = "";
 
-
     /**
      * Specifies an award end date for basing the database query
      */
     @Option(name = "-z", aliases = {"-awardEndDate", "--awardEndDate"},
             usage = "Date for the AWARD_END to start the query against Grant source system. This will cause " +
-                    "a return of all records having an AWARD_END after the supplied date. Syntax must be MM/dd/yyyy. " +
-                    "If not specified, the default will be " +
-                    "01/01/2011")
-    protected static String awardEndDate = "01/01/2011";
+                "a return of all grants having a record with an AWARD_END after the supplied date. " +
+                "Syntax must be yyyy-mm-dd. If not specified, the default will be 2011-01-01")
+    protected static String awardEndDate = "2011-01-01";
 
     /**
      * Specifies an optional action - either "pull" or "load" - to restrict the operation of the application to only
@@ -81,11 +79,10 @@ public class GrantLoaderCLIRunner implements CommandLineRunner {
      */
     @Option(name = "-a", aliases = {"-action", "--action"},
             usage = "Action to be taken - 'pull' is for Grant source system pull only," +
-                    "'load' is for PASS load only. Either option requires a file path specified as an argument " +
-                    "after all options - an" +
-                    "output file in the case of 'pull', and an input file in the case of 'load'. If no action is " +
-                    "specified, " +
-                    "the data will be pulled from Grant source system and loaded directly into PASS")
+                "'load' is for PASS load only. Either option requires a file path specified as an argument " +
+                "after all options - an output file in the case of 'pull', and an input file in the case of " +
+                "'load'. If no action is specified, the data will be pulled from Grant source system and loaded " +
+                "directly into PASS")
     protected static String action = "";
 
     /**

pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/AbstractDefaultPassUpdater.java

@@ -611,7 +611,7 @@ private User updateUserInPass(User systemUser) throws IOException {
         if (Objects.nonNull(passUser)) {
             User updatedUser = updateUserIfNeeded(systemUser, passUser);
             if (Objects.nonNull(updatedUser)) { //need to update
-                //post COEUS processing goes here
+                //post Grant database processing goes here
                 if (!updatedUser.getRoles().contains(UserRole.SUBMITTER)) {
                     updatedUser.getRoles().add(UserRole.SUBMITTER);
                 }

pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/DateTimeUtil.java

@@ -15,14 +15,12 @@
  */
 package org.eclipse.pass.support.grant.data;
 
-import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeFormatterBuilder;
 import java.time.format.DateTimeParseException;
-import java.time.format.ResolverStyle;
 import java.time.temporal.ChronoField;
 
 import org.apache.commons.lang3.StringUtils;
@@ -38,9 +36,6 @@ public class DateTimeUtil {
                 .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
                 .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
                 .toFormatter();
-    static final DateTimeFormatter DATE_FORMATTER =
-            DateTimeFormatter.ofPattern("M/d/uuuu")
-                .withResolverStyle(ResolverStyle.STRICT);
 
     private DateTimeUtil () {
         //never called
@@ -57,10 +52,6 @@ public static ZonedDateTime createZonedDateTime(String dateString) throws GrantD
             LocalDateTime localDateTime = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER);
             return localDateTime.atZone(ZoneOffset.UTC);
         }
-        if (verifyDate(dateString)) { //we may have just a date - date format is mm/day/year
-            LocalDate localDate = LocalDate.parse(dateString, DATE_FORMATTER);
-            return localDate.atStartOfDay(ZoneOffset.UTC);
-        }
         if (StringUtils.isNotBlank(dateString)) {
             throw new GrantDataException("Invalid Format for " + dateString +
                 ".  Valid Format is " + DATE_TIME_PATTERN);
@@ -73,28 +64,15 @@ public static ZonedDateTime createZonedDateTime(String dateString) throws GrantD
      * validity
      * (for example, "2018-02-31 ... " passes)
      *
-     * @param dateStr the date string to be checked
+     * @param dateTimeStr the datetime string to be checked
      * @return a boolean indicating whether the date matches the required format
      */
-    public static boolean verifyDateTimeFormat(String dateStr) {
-        return checkDateTimeFormat(dateStr, DATE_TIME_FORMATTER);
-    }
-
-    /**
-     * Date must be in the form "mm/dd/yyyy"
-     * @param date the date to verify
-     * @return true if date format is valid, false if not
-     */
-    public static boolean verifyDate(String date) {
-        return checkDateTimeFormat(date, DATE_FORMATTER);
-    }
-
-    private static boolean checkDateTimeFormat(String dateTimeStr, DateTimeFormatter formatter) {
+    public static boolean verifyDateTimeFormat(String dateTimeStr) {
         if (dateTimeStr == null) {
             return false;
         }
         try {
-            formatter.parse(dateTimeStr);
+            DATE_TIME_FORMATTER.parse(dateTimeStr);
         } catch (DateTimeParseException e) {
             return false;
         }

pass-grant-loader/src/main/java/org/eclipse/pass/support/grant/data/GrantIngestUserRole.java

@@ -26,9 +26,5 @@ public enum GrantIngestUserRole {
     /**
      * Co-PI
      */
-    C,
-    /**
-     * K
-     */
-    K
+    C
 }