Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove the user directory service api call from grant loader #60

Merged
merged 11 commits into from
Oct 5, 2023
Merged

Remove the user directory service api call from grant loader #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
222667d
Remove user dir api call from coeus call
rpoet-jh Oct 2, 2023
ee9f96e
Add user dir call to pass update logic
rpoet-jh Oct 2, 2023
4ffe60b
Change update rules for locatorIds on user
rpoet-jh Oct 2, 2023
c830050
Consolidate pass updater and entity utils
rpoet-jh Oct 2, 2023
2cb2fd7
Clean code
rpoet-jh Oct 2, 2023
9e46b51
Clean code
rpoet-jh Oct 3, 2023
d0f3d24
Remove call to user dir service
rpoet-jh Oct 3, 2023
cafc3e6
Change pass user lookup to use all locatorIDs
rpoet-jh Oct 3, 2023
208820a
Add grant diff logging for init mode
rpoet-jh Oct 4, 2023
d88f26f
Add logger level check
rpoet-jh Oct 4, 2023
528ae84
Clean difference logging
rpoet-jh Oct 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 67 additions & 116 deletions ...upport/grant/data/DefaultPassUpdater.java → ...rant/data/AbstractDefaultPassUpdater.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,13 @@
import static org.eclipse.pass.support.grant.data.CoeusFieldNames.C_PRIMARY_FUNDER_NAME;
import static org.eclipse.pass.support.grant.data.CoeusFieldNames.C_PRIMARY_FUNDER_POLICY;
import static org.eclipse.pass.support.grant.data.CoeusFieldNames.C_UPDATE_TIMESTAMP;
import static org.eclipse.pass.support.grant.data.CoeusFieldNames.C_USER_EMAIL;
import static org.eclipse.pass.support.grant.data.CoeusFieldNames.C_USER_EMPLOYEE_ID;
import static org.eclipse.pass.support.grant.data.CoeusFieldNames.C_USER_FIRST_NAME;
import static org.eclipse.pass.support.grant.data.CoeusFieldNames.C_USER_LAST_NAME;
import static org.eclipse.pass.support.grant.data.CoeusFieldNames.C_USER_MIDDLE_NAME;
import static org.eclipse.pass.support.grant.data.DateTimeUtil.createZonedDateTime;

import java.io.IOException;
import java.time.ZonedDateTime;
import java.util.Collection;
import java.util.HashMap;
import java.util.ListIterator;
import java.util.Map;
import java.util.Objects;

Expand All @@ -68,19 +63,17 @@
* @author [email protected]
*/

public class DefaultPassUpdater implements PassUpdater {
private static final Logger LOG = LoggerFactory.getLogger(DefaultPassUpdater.class);
abstract class AbstractDefaultPassUpdater implements PassUpdater {
private static final Logger LOG = LoggerFactory.getLogger(AbstractDefaultPassUpdater.class);

private static final String GRANT_ID_TYPE = "grant";
private static final String EMPLOYEE_ID_TYPE = "employeeid";
private static final String FUNDER_ID_TYPE = "funder";

private String domain = "default.domain";
private String latestUpdateString = "";

private final PassClient passClient;
private final PassUpdateStatistics statistics = new PassUpdateStatistics();
private final PassEntityUtil passEntityUtil;

private final Map<String, Grant> grantResultMap = new HashMap<>();

Expand All @@ -92,38 +85,61 @@ public class DefaultPassUpdater implements PassUpdater {

private String mode;

DefaultPassUpdater(PassEntityUtil passEntityUtil) {
this.passEntityUtil = passEntityUtil;
AbstractDefaultPassUpdater() {
this.passClient = PassClient.newInstance();
}

//used in unit testing for injecting a mock client
DefaultPassUpdater(PassEntityUtil passEntityUtil, PassClient passClient) {
this.passEntityUtil = passEntityUtil;
this.passClient = passClient;
}

public void updatePass(Collection<Map<String, String>> results, String mode) {
this.mode = mode;
userMap.clear();
funderMap.clear();
statistics.reset();
statistics.setType(mode);
switch (mode) {
case "grant":
updateGrants(results);
break;
case "user":
updateUsers(results);
break;
case "funder":
updateFunders(results);
break;
default:
break;
case "grant" -> updateGrants(results);
case "user" -> updateUsers(results);
case "funder" -> updateFunders(results);
default -> {
}
}
}

void setDomain(String domain) {
this.domain = domain;
}

/**
* This method provides the latest timestamp of all records processed. After processing, this timestamp
* will be used to be tha base timestamp for the next run of the app
*
* @return the latest update timestamp string
*/
public String getLatestUpdate() {
return this.latestUpdateString;
}

/**
* This returns the final statistics of the processing of the Grant or User Set
*
* @return the report
*/
public String getReport() {
return statistics.getReport();
}

/**
* This returns the final statistics Object - useful in testing
*
* @return the statistics object
*/
public PassUpdateStatistics getStatistics() {
return statistics;
}

public Map<String, Grant> getGrantResultMap() {
return grantResultMap;
}

/**
* Build a Collection of Grants from a ResultSet, then update the grants in Pass
* Because we need to make sure we catch any updates to fields referenced by URIs, we construct
Expand Down Expand Up @@ -272,7 +288,7 @@ private void updateGrants(Collection<Map<String, String>> results) {
? grantUpdateString
: DateTimeUtil.returnLaterUpdate(grantUpdateString, latestUpdateString);
}
} catch (IOException | GrantDataException e) {
} catch (Exception e) {
LOG.error("Error building Grant Row with localKey: " + grantLocalKey, e);
}
}
Expand Down Expand Up @@ -324,7 +340,7 @@ private void updateUsers(Collection<Map<String, String>> results) {
? userUpdateString
: DateTimeUtil.returnLaterUpdate(userUpdateString, latestUpdateString);
}
} catch (IOException | GrantDataException e) {
} catch (Exception e) {
LOG.error("Error processing User: " + rowUser, e);
}
}
Expand Down Expand Up @@ -369,24 +385,6 @@ private void updateFunders(Collection<Map<String, String>> results) {
statistics.setReport(results.size(), funderProcessedCounter);
}

User buildUser(Map<String, String> rowMap) {
User user = new User();
user.setFirstName(rowMap.get(C_USER_FIRST_NAME));
user.setMiddleName(rowMap.getOrDefault(C_USER_MIDDLE_NAME, null));
user.setLastName(rowMap.get(C_USER_LAST_NAME));
user.setDisplayName(rowMap.get(C_USER_FIRST_NAME) + " " + rowMap.get(C_USER_LAST_NAME));
user.setEmail(rowMap.get(C_USER_EMAIL));
String employeeId = rowMap.get(C_USER_EMPLOYEE_ID);
//Build the List of locatorIds - put the most reliable ids first
if (employeeId != null) {
String localKey = GrantDataUtils.buildLocalKey(domain, EMPLOYEE_ID_TYPE, employeeId);
user.getLocatorIds().add(localKey);
}
user.getRoles().add(UserRole.SUBMITTER);
LOG.debug("Built user with employee ID {}", employeeId);
return user;
}

/**
* this method gets called on a grant mode process if the primary funder is different from direct, and also
* any time the updater is called in funder mode
Expand Down Expand Up @@ -443,7 +441,7 @@ private Funder updateFunderInPass(Funder systemFunder) throws IOException, Grant

if (!result.getObjects().isEmpty()) {
Funder storedFunder = getSingleObject(result, fullLocalKey);
Funder updatedFunder = passEntityUtil.update(systemFunder, storedFunder);
Funder updatedFunder = updateFunderIfNeeded(systemFunder, storedFunder);
if (Objects.nonNull(updatedFunder)) { //need to update
passClient.updateObject(updatedFunder);
statistics.addFundersUpdated();
Expand All @@ -466,26 +464,15 @@ private Funder updateFunderInPass(Funder systemFunder) throws IOException, Grant
* @param systemUser the new User object populated from COEUS
* @return the URI for the resource representing the updated User in Pass
*/
private User updateUserInPass(User systemUser) throws IOException, GrantDataException {
//we first check to see if the user is known by the Hopkins ID. If not, we check the employee ID.
//last attempt is the JHED ID. this order is specified by the order of the List as constructed on updatedUser
User passUser = null;
ListIterator<String> idIterator = systemUser.getLocatorIds().listIterator();

while (passUser == null && idIterator.hasNext()) {
String id = String.valueOf(idIterator.next());
if (id != null) {
PassClientSelector<User> selector = new PassClientSelector<>(User.class);
selector.setFilter(RSQL.hasMember("locatorIds", id));
PassClientResult<User> result = passClient.selectObjects(selector);
passUser = result.getObjects().isEmpty()
? null
: getSingleObject(result, id);;
}
}
private User updateUserInPass(User systemUser) throws IOException {
User passUser = systemUser.getLocatorIds().stream()
.map(this::lookupPassUser)
.filter(Objects::nonNull)
.findFirst()
.orElse(null);

if (Objects.nonNull(passUser)) {
User updatedUser = passEntityUtil.update(systemUser, passUser);
User updatedUser = updateUserIfNeeded(systemUser, passUser);
if (Objects.nonNull(updatedUser)) { //need to update
//post COEUS processing goes here
if (!updatedUser.getRoles().contains(UserRole.SUBMITTER)) {
Expand All @@ -495,15 +482,25 @@ private User updateUserInPass(User systemUser) throws IOException, GrantDataExce
statistics.addUsersUpdated();
return updatedUser;
}
} else if (!mode.equals("user")) { //don't have a stored User for this URI - this one is new to Pass
//but don't update if we are in user mode - just update existing users
} else if (!mode.equals("user")) {
passClient.createObject(systemUser);
statistics.addUsersCreated();
return systemUser;
}
return passUser;
}

private User lookupPassUser(String locatorId) {
try {
PassClientSelector<User> selector = new PassClientSelector<>(User.class);
selector.setFilter(RSQL.hasMember("locatorIds", locatorId));
PassClientResult<User> result = passClient.selectObjects(selector);
return result.getObjects().isEmpty() ? null : getSingleObject(result, locatorId);
} catch (IOException | GrantDataException e) {
throw new RuntimeException(e);
}
}

/**
* Take a new Grant object populated as fully as possible from the COEUS pull, and use this
* new information to update an object for the same Grant in Pass (if it exists)
Expand All @@ -525,7 +522,7 @@ private Grant updateGrantInPass(Grant systemGrant) throws IOException, GrantData
if (!result.getObjects().isEmpty()) {
LOG.debug("Found grant with localKey {}", fullLocalKey);
Grant storedGrant = getSingleObject(result, fullLocalKey);
Grant updatedGrant = passEntityUtil.update(systemGrant, storedGrant);
Grant updatedGrant = updateGrantIfNeeded(systemGrant, storedGrant);
if (Objects.nonNull(updatedGrant)) { //need to update
passClient.updateObject(updatedGrant);
statistics.addGrantsUpdated();
Expand All @@ -548,50 +545,4 @@ private <T extends PassEntity> T getSingleObject(PassClientResult<T> result, Str
return result.getObjects().get(0);
}

/**
* This method provides the latest timestamp of all records processed. After processing, this timestamp
* will be used to be tha base timestamp for the next run of the app
*
* @return the latest update timestamp string
*/
public String getLatestUpdate() {
return this.latestUpdateString;
}

/**
* This returns the final statistics of the processing of the Grant or User Set
*
* @return the report
*/
public String getReport() {
return statistics.getReport();
}

/**
* This returns the final statistics Object - useful in testing
*
* @return the statistics object
*/
public PassUpdateStatistics getStatistics() {
return statistics;
}

public Map<String, Grant> getGrantResultMap() {
return grantResultMap;
}

//used in unit test
Map<String, Funder> getFunderMap() {
return funderMap;
}

//used in unit test
Map<String, User> getUserMap() {
return userMap;
}

void setDomain(String domain) {
this.domain = domain;
}

}
18 changes: 1 addition & 17 deletions ...der/pass-grant-data/src/main/java/org/eclipse/pass/support/grant/data/CoeusConnector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,6 @@ public class CoeusConnector implements GrantConnector {

private final Properties funderPolicyProperties;

private DirectoryServiceUtil directoryServiceUtil;

/**
* Class constructor.
* @param connectionProperties the connection props
Expand All @@ -69,7 +67,6 @@ public CoeusConnector(Properties connectionProperties, Properties funderPolicyPr
if (connectionProperties.getProperty(COEUS_PASS) != null) {
this.coeusPassword = connectionProperties.getProperty(COEUS_PASS);
}
this.directoryServiceUtil = new DirectoryServiceUtil(connectionProperties);
}

this.funderPolicyProperties = funderPolicyProperties;
Expand All @@ -94,7 +91,7 @@ public List<Map<String, String>> retrieveUpdates(String queryString, String mode
* @return the {@code ResultSet} from the query
*/
private List<Map<String, String>> retrieveGrantUpdates(String queryString)
throws ClassNotFoundException, SQLException, IOException {
throws ClassNotFoundException, SQLException {

List<Map<String, String>> mapList = new ArrayList<>();

Expand Down Expand Up @@ -130,12 +127,6 @@ private List<Map<String, String>> retrieveGrantUpdates(String queryString)
rowMap.put(CoeusFieldNames.C_UPDATE_TIMESTAMP, rs.getString(CoeusFieldNames.C_UPDATE_TIMESTAMP));
rowMap.put(CoeusFieldNames.C_ABBREVIATED_ROLE, rs.getString(CoeusFieldNames.C_ABBREVIATED_ROLE));

String employeeId = rs.getString(CoeusFieldNames.C_USER_EMPLOYEE_ID);
if (employeeId != null) {
rowMap.put(CoeusFieldNames.C_USER_HOPKINS_ID,
directoryServiceUtil.getHopkinsIdForEmployeeId(employeeId));
}

String primaryFunderLocalKey = rs.getString(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY);
rowMap.put(CoeusFieldNames.C_PRIMARY_FUNDER_LOCAL_KEY, primaryFunderLocalKey);
if (primaryFunderLocalKey != null &&
Expand Down Expand Up @@ -225,18 +216,11 @@ private List<Map<String, String>> retrieveUserUpdates(String queryString)
rs.getString(CoeusFieldNames.C_USER_INSTITUTIONAL_ID));
rowMap.put(CoeusFieldNames.C_USER_EMPLOYEE_ID, rs.getString(CoeusFieldNames.C_USER_EMPLOYEE_ID));
rowMap.put(CoeusFieldNames.C_UPDATE_TIMESTAMP, rs.getString(CoeusFieldNames.C_UPDATE_TIMESTAMP));
String employeeId = rs.getString(CoeusFieldNames.C_USER_EMPLOYEE_ID);
if (employeeId != null) {
rowMap.put(CoeusFieldNames.C_USER_HOPKINS_ID,
directoryServiceUtil.getHopkinsIdForEmployeeId(employeeId));
}
LOG.debug("Record processed: {}", rowMap);
if (!mapList.contains(rowMap)) {
mapList.add(rowMap);
}
}
} catch (IOException e) {
e.printStackTrace();
}
LOG.info("Retrieved result set from COEUS: {} records processed", mapList.size());
return mapList;
Expand Down
Loading
Loading