diff --git a/config/indices.sql b/config/indices.sql index 72a69afbf8..8deb24b902 100644 --- a/config/indices.sql +++ b/config/indices.sql @@ -5,11 +5,14 @@ -- UGH, i guess _IDX (in caps!) should be what we standardize on. that is what datanucleus does. -- to make matters worse, we have some places we named them explicitely in package.jdo (but those should fix themselves?) +BEGIN; + CREATE INDEX IF NOT EXISTS "MEDIAASSET_PARENTID_idx" ON "MEDIAASSET" ("PARENTID"); CREATE INDEX IF NOT EXISTS "MEDIAASSET_HASHCODE_idx" ON "MEDIAASSET" ("HASHCODE"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_LOCATIONID_idx" ON "ENCOUNTER" ("LOCATIONID"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_STATE_idx" ON "ENCOUNTER" ("STATE"); +CREATE INDEX IF NOT EXISTS "ENCOUNTER_MODIFIED_idx" ON "ENCOUNTER" ("MODIFIED"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_INDIVIDUALID_idx" ON "ENCOUNTER" ("INDIVIDUALID"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_DATEINMILLISECONDS_idx" ON "ENCOUNTER" ("DATEINMILLISECONDS"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_DECIMALLATITUDE_idx" ON "ENCOUNTER" ("DECIMALLATITUDE"); @@ -45,3 +48,5 @@ CREATE INDEX IF NOT EXISTS "TASK_CREATED_IDX" ON "TASK" ("CREATED"); INSERT INTO "RELATIONSHIP" ("RELATIONSHIP_ID", "MARKEDINDIVIDUALNAME1", "MARKEDINDIVIDUALNAME2", "MARKEDINDIVIDUALROLE1", "MARKEDINDIVIDUALROLE2", "TYPE", "STARTTIME", "ENDTIME") VALUES (0, (SELECT "INDIVIDUALID" FROM "MARKEDINDIVIDUAL" ORDER BY random() LIMIT 1), (SELECT "INDIVIDUALID" FROM "MARKEDINDIVIDUAL" ORDER BY random() LIMIT 1), 'placeholder', 'placeholder', 'placeholder-to-prevent-empty-table', 0, 0); +END; + diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index b1afc85ee3..ff94170d5d 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -77,8 +77,10 @@ */ public abstract void addComments(final String newComments); - public abstract List userIdsWithViewAccess(Shepherd myShepherd); - public abstract List userIdsWithEditAccess(Shepherd myShepherd); + // issue 785 makes this no longer necessary; the overrides are left on Occurrence and MarkedIndividual + // for now as reference -- but are not called. they will need to be addressed when these classes are searchable + // public abstract List userIdsWithViewAccess(Shepherd myShepherd); + // public abstract List userIdsWithEditAccess(Shepherd myShepherd); public abstract String opensearchIndexName(); @@ -97,6 +99,8 @@ public JSONObject opensearchMapping() { map.put("version", new org.json.JSONObject("{\"type\": \"long\"}")); // id should be keyword for the sake of sorting map.put("id", new org.json.JSONObject("{\"type\": \"keyword\"}")); + map.put("viewUsers", new org.json.JSONObject("{\"type\": \"keyword\"}")); + map.put("editUsers", new org.json.JSONObject("{\"type\": \"keyword\"}")); return map; } @@ -153,11 +157,24 @@ public void opensearchUnindexDeep() this.opensearchUnindex(); } + public void opensearchUpdate(final JSONObject updateData) + throws IOException { + if (updateData == null) return; + OpenSearch opensearch = new OpenSearch(); + + opensearch.indexUpdate(this.opensearchIndexName(), this.getId(), updateData); + } + // should be overridden public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd) throws IOException, JsonProcessingException { jgen.writeStringField("id", this.getId()); jgen.writeNumberField("version", this.getVersion()); + jgen.writeNumberField("indexTimestamp", System.currentTimeMillis()); + +/* + these are no longer computed in the general opensearchIndex() call. + they are too expensive. see Encounter.opensearchIndexPermission() jgen.writeFieldName("viewUsers"); jgen.writeStartArray(); @@ -172,6 +189,7 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeString(id); } jgen.writeEndArray(); + */ } public void opensearchDocumentSerializer(JsonGenerator jgen) @@ -196,6 +214,11 @@ public static JSONObject opensearchQuery(final String indexname, final JSONObjec return res; } + // this is so we can call it on Base obj, but really is only needed by [overridden by] Encounter (currently) + public boolean getOpensearchProcessPermissions() { + return false; + } + public static Map getAllVersions(Shepherd myShepherd, String sql) { Query query = myShepherd.getPM().newQuery("javax.jdo.query.SQL", sql); Map rtn = new HashMap(); diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 6c28686376..99303edfb1 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -17,6 +17,7 @@ import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; @@ -118,6 +119,7 @@ public class Encounter extends Base implements java.io.Serializable { private Double immunoglobin; private Boolean sampleTakenForDiet; private Boolean injured; + private boolean opensearchProcessPermissions = false; private ArrayList observations = new ArrayList(); @@ -3141,25 +3143,31 @@ public boolean isUserOwner(User user) { // the definition of this might change? return false; } - @Override public List userIdsWithViewAccess(Shepherd myShepherd) { + // new logic means we only need users who are in collab with submitting user + // and if public, we dont need to do this at all + public List userIdsWithViewAccess(Shepherd myShepherd) { List ids = new ArrayList(); - for (User user : myShepherd.getAllUsers()) { - if ((user.getId() != null) && this.canUserView(user, myShepherd)) - ids.add(user.getId()); + if (this.isPubliclyReadable()) return ids; + List collabs = Collaboration.collaborationsForUser(myShepherd, + this.getSubmitterID()); + for (Collaboration collab : collabs) { + User user = myShepherd.getUser(collab.getOtherUsername(this.getSubmitterID())); + if (user != null) ids.add(user.getId()); } return ids; } - @Override public List userIdsWithEditAccess(Shepherd myShepherd) { +/* + public List userIdsWithEditAccess(Shepherd myShepherd) { List ids = new ArrayList(); - for (User user : myShepherd.getAllUsers()) { + for (User user : myShepherd.getUsersWithUsername()) { if ((user.getId() != null) && this.canUserEdit(user)) ids.add(user.getId()); } return ids; } - + */ public JSONObject sanitizeJson(HttpServletRequest request, JSONObject jobj) throws JSONException { boolean fullAccess = this.canUserAccess(request); @@ -3853,6 +3861,163 @@ public int hashCode() { // we need this along with equals() for collections meth return this.getCatalogNumber().hashCode(); } + // sadly, this mess needs to carry on the tradition set up in User.isUsernameAnonymous() + // thanks to the logic in Collaboration.canUserAccessOwnedObject() + public boolean isPubliclyReadable() { + if (!Collaboration.securityEnabled("context0")) return true; + return User.isUsernameAnonymous(this.submitterID); + } + + public boolean getOpensearchProcessPermissions() { + return opensearchProcessPermissions; + } + + public void setOpensearchProcessPermissions(boolean value) { + opensearchProcessPermissions = value; + } + + // wrapper for below, that checks if we really need to be run + public static void opensearchIndexPermissionsBackground(Shepherd myShepherd) { + boolean runIt = false; + Long lastRun = OpenSearch.getPermissionsTimestamp(myShepherd); + long now = System.currentTimeMillis(); + + if ((lastRun == null) || + ((now - lastRun) > OpenSearch.BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES * 60000)) { + System.out.println( + "opensearchIndexPermissionsBackground: forced run due to max time since previous"); + runIt = true; + } + boolean needed = OpenSearch.getPermissionsNeeded(myShepherd); + if (needed && !runIt) { + System.out.println("opensearchIndexPermissionsBackground: running due to needed=true"); + runIt = true; + } + if (!runIt) { + System.out.println("opensearchIndexPermissionsBackground: running not required; done"); + return; + } + // i think we should set these first... tho they may not get persisted til after? + OpenSearch.setPermissionsTimestamp(myShepherd); + OpenSearch.setPermissionsNeeded(myShepherd, false); + opensearchIndexPermissions(); + System.out.println("opensearchIndexPermissionsBackground: running completed"); + } + +/* note: there are a great deal of users with *no username* that seem to appear in enc.submitters array. + however, very few (2 out of 5600+) encounters with such .submitters have a blank submitterID value + therefore: submitterID will be assumed to be a required value on users which need to be + + this seems further validated by the facts that: + - canUserAccess(user) returns false if no username on user + - a user wihtout a username cant be logged in (and thus cant search) + + "admin" users are just ignored entirely, as they will be exempt from the viewUsers criteria during searching. + + other than "ownership" (via submitterID), a user can view if they have view or edit collab with + another user. so we frontload *approved* collabs for every user here too. + + in terms of "public" encounters, it seems that (based on Collaboration.canUserAccessEncounter()), + encounters with submitterID in (NULL, "public", "", "N/A" [ugh]) is readable by anyone; so we will + skip these from processing as they should be flagged with the boolean isPubliclyReadable in indexing + */ + public static void opensearchIndexPermissions() { + Util.mark("perm start"); + long startT = System.currentTimeMillis(); + System.out.println("opensearchIndexPermissions(): begin..."); + // no security => everything publiclyReadable - saves us work, no? + if (!Collaboration.securityEnabled("context0")) return; + OpenSearch os = new OpenSearch(); + Map > collab = new HashMap >(); + Map usernameToId = new HashMap(); + Shepherd myShepherd = new Shepherd("context0"); + myShepherd.setAction("Encounter.opensearchIndexPermissions"); + myShepherd.beginDBTransaction(); + int nonAdminCt = 0; + // it seems as though user.uuid is *required* so we can trust that + try { + for (User user : myShepherd.getUsersWithUsername()) { + usernameToId.put(user.getUsername(), user.getId()); + if (user.isAdmin(myShepherd)) continue; + nonAdminCt++; + List collabsFor = Collaboration.collaborationsForUser(myShepherd, + user.getUsername()); + if (Util.collectionIsEmptyOrNull(collabsFor)) continue; + for (Collaboration col : collabsFor) { + if (!col.isApproved() && !col.isEditApproved()) continue; + if (!collab.containsKey(user.getId())) + collab.put(user.getId(), new HashSet()); + collab.get(user.getId()).add(col.getOtherUsername(user.getUsername())); + } + } + } catch (Exception ex) { + ex.printStackTrace(); + } + Util.mark("perm: user build done", startT); + System.out.println("opensearchIndexPermissions(): " + usernameToId.size() + + " total users; " + nonAdminCt + " non-admin; " + collab.size() + " have active collab"); + // now iterated over (non-public) encounters + int encCount = 0; + org.json.JSONObject updateData = new org.json.JSONObject(); + // we do not need full Encounter objects here to update index docs, so lets do this via sql/fields - much faster + String sql = + "SELECT \"CATALOGNUMBER\", \"SUBMITTERID\" FROM \"ENCOUNTER\" WHERE \"SUBMITTERID\" IS NOT NULL AND \"SUBMITTERID\" != '' AND \"SUBMITTERID\" != 'N/A' AND \"SUBMITTERID\" != 'public'"; + Query q = null; + try { + q = myShepherd.getPM().newQuery("javax.jdo.query.SQL", sql); + List results = (List)q.execute(); + Iterator it = results.iterator(); + Util.mark("perm: start encs, size=" + results.size(), startT); + while (it.hasNext()) { + Object[] row = (Object[])it.next(); + String id = (String)row[0]; + String submitterId = (String)row[1]; + org.json.JSONArray viewUsers = new org.json.JSONArray(); + String uid = usernameToId.get(submitterId); + if (uid == null) { + // see issue 939 for example :( + System.out.println("opensearchIndexPermissions(): WARNING invalid username " + + submitterId + " on enc " + id); + continue; + } + encCount++; + if (encCount % 1000 == 0) Util.mark("enc[" + encCount + "]", startT); + // viewUsers.put(uid); // we no longer do this as we use submitterUserId from regular indexing in query filter + if (collab.containsKey(uid)) { + for (String colUsername : collab.get(uid)) { + String colId = usernameToId.get(colUsername); + if (colId == null) { + System.out.println( + "opensearchIndexPermissions(): WARNING invalid username " + + colUsername + " in collaboration with userId=" + uid); + continue; + } + viewUsers.put(colId); + } + } + if (viewUsers.length() > 0) { + updateData.put("viewUsers", viewUsers); + try { + os.indexUpdate("encounter", id, updateData); + } catch (Exception ex) { + // keeping this quiet cuz it can get noise while index builds + // System.out.println("opensearchIndexPermissions(): WARNING failed to update viewUsers on enc " + enc.getId() + "; likely has not been indexed yet: " + ex); + } + } + } + q.closeAll(); + } catch (Exception ex) { + System.out.println("opensearchIndexPermissions(): failed during encounter loop: " + ex); + ex.printStackTrace(); + } finally { + if (q != null) q.closeAll(); + } + Util.mark("perm: done encs", startT); + myShepherd.rollbackAndClose(); + System.out.println("opensearchIndexPermissions(): ...end [" + encCount + " encs; " + + Math.round((System.currentTimeMillis() - startT) / 1000) + "sec]"); + } + public static org.json.JSONObject opensearchQuery(final org.json.JSONObject query, int numFrom, int pageSize, String sort, String sortOrder) throws IOException { @@ -3896,6 +4061,7 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeStringField("state", this.getState()); jgen.writeStringField("occurrenceRemarks", this.getOccurrenceRemarks()); jgen.writeStringField("otherCatalogNumbers", this.getOtherCatalogNumbers()); + jgen.writeBooleanField("publiclyReadable", this.isPubliclyReadable()); String featuredAssetId = null; List mas = this.getMedia(); @@ -3906,8 +4072,11 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeStartObject(); jgen.writeNumberField("id", ma.getId()); jgen.writeStringField("uuid", ma.getUUID()); - java.net.URL url = ma.safeURL(myShepherd); - if (url != null) jgen.writeStringField("url", url.toString()); + try { + // historic data might throw IllegalArgumentException: Path not under given root + java.net.URL url = ma.safeURL(myShepherd); + if (url != null) jgen.writeStringField("url", url.toString()); + } catch (Exception ex) {} jgen.writeEndObject(); if (featuredAssetId == null) featuredAssetId = ma.getUUID(); } @@ -3917,6 +4086,8 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeNullField("assignedUsername"); } else { jgen.writeStringField("assignedUsername", this.submitterID); + User submitter = this.getSubmitterUser(myShepherd); + if (submitter != null) jgen.writeStringField("submitterUserId", submitter.getId()); } jgen.writeArrayFieldStart("submitters"); for (String id : this.getAllSubmitterIds(myShepherd)) { @@ -4027,7 +4198,8 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd Double dlat = this.getDecimalLatitudeAsDouble(); Double dlon = this.getDecimalLongitudeAsDouble(); - if ((dlat == null) || (dlon == null)) { + if ((dlat == null) || !Util.isValidDecimalLatitude(dlat) || (dlon == null) || + !Util.isValidDecimalLongitude(dlon)) { jgen.writeNullField("locationGeoPoint"); } else { jgen.writeObjectFieldStart("locationGeoPoint"); @@ -4127,6 +4299,19 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeNumberField(type, bmeas.get(type).getValue()); } jgen.writeEndObject(); + // this gets set on specific single-encounter-only actions, when extra expense is okay + // otherwise this will be computed by permissions backgrounding + if (this.getOpensearchProcessPermissions()) { + System.out.println("opensearchProcessPermissions=true for " + this.getId() + + "; indexing permissions"); + jgen.writeFieldName("viewUsers"); + jgen.writeStartArray(); + for (String id : this.userIdsWithViewAccess(myShepherd)) { + System.out.println("opensearch whhhh: " + id); + jgen.writeString(id); + } + jgen.writeEndArray(); + } } @Override public long getVersion() { @@ -4155,6 +4340,7 @@ public org.json.JSONObject opensearchMapping() { map.put("taxonomy", keywordType); map.put("occurrenceId", keywordType); map.put("state", keywordType); + map.put("submitterUserId", keywordType); // all case-insensitive keyword-ish types map.put("locationId", keywordNormalType); @@ -4214,7 +4400,13 @@ public static int[] opensearchSyncIndex(Shepherd myShepherd, int stopAfter) int ct = 0; for (String id : needIndexing) { Encounter enc = myShepherd.getEncounter(id); - if (enc != null) os.index(indexName, enc); + try { + if (enc != null) os.index(indexName, enc); + } catch (Exception ex) { + System.out.println("Encounter.opensearchSyncIndex(): index failed " + enc + " => " + + ex.toString()); + ex.printStackTrace(); + } if (ct % 500 == 0) System.out.println("Encounter.opensearchSyncIndex needIndexing: " + ct + "/" + rtn[0]); @@ -4504,6 +4696,7 @@ public void sendCreationEmails(Shepherd myShepherd, String langCode) { public void opensearchIndexDeep() throws IOException { final String encId = this.getId(); + final Encounter origEnc = this; ExecutorService executor = Executors.newFixedThreadPool(4); Runnable rn = new Runnable() { public void run() { @@ -4513,6 +4706,8 @@ public void run() { try { Encounter enc = bgShepherd.getEncounter(encId); if (enc == null) { + // we use origEnc if we can (especially necessary on initial creation of Encounter) + if (origEnc != null) origEnc.opensearchIndex(); bgShepherd.rollbackAndClose(); executor.shutdown(); return; diff --git a/src/main/java/org/ecocean/EncounterQueryProcessor.java b/src/main/java/org/ecocean/EncounterQueryProcessor.java index a1f1a2934f..04d3ba505e 100644 --- a/src/main/java/org/ecocean/EncounterQueryProcessor.java +++ b/src/main/java/org/ecocean/EncounterQueryProcessor.java @@ -61,7 +61,14 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer String indexName = searchQuery.optString("indexName", null); if (indexName == null) return failed; searchQuery = OpenSearch.queryScrubStored(searchQuery); - JSONObject sanitized = OpenSearch.querySanitize(searchQuery, user); + JSONObject sanitized = null; + try { + sanitized = OpenSearch.querySanitize(searchQuery, user, myShepherd); + } catch (IOException ex) { + ex.printStackTrace(); + // this should be unlikely, so fail hard + throw new RuntimeException("query failed"); + } OpenSearch os = new OpenSearch(); String sort = request.getParameter("sort"); String sortOrder = request.getParameter("sortOrder"); @@ -134,8 +141,7 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer String variables_statement = " VARIABLES org.ecocean.User user; org.ecocean.Organization org"; jdoqlVariableDeclaration = addOrgVars(variables_statement, filter); - } else { - } + } else {} // end filter for organization------------------ // filter for projectName------------------- if (Util.isUUID(request.getParameter("projectId"))) { @@ -169,8 +175,7 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer } String variables_statement = " VARIABLES org.ecocean.Project proj"; jdoqlVariableDeclaration = addOrgVars(variables_statement, filter); - } else { - } + } else {} // end filter for projectName------------------ // username filters------------------------------------------------- String[] usernames = request.getParameterValues("username"); @@ -1356,7 +1361,6 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer (!request.getParameter("nameField").equals(""))) { String nameString = request.getParameter("nameField").replaceAll("%20", " ").toLowerCase().trim(); - String filterString = "" + "(" + "(submitters.contains(submitter) && ((submitter.fullName.toLowerCase().indexOf('" + nameString + "') != -1)||(submitter.emailAddress.toLowerCase().indexOf('" + @@ -1529,7 +1533,6 @@ public static EncounterQueryResult processQuery(Shepherd myShepherd, HttpServlet String currentUser = null; if (request.getUserPrincipal() != null) currentUser = request.getUserPrincipal().getName(); - String searchQueryId = request.getParameter("searchQueryId"); long startTime = System.currentTimeMillis(); if (searchQueryId != null) { @@ -1546,7 +1549,14 @@ public static EncounterQueryResult processQuery(Shepherd myShepherd, HttpServlet return new EncounterQueryResult(rEncounters, "searchQuery has no indexName", "OpenSearch id " + searchQueryId); searchQuery = OpenSearch.queryScrubStored(searchQuery); - JSONObject sanitized = OpenSearch.querySanitize(searchQuery, user); + JSONObject sanitized = null; + try { + sanitized = OpenSearch.querySanitize(searchQuery, user, myShepherd); + } catch (IOException ex) { + ex.printStackTrace(); + // this should be unlikely, so fail hard + throw new RuntimeException("query failed"); + } OpenSearch os = new OpenSearch(); String sort = request.getParameter("sort"); String sortOrder = request.getParameter("sortOrder"); @@ -1620,7 +1630,6 @@ public static EncounterQueryResult processQuery(Shepherd myShepherd, HttpServlet rEncounters.add(temp_enc); } } - query.closeAll(); // silo security logging diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index bca1cf48d5..3eaa80005c 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2018,7 +2018,8 @@ public boolean canUserAccess(HttpServletRequest request) { return Collaboration.canUserAccessMarkedIndividual(this, request); } - @Override public List userIdsWithViewAccess(Shepherd myShepherd) { + // see note on Base class + public List userIdsWithViewAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getAllUsers()) { @@ -2027,7 +2028,8 @@ public boolean canUserAccess(HttpServletRequest request) { return ids; } - @Override public List userIdsWithEditAccess(Shepherd myShepherd) { + // see note on Base class + public List userIdsWithEditAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getAllUsers()) { diff --git a/src/main/java/org/ecocean/Occurrence.java b/src/main/java/org/ecocean/Occurrence.java index 04ac4783e2..1f4cce4a65 100644 --- a/src/main/java/org/ecocean/Occurrence.java +++ b/src/main/java/org/ecocean/Occurrence.java @@ -817,7 +817,8 @@ public boolean canUserAccess(HttpServletRequest request) { return Collaboration.canUserAccessOccurrence(this, request); } - @Override public List userIdsWithViewAccess(Shepherd myShepherd) { + // see note on Base class + public List userIdsWithViewAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getAllUsers()) { @@ -829,7 +830,8 @@ public boolean canUserAccess(HttpServletRequest request) { return ids; } - @Override public List userIdsWithEditAccess(Shepherd myShepherd) { + // see note on Base class + public List userIdsWithEditAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getAllUsers()) { diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 38edc3fdac..e20c100950 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -66,6 +66,12 @@ public class OpenSearch { "backgroundDelayMinutes", 20); public static int BACKGROUND_SLICE_SIZE = (Integer)getConfigurationValue("backgroundSliceSize", 2500); + public static int BACKGROUND_PERMISSIONS_MINUTES = (Integer)getConfigurationValue( + "backgroundPermissionsMinutes", 10); + public static int BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES = (Integer)getConfigurationValue( + "backgroundPermissionsMaxForceMinutes", 45); + public static String PERMISSIONS_LAST_RUN_KEY = "OpenSearch_permissions_last_run_timestamp"; + public static String PERMISSIONS_NEEDED_KEY = "OpenSearch_permissions_needed"; public static String QUERY_STORAGE_DIR = "/tmp"; // FIXME static String ACTIVE_TYPE_FOREGROUND = "opensearch_indexing_foreground"; static String ACTIVE_TYPE_BACKGROUND = "opensearch_indexing_background"; @@ -136,25 +142,45 @@ public static boolean skipAutoIndexing() { // http://localhost:9200/_cat/indices?v public static void backgroundStartup(String context) { - final ScheduledExecutorService schedExec = Executors.newScheduledThreadPool(2); - final ScheduledFuture schedFuture = schedExec.scheduleWithFixedDelay(new Runnable() { - public void run() { - Shepherd myShepherd = new Shepherd(context); - myShepherd.setAction("OpenSearch.background"); - try { - myShepherd.beginDBTransaction(); - System.out.println("OpenSearch background running..."); - Encounter.opensearchSyncIndex(myShepherd, BACKGROUND_SLICE_SIZE); - System.out.println("OpenSearch background finished."); - myShepherd.rollbackAndClose(); - } catch (Exception ex) { - ex.printStackTrace(); - myShepherd.rollbackAndClose(); + final ScheduledExecutorService schedExec = Executors.newScheduledThreadPool(8); + final ScheduledFuture schedFutureIndexing = schedExec.scheduleWithFixedDelay( + new Runnable() { + public void run() { + Shepherd myShepherd = new Shepherd(context); + myShepherd.setAction("OpenSearch.backgroundIndexing"); + try { + myShepherd.beginDBTransaction(); + System.out.println("OpenSearch background indexing running..."); + Encounter.opensearchSyncIndex(myShepherd, BACKGROUND_SLICE_SIZE); + System.out.println("OpenSearch background indexing finished."); + myShepherd.rollbackAndClose(); + } catch (Exception ex) { + ex.printStackTrace(); + myShepherd.rollbackAndClose(); + } } - } - }, 2, // initial delay + }, 2, // initial delay BACKGROUND_DELAY_MINUTES, // period delay *after* execution finishes TimeUnit.MINUTES); // unit of delays above + final ScheduledFuture schedFuturePermissions = schedExec.scheduleWithFixedDelay( + new Runnable() { + public void run() { + Shepherd myShepherd = new Shepherd(context); + myShepherd.setAction("OpenSearch.backgroundPermissions"); + try { + myShepherd.beginDBTransaction(); + System.out.println("OpenSearch background permissions running..."); + Encounter.opensearchIndexPermissionsBackground(myShepherd); + System.out.println("OpenSearch background permissions finished."); + myShepherd.commitDBTransaction(); // need commit since we might have changed SystemValues + myShepherd.closeDBTransaction(); + } catch (Exception ex) { + ex.printStackTrace(); + myShepherd.rollbackAndClose(); + } + } + }, 8, // initial delay + BACKGROUND_PERMISSIONS_MINUTES, TimeUnit.MINUTES); // unit of delays above try { schedExec.awaitTermination(5000, TimeUnit.MILLISECONDS); @@ -472,6 +498,17 @@ public void indexClose(final String indexName) System.out.println("OpenSearch.indexClose() on " + indexName + ": " + rtn); } + // updateData is { field0: value0, field1: value1, ... } + public void indexUpdate(final String indexName, String id, JSONObject updateData) + throws IOException { + if ((id == null) || (updateData == null)) throw new IOException("missing id or updateData"); + JSONObject doc = new JSONObject(); + doc.put("doc", updateData); + Request updateRequest = new Request("POST", indexName + "/_update/" + id); + updateRequest.setJsonEntity(doc.toString()); + getRestResponse(updateRequest); + } + // returns 2 lists: (1) items needing (re-)indexing; (2) items needing removal public static List > resolveVersions(Map objVersions, Map indexVersions) { @@ -576,6 +613,77 @@ public Long getIndexTimestamp(Shepherd myShepherd, String indexName) { return SystemValue.getLong(myShepherd, INDEX_TIMESTAMP_PREFIX + indexName); } + public static long setPermissionsTimestamp(Shepherd myShepherd) { + long now = System.currentTimeMillis(); + + SystemValue.set(myShepherd, PERMISSIONS_LAST_RUN_KEY, now); + return now; + } + + public static Long getPermissionsTimestamp(Shepherd myShepherd) { + return SystemValue.getLong(myShepherd, PERMISSIONS_LAST_RUN_KEY); + } + + public static void setPermissionsNeeded(Shepherd myShepherd, boolean value) { + SystemValue.set(myShepherd, PERMISSIONS_NEEDED_KEY, value); + } + + public static void setPermissionsNeeded(boolean value) { + Shepherd myShepherd = new Shepherd("context0"); + + myShepherd.setAction("OpenSearch.setPermissionsNeeded"); + myShepherd.beginDBTransaction(); + try { + setPermissionsNeeded(myShepherd, value); + myShepherd.commitDBTransaction(); + myShepherd.closeDBTransaction(); + } catch (Exception ex) { + ex.printStackTrace(); + myShepherd.rollbackAndClose(); + } + } + + public static boolean getPermissionsNeeded(Shepherd myShepherd) { + Boolean value = SystemValue.getBoolean(myShepherd, PERMISSIONS_NEEDED_KEY); + + if (value == null) return false; + return value; + } + + public static JSONObject querySanitize(JSONObject query, User user, Shepherd myShepherd) + throws IOException { + if ((query == null) || (user == null)) throw new IOException("empty query or user"); + // do not add permissions clause when we are admin, as user has no restriction + if (user.isAdmin(myShepherd)) return query; + // if (!Collaboration.securityEnabled("context0")) TODO do we want to allow everything searchable? +/* + JSONObject permClause = new JSONObject("{\"bool\": {\"should\": [] }}"); + "{\"bool\": {\"should\": [{\"term\": {\"publiclyReadable\": true}}, {\"term\": {\"viewUsers\": \"" + + user.getId() + "\"}} ] }}"); + */ + JSONArray shouldArr = new JSONArray(); + shouldArr.put(new JSONObject("{\"term\": {\"publiclyReadable\": true}}")); + shouldArr.put(new JSONObject("{\"term\": {\"submitterUserId\": \"" + user.getId() + + "\"}}")); + shouldArr.put(new JSONObject("{\"term\": {\"viewUsers\": \"" + user.getId() + "\"}}")); + JSONObject pshould = new JSONObject(); + pshould.put("should", shouldArr); + JSONObject permClause = new JSONObject(); + permClause.put("bool", pshould); + JSONObject newQuery = new JSONObject(query.toString()); + try { + JSONArray filter = newQuery.getJSONObject("query").getJSONObject("bool").getJSONArray( + "filter"); + filter.put(permClause); + } catch (Exception ex) { + System.out.println( + "OpenSearch.querySanitize() failed to find placement for permissions in query=" + + query + "; cause: " + ex); + throw new IOException("unable to find placement for permissions clause in query"); + } + return newQuery; + } + public static boolean indexingActive() { return indexingActiveBackground() || indexingActiveForeground(); } @@ -652,19 +760,6 @@ static boolean getActive(String type) { return active; } - public static JSONObject querySanitize(JSONObject query, User user) { - if ((query == null) || (user == null)) return query; - JSONObject newQuery = new JSONObject(query.toString()); - try { - JSONArray filter = newQuery.getJSONObject("query").getJSONObject("bool").getJSONArray( - "filter"); - filter.put(new JSONObject("{\"match\": {\"viewUsers\": \"" + user.getId() + "\"}}")); - } catch (Exception ex) { - System.out.println("OpenSearch.querySanitize() failed to find filter element: " + ex); - } - return newQuery; - } - // TODO: right now this respects index timestamp and only indexes objects with versions > timestamp. // want to make an option to index everything and ignore version/timestamp. public void indexAll(Shepherd myShepherd, Base obj) diff --git a/src/main/java/org/ecocean/SystemValue.java b/src/main/java/org/ecocean/SystemValue.java index f66a9c0ed6..250d9fbe0f 100644 --- a/src/main/java/org/ecocean/SystemValue.java +++ b/src/main/java/org/ecocean/SystemValue.java @@ -57,7 +57,11 @@ public static SystemValue obtain(Shepherd myShepherd, String key) { public void store(Shepherd myShepherd) { this.version = System.currentTimeMillis(); - myShepherd.getPM().makePersistent(this); + try { + myShepherd.getPM().makePersistent(this); + } catch (Exception ex) { + System.out.println("SystemValue.store() failed to store " + this + ": " + ex); + } } private static SystemValue _set(Shepherd myShepherd, String key, String type, Object val) { diff --git a/src/main/java/org/ecocean/WildbookLifecycleListener.java b/src/main/java/org/ecocean/WildbookLifecycleListener.java index 9666e0e376..2fe161e2a3 100644 --- a/src/main/java/org/ecocean/WildbookLifecycleListener.java +++ b/src/main/java/org/ecocean/WildbookLifecycleListener.java @@ -5,6 +5,7 @@ import org.datanucleus.enhancement.Persistable; import org.ecocean.Base; import org.ecocean.OpenSearch; +import org.ecocean.security.Collaboration; // https://www.datanucleus.org/products/accessplatform_4_1/jdo/lifecycle_callbacks.html#listeners @@ -64,6 +65,10 @@ public void postStore(InstanceLifecycleEvent event) { } catch (IOException ex) { ex.printStackTrace(); } + } else if (Collaboration.class.isInstance(obj)) { + System.out.println("WildbookLifecycleListener postStore() event on " + obj + + " triggering permissionsNeeded=true"); + OpenSearch.setPermissionsNeeded(true); } } diff --git a/src/main/java/org/ecocean/api/BaseObject.java b/src/main/java/org/ecocean/api/BaseObject.java index 30cd991822..8e1abbff28 100644 --- a/src/main/java/org/ecocean/api/BaseObject.java +++ b/src/main/java/org/ecocean/api/BaseObject.java @@ -24,6 +24,7 @@ import org.ecocean.media.MediaAssetFactory; import org.ecocean.MarkedIndividual; import org.ecocean.Occurrence; +import org.ecocean.OpenSearch; import org.ecocean.Project; import org.ecocean.resumableupload.UploadServlet; import org.ecocean.servlet.importer.ImportTask; @@ -175,6 +176,7 @@ protected JSONObject processPost(HttpServletRequest request, String[] args, JSON if ((obj != null) && (rtn.optInt("statusCode", 0) == 200)) { System.out.println("BaseObject.processPost() success (200) creating " + obj + " from payload " + payload); + OpenSearch.setPermissionsNeeded(myShepherd, true); myShepherd.commitDBTransaction(); MediaAsset.updateStandardChildrenBackground(context, maIds); if (encounterForIA != null) { diff --git a/src/main/java/org/ecocean/api/Login.java b/src/main/java/org/ecocean/api/Login.java index 748c43e31f..0fd50232d2 100644 --- a/src/main/java/org/ecocean/api/Login.java +++ b/src/main/java/org/ecocean/api/Login.java @@ -16,9 +16,9 @@ import org.apache.shiro.authc.UsernamePasswordToken; import org.apache.shiro.session.Session; import org.apache.shiro.subject.Subject; +import org.apache.shiro.SecurityUtils; import org.apache.shiro.web.util.SavedRequest; import org.apache.shiro.web.util.WebUtils; -import org.apache.shiro.SecurityUtils; import org.ecocean.servlet.ServletUtilities; import org.ecocean.Shepherd; @@ -74,15 +74,12 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) success = true; results = user.infoJSONObject(context, true); results.put("success", true); - - //check for redirect URL - SavedRequest saved=WebUtils.getAndClearSavedRequest(request); - if(saved!=null) { - results.put("redirectUrl",saved.getRequestUrl()); - } - - + // check for redirect URL + SavedRequest saved = WebUtils.getAndClearSavedRequest(request); + if (saved != null) { + results.put("redirectUrl", saved.getRequestUrl()); + } } catch (UnknownAccountException ex) { // username not found ex.printStackTrace(); diff --git a/src/main/java/org/ecocean/api/SearchApi.java b/src/main/java/org/ecocean/api/SearchApi.java index 4751baa255..42df780548 100644 --- a/src/main/java/org/ecocean/api/SearchApi.java +++ b/src/main/java/org/ecocean/api/SearchApi.java @@ -62,7 +62,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) String sort = request.getParameter("sort"); String sortOrder = request.getParameter("sortOrder"); // for now, we delete pit by default. TODO: let frontend decide when to keep it - // by passing in the previous pit (e.g. for pagination) + // by passing in the previous pit (e.g. for pagination) // boolean deletePit = Util.requestParameterSet(request.getParameter("deletePit")); boolean deletePit = true; int numFrom = 0; @@ -77,7 +77,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) indexName = query.optString("indexName", null); query = OpenSearch.queryScrubStored(query); } - query = OpenSearch.querySanitize(query, currentUser); + query = OpenSearch.querySanitize(query, currentUser, myShepherd); System.out.println("SearchApi (sanitized) indexName=" + indexName + "; query=" + query); diff --git a/src/main/java/org/ecocean/security/Collaboration.java b/src/main/java/org/ecocean/security/Collaboration.java index f65324bf8a..3d238b022b 100644 --- a/src/main/java/org/ecocean/security/Collaboration.java +++ b/src/main/java/org/ecocean/security/Collaboration.java @@ -66,6 +66,13 @@ public void setUsername1(String name) { this.setId(); } + public String getOtherUsername(String name) { + if (name == null) return null; + if (name.equals(username1)) return username2; + if (name.equals(username2)) return username1; + return null; + } + public String getUsername2() { return this.username2; } @@ -109,6 +116,10 @@ public boolean isApproved() { return (this.state != null && this.state.equals(STATE_APPROVED)); } + public boolean isEditApproved() { + return STATE_EDIT_PRIV.equals(this.state); + } + public String getState() { return this.state; } @@ -129,6 +140,7 @@ public void setId() { // NOTE the first user, by convention, is the initiator public static Collaboration create(String u1, String u2) { Collaboration c = new Collaboration(u1, u2); + return c; } @@ -274,7 +286,7 @@ public static boolean canCollaborate(User u1, User u2, String context) { } public static boolean canCollaborate(String context, String u1, String u2) { - if (User.isUsernameAnonymous(u1) || User.isUsernameAnonymous(u2)) return true; + if (User.isUsernameAnonymous(u1) || User.isUsernameAnonymous(u2)) return true; if (u1.equals(u2)) return true; Collaboration c = collaborationBetweenUsers(u1, u2, context); // System.out.println("canCollaborate(String context, String u1, String u2)"); @@ -377,7 +389,7 @@ public static boolean securityEnabled(String context) { // "View" means "you can see that the data exists but may not necessarily access the data" public static boolean canUserViewOwnedObject(String ownerName, HttpServletRequest request, Shepherd myShepherd) { - if (request.isUserInRole("admin")) return true; + if (request.isUserInRole("admin")) return true; if (ownerName == null || request.isUserInRole("admin")) return true; User viewer = myShepherd.getUser(request); User owner = myShepherd.getUser(ownerName); @@ -400,7 +412,7 @@ public static boolean canUserAccessOwnedObject(String ownerName, HttpServletRequ String context = ServletUtilities.getContext(request); if (!securityEnabled(context)) return true; - if (request.isUserInRole("admin")) return true; + if (request.isUserInRole("admin")) return true; if (User.isUsernameAnonymous(ownerName)) return true; // anon-owned is "fair game" to anyone if (request.getUserPrincipal() == null) { return canCollaborate(context, ownerName, "public"); @@ -463,7 +475,7 @@ public static boolean canUserAccessMarkedIndividual(MarkedIndividual mi, // Check if User (via request) has edit access to every Encounter in this Individual public static boolean canUserFullyEditMarkedIndividual(MarkedIndividual mi, HttpServletRequest request) { - if (request.isUserInRole("admin")) return true; + if (request.isUserInRole("admin")) return true; Vector all = mi.getEncounters(); if ((all == null) || (all.size() < 1)) return false; for (Encounter enc : all) { diff --git a/src/main/java/org/ecocean/servlet/Collaborate.java b/src/main/java/org/ecocean/servlet/Collaborate.java index 8dd4557aa5..d3d5c5a141 100644 --- a/src/main/java/org/ecocean/servlet/Collaborate.java +++ b/src/main/java/org/ecocean/servlet/Collaborate.java @@ -253,6 +253,7 @@ else if ((approve != null) && !approve.equals("")) { System.out.println("/Collaborate: new .getState() = " + collab.getState() + " for collab " + collab); rtn.put("success", true); + OpenSearch.setPermissionsNeeded(myShepherd, true); myShepherd.updateDBTransaction(); // myShepherd.commitDBTransaction(); } diff --git a/src/main/java/org/ecocean/servlet/EncounterForm.java b/src/main/java/org/ecocean/servlet/EncounterForm.java index 6dfb53f4b2..8857a84c19 100644 --- a/src/main/java/org/ecocean/servlet/EncounterForm.java +++ b/src/main/java/org/ecocean/servlet/EncounterForm.java @@ -37,6 +37,7 @@ import org.ecocean.Measurement; import org.ecocean.NotificationMailer; import org.ecocean.Occurrence; +import org.ecocean.OpenSearch; import org.ecocean.Project; import org.ecocean.Shepherd; import org.ecocean.ShepherdProperties; @@ -104,7 +105,7 @@ private AcousticTag getAcousticTag(Map formValues) { private List getMetalTags(Map formValues) { List list = new ArrayList(); - List keys = Arrays.asList("left", "right"); + List keys = Arrays.asList("left", "right"); for (String key : keys) { // The keys are the location @@ -131,8 +132,7 @@ private List getMeasurements(Map formValues, String encID, String c try { Double doubleVal = Double.valueOf(value); list.add(new Measurement(encID, key, doubleVal, units, samplingProtocol)); - } catch (Exception ex) { - } + } catch (Exception ex) {} } } return list; @@ -188,7 +188,7 @@ private List getMeasurements(Map formValues, String encID, String c if (item.isFormField()) { // plain field formValues.put(item.getFieldName(), ServletUtilities.preventCrossSiteScriptingAttacks(item.getString( - "UTF-8").trim())); + "UTF-8").trim())); if (item.getFieldName().equals("defaultProject")) { if (!projectIdSelection.contains(item.getString().trim())) { projectIdSelection.add(item.getString().trim()); @@ -262,8 +262,7 @@ private List getMeasurements(Map formValues, String encID, String c if (badmsg.equals("")) { badmsg = "none"; } session.setAttribute("filesBadMessage", badmsg); if (fileSuccess) { - - // check for spamBots + // check for spamBots boolean spamBot = false; String[] spamFieldsToCheck = new String[] { "submitterPhone", "submitterName", "photographerName", "" + "Phone", "location", @@ -763,7 +762,6 @@ else if (formValues.get("location") != null) { if ((formValues.get("lat") != null) && (formValues.get("longitude") != null) && !formValues.get("lat").toString().equals("") && !formValues.get("longitude").toString().equals("")) { - try { double degrees = (new Double(formValues.get("lat").toString())).doubleValue(); double position = degrees; @@ -780,7 +778,6 @@ else if (formValues.get("location") != null) { e.printStackTrace(); } } - enc.addComments("

Submitted on " + (new java.util.Date()).toString() + " from address: " + ServletUtilities.getRemoteHost(request) + "

"); // enc.approved = false; @@ -897,7 +894,7 @@ else if (formValues.get("location") != null) { parentTask.setParameters(tp); } Task task = org.ecocean.ia.IA.intakeMediaAssets(myShepherd, enc.getMedia(), - parentTask); + parentTask); myShepherd.storeNewTask(task); Logger log = LoggerFactory.getLogger(EncounterForm.class); log.info("New encounter submission:
Changed location code from " + oldCode + " to " + request.getParameter("code") + ".

"); // update numberLocations on a dependent MarkedIndividual too +/* if (changeMe.getIndividual() != null) { MarkedIndividual indy = changeMe.getIndividual(); indy.refreshDependentProperties(); } + */ } catch (Exception le) { locked = true; le.printStackTrace(); diff --git a/src/main/java/org/ecocean/servlet/UserCreate.java b/src/main/java/org/ecocean/servlet/UserCreate.java index aa2bfc73c7..d2b387612f 100644 --- a/src/main/java/org/ecocean/servlet/UserCreate.java +++ b/src/main/java/org/ecocean/servlet/UserCreate.java @@ -113,6 +113,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) newUser.setSalt(salt); } } + OpenSearch.setPermissionsNeeded(myShepherd, true); // here handle all of the other User fields (e.g., email address, etc.) if ((request.getParameter("username") != null) && (!request.getParameter("username").trim().equals(""))) { diff --git a/src/main/java/org/ecocean/servlet/importer/StandardImport.java b/src/main/java/org/ecocean/servlet/importer/StandardImport.java index d2338cdd97..ae18a23afe 100644 --- a/src/main/java/org/ecocean/servlet/importer/StandardImport.java +++ b/src/main/java/org/ecocean/servlet/importer/StandardImport.java @@ -442,6 +442,7 @@ public void doImport(String filename, File dataFile, HttpServletRequest request, if (itask != null) sendforACMID(itask, myShepherd, context); // let's finish up and be done if (itask != null) itask.setStatus("complete"); + OpenSearch.setPermissionsNeeded(myShepherd, true); myShepherd.commitDBTransaction(); myShepherd.closeDBTransaction(); if (itask != null) diff --git a/src/main/resources/bundles/OpenSearch.properties b/src/main/resources/bundles/OpenSearch.properties index 27beb9dc8d..6e10c4e8db 100644 --- a/src/main/resources/bundles/OpenSearch.properties +++ b/src/main/resources/bundles/OpenSearch.properties @@ -2,6 +2,12 @@ #backgroundDelayMinutes=20 #backgroundSliceSize=2500 +# how often to check to see if permissions *needs* to run +#backgroundPermissionsMinutes=10 +# how often it is *forced* to run +#backgroundPermissionsMaxForceMinutes=45 + + # these probably should not be adjusted #searchScrollTime=10m #searchPitTime=10m diff --git a/src/main/resources/org/ecocean/package.jdo b/src/main/resources/org/ecocean/package.jdo index 3c6370195e..8c494ece85 100755 --- a/src/main/resources/org/ecocean/package.jdo +++ b/src/main/resources/org/ecocean/package.jdo @@ -434,6 +434,8 @@ + + diff --git a/src/main/webapp/appadmin/opensearchInfo.jsp b/src/main/webapp/appadmin/opensearchInfo.jsp index 0c2e5d57cd..8e90b593c7 100644 --- a/src/main/webapp/appadmin/opensearchInfo.jsp +++ b/src/main/webapp/appadmin/opensearchInfo.jsp @@ -30,6 +30,11 @@ out.println("

SEARCH_SCROLL_TIME=" + os.SEARCH_SCROLL_TIME + "
"); out.println("SEARCH_PIT_TIME=" + os.SEARCH_PIT_TIME + "
"); out.println("BACKGROUND_DELAY_MINUTES=" + os.BACKGROUND_DELAY_MINUTES + "
"); out.println("BACKGROUND_SLICE_SIZE=" + os.BACKGROUND_SLICE_SIZE + "

"); +out.println("BACKGROUND_PERMISSIONS_MINUTES=" + os.BACKGROUND_PERMISSIONS_MINUTES + "
"); +out.println("BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES=" + os.BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES + "

"); + +out.println("

active indexing: foreground=" + String.valueOf(os.indexingActiveForeground())); +out.println(" / background=" + String.valueOf(os.indexingActiveBackground()) + "

"); Request req = new Request("GET", "_cat/indices?v"); //req.setJsonEntity(query.toString()); diff --git a/src/main/webapp/appadmin/opensearchSync.jsp b/src/main/webapp/appadmin/opensearchSync.jsp index a1413d87d2..f26fe99e48 100644 --- a/src/main/webapp/appadmin/opensearchSync.jsp +++ b/src/main/webapp/appadmin/opensearchSync.jsp @@ -8,20 +8,31 @@ org.ecocean.* <% System.out.println("opensearchSync.jsp begun..."); +long timer = System.currentTimeMillis(); +int numProcessed = -1; +Util.mark("opensearchSync begin"); boolean resetIndex = Util.requestParameterSet(request.getParameter("resetIndex")); -String fstr = request.getParameter("forceNum"); -int forceNum = -1; +String fstr = request.getParameter("endNum"); +int endNum = -1; if ("".equals(fstr)) { - forceNum = 500; + endNum = 500; } else if (fstr != null) { try { - forceNum = Integer.parseInt(fstr); + endNum = Integer.parseInt(fstr); } catch (Exception ex) {} } -if (forceNum == 0) forceNum = 999999; +if (endNum == 0) endNum = 999999; + +String sstr = request.getParameter("startNum"); +int startNum = -1; +if (sstr != null) { + try { + startNum = Integer.parseInt(sstr); + } catch (Exception ex) {} +} Shepherd myShepherd = new Shepherd(request); OpenSearch os = new OpenSearch(); @@ -48,18 +59,32 @@ if (!os.existsIndex("encounter")) { } -if (forceNum > 0) { - out.println("

indexing " + forceNum + " Encounters

"); +if (endNum > 0) { + if (startNum > 0) { + out.println("

indexing " + startNum + "-" + endNum + " Encounters

"); + } else { + out.println("

indexing through " + endNum + " Encounters

"); + } int ct = 0; Iterator itr = myShepherd.getAllEncounters("catalogNumber"); while (itr.hasNext()) { Encounter enc = (Encounter)itr.next(); if (!Util.stringExists(enc.getId())) continue; + ct++; + if (startNum > 0) { + if (ct < startNum) continue; + if (ct == startNum) System.out.println("opensearchSync.jsp: starting at " + startNum); + } //System.out.println(enc.getId() + ": " + enc.getVersion()); - enc.opensearchIndex(); + try { + enc.opensearchIndex(); + numProcessed++; + } catch (Exception ex) { + System.out.println("opensearchSync.jsp: exception failure on " + enc); + ex.printStackTrace(); + } if (ct % 100 == 0) System.out.println("opensearchSync.jsp: count " + ct); - ct++; - if (ct > forceNum) break; + if (ct > endNum) break; } } else { @@ -72,6 +97,14 @@ myShepherd.rollbackAndClose(); OpenSearch.unsetActiveIndexingForeground(); os.deleteAllPits(); -System.out.println("opensearchSync.jsp finished"); + +double totalMin = System.currentTimeMillis() - timer; +totalMin = totalMin / 60000D; +if (numProcessed > 0) { + System.out.println("opensearchSync.jsp finished: " + numProcessed + " in " + String.format("%.2f", totalMin) + " min (" + String.format("%.2f", numProcessed / totalMin) + " per min)"); +} else { + System.out.println("opensearchSync.jsp finished: " + totalMin + " min"); +} +Util.mark("opensearchSync ended", timer); %>