From 1661acdcae974a3e5834a4e279d23900e157c1c7 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 11 Dec 2024 15:11:46 -0700 Subject: [PATCH 01/26] publicness on encounter/index; catch index-busting exception --- src/main/java/org/ecocean/Encounter.java | 25 +++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index d69911f42c..bfe3715262 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -154,7 +154,6 @@ public void setSampleTakenForDiet(Boolean sampleTakenForDiet) { private static HashMap > _matchEncounterCache = new HashMap >(); - // An URL to a thumbnail image representing the encounter. private String dwcImageURL; @@ -678,13 +677,12 @@ public boolean hasRightSpotImage() { return (this.getNumRightSpots() > 0); } - // Sets the recorded length of the shark for this encounter. public void setSize(Double mysize) { if (mysize != null) { size = mysize; } else { size = null; } } - // @return the length of the shark + // @return the length of the shark public double getSize() { return size.doubleValue(); } @@ -2450,9 +2448,10 @@ public void setTissueSamples(List samps) { public Set getTissueSampleIDs() { Set ids = new HashSet(); - if (tissueSamples != null) for (TissueSample ts : tissueSamples) { - ids.add(ts.getSampleID()); - } + if (tissueSamples != null) + for (TissueSample ts : tissueSamples) { + ids.add(ts.getSampleID()); + } return ids; } @@ -3852,6 +3851,11 @@ public int hashCode() { // we need this along with equals() for collections meth return this.getCatalogNumber().hashCode(); } + // "true" public (not null submitterID) + public boolean isPublic() { + return "public".equals(this.submitterID); + } + public static org.json.JSONObject opensearchQuery(final org.json.JSONObject query, int numFrom, int pageSize, String sort, String sortOrder) throws IOException { @@ -3885,6 +3889,7 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeStringField("state", this.getState()); jgen.writeStringField("occurrenceRemarks", this.getOccurrenceRemarks()); jgen.writeStringField("otherCatalogNumbers", this.getOtherCatalogNumbers()); + jgen.writeBooleanField("public", this.isPublic()); String featuredAssetId = null; List mas = this.getMedia(); @@ -3895,8 +3900,11 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeStartObject(); jgen.writeNumberField("id", ma.getId()); jgen.writeStringField("uuid", ma.getUUID()); - java.net.URL url = ma.safeURL(myShepherd); - if (url != null) jgen.writeStringField("url", url.toString()); + try { + // historic data might throw IllegalArgumentException: Path not under given root + java.net.URL url = ma.safeURL(myShepherd); + if (url != null) jgen.writeStringField("url", url.toString()); + } catch (Exception ex) {} jgen.writeEndObject(); if (featuredAssetId == null) featuredAssetId = ma.getUUID(); } @@ -4052,7 +4060,6 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) encDate = Util.getISO8601Date(encs[encs.length - 1].getDate()); if (encDate != null) jgen.writeStringField("individualLastEncounterDate", encDate); } - jgen.writeArrayFieldStart("individualSocialUnits"); for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(indiv)) { Membership mem = su.getMembershipForMarkedIndividual(indiv); From d409db2343c653445e588620eb90e43c7902c0b1 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 11 Dec 2024 17:01:03 -0700 Subject: [PATCH 02/26] take 1 of N for offloading permissions indexing --- src/main/java/org/ecocean/Base.java | 7 +++ src/main/java/org/ecocean/Encounter.java | 57 +++++++++++++++++-- .../org/ecocean/security/Collaboration.java | 20 +++++-- 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index 3af6492495..d79e7de504 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -94,6 +94,8 @@ public JSONObject opensearchMapping() { map.put("version", new org.json.JSONObject("{\"type\": \"long\"}")); // id should be keyword for the sake of sorting map.put("id", new org.json.JSONObject("{\"type\": \"keyword\"}")); + map.put("viewUsers", new org.json.JSONObject("{\"type\": \"keyword\"}")); + map.put("editUsers", new org.json.JSONObject("{\"type\": \"keyword\"}")); return map; } @@ -144,6 +146,10 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeStringField("id", this.getId()); jgen.writeNumberField("version", this.getVersion()); +/* + these are no longer computed in the general opensearchIndex() call. + they are too expensive. see Encounter.opensearchIndexPermission() + jgen.writeFieldName("viewUsers"); jgen.writeStartArray(); for (String id : this.userIdsWithViewAccess(myShepherd)) { @@ -157,6 +163,7 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeString(id); } jgen.writeEndArray(); + */ myShepherd.rollbackDBTransaction(); myShepherd.closeDBTransaction(); } diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index bfe3715262..6f9c7bf711 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3142,7 +3142,7 @@ public boolean isUserOwner(User user) { // the definition of this might change? @Override public List userIdsWithViewAccess(Shepherd myShepherd) { List ids = new ArrayList(); - for (User user : myShepherd.getAllUsers()) { + for (User user : myShepherd.getUsersWithUsername()) { if ((user.getId() != null) && this.canUserView(user, myShepherd)) ids.add(user.getId()); } @@ -3152,7 +3152,7 @@ public boolean isUserOwner(User user) { // the definition of this might change? @Override public List userIdsWithEditAccess(Shepherd myShepherd) { List ids = new ArrayList(); - for (User user : myShepherd.getAllUsers()) { + for (User user : myShepherd.getUsersWithUsername()) { if ((user.getId() != null) && this.canUserEdit(user)) ids.add(user.getId()); } return ids; @@ -3851,9 +3851,54 @@ public int hashCode() { // we need this along with equals() for collections meth return this.getCatalogNumber().hashCode(); } - // "true" public (not null submitterID) - public boolean isPublic() { - return "public".equals(this.submitterID); + // sadly, this mess needs to carry on the tradition set up in User.isUsernameAnonymous() + // thanks to the logic in Collaboration.canUserAccessOwnedObject() + public boolean isPubliclyReadable() { + if (!Collaboration.securityEnabled("context0")) return true; + return User.isUsernameAnonymous(this.submitterID); + } + +/* note: there are a great deal of users with *no username* that seem to appear in enc.submitters array. + however, very few (2 out of 5600+) encounters with such .submitters have a blank submitterID value + therefore: submitterID will be assumed to be a required value on users which need to be + + this seems further validated by the facts that: + - canUserAccess(user) returns false if no username on user + - a user wihtout a username cant be logged in (and thus cant search) + + "admin" users are just ignored entirely, as they will be exempt from the viewUsers criteria during searching. + + other than "ownership" (via submitterID), a user can view if they have view or edit collab with + another user. so we frontload *approved* collabs for every user here too. + + in terms of "public" encounters, it seems that (based on Collaboration.canUserAccessEncounter()), + encounters with submitterID in (NULL, "public", "", "N/A" [ugh]) is readable by anyone; so we will + skip these from processing as they should be flagged with the boolean isPubliclyReadable in indexing + */ + public static void opensearchIndexPermissions() { + // no security => everything publiclyReadable - saves us work, no? + if (!Collaboration.securityEnabled("context0")) return; + Map > collab = new HashMap >(); + Map usernameToId = new HashMap(); + Shepherd myShepherd = new Shepherd("context0"); + myShepherd.setAction("Encounter.opensearchIndexPermissions"); + myShepherd.beginDBTransaction(); + // it seems as though user.uuid is *required* so we can trust that + for (User user : myShepherd.getUsersWithUsername()) { + usernameToId.put(user.getUsername(), user.getId()); + if (user.isAdmin(myShepherd)) continue; + List collabsFor = Collaboration.collaborationsForUser(myShepherd, + user.getUsername()); + if (Util.collectionIsEmptyOrNull(collabsFor)) continue; + for (Collaboration col : collabsFor) { + if (!col.isApproved() && !col.isEditApproved()) continue; + if (!collab.containsKey(user.getId())) + collab.put(user.getId(), new HashSet()); + collab.get(user.getId()).add(col.getOtherUsername(user.getUsername())); + } + } + // now iterated over NECESSARY encounters + myShepherd.rollbackAndClose(); } public static org.json.JSONObject opensearchQuery(final org.json.JSONObject query, int numFrom, @@ -3889,7 +3934,7 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeStringField("state", this.getState()); jgen.writeStringField("occurrenceRemarks", this.getOccurrenceRemarks()); jgen.writeStringField("otherCatalogNumbers", this.getOtherCatalogNumbers()); - jgen.writeBooleanField("public", this.isPublic()); + jgen.writeBooleanField("publiclyReadable", this.isPubliclyReadable()); String featuredAssetId = null; List mas = this.getMedia(); diff --git a/src/main/java/org/ecocean/security/Collaboration.java b/src/main/java/org/ecocean/security/Collaboration.java index f65324bf8a..3d238b022b 100644 --- a/src/main/java/org/ecocean/security/Collaboration.java +++ b/src/main/java/org/ecocean/security/Collaboration.java @@ -66,6 +66,13 @@ public void setUsername1(String name) { this.setId(); } + public String getOtherUsername(String name) { + if (name == null) return null; + if (name.equals(username1)) return username2; + if (name.equals(username2)) return username1; + return null; + } + public String getUsername2() { return this.username2; } @@ -109,6 +116,10 @@ public boolean isApproved() { return (this.state != null && this.state.equals(STATE_APPROVED)); } + public boolean isEditApproved() { + return STATE_EDIT_PRIV.equals(this.state); + } + public String getState() { return this.state; } @@ -129,6 +140,7 @@ public void setId() { // NOTE the first user, by convention, is the initiator public static Collaboration create(String u1, String u2) { Collaboration c = new Collaboration(u1, u2); + return c; } @@ -274,7 +286,7 @@ public static boolean canCollaborate(User u1, User u2, String context) { } public static boolean canCollaborate(String context, String u1, String u2) { - if (User.isUsernameAnonymous(u1) || User.isUsernameAnonymous(u2)) return true; + if (User.isUsernameAnonymous(u1) || User.isUsernameAnonymous(u2)) return true; if (u1.equals(u2)) return true; Collaboration c = collaborationBetweenUsers(u1, u2, context); // System.out.println("canCollaborate(String context, String u1, String u2)"); @@ -377,7 +389,7 @@ public static boolean securityEnabled(String context) { // "View" means "you can see that the data exists but may not necessarily access the data" public static boolean canUserViewOwnedObject(String ownerName, HttpServletRequest request, Shepherd myShepherd) { - if (request.isUserInRole("admin")) return true; + if (request.isUserInRole("admin")) return true; if (ownerName == null || request.isUserInRole("admin")) return true; User viewer = myShepherd.getUser(request); User owner = myShepherd.getUser(ownerName); @@ -400,7 +412,7 @@ public static boolean canUserAccessOwnedObject(String ownerName, HttpServletRequ String context = ServletUtilities.getContext(request); if (!securityEnabled(context)) return true; - if (request.isUserInRole("admin")) return true; + if (request.isUserInRole("admin")) return true; if (User.isUsernameAnonymous(ownerName)) return true; // anon-owned is "fair game" to anyone if (request.getUserPrincipal() == null) { return canCollaborate(context, ownerName, "public"); @@ -463,7 +475,7 @@ public static boolean canUserAccessMarkedIndividual(MarkedIndividual mi, // Check if User (via request) has edit access to every Encounter in this Individual public static boolean canUserFullyEditMarkedIndividual(MarkedIndividual mi, HttpServletRequest request) { - if (request.isUserInRole("admin")) return true; + if (request.isUserInRole("admin")) return true; Vector all = mi.getEncounters(); if ((all == null) || (all.size() < 1)) return false; for (Encounter enc : all) { From 238059ef1f9a74b089d417799dd491b8e9516243 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 11 Dec 2024 22:19:41 -0700 Subject: [PATCH 03/26] take 2: iterate over encounters --- src/main/java/org/ecocean/Encounter.java | 40 +++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 6f9c7bf711..4391c75198 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -15,6 +15,7 @@ import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; @@ -3876,6 +3877,8 @@ however, very few (2 out of 5600+) encounters with such .submitters have a blank skip these from processing as they should be flagged with the boolean isPubliclyReadable in indexing */ public static void opensearchIndexPermissions() { +// Util.mark("perm start"); long t = System.currentTimeMillis(); + System.out.println("opensearchIndexPermissions(): begin..."); // no security => everything publiclyReadable - saves us work, no? if (!Collaboration.securityEnabled("context0")) return; Map > collab = new HashMap >(); @@ -3883,10 +3886,12 @@ public static void opensearchIndexPermissions() { Shepherd myShepherd = new Shepherd("context0"); myShepherd.setAction("Encounter.opensearchIndexPermissions"); myShepherd.beginDBTransaction(); + int nonAdminCt = 0; // it seems as though user.uuid is *required* so we can trust that for (User user : myShepherd.getUsersWithUsername()) { usernameToId.put(user.getUsername(), user.getId()); if (user.isAdmin(myShepherd)) continue; + nonAdminCt++; List collabsFor = Collaboration.collaborationsForUser(myShepherd, user.getUsername()); if (Util.collectionIsEmptyOrNull(collabsFor)) continue; @@ -3897,8 +3902,41 @@ public static void opensearchIndexPermissions() { collab.get(user.getId()).add(col.getOtherUsername(user.getUsername())); } } - // now iterated over NECESSARY encounters +// Util.mark("perm: user build done", t); + System.out.println("opensearchIndexPermissions(): " + usernameToId.size() + + " total users; " + nonAdminCt + " non-admin; " + collab.size() + " have active collab"); + // now iterated over (non-public) encounters + int encCount = 0; + Query query = myShepherd.getPM().newQuery( + "SELECT FROM org.ecocean.Encounter WHERE (submitterID != null) && (submitterID != '') && (submitterID != 'N/A') && (submitterID != 'public')"); + Iterator it = myShepherd.getAllEncounters(query); +// Util.mark("perm: start encs", t); + while (it.hasNext()) { + Set viewers = new HashSet(); + Encounter enc = (Encounter)it.next(); + String uid = usernameToId.get(enc.getSubmitterID()); + if (uid == null) { + // see issue 939 for example :( + System.out.println("opensearchIndexPermissions(): WARNING invalid username " + + enc.getSubmitterID() + " on enc " + enc.getId()); + continue; + } + encCount++; + viewers.add(uid); + if (!collab.containsKey(uid)) continue; + for (String colUsername : collab.get(uid)) { + String colId = usernameToId.get(colUsername); + if (colId == null) { + System.out.println("opensearchIndexPermissions(): WARNING invalid username " + + colUsername + " in collaboration with userId=" + uid); + continue; + } + viewers.add(colId); + } + } +// Util.mark("perm: done encs", t); myShepherd.rollbackAndClose(); + System.out.println("opensearchIndexPermissions(): ...end [" + encCount + " encs]"); } public static org.json.JSONObject opensearchQuery(final org.json.JSONObject query, int numFrom, From 03783619627d10a2b25e89042c4a3f36a1a52a8a Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 11 Dec 2024 22:44:36 -0700 Subject: [PATCH 04/26] base.opensearchUpdate() and friends --- src/main/java/org/ecocean/Base.java | 8 ++++++++ src/main/java/org/ecocean/OpenSearch.java | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index d79e7de504..90958ad970 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -136,6 +136,14 @@ public void opensearchUnindexDeep() this.opensearchUnindex(); } + public void opensearchUpdate(final JSONObject updateData) + throws IOException { + if (updateData == null) return; + OpenSearch opensearch = new OpenSearch(); + + opensearch.indexUpdate(this.opensearchIndexName(), this.getId(), updateData); + } + // should be overridden public void opensearchDocumentSerializer(JsonGenerator jgen) throws IOException, JsonProcessingException { diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 64b3c78be7..0b1c998873 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -466,6 +466,17 @@ public void indexClose(final String indexName) System.out.println("OpenSearch.indexClose() on " + indexName + ": " + rtn); } + // updateData is { field0: value0, field1: value1, ... } + public void indexUpdate(final String indexName, String id, JSONObject updateData) + throws IOException { + if ((id == null) || (updateData == null)) throw new IOException("missing id or updateData"); + JSONObject doc = new JSONObject(); + doc.put("doc", updateData); + Request updateRequest = new Request("POST", indexName + "/_update/" + id); + updateRequest.setJsonEntity(doc.toString()); + getRestResponse(updateRequest); + } + // returns 2 lists: (1) items needing (re-)indexing; (2) items needing removal public static List > resolveVersions(Map objVersions, Map indexVersions) { From c038cc99634fed8d7bc0401535e9f834e26b66d3 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 11 Dec 2024 23:09:23 -0700 Subject: [PATCH 05/26] actually update index documents woohoo --- src/main/java/org/ecocean/Encounter.java | 34 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 4391c75198..4fc0858f01 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3877,7 +3877,8 @@ however, very few (2 out of 5600+) encounters with such .submitters have a blank skip these from processing as they should be flagged with the boolean isPubliclyReadable in indexing */ public static void opensearchIndexPermissions() { -// Util.mark("perm start"); long t = System.currentTimeMillis(); + Util.mark("perm start"); + long t = System.currentTimeMillis(); System.out.println("opensearchIndexPermissions(): begin..."); // no security => everything publiclyReadable - saves us work, no? if (!Collaboration.securityEnabled("context0")) return; @@ -3907,12 +3908,13 @@ public static void opensearchIndexPermissions() { " total users; " + nonAdminCt + " non-admin; " + collab.size() + " have active collab"); // now iterated over (non-public) encounters int encCount = 0; + org.json.JSONObject updateData = new org.json.JSONObject(); Query query = myShepherd.getPM().newQuery( "SELECT FROM org.ecocean.Encounter WHERE (submitterID != null) && (submitterID != '') && (submitterID != 'N/A') && (submitterID != 'public')"); Iterator it = myShepherd.getAllEncounters(query); // Util.mark("perm: start encs", t); while (it.hasNext()) { - Set viewers = new HashSet(); + org.json.JSONArray viewUsers = new org.json.JSONArray(); Encounter enc = (Encounter)it.next(); String uid = usernameToId.get(enc.getSubmitterID()); if (uid == null) { @@ -3922,16 +3924,26 @@ public static void opensearchIndexPermissions() { continue; } encCount++; - viewers.add(uid); - if (!collab.containsKey(uid)) continue; - for (String colUsername : collab.get(uid)) { - String colId = usernameToId.get(colUsername); - if (colId == null) { - System.out.println("opensearchIndexPermissions(): WARNING invalid username " + - colUsername + " in collaboration with userId=" + uid); - continue; + if (encCount % 1000 == 0) Util.mark("enc[" + encCount + "]", t); + viewUsers.put(uid); + if (collab.containsKey(uid)) { + for (String colUsername : collab.get(uid)) { + String colId = usernameToId.get(colUsername); + if (colId == null) { + System.out.println( + "opensearchIndexPermissions(): WARNING invalid username " + + colUsername + " in collaboration with userId=" + uid); + continue; + } + viewUsers.put(colId); } - viewers.add(colId); + } + updateData.put("viewUsers", viewUsers); + try { + enc.opensearchUpdate(updateData); + } catch (Exception ex) { + // keeping this quiet cuz it can get noise while index builds + // System.out.println("opensearchIndexPermissions(): WARNING failed to update viewUsers on enc " + enc.getId() + "; likely has not been indexed yet: " + ex); } } // Util.mark("perm: done encs", t); From 4f748182d8fe507aa7e7a2dbe12bb9d256220bcf Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 12 Dec 2024 10:27:03 -0700 Subject: [PATCH 06/26] take1 at improved search permission query --- .../java/org/ecocean/EncounterQueryProcessor.java | 13 ++++--------- src/main/java/org/ecocean/OpenSearch.java | 9 +++++++-- src/main/java/org/ecocean/api/SearchApi.java | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/ecocean/EncounterQueryProcessor.java b/src/main/java/org/ecocean/EncounterQueryProcessor.java index a1f1a2934f..83fec9249f 100644 --- a/src/main/java/org/ecocean/EncounterQueryProcessor.java +++ b/src/main/java/org/ecocean/EncounterQueryProcessor.java @@ -61,7 +61,7 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer String indexName = searchQuery.optString("indexName", null); if (indexName == null) return failed; searchQuery = OpenSearch.queryScrubStored(searchQuery); - JSONObject sanitized = OpenSearch.querySanitize(searchQuery, user); + JSONObject sanitized = OpenSearch.querySanitize(searchQuery, user, myShepherd); OpenSearch os = new OpenSearch(); String sort = request.getParameter("sort"); String sortOrder = request.getParameter("sortOrder"); @@ -134,8 +134,7 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer String variables_statement = " VARIABLES org.ecocean.User user; org.ecocean.Organization org"; jdoqlVariableDeclaration = addOrgVars(variables_statement, filter); - } else { - } + } else {} // end filter for organization------------------ // filter for projectName------------------- if (Util.isUUID(request.getParameter("projectId"))) { @@ -169,8 +168,7 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer } String variables_statement = " VARIABLES org.ecocean.Project proj"; jdoqlVariableDeclaration = addOrgVars(variables_statement, filter); - } else { - } + } else {} // end filter for projectName------------------ // username filters------------------------------------------------- String[] usernames = request.getParameterValues("username"); @@ -1356,7 +1354,6 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer (!request.getParameter("nameField").equals(""))) { String nameString = request.getParameter("nameField").replaceAll("%20", " ").toLowerCase().trim(); - String filterString = "" + "(" + "(submitters.contains(submitter) && ((submitter.fullName.toLowerCase().indexOf('" + nameString + "') != -1)||(submitter.emailAddress.toLowerCase().indexOf('" + @@ -1529,7 +1526,6 @@ public static EncounterQueryResult processQuery(Shepherd myShepherd, HttpServlet String currentUser = null; if (request.getUserPrincipal() != null) currentUser = request.getUserPrincipal().getName(); - String searchQueryId = request.getParameter("searchQueryId"); long startTime = System.currentTimeMillis(); if (searchQueryId != null) { @@ -1546,7 +1542,7 @@ public static EncounterQueryResult processQuery(Shepherd myShepherd, HttpServlet return new EncounterQueryResult(rEncounters, "searchQuery has no indexName", "OpenSearch id " + searchQueryId); searchQuery = OpenSearch.queryScrubStored(searchQuery); - JSONObject sanitized = OpenSearch.querySanitize(searchQuery, user); + JSONObject sanitized = OpenSearch.querySanitize(searchQuery, user, myShepherd); OpenSearch os = new OpenSearch(); String sort = request.getParameter("sort"); String sortOrder = request.getParameter("sortOrder"); @@ -1620,7 +1616,6 @@ public static EncounterQueryResult processQuery(Shepherd myShepherd, HttpServlet rEncounters.add(temp_enc); } } - query.closeAll(); // silo security logging diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 0b1c998873..281a8c204d 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -581,13 +581,18 @@ public Long getIndexTimestamp(Shepherd myShepherd, String indexName) { return SystemValue.getLong(myShepherd, INDEX_TIMESTAMP_PREFIX + indexName); } - public static JSONObject querySanitize(JSONObject query, User user) { + public static JSONObject querySanitize(JSONObject query, User user, Shepherd myShepherd) { if ((query == null) || (user == null)) return query; + // do not add viewUsers query when we are admin, as user has no restriction + if (user.isAdmin(myShepherd)) return query; + JSONObject permClause = new JSONObject( + "{\"bool\": {\"should\": [{\"term\": {\"publiclyReadable\": true}}, {\"term\": {\"viewUsers\": \"" + + user.getId() + "\"}} ] }}"); JSONObject newQuery = new JSONObject(query.toString()); try { JSONArray filter = newQuery.getJSONObject("query").getJSONObject("bool").getJSONArray( "filter"); - filter.put(new JSONObject("{\"match\": {\"viewUsers\": \"" + user.getId() + "\"}}")); + filter.put(permClause); } catch (Exception ex) { System.out.println("OpenSearch.querySanitize() failed to find filter element: " + ex); } diff --git a/src/main/java/org/ecocean/api/SearchApi.java b/src/main/java/org/ecocean/api/SearchApi.java index 4751baa255..42df780548 100644 --- a/src/main/java/org/ecocean/api/SearchApi.java +++ b/src/main/java/org/ecocean/api/SearchApi.java @@ -62,7 +62,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) String sort = request.getParameter("sort"); String sortOrder = request.getParameter("sortOrder"); // for now, we delete pit by default. TODO: let frontend decide when to keep it - // by passing in the previous pit (e.g. for pagination) + // by passing in the previous pit (e.g. for pagination) // boolean deletePit = Util.requestParameterSet(request.getParameter("deletePit")); boolean deletePit = true; int numFrom = 0; @@ -77,7 +77,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) indexName = query.optString("indexName", null); query = OpenSearch.queryScrubStored(query); } - query = OpenSearch.querySanitize(query, currentUser); + query = OpenSearch.querySanitize(query, currentUser, myShepherd); System.out.println("SearchApi (sanitized) indexName=" + indexName + "; query=" + query); From 0b40acf6fc1e385b792ff7ef183b0f0190061675 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 12 Dec 2024 10:45:10 -0700 Subject: [PATCH 07/26] [#779] catch bad lat/lon; catch index exceptions; startNum on opensearchSync --- src/main/java/org/ecocean/Encounter.java | 11 ++++++++-- src/main/webapp/appadmin/opensearchSync.jsp | 24 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 4fc0858f01..da7d7d2d0f 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -4119,7 +4119,8 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) Double dlat = this.getDecimalLatitudeAsDouble(); Double dlon = this.getDecimalLongitudeAsDouble(); - if ((dlat == null) || (dlon == null)) { + if ((dlat == null) || !Util.isValidDecimalLatitude(dlat) || (dlon == null) || + !Util.isValidDecimalLongitude(dlon)) { jgen.writeNullField("locationGeoPoint"); } else { jgen.writeObjectFieldStart("locationGeoPoint"); @@ -4300,7 +4301,13 @@ public static int[] opensearchSyncIndex(Shepherd myShepherd, int stopAfter) int ct = 0; for (String id : needIndexing) { Encounter enc = myShepherd.getEncounter(id); - if (enc != null) os.index(indexName, enc); + try { + if (enc != null) os.index(indexName, enc); + } catch (Exception ex) { + System.out.println("Encounter.opensearchSyncIndex(): index failed " + enc + " => " + + ex.toString()); + ex.printStackTrace(); + } if (ct % 500 == 0) System.out.println("Encounter.opensearchSyncIndex needIndexing: " + ct + "/" + rtn[0]); diff --git a/src/main/webapp/appadmin/opensearchSync.jsp b/src/main/webapp/appadmin/opensearchSync.jsp index b913c7a9a8..ccde192a20 100644 --- a/src/main/webapp/appadmin/opensearchSync.jsp +++ b/src/main/webapp/appadmin/opensearchSync.jsp @@ -8,6 +8,8 @@ org.ecocean.* <% System.out.println("opensearchSync.jsp begun..."); +long timer = System.currentTimeMillis(); +Util.mark("opensearchSync begin"); boolean resetIndex = Util.requestParameterSet(request.getParameter("resetIndex")); @@ -23,6 +25,14 @@ if ("".equals(fstr)) { if (forceNum == 0) forceNum = 999999; +String sstr = request.getParameter("startNum"); +int startNum = -1; +if (sstr != null) { + try { + startNum = Integer.parseInt(sstr); + } catch (Exception ex) {} +} + Shepherd myShepherd = new Shepherd(request); OpenSearch os = new OpenSearch(); @@ -45,10 +55,19 @@ if (forceNum > 0) { while (itr.hasNext()) { Encounter enc = (Encounter)itr.next(); if (!Util.stringExists(enc.getId())) continue; + ct++; + if (startNum > 0) { + if (ct < startNum) continue; + if (ct == startNum) System.out.println("opensearchSync.jsp: starting at " + startNum); + } //System.out.println(enc.getId() + ": " + enc.getVersion()); - enc.opensearchIndex(); + try { + enc.opensearchIndex(); + } catch (Exception ex) { + System.out.println("opensearchSync.jsp: exception failure on " + enc); + ex.printStackTrace(); + } if (ct % 100 == 0) System.out.println("opensearchSync.jsp: count " + ct); - ct++; if (ct > forceNum) break; } @@ -62,5 +81,6 @@ myShepherd.rollbackAndClose(); os.deleteAllPits(); System.out.println("opensearchSync.jsp finished"); +Util.mark("opensearchSync ended", timer); %> From 3f75d2e12d0186873e9b940456def847fb879c09 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 12 Dec 2024 11:44:55 -0700 Subject: [PATCH 08/26] take1 of actual permissions backgroundness --- src/main/java/org/ecocean/Encounter.java | 28 ++++++++ src/main/java/org/ecocean/OpenSearch.java | 79 ++++++++++++++++++----- 2 files changed, 91 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index da7d7d2d0f..e12790a359 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3859,6 +3859,34 @@ public boolean isPubliclyReadable() { return User.isUsernameAnonymous(this.submitterID); } + // wrapper for below, that checks if we really need to be run + public static void opensearchIndexPermissionsBackground(Shepherd myShepherd) { + boolean runIt = false; + Long lastRun = OpenSearch.getPermissionsTimestamp(myShepherd); + long now = System.currentTimeMillis(); + + if ((lastRun == null) || + ((now - lastRun) > OpenSearch.BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES * 60000)) { + System.out.println( + "opensearchIndexPermissionsBackground: forced run due to max time since previous"); + runIt = true; + } + boolean needed = OpenSearch.getPermissionsNeeded(myShepherd); + if (needed && !runIt) { + System.out.println("opensearchIndexPermissionsBackground: running due to needed=true"); + runIt = true; + } + if (!runIt) { + System.out.println("opensearchIndexPermissionsBackground: running not required; done"); + return; + } + // i think we should set these first... tho they may not get persisted til after? + OpenSearch.setPermissionsTimestamp(myShepherd); + OpenSearch.setPermissionsNeeded(myShepherd, false); + opensearchIndexPermissions(); + System.out.println("opensearchIndexPermissionsBackground: running completed"); + } + /* note: there are a great deal of users with *no username* that seem to appear in enc.submitters array. however, very few (2 out of 5600+) encounters with such .submitters have a blank submitterID value therefore: submitterID will be assumed to be a required value on users which need to be diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 281a8c204d..34e16dd51e 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -62,6 +62,10 @@ public class OpenSearch { public static String[] VALID_INDICES = { "encounter", "individual", "occurrence" }; public static int BACKGROUND_DELAY_MINUTES = 20; public static int BACKGROUND_SLICE_SIZE = 2500; + public static int BACKGROUND_PERMISSIONS_MINUTES = 10; // how often it checks if NEED to run + public static int BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES = 45; // how often it forces a run + public static String PERMISSIONS_LAST_RUN_KEY = "OpenSearch_permissions_last_run_timestamp"; + public static String PERMISSIONS_NEEDED_KEY = "OpenSearch_permissions_needed"; public static String QUERY_STORAGE_DIR = "/tmp"; // FIXME private int pitRetry = 0; @@ -130,25 +134,45 @@ public static boolean skipAutoIndexing() { // http://localhost:9200/_cat/indices?v public static void backgroundStartup(String context) { - final ScheduledExecutorService schedExec = Executors.newScheduledThreadPool(2); - final ScheduledFuture schedFuture = schedExec.scheduleWithFixedDelay(new Runnable() { - public void run() { - Shepherd myShepherd = new Shepherd(context); - myShepherd.setAction("OpenSearch.background"); - try { - myShepherd.beginDBTransaction(); - System.out.println("OpenSearch background running..."); - Encounter.opensearchSyncIndex(myShepherd, BACKGROUND_SLICE_SIZE); - System.out.println("OpenSearch background finished."); - myShepherd.rollbackAndClose(); - } catch (Exception ex) { - ex.printStackTrace(); - myShepherd.rollbackAndClose(); + final ScheduledExecutorService schedExec = Executors.newScheduledThreadPool(8); + final ScheduledFuture schedFutureIndexing = schedExec.scheduleWithFixedDelay( + new Runnable() { + public void run() { + Shepherd myShepherd = new Shepherd(context); + myShepherd.setAction("OpenSearch.backgroundIndexing"); + try { + myShepherd.beginDBTransaction(); + System.out.println("OpenSearch background indexing running..."); + Encounter.opensearchSyncIndex(myShepherd, BACKGROUND_SLICE_SIZE); + System.out.println("OpenSearch background indexing finished."); + myShepherd.rollbackAndClose(); + } catch (Exception ex) { + ex.printStackTrace(); + myShepherd.rollbackAndClose(); + } } - } - }, 2, // initial delay + }, 2, // initial delay BACKGROUND_DELAY_MINUTES, // period delay *after* execution finishes TimeUnit.MINUTES); // unit of delays above + final ScheduledFuture schedFuturePermissions = schedExec.scheduleWithFixedDelay( + new Runnable() { + public void run() { + Shepherd myShepherd = new Shepherd(context); + myShepherd.setAction("OpenSearch.backgroundPermissions"); + try { + myShepherd.beginDBTransaction(); + System.out.println("OpenSearch background permissions running..."); + Encounter.opensearchIndexPermissionsBackground(myShepherd); + System.out.println("OpenSearch background permissions finished."); + myShepherd.commitDBTransaction(); // need commit since we might have changed SystemValues + myShepherd.closeDBTransaction(); + } catch (Exception ex) { + ex.printStackTrace(); + myShepherd.rollbackAndClose(); + } + } + }, 8, // initial delay + BACKGROUND_PERMISSIONS_MINUTES, TimeUnit.MINUTES); // unit of delays above try { schedExec.awaitTermination(5000, TimeUnit.MILLISECONDS); @@ -581,6 +605,29 @@ public Long getIndexTimestamp(Shepherd myShepherd, String indexName) { return SystemValue.getLong(myShepherd, INDEX_TIMESTAMP_PREFIX + indexName); } + public static long setPermissionsTimestamp(Shepherd myShepherd) { + long now = System.currentTimeMillis(); + + SystemValue.set(myShepherd, PERMISSIONS_LAST_RUN_KEY, now); + return now; + } + + public static Long getPermissionsTimestamp(Shepherd myShepherd) { + return SystemValue.getLong(myShepherd, PERMISSIONS_LAST_RUN_KEY); + } + + // FIXME: when we have Boolean support in SystemValue, modify these! + public static void setPermissionsNeeded(Shepherd myShepherd, boolean value) { + SystemValue.set(myShepherd, PERMISSIONS_NEEDED_KEY, String.valueOf(value)); + } + + // TODO see above + public static boolean getPermissionsNeeded(Shepherd myShepherd) { + String value = SystemValue.getString(myShepherd, PERMISSIONS_NEEDED_KEY); + + return "true".equals(value); + } + public static JSONObject querySanitize(JSONObject query, User user, Shepherd myShepherd) { if ((query == null) || (user == null)) return query; // do not add viewUsers query when we are admin, as user has no restriction From a8d28d8be80eccc189ace963be0dde8acce850fc Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 12 Dec 2024 18:59:27 -0700 Subject: [PATCH 09/26] setPermissionsNeeded() in a few places to start --- src/main/java/org/ecocean/Encounter.java | 13 +++++++------ src/main/java/org/ecocean/OpenSearch.java | 1 + src/main/java/org/ecocean/api/BaseObject.java | 2 ++ .../java/org/ecocean/servlet/Collaborate.java | 1 + .../java/org/ecocean/servlet/EncounterForm.java | 17 +++++++---------- .../java/org/ecocean/servlet/UserCreate.java | 1 + .../servlet/importer/StandardImport.java | 1 + 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index e12790a359..f62470685b 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3906,7 +3906,7 @@ however, very few (2 out of 5600+) encounters with such .submitters have a blank */ public static void opensearchIndexPermissions() { Util.mark("perm start"); - long t = System.currentTimeMillis(); + long startT = System.currentTimeMillis(); System.out.println("opensearchIndexPermissions(): begin..."); // no security => everything publiclyReadable - saves us work, no? if (!Collaboration.securityEnabled("context0")) return; @@ -3931,7 +3931,7 @@ public static void opensearchIndexPermissions() { collab.get(user.getId()).add(col.getOtherUsername(user.getUsername())); } } -// Util.mark("perm: user build done", t); +// Util.mark("perm: user build done", startT); System.out.println("opensearchIndexPermissions(): " + usernameToId.size() + " total users; " + nonAdminCt + " non-admin; " + collab.size() + " have active collab"); // now iterated over (non-public) encounters @@ -3940,7 +3940,7 @@ public static void opensearchIndexPermissions() { Query query = myShepherd.getPM().newQuery( "SELECT FROM org.ecocean.Encounter WHERE (submitterID != null) && (submitterID != '') && (submitterID != 'N/A') && (submitterID != 'public')"); Iterator it = myShepherd.getAllEncounters(query); -// Util.mark("perm: start encs", t); +// Util.mark("perm: start encs", startT); while (it.hasNext()) { org.json.JSONArray viewUsers = new org.json.JSONArray(); Encounter enc = (Encounter)it.next(); @@ -3952,7 +3952,7 @@ public static void opensearchIndexPermissions() { continue; } encCount++; - if (encCount % 1000 == 0) Util.mark("enc[" + encCount + "]", t); + if (encCount % 1000 == 0) Util.mark("enc[" + encCount + "]", startT); viewUsers.put(uid); if (collab.containsKey(uid)) { for (String colUsername : collab.get(uid)) { @@ -3974,9 +3974,10 @@ public static void opensearchIndexPermissions() { // System.out.println("opensearchIndexPermissions(): WARNING failed to update viewUsers on enc " + enc.getId() + "; likely has not been indexed yet: " + ex); } } -// Util.mark("perm: done encs", t); +// Util.mark("perm: done encs", startT); myShepherd.rollbackAndClose(); - System.out.println("opensearchIndexPermissions(): ...end [" + encCount + " encs]"); + System.out.println("opensearchIndexPermissions(): ...end [" + encCount + " encs; " + + Math.round((System.currentTimeMillis() - startT) / 1000) + "sec]"); } public static org.json.JSONObject opensearchQuery(final org.json.JSONObject query, int numFrom, diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 34e16dd51e..6c6fd1b05b 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -632,6 +632,7 @@ public static JSONObject querySanitize(JSONObject query, User user, Shepherd myS if ((query == null) || (user == null)) return query; // do not add viewUsers query when we are admin, as user has no restriction if (user.isAdmin(myShepherd)) return query; + // if (!Collaboration.securityEnabled("context0")) TODO do we want to allow everything searchable? JSONObject permClause = new JSONObject( "{\"bool\": {\"should\": [{\"term\": {\"publiclyReadable\": true}}, {\"term\": {\"viewUsers\": \"" + user.getId() + "\"}} ] }}"); diff --git a/src/main/java/org/ecocean/api/BaseObject.java b/src/main/java/org/ecocean/api/BaseObject.java index 30cd991822..8e1abbff28 100644 --- a/src/main/java/org/ecocean/api/BaseObject.java +++ b/src/main/java/org/ecocean/api/BaseObject.java @@ -24,6 +24,7 @@ import org.ecocean.media.MediaAssetFactory; import org.ecocean.MarkedIndividual; import org.ecocean.Occurrence; +import org.ecocean.OpenSearch; import org.ecocean.Project; import org.ecocean.resumableupload.UploadServlet; import org.ecocean.servlet.importer.ImportTask; @@ -175,6 +176,7 @@ protected JSONObject processPost(HttpServletRequest request, String[] args, JSON if ((obj != null) && (rtn.optInt("statusCode", 0) == 200)) { System.out.println("BaseObject.processPost() success (200) creating " + obj + " from payload " + payload); + OpenSearch.setPermissionsNeeded(myShepherd, true); myShepherd.commitDBTransaction(); MediaAsset.updateStandardChildrenBackground(context, maIds); if (encounterForIA != null) { diff --git a/src/main/java/org/ecocean/servlet/Collaborate.java b/src/main/java/org/ecocean/servlet/Collaborate.java index 8dd4557aa5..d3d5c5a141 100644 --- a/src/main/java/org/ecocean/servlet/Collaborate.java +++ b/src/main/java/org/ecocean/servlet/Collaborate.java @@ -253,6 +253,7 @@ else if ((approve != null) && !approve.equals("")) { System.out.println("/Collaborate: new .getState() = " + collab.getState() + " for collab " + collab); rtn.put("success", true); + OpenSearch.setPermissionsNeeded(myShepherd, true); myShepherd.updateDBTransaction(); // myShepherd.commitDBTransaction(); } diff --git a/src/main/java/org/ecocean/servlet/EncounterForm.java b/src/main/java/org/ecocean/servlet/EncounterForm.java index 6dfb53f4b2..8857a84c19 100644 --- a/src/main/java/org/ecocean/servlet/EncounterForm.java +++ b/src/main/java/org/ecocean/servlet/EncounterForm.java @@ -37,6 +37,7 @@ import org.ecocean.Measurement; import org.ecocean.NotificationMailer; import org.ecocean.Occurrence; +import org.ecocean.OpenSearch; import org.ecocean.Project; import org.ecocean.Shepherd; import org.ecocean.ShepherdProperties; @@ -104,7 +105,7 @@ private AcousticTag getAcousticTag(Map formValues) { private List getMetalTags(Map formValues) { List list = new ArrayList(); - List keys = Arrays.asList("left", "right"); + List keys = Arrays.asList("left", "right"); for (String key : keys) { // The keys are the location @@ -131,8 +132,7 @@ private List getMeasurements(Map formValues, String encID, String c try { Double doubleVal = Double.valueOf(value); list.add(new Measurement(encID, key, doubleVal, units, samplingProtocol)); - } catch (Exception ex) { - } + } catch (Exception ex) {} } } return list; @@ -188,7 +188,7 @@ private List getMeasurements(Map formValues, String encID, String c if (item.isFormField()) { // plain field formValues.put(item.getFieldName(), ServletUtilities.preventCrossSiteScriptingAttacks(item.getString( - "UTF-8").trim())); + "UTF-8").trim())); if (item.getFieldName().equals("defaultProject")) { if (!projectIdSelection.contains(item.getString().trim())) { projectIdSelection.add(item.getString().trim()); @@ -262,8 +262,7 @@ private List getMeasurements(Map formValues, String encID, String c if (badmsg.equals("")) { badmsg = "none"; } session.setAttribute("filesBadMessage", badmsg); if (fileSuccess) { - - // check for spamBots + // check for spamBots boolean spamBot = false; String[] spamFieldsToCheck = new String[] { "submitterPhone", "submitterName", "photographerName", "" + "Phone", "location", @@ -763,7 +762,6 @@ else if (formValues.get("location") != null) { if ((formValues.get("lat") != null) && (formValues.get("longitude") != null) && !formValues.get("lat").toString().equals("") && !formValues.get("longitude").toString().equals("")) { - try { double degrees = (new Double(formValues.get("lat").toString())).doubleValue(); double position = degrees; @@ -780,7 +778,6 @@ else if (formValues.get("location") != null) { e.printStackTrace(); } } - enc.addComments("

Submitted on " + (new java.util.Date()).toString() + " from address: " + ServletUtilities.getRemoteHost(request) + "

"); // enc.approved = false; @@ -897,7 +894,7 @@ else if (formValues.get("location") != null) { parentTask.setParameters(tp); } Task task = org.ecocean.ia.IA.intakeMediaAssets(myShepherd, enc.getMedia(), - parentTask); + parentTask); myShepherd.storeNewTask(task); Logger log = LoggerFactory.getLogger(EncounterForm.class); log.info("New encounter submission: Date: Fri, 13 Dec 2024 12:59:55 -0700 Subject: [PATCH 10/26] submitterUserId in index; smarter viewUsers perm logic; alpha new permissions on searching api --- src/main/java/org/ecocean/Encounter.java | 19 ++++++++++++------- src/main/java/org/ecocean/OpenSearch.java | 17 ++++++++++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index f62470685b..6fc77cb567 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3953,7 +3953,7 @@ public static void opensearchIndexPermissions() { } encCount++; if (encCount % 1000 == 0) Util.mark("enc[" + encCount + "]", startT); - viewUsers.put(uid); + // viewUsers.put(uid); // we no longer do this as we use submitterUserId from regular indexing in query filter if (collab.containsKey(uid)) { for (String colUsername : collab.get(uid)) { String colId = usernameToId.get(colUsername); @@ -3966,12 +3966,14 @@ public static void opensearchIndexPermissions() { viewUsers.put(colId); } } - updateData.put("viewUsers", viewUsers); - try { - enc.opensearchUpdate(updateData); - } catch (Exception ex) { - // keeping this quiet cuz it can get noise while index builds - // System.out.println("opensearchIndexPermissions(): WARNING failed to update viewUsers on enc " + enc.getId() + "; likely has not been indexed yet: " + ex); + if (viewUsers.length() > 0) { + updateData.put("viewUsers", viewUsers); + try { + enc.opensearchUpdate(updateData); + } catch (Exception ex) { + // keeping this quiet cuz it can get noise while index builds + // System.out.println("opensearchIndexPermissions(): WARNING failed to update viewUsers on enc " + enc.getId() + "; likely has not been indexed yet: " + ex); + } } } // Util.mark("perm: done encs", startT); @@ -4038,6 +4040,8 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeNullField("assignedUsername"); } else { jgen.writeStringField("assignedUsername", this.submitterID); + User submitter = this.getSubmitterUser(myShepherd); + if (submitter != null) jgen.writeStringField("submitterUserId", submitter.getId()); } jgen.writeArrayFieldStart("submitters"); for (String id : this.getAllSubmitterIds(myShepherd)) { @@ -4278,6 +4282,7 @@ public org.json.JSONObject opensearchMapping() { map.put("taxonomy", keywordType); map.put("occurrenceId", keywordType); map.put("state", keywordType); + map.put("submitterUserId", keywordType); // all case-insensitive keyword-ish types map.put("locationId", keywordNormalType); diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 6c6fd1b05b..320c5ee87b 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -630,12 +630,23 @@ public static boolean getPermissionsNeeded(Shepherd myShepherd) { public static JSONObject querySanitize(JSONObject query, User user, Shepherd myShepherd) { if ((query == null) || (user == null)) return query; - // do not add viewUsers query when we are admin, as user has no restriction + // do not add permissions clause when we are admin, as user has no restriction if (user.isAdmin(myShepherd)) return query; // if (!Collaboration.securityEnabled("context0")) TODO do we want to allow everything searchable? - JSONObject permClause = new JSONObject( +/* + JSONObject permClause = new JSONObject("{\"bool\": {\"should\": [] }}"); "{\"bool\": {\"should\": [{\"term\": {\"publiclyReadable\": true}}, {\"term\": {\"viewUsers\": \"" - + user.getId() + "\"}} ] }}"); + + user.getId() + "\"}} ] }}"); + */ + JSONArray shouldArr = new JSONArray(); + shouldArr.put(new JSONObject("{\"term\": {\"publiclyReadable\": true}}")); + shouldArr.put(new JSONObject("{\"term\": {\"submitterUserId\": \"" + user.getId() + + "\"}}")); + shouldArr.put(new JSONObject("{\"term\": {\"viewUsers\": \"" + user.getId() + "\"}}")); + JSONObject pshould = new JSONObject(); + pshould.put("should", shouldArr); + JSONObject permClause = new JSONObject(); + permClause.put("bool", pshould); JSONObject newQuery = new JSONObject(query.toString()); try { JSONArray filter = newQuery.getJSONObject("query").getJSONObject("bool").getJSONArray( From 89368c2cebf4212c3ffdeb7dfaceee6ee5df06c5 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 16 Dec 2024 10:20:20 -0700 Subject: [PATCH 11/26] userIdsWithViewAccess() no longer abstract on base class, used only for encounters; introducing opensearchProcessPermissions on encounters to force permissions during (single) indexing --- src/main/java/org/ecocean/Base.java | 12 +++++- src/main/java/org/ecocean/Encounter.java | 40 ++++++++++++++++--- .../java/org/ecocean/MarkedIndividual.java | 6 ++- src/main/java/org/ecocean/Occurrence.java | 16 ++++---- .../servlet/EncounterSetLocationID.java | 3 ++ src/main/resources/org/ecocean/package.jdo | 2 + 6 files changed, 62 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index 90958ad970..2a3b331990 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -74,8 +74,10 @@ */ public abstract void addComments(final String newComments); - public abstract List userIdsWithViewAccess(Shepherd myShepherd); - public abstract List userIdsWithEditAccess(Shepherd myShepherd); + // issue 785 makes this no longer necessary; they overrides are left on Occurrence and MarkedIndividual + // for now as reference -- but are not called. they will need to be addressed when these classes are searchable + // public abstract List userIdsWithViewAccess(Shepherd myShepherd); + // public abstract List userIdsWithEditAccess(Shepherd myShepherd); public abstract String opensearchIndexName(); @@ -153,6 +155,7 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) myShepherd.beginDBTransaction(); jgen.writeStringField("id", this.getId()); jgen.writeNumberField("version", this.getVersion()); + jgen.writeNumberField("indexTimestamp", System.currentTimeMillis()); /* these are no longer computed in the general opensearchIndex() call. @@ -185,6 +188,11 @@ public static JSONObject opensearchQuery(final String indexname, final JSONObjec return res; } + // this is so we can call it on Base obj, but really is only needed by [overridden by] Encounter (currently) + public boolean getOpensearchProcessPermissions() { + return false; + } + public static Map getAllVersions(Shepherd myShepherd, String sql) { Query query = myShepherd.getPM().newQuery("javax.jdo.query.SQL", sql); Map rtn = new HashMap(); diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 6fc77cb567..8066071022 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -117,6 +117,7 @@ public class Encounter extends Base implements java.io.Serializable { private Double immunoglobin; private Boolean sampleTakenForDiet; private Boolean injured; + private boolean opensearchProcessPermissions = false; private ArrayList observations = new ArrayList(); @@ -3140,17 +3141,23 @@ public boolean isUserOwner(User user) { // the definition of this might change? return false; } - @Override public List userIdsWithViewAccess(Shepherd myShepherd) { + // new logic means we only need users who are in collab with submitting user + // and if public, we dont need to do this at all + public List userIdsWithViewAccess(Shepherd myShepherd) { List ids = new ArrayList(); - for (User user : myShepherd.getUsersWithUsername()) { - if ((user.getId() != null) && this.canUserView(user, myShepherd)) - ids.add(user.getId()); + if (this.isPubliclyReadable()) return ids; + List collabs = Collaboration.collaborationsForUser(myShepherd, + this.getSubmitterID()); + for (Collaboration collab : collabs) { + User user = myShepherd.getUser(collab.getOtherUsername(this.getSubmitterID())); + if (user != null) ids.add(user.getId()); } return ids; } - @Override public List userIdsWithEditAccess(Shepherd myShepherd) { +/* + public List userIdsWithEditAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getUsersWithUsername()) { @@ -3158,7 +3165,7 @@ public boolean isUserOwner(User user) { // the definition of this might change? } return ids; } - + */ public JSONObject sanitizeJson(HttpServletRequest request, JSONObject jobj) throws JSONException { boolean fullAccess = this.canUserAccess(request); @@ -3859,6 +3866,14 @@ public boolean isPubliclyReadable() { return User.isUsernameAnonymous(this.submitterID); } + public boolean getOpensearchProcessPermissions() { + return opensearchProcessPermissions; + } + + public void setOpensearchProcessPermissions(boolean value) { + opensearchProcessPermissions = value; + } + // wrapper for below, that checks if we really need to be run public static void opensearchIndexPermissionsBackground(Shepherd myShepherd) { boolean runIt = false; @@ -4253,6 +4268,19 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeNumberField(type, bmeas.get(type).getValue()); } jgen.writeEndObject(); + // this gets set on specific single-encounter-only actions, when extra expense is okay + // otherwise this will be computed by permissions backgrounding + if (this.getOpensearchProcessPermissions()) { + System.out.println("opensearchProcessPermissions=true for " + this.getId() + + "; indexing permissions"); + jgen.writeFieldName("viewUsers"); + jgen.writeStartArray(); + for (String id : this.userIdsWithViewAccess(myShepherd)) { + System.out.println("opensearch whhhh: " + id); + jgen.writeString(id); + } + jgen.writeEndArray(); + } myShepherd.rollbackAndClose(); } diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 858a3f4fba..04f580326b 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2018,7 +2018,8 @@ public boolean canUserAccess(HttpServletRequest request) { return Collaboration.canUserAccessMarkedIndividual(this, request); } - @Override public List userIdsWithViewAccess(Shepherd myShepherd) { + // see note on Base class + public List userIdsWithViewAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getAllUsers()) { @@ -2027,7 +2028,8 @@ public boolean canUserAccess(HttpServletRequest request) { return ids; } - @Override public List userIdsWithEditAccess(Shepherd myShepherd) { + // see note on Base class + public List userIdsWithEditAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getAllUsers()) { diff --git a/src/main/java/org/ecocean/Occurrence.java b/src/main/java/org/ecocean/Occurrence.java index c66d3a33d6..ef937380c2 100644 --- a/src/main/java/org/ecocean/Occurrence.java +++ b/src/main/java/org/ecocean/Occurrence.java @@ -95,7 +95,7 @@ public class Occurrence extends Base implements java.io.Serializable { private Integer numCalves; private String observer; - private String submitterID; + private String submitterID; private List submitters; private List informOthers; @@ -383,12 +383,12 @@ public ArrayList getMarkedIndividualNamesForThisOccurrence() { return names; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public void setID(String id) { occurrenceID = id; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public String getID() { return occurrenceID; } @@ -401,7 +401,7 @@ public String getWebUrl(HttpServletRequest req) { return getWebUrl(getOccurrenceID(), req); } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public String getOccurrenceID() { return occurrenceID; } @@ -416,7 +416,7 @@ public String getOccurrenceID() { occurrenceID = id; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public void setOccurrenceID(String id) { occurrenceID = id; } @@ -815,7 +815,8 @@ public boolean canUserAccess(HttpServletRequest request) { return Collaboration.canUserAccessOccurrence(this, request); } - @Override public List userIdsWithViewAccess(Shepherd myShepherd) { + // see note on Base class + public List userIdsWithViewAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getAllUsers()) { @@ -827,7 +828,8 @@ public boolean canUserAccess(HttpServletRequest request) { return ids; } - @Override public List userIdsWithEditAccess(Shepherd myShepherd) { + // see note on Base class + public List userIdsWithEditAccess(Shepherd myShepherd) { List ids = new ArrayList(); for (User user : myShepherd.getAllUsers()) { diff --git a/src/main/java/org/ecocean/servlet/EncounterSetLocationID.java b/src/main/java/org/ecocean/servlet/EncounterSetLocationID.java index 3776425517..fc0d6b0ef7 100644 --- a/src/main/java/org/ecocean/servlet/EncounterSetLocationID.java +++ b/src/main/java/org/ecocean/servlet/EncounterSetLocationID.java @@ -70,6 +70,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) myShepherd.beginDBTransaction(); String encNum = request.getParameter("number").trim(); Encounter changeMe = myShepherd.getEncounter(encNum); + changeMe.setOpensearchProcessPermissions(true); setDateLastModified(changeMe); try { oldCode = changeMe.getLocationCode(); @@ -78,10 +79,12 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) (new java.util.Date()).toString() + "
Changed location code from " + oldCode + " to " + request.getParameter("code") + ".

"); // update numberLocations on a dependent MarkedIndividual too +/* if (changeMe.getIndividual() != null) { MarkedIndividual indy = changeMe.getIndividual(); indy.refreshDependentProperties(); } + */ } catch (Exception le) { locked = true; le.printStackTrace(); diff --git a/src/main/resources/org/ecocean/package.jdo b/src/main/resources/org/ecocean/package.jdo index 3c6370195e..8c494ece85 100755 --- a/src/main/resources/org/ecocean/package.jdo +++ b/src/main/resources/org/ecocean/package.jdo @@ -434,6 +434,8 @@ + + From 091a3e38ef06efb5219410d95cb0c084a972c589 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 16 Dec 2024 11:31:16 -0700 Subject: [PATCH 12/26] linting --- src/main/java/org/ecocean/Encounter.java | 1 - src/main/java/org/ecocean/api/Login.java | 15 ++++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 3006abe5d5..a540c34a10 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -4280,7 +4280,6 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeNumberField(type, bmeas.get(type).getValue()); } jgen.writeEndObject(); - // this gets set on specific single-encounter-only actions, when extra expense is okay // otherwise this will be computed by permissions backgrounding if (this.getOpensearchProcessPermissions()) { diff --git a/src/main/java/org/ecocean/api/Login.java b/src/main/java/org/ecocean/api/Login.java index 748c43e31f..0fd50232d2 100644 --- a/src/main/java/org/ecocean/api/Login.java +++ b/src/main/java/org/ecocean/api/Login.java @@ -16,9 +16,9 @@ import org.apache.shiro.authc.UsernamePasswordToken; import org.apache.shiro.session.Session; import org.apache.shiro.subject.Subject; +import org.apache.shiro.SecurityUtils; import org.apache.shiro.web.util.SavedRequest; import org.apache.shiro.web.util.WebUtils; -import org.apache.shiro.SecurityUtils; import org.ecocean.servlet.ServletUtilities; import org.ecocean.Shepherd; @@ -74,15 +74,12 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) success = true; results = user.infoJSONObject(context, true); results.put("success", true); - - //check for redirect URL - SavedRequest saved=WebUtils.getAndClearSavedRequest(request); - if(saved!=null) { - results.put("redirectUrl",saved.getRequestUrl()); - } - - + // check for redirect URL + SavedRequest saved = WebUtils.getAndClearSavedRequest(request); + if (saved != null) { + results.put("redirectUrl", saved.getRequestUrl()); + } } catch (UnknownAccountException ex) { // username not found ex.printStackTrace(); From e28aa3610d1dcb778d61cb94bcd88577695e7e29 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 16 Dec 2024 11:43:01 -0700 Subject: [PATCH 13/26] tweaks post-commit --- src/main/java/org/ecocean/Base.java | 2 +- src/main/java/org/ecocean/Encounter.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index 560e8a4ee8..ff94170d5d 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -77,7 +77,7 @@ */ public abstract void addComments(final String newComments); - // issue 785 makes this no longer necessary; they overrides are left on Occurrence and MarkedIndividual + // issue 785 makes this no longer necessary; the overrides are left on Occurrence and MarkedIndividual // for now as reference -- but are not called. they will need to be addressed when these classes are searchable // public abstract List userIdsWithViewAccess(Shepherd myShepherd); // public abstract List userIdsWithEditAccess(Shepherd myShepherd); diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index a540c34a10..73e8715693 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -4293,7 +4293,6 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd } jgen.writeEndArray(); } - myShepherd.rollbackAndClose(); } @Override public long getVersion() { From ae4013374cd920b4c22b03cade35e8f64f22365c Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 16 Dec 2024 12:11:21 -0700 Subject: [PATCH 14/26] set these values via properties, now that we have that option --- src/main/java/org/ecocean/OpenSearch.java | 6 ++++-- src/main/resources/bundles/OpenSearch.properties | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 9e409087e2..b821e5f9f6 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -66,8 +66,10 @@ public class OpenSearch { "backgroundDelayMinutes", 20); public static int BACKGROUND_SLICE_SIZE = (Integer)getConfigurationValue("backgroundSliceSize", 2500); - public static int BACKGROUND_PERMISSIONS_MINUTES = 10; // how often it checks if NEED to run - public static int BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES = 45; // how often it forces a run + public static int BACKGROUND_PERMISSIONS_MINUTES = (Integer)getConfigurationValue( + "backgroundPermissionsMinutes", 10); + public static int BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES = (Integer)getConfigurationValue( + "backgroundPermissionsMaxForceMinutes", 45); public static String PERMISSIONS_LAST_RUN_KEY = "OpenSearch_permissions_last_run_timestamp"; public static String PERMISSIONS_NEEDED_KEY = "OpenSearch_permissions_needed"; public static String QUERY_STORAGE_DIR = "/tmp"; // FIXME diff --git a/src/main/resources/bundles/OpenSearch.properties b/src/main/resources/bundles/OpenSearch.properties index 27beb9dc8d..6e10c4e8db 100644 --- a/src/main/resources/bundles/OpenSearch.properties +++ b/src/main/resources/bundles/OpenSearch.properties @@ -2,6 +2,12 @@ #backgroundDelayMinutes=20 #backgroundSliceSize=2500 +# how often to check to see if permissions *needs* to run +#backgroundPermissionsMinutes=10 +# how often it is *forced* to run +#backgroundPermissionsMaxForceMinutes=45 + + # these probably should not be adjusted #searchScrollTime=10m #searchPitTime=10m From 9e1deb5b47bb09fa6675c4bda12a44039ad00e4d Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 16 Dec 2024 12:24:15 -0700 Subject: [PATCH 15/26] throw exception rather than be forgiving when adding permissions to query --- .../org/ecocean/EncounterQueryProcessor.java | 18 ++++++++++++++++-- src/main/java/org/ecocean/OpenSearch.java | 10 +++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/ecocean/EncounterQueryProcessor.java b/src/main/java/org/ecocean/EncounterQueryProcessor.java index 83fec9249f..04d3ba505e 100644 --- a/src/main/java/org/ecocean/EncounterQueryProcessor.java +++ b/src/main/java/org/ecocean/EncounterQueryProcessor.java @@ -61,7 +61,14 @@ public static String queryStringBuilder(HttpServletRequest request, StringBuffer String indexName = searchQuery.optString("indexName", null); if (indexName == null) return failed; searchQuery = OpenSearch.queryScrubStored(searchQuery); - JSONObject sanitized = OpenSearch.querySanitize(searchQuery, user, myShepherd); + JSONObject sanitized = null; + try { + sanitized = OpenSearch.querySanitize(searchQuery, user, myShepherd); + } catch (IOException ex) { + ex.printStackTrace(); + // this should be unlikely, so fail hard + throw new RuntimeException("query failed"); + } OpenSearch os = new OpenSearch(); String sort = request.getParameter("sort"); String sortOrder = request.getParameter("sortOrder"); @@ -1542,7 +1549,14 @@ public static EncounterQueryResult processQuery(Shepherd myShepherd, HttpServlet return new EncounterQueryResult(rEncounters, "searchQuery has no indexName", "OpenSearch id " + searchQueryId); searchQuery = OpenSearch.queryScrubStored(searchQuery); - JSONObject sanitized = OpenSearch.querySanitize(searchQuery, user, myShepherd); + JSONObject sanitized = null; + try { + sanitized = OpenSearch.querySanitize(searchQuery, user, myShepherd); + } catch (IOException ex) { + ex.printStackTrace(); + // this should be unlikely, so fail hard + throw new RuntimeException("query failed"); + } OpenSearch os = new OpenSearch(); String sort = request.getParameter("sort"); String sortOrder = request.getParameter("sortOrder"); diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index b821e5f9f6..cdbe02838a 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -634,8 +634,9 @@ public static boolean getPermissionsNeeded(Shepherd myShepherd) { return "true".equals(value); } - public static JSONObject querySanitize(JSONObject query, User user, Shepherd myShepherd) { - if ((query == null) || (user == null)) return query; + public static JSONObject querySanitize(JSONObject query, User user, Shepherd myShepherd) + throws IOException { + if ((query == null) || (user == null)) throw new IOException("empty query or user"); // do not add permissions clause when we are admin, as user has no restriction if (user.isAdmin(myShepherd)) return query; // if (!Collaboration.securityEnabled("context0")) TODO do we want to allow everything searchable? @@ -659,7 +660,10 @@ public static JSONObject querySanitize(JSONObject query, User user, Shepherd myS "filter"); filter.put(permClause); } catch (Exception ex) { - System.out.println("OpenSearch.querySanitize() failed to find filter element: " + ex); + System.out.println( + "OpenSearch.querySanitize() failed to find placement for permissions in query=" + + query + "; cause: " + ex); + throw new IOException("unable to find placement for permissions clause in query"); } return newQuery; } From 9742308263638e1293f01edb05b2753f95d8787d Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 16 Dec 2024 15:47:42 -0700 Subject: [PATCH 16/26] use the newly available Boolean SystemValue --- src/main/java/org/ecocean/OpenSearch.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index a8977cfbd3..d28806dfce 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -624,16 +624,15 @@ public static Long getPermissionsTimestamp(Shepherd myShepherd) { return SystemValue.getLong(myShepherd, PERMISSIONS_LAST_RUN_KEY); } - // FIXME: when we have Boolean support in SystemValue, modify these! public static void setPermissionsNeeded(Shepherd myShepherd, boolean value) { - SystemValue.set(myShepherd, PERMISSIONS_NEEDED_KEY, String.valueOf(value)); + SystemValue.set(myShepherd, PERMISSIONS_NEEDED_KEY, value); } - // TODO see above public static boolean getPermissionsNeeded(Shepherd myShepherd) { - String value = SystemValue.getString(myShepherd, PERMISSIONS_NEEDED_KEY); + Boolean value = SystemValue.getBoolean(myShepherd, PERMISSIONS_NEEDED_KEY); - return "true".equals(value); + if (value == null) return false; + return value; } public static JSONObject querySanitize(JSONObject query, User user, Shepherd myShepherd) From ad98174c6013035d47f3e4d4a777e15018f0dad3 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 16 Dec 2024 17:45:11 -0700 Subject: [PATCH 17/26] little more info --- src/main/webapp/appadmin/opensearchInfo.jsp | 5 +++++ src/main/webapp/appadmin/opensearchSync.jsp | 11 ++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/appadmin/opensearchInfo.jsp b/src/main/webapp/appadmin/opensearchInfo.jsp index 0c2e5d57cd..8e90b593c7 100644 --- a/src/main/webapp/appadmin/opensearchInfo.jsp +++ b/src/main/webapp/appadmin/opensearchInfo.jsp @@ -30,6 +30,11 @@ out.println("

SEARCH_SCROLL_TIME=" + os.SEARCH_SCROLL_TIME + "
"); out.println("SEARCH_PIT_TIME=" + os.SEARCH_PIT_TIME + "
"); out.println("BACKGROUND_DELAY_MINUTES=" + os.BACKGROUND_DELAY_MINUTES + "
"); out.println("BACKGROUND_SLICE_SIZE=" + os.BACKGROUND_SLICE_SIZE + "

"); +out.println("BACKGROUND_PERMISSIONS_MINUTES=" + os.BACKGROUND_PERMISSIONS_MINUTES + "
"); +out.println("BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES=" + os.BACKGROUND_PERMISSIONS_MAX_FORCE_MINUTES + "

"); + +out.println("

active indexing: foreground=" + String.valueOf(os.indexingActiveForeground())); +out.println(" / background=" + String.valueOf(os.indexingActiveBackground()) + "

"); Request req = new Request("GET", "_cat/indices?v"); //req.setJsonEntity(query.toString()); diff --git a/src/main/webapp/appadmin/opensearchSync.jsp b/src/main/webapp/appadmin/opensearchSync.jsp index 968b3749a0..ad32b6cfa9 100644 --- a/src/main/webapp/appadmin/opensearchSync.jsp +++ b/src/main/webapp/appadmin/opensearchSync.jsp @@ -9,6 +9,7 @@ org.ecocean.* <% System.out.println("opensearchSync.jsp begun..."); long timer = System.currentTimeMillis(); +int numProcessed = -1; Util.mark("opensearchSync begin"); boolean resetIndex = Util.requestParameterSet(request.getParameter("resetIndex")); @@ -73,6 +74,7 @@ if (forceNum > 0) { //System.out.println(enc.getId() + ": " + enc.getVersion()); try { enc.opensearchIndex(); + numProcessed++; } catch (Exception ex) { System.out.println("opensearchSync.jsp: exception failure on " + enc); ex.printStackTrace(); @@ -91,7 +93,14 @@ myShepherd.rollbackAndClose(); OpenSearch.unsetActiveIndexingForeground(); os.deleteAllPits(); -System.out.println("opensearchSync.jsp finished"); + +double totalMin = System.currentTimeMillis() - timer; +totalMin = totalMin / 60000D; +if (numProcessed > 0) { + System.out.println("opensearchSync.jsp finished: " + numProcessed + " in " + String.format("%.2f", totalMin) + " min (" + String.format("%.2f", numProcessed / totalMin) + " per min)"); +} else { + System.out.println("opensearchSync.jsp finished: " + totalMin + " min"); +} Util.mark("opensearchSync ended", timer); %> From ea7014cddf1835c05f68414ec651672ba4f878a2 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 16 Dec 2024 17:49:10 -0700 Subject: [PATCH 18/26] enc.modified is nice to index --- config/indices.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/indices.sql b/config/indices.sql index 72a69afbf8..8deb24b902 100644 --- a/config/indices.sql +++ b/config/indices.sql @@ -5,11 +5,14 @@ -- UGH, i guess _IDX (in caps!) should be what we standardize on. that is what datanucleus does. -- to make matters worse, we have some places we named them explicitely in package.jdo (but those should fix themselves?) +BEGIN; + CREATE INDEX IF NOT EXISTS "MEDIAASSET_PARENTID_idx" ON "MEDIAASSET" ("PARENTID"); CREATE INDEX IF NOT EXISTS "MEDIAASSET_HASHCODE_idx" ON "MEDIAASSET" ("HASHCODE"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_LOCATIONID_idx" ON "ENCOUNTER" ("LOCATIONID"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_STATE_idx" ON "ENCOUNTER" ("STATE"); +CREATE INDEX IF NOT EXISTS "ENCOUNTER_MODIFIED_idx" ON "ENCOUNTER" ("MODIFIED"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_INDIVIDUALID_idx" ON "ENCOUNTER" ("INDIVIDUALID"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_DATEINMILLISECONDS_idx" ON "ENCOUNTER" ("DATEINMILLISECONDS"); CREATE INDEX IF NOT EXISTS "ENCOUNTER_DECIMALLATITUDE_idx" ON "ENCOUNTER" ("DECIMALLATITUDE"); @@ -45,3 +48,5 @@ CREATE INDEX IF NOT EXISTS "TASK_CREATED_IDX" ON "TASK" ("CREATED"); INSERT INTO "RELATIONSHIP" ("RELATIONSHIP_ID", "MARKEDINDIVIDUALNAME1", "MARKEDINDIVIDUALNAME2", "MARKEDINDIVIDUALROLE1", "MARKEDINDIVIDUALROLE2", "TYPE", "STARTTIME", "ENDTIME") VALUES (0, (SELECT "INDIVIDUALID" FROM "MARKEDINDIVIDUAL" ORDER BY random() LIMIT 1), (SELECT "INDIVIDUALID" FROM "MARKEDINDIVIDUAL" ORDER BY random() LIMIT 1), 'placeholder', 'placeholder', 'placeholder-to-prevent-empty-table', 0, 0); +END; + From 1fc60be5cf3782fb4b6ef5cd915a17e70dde5c37 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Tue, 17 Dec 2024 10:33:24 -0700 Subject: [PATCH 19/26] name change for clarity --- src/main/webapp/appadmin/opensearchSync.jsp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/webapp/appadmin/opensearchSync.jsp b/src/main/webapp/appadmin/opensearchSync.jsp index ad32b6cfa9..f26fe99e48 100644 --- a/src/main/webapp/appadmin/opensearchSync.jsp +++ b/src/main/webapp/appadmin/opensearchSync.jsp @@ -14,17 +14,17 @@ Util.mark("opensearchSync begin"); boolean resetIndex = Util.requestParameterSet(request.getParameter("resetIndex")); -String fstr = request.getParameter("forceNum"); -int forceNum = -1; +String fstr = request.getParameter("endNum"); +int endNum = -1; if ("".equals(fstr)) { - forceNum = 500; + endNum = 500; } else if (fstr != null) { try { - forceNum = Integer.parseInt(fstr); + endNum = Integer.parseInt(fstr); } catch (Exception ex) {} } -if (forceNum == 0) forceNum = 999999; +if (endNum == 0) endNum = 999999; String sstr = request.getParameter("startNum"); int startNum = -1; @@ -59,8 +59,12 @@ if (!os.existsIndex("encounter")) { } -if (forceNum > 0) { - out.println("

indexing " + forceNum + " Encounters

"); +if (endNum > 0) { + if (startNum > 0) { + out.println("

indexing " + startNum + "-" + endNum + " Encounters

"); + } else { + out.println("

indexing through " + endNum + " Encounters

"); + } int ct = 0; Iterator itr = myShepherd.getAllEncounters("catalogNumber"); while (itr.hasNext()) { @@ -80,7 +84,7 @@ if (forceNum > 0) { ex.printStackTrace(); } if (ct % 100 == 0) System.out.println("opensearchSync.jsp: count " + ct); - if (ct > forceNum) break; + if (ct > endNum) break; } } else { From 979962512d9a4b8f700d870a078547a9d78c23f7 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Tue, 17 Dec 2024 12:55:43 -0700 Subject: [PATCH 20/26] use raw sql instead of Encounter objects, for speed; also some debugging on --- src/main/java/org/ecocean/Encounter.java | 27 +++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 5c247ff09e..ba586e3180 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3927,6 +3927,7 @@ public static void opensearchIndexPermissions() { System.out.println("opensearchIndexPermissions(): begin..."); // no security => everything publiclyReadable - saves us work, no? if (!Collaboration.securityEnabled("context0")) return; + OpenSearch os = new OpenSearch(); Map > collab = new HashMap >(); Map usernameToId = new HashMap(); Shepherd myShepherd = new Shepherd("context0"); @@ -3948,24 +3949,29 @@ public static void opensearchIndexPermissions() { collab.get(user.getId()).add(col.getOtherUsername(user.getUsername())); } } -// Util.mark("perm: user build done", startT); + Util.mark("perm: user build done", startT); System.out.println("opensearchIndexPermissions(): " + usernameToId.size() + " total users; " + nonAdminCt + " non-admin; " + collab.size() + " have active collab"); // now iterated over (non-public) encounters int encCount = 0; org.json.JSONObject updateData = new org.json.JSONObject(); - Query query = myShepherd.getPM().newQuery( - "SELECT FROM org.ecocean.Encounter WHERE (submitterID != null) && (submitterID != '') && (submitterID != 'N/A') && (submitterID != 'public')"); - Iterator it = myShepherd.getAllEncounters(query); -// Util.mark("perm: start encs", startT); + // we do not need full Encounter objects here to update index docs, so lets do this via sql/fields - much faster + String sql = + "SELECT \"CATALOGNUMBER\", \"SUBMITTERID\" FROM \"ENCOUNTER\" WHERE \"SUBMITTERID\" IS NOT NULL AND \"SUBMITTERID\" != '' AND \"SUBMITTERID\" != 'N/A' AND \"SUBMITTERID\" != 'public'"; + Query q = myShepherd.getPM().newQuery("javax.jdo.query.SQL", sql); + List results = (List)q.execute(); + Iterator it = results.iterator(); + Util.mark("perm: start encs, size=" + results.size(), startT); while (it.hasNext()) { + Object[] row = (Object[])it.next(); + String id = (String)row[0]; + String submitterId = (String)row[1]; org.json.JSONArray viewUsers = new org.json.JSONArray(); - Encounter enc = (Encounter)it.next(); - String uid = usernameToId.get(enc.getSubmitterID()); + String uid = usernameToId.get(submitterId); if (uid == null) { // see issue 939 for example :( System.out.println("opensearchIndexPermissions(): WARNING invalid username " + - enc.getSubmitterID() + " on enc " + enc.getId()); + submitterId + " on enc " + id); continue; } encCount++; @@ -3986,14 +3992,15 @@ public static void opensearchIndexPermissions() { if (viewUsers.length() > 0) { updateData.put("viewUsers", viewUsers); try { - enc.opensearchUpdate(updateData); + os.indexUpdate("encounter", id, updateData); } catch (Exception ex) { // keeping this quiet cuz it can get noise while index builds // System.out.println("opensearchIndexPermissions(): WARNING failed to update viewUsers on enc " + enc.getId() + "; likely has not been indexed yet: " + ex); } } } -// Util.mark("perm: done encs", startT); + q.closeAll(); + Util.mark("perm: done encs", startT); myShepherd.rollbackAndClose(); System.out.println("opensearchIndexPermissions(): ...end [" + encCount + " encs; " + Math.round((System.currentTimeMillis() - startT) / 1000) + "sec]"); From 5a4e4d023596bef1b2f880511ed2fd5586260b50 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Tue, 17 Dec 2024 16:52:08 -0700 Subject: [PATCH 21/26] try/catch safety --- src/main/java/org/ecocean/SystemValue.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/SystemValue.java b/src/main/java/org/ecocean/SystemValue.java index f66a9c0ed6..250d9fbe0f 100644 --- a/src/main/java/org/ecocean/SystemValue.java +++ b/src/main/java/org/ecocean/SystemValue.java @@ -57,7 +57,11 @@ public static SystemValue obtain(Shepherd myShepherd, String key) { public void store(Shepherd myShepherd) { this.version = System.currentTimeMillis(); - myShepherd.getPM().makePersistent(this); + try { + myShepherd.getPM().makePersistent(this); + } catch (Exception ex) { + System.out.println("SystemValue.store() failed to store " + this + ": " + ex); + } } private static SystemValue _set(Shepherd myShepherd, String key, String type, Object val) { From 9524b0e931c62884194b6d8d07765a0ef947ca24 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Tue, 17 Dec 2024 16:52:27 -0700 Subject: [PATCH 22/26] shepherd-less flavor --- src/main/java/org/ecocean/OpenSearch.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index d28806dfce..e20c100950 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -628,6 +628,21 @@ public static void setPermissionsNeeded(Shepherd myShepherd, boolean value) { SystemValue.set(myShepherd, PERMISSIONS_NEEDED_KEY, value); } + public static void setPermissionsNeeded(boolean value) { + Shepherd myShepherd = new Shepherd("context0"); + + myShepherd.setAction("OpenSearch.setPermissionsNeeded"); + myShepherd.beginDBTransaction(); + try { + setPermissionsNeeded(myShepherd, value); + myShepherd.commitDBTransaction(); + myShepherd.closeDBTransaction(); + } catch (Exception ex) { + ex.printStackTrace(); + myShepherd.rollbackAndClose(); + } + } + public static boolean getPermissionsNeeded(Shepherd myShepherd) { Boolean value = SystemValue.getBoolean(myShepherd, PERMISSIONS_NEEDED_KEY); From b169ece2392175118ef8f791d193f2a7c493d936 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Tue, 17 Dec 2024 16:53:04 -0700 Subject: [PATCH 23/26] collab persistence triggers permissionsNeeded --- src/main/java/org/ecocean/WildbookLifecycleListener.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/ecocean/WildbookLifecycleListener.java b/src/main/java/org/ecocean/WildbookLifecycleListener.java index 9666e0e376..2fe161e2a3 100644 --- a/src/main/java/org/ecocean/WildbookLifecycleListener.java +++ b/src/main/java/org/ecocean/WildbookLifecycleListener.java @@ -5,6 +5,7 @@ import org.datanucleus.enhancement.Persistable; import org.ecocean.Base; import org.ecocean.OpenSearch; +import org.ecocean.security.Collaboration; // https://www.datanucleus.org/products/accessplatform_4_1/jdo/lifecycle_callbacks.html#listeners @@ -64,6 +65,10 @@ public void postStore(InstanceLifecycleEvent event) { } catch (IOException ex) { ex.printStackTrace(); } + } else if (Collaboration.class.isInstance(obj)) { + System.out.println("WildbookLifecycleListener postStore() event on " + obj + + " triggering permissionsNeeded=true"); + OpenSearch.setPermissionsNeeded(true); } } From ce8d810859e8ac1efe0f76e431fe7b59d91e4151 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Tue, 17 Dec 2024 17:15:14 -0700 Subject: [PATCH 24/26] attempt to handle the case where Encounter is first created (so cant load from new shepherd) --- src/main/java/org/ecocean/Encounter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index ba586e3180..28715f9e2c 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -4684,6 +4684,7 @@ public void sendCreationEmails(Shepherd myShepherd, String langCode) { public void opensearchIndexDeep() throws IOException { final String encId = this.getId(); + final Encounter origEnc = this; ExecutorService executor = Executors.newFixedThreadPool(4); Runnable rn = new Runnable() { public void run() { @@ -4693,6 +4694,8 @@ public void run() { try { Encounter enc = bgShepherd.getEncounter(encId); if (enc == null) { + // we use origEnc if we can (especially necessary on initial creation of Encounter) + if (origEnc != null) origEnc.opensearchIndex(); bgShepherd.rollbackAndClose(); executor.shutdown(); return; From 1d06bb86c944acd1d30d3842926cd3d4c099315e Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 18 Dec 2024 14:15:45 -0700 Subject: [PATCH 25/26] add a couple try/catch to opensearchIndexPermissions() --- src/main/java/org/ecocean/Encounter.java | 107 ++++++++++++----------- 1 file changed, 58 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 28715f9e2c..a9c73acb2b 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3935,19 +3935,23 @@ public static void opensearchIndexPermissions() { myShepherd.beginDBTransaction(); int nonAdminCt = 0; // it seems as though user.uuid is *required* so we can trust that - for (User user : myShepherd.getUsersWithUsername()) { - usernameToId.put(user.getUsername(), user.getId()); - if (user.isAdmin(myShepherd)) continue; - nonAdminCt++; - List collabsFor = Collaboration.collaborationsForUser(myShepherd, - user.getUsername()); - if (Util.collectionIsEmptyOrNull(collabsFor)) continue; - for (Collaboration col : collabsFor) { - if (!col.isApproved() && !col.isEditApproved()) continue; - if (!collab.containsKey(user.getId())) - collab.put(user.getId(), new HashSet()); - collab.get(user.getId()).add(col.getOtherUsername(user.getUsername())); + try { + for (User user : myShepherd.getUsersWithUsername()) { + usernameToId.put(user.getUsername(), user.getId()); + if (user.isAdmin(myShepherd)) continue; + nonAdminCt++; + List collabsFor = Collaboration.collaborationsForUser(myShepherd, + user.getUsername()); + if (Util.collectionIsEmptyOrNull(collabsFor)) continue; + for (Collaboration col : collabsFor) { + if (!col.isApproved() && !col.isEditApproved()) continue; + if (!collab.containsKey(user.getId())) + collab.put(user.getId(), new HashSet()); + collab.get(user.getId()).add(col.getOtherUsername(user.getUsername())); + } } + } catch (Exception ex) { + ex.printStackTrace(); } Util.mark("perm: user build done", startT); System.out.println("opensearchIndexPermissions(): " + usernameToId.size() + @@ -3958,48 +3962,53 @@ public static void opensearchIndexPermissions() { // we do not need full Encounter objects here to update index docs, so lets do this via sql/fields - much faster String sql = "SELECT \"CATALOGNUMBER\", \"SUBMITTERID\" FROM \"ENCOUNTER\" WHERE \"SUBMITTERID\" IS NOT NULL AND \"SUBMITTERID\" != '' AND \"SUBMITTERID\" != 'N/A' AND \"SUBMITTERID\" != 'public'"; - Query q = myShepherd.getPM().newQuery("javax.jdo.query.SQL", sql); - List results = (List)q.execute(); - Iterator it = results.iterator(); - Util.mark("perm: start encs, size=" + results.size(), startT); - while (it.hasNext()) { - Object[] row = (Object[])it.next(); - String id = (String)row[0]; - String submitterId = (String)row[1]; - org.json.JSONArray viewUsers = new org.json.JSONArray(); - String uid = usernameToId.get(submitterId); - if (uid == null) { - // see issue 939 for example :( - System.out.println("opensearchIndexPermissions(): WARNING invalid username " + - submitterId + " on enc " + id); - continue; - } - encCount++; - if (encCount % 1000 == 0) Util.mark("enc[" + encCount + "]", startT); - // viewUsers.put(uid); // we no longer do this as we use submitterUserId from regular indexing in query filter - if (collab.containsKey(uid)) { - for (String colUsername : collab.get(uid)) { - String colId = usernameToId.get(colUsername); - if (colId == null) { - System.out.println( - "opensearchIndexPermissions(): WARNING invalid username " + - colUsername + " in collaboration with userId=" + uid); - continue; + try { + Query q = myShepherd.getPM().newQuery("javax.jdo.query.SQL", sql); + List results = (List)q.execute(); + Iterator it = results.iterator(); + Util.mark("perm: start encs, size=" + results.size(), startT); + while (it.hasNext()) { + Object[] row = (Object[])it.next(); + String id = (String)row[0]; + String submitterId = (String)row[1]; + org.json.JSONArray viewUsers = new org.json.JSONArray(); + String uid = usernameToId.get(submitterId); + if (uid == null) { + // see issue 939 for example :( + System.out.println("opensearchIndexPermissions(): WARNING invalid username " + + submitterId + " on enc " + id); + continue; + } + encCount++; + if (encCount % 1000 == 0) Util.mark("enc[" + encCount + "]", startT); + // viewUsers.put(uid); // we no longer do this as we use submitterUserId from regular indexing in query filter + if (collab.containsKey(uid)) { + for (String colUsername : collab.get(uid)) { + String colId = usernameToId.get(colUsername); + if (colId == null) { + System.out.println( + "opensearchIndexPermissions(): WARNING invalid username " + + colUsername + " in collaboration with userId=" + uid); + continue; + } + viewUsers.put(colId); } - viewUsers.put(colId); } - } - if (viewUsers.length() > 0) { - updateData.put("viewUsers", viewUsers); - try { - os.indexUpdate("encounter", id, updateData); - } catch (Exception ex) { - // keeping this quiet cuz it can get noise while index builds - // System.out.println("opensearchIndexPermissions(): WARNING failed to update viewUsers on enc " + enc.getId() + "; likely has not been indexed yet: " + ex); + if (viewUsers.length() > 0) { + updateData.put("viewUsers", viewUsers); + try { + os.indexUpdate("encounter", id, updateData); + } catch (Exception ex) { + // keeping this quiet cuz it can get noise while index builds + // System.out.println("opensearchIndexPermissions(): WARNING failed to update viewUsers on enc " + enc.getId() + "; likely has not been indexed yet: " + ex); + } } } + q.closeAll(); + } catch (Exception ex) { + System.out.println("opensearchIndexPermissions(): failed during encounter loop: " + ex); + ex.printStackTrace(); } - q.closeAll(); Util.mark("perm: done encs", startT); myShepherd.rollbackAndClose(); System.out.println("opensearchIndexPermissions(): ...end [" + encCount + " encs; " + From fa16a5438357df10ee265e7d3abfe1e8502dc162 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 18 Dec 2024 15:42:29 -0700 Subject: [PATCH 26/26] close query in finally --- src/main/java/org/ecocean/Encounter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index a9c73acb2b..99303edfb1 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3962,8 +3962,9 @@ public static void opensearchIndexPermissions() { // we do not need full Encounter objects here to update index docs, so lets do this via sql/fields - much faster String sql = "SELECT \"CATALOGNUMBER\", \"SUBMITTERID\" FROM \"ENCOUNTER\" WHERE \"SUBMITTERID\" IS NOT NULL AND \"SUBMITTERID\" != '' AND \"SUBMITTERID\" != 'N/A' AND \"SUBMITTERID\" != 'public'"; + Query q = null; try { - Query q = myShepherd.getPM().newQuery("javax.jdo.query.SQL", sql); + q = myShepherd.getPM().newQuery("javax.jdo.query.SQL", sql); List results = (List)q.execute(); Iterator it = results.iterator(); Util.mark("perm: start encs, size=" + results.size(), startT); @@ -4008,6 +4009,8 @@ public static void opensearchIndexPermissions() { } catch (Exception ex) { System.out.println("opensearchIndexPermissions(): failed during encounter loop: " + ex); ex.printStackTrace(); + } finally { + if (q != null) q.closeAll(); } Util.mark("perm: done encs", startT); myShepherd.rollbackAndClose();