diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index 3af6492495..b1afc85ee3 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -6,10 +6,13 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.File; import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; + import javax.jdo.Query; import org.ecocean.api.ApiException; import org.ecocean.OpenSearch; @@ -113,9 +116,25 @@ public void opensearchIndexDeep() public void opensearchUnindex() throws IOException { - OpenSearch opensearch = new OpenSearch(); + // unindexing should be non-blocking and backgrounded + String opensearchIndexName = this.opensearchIndexName(); + String objectId = this.getId(); + ExecutorService executor = Executors.newFixedThreadPool(4); + Runnable rn = new Runnable() { + OpenSearch opensearch = new OpenSearch(); + public void run() { + try { + opensearch.delete(opensearchIndexName, objectId); + } catch (Exception e) { + System.out.println("opensearchUnindex() backgrounding Object " + objectId + + " hit an exception."); + e.printStackTrace(); + } + executor.shutdown(); + } + }; - opensearch.delete(this.opensearchIndexName(), this); + executor.execute(rn); } public void opensearchUnindexQuiet() { @@ -135,12 +154,8 @@ public void opensearchUnindexDeep() } // should be overridden - public void opensearchDocumentSerializer(JsonGenerator jgen) + public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd) throws IOException, JsonProcessingException { - Shepherd myShepherd = new Shepherd("context0"); - - myShepherd.setAction("BaseSerializer"); - myShepherd.beginDBTransaction(); jgen.writeStringField("id", this.getId()); jgen.writeNumberField("version", this.getVersion()); @@ -157,8 +172,19 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeString(id); } jgen.writeEndArray(); - myShepherd.rollbackDBTransaction(); - myShepherd.closeDBTransaction(); + } + + public void opensearchDocumentSerializer(JsonGenerator jgen) + throws IOException, JsonProcessingException { + Shepherd myShepherd = new Shepherd("context0"); + + myShepherd.setAction("BaseSerializer"); + myShepherd.beginDBTransaction(); + try { + opensearchDocumentSerializer(jgen, myShepherd); + } catch (Exception e) {} finally { + myShepherd.rollbackAndClose(); + } } public static JSONObject opensearchQuery(final String indexname, final JSONObject query, diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index d69911f42c..307837b5b7 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -8,6 +8,8 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.Calendar; import java.util.Collection; @@ -154,7 +156,6 @@ public void setSampleTakenForDiet(Boolean sampleTakenForDiet) { private static HashMap > _matchEncounterCache = new HashMap >(); - // An URL to a thumbnail image representing the encounter. private String dwcImageURL; @@ -678,13 +679,12 @@ public boolean hasRightSpotImage() { return (this.getNumRightSpots() > 0); } - // Sets the recorded length of the shark for this encounter. public void setSize(Double mysize) { if (mysize != null) { size = mysize; } else { size = null; } } - // @return the length of the shark + // @return the length of the shark public double getSize() { return size.doubleValue(); } @@ -2450,9 +2450,10 @@ public void setTissueSamples(List samps) { public Set getTissueSampleIDs() { Set ids = new HashSet(); - if (tissueSamples != null) for (TissueSample ts : tissueSamples) { - ids.add(ts.getSampleID()); - } + if (tissueSamples != null) + for (TissueSample ts : tissueSamples) { + ids.add(ts.getSampleID()); + } return ids; } @@ -3860,10 +3861,20 @@ public static org.json.JSONObject opensearchQuery(final org.json.JSONObject quer public void opensearchDocumentSerializer(JsonGenerator jgen) throws IOException, JsonProcessingException { - super.opensearchDocumentSerializer(jgen); Shepherd myShepherd = new Shepherd("context0"); + myShepherd.setAction("Encounter.opensearchDocumentSerializer"); myShepherd.beginDBTransaction(); + try { + opensearchDocumentSerializer(jgen, myShepherd); + } catch (Exception e) {} finally { + myShepherd.rollbackAndClose(); + } + } + + public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd) + throws IOException, JsonProcessingException { + super.opensearchDocumentSerializer(jgen, myShepherd); jgen.writeStringField("locationId", this.getLocationID()); jgen.writeStringField("locationName", this.getLocationName()); @@ -4052,7 +4063,6 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) encDate = Util.getISO8601Date(encs[encs.length - 1].getDate()); if (encDate != null) jgen.writeStringField("individualLastEncounterDate", encDate); } - jgen.writeArrayFieldStart("individualSocialUnits"); for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(indiv)) { Membership mem = su.getMembershipForMarkedIndividual(indiv); @@ -4117,7 +4127,6 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeNumberField(type, bmeas.get(type).getValue()); } jgen.writeEndObject(); - myShepherd.rollbackAndClose(); } @Override public long getVersion() { @@ -4335,9 +4344,13 @@ public static Object validateFieldValue(String fieldName, org.json.JSONObject da // this is throwaway read-only shepherd Shepherd myShepherd = new Shepherd("context0"); myShepherd.setAction("Encounter.validateFieldValue"); + boolean validTaxonomy = false; myShepherd.beginDBTransaction(); - boolean validTaxonomy = myShepherd.isValidTaxonomyName((String)returnValue); - myShepherd.rollbackDBTransaction(); + try { + validTaxonomy = myShepherd.isValidTaxonomyName((String)returnValue); + } catch (Exception e) { e.printStackTrace(); } finally { + myShepherd.rollbackAndClose(); + } if (!validTaxonomy) { error.put("code", ApiException.ERROR_RETURN_CODE_INVALID); error.put("value", returnValue); @@ -4479,4 +4492,33 @@ public void sendCreationEmails(Shepherd myShepherd, String langCode) { myShepherd.rollbackDBTransaction(); } } + + public void opensearchIndexDeep() + throws IOException { + final String encId = this.getId(); + ExecutorService executor = Executors.newFixedThreadPool(4); + Runnable rn = new Runnable() { + public void run() { + Shepherd bgShepherd = new Shepherd("context0"); + bgShepherd.setAction("Encounter.opensearchIndexDeep_" + encId); + bgShepherd.beginDBTransaction(); + try { + Encounter enc = bgShepherd.getEncounter(encId); + if (enc == null) { + bgShepherd.rollbackAndClose(); + executor.shutdown(); + return; + } + enc.opensearchIndex(); + } catch (Exception e) { + e.printStackTrace(); + } finally { + bgShepherd.rollbackAndClose(); + } + executor.shutdown(); + } + }; + + executor.execute(rn); + } } diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 858a3f4fba..bca1cf48d5 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2602,7 +2602,7 @@ public void opensearchIndexDeep() Runnable rn = new Runnable() { public void run() { Shepherd bgShepherd = new Shepherd("context0"); - bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep"); + bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep_" + indivId); bgShepherd.beginDBTransaction(); try { MarkedIndividual indiv = bgShepherd.getMarkedIndividual(indivId); @@ -2625,6 +2625,10 @@ public void run() { ex.printStackTrace(); } } + } catch (Exception e) { + System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " + + indivId + " hit an exception."); + e.printStackTrace(); } finally { bgShepherd.rollbackAndClose(); } diff --git a/src/main/java/org/ecocean/Occurrence.java b/src/main/java/org/ecocean/Occurrence.java index c66d3a33d6..04ac4783e2 100644 --- a/src/main/java/org/ecocean/Occurrence.java +++ b/src/main/java/org/ecocean/Occurrence.java @@ -3,6 +3,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -95,7 +97,7 @@ public class Occurrence extends Base implements java.io.Serializable { private Integer numCalves; private String observer; - private String submitterID; + private String submitterID; private List submitters; private List informOthers; @@ -383,12 +385,12 @@ public ArrayList getMarkedIndividualNamesForThisOccurrence() { return names; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public void setID(String id) { occurrenceID = id; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public String getID() { return occurrenceID; } @@ -401,7 +403,7 @@ public String getWebUrl(HttpServletRequest req) { return getWebUrl(getOccurrenceID(), req); } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public String getOccurrenceID() { return occurrenceID; } @@ -416,7 +418,7 @@ public String getOccurrenceID() { occurrenceID = id; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public void setOccurrenceID(String id) { occurrenceID = id; } @@ -1354,11 +1356,50 @@ public org.json.JSONObject getJSONSummary() { // has to be handled at the point of removal, e.g. OccurrenceRemoveEncounter servlet public void opensearchIndexDeep() throws IOException { - if (this.encounters != null) - for (Encounter enc : this.encounters) { - enc.opensearchIndex(); - } this.opensearchIndex(); + + final String occurId = this.getId(); + ExecutorService executor = Executors.newFixedThreadPool(4); + Runnable rn = new Runnable() { + public void run() { + Shepherd bgShepherd = new Shepherd("context0"); + bgShepherd.setAction("Occurrence.opensearchIndexDeep_" + occurId); + bgShepherd.beginDBTransaction(); + try { + Occurrence occur = bgShepherd.getOccurrence(occurId); + if ((occur == null) || (occur.getEncounters() == null)) { + bgShepherd.rollbackAndClose(); + executor.shutdown(); + return; + } + int total = occur.getNumberEncounters(); + int ct = 0; + for (Encounter enc : occur.getEncounters()) { + ct++; + System.out.println("opensearchIndexDeep() background indexing " + + enc.getId() + " via " + occurId + " [" + ct + "/" + total + "]"); + try { + enc.opensearchIndex(); + } catch (Exception ex) { + System.out.println("opensearchIndexDeep() background indexing " + + enc.getId() + " FAILED: " + ex.toString()); + ex.printStackTrace(); + } + } + } catch (Exception e) { + System.out.println("opensearchIndexDeep() backgrounding Occurrence " + occurId + + " hit an exception."); + e.printStackTrace(); + } finally { + bgShepherd.rollbackAndClose(); + } + System.out.println("opensearchIndexDeep() backgrounding Occurrence " + occurId + + " finished."); + executor.shutdown(); + } + }; + + executor.execute(rn); } @Override public long getVersion() { diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 64b3c78be7..8155cfcd1f 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -9,6 +9,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import javax.jdo.Query; import javax.net.ssl.SSLContext; @@ -56,12 +57,15 @@ public class OpenSearch { public static RestClient restClient = null; public static Map INDEX_EXISTS_CACHE = new HashMap(); public static Map PIT_CACHE = new HashMap(); - public static String SEARCH_SCROLL_TIME = "10m"; - public static String SEARCH_PIT_TIME = "10m"; + public static String SEARCH_SCROLL_TIME = (String)getConfigurationValue("searchScrollTime", + "10m"); + public static String SEARCH_PIT_TIME = (String)getConfigurationValue("searchPitTime", "10m"); public static String INDEX_TIMESTAMP_PREFIX = "OpenSearch_index_timestamp_"; public static String[] VALID_INDICES = { "encounter", "individual", "occurrence" }; - public static int BACKGROUND_DELAY_MINUTES = 20; - public static int BACKGROUND_SLICE_SIZE = 2500; + public static int BACKGROUND_DELAY_MINUTES = (Integer)getConfigurationValue( + "backgroundDelayMinutes", 20); + public static int BACKGROUND_SLICE_SIZE = (Integer)getConfigurationValue("backgroundSliceSize", + 2500); public static String QUERY_STORAGE_DIR = "/tmp"; // FIXME private int pitRetry = 0; @@ -670,4 +674,42 @@ public static JSONObject queryScrubStored(final JSONObject query) { scrubbed.put("query", query.optJSONObject("query")); return scrubbed; } + + public static Object getConfigurationValue(String key, Object defaultValue) { + return getConfigurationValue("context0", key, defaultValue); + } + + public static Object getConfigurationValue(String context, String key, Object defaultValue) { + if (key == null) return null; + Properties props = getConfigurationProperties(context); + if (props == null) { + System.out.println( + "OpenSearch.getConfigurationValue(): WARNING could not get properties file; using defaultValue [" + + defaultValue + "] for " + key); + return defaultValue; + } + String propValue = props.getProperty(key); + // TODO can we actually set a NULL from a properties file? if so: we need to return that as null here + if (propValue == null) return defaultValue; + if (defaultValue instanceof Integer) { // get int from string + try { + return Integer.parseInt(propValue); + } catch (NumberFormatException nfe) { + return defaultValue; + } + } + if (defaultValue instanceof Double) { // get int from string + try { + return Double.parseDouble(propValue); + } catch (NumberFormatException nfe) { + return defaultValue; + } + } + // guess we are just a string + return propValue; + } + + static Properties getConfigurationProperties(String context) { + return ShepherdProperties.getProperties("OpenSearch.properties", "", context); + } } diff --git a/src/main/java/org/ecocean/api/Login.java b/src/main/java/org/ecocean/api/Login.java index b9fe39acba..748c43e31f 100644 --- a/src/main/java/org/ecocean/api/Login.java +++ b/src/main/java/org/ecocean/api/Login.java @@ -57,7 +57,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) // user.setAcceptedUserAgreement(true); // myShepherd.commitDBTransaction(); } catch (Exception ex) { - myShepherd.rollbackDBTransaction(); + myShepherd.rollbackAndClose(); results.put("error", "invalid_credentials"); } if (user != null) { diff --git a/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java b/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java index 6e09e3734d..d3a1cd8167 100644 --- a/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java +++ b/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java @@ -107,7 +107,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) } if (!locked) { myShepherd.commitDBTransaction(); - if (enc2remove != null) enc2remove.opensearchIndex(); + if (enc2remove != null) enc2remove.opensearchIndexDeep(); out.println(ServletUtilities.getHeader(request)); response.setStatus(HttpServletResponse.SC_OK); out.println("Success: Encounter #" + diff --git a/src/main/java/org/ecocean/servlet/OccurrenceRemoveEncounter.java b/src/main/java/org/ecocean/servlet/OccurrenceRemoveEncounter.java index 427461fa89..b6dc7c135e 100644 --- a/src/main/java/org/ecocean/servlet/OccurrenceRemoveEncounter.java +++ b/src/main/java/org/ecocean/servlet/OccurrenceRemoveEncounter.java @@ -83,7 +83,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) if (!locked) { myShepherd.commitDBTransaction(); // out.println(ServletUtilities.getHeader(request)); - if (enc2remove != null) enc2remove.opensearchIndex(); + if (enc2remove != null) enc2remove.opensearchIndexDeep(); out.println("Success: Encounter " + request.getParameter("number") + " was successfully removed from occurrence " + old_name + "."); diff --git a/src/main/resources/bundles/OpenSearch.properties b/src/main/resources/bundles/OpenSearch.properties new file mode 100644 index 0000000000..27beb9dc8d --- /dev/null +++ b/src/main/resources/bundles/OpenSearch.properties @@ -0,0 +1,7 @@ +# these might need adjusting if background indexing seems to be taking longer than the delay time +#backgroundDelayMinutes=20 +#backgroundSliceSize=2500 + +# these probably should not be adjusted +#searchScrollTime=10m +#searchPitTime=10m diff --git a/src/main/webapp/appadmin/opensearchInfo.jsp b/src/main/webapp/appadmin/opensearchInfo.jsp index a031b34c46..0c2e5d57cd 100644 --- a/src/main/webapp/appadmin/opensearchInfo.jsp +++ b/src/main/webapp/appadmin/opensearchInfo.jsp @@ -26,6 +26,11 @@ private String wrap(final String input, int len) { Shepherd myShepherd = new Shepherd(request); OpenSearch os = new OpenSearch(); +out.println("

SEARCH_SCROLL_TIME=" + os.SEARCH_SCROLL_TIME + "
"); +out.println("SEARCH_PIT_TIME=" + os.SEARCH_PIT_TIME + "
"); +out.println("BACKGROUND_DELAY_MINUTES=" + os.BACKGROUND_DELAY_MINUTES + "
"); +out.println("BACKGROUND_SLICE_SIZE=" + os.BACKGROUND_SLICE_SIZE + "

"); + Request req = new Request("GET", "_cat/indices?v"); //req.setJsonEntity(query.toString()); String rtn = os.getRestResponse(req); @@ -73,9 +78,6 @@ out.println(res.toString(4)); -myShepherd.rollbackAndClose(); - - myShepherd.rollbackAndClose();