From 52df8e121f07064ece83818feb1aa299e1d8bfc2 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 6 Dec 2024 13:00:05 -0700 Subject: [PATCH 01/12] wip --- src/main/java/org/ecocean/OpenSearch.java | 35 +++++++++++++++++++++ src/main/webapp/appadmin/opensearchInfo.jsp | 5 ++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 64b3c78be7..22bf07e1e3 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -9,6 +9,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import javax.jdo.Query; import javax.net.ssl.SSLContext; @@ -670,4 +671,38 @@ public static JSONObject queryScrubStored(final JSONObject query) { scrubbed.put("query", query.optJSONObject("query")); return scrubbed; } + + public static Object getConfigurationValue(String context, String key, Object defaultValue) { + if (key == null) return null; + Properties props = getConfigurationProperties(context); + if (props == null) { + System.out.println( + "OpenSearch.getConfigurationValue(): WARNING could not get properties file; using defaultValue [" + + defaultValue + "] for " + key); + return defaultValue; + } + String propValue = props.getProperty(key); + // TODO can we actually set a NULL from a properties file? if so: we need to return that as null here + if (propValue == null) return defaultValue; + if (defaultValue instanceof Integer) { // get int from string + try { + return Integer.parseInt(propValue); + } catch (NumberFormatException nfe) { + return defaultValue; + } + } + if (defaultValue instanceof Double) { // get int from string + try { + return Double.parseDouble(propValue); + } catch (NumberFormatException nfe) { + return defaultValue; + } + } + // guess we are just a string + return propValue; + } + + static Properties getConfigurationProperties(String context) { + return ShepherdProperties.getProperties("OpenSearch.properties", "", context); + } } diff --git a/src/main/webapp/appadmin/opensearchInfo.jsp b/src/main/webapp/appadmin/opensearchInfo.jsp index a031b34c46..d5716b6837 100644 --- a/src/main/webapp/appadmin/opensearchInfo.jsp +++ b/src/main/webapp/appadmin/opensearchInfo.jsp @@ -26,6 +26,8 @@ private String wrap(final String input, int len) { Shepherd myShepherd = new Shepherd(request); OpenSearch os = new OpenSearch(); +out.println("

fubar=" + os.getConfigurationValue("context0", "fubar", 1234) + "

"); + Request req = new Request("GET", "_cat/indices?v"); //req.setJsonEntity(query.toString()); String rtn = os.getRestResponse(req); @@ -73,9 +75,6 @@ out.println(res.toString(4)); -myShepherd.rollbackAndClose(); - - myShepherd.rollbackAndClose(); From d803747ef5e51047741fd676b501ebb59c55709c Mon Sep 17 00:00:00 2001 From: holmbergius Date: Fri, 6 Dec 2024 12:07:17 -0800 Subject: [PATCH 02/12] Allow for reuse of single Shepherd to speed indexing --- src/main/java/org/ecocean/Base.java | 26 +- src/main/java/org/ecocean/Encounter.java | 527 ++++++++++++----------- 2 files changed, 290 insertions(+), 263 deletions(-) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index 3af6492495..6d03ab31ad 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -135,12 +135,9 @@ public void opensearchUnindexDeep() } // should be overridden - public void opensearchDocumentSerializer(JsonGenerator jgen) + public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd) throws IOException, JsonProcessingException { - Shepherd myShepherd = new Shepherd("context0"); - myShepherd.setAction("BaseSerializer"); - myShepherd.beginDBTransaction(); jgen.writeStringField("id", this.getId()); jgen.writeNumberField("version", this.getVersion()); @@ -157,9 +154,26 @@ public void opensearchDocumentSerializer(JsonGenerator jgen) jgen.writeString(id); } jgen.writeEndArray(); - myShepherd.rollbackDBTransaction(); - myShepherd.closeDBTransaction(); + + } + + public void opensearchDocumentSerializer(JsonGenerator jgen) + throws IOException, JsonProcessingException { + + Shepherd myShepherd = new Shepherd("context0"); + + myShepherd.setAction("BaseSerializer"); + myShepherd.beginDBTransaction(); + try { + opensearchDocumentSerializer(jgen, myShepherd); + } + catch(Exception e) {} + finally { + myShepherd.rollbackAndClose(); + } + } + public static JSONObject opensearchQuery(final String indexname, final JSONObject query, int numFrom, int pageSize, String sort, String sortOrder) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index d69911f42c..d2ad5ffb98 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -3857,267 +3857,280 @@ public static org.json.JSONObject opensearchQuery(final org.json.JSONObject quer throws IOException { return Base.opensearchQuery("encounter", query, numFrom, pageSize, sort, sortOrder); } - + public void opensearchDocumentSerializer(JsonGenerator jgen) + throws IOException, JsonProcessingException { + Shepherd myShepherd = new Shepherd("context0"); + myShepherd.setAction("Encounter.opensearchDocumentSerializer"); + myShepherd.beginDBTransaction(); + try { + opensearchDocumentSerializer(jgen, myShepherd); + } + catch(Exception e) {} + finally { + myShepherd.rollbackAndClose(); + } + + } + + public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd) throws IOException, JsonProcessingException { - super.opensearchDocumentSerializer(jgen); - Shepherd myShepherd = new Shepherd("context0"); - myShepherd.setAction("Encounter.opensearchDocumentSerializer"); - myShepherd.beginDBTransaction(); - - jgen.writeStringField("locationId", this.getLocationID()); - jgen.writeStringField("locationName", this.getLocationName()); - Long dim = this.getDateInMillisecondsFallback(); - if (dim != null) jgen.writeNumberField("dateMillis", dim); - String date = Util.getISO8601Date(this.getDate()); - if (date != null) jgen.writeStringField("date", date); - date = Util.getISO8601Date(this.getDWCDateAdded()); - if (date != null) jgen.writeStringField("dateSubmitted", date); - jgen.writeStringField("verbatimEventDate", this.getVerbatimEventDate()); - jgen.writeStringField("sex", this.getSex()); - jgen.writeStringField("taxonomy", this.getTaxonomyString()); - jgen.writeStringField("lifeStage", this.getLifeStage()); - jgen.writeStringField("livingStatus", this.getLivingStatus()); - jgen.writeStringField("verbatimLocality", this.getVerbatimLocality()); - jgen.writeStringField("country", this.getCountry()); - jgen.writeStringField("behavior", this.getBehavior()); - jgen.writeStringField("patterningCode", this.getPatterningCode()); - jgen.writeStringField("state", this.getState()); - jgen.writeStringField("occurrenceRemarks", this.getOccurrenceRemarks()); - jgen.writeStringField("otherCatalogNumbers", this.getOtherCatalogNumbers()); - - String featuredAssetId = null; - List mas = this.getMedia(); - jgen.writeNumberField("numberAnnotations", this.numNonTrivialAnnotations()); - jgen.writeNumberField("numberMediaAssets", mas.size()); - jgen.writeArrayFieldStart("mediaAssets"); - for (MediaAsset ma : mas) { - jgen.writeStartObject(); - jgen.writeNumberField("id", ma.getId()); - jgen.writeStringField("uuid", ma.getUUID()); - java.net.URL url = ma.safeURL(myShepherd); - if (url != null) jgen.writeStringField("url", url.toString()); - jgen.writeEndObject(); - if (featuredAssetId == null) featuredAssetId = ma.getUUID(); - } - jgen.writeEndArray(); - if (featuredAssetId != null) jgen.writeStringField("featuredAssetUuid", featuredAssetId); - if (this.submitterID == null) { - jgen.writeNullField("assignedUsername"); - } else { - jgen.writeStringField("assignedUsername", this.submitterID); - } - jgen.writeArrayFieldStart("submitters"); - for (String id : this.getAllSubmitterIds(myShepherd)) { - jgen.writeString(id); - } - jgen.writeEndArray(); - jgen.writeArrayFieldStart("photographers"); - for (String id : this.getAllPhotographerIds()) { - jgen.writeString(id); - } - jgen.writeEndArray(); - jgen.writeArrayFieldStart("informOthers"); - for (String id : this.getAllInformOtherIds()) { - jgen.writeString(id); - } - jgen.writeEndArray(); - - List kws = new ArrayList(); - Map lkws = new HashMap(); - for (Keyword kw : this.getMediaAssetKeywords()) { - if (kw instanceof LabeledKeyword) { - LabeledKeyword lkw = (LabeledKeyword)kw; - lkws.put(lkw.getLabel(), lkw.getValue()); - } else { - String name = kw.getDisplayName(); - if (!kws.contains(name)) kws.add(name); - } - } - jgen.writeArrayFieldStart("mediaAssetKeywords"); - for (String kw : kws) { - jgen.writeString(kw); - } - jgen.writeEndArray(); - jgen.writeObjectFieldStart("mediaAssetLabeledKeywords"); - for (String kwLabel : lkws.keySet()) { - jgen.writeStringField(kwLabel, lkws.get(kwLabel)); - } - jgen.writeEndObject(); - - List projs = this.getProjects(myShepherd); - jgen.writeArrayFieldStart("projects"); - if (projs != null) - for (Project proj : projs) { - jgen.writeString(proj.getId()); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("annotationViewpoints"); - for (String vp : this.getAnnotationViewpoints()) { - jgen.writeString(vp); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("annotationIAClasses"); - for (String cls : this.getAnnotationIAClasses()) { - jgen.writeString(cls); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("measurements"); - if (this.measurements != null) - for (Measurement meas : this.measurements) { - if (meas.getValue() == null) continue; // no value means we should skip - jgen.writeStartObject(); - jgen.writeNumberField("value", meas.getValue()); - if (meas.getType() != null) jgen.writeStringField("type", meas.getType()); - if (meas.getUnits() != null) jgen.writeStringField("units", meas.getUnits()); - if (meas.getSamplingProtocol() != null) - jgen.writeStringField("samplingProtocol", meas.getSamplingProtocol()); - jgen.writeEndObject(); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("metalTags"); - if (this.getMetalTags() != null) - for (MetalTag tag : this.getMetalTags()) { - jgen.writeStartObject(); - jgen.writeStringField("number", tag.getTagNumber()); - jgen.writeStringField("location", tag.getLocation()); - jgen.writeEndObject(); - } - jgen.writeEndArray(); - if (this.getAcousticTag() != null) { - jgen.writeObjectFieldStart("acousticTag"); - jgen.writeStringField("idNumber", this.getAcousticTag().getIdNumber()); - jgen.writeStringField("serialNumber", this.getAcousticTag().getSerialNumber()); - jgen.writeEndObject(); - } - if (this.getSatelliteTag() != null) { - jgen.writeObjectFieldStart("satelliteTag"); - jgen.writeStringField("name", this.getSatelliteTag().getName()); - jgen.writeStringField("serialNumber", this.getSatelliteTag().getSerialNumber()); - jgen.writeStringField("argosPttNumber", this.getSatelliteTag().getArgosPttNumber()); - jgen.writeEndObject(); - } - if (this.getDTag() != null) { - jgen.writeObjectFieldStart("digitalArchiveTag"); - jgen.writeStringField("dTagID", this.getDTag().getDTagID()); - jgen.writeStringField("serialNumber", this.getDTag().getSerialNumber()); - jgen.writeEndObject(); - } - org.json.JSONObject dpj = this.getDynamicPropertiesJSONObject(); - jgen.writeObjectFieldStart("dynamicProperties"); - for (String key : (Set)dpj.keySet()) { - jgen.writeStringField(key, dpj.optString(key, null)); - } - jgen.writeEndObject(); - - Double dlat = this.getDecimalLatitudeAsDouble(); - Double dlon = this.getDecimalLongitudeAsDouble(); - if ((dlat == null) || (dlon == null)) { - jgen.writeNullField("locationGeoPoint"); - } else { - jgen.writeObjectFieldStart("locationGeoPoint"); - jgen.writeNumberField("lat", dlat); - jgen.writeNumberField("lon", dlon); - jgen.writeEndObject(); - } - MarkedIndividual indiv = this.getIndividual(); - if (indiv == null) { - jgen.writeNullField("individualId"); - } else { - jgen.writeStringField("individualId", indiv.getId()); - jgen.writeStringField("individualSex", indiv.getSex()); - jgen.writeNumberField("individualNumberEncounters", indiv.getNumEncounters()); - jgen.writeStringField("individualDisplayName", indiv.getDisplayName()); - jgen.writeArrayFieldStart("individualNames"); - Set names = indiv.getAllNamesList(); - if (names != null) - for (String name : names) { - jgen.writeString(name); - } - jgen.writeEndArray(); - jgen.writeStringField("individualNickName", indiv.getNickName()); - if (indiv.getTimeOfBirth() > 0) { - String birthTime = Util.getISO8601Date(new DateTime( - indiv.getTimeOfBirth()).toString()); - jgen.writeStringField("individualTimeOfBirth", birthTime); - } - Encounter[] encs = indiv.getDateSortedEncounters(true); - if ((encs != null) && (encs.length > 0)) { - String encDate = Util.getISO8601Date(encs[0].getDate()); - if (encDate != null) jgen.writeStringField("individualFirstEncounterDate", encDate); - encDate = Util.getISO8601Date(encs[encs.length - 1].getDate()); - if (encDate != null) jgen.writeStringField("individualLastEncounterDate", encDate); - } - - jgen.writeArrayFieldStart("individualSocialUnits"); - for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(indiv)) { - Membership mem = su.getMembershipForMarkedIndividual(indiv); - if (mem != null) jgen.writeString(su.getSocialUnitName()); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("individualRelationshipRoles"); - for (String relRole : myShepherd.getAllRoleNamesForMarkedIndividual(indiv.getId())) { - jgen.writeString(relRole); - } - jgen.writeEndArray(); - } - DateTime occdt = getOccurrenceDateTime(myShepherd); - if (occdt != null) jgen.writeStringField("occurrenceDate", occdt.toString()); - jgen.writeStringField("geneticSex", this.getGeneticSex()); - jgen.writeStringField("haplotype", this.getHaplotype()); - - jgen.writeArrayFieldStart("microsatelliteMarkers"); - for (MicrosatelliteMarkersAnalysis msm : getMicrosatelliteMarkers()) { - jgen.writeStartObject(); - jgen.writeStringField("analysisId", msm.getAnalysisID()); - jgen.writeObjectFieldStart("loci"); - if (msm.getLoci() != null) - for (Locus locus : msm.getLoci()) { - if (locus.getName() == null) continue; // snh - jgen.writeObjectFieldStart(locus.getName()); - for (int i = 0; i < 4; i++) { - Integer allele = locus.getAllele(i); - if (allele != null) jgen.writeNumberField("allele" + i, allele); - } - jgen.writeEndObject(); - } - jgen.writeEndObject(); - jgen.writeEndObject(); - } - jgen.writeEndArray(); - Occurrence occ = this.getOccurrence(myShepherd); - if (occ == null) { - jgen.writeNullField("occurrenceId"); - } else { - jgen.writeStringField("occurrenceId", occ.getId()); - } - jgen.writeArrayFieldStart("organizations"); - User owner = this.getSubmitterUser(myShepherd); - if ((owner != null) && (owner.getOrganizations() != null)) - for (Organization org : owner.getOrganizations()) { - jgen.writeString(org.getId()); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("tissueSampleIds"); - for (String id : this.getTissueSampleIDs()) { - jgen.writeString(id); - } - jgen.writeEndArray(); + super.opensearchDocumentSerializer(jgen,myShepherd); + + jgen.writeStringField("locationId", this.getLocationID()); + jgen.writeStringField("locationName", this.getLocationName()); + Long dim = this.getDateInMillisecondsFallback(); + if (dim != null) jgen.writeNumberField("dateMillis", dim); + String date = Util.getISO8601Date(this.getDate()); + if (date != null) jgen.writeStringField("date", date); + date = Util.getISO8601Date(this.getDWCDateAdded()); + if (date != null) jgen.writeStringField("dateSubmitted", date); + jgen.writeStringField("verbatimEventDate", this.getVerbatimEventDate()); + jgen.writeStringField("sex", this.getSex()); + jgen.writeStringField("taxonomy", this.getTaxonomyString()); + jgen.writeStringField("lifeStage", this.getLifeStage()); + jgen.writeStringField("livingStatus", this.getLivingStatus()); + jgen.writeStringField("verbatimLocality", this.getVerbatimLocality()); + jgen.writeStringField("country", this.getCountry()); + jgen.writeStringField("behavior", this.getBehavior()); + jgen.writeStringField("patterningCode", this.getPatterningCode()); + jgen.writeStringField("state", this.getState()); + jgen.writeStringField("occurrenceRemarks", this.getOccurrenceRemarks()); + jgen.writeStringField("otherCatalogNumbers", this.getOtherCatalogNumbers()); + + String featuredAssetId = null; + List mas = this.getMedia(); + jgen.writeNumberField("numberAnnotations", this.numNonTrivialAnnotations()); + jgen.writeNumberField("numberMediaAssets", mas.size()); + jgen.writeArrayFieldStart("mediaAssets"); + for (MediaAsset ma : mas) { + jgen.writeStartObject(); + jgen.writeNumberField("id", ma.getId()); + jgen.writeStringField("uuid", ma.getUUID()); + java.net.URL url = ma.safeURL(myShepherd); + if (url != null) jgen.writeStringField("url", url.toString()); + jgen.writeEndObject(); + if (featuredAssetId == null) featuredAssetId = ma.getUUID(); + } + jgen.writeEndArray(); + if (featuredAssetId != null) jgen.writeStringField("featuredAssetUuid", featuredAssetId); + if (this.submitterID == null) { + jgen.writeNullField("assignedUsername"); + } else { + jgen.writeStringField("assignedUsername", this.submitterID); + } + jgen.writeArrayFieldStart("submitters"); + for (String id : this.getAllSubmitterIds(myShepherd)) { + jgen.writeString(id); + } + jgen.writeEndArray(); + jgen.writeArrayFieldStart("photographers"); + for (String id : this.getAllPhotographerIds()) { + jgen.writeString(id); + } + jgen.writeEndArray(); + jgen.writeArrayFieldStart("informOthers"); + for (String id : this.getAllInformOtherIds()) { + jgen.writeString(id); + } + jgen.writeEndArray(); + + List kws = new ArrayList(); + Map lkws = new HashMap(); + for (Keyword kw : this.getMediaAssetKeywords()) { + if (kw instanceof LabeledKeyword) { + LabeledKeyword lkw = (LabeledKeyword)kw; + lkws.put(lkw.getLabel(), lkw.getValue()); + } else { + String name = kw.getDisplayName(); + if (!kws.contains(name)) kws.add(name); + } + } + jgen.writeArrayFieldStart("mediaAssetKeywords"); + for (String kw : kws) { + jgen.writeString(kw); + } + jgen.writeEndArray(); + jgen.writeObjectFieldStart("mediaAssetLabeledKeywords"); + for (String kwLabel : lkws.keySet()) { + jgen.writeStringField(kwLabel, lkws.get(kwLabel)); + } + jgen.writeEndObject(); + + List projs = this.getProjects(myShepherd); + jgen.writeArrayFieldStart("projects"); + if (projs != null) + for (Project proj : projs) { + jgen.writeString(proj.getId()); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("annotationViewpoints"); + for (String vp : this.getAnnotationViewpoints()) { + jgen.writeString(vp); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("annotationIAClasses"); + for (String cls : this.getAnnotationIAClasses()) { + jgen.writeString(cls); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("measurements"); + if (this.measurements != null) + for (Measurement meas : this.measurements) { + if (meas.getValue() == null) continue; // no value means we should skip + jgen.writeStartObject(); + jgen.writeNumberField("value", meas.getValue()); + if (meas.getType() != null) jgen.writeStringField("type", meas.getType()); + if (meas.getUnits() != null) jgen.writeStringField("units", meas.getUnits()); + if (meas.getSamplingProtocol() != null) + jgen.writeStringField("samplingProtocol", meas.getSamplingProtocol()); + jgen.writeEndObject(); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("metalTags"); + if (this.getMetalTags() != null) + for (MetalTag tag : this.getMetalTags()) { + jgen.writeStartObject(); + jgen.writeStringField("number", tag.getTagNumber()); + jgen.writeStringField("location", tag.getLocation()); + jgen.writeEndObject(); + } + jgen.writeEndArray(); + if (this.getAcousticTag() != null) { + jgen.writeObjectFieldStart("acousticTag"); + jgen.writeStringField("idNumber", this.getAcousticTag().getIdNumber()); + jgen.writeStringField("serialNumber", this.getAcousticTag().getSerialNumber()); + jgen.writeEndObject(); + } + if (this.getSatelliteTag() != null) { + jgen.writeObjectFieldStart("satelliteTag"); + jgen.writeStringField("name", this.getSatelliteTag().getName()); + jgen.writeStringField("serialNumber", this.getSatelliteTag().getSerialNumber()); + jgen.writeStringField("argosPttNumber", this.getSatelliteTag().getArgosPttNumber()); + jgen.writeEndObject(); + } + if (this.getDTag() != null) { + jgen.writeObjectFieldStart("digitalArchiveTag"); + jgen.writeStringField("dTagID", this.getDTag().getDTagID()); + jgen.writeStringField("serialNumber", this.getDTag().getSerialNumber()); + jgen.writeEndObject(); + } + org.json.JSONObject dpj = this.getDynamicPropertiesJSONObject(); + jgen.writeObjectFieldStart("dynamicProperties"); + for (String key : (Set)dpj.keySet()) { + jgen.writeStringField(key, dpj.optString(key, null)); + } + jgen.writeEndObject(); + + Double dlat = this.getDecimalLatitudeAsDouble(); + Double dlon = this.getDecimalLongitudeAsDouble(); + if ((dlat == null) || (dlon == null)) { + jgen.writeNullField("locationGeoPoint"); + } else { + jgen.writeObjectFieldStart("locationGeoPoint"); + jgen.writeNumberField("lat", dlat); + jgen.writeNumberField("lon", dlon); + jgen.writeEndObject(); + } + MarkedIndividual indiv = this.getIndividual(); + if (indiv == null) { + jgen.writeNullField("individualId"); + } else { + jgen.writeStringField("individualId", indiv.getId()); + jgen.writeStringField("individualSex", indiv.getSex()); + jgen.writeNumberField("individualNumberEncounters", indiv.getNumEncounters()); + jgen.writeStringField("individualDisplayName", indiv.getDisplayName()); + jgen.writeArrayFieldStart("individualNames"); + Set names = indiv.getAllNamesList(); + if (names != null) + for (String name : names) { + jgen.writeString(name); + } + jgen.writeEndArray(); + jgen.writeStringField("individualNickName", indiv.getNickName()); + if (indiv.getTimeOfBirth() > 0) { + String birthTime = Util.getISO8601Date(new DateTime( + indiv.getTimeOfBirth()).toString()); + jgen.writeStringField("individualTimeOfBirth", birthTime); + } + Encounter[] encs = indiv.getDateSortedEncounters(true); + if ((encs != null) && (encs.length > 0)) { + String encDate = Util.getISO8601Date(encs[0].getDate()); + if (encDate != null) jgen.writeStringField("individualFirstEncounterDate", encDate); + encDate = Util.getISO8601Date(encs[encs.length - 1].getDate()); + if (encDate != null) jgen.writeStringField("individualLastEncounterDate", encDate); + } + + jgen.writeArrayFieldStart("individualSocialUnits"); + for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(indiv)) { + Membership mem = su.getMembershipForMarkedIndividual(indiv); + if (mem != null) jgen.writeString(su.getSocialUnitName()); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("individualRelationshipRoles"); + for (String relRole : myShepherd.getAllRoleNamesForMarkedIndividual(indiv.getId())) { + jgen.writeString(relRole); + } + jgen.writeEndArray(); + } + DateTime occdt = getOccurrenceDateTime(myShepherd); + if (occdt != null) jgen.writeStringField("occurrenceDate", occdt.toString()); + jgen.writeStringField("geneticSex", this.getGeneticSex()); + jgen.writeStringField("haplotype", this.getHaplotype()); + + jgen.writeArrayFieldStart("microsatelliteMarkers"); + for (MicrosatelliteMarkersAnalysis msm : getMicrosatelliteMarkers()) { + jgen.writeStartObject(); + jgen.writeStringField("analysisId", msm.getAnalysisID()); + jgen.writeObjectFieldStart("loci"); + if (msm.getLoci() != null) + for (Locus locus : msm.getLoci()) { + if (locus.getName() == null) continue; // snh + jgen.writeObjectFieldStart(locus.getName()); + for (int i = 0; i < 4; i++) { + Integer allele = locus.getAllele(i); + if (allele != null) jgen.writeNumberField("allele" + i, allele); + } + jgen.writeEndObject(); + } + jgen.writeEndObject(); + jgen.writeEndObject(); + } + jgen.writeEndArray(); + + Occurrence occ = this.getOccurrence(myShepherd); + if (occ == null) { + jgen.writeNullField("occurrenceId"); + } else { + jgen.writeStringField("occurrenceId", occ.getId()); + } + jgen.writeArrayFieldStart("organizations"); + User owner = this.getSubmitterUser(myShepherd); + if ((owner != null) && (owner.getOrganizations() != null)) + for (Organization org : owner.getOrganizations()) { + jgen.writeString(org.getId()); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("tissueSampleIds"); + for (String id : this.getTissueSampleIDs()) { + jgen.writeString(id); + } + jgen.writeEndArray(); + + Map bmeas = this.getBiologicalMeasurementsByType(); + jgen.writeObjectFieldStart("biologicalMeasurements"); + for (String type : (Set)bmeas.keySet()) { + jgen.writeNumberField(type, bmeas.get(type).getValue()); + } + jgen.writeEndObject(); - Map bmeas = this.getBiologicalMeasurementsByType(); - jgen.writeObjectFieldStart("biologicalMeasurements"); - for (String type : (Set)bmeas.keySet()) { - jgen.writeNumberField(type, bmeas.get(type).getValue()); - } - jgen.writeEndObject(); - myShepherd.rollbackAndClose(); } @Override public long getVersion() { From 4095265716885886d4f1a37db91c2c9898d6ea48 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 6 Dec 2024 13:18:08 -0700 Subject: [PATCH 03/12] config settings --- src/main/java/org/ecocean/OpenSearch.java | 15 +++++++++++---- src/main/resources/bundles/OpenSearch.properties | 7 +++++++ src/main/webapp/appadmin/opensearchInfo.jsp | 5 ++++- 3 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 src/main/resources/bundles/OpenSearch.properties diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 22bf07e1e3..8155cfcd1f 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -57,12 +57,15 @@ public class OpenSearch { public static RestClient restClient = null; public static Map INDEX_EXISTS_CACHE = new HashMap(); public static Map PIT_CACHE = new HashMap(); - public static String SEARCH_SCROLL_TIME = "10m"; - public static String SEARCH_PIT_TIME = "10m"; + public static String SEARCH_SCROLL_TIME = (String)getConfigurationValue("searchScrollTime", + "10m"); + public static String SEARCH_PIT_TIME = (String)getConfigurationValue("searchPitTime", "10m"); public static String INDEX_TIMESTAMP_PREFIX = "OpenSearch_index_timestamp_"; public static String[] VALID_INDICES = { "encounter", "individual", "occurrence" }; - public static int BACKGROUND_DELAY_MINUTES = 20; - public static int BACKGROUND_SLICE_SIZE = 2500; + public static int BACKGROUND_DELAY_MINUTES = (Integer)getConfigurationValue( + "backgroundDelayMinutes", 20); + public static int BACKGROUND_SLICE_SIZE = (Integer)getConfigurationValue("backgroundSliceSize", + 2500); public static String QUERY_STORAGE_DIR = "/tmp"; // FIXME private int pitRetry = 0; @@ -672,6 +675,10 @@ public static JSONObject queryScrubStored(final JSONObject query) { return scrubbed; } + public static Object getConfigurationValue(String key, Object defaultValue) { + return getConfigurationValue("context0", key, defaultValue); + } + public static Object getConfigurationValue(String context, String key, Object defaultValue) { if (key == null) return null; Properties props = getConfigurationProperties(context); diff --git a/src/main/resources/bundles/OpenSearch.properties b/src/main/resources/bundles/OpenSearch.properties new file mode 100644 index 0000000000..27beb9dc8d --- /dev/null +++ b/src/main/resources/bundles/OpenSearch.properties @@ -0,0 +1,7 @@ +# these might need adjusting if background indexing seems to be taking longer than the delay time +#backgroundDelayMinutes=20 +#backgroundSliceSize=2500 + +# these probably should not be adjusted +#searchScrollTime=10m +#searchPitTime=10m diff --git a/src/main/webapp/appadmin/opensearchInfo.jsp b/src/main/webapp/appadmin/opensearchInfo.jsp index d5716b6837..0c2e5d57cd 100644 --- a/src/main/webapp/appadmin/opensearchInfo.jsp +++ b/src/main/webapp/appadmin/opensearchInfo.jsp @@ -26,7 +26,10 @@ private String wrap(final String input, int len) { Shepherd myShepherd = new Shepherd(request); OpenSearch os = new OpenSearch(); -out.println("

fubar=" + os.getConfigurationValue("context0", "fubar", 1234) + "

"); +out.println("

SEARCH_SCROLL_TIME=" + os.SEARCH_SCROLL_TIME + "
"); +out.println("SEARCH_PIT_TIME=" + os.SEARCH_PIT_TIME + "
"); +out.println("BACKGROUND_DELAY_MINUTES=" + os.BACKGROUND_DELAY_MINUTES + "
"); +out.println("BACKGROUND_SLICE_SIZE=" + os.BACKGROUND_SLICE_SIZE + "

"); Request req = new Request("GET", "_cat/indices?v"); //req.setJsonEntity(query.toString()); From 1709b5efa4eba3309d822758cc9bf0c2c276be4b Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 6 Dec 2024 13:28:31 -0700 Subject: [PATCH 04/12] linting --- src/main/java/org/ecocean/Base.java | 27 +- src/main/java/org/ecocean/Encounter.java | 544 +++++++++++------------ 2 files changed, 279 insertions(+), 292 deletions(-) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index 6d03ab31ad..2eb477c6f2 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -137,7 +137,6 @@ public void opensearchUnindexDeep() // should be overridden public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd) throws IOException, JsonProcessingException { - jgen.writeStringField("id", this.getId()); jgen.writeNumberField("version", this.getVersion()); @@ -154,26 +153,20 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeString(id); } jgen.writeEndArray(); - } - + public void opensearchDocumentSerializer(JsonGenerator jgen) throws IOException, JsonProcessingException { - - Shepherd myShepherd = new Shepherd("context0"); - - myShepherd.setAction("BaseSerializer"); - myShepherd.beginDBTransaction(); - try { - opensearchDocumentSerializer(jgen, myShepherd); - } - catch(Exception e) {} - finally { - myShepherd.rollbackAndClose(); - } - + Shepherd myShepherd = new Shepherd("context0"); + + myShepherd.setAction("BaseSerializer"); + myShepherd.beginDBTransaction(); + try { + opensearchDocumentSerializer(jgen, myShepherd); + } catch (Exception e) {} finally { + myShepherd.rollbackAndClose(); + } } - public static JSONObject opensearchQuery(final String indexname, final JSONObject query, int numFrom, int pageSize, String sort, String sortOrder) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index d2ad5ffb98..6355dfea15 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -154,7 +154,6 @@ public void setSampleTakenForDiet(Boolean sampleTakenForDiet) { private static HashMap > _matchEncounterCache = new HashMap >(); - // An URL to a thumbnail image representing the encounter. private String dwcImageURL; @@ -678,13 +677,12 @@ public boolean hasRightSpotImage() { return (this.getNumRightSpots() > 0); } - // Sets the recorded length of the shark for this encounter. public void setSize(Double mysize) { if (mysize != null) { size = mysize; } else { size = null; } } - // @return the length of the shark + // @return the length of the shark public double getSize() { return size.doubleValue(); } @@ -2450,9 +2448,10 @@ public void setTissueSamples(List samps) { public Set getTissueSampleIDs() { Set ids = new HashSet(); - if (tissueSamples != null) for (TissueSample ts : tissueSamples) { - ids.add(ts.getSampleID()); - } + if (tissueSamples != null) + for (TissueSample ts : tissueSamples) { + ids.add(ts.getSampleID()); + } return ids; } @@ -3857,280 +3856,275 @@ public static org.json.JSONObject opensearchQuery(final org.json.JSONObject quer throws IOException { return Base.opensearchQuery("encounter", query, numFrom, pageSize, sort, sortOrder); } - + public void opensearchDocumentSerializer(JsonGenerator jgen) - throws IOException, JsonProcessingException { - Shepherd myShepherd = new Shepherd("context0"); - myShepherd.setAction("Encounter.opensearchDocumentSerializer"); - myShepherd.beginDBTransaction(); - try { - opensearchDocumentSerializer(jgen, myShepherd); - } - catch(Exception e) {} - finally { - myShepherd.rollbackAndClose(); - } - - } - + throws IOException, JsonProcessingException { + Shepherd myShepherd = new Shepherd("context0"); + + myShepherd.setAction("Encounter.opensearchDocumentSerializer"); + myShepherd.beginDBTransaction(); + try { + opensearchDocumentSerializer(jgen, myShepherd); + } catch (Exception e) {} finally { + myShepherd.rollbackAndClose(); + } + } + public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd) throws IOException, JsonProcessingException { + super.opensearchDocumentSerializer(jgen, myShepherd); + + jgen.writeStringField("locationId", this.getLocationID()); + jgen.writeStringField("locationName", this.getLocationName()); + Long dim = this.getDateInMillisecondsFallback(); + if (dim != null) jgen.writeNumberField("dateMillis", dim); + String date = Util.getISO8601Date(this.getDate()); + if (date != null) jgen.writeStringField("date", date); + date = Util.getISO8601Date(this.getDWCDateAdded()); + if (date != null) jgen.writeStringField("dateSubmitted", date); + jgen.writeStringField("verbatimEventDate", this.getVerbatimEventDate()); + jgen.writeStringField("sex", this.getSex()); + jgen.writeStringField("taxonomy", this.getTaxonomyString()); + jgen.writeStringField("lifeStage", this.getLifeStage()); + jgen.writeStringField("livingStatus", this.getLivingStatus()); + jgen.writeStringField("verbatimLocality", this.getVerbatimLocality()); + jgen.writeStringField("country", this.getCountry()); + jgen.writeStringField("behavior", this.getBehavior()); + jgen.writeStringField("patterningCode", this.getPatterningCode()); + jgen.writeStringField("state", this.getState()); + jgen.writeStringField("occurrenceRemarks", this.getOccurrenceRemarks()); + jgen.writeStringField("otherCatalogNumbers", this.getOtherCatalogNumbers()); + + String featuredAssetId = null; + List mas = this.getMedia(); + jgen.writeNumberField("numberAnnotations", this.numNonTrivialAnnotations()); + jgen.writeNumberField("numberMediaAssets", mas.size()); + jgen.writeArrayFieldStart("mediaAssets"); + for (MediaAsset ma : mas) { + jgen.writeStartObject(); + jgen.writeNumberField("id", ma.getId()); + jgen.writeStringField("uuid", ma.getUUID()); + java.net.URL url = ma.safeURL(myShepherd); + if (url != null) jgen.writeStringField("url", url.toString()); + jgen.writeEndObject(); + if (featuredAssetId == null) featuredAssetId = ma.getUUID(); + } + jgen.writeEndArray(); + if (featuredAssetId != null) jgen.writeStringField("featuredAssetUuid", featuredAssetId); + if (this.submitterID == null) { + jgen.writeNullField("assignedUsername"); + } else { + jgen.writeStringField("assignedUsername", this.submitterID); + } + jgen.writeArrayFieldStart("submitters"); + for (String id : this.getAllSubmitterIds(myShepherd)) { + jgen.writeString(id); + } + jgen.writeEndArray(); + jgen.writeArrayFieldStart("photographers"); + for (String id : this.getAllPhotographerIds()) { + jgen.writeString(id); + } + jgen.writeEndArray(); + jgen.writeArrayFieldStart("informOthers"); + for (String id : this.getAllInformOtherIds()) { + jgen.writeString(id); + } + jgen.writeEndArray(); + + List kws = new ArrayList(); + Map lkws = new HashMap(); + for (Keyword kw : this.getMediaAssetKeywords()) { + if (kw instanceof LabeledKeyword) { + LabeledKeyword lkw = (LabeledKeyword)kw; + lkws.put(lkw.getLabel(), lkw.getValue()); + } else { + String name = kw.getDisplayName(); + if (!kws.contains(name)) kws.add(name); + } + } + jgen.writeArrayFieldStart("mediaAssetKeywords"); + for (String kw : kws) { + jgen.writeString(kw); + } + jgen.writeEndArray(); + jgen.writeObjectFieldStart("mediaAssetLabeledKeywords"); + for (String kwLabel : lkws.keySet()) { + jgen.writeStringField(kwLabel, lkws.get(kwLabel)); + } + jgen.writeEndObject(); + + List projs = this.getProjects(myShepherd); + jgen.writeArrayFieldStart("projects"); + if (projs != null) + for (Project proj : projs) { + jgen.writeString(proj.getId()); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("annotationViewpoints"); + for (String vp : this.getAnnotationViewpoints()) { + jgen.writeString(vp); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("annotationIAClasses"); + for (String cls : this.getAnnotationIAClasses()) { + jgen.writeString(cls); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("measurements"); + if (this.measurements != null) + for (Measurement meas : this.measurements) { + if (meas.getValue() == null) continue; // no value means we should skip + jgen.writeStartObject(); + jgen.writeNumberField("value", meas.getValue()); + if (meas.getType() != null) jgen.writeStringField("type", meas.getType()); + if (meas.getUnits() != null) jgen.writeStringField("units", meas.getUnits()); + if (meas.getSamplingProtocol() != null) + jgen.writeStringField("samplingProtocol", meas.getSamplingProtocol()); + jgen.writeEndObject(); + } + jgen.writeEndArray(); + + jgen.writeArrayFieldStart("metalTags"); + if (this.getMetalTags() != null) + for (MetalTag tag : this.getMetalTags()) { + jgen.writeStartObject(); + jgen.writeStringField("number", tag.getTagNumber()); + jgen.writeStringField("location", tag.getLocation()); + jgen.writeEndObject(); + } + jgen.writeEndArray(); + if (this.getAcousticTag() != null) { + jgen.writeObjectFieldStart("acousticTag"); + jgen.writeStringField("idNumber", this.getAcousticTag().getIdNumber()); + jgen.writeStringField("serialNumber", this.getAcousticTag().getSerialNumber()); + jgen.writeEndObject(); + } + if (this.getSatelliteTag() != null) { + jgen.writeObjectFieldStart("satelliteTag"); + jgen.writeStringField("name", this.getSatelliteTag().getName()); + jgen.writeStringField("serialNumber", this.getSatelliteTag().getSerialNumber()); + jgen.writeStringField("argosPttNumber", this.getSatelliteTag().getArgosPttNumber()); + jgen.writeEndObject(); + } + if (this.getDTag() != null) { + jgen.writeObjectFieldStart("digitalArchiveTag"); + jgen.writeStringField("dTagID", this.getDTag().getDTagID()); + jgen.writeStringField("serialNumber", this.getDTag().getSerialNumber()); + jgen.writeEndObject(); + } + org.json.JSONObject dpj = this.getDynamicPropertiesJSONObject(); + jgen.writeObjectFieldStart("dynamicProperties"); + for (String key : (Set)dpj.keySet()) { + jgen.writeStringField(key, dpj.optString(key, null)); + } + jgen.writeEndObject(); + + Double dlat = this.getDecimalLatitudeAsDouble(); + Double dlon = this.getDecimalLongitudeAsDouble(); + if ((dlat == null) || (dlon == null)) { + jgen.writeNullField("locationGeoPoint"); + } else { + jgen.writeObjectFieldStart("locationGeoPoint"); + jgen.writeNumberField("lat", dlat); + jgen.writeNumberField("lon", dlon); + jgen.writeEndObject(); + } + MarkedIndividual indiv = this.getIndividual(); + if (indiv == null) { + jgen.writeNullField("individualId"); + } else { + jgen.writeStringField("individualId", indiv.getId()); + jgen.writeStringField("individualSex", indiv.getSex()); + jgen.writeNumberField("individualNumberEncounters", indiv.getNumEncounters()); + jgen.writeStringField("individualDisplayName", indiv.getDisplayName()); + jgen.writeArrayFieldStart("individualNames"); + Set names = indiv.getAllNamesList(); + if (names != null) + for (String name : names) { + jgen.writeString(name); + } + jgen.writeEndArray(); + jgen.writeStringField("individualNickName", indiv.getNickName()); + if (indiv.getTimeOfBirth() > 0) { + String birthTime = Util.getISO8601Date(new DateTime( + indiv.getTimeOfBirth()).toString()); + jgen.writeStringField("individualTimeOfBirth", birthTime); + } + Encounter[] encs = indiv.getDateSortedEncounters(true); + if ((encs != null) && (encs.length > 0)) { + String encDate = Util.getISO8601Date(encs[0].getDate()); + if (encDate != null) jgen.writeStringField("individualFirstEncounterDate", encDate); + encDate = Util.getISO8601Date(encs[encs.length - 1].getDate()); + if (encDate != null) jgen.writeStringField("individualLastEncounterDate", encDate); + } + jgen.writeArrayFieldStart("individualSocialUnits"); + for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(indiv)) { + Membership mem = su.getMembershipForMarkedIndividual(indiv); + if (mem != null) jgen.writeString(su.getSocialUnitName()); + } + jgen.writeEndArray(); - super.opensearchDocumentSerializer(jgen,myShepherd); - - jgen.writeStringField("locationId", this.getLocationID()); - jgen.writeStringField("locationName", this.getLocationName()); - Long dim = this.getDateInMillisecondsFallback(); - if (dim != null) jgen.writeNumberField("dateMillis", dim); - String date = Util.getISO8601Date(this.getDate()); - if (date != null) jgen.writeStringField("date", date); - date = Util.getISO8601Date(this.getDWCDateAdded()); - if (date != null) jgen.writeStringField("dateSubmitted", date); - jgen.writeStringField("verbatimEventDate", this.getVerbatimEventDate()); - jgen.writeStringField("sex", this.getSex()); - jgen.writeStringField("taxonomy", this.getTaxonomyString()); - jgen.writeStringField("lifeStage", this.getLifeStage()); - jgen.writeStringField("livingStatus", this.getLivingStatus()); - jgen.writeStringField("verbatimLocality", this.getVerbatimLocality()); - jgen.writeStringField("country", this.getCountry()); - jgen.writeStringField("behavior", this.getBehavior()); - jgen.writeStringField("patterningCode", this.getPatterningCode()); - jgen.writeStringField("state", this.getState()); - jgen.writeStringField("occurrenceRemarks", this.getOccurrenceRemarks()); - jgen.writeStringField("otherCatalogNumbers", this.getOtherCatalogNumbers()); - - String featuredAssetId = null; - List mas = this.getMedia(); - jgen.writeNumberField("numberAnnotations", this.numNonTrivialAnnotations()); - jgen.writeNumberField("numberMediaAssets", mas.size()); - jgen.writeArrayFieldStart("mediaAssets"); - for (MediaAsset ma : mas) { - jgen.writeStartObject(); - jgen.writeNumberField("id", ma.getId()); - jgen.writeStringField("uuid", ma.getUUID()); - java.net.URL url = ma.safeURL(myShepherd); - if (url != null) jgen.writeStringField("url", url.toString()); - jgen.writeEndObject(); - if (featuredAssetId == null) featuredAssetId = ma.getUUID(); - } - jgen.writeEndArray(); - if (featuredAssetId != null) jgen.writeStringField("featuredAssetUuid", featuredAssetId); - if (this.submitterID == null) { - jgen.writeNullField("assignedUsername"); - } else { - jgen.writeStringField("assignedUsername", this.submitterID); - } - jgen.writeArrayFieldStart("submitters"); - for (String id : this.getAllSubmitterIds(myShepherd)) { - jgen.writeString(id); - } - jgen.writeEndArray(); - jgen.writeArrayFieldStart("photographers"); - for (String id : this.getAllPhotographerIds()) { - jgen.writeString(id); - } - jgen.writeEndArray(); - jgen.writeArrayFieldStart("informOthers"); - for (String id : this.getAllInformOtherIds()) { - jgen.writeString(id); - } - jgen.writeEndArray(); - - List kws = new ArrayList(); - Map lkws = new HashMap(); - for (Keyword kw : this.getMediaAssetKeywords()) { - if (kw instanceof LabeledKeyword) { - LabeledKeyword lkw = (LabeledKeyword)kw; - lkws.put(lkw.getLabel(), lkw.getValue()); - } else { - String name = kw.getDisplayName(); - if (!kws.contains(name)) kws.add(name); - } - } - jgen.writeArrayFieldStart("mediaAssetKeywords"); - for (String kw : kws) { - jgen.writeString(kw); - } - jgen.writeEndArray(); - jgen.writeObjectFieldStart("mediaAssetLabeledKeywords"); - for (String kwLabel : lkws.keySet()) { - jgen.writeStringField(kwLabel, lkws.get(kwLabel)); - } - jgen.writeEndObject(); - - List projs = this.getProjects(myShepherd); - jgen.writeArrayFieldStart("projects"); - if (projs != null) - for (Project proj : projs) { - jgen.writeString(proj.getId()); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("annotationViewpoints"); - for (String vp : this.getAnnotationViewpoints()) { - jgen.writeString(vp); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("annotationIAClasses"); - for (String cls : this.getAnnotationIAClasses()) { - jgen.writeString(cls); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("measurements"); - if (this.measurements != null) - for (Measurement meas : this.measurements) { - if (meas.getValue() == null) continue; // no value means we should skip - jgen.writeStartObject(); - jgen.writeNumberField("value", meas.getValue()); - if (meas.getType() != null) jgen.writeStringField("type", meas.getType()); - if (meas.getUnits() != null) jgen.writeStringField("units", meas.getUnits()); - if (meas.getSamplingProtocol() != null) - jgen.writeStringField("samplingProtocol", meas.getSamplingProtocol()); - jgen.writeEndObject(); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("metalTags"); - if (this.getMetalTags() != null) - for (MetalTag tag : this.getMetalTags()) { - jgen.writeStartObject(); - jgen.writeStringField("number", tag.getTagNumber()); - jgen.writeStringField("location", tag.getLocation()); - jgen.writeEndObject(); - } - jgen.writeEndArray(); - if (this.getAcousticTag() != null) { - jgen.writeObjectFieldStart("acousticTag"); - jgen.writeStringField("idNumber", this.getAcousticTag().getIdNumber()); - jgen.writeStringField("serialNumber", this.getAcousticTag().getSerialNumber()); - jgen.writeEndObject(); - } - if (this.getSatelliteTag() != null) { - jgen.writeObjectFieldStart("satelliteTag"); - jgen.writeStringField("name", this.getSatelliteTag().getName()); - jgen.writeStringField("serialNumber", this.getSatelliteTag().getSerialNumber()); - jgen.writeStringField("argosPttNumber", this.getSatelliteTag().getArgosPttNumber()); - jgen.writeEndObject(); - } - if (this.getDTag() != null) { - jgen.writeObjectFieldStart("digitalArchiveTag"); - jgen.writeStringField("dTagID", this.getDTag().getDTagID()); - jgen.writeStringField("serialNumber", this.getDTag().getSerialNumber()); - jgen.writeEndObject(); - } - org.json.JSONObject dpj = this.getDynamicPropertiesJSONObject(); - jgen.writeObjectFieldStart("dynamicProperties"); - for (String key : (Set)dpj.keySet()) { - jgen.writeStringField(key, dpj.optString(key, null)); - } - jgen.writeEndObject(); - - Double dlat = this.getDecimalLatitudeAsDouble(); - Double dlon = this.getDecimalLongitudeAsDouble(); - if ((dlat == null) || (dlon == null)) { - jgen.writeNullField("locationGeoPoint"); - } else { - jgen.writeObjectFieldStart("locationGeoPoint"); - jgen.writeNumberField("lat", dlat); - jgen.writeNumberField("lon", dlon); - jgen.writeEndObject(); - } - MarkedIndividual indiv = this.getIndividual(); - if (indiv == null) { - jgen.writeNullField("individualId"); - } else { - jgen.writeStringField("individualId", indiv.getId()); - jgen.writeStringField("individualSex", indiv.getSex()); - jgen.writeNumberField("individualNumberEncounters", indiv.getNumEncounters()); - jgen.writeStringField("individualDisplayName", indiv.getDisplayName()); - jgen.writeArrayFieldStart("individualNames"); - Set names = indiv.getAllNamesList(); - if (names != null) - for (String name : names) { - jgen.writeString(name); - } - jgen.writeEndArray(); - jgen.writeStringField("individualNickName", indiv.getNickName()); - if (indiv.getTimeOfBirth() > 0) { - String birthTime = Util.getISO8601Date(new DateTime( - indiv.getTimeOfBirth()).toString()); - jgen.writeStringField("individualTimeOfBirth", birthTime); - } - Encounter[] encs = indiv.getDateSortedEncounters(true); - if ((encs != null) && (encs.length > 0)) { - String encDate = Util.getISO8601Date(encs[0].getDate()); - if (encDate != null) jgen.writeStringField("individualFirstEncounterDate", encDate); - encDate = Util.getISO8601Date(encs[encs.length - 1].getDate()); - if (encDate != null) jgen.writeStringField("individualLastEncounterDate", encDate); - } - - jgen.writeArrayFieldStart("individualSocialUnits"); - for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(indiv)) { - Membership mem = su.getMembershipForMarkedIndividual(indiv); - if (mem != null) jgen.writeString(su.getSocialUnitName()); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("individualRelationshipRoles"); - for (String relRole : myShepherd.getAllRoleNamesForMarkedIndividual(indiv.getId())) { - jgen.writeString(relRole); - } - jgen.writeEndArray(); - } - DateTime occdt = getOccurrenceDateTime(myShepherd); - if (occdt != null) jgen.writeStringField("occurrenceDate", occdt.toString()); - jgen.writeStringField("geneticSex", this.getGeneticSex()); - jgen.writeStringField("haplotype", this.getHaplotype()); - - jgen.writeArrayFieldStart("microsatelliteMarkers"); - for (MicrosatelliteMarkersAnalysis msm : getMicrosatelliteMarkers()) { - jgen.writeStartObject(); - jgen.writeStringField("analysisId", msm.getAnalysisID()); - jgen.writeObjectFieldStart("loci"); - if (msm.getLoci() != null) - for (Locus locus : msm.getLoci()) { - if (locus.getName() == null) continue; // snh - jgen.writeObjectFieldStart(locus.getName()); - for (int i = 0; i < 4; i++) { - Integer allele = locus.getAllele(i); - if (allele != null) jgen.writeNumberField("allele" + i, allele); - } - jgen.writeEndObject(); - } - jgen.writeEndObject(); - jgen.writeEndObject(); - } - jgen.writeEndArray(); - - Occurrence occ = this.getOccurrence(myShepherd); - if (occ == null) { - jgen.writeNullField("occurrenceId"); - } else { - jgen.writeStringField("occurrenceId", occ.getId()); - } - jgen.writeArrayFieldStart("organizations"); - User owner = this.getSubmitterUser(myShepherd); - if ((owner != null) && (owner.getOrganizations() != null)) - for (Organization org : owner.getOrganizations()) { - jgen.writeString(org.getId()); - } - jgen.writeEndArray(); - - jgen.writeArrayFieldStart("tissueSampleIds"); - for (String id : this.getTissueSampleIDs()) { - jgen.writeString(id); - } - jgen.writeEndArray(); - - Map bmeas = this.getBiologicalMeasurementsByType(); - jgen.writeObjectFieldStart("biologicalMeasurements"); - for (String type : (Set)bmeas.keySet()) { - jgen.writeNumberField(type, bmeas.get(type).getValue()); - } - jgen.writeEndObject(); + jgen.writeArrayFieldStart("individualRelationshipRoles"); + for (String relRole : myShepherd.getAllRoleNamesForMarkedIndividual(indiv.getId())) { + jgen.writeString(relRole); + } + jgen.writeEndArray(); + } + DateTime occdt = getOccurrenceDateTime(myShepherd); + if (occdt != null) jgen.writeStringField("occurrenceDate", occdt.toString()); + jgen.writeStringField("geneticSex", this.getGeneticSex()); + jgen.writeStringField("haplotype", this.getHaplotype()); + + jgen.writeArrayFieldStart("microsatelliteMarkers"); + for (MicrosatelliteMarkersAnalysis msm : getMicrosatelliteMarkers()) { + jgen.writeStartObject(); + jgen.writeStringField("analysisId", msm.getAnalysisID()); + jgen.writeObjectFieldStart("loci"); + if (msm.getLoci() != null) + for (Locus locus : msm.getLoci()) { + if (locus.getName() == null) continue; // snh + jgen.writeObjectFieldStart(locus.getName()); + for (int i = 0; i < 4; i++) { + Integer allele = locus.getAllele(i); + if (allele != null) jgen.writeNumberField("allele" + i, allele); + } + jgen.writeEndObject(); + } + jgen.writeEndObject(); + jgen.writeEndObject(); + } + jgen.writeEndArray(); + + Occurrence occ = this.getOccurrence(myShepherd); + if (occ == null) { + jgen.writeNullField("occurrenceId"); + } else { + jgen.writeStringField("occurrenceId", occ.getId()); + } + jgen.writeArrayFieldStart("organizations"); + User owner = this.getSubmitterUser(myShepherd); + if ((owner != null) && (owner.getOrganizations() != null)) + for (Organization org : owner.getOrganizations()) { + jgen.writeString(org.getId()); + } + jgen.writeEndArray(); + jgen.writeArrayFieldStart("tissueSampleIds"); + for (String id : this.getTissueSampleIDs()) { + jgen.writeString(id); + } + jgen.writeEndArray(); + + Map bmeas = this.getBiologicalMeasurementsByType(); + jgen.writeObjectFieldStart("biologicalMeasurements"); + for (String type : (Set)bmeas.keySet()) { + jgen.writeNumberField(type, bmeas.get(type).getValue()); + } + jgen.writeEndObject(); } @Override public long getVersion() { From 8b13b3624844a15a6d05deaa158e95f14c7914a1 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Fri, 6 Dec 2024 14:14:15 -0800 Subject: [PATCH 05/12] finish the close in this condition --- src/main/java/org/ecocean/api/Login.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/api/Login.java b/src/main/java/org/ecocean/api/Login.java index b9fe39acba..4d6338f3ab 100644 --- a/src/main/java/org/ecocean/api/Login.java +++ b/src/main/java/org/ecocean/api/Login.java @@ -57,7 +57,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) // user.setAcceptedUserAgreement(true); // myShepherd.commitDBTransaction(); } catch (Exception ex) { - myShepherd.rollbackDBTransaction(); + myShepherd.rollbackAndCloseDBTransaction(); results.put("error", "invalid_credentials"); } if (user != null) { From 43259d278d200902931061d3fcb4a07b536b75f9 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Fri, 6 Dec 2024 17:21:33 -0800 Subject: [PATCH 06/12] correct method name --- src/main/java/org/ecocean/api/Login.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/api/Login.java b/src/main/java/org/ecocean/api/Login.java index 4d6338f3ab..748c43e31f 100644 --- a/src/main/java/org/ecocean/api/Login.java +++ b/src/main/java/org/ecocean/api/Login.java @@ -57,7 +57,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) // user.setAcceptedUserAgreement(true); // myShepherd.commitDBTransaction(); } catch (Exception ex) { - myShepherd.rollbackAndCloseDBTransaction(); + myShepherd.rollbackAndClose(); results.put("error", "invalid_credentials"); } if (user != null) { From 103dd21278c65e646332df8ef46d1401d106a7b9 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Sat, 7 Dec 2024 06:55:08 -0800 Subject: [PATCH 07/12] close DB transaction and handle potential exceptions --- src/main/java/org/ecocean/Encounter.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 6355dfea15..8db3dc2325 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -4342,9 +4342,15 @@ public static Object validateFieldValue(String fieldName, org.json.JSONObject da // this is throwaway read-only shepherd Shepherd myShepherd = new Shepherd("context0"); myShepherd.setAction("Encounter.validateFieldValue"); + boolean validTaxonomy=false; myShepherd.beginDBTransaction(); - boolean validTaxonomy = myShepherd.isValidTaxonomyName((String)returnValue); - myShepherd.rollbackDBTransaction(); + try { + validTaxonomy = myShepherd.isValidTaxonomyName((String)returnValue); + } + catch(Exception e) {e.printStackTrace();} + finally { + myShepherd.rollbackAndClose(); + } if (!validTaxonomy) { error.put("code", ApiException.ERROR_RETURN_CODE_INVALID); error.put("value", returnValue); From e0fa91492e4229170051cac8e48e94fee4429dd1 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Sun, 8 Dec 2024 11:37:51 -0800 Subject: [PATCH 08/12] Better backgrounding for unindex and exception handling for individual indexing --- src/main/java/org/ecocean/Base.java | 27 +++++++++++++++++-- .../java/org/ecocean/MarkedIndividual.java | 8 +++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index 2eb477c6f2..b8bb68c276 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -10,6 +10,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + import javax.jdo.Query; import org.ecocean.api.ApiException; import org.ecocean.OpenSearch; @@ -113,9 +116,29 @@ public void opensearchIndexDeep() public void opensearchUnindex() throws IOException { - OpenSearch opensearch = new OpenSearch(); + + //unindexing should be non-blocking and backgrounded + OpenSearch opensearch = new OpenSearch(); + String opensearchIndexName=this.opensearchIndexName(); + String objectId=this.getId(); + + ExecutorService executor = Executors.newFixedThreadPool(4); + Runnable rn = new Runnable() { + public void run() { + try { + opensearch.delete(opensearchIndexName, objectId); + } + catch(Exception e) { + System.out.println("opensearchUnindex() backgrounding Object " + + objectId + " hit an exception."); + e.printStackTrace(); + } + + executor.shutdown(); + } + }; + executor.execute(rn); - opensearch.delete(this.opensearchIndexName(), this); } public void opensearchUnindexQuiet() { diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 858a3f4fba..e326115e2f 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2625,7 +2625,13 @@ public void run() { ex.printStackTrace(); } } - } finally { + } + catch(Exception e) { + System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " + + indivId + " hit an exception."); + e.printStackTrace(); + } + finally { bgShepherd.rollbackAndClose(); } System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " + From 26af9a9217db3a159a9f871fe6df501f2a125a32 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Sun, 8 Dec 2024 12:10:55 -0800 Subject: [PATCH 09/12] Move OpenSearch instantiation and its embedded http call into runnable thread for more nonblocking behavior and speed --- src/main/java/org/ecocean/Base.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index b8bb68c276..a4e5e72f7c 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -118,12 +118,12 @@ public void opensearchUnindex() throws IOException { //unindexing should be non-blocking and backgrounded - OpenSearch opensearch = new OpenSearch(); String opensearchIndexName=this.opensearchIndexName(); String objectId=this.getId(); ExecutorService executor = Executors.newFixedThreadPool(4); Runnable rn = new Runnable() { + OpenSearch opensearch = new OpenSearch(); public void run() { try { opensearch.delete(opensearchIndexName, objectId); From 79058a96ce5484fde52f78b353c354ad061b1a59 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Mon, 9 Dec 2024 13:50:48 -0800 Subject: [PATCH 10/12] Send indexing out to threads --- src/main/java/org/ecocean/Encounter.java | 38 ++++++++++++++ .../java/org/ecocean/MarkedIndividual.java | 2 +- src/main/java/org/ecocean/Occurrence.java | 52 +++++++++++++++++-- 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 8db3dc2325..4fa900cb84 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -8,6 +8,8 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.Calendar; import java.util.Collection; @@ -4492,4 +4494,40 @@ public void sendCreationEmails(Shepherd myShepherd, String langCode) { myShepherd.rollbackDBTransaction(); } } + + public void opensearchIndexDeep() + throws IOException { + + + final String encId = this.getId(); + ExecutorService executor = Executors.newFixedThreadPool(4); + Runnable rn = new Runnable() { + public void run() { + Shepherd bgShepherd = new Shepherd("context0"); + bgShepherd.setAction("Encounter.opensearchIndexDeep_"+encId); + bgShepherd.beginDBTransaction(); + try { + Encounter enc = bgShepherd.getEncounter(encId); + if (enc == null) { + bgShepherd.rollbackAndClose(); + executor.shutdown(); + return; + } + enc.opensearchIndex(); + } + catch(Exception e) { + + e.printStackTrace(); + } + finally { + bgShepherd.rollbackAndClose(); + } + executor.shutdown(); + } + }; + + executor.execute(rn); + + } + } diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index e326115e2f..21398611bb 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2602,7 +2602,7 @@ public void opensearchIndexDeep() Runnable rn = new Runnable() { public void run() { Shepherd bgShepherd = new Shepherd("context0"); - bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep"); + bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep_"+indivId); bgShepherd.beginDBTransaction(); try { MarkedIndividual indiv = bgShepherd.getMarkedIndividual(indivId); diff --git a/src/main/java/org/ecocean/Occurrence.java b/src/main/java/org/ecocean/Occurrence.java index c66d3a33d6..84d28a7519 100644 --- a/src/main/java/org/ecocean/Occurrence.java +++ b/src/main/java/org/ecocean/Occurrence.java @@ -9,6 +9,8 @@ import java.util.Map; import java.util.Set; import java.util.Vector; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.text.SimpleDateFormat; import org.ecocean.media.MediaAsset; @@ -1354,11 +1356,53 @@ public org.json.JSONObject getJSONSummary() { // has to be handled at the point of removal, e.g. OccurrenceRemoveEncounter servlet public void opensearchIndexDeep() throws IOException { - if (this.encounters != null) - for (Encounter enc : this.encounters) { - enc.opensearchIndex(); - } this.opensearchIndex(); + + final String occurId = this.getId(); + ExecutorService executor = Executors.newFixedThreadPool(4); + Runnable rn = new Runnable() { + public void run() { + Shepherd bgShepherd = new Shepherd("context0"); + bgShepherd.setAction("Occurrence.opensearchIndexDeep_"+occurId); + bgShepherd.beginDBTransaction(); + try { + Occurrence occur = bgShepherd.getOccurrence(occurId); + if ((occur == null) || (occur.getEncounters() == null)) { + bgShepherd.rollbackAndClose(); + executor.shutdown(); + return; + } + int total = occur.getNumberEncounters(); + int ct = 0; + for (Encounter enc : occur.getEncounters()) { + ct++; + System.out.println("opensearchIndexDeep() background indexing " + + enc.getId() + " via " + occurId + " [" + ct + "/" + total + "]"); + try { + enc.opensearchIndex(); + } catch (Exception ex) { + System.out.println("opensearchIndexDeep() background indexing " + + enc.getId() + " FAILED: " + ex.toString()); + ex.printStackTrace(); + } + } + } + catch(Exception e) { + System.out.println("opensearchIndexDeep() backgrounding Occurrence " + + occurId + " hit an exception."); + e.printStackTrace(); + } + finally { + bgShepherd.rollbackAndClose(); + } + System.out.println("opensearchIndexDeep() backgrounding Occurrence " + + occurId + " finished."); + executor.shutdown(); + } + }; + + executor.execute(rn); + } @Override public long getVersion() { From 2f9c07d526f06ed53f813b6a616246401248b2af Mon Sep 17 00:00:00 2001 From: holmbergius Date: Mon, 9 Dec 2024 14:10:31 -0800 Subject: [PATCH 11/12] Switch these to deep for threading --- .../java/org/ecocean/servlet/IndividualRemoveEncounter.java | 2 +- .../java/org/ecocean/servlet/OccurrenceRemoveEncounter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java b/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java index 6e09e3734d..d3a1cd8167 100644 --- a/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java +++ b/src/main/java/org/ecocean/servlet/IndividualRemoveEncounter.java @@ -107,7 +107,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) } if (!locked) { myShepherd.commitDBTransaction(); - if (enc2remove != null) enc2remove.opensearchIndex(); + if (enc2remove != null) enc2remove.opensearchIndexDeep(); out.println(ServletUtilities.getHeader(request)); response.setStatus(HttpServletResponse.SC_OK); out.println("Success: Encounter #" + diff --git a/src/main/java/org/ecocean/servlet/OccurrenceRemoveEncounter.java b/src/main/java/org/ecocean/servlet/OccurrenceRemoveEncounter.java index 427461fa89..b6dc7c135e 100644 --- a/src/main/java/org/ecocean/servlet/OccurrenceRemoveEncounter.java +++ b/src/main/java/org/ecocean/servlet/OccurrenceRemoveEncounter.java @@ -83,7 +83,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) if (!locked) { myShepherd.commitDBTransaction(); // out.println(ServletUtilities.getHeader(request)); - if (enc2remove != null) enc2remove.opensearchIndex(); + if (enc2remove != null) enc2remove.opensearchIndexDeep(); out.println("Success: Encounter " + request.getParameter("number") + " was successfully removed from occurrence " + old_name + "."); From da7bd44f33712595c39175712ead1cd101847de7 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 11 Dec 2024 11:00:52 -0700 Subject: [PATCH 12/12] linting --- src/main/java/org/ecocean/Base.java | 28 +++++++-------- src/main/java/org/ecocean/Encounter.java | 29 ++++++--------- .../java/org/ecocean/MarkedIndividual.java | 12 +++---- src/main/java/org/ecocean/Occurrence.java | 35 +++++++++---------- 4 files changed, 43 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/ecocean/Base.java b/src/main/java/org/ecocean/Base.java index a4e5e72f7c..b1afc85ee3 100644 --- a/src/main/java/org/ecocean/Base.java +++ b/src/main/java/org/ecocean/Base.java @@ -6,12 +6,12 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.io.File; import java.io.IOException; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import javax.jdo.Query; import org.ecocean.api.ApiException; @@ -116,29 +116,25 @@ public void opensearchIndexDeep() public void opensearchUnindex() throws IOException { - - //unindexing should be non-blocking and backgrounded - String opensearchIndexName=this.opensearchIndexName(); - String objectId=this.getId(); - + // unindexing should be non-blocking and backgrounded + String opensearchIndexName = this.opensearchIndexName(); + String objectId = this.getId(); ExecutorService executor = Executors.newFixedThreadPool(4); Runnable rn = new Runnable() { - OpenSearch opensearch = new OpenSearch(); + OpenSearch opensearch = new OpenSearch(); public void run() { try { - opensearch.delete(opensearchIndexName, objectId); + opensearch.delete(opensearchIndexName, objectId); + } catch (Exception e) { + System.out.println("opensearchUnindex() backgrounding Object " + objectId + + " hit an exception."); + e.printStackTrace(); } - catch(Exception e) { - System.out.println("opensearchUnindex() backgrounding Object " + - objectId + " hit an exception."); - e.printStackTrace(); - } - executor.shutdown(); } }; - executor.execute(rn); + executor.execute(rn); } public void opensearchUnindexQuiet() { diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index 4fa900cb84..307837b5b7 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -8,8 +8,8 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.Calendar; import java.util.Collection; @@ -4344,14 +4344,12 @@ public static Object validateFieldValue(String fieldName, org.json.JSONObject da // this is throwaway read-only shepherd Shepherd myShepherd = new Shepherd("context0"); myShepherd.setAction("Encounter.validateFieldValue"); - boolean validTaxonomy=false; + boolean validTaxonomy = false; myShepherd.beginDBTransaction(); try { - validTaxonomy = myShepherd.isValidTaxonomyName((String)returnValue); - } - catch(Exception e) {e.printStackTrace();} - finally { - myShepherd.rollbackAndClose(); + validTaxonomy = myShepherd.isValidTaxonomyName((String)returnValue); + } catch (Exception e) { e.printStackTrace(); } finally { + myShepherd.rollbackAndClose(); } if (!validTaxonomy) { error.put("code", ApiException.ERROR_RETURN_CODE_INVALID); @@ -4494,17 +4492,15 @@ public void sendCreationEmails(Shepherd myShepherd, String langCode) { myShepherd.rollbackDBTransaction(); } } - + public void opensearchIndexDeep() throws IOException { - - final String encId = this.getId(); ExecutorService executor = Executors.newFixedThreadPool(4); Runnable rn = new Runnable() { public void run() { Shepherd bgShepherd = new Shepherd("context0"); - bgShepherd.setAction("Encounter.opensearchIndexDeep_"+encId); + bgShepherd.setAction("Encounter.opensearchIndexDeep_" + encId); bgShepherd.beginDBTransaction(); try { Encounter enc = bgShepherd.getEncounter(encId); @@ -4514,12 +4510,9 @@ public void run() { return; } enc.opensearchIndex(); - } - catch(Exception e) { - - e.printStackTrace(); - } - finally { + } catch (Exception e) { + e.printStackTrace(); + } finally { bgShepherd.rollbackAndClose(); } executor.shutdown(); @@ -4527,7 +4520,5 @@ public void run() { }; executor.execute(rn); - } - } diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 21398611bb..bca1cf48d5 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2602,7 +2602,7 @@ public void opensearchIndexDeep() Runnable rn = new Runnable() { public void run() { Shepherd bgShepherd = new Shepherd("context0"); - bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep_"+indivId); + bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep_" + indivId); bgShepherd.beginDBTransaction(); try { MarkedIndividual indiv = bgShepherd.getMarkedIndividual(indivId); @@ -2625,13 +2625,11 @@ public void run() { ex.printStackTrace(); } } - } - catch(Exception e) { - System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " + + } catch (Exception e) { + System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " + indivId + " hit an exception."); - e.printStackTrace(); - } - finally { + e.printStackTrace(); + } finally { bgShepherd.rollbackAndClose(); } System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " + diff --git a/src/main/java/org/ecocean/Occurrence.java b/src/main/java/org/ecocean/Occurrence.java index 84d28a7519..04ac4783e2 100644 --- a/src/main/java/org/ecocean/Occurrence.java +++ b/src/main/java/org/ecocean/Occurrence.java @@ -3,14 +3,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.Vector; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.text.SimpleDateFormat; import org.ecocean.media.MediaAsset; @@ -97,7 +97,7 @@ public class Occurrence extends Base implements java.io.Serializable { private Integer numCalves; private String observer; - private String submitterID; + private String submitterID; private List submitters; private List informOthers; @@ -385,12 +385,12 @@ public ArrayList getMarkedIndividualNamesForThisOccurrence() { return names; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public void setID(String id) { occurrenceID = id; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public String getID() { return occurrenceID; } @@ -403,7 +403,7 @@ public String getWebUrl(HttpServletRequest req) { return getWebUrl(getOccurrenceID(), req); } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public String getOccurrenceID() { return occurrenceID; } @@ -418,7 +418,7 @@ public String getOccurrenceID() { occurrenceID = id; } - //TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public void setOccurrenceID(String id) { occurrenceID = id; } @@ -1363,7 +1363,7 @@ public void opensearchIndexDeep() Runnable rn = new Runnable() { public void run() { Shepherd bgShepherd = new Shepherd("context0"); - bgShepherd.setAction("Occurrence.opensearchIndexDeep_"+occurId); + bgShepherd.setAction("Occurrence.opensearchIndexDeep_" + occurId); bgShepherd.beginDBTransaction(); try { Occurrence occur = bgShepherd.getOccurrence(occurId); @@ -1386,23 +1386,20 @@ public void run() { ex.printStackTrace(); } } - } - catch(Exception e) { - System.out.println("opensearchIndexDeep() backgrounding Occurrence " + - occurId + " hit an exception."); - e.printStackTrace(); - } - finally { + } catch (Exception e) { + System.out.println("opensearchIndexDeep() backgrounding Occurrence " + occurId + + " hit an exception."); + e.printStackTrace(); + } finally { bgShepherd.rollbackAndClose(); } - System.out.println("opensearchIndexDeep() backgrounding Occurrence " + - occurId + " finished."); + System.out.println("opensearchIndexDeep() backgrounding Occurrence " + occurId + + " finished."); executor.shutdown(); } }; - + executor.execute(rn); - } @Override public long getVersion() {