Skip to content

Commit

Permalink
Merge pull request #929 from WildMeOrg/783_opensearch_settings
Browse files Browse the repository at this point in the history 
replace hard-coded config values with properties settings
  • Loading branch information
@holmbergius
holmbergius authored Dec 14, 2024
2 parents 5572d12 + 3868b72 commit f5c1a9c
Show file tree
Hide file tree
Showing 10 changed files with 204 additions and 40 deletions.
44 changes: 35 additions & 9 deletions src/main/java/org/ecocean/Base.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import javax.jdo.Query;
import org.ecocean.api.ApiException;
import org.ecocean.OpenSearch;
Expand Down Expand Up @@ -113,9 +116,25 @@ public void opensearchIndexDeep()

public void opensearchUnindex()
throws IOException {
OpenSearch opensearch = new OpenSearch();
// unindexing should be non-blocking and backgrounded
String opensearchIndexName = this.opensearchIndexName();
String objectId = this.getId();
ExecutorService executor = Executors.newFixedThreadPool(4);
Runnable rn = new Runnable() {
OpenSearch opensearch = new OpenSearch();
public void run() {
try {
opensearch.delete(opensearchIndexName, objectId);
} catch (Exception e) {
System.out.println("opensearchUnindex() backgrounding Object " + objectId +
" hit an exception.");
e.printStackTrace();
}
executor.shutdown();
}
};

opensearch.delete(this.opensearchIndexName(), this);
executor.execute(rn);
}

public void opensearchUnindexQuiet() {
Expand All @@ -135,12 +154,8 @@ public void opensearchUnindexDeep()
}

// should be overridden
public void opensearchDocumentSerializer(JsonGenerator jgen)
public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd)
throws IOException, JsonProcessingException {
Shepherd myShepherd = new Shepherd("context0");

myShepherd.setAction("BaseSerializer");
myShepherd.beginDBTransaction();
jgen.writeStringField("id", this.getId());
jgen.writeNumberField("version", this.getVersion());

Expand All @@ -157,8 +172,19 @@ public void opensearchDocumentSerializer(JsonGenerator jgen)
jgen.writeString(id);
}
jgen.writeEndArray();
myShepherd.rollbackDBTransaction();
myShepherd.closeDBTransaction();
}

public void opensearchDocumentSerializer(JsonGenerator jgen)
throws IOException, JsonProcessingException {
Shepherd myShepherd = new Shepherd("context0");

myShepherd.setAction("BaseSerializer");
myShepherd.beginDBTransaction();
try {
opensearchDocumentSerializer(jgen, myShepherd);
} catch (Exception e) {} finally {
myShepherd.rollbackAndClose();
}
}

public static JSONObject opensearchQuery(final String indexname, final JSONObject query,
Expand Down
64 changes: 53 additions & 11 deletions src/main/java/org/ecocean/Encounter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.Calendar;
import java.util.Collection;
Expand Down Expand Up @@ -154,7 +156,6 @@ public void setSampleTakenForDiet(Boolean sampleTakenForDiet) {
private static HashMap<String, ArrayList<Encounter> > _matchEncounterCache = new HashMap<String,
ArrayList<Encounter> >();


// An URL to a thumbnail image representing the encounter.
private String dwcImageURL;

Expand Down Expand Up @@ -678,13 +679,12 @@ public boolean hasRightSpotImage() {
return (this.getNumRightSpots() > 0);
}


// Sets the recorded length of the shark for this encounter.
public void setSize(Double mysize) {
if (mysize != null) { size = mysize; } else { size = null; }
}

// @return the length of the shark
// @return the length of the shark
public double getSize() {
return size.doubleValue();
}
Expand Down Expand Up @@ -2450,9 +2450,10 @@ public void setTissueSamples(List<TissueSample> samps) {
public Set<String> getTissueSampleIDs() {
Set<String> ids = new HashSet<String>();

if (tissueSamples != null) for (TissueSample ts : tissueSamples) {
ids.add(ts.getSampleID());
}
if (tissueSamples != null)
for (TissueSample ts : tissueSamples) {
ids.add(ts.getSampleID());
}
return ids;
}

Expand Down Expand Up @@ -3860,10 +3861,20 @@ public static org.json.JSONObject opensearchQuery(final org.json.JSONObject quer

public void opensearchDocumentSerializer(JsonGenerator jgen)
throws IOException, JsonProcessingException {
super.opensearchDocumentSerializer(jgen);
Shepherd myShepherd = new Shepherd("context0");

myShepherd.setAction("Encounter.opensearchDocumentSerializer");
myShepherd.beginDBTransaction();
try {
opensearchDocumentSerializer(jgen, myShepherd);
} catch (Exception e) {} finally {
myShepherd.rollbackAndClose();
}
}

public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd)
throws IOException, JsonProcessingException {
super.opensearchDocumentSerializer(jgen, myShepherd);

jgen.writeStringField("locationId", this.getLocationID());
jgen.writeStringField("locationName", this.getLocationName());
Expand Down Expand Up @@ -4052,7 +4063,6 @@ public void opensearchDocumentSerializer(JsonGenerator jgen)
encDate = Util.getISO8601Date(encs[encs.length - 1].getDate());
if (encDate != null) jgen.writeStringField("individualLastEncounterDate", encDate);
}

jgen.writeArrayFieldStart("individualSocialUnits");
for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(indiv)) {
Membership mem = su.getMembershipForMarkedIndividual(indiv);
Expand Down Expand Up @@ -4117,7 +4127,6 @@ public void opensearchDocumentSerializer(JsonGenerator jgen)
jgen.writeNumberField(type, bmeas.get(type).getValue());
}
jgen.writeEndObject();
myShepherd.rollbackAndClose();
}

@Override public long getVersion() {
Expand Down Expand Up @@ -4335,9 +4344,13 @@ public static Object validateFieldValue(String fieldName, org.json.JSONObject da
// this is throwaway read-only shepherd
Shepherd myShepherd = new Shepherd("context0");
myShepherd.setAction("Encounter.validateFieldValue");
boolean validTaxonomy = false;
myShepherd.beginDBTransaction();
boolean validTaxonomy = myShepherd.isValidTaxonomyName((String)returnValue);
myShepherd.rollbackDBTransaction();
try {
validTaxonomy = myShepherd.isValidTaxonomyName((String)returnValue);
} catch (Exception e) { e.printStackTrace(); } finally {
myShepherd.rollbackAndClose();
}
if (!validTaxonomy) {
error.put("code", ApiException.ERROR_RETURN_CODE_INVALID);
error.put("value", returnValue);
Expand Down Expand Up @@ -4479,4 +4492,33 @@ public void sendCreationEmails(Shepherd myShepherd, String langCode) {
myShepherd.rollbackDBTransaction();
}
}

public void opensearchIndexDeep()
throws IOException {
final String encId = this.getId();
ExecutorService executor = Executors.newFixedThreadPool(4);
Runnable rn = new Runnable() {
public void run() {
Shepherd bgShepherd = new Shepherd("context0");
bgShepherd.setAction("Encounter.opensearchIndexDeep_" + encId);
bgShepherd.beginDBTransaction();
try {
Encounter enc = bgShepherd.getEncounter(encId);
if (enc == null) {
bgShepherd.rollbackAndClose();
executor.shutdown();
return;
}
enc.opensearchIndex();
} catch (Exception e) {
e.printStackTrace();
} finally {
bgShepherd.rollbackAndClose();
}
executor.shutdown();
}
};

executor.execute(rn);
}
}
6 changes: 5 additions & 1 deletion src/main/java/org/ecocean/MarkedIndividual.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2602,7 +2602,7 @@ public void opensearchIndexDeep()
Runnable rn = new Runnable() {
public void run() {
Shepherd bgShepherd = new Shepherd("context0");
bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep");
bgShepherd.setAction("MarkedIndividual.opensearchIndexDeep_" + indivId);
bgShepherd.beginDBTransaction();
try {
MarkedIndividual indiv = bgShepherd.getMarkedIndividual(indivId);
Expand All @@ -2625,6 +2625,10 @@ public void run() {
ex.printStackTrace();
}
}
} catch (Exception e) {
System.out.println("opensearchIndexDeep() backgrounding MarkedIndividual " +
indivId + " hit an exception.");
e.printStackTrace();
} finally {
bgShepherd.rollbackAndClose();
}
Expand Down
59 changes: 50 additions & 9 deletions src/main/java/org/ecocean/Occurrence.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand Down Expand Up @@ -95,7 +97,7 @@ public class Occurrence extends Base implements java.io.Serializable {
private Integer numCalves;
private String observer;

private String submitterID;
private String submitterID;
private List<User> submitters;
private List<User> informOthers;

Expand Down Expand Up @@ -383,12 +385,12 @@ public ArrayList<String> getMarkedIndividualNamesForThisOccurrence() {
return names;
}

//TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method
// TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method
public void setID(String id) {
occurrenceID = id;
}

//TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method
// TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method
public String getID() {
return occurrenceID;
}
Expand All @@ -401,7 +403,7 @@ public String getWebUrl(HttpServletRequest req) {
return getWebUrl(getOccurrenceID(), req);
}

//TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method
// TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method
public String getOccurrenceID() {
return occurrenceID;
}
Expand All @@ -416,7 +418,7 @@ public String getOccurrenceID() {
occurrenceID = id;
}

//TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method
// TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method
public void setOccurrenceID(String id) {
occurrenceID = id;
}
Expand Down Expand Up @@ -1354,11 +1356,50 @@ public org.json.JSONObject getJSONSummary() {
// has to be handled at the point of removal, e.g. OccurrenceRemoveEncounter servlet
public void opensearchIndexDeep()
throws IOException {
if (this.encounters != null)
for (Encounter enc : this.encounters) {
enc.opensearchIndex();
}
this.opensearchIndex();

final String occurId = this.getId();
ExecutorService executor = Executors.newFixedThreadPool(4);
Runnable rn = new Runnable() {
public void run() {
Shepherd bgShepherd = new Shepherd("context0");
bgShepherd.setAction("Occurrence.opensearchIndexDeep_" + occurId);
bgShepherd.beginDBTransaction();
try {
Occurrence occur = bgShepherd.getOccurrence(occurId);
if ((occur == null) || (occur.getEncounters() == null)) {
bgShepherd.rollbackAndClose();
executor.shutdown();
return;
}
int total = occur.getNumberEncounters();
int ct = 0;
for (Encounter enc : occur.getEncounters()) {
ct++;
System.out.println("opensearchIndexDeep() background indexing " +
enc.getId() + " via " + occurId + " [" + ct + "/" + total + "]");
try {
enc.opensearchIndex();
} catch (Exception ex) {
System.out.println("opensearchIndexDeep() background indexing " +
enc.getId() + " FAILED: " + ex.toString());
ex.printStackTrace();
}
}
} catch (Exception e) {
System.out.println("opensearchIndexDeep() backgrounding Occurrence " + occurId +
" hit an exception.");
e.printStackTrace();
} finally {
bgShepherd.rollbackAndClose();
}
System.out.println("opensearchIndexDeep() backgrounding Occurrence " + occurId +
" finished.");
executor.shutdown();
}
};

executor.execute(rn);
}

@Override public long getVersion() {
Expand Down
Loading

0 comments on commit f5c1a9c

Please sign in to comment.