#622 impl(index distance to edl to solr)

gbif · Nov 28, 2021 · 8d41ef8 · 8d41ef8
1 parent 28cbab8
commit 8d41ef8
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 12 deletions.
diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/IndexRecordToSolrPipeline.java b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/IndexRecordToSolrPipeline.java
@@ -560,15 +560,26 @@ public void processElement(ProcessContext c) {
       public void processElement(ProcessContext c) {
 
         KV<String, KV<IndexRecord, DistributionOutlierRecord>> e = c.element();
-
-        IndexRecord indexRecord = e.getValue().getKey();
-        String id = indexRecord.getId();
+        String id = e.getKey();
 
         DistributionOutlierRecord outlierRecord = e.getValue().getValue();
 
-        indexRecord.getDoubles().put(DISTANCE_TO_EDL, outlierRecord.getDistanceOutOfEDL());
-
-        c.output(KV.of(id, indexRecord));
+        IndexRecord indexRecord = e.getValue().getKey();
+        IndexRecord ouputIR =
+            IndexRecord.newBuilder()
+                .setId(indexRecord.getId())
+                .setTaxonID(indexRecord.getTaxonID())
+                .setLatLng(indexRecord.getLatLng())
+                .setMultiValues(indexRecord.getMultiValues())
+                .setDates(indexRecord.getDates())
+                .setLongs(indexRecord.getLongs())
+                .setBooleans(indexRecord.getBooleans())
+                .setInts(indexRecord.getInts())
+                .build();
+
+        ouputIR.getDoubles().put(DISTANCE_TO_EDL, outlierRecord.getDistanceOutOfEDL());
+
+        c.output(KV.of(id, ouputIR));
       }
     };
   }

diff --git a/...pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java b/...pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java
@@ -7,13 +7,15 @@
 import au.org.ala.pipelines.interpreters.DistributionOutlierInterpreter;
 import java.util.*;
 import java.util.stream.StreamSupport;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.beam.sdk.transforms.*;
 import org.apache.beam.sdk.values.*;
 import org.gbif.pipelines.core.functions.SerializableConsumer;
 import org.gbif.pipelines.core.interpreters.Interpretation;
 import org.gbif.pipelines.io.avro.*;
 import org.gbif.pipelines.transforms.Transform;
 
+@Slf4j
 public class DistributionOutlierTransform
     extends Transform<IndexRecord, DistributionOutlierRecord> {
 
@@ -71,11 +73,13 @@ public Iterable<DistributionOutlierRecord> apply(
               DistributionServiceImpl distributionService =
                   DistributionServiceImpl.init(spatialUrl);
               List<DistributionLayer> edl = distributionService.findLayersByLsid(lsid);
+              boolean hasEDL = edl.size() > 0 ? true : false;
+              double distanceToEDL = hasEDL ? 0 : -1; // 0 -inside, -1: no EDL
               // Available EDLD of this species
               Map points = new HashMap();
               while (iter.hasNext()) {
                 IndexRecord record = iter.next();
-                DistributionOutlierRecord dr = convertToDistribution(record);
+                DistributionOutlierRecord dr = convertToDistribution(record, distanceToEDL);
                 outputs.add(dr);
 
                 Map point = new HashMap();
@@ -84,7 +88,7 @@ public Iterable<DistributionOutlierRecord> apply(
                 points.put(dr.getOccurrenceID(), point);
               }
 
-              if (edl.size() > 0) {
+              if (hasEDL) {
                 Map<String, Double> results = distributionService.outliers(lsid, points);
                 Iterator<Map.Entry<String, Double>> iterator = results.entrySet().iterator();
                 while (iterator.hasNext()) {
@@ -96,8 +100,9 @@ public Iterable<DistributionOutlierRecord> apply(
               }
 
             } catch (ExpertDistributionException e) {
+              log.error(e.getMessage());
               throw new RuntimeException(
-                  "Expert distribution service returns error: " + e.getMessage());
+                  "Expert distribution service throws a runtime error, please check logs");
             } catch (Exception e) {
               throw new RuntimeException("Runtime error: " + e.getMessage());
             }
@@ -119,18 +124,19 @@ public MapElements<DistributionOutlierRecord, String> flatToString() {
 
   /**
    * Only can be used when EDL exists - which means the record can only in/out EDL Force to reset
-   * distanceOutOfEDL 0 because Spatial EDL service only return distance of outlier records
+   * distanceOutOfEDL 0: inside edl, -1: no edl
    *
    * @param record
    * @return
    */
-  private DistributionOutlierRecord convertToDistribution(IndexRecord record) {
+  private DistributionOutlierRecord convertToDistribution(
+      IndexRecord record, double distanceToEDL) {
     DistributionOutlierRecord newRecord =
         DistributionOutlierRecord.newBuilder()
             .setId(record.getId())
             .setOccurrenceID(record.getId())
-            .setDistanceOutOfEDL(0.0d)
             .setSpeciesID(record.getTaxonID())
+            .setDistanceOutOfEDL(distanceToEDL)
             .build();
 
     String latlng = record.getLatLng();

diff --git a/livingatlas/solr/conf/managed-schema b/livingatlas/solr/conf/managed-schema
@@ -440,6 +440,9 @@
   <field name="matchType"  type="string" docValues="true" indexed="true"/>
   <field name="nameType"   type="string" docValues="true" indexed="true"/>
 
+  <!-- ALA 'Data Quality' additional fields -->
+  <field name="distanceToEDL"   type="double" docValues="true" indexed="true"/>
+
   <!-- iNaturalist additions -->
   <field name="nick" type="string" docValues="true" indexed="true"/>
   <field name="numIdentificationAgreements" type="string" docValues="true" indexed="true"/>