#622 impl(only output records which have outlier)

gbif · Dec 20, 2021 · 578f75f · 578f75f
1 parent 7a9ee6c
commit 578f75f
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 21 deletions.
diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/IndexRecordToSolrPipeline.java b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/IndexRecordToSolrPipeline.java
@@ -576,8 +576,9 @@ public void processElement(ProcessContext c) {
                 .setBooleans(indexRecord.getBooleans())
                 .setInts(indexRecord.getInts())
                 .build();
-
-        ouputIR.getDoubles().put(DISTANCE_TO_EDL, outlierRecord.getDistanceOutOfEDL());
+        if (outlierRecord != null) {
+          ouputIR.getDoubles().put(DISTANCE_TO_EDL, outlierRecord.getDistanceOutOfEDL());
+        }
 
         c.output(KV.of(id, ouputIR));
       }

diff --git a/...pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java b/...pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java
@@ -76,20 +76,23 @@ public Iterable<DistributionOutlierRecord> apply(
               boolean hasEDL = edl.size() > 0 ? true : false;
               double distanceToEDL = hasEDL ? 0 : -1; // 0 -inside, -1: no EDL
               // Available EDLD of this species
-              Map points = new HashMap();
-              while (iter.hasNext()) {
-                IndexRecord record = iter.next();
-                DistributionOutlierRecord dr = convertToDistribution(record, distanceToEDL);
-                if (dr != null) {
-                  outputs.add(dr);
-                  Map point = new HashMap();
-                  point.put("decimalLatitude", dr.getDecimalLatitude());
-                  point.put("decimalLongitude", dr.getDecimalLongitude());
-                  points.put(dr.getId(), point);
-                }
-              }
 
               if (hasEDL) {
+                Map points = new HashMap();
+                while (iter.hasNext()) {
+                  IndexRecord record = iter.next();
+                  DistributionOutlierRecord dr = convertToDistribution(record, distanceToEDL);
+                  if (dr != null) {
+                    outputs.add(dr);
+                    Map point = new HashMap();
+                    point.put("decimalLatitude", dr.getDecimalLatitude());
+                    point.put("decimalLongitude", dr.getDecimalLongitude());
+                    points.put(dr.getId(), point);
+                  }
+                }
+
+                log.debug(
+                    String.format("Calculating %d records of the species %s", points.size(), lsid));
                 Map<String, Double> results = distributionService.outliers(lsid, points);
                 Iterator<Map.Entry<String, Double>> iterator = results.entrySet().iterator();
                 while (iterator.hasNext()) {
@@ -99,14 +102,16 @@ public Iterable<DistributionOutlierRecord> apply(
                       .forEach(it -> it.setDistanceOutOfEDL(entry.getValue()));
                 }
               }
-
             } catch (ExpertDistributionException e) {
+              log.error("Error in processing the species: " + lsid + " . Ignored");
               log.error(e.getMessage());
-              throw new RuntimeException(
-                  "Expert distribution service throws a runtime error, please check logs");
+              // throw new RuntimeException(
+              //  "Expert distribution service throws a runtime error, please check
+              // logs");
             } catch (Exception e) {
+              log.error("Error in processing the species: " + lsid + " . Ignored");
               log.error(e.getMessage());
-              throw new RuntimeException("Runtime error: " + e.getMessage());
+              // throw new RuntimeException("Runtime error: " + e.getMessage());
             }
 
             return outputs;

diff --git a/livingatlas/pipelines/src/main/resources/logback.xml b/livingatlas/pipelines/src/main/resources/logback.xml
@@ -7,9 +7,6 @@
 
     <property name="defaultPattern" value="%-5level [%date{'yyyy-MM-dd HH:mm:ss,SSSZ'}] [%thread] %logger: %msg%n%xEx"/>
 
-    log4j.appender.console.layout.ConversionPattern=%d{dd-MMM} %color-black{%d{HH:mm:ss}} [%color-magenta{%X{step}}] [%color-blue{%X{datasetId}}] [%color{%p}] %color-cyan{%c{1}}: %color-blue{%m%n}
-
-
     <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>${defaultPattern}</pattern>