From 578f75fdcf14b29c9a50231e7fc46ec371723998 Mon Sep 17 00:00:00 2001
From: Qifeng <qifeng.bai@csiro.au>
Date: Tue, 21 Dec 2021 10:46:45 +1100
Subject: [PATCH] #622 impl(only output records which have outlier)

---
 .../beam/IndexRecordToSolrPipeline.java       |  5 ++-
 .../DistributionOutlierTransform.java         | 37 +++++++++++--------
 .../pipelines/src/main/resources/logback.xml  |  3 --
 3 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/IndexRecordToSolrPipeline.java b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/IndexRecordToSolrPipeline.java
index 88a801e344..38f49a57cb 100644
--- a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/IndexRecordToSolrPipeline.java
+++ b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/IndexRecordToSolrPipeline.java
@@ -576,8 +576,9 @@ public void processElement(ProcessContext c) {
                 .setBooleans(indexRecord.getBooleans())
                 .setInts(indexRecord.getInts())
                 .build();
-
-        ouputIR.getDoubles().put(DISTANCE_TO_EDL, outlierRecord.getDistanceOutOfEDL());
+        if (outlierRecord != null) {
+          ouputIR.getDoubles().put(DISTANCE_TO_EDL, outlierRecord.getDistanceOutOfEDL());
+        }
 
         c.output(KV.of(id, ouputIR));
       }
diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java
index c389472b5a..1204f537be 100644
--- a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java
+++ b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java
@@ -76,20 +76,23 @@ public Iterable<DistributionOutlierRecord> apply(
               boolean hasEDL = edl.size() > 0 ? true : false;
               double distanceToEDL = hasEDL ? 0 : -1; // 0 -inside, -1: no EDL
               // Available EDLD of this species
-              Map points = new HashMap();
-              while (iter.hasNext()) {
-                IndexRecord record = iter.next();
-                DistributionOutlierRecord dr = convertToDistribution(record, distanceToEDL);
-                if (dr != null) {
-                  outputs.add(dr);
-                  Map point = new HashMap();
-                  point.put("decimalLatitude", dr.getDecimalLatitude());
-                  point.put("decimalLongitude", dr.getDecimalLongitude());
-                  points.put(dr.getId(), point);
-                }
-              }
 
               if (hasEDL) {
+                Map points = new HashMap();
+                while (iter.hasNext()) {
+                  IndexRecord record = iter.next();
+                  DistributionOutlierRecord dr = convertToDistribution(record, distanceToEDL);
+                  if (dr != null) {
+                    outputs.add(dr);
+                    Map point = new HashMap();
+                    point.put("decimalLatitude", dr.getDecimalLatitude());
+                    point.put("decimalLongitude", dr.getDecimalLongitude());
+                    points.put(dr.getId(), point);
+                  }
+                }
+
+                log.debug(
+                    String.format("Calculating %d records of the species %s", points.size(), lsid));
                 Map<String, Double> results = distributionService.outliers(lsid, points);
                 Iterator<Map.Entry<String, Double>> iterator = results.entrySet().iterator();
                 while (iterator.hasNext()) {
@@ -99,14 +102,16 @@ public Iterable<DistributionOutlierRecord> apply(
                       .forEach(it -> it.setDistanceOutOfEDL(entry.getValue()));
                 }
               }
-
             } catch (ExpertDistributionException e) {
+              log.error("Error in processing the species: " + lsid + " . Ignored");
               log.error(e.getMessage());
-              throw new RuntimeException(
-                  "Expert distribution service throws a runtime error, please check logs");
+              // throw new RuntimeException(
+              //  "Expert distribution service throws a runtime error, please check
+              // logs");
             } catch (Exception e) {
+              log.error("Error in processing the species: " + lsid + " . Ignored");
               log.error(e.getMessage());
-              throw new RuntimeException("Runtime error: " + e.getMessage());
+              // throw new RuntimeException("Runtime error: " + e.getMessage());
             }
 
             return outputs;
diff --git a/livingatlas/pipelines/src/main/resources/logback.xml b/livingatlas/pipelines/src/main/resources/logback.xml
index 99c9db2419..8251880ab9 100644
--- a/livingatlas/pipelines/src/main/resources/logback.xml
+++ b/livingatlas/pipelines/src/main/resources/logback.xml
@@ -7,9 +7,6 @@
 
     <property name="defaultPattern" value="%-5level [%date{'yyyy-MM-dd HH:mm:ss,SSSZ'}] [%thread] %logger: %msg%n%xEx"/>
 
-    log4j.appender.console.layout.ConversionPattern=%d{dd-MMM} %color-black{%d{HH:mm:ss}} [%color-magenta{%X{step}}] [%color-blue{%X{datasetId}}] [%color{%p}] %color-cyan{%c{1}}: %color-blue{%m%n}
-
-
     <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>${defaultPattern}</pattern>