#622 fix(using logback, not log4j )

gbif · Feb 3, 2022 · 7c36ac6 · 7c36ac6
1 parent c1da691
commit 7c36ac6
Show file tree

Hide file tree

Showing 6 changed files with 88 additions and 104 deletions.
diff --git a/livingatlas/configs/la-pipelines.yaml b/livingatlas/configs/la-pipelines.yaml
@@ -344,6 +344,18 @@ sampling-sh-args:
     executor-memory: 16G
     driver-memory: 4G
 
+outlier-sh-args:
+  local:
+    jvm: -Xmx8g -XX:+UseG1GC -Dspark.master=local[*]
+  spark-embedded:
+    jvm: -Xmx8g -XX:+UseG1GC -Dspark.master=local[*]
+  spark-cluster:
+    conf: spark.default.parallelism=144
+    num-executors: 8
+    executor-cores: 8
+    executor-memory: 16G
+    driver-memory: 4G
+
 sensitive-sh-args:
   spark-embedded:
     jvm: -Xmx8g -XX:+UseG1GC

diff --git a/...gatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/DistributionOutlierPipeline.java b/...gatlas/pipelines/src/main/java/au/org/ala/pipelines/beam/DistributionOutlierPipeline.java
@@ -27,10 +27,7 @@
  * <p>Example: java au.org.ala.pipelines.beam.DistributionOutlierPipeline
  * --config=/data/la-pipelines/config/la-pipelines.yaml --fsPath=/data
  *
- * <p>Running with Jar java
- * -Dlog4j.configuration=file://../pipelines/src/main/resources/log4j-colorized.properties
- * -Dlog4j.configurationFile=file://../pipelines/src/main/resources/log4j-colorized.properties -cp
- * ../pipelines/target/pipelines-2.10.0-SNAPSHOT-shaded.jar
+ * <p>Running with Jar java -cp ../pipelines/target/pipelines-2.10.0-SNAPSHOT-shaded.jar
  * au.org.ala.pipelines.beam.DistributionOutlierPipeline
  * --config=/data/la-pipelines/config/la-pipelines.yaml,la-pipelines-local.yaml --fsPath=/data
  */
@@ -39,6 +36,7 @@
 public class DistributionOutlierPipeline {
 
   public static void main(String[] args) throws Exception {
+    log.debug("debug test");
     VersionInfo.print();
     CombinedYamlConfiguration conf = new CombinedYamlConfiguration(args);
     String[] combinedArgs = conf.toArgs("general", "outlier");

diff --git a/...pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java b/...pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java
@@ -10,7 +10,6 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.beam.sdk.transforms.*;
 import org.apache.beam.sdk.values.*;
-import org.apache.commons.lang3.StringUtils;
 import org.gbif.pipelines.core.functions.SerializableConsumer;
 import org.gbif.pipelines.core.interpreters.Interpretation;
 import org.gbif.pipelines.io.avro.*;
@@ -126,33 +125,31 @@ public MapElements<DistributionOutlierRecord, String> flatToString() {
   }
 
   /**
-   * distanceOutOfEDL 0: inside edl, -1: no edl
+   * ID / Taxon ID / LatLng MUST be valid
+   *
+   * <p>distanceOutOfEDL 0: inside edl, -1: no edl
    *
    * @param record
    * @return
    */
   private DistributionOutlierRecord convertToDistribution(
       IndexRecord record, double distanceToEDL) {
     try {
-      if (!StringUtils.isEmpty(record.getId())
-          && !StringUtils.isEmpty(record.getTaxonID())
-          && !StringUtils.isEmpty(record.getLatLng())) {
-        DistributionOutlierRecord newRecord =
-            DistributionOutlierRecord.newBuilder()
-                .setId(record.getId())
-                .setSpeciesID(record.getTaxonID())
-                .setDistanceOutOfEDL(distanceToEDL)
-                .build();
-
-        String latlng = record.getLatLng();
-        String[] coordinates = latlng.split(",");
-        newRecord.setDecimalLatitude(Double.parseDouble(coordinates[0]));
-        newRecord.setDecimalLongitude(Double.parseDouble(coordinates[1]));
-
-        return newRecord;
-      }
+      DistributionOutlierRecord newRecord =
+          DistributionOutlierRecord.newBuilder()
+              .setId(record.getId())
+              .setSpeciesID(record.getTaxonID())
+              .setDistanceOutOfEDL(distanceToEDL)
+              .build();
+
+      String latlng = record.getLatLng();
+      String[] coordinates = latlng.split(",");
+      newRecord.setDecimalLatitude(Double.parseDouble(coordinates[0]));
+      newRecord.setDecimalLongitude(Double.parseDouble(coordinates[1]));
+
+      return newRecord;
     } catch (Exception ex) {
-      log.debug(record.getId() + " does not have lat/lng or taxon. ignored..");
+      log.debug(record.getId() + " has incorrect lat/lng or taxon. ignored..");
     }
     return null;
   }

diff --git a/livingatlas/pipelines/src/main/resources/log4j-colorized.properties b/livingatlas/pipelines/src/main/resources/log4j-colorized.properties
diff --git a/livingatlas/pipelines/src/main/resources/log4j.properties b/livingatlas/pipelines/src/main/resources/log4j.properties
diff --git a/livingatlas/pipelines/src/main/resources/logback-test.xml b/livingatlas/pipelines/src/main/resources/logback-test.xml
@@ -0,0 +1,57 @@
+<configuration>
+    <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
+        <resetJUL>true</resetJUL>
+    </contextListener>
+    <!-- Stop output INFO at start -->
+    <statusListener class="ch.qos.logback.core.status.NopStatusListener" />
+
+    <property name="defaultPattern" value="%-5level [%date{'yyyy-MM-dd HH:mm:ss,SSSZ'}] [%thread] %logger: %msg%n%xEx"/>
+
+    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>${defaultPattern}</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="ch.qos.logback" level="error" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <logger name="au.org.ala" level="info" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <logger name="org.gbif.pipelines" level="info" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <logger name="org.spark-project.jetty" level="warn" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="org.gbif.common.parsers" level="error" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="org.gbif.geocode.api.cache" level="error" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="org.gbif.pipelines.core.parsers.location.cache.GeocodeBitmapCache" level="error" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <logger name="org.gbif.dwc.terms.TermFactory" level="error" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="org.gbif.vocabulary.lookup" level="error" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+    <logger name="org.apache.beam.runners.spark.translation" level="error" additivity="false">
+        <appender-ref ref="CONSOLE"/>
+    </logger>
+
+    <root level="error">
+        <appender-ref ref="CONSOLE"/>
+    </root>
+
+    <include file="${logback.configurationFile}"/>
+
+</configuration>