From 27436e89bd8d05a3f594d4fbb9e6162e194c440f Mon Sep 17 00:00:00 2001
From: Qifeng <qifeng.bai@csiro.au>
Date: Fri, 17 Dec 2021 10:08:25 +1100
Subject: [PATCH] #622 doc(how to use external logback config)

---
 livingatlas/configs/la-pipelines-local.yaml   |  3 ++
 .../main/resources/log4j-colorized.properties | 42 +++++++++++++++++++
 .../src/main/resources/log4j.properties       |  5 +++
 .../src/main/resources/logback-test.xml       | 13 +++++-
 4 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 livingatlas/pipelines/src/main/resources/log4j-colorized.properties
 create mode 100644 livingatlas/pipelines/src/main/resources/log4j.properties

diff --git a/livingatlas/configs/la-pipelines-local.yaml b/livingatlas/configs/la-pipelines-local.yaml
index d5a85c0af3..f9c5fd9967 100644
--- a/livingatlas/configs/la-pipelines-local.yaml
+++ b/livingatlas/configs/la-pipelines-local.yaml
@@ -1 +1,4 @@
 # which filesystem to use: local or hdfs
+outlier:
+  baseUrl: https://spatial-test.ala.org.au/ws/
+  runner: DirectRunner
\ No newline at end of file
diff --git a/livingatlas/pipelines/src/main/resources/log4j-colorized.properties b/livingatlas/pipelines/src/main/resources/log4j-colorized.properties
new file mode 100644
index 0000000000..07f882e378
--- /dev/null
+++ b/livingatlas/pipelines/src/main/resources/log4j-colorized.properties
@@ -0,0 +1,42 @@
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.out
+log4j.appender.console.layout=com.jcabi.log.MulticolorLayout
+# https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
+log4j.appender.console.layout.Levels=INFO:0;32,WARN:0;33
+log4j.appender.console.layout.Colors=black:90
+# Supported colors: red, blue, yellow, cyan, black, and white.
+log4j.appender.console.layout.ConversionPattern=%d{dd-MMM} %color-black{%d{HH:mm:ss}} [%color-magenta{%X{step}}] [%color-blue{%X{datasetId}}] [%color{%p}] %color-cyan{%c{1}}: %color-blue{%m%n}
+# Settings to quiet third party logs that are too verbose
+log4j.logger.au.org.ala=INFO
+log4j.logger.org.spark-project.jetty=WARN
+log4j.logger.org.gbif.common.parsers=ERROR
+log4j.logger.org.gbif.geocode.api.cache=ERROR
+log4j.logger.org.gbif.geocode.api.cache.GeocodeBitmapCache=ERROR
+log4j.logger.org.gbif.pipelines.core.parsers.location.cache.GeocodeBitmapCache=ERROR
+log4j.logger.org.spark_project.jetty.servlet.ServletContextHandler=ERROR
+log4j.logger.org.eclipse.jetty.server.handler.ContextHandler=ERROR
+log4j.logger.org.gbif.dwc.terms.TermFactory=ERROR
+log4j.logger.org.gbif.vocabulary.lookup=ERROR
+log4j.logger.org.apache.beam.runners.spark.translation=ERROR
+log4j.logger.ServletContextHandler=ERROR
+project.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark=ERROR
+log4j.logger.org.eclipse.jetty=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=ERROR
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=ERROR
+log4j.logger.org.apache.parquet=ERROR
+log4j.logger.parquet=ERROR
+log4j.logger.org.apache.beam=ERROR
+# SPARK-9183: Settings to avoid annoying messages when looking up
+# nonexistent UDFs in SparkSQL with Hive support
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
+log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
+
+
+log4j.logger.org.apache.spark.executor.Executor=ERROR
+log4j.logger.org.apache.spark.storage.memory.MemoryStore=ERROR
+log4j.logger.org.apache.spark.storage.BlockManager=ERROR
+log4j.logger.org.apache.spark.executor=ERROR
+log4j.logger.org.apache.spark.storage=ERROR
\ No newline at end of file
diff --git a/livingatlas/pipelines/src/main/resources/log4j.properties b/livingatlas/pipelines/src/main/resources/log4j.properties
new file mode 100644
index 0000000000..3900a938b0
--- /dev/null
+++ b/livingatlas/pipelines/src/main/resources/log4j.properties
@@ -0,0 +1,5 @@
+log4j.rootLogger=DEBUG, consoleAppender
+
+log4j.appender.consoleAppender=org.apache.log4j.ConsoleAppender
+log4j.appender.consoleAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.consoleAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n
diff --git a/livingatlas/pipelines/src/main/resources/logback-test.xml b/livingatlas/pipelines/src/main/resources/logback-test.xml
index c1e164fcd7..110933b120 100644
--- a/livingatlas/pipelines/src/main/resources/logback-test.xml
+++ b/livingatlas/pipelines/src/main/resources/logback-test.xml
@@ -3,10 +3,13 @@
         <resetJUL>true</resetJUL>
     </contextListener>
     <!-- Stop output INFO at start -->
-    <statusListener class="ch.qos.logback.core.status.NopStatusListener" />
+<!--    <statusListener class="ch.qos.logback.core.status.NopStatusListener" />-->
 
     <property name="defaultPattern" value="%-5level [%date{'yyyy-MM-dd HH:mm:ss,SSSZ'}] [%thread] %logger: %msg%n%xEx"/>
 
+    log4j.appender.console.layout.ConversionPattern=%d{dd-MMM} %color-black{%d{HH:mm:ss}} [%color-magenta{%X{step}}] [%color-blue{%X{datasetId}}] [%color{%p}] %color-cyan{%c{1}}: %color-blue{%m%n}
+
+
     <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>${defaultPattern}</pattern>
@@ -52,6 +55,14 @@
         <appender-ref ref="CONSOLE"/>
     </root>
 
+    <!--    Passing external config with -Dlogback.configurationFile=xxxx   -->
+    <!--    Example:                                                        -->
+    <!--    <included>                                                      -->
+    <!--        <logger name="org.gbif" level="info" additivity="false">    -->
+    <!--            <appender-ref ref="CONSOLE"/>                           -->
+    <!--        </logger>                                                   -->
+    <!--    </included>                                                     -->
+
     <include file="${logback.configurationFile}"/>
 
 </configuration>
\ No newline at end of file