#622 doc(how to use external logback config)

livingatlas/configs/la-pipelines-local.yaml

@@ -1 +1,4 @@
 # which filesystem to use: local or hdfs
+outlier:
+  baseUrl: https://spatial-test.ala.org.au/ws/
+  runner: DirectRunner

livingatlas/pipelines/src/main/resources/log4j-colorized.properties

@@ -0,0 +1,42 @@
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.out
+log4j.appender.console.layout=com.jcabi.log.MulticolorLayout
+# https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
+log4j.appender.console.layout.Levels=INFO:0;32,WARN:0;33
+log4j.appender.console.layout.Colors=black:90
+# Supported colors: red, blue, yellow, cyan, black, and white.
+log4j.appender.console.layout.ConversionPattern=%d{dd-MMM} %color-black{%d{HH:mm:ss}} [%color-magenta{%X{step}}] [%color-blue{%X{datasetId}}] [%color{%p}] %color-cyan{%c{1}}: %color-blue{%m%n}
+# Settings to quiet third party logs that are too verbose
+log4j.logger.au.org.ala=INFO
+log4j.logger.org.spark-project.jetty=WARN
+log4j.logger.org.gbif.common.parsers=ERROR
+log4j.logger.org.gbif.geocode.api.cache=ERROR
+log4j.logger.org.gbif.geocode.api.cache.GeocodeBitmapCache=ERROR
+log4j.logger.org.gbif.pipelines.core.parsers.location.cache.GeocodeBitmapCache=ERROR
+log4j.logger.org.spark_project.jetty.servlet.ServletContextHandler=ERROR
+log4j.logger.org.eclipse.jetty.server.handler.ContextHandler=ERROR
+log4j.logger.org.gbif.dwc.terms.TermFactory=ERROR
+log4j.logger.org.gbif.vocabulary.lookup=ERROR
+log4j.logger.org.apache.beam.runners.spark.translation=ERROR
+log4j.logger.ServletContextHandler=ERROR
+project.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark=ERROR
+log4j.logger.org.eclipse.jetty=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=ERROR
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=ERROR
+log4j.logger.org.apache.parquet=ERROR
+log4j.logger.parquet=ERROR
+log4j.logger.org.apache.beam=ERROR
+# SPARK-9183: Settings to avoid annoying messages when looking up
+# nonexistent UDFs in SparkSQL with Hive support
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
+log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
+
+
+log4j.logger.org.apache.spark.executor.Executor=ERROR
+log4j.logger.org.apache.spark.storage.memory.MemoryStore=ERROR
+log4j.logger.org.apache.spark.storage.BlockManager=ERROR
+log4j.logger.org.apache.spark.executor=ERROR
+log4j.logger.org.apache.spark.storage=ERROR

livingatlas/pipelines/src/main/resources/log4j.properties

@@ -0,0 +1,5 @@
+log4j.rootLogger=DEBUG, consoleAppender
+
+log4j.appender.consoleAppender=org.apache.log4j.ConsoleAppender
+log4j.appender.consoleAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.consoleAppender.layout.ConversionPattern=[%t] %-5p %c %x - %m%n

livingatlas/pipelines/src/main/resources/logback-test.xml

@@ -3,10 +3,13 @@
         <resetJUL>true</resetJUL>
     </contextListener>
     <!-- Stop output INFO at start -->
-    <statusListener class="ch.qos.logback.core.status.NopStatusListener" />
+<!--    <statusListener class="ch.qos.logback.core.status.NopStatusListener" />-->
 
     <property name="defaultPattern" value="%-5level [%date{'yyyy-MM-dd HH:mm:ss,SSSZ'}] [%thread] %logger: %msg%n%xEx"/>
 
+    log4j.appender.console.layout.ConversionPattern=%d{dd-MMM} %color-black{%d{HH:mm:ss}} [%color-magenta{%X{step}}] [%color-blue{%X{datasetId}}] [%color{%p}] %color-cyan{%c{1}}: %color-blue{%m%n}
+
+
     <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>${defaultPattern}</pattern>
@@ -52,6 +55,14 @@
         <appender-ref ref="CONSOLE"/>
     </root>
 
+    <!--    Passing external config with -Dlogback.configurationFile=xxxx   -->
+    <!--    Example:                                                        -->
+    <!--    <included>                                                      -->
+    <!--        <logger name="org.gbif" level="info" additivity="false">    -->
+    <!--            <appender-ref ref="CONSOLE"/>                           -->
+    <!--        </logger>                                                   -->
+    <!--    </included>                                                     -->
+
     <include file="${logback.configurationFile}"/>
 
 </configuration>