Oracle-ES consistency

Utils/Dataflow/run/data4es-consistency-check

@@ -3,29 +3,15 @@
 DEBUG=
 
 base_dir=$( cd "$(dirname "$(readlink -f "$0")")"; pwd)
+lib="$base_dir/../shell_lib"
 
 # Directories with configuration files
 [ -n "$DATA4ES_CONSISTENCY_CONFIG_PATH" ] && \
     CONFIG_PATH="$DATA4ES_CONSISTENCY_CONFIG_PATH" || \
     CONFIG_PATH="${base_dir}/../config:${base_dir}/../../Elasticsearch/config"
 
-# Find configuration file in $CONFIG_PATH
-get_config() {
-  [ -z "$1" ] && echo "get_config(): no arguments passed." && return 1
-  dirs=$CONFIG_PATH
-  while [ -n "$dirs" ]; do
-    dir=${dirs%%:*}
-    [ "$dirs" = "$dir" ] && \
-        dirs='' || \
-        dirs="${dirs#*:}"
-    [ -f "${dir}/${1}" ] && readlink -f "${dir}/${1}" && return 0
-  done
-}
-
-# EOP filter (required due to the unconfigurable EOP marker in pyDKB)
-eop_filter() {
-  sed -e"s/\\x00//g"
-}
+source $lib/get_config
+source $lib/eop_filter
 
 # Oracle
 cfg009=`get_config "consistency009.cfg"`

Utils/Dataflow/run/data4es-start

@@ -6,14 +6,9 @@ BATCH_SIZE=100
 DEBUG=
 
 base_dir=$( cd "$(dirname "$(readlink -f "$0")")"; pwd)
+lib="$base_dir/../shell_lib"
 
-log() {
-  level=INFO
-  [ $# -eq 2 ] && level="$1" && shift
-  [ -z "$SCRIPT_NAME" ] && SCRIPT_NAME="$(basename "$0")"
-  msg="$1"
-  echo "($level) `date +'%d-%m-%Y %T'` ($SCRIPT_NAME) $msg" >&2
-}
+source $lib/log
 
 # Directories with configuration files
 [ -n "$DATA4ES_CONFIG_PATH" ] && \
@@ -49,18 +44,7 @@ pidfile="$HOME_DIR/pid"
 # Define commands to be used as dataflow nodes
 # ---
 
-# Find configuration file in $CONFIG_PATH
-get_config() {
-  [ -z "$1" ] && log ERROR "get_config(): no arguments passed." && return 1
-  dirs=$CONFIG_PATH
-  while [ -n "$dirs" ]; do
-    dir=${dirs%%:*}
-    [ "$dirs" = "$dir" ] && \
-        dirs='' || \
-        dirs="${dirs#*:}"
-    [ -f "${dir}/${1}" ] && readlink -f "${dir}/${1}" && return 0
-  done
-}
+source $lib/get_config
 
 # Oracle Connector
 cmd_09="${base_dir}/../009_oracleConnector/Oracle2JSON.py"
@@ -101,10 +85,7 @@ cmd_95="${base_dir}/../095_datasetInfoAMI/amiDatasets.py -m s --userkey $AUTH_KE
 
 # Service (glue) functions
 # ---
-# EOP filter (required due to the unconfigurable EOP marker in pyDKB)
-eop_filter() {
-  sed -e"s/\\x00//g"
-}
+source $lib/eop_filter
 
 # Buffer file name
 get_buffer() {

Utils/Dataflow/shell_lib/eop_filter

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# EOP filter (required due to the unconfigurable EOP marker in pyDKB)
+eop_filter() {
+  sed -e"s/\\x00//g"
+}

Utils/Dataflow/shell_lib/get_config

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+get_config_lib=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+source $get_config_lib/log
+
+# Find configuration file in $CONFIG_PATH
+get_config() {
+  [ -z "$1" ] && log ERROR "get_config(): no arguments passed." && return 1
+  [ -z "$CONFIG_PATH" ] && CONFIG_PATH=`pwd`
+  dirs=$CONFIG_PATH
+  while [ -n "$dirs" ]; do
+    dir=${dirs%%:*}
+    [ "$dirs" = "$dir" ] && \
+        dirs='' || \
+        dirs="${dirs#*:}"
+    [ -f "${dir}/${1}" ] && readlink -f "${dir}/${1}" && return 0
+  done
+}

Utils/Dataflow/shell_lib/log

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+log() {
+  level=INFO
+  [ $# -eq 2 ] && level="$1" && shift
+  [ -z "$SCRIPT_NAME" ] && SCRIPT_NAME="$(basename "$0")"
+  msg="$1"
+  echo "($level) `date +'%d-%m-%Y %T'` ($SCRIPT_NAME) $msg" >&2
+}