From 353d80713ffb23387b9348436dd36b95e27ffee6 Mon Sep 17 00:00:00 2001
From: Benedikt Volkel <benedikt.volkel@cern.ch>
Date: Fri, 1 Mar 2024 15:32:50 +0100
Subject: [PATCH] [SimCI] Revise tests

* standalone AnalysisQC test has been remobved in favor of developments
  in workflow test

* generators
  * do not blindly test all INI files when run_generator_tests.sh changes
  * find files correctly when there are other changed files not related
    to generator testing

* workflows
  revised logic such that
  * if MC/bin changes
    --> run anchored,
    --> check correct creation of workflows implemented by PWGs,
    --> test AnalysisQC and QC
  * if MC/analysis_testing or MC-related QC configurations change
    --> test AnalysisQC and QC,
    --> test O2DPG AnalysisQC CLI
  * if anchored-related shell scripts change
    --> run anchored

* relval
  * no changes
---
 test/common/utils/utils.sh   |  34 +++-
 test/run_analysisqc_tests.sh | 158 +-----------------
 test/run_generator_tests.sh  |  48 +++---
 test/run_workflow_tests.sh   | 314 +++++++++++++++++------------------
 4 files changed, 207 insertions(+), 347 deletions(-)

diff --git a/test/common/utils/utils.sh b/test/common/utils/utils.sh
index 0c34c5395..03c01d96f 100644
--- a/test/common/utils/utils.sh
+++ b/test/common/utils/utils.sh
@@ -4,6 +4,33 @@
 # Test utility functionality
 #
 
+# a global counter for tests
+TEST_COUNTER=0
+
+# Prepare some colored output
+SRED="\033[0;31m"
+SGREEN="\033[0;32m"
+SYELLOW="\033[0;33m"
+SEND="\033[0m"
+
+echo_green()
+{
+    echo -e "${SGREEN}${*}${SEND}"
+}
+
+
+echo_red()
+{
+    echo -e "${SRED}${*}${SEND}"
+}
+
+
+echo_yellow()
+{
+    echo -e "${SYELLOW}${*}${SEND}"
+}
+
+
 remove_artifacts()
 {
     [[ "${KEEP_ONLY_LOGS}" == "1" ]] && find . -type f ! -name '*.log' -and ! -name "*serverlog*" -and ! -name "*mergerlog*" -and ! -name "*workerlog*" -delete
@@ -25,7 +52,12 @@ get_changed_files()
     [[ ! -z "$(git diff)" && -z ${ALIBUILD_HEAD_HASH+x} && -z ${O2DPG_TEST_HASH_HEAD+x} ]] && hash_head=""
     # if there are unstaged changes and no base from user, set to HEAD
     [[ ! -z "$(git diff)" && -z ${ALIBUILD_HEAD_HASH+x} && -z ${O2DPG_TEST_HASH_BASE+x} ]] && hash_base="HEAD"
-    git diff --diff-filter=AMR --name-only ${hash_base} ${hash_head}
+    local paths=$(git diff --diff-filter=AMR --name-only ${hash_base} ${hash_head})
+    local absolute_paths=
+    for p in ${paths} ; do
+        absolute_paths+="$(realpath ${p}) "
+    done
+    echo "${absolute_paths}"
 }
 
 
diff --git a/test/run_analysisqc_tests.sh b/test/run_analysisqc_tests.sh
index bd57493fd..a4064ba30 100755
--- a/test/run_analysisqc_tests.sh
+++ b/test/run_analysisqc_tests.sh
@@ -1,160 +1,4 @@
 #!/bin/bash
 
-# The test parent dir to be cretaed in current directory
-TEST_PARENT_DIR="o2dpg_tests/analysisqc"
-
-# unified names of log files
-LOG_FILE="o2dpg-test-analysisqc.log"
-
-# Prepare some colored output
-SRED="\033[0;31m"
-SGREEN="\033[0;32m"
-SEND="\033[0m"
-
-
-echo_green()
-{
-    echo -e "${SGREEN}${*}${SEND}"
-}
-
-
-echo_red()
-{
-    echo -e "${SRED}${*}${SEND}"
-}
-
-
-get_git_repo_directory()
-{
-    local repo=
-    if [[ -d .git ]] ; then
-        pwd
-    else
-        repo=$(git rev-parse --git-dir 2> /dev/null)
-    fi
-    [[ "${repo}" != "" ]] && repo=${repo%%/.git}
-    echo ${repo}
-}
-
-
-test_analysisqc()
-{
-    echo "### Testing AnalysisQC creation for MC ###" > ${LOG_FILE}
-    ${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f AO2D.root --is-mc -o wokflow_test_mc.json >> ${LOG_FILE} 2>&1
-    local ret=${?}
-    [[ "${ret}" != "0" ]] && echo "[FATAL]: O2DPG_TEST failed" >> ${LOG_FILE}
-    echo "### Testing AnalysisQC creation for data ###" >> ${LOG_FILE}
-    ${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f AO2D.root -o wokflow_test_data.json >> ${LOG_FILE} 2>&1
-    local ret_data=${?}
-    [[ "${ret_data}" != "0" ]] && { echo "[FATAL]: O2DPG_TEST failed" >> ${LOG_FILE} ; ret=${ret_data} ; }
-    return ${ret}
-}
-
-
-print_usage()
-{
-    echo
-    echo "usage: run_workflow_tests.sh"
-    echo
-    echo "  ENVIRONMENT VARIABLES:"
-    echo
-    echo "  O2DPG_TEST_REPO_DIR : Point to the source repository you want to test."
-    echo "  O2DPG_TEST_HASH_BASE : The base hash you want to use for comparison (optional)"
-    echo "  O2DPG_TEST_HASH_HEAD : The head hash you want to use for comparison (optional)"
-    echo
-    echo "  If O2DPG_TEST_HASH_BASE is not set, it will be looked for ALIBUILD_BASE_HASH."
-    echo "  If also not set, this will be set to HEAD~1. However, if there are unstaged"
-    echo "  changes, it will be set to HEAD."
-    echo
-    echo "  If O2DPG_TEST_HASH_HEAD is not set, it will be looked for ALIBUILD_HEAD_HASH."
-    echo "  If also not set, this will be set to HEAD. However, if there are unstaged"
-    echo "  changes, it will left blank."
-    echo
-}
-
-while [ "$1" != "" ] ; do
-    case $1 in
-        --help|-h )          print_usage
-                             exit 1
-                             ;;
-        * )                  echo "Unknown argument ${1}"
-                             exit 1
-                             ;;
-    esac
-done
-
-echo
-echo "################################"
-echo "# Run O2DPG AnalysisQC testing #"
-echo "################################"
-echo
-
-REPO_DIR=${O2DPG_TEST_REPO_DIR:-$(get_git_repo_directory)}
-if [[ ! -d ${REPO_DIR}/.git ]] ; then
-    echo_red "Directory \"${REPO_DIR}\" is not a git repository."
-    exit 1
-fi
-
-if [[ -z ${O2DPG_ROOT+x} ]] ; then
-    echo_red "O2DPG is not loaded, probably other packages are missing as well in this environment."
-    exit 1
-fi
-
-# source the utilities
-source ${REPO_DIR}/test/common/utils/utils.sh
-
-# Do the initial steps in the source dir where we have the full git repo
-pushd ${REPO_DIR} > /dev/null
-
-# flag if anything changed for AnalysisQC
-need_testing=$(get_changed_files | grep "MC/.*analysis_testing")
-
-# go back to where we came from
-popd > /dev/null
-REPO_DIR=$(realpath ${REPO_DIR})
-
-# Now, do the trick:
-# We just use the source dir since O2DPG's installation is basically just a copy of the whole repo.
-# This makes sense in particular for local testing but also in the CI it works in the same way. We could do
-#         [[ -z {ALIBUILD_HEAD_HASH+x} ]] && export O2DPG_ROOT=${REPO_DIR}
-# but let's do the same for both local and CI consistently
-export O2DPG_ROOT=${REPO_DIR}
-
-
-###############
-# Let's do it #
-###############
-ret_global=0
-# prepare our local test directory for PWG tests
-rm -rf ${TEST_PARENT_DIR} 2>/dev/null
-mkdir -p ${TEST_PARENT_DIR} 2>/dev/null
-pushd ${TEST_PARENT_DIR} > /dev/null
-
-# Test what we found
-if [[ "${need_testing}" != "" ]] ; then
-    test_analysisqc
-    ret_global=${?}
-else
-    echo "Nothing to test"
-    exit 0
-fi
-
-# return to where we came from
-popd > /dev/null
-
-# However, if a central test fails, exit code will be !=0
-if [[ "${ret_global}" != "0" ]] ; then
-    echo
-    echo "########################"
-    echo "# ERROR for AnalysisQC #"
-    echo "########################"
-    echo
-    print_error_logs ${TEST_PARENT_DIR}
-    exit ${ret_global}
-fi
-
-echo
-echo_green "AnalysisQC tests successful"
-echo
-
+# for now, obsolete
 exit 0
diff --git a/test/run_generator_tests.sh b/test/run_generator_tests.sh
index d5a4d3c74..79addd3ea 100755
--- a/test/run_generator_tests.sh
+++ b/test/run_generator_tests.sh
@@ -250,35 +250,26 @@ add_ini_files_from_tests()
     done
 }
 
-add_ini_files_from_all_tests()
-{
-    # Collect also those INI files for which the test has been changed
-    local all_tests=$(find ${REPO_DIR} -name "*.C" | grep "MC/.*/ini/tests")
-    local repo_dir_head=${REPO_DIR}
-    for t in ${all_tests} ; do
-        local this_test=$(realpath ${t})
-        this_test=${this_test##${repo_dir_head}/}
-        local tc=$(basename ${this_test})
-        this_test=${this_test%%/tests/*}
-        tc=${tc%.C}.ini
-        tc=${this_test}/${tc}
-        [[ "${INI_FILES}" == *"${tc}"* ]] && continue
-        INI_FILES+=" ${tc} "
-    done
-}
-
 
 collect_ini_files()
 {
     # Collect all INI files which have changed
-    local ini_files=$(get_changed_files | grep ".ini$" | grep "MC/config")
-    for ini in ${ini_files} ; do
+    local changed_files=$(get_changed_files)
+    for ini in ${changed_files} ; do
+        [[ "${ini}" != *"MC/config"*".ini" ]] && continue
         [[ "${INI_FILES}" == *"${ini}"* ]] && continue || INI_FILES+=" ${ini} "
     done
 
     # this relies on INI_FILES and MACRO_FILES_POTENTIALLY_INCLUDED
     # collect all INI files that might include some changed macros
-    add_ini_files_from_macros $(get_changed_files | grep ".C$" | grep "MC/config")
+    changed_files=$(get_changed_files)
+    local macros=
+    for m in ${changed_files} ; do
+        [[ "${m}" != *"MC/config"*".C" ]] && continue
+        macros+=" ${m} "
+    done
+
+    add_ini_files_from_macros ${macros}
 
     # this relies on MACRO_FILES_POTENTIALLY_INCLUDED
     # collect all INI files that might contain macros which in turn include changed macros
@@ -286,7 +277,13 @@ collect_ini_files()
     add_ini_files_from_macros $(find_including_macros)
 
     # also tests might have changed in which case we run them
-    add_ini_files_from_tests $(get_changed_files | grep ".C$" | grep "MC/.*/ini/tests")
+    changed_files=$(get_changed_files)
+    local macros=
+    for m in ${changed_files} ; do
+        [[ "${m}" != *"MC/"*"ini/tests"*".C" ]] && continue
+        macros+=" ${m} "
+    done
+    add_ini_files_from_tests ${macros}
 }
 
 
@@ -361,12 +358,12 @@ echo
 
 REPO_DIR=${O2DPG_TEST_REPO_DIR:-$(get_git_repo_directory)}
 if [[ ! -d ${REPO_DIR}/.git ]] ; then
-    echo_red "Directory \"${REPO_DIR}\" is not a git repository."
+    echo "ERROR: Directory \"${REPO_DIR}\" is not a git repository."
     exit 1
 fi
 
 if [[ -z ${O2DPG_ROOT+x} ]] ; then
-    echo_red "O2DPG is not loaded, probably other packages are missing as well in this environment."
+    echo "ERROR: O2DPG is not loaded, probably other packages are missing as well in this environment."
     exit 1
 fi
 
@@ -376,11 +373,6 @@ source ${REPO_DIR}/test/common/utils/utils.sh
 # Do the initial steps in the source dir where we have the full git repo
 pushd ${REPO_DIR} > /dev/null
 
-# First check, if testing itself has changed. In that case this will add INI files
-# for which a test can be found
-global_testing_changed=$(get_changed_files | grep -E "common/kine_tests/test_generic_kine.C|run_generator_tests.sh" | grep "^test/")
-[[ "${global_testing_changed}" != "" ]] && add_ini_files_from_all_tests
-
 # Then add the ini files that have changed as well. We need to do that so we get information
 # about missing tests etc.
 collect_ini_files
diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh
index 2acde102e..9962b3293 100755
--- a/test/run_workflow_tests.sh
+++ b/test/run_workflow_tests.sh
@@ -2,56 +2,30 @@
 
 # The test parent dir to be cretaed in current directory
 TEST_PARENT_DIR_PWG="o2dpg_tests/workflows_pwgs"
-TEST_PARENT_DIR_BIN="o2dpg_tests/workflows_bin"
+TEST_PARENT_DIR_BIN="o2dpg_tests/workflows_analysisqc"
 TEST_PARENT_DIR_ANCHORED="o2dpg_tests/anchored"
 
-# a global counter for tests
-TEST_COUNTER=0
-
 # unified names of log files
 LOG_FILE_WF="o2dpg-test-wf.log"
 LOG_FILE_ANCHORED="o2dpg-test-anchored.log"
-
-# Prepare some colored output
-SRED="\033[0;31m"
-SGREEN="\033[0;32m"
-SEND="\033[0m"
-
-
-echo_green()
-{
-    echo -e "${SGREEN}${*}${SEND}"
-}
+LOG_FILE_ANALYSISQC="o2dpg-test_analysisqc.log"
 
 
-echo_red()
-{
-    echo -e "${SRED}${*}${SEND}"
-}
-
 get_git_repo_directory()
 {
+    local look_dir=${1:-$(pwd)}
+    look_dir=$(realpath "${look_dir}")
+    look_dir=${look_dir%%/.git}
     local repo=
-    if [[ -d .git ]] ; then
-        pwd
-    else
+    (
+        cd "${look_dir}"
         repo=$(git rev-parse --git-dir 2> /dev/null)
-    fi
-    [[ "${repo}" != "" ]] && repo=${repo%%/.git}
+        [[ "${repo}" != "" ]] && { repo=$(realpath "${repo}") ; repo=${repo%%/.git} ; }
+    )
     echo ${repo}
 }
 
 
-get_all_workflows()
-{
-    # Collect also those INI files for which the test has been changed
-    local repo_dir_head=${REPO_DIR}
-    local grep_dir=${1}
-    local all_workflows=$(find ${repo_dir_head} -name "*.sh" | grep "${grep_dir}")
-    echo ${all_workflows}
-}
-
-
 test_single_wf()
 {
     local wf_script=${1}
@@ -59,12 +33,12 @@ test_single_wf()
     make_wf_creation_script ${wf_script} ${wf_script_local}
     local has_wf_script_local=${?}
     echo -n "Test ${TEST_COUNTER}: ${wfs}"
-    [[ "${has_wf_script_local}" != "0" ]] && { echo "No WF creation in script ${wfs} ##########" ; return 1 ; }
+    [[ "${has_wf_script_local}" != "0" ]] && { echo -n " (No WF creation in script)" ; echo_red " -> FAILED" ; return 1 ; }
     # Check if there is an "exit" other than the usual
     # [ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1
     # like ones.
     # This is not perfect but might prevent us from running into some checks the WF script does before launching the WF creation
-    [[ "$(grep exit ${wf_script_local} | grep -v "This needs")" != "" ]] && { echo -e -n "\nFound \"exit\" in ${wfs} so will not test automatically" ; return 0 ; }
+    [[ "$(grep exit ${wf_script_local} | grep -v "This needs")" != "" ]] && { echo -n " (Found \"exit\" in script, not testing automatically)" ; echo_yellow " -> WARNING" ; return 0 ; }
     # one single test
     echo "Test ${wf_line} from ${wfs}" > ${LOG_FILE_WF}
     bash ${wf_script_local} >> ${LOG_FILE_WF} 2>&1
@@ -72,6 +46,7 @@ test_single_wf()
     local ret_this_qc=0
     local ret_this_analysis=0
     if [[ "${ret_this}" != "0" ]] ; then
+        echo_red " -> FAILED"
         echo "[FATAL]: O2DPG_TEST Workflow creation failed" >> ${LOG_FILE_WF}
     elif [[ "${execute}" != "" ]] ; then
         local memlimit=${O2DPG_TEST_WORKFLOW_MEMLIMIT:+--mem-limit ${O2DPG_TEST_WORKFLOW_MEMLIMIT}}
@@ -80,7 +55,9 @@ test_single_wf()
         [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels QC ${memlimit} >> ${LOG_FILE_WF} 2>&1 ; ret_this_qc=${?} ; }
         [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels Analysis ${memlimit} >> ${LOG_FILE_WF} 2>&1 ; ret_this_analysis=${?} ; }
         ret_this=$((ret_this + ret_this_qc + ret_this_analysis))
-        [[ "${ret_this}" != "0" ]] && echo "[FATAL]: O2DPG_TEST Workflow execution failed" >> ${LOG_FILE_WF}
+        [[ "${ret_this}" != "0" ]] && echo "[FATAL]: O2DPG_TEST Workflow execution failed" >> ${LOG_FILE_WF} || echo_green " -> PASSED"
+    else
+        echo_green " -> PASSED"
     fi
     return ${ret_this}
 }
@@ -91,8 +68,8 @@ run_workflow_creation()
     local execute=
     while [ "$1" != "" ] ; do
         case $1 in
-            --execute ) shift
-                        execute=1
+            --execute ) execute=1
+                        shift
                         ;;
             * )         wf_scripts+="${1} "
                         shift
@@ -116,16 +93,34 @@ run_workflow_creation()
             local ret_this=${?}
             [[ "${ret_this}" != "0" ]] && RET=${ret_this}
         popd > /dev/null
-        if [[ "${ret_this}" != "0" ]] ; then
-            echo_red " -> FAILED"
-        else
-            echo_green " -> PASSED"
-        fi
     done
 
     return ${RET}
 }
 
+
+test_analysisqc_cli()
+{
+    ((TEST_COUNTER++))
+    local test_dir="${TEST_COUNTER}_analysisqc_cli"
+    rm -rf ${test_dir} 2> /dev/null
+    mkdir ${test_dir}
+    pushd ${test_dir} > /dev/null
+        echo "### Testing AnalysisQC creation for MC ###" > ${LOG_FILE_ANALYSISQC}
+        echo -n "Test ${TEST_COUNTER}: Running AnalysisQC CLI"
+        ${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f AO2D.root --is-mc -o wokflow_test_mc.json >> ${LOG_FILE_ANALYSISQC} 2>&1
+        local ret=${?}
+        [[ "${ret}" != "0" ]] && echo "[FATAL]: O2DPG_TEST failed" >> ${LOG_FILE_ANALYSISQC}
+        echo "### Testing AnalysisQC creation for data ###" >> ${LOG_FILE_ANALYSISQC}
+        ${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f AO2D.root -o wokflow_test_data.json >> ${LOG_FILE_ANALYSISQC} 2>&1
+        local ret_data=${?}
+        [[ "${ret_data}" != "0" ]] && { echo "[FATAL]: O2DPG_TEST failed" >> ${LOG_FILE_ANALYSISQC} ; ret=${ret_data} ; }
+    popd > /dev/null
+    [[ "${ret}" != "0" ]] && echo_red " -> FAILED" || echo_green " -> PASSED"
+    return ${ret}
+}
+
+
 test_anchored()
 {
     local to_run="${1:-${O2DPG_ROOT}/MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh}"
@@ -140,27 +135,24 @@ test_anchored()
             echo -n "Test ${TEST_COUNTER}: ${anchored_script}"
             ${anchored_script} >> ${LOG_FILE_ANCHORED} 2>&1
             local ret_this=${?}
-            [[ "${ret_this}" != "0" ]] && RET=${ret_this}
+            if [[ "${ret_this}" != "0" ]] ; then
+                echo_red " -> FAILED"
+                RET=${ret_this}
+            else
+                echo_green " -> PASSED"
+            fi
         popd > /dev/null
     done
     return ${RET}
 }
 
-collect_changed_pwg_wf_files()
-{
-    # Collect all INI files which have changed
-    local wf_scripts=$(get_changed_files | grep ".sh$" | grep "MC/run")
-    for wfs in ${wf_scripts} ; do
-        [[ "${WF_FILES}" == *"${wfs}"* ]] && continue || WF_FILES+=" ${wfs} "
-    done
-}
-
 print_usage()
 {
+
     echo
     echo "usage: run_workflow_tests.sh"
     echo
-    echo "  ENVIRONMENT VARIABLES:"
+    echo "  ENVIRONMENT VARIABLES TO DETERMINE WHAT TO COMPARE:"
     echo
     echo "  O2DPG_TEST_REPO_DIR : Point to the source repository you want to test."
     echo "  O2DPG_TEST_HASH_BASE : The base hash you want to use for comparison (optional)"
@@ -174,10 +166,15 @@ print_usage()
     echo "  If also not set, this will be set to HEAD. However, if there are unstaged"
     echo "  changes, it will left blank."
     echo
+    echo "  SPECIFIC ENVIRONMENT VARIABLES FOR THIS TEST:"
     echo "  O2DPG_TEST_WORKFLOW_MEMLIMIT : The memory limit that is passed to the workflow runner in case a workflow is executed (optional)"
     echo
 }
 
+
+#############
+# Main part #
+#############
 while [ "$1" != "" ] ; do
     case $1 in
         --help|-h )          print_usage
@@ -189,47 +186,70 @@ while [ "$1" != "" ] ; do
     esac
 done
 
-echo
-echo "##############################"
-echo "# Run O2DPG workflow testing #"
-echo "##############################"
-echo
-
+# determine the repository directory
 REPO_DIR=${O2DPG_TEST_REPO_DIR:-$(get_git_repo_directory)}
 if [[ ! -d ${REPO_DIR}/.git ]] ; then
-    echo_red "Directory \"${REPO_DIR}\" is not a git repository."
+    echo "ERROR: Directory \"${REPO_DIR}\" is not a git repository."
     exit 1
 fi
 
 if [[ -z ${O2DPG_ROOT+x} ]] ; then
-    echo_red "O2DPG is not loaded, probably other packages are missing as well in this environment."
+    echo "ERROR: O2DPG is not loaded, probably other packages are missing as well in this environment."
     exit 1
 fi
 
 # source the utilities
 source ${REPO_DIR}/test/common/utils/utils.sh
 
+
+echo "##############################"
+echo "# Run O2DPG workflow testing #"
+echo "##############################"
+
 # Do the initial steps in the source dir where we have the full git repo
 pushd ${REPO_DIR} > /dev/null
 
 # flag if anything changed in the sim workflow bin dir
-changed_wf_bin=$(get_changed_files | grep -E "MC/bin")
-changed_wf_bin_related=$(get_changed_files | grep -E "MC/analysis_testing|MC/config/analysis_testing/json|MC/config/QC/json")
-changed_anchored_related=$(get_changed_files | grep -E "MC/run/ANCHOR/anchorMC.sh|MC/run/ANCHOR/tests|MC/bin|UTILS/parse-async-WorkflowConfig.py")
-
+changed_sim_bin=$(get_changed_files | grep -E "MC/bin")
+# collect if anything has changed related to AnalysisQC
+changed_analysis_qc=$(get_changed_files | grep -E "MC/analysis_testing|MC/config/analysis_testing/json|MC/config/QC/json")
+# check if anything has changed concerning anchoring
+changed_anchored=$(get_changed_files | grep -E "MC/bin|MC/run/ANCHOR/anchorMC.sh|MC/run/ANCHOR/tests|MC/bin|UTILS/parse-async-WorkflowConfig.py|DATA/production/configurations/asyncReco/setenv_extra.sh|DATA/production/configurations/asyncReco/async_pass.sh|DATA/common/setenv.sh|DATA/production/workflow-multiplicities.sh")
+# collect changed workflow scripts
+changed_workflows=
+# workflows to be executed
+execute_workflows=
+echo "==> Test outline"
+if [[ "${changed_sim_bin}" != "" ]] ; then
+    # in this case, something central has changed, test creation of all workflows against it
+    echo "  - The creation of simulation workflows from all run scripts (MC/run/**/*.sh) will be tested."
+    for p in $(find MC/run -name "*.sh") ; do
+        changed_workflows+="$(realpath ${p}) "
+    done
+    # definitely run anchored if central python scripts have changed
+    echo "    - Changes in MC/bin/ detected, mark anchored MC test to be run."
+    changed_anchored="1"
+else
+    # otherwise, only take the changed shell scripts
+    changed_workflows=
+    changed_files=$(get_changed_files)
+    for cf in ${changed_files} ; do
+        [[ "${cf}" != *"MC/run"*".sh" ]] && continue
+        changed_workflows+="${cf} "
+    done
+    [[ "${changed_workflows}" != "" ]] && echo "  - The creation of simulation workflows from changed run scripts (sub-sect of MC/run/**/*.sh) will be tested."
+fi
 
-# collect what has changed for PWGs
-collect_changed_pwg_wf_files
+if [[ "${changed_analysis_qc}" != "" || "${changed_sim_bin}" ]] ; then
+    for p in $(find "MC/bin/tests" -name "*.sh") ; do
+        execute_workflows+="$(realpath ${p}) "
+    done
+    echo "  - Test AnalysisQC CLI and execution with a simulation."
+fi
 
-# get realpaths for all changes
-wf_files_tmp=${WF_FILES}
-WF_FILES=
-for wf_tmp in ${wf_files_tmp} ; do
-    # convert to full path so that we can find it from anywhere
-    WF_FILES+="$(realpath ${wf_tmp}) "
-done
+[[ "${changed_anchored}" != "" ]] && echo "  - Test anchored simulation."
 
-# go back to where we came from
+# everything collected, go back to where we came from
 popd > /dev/null
 REPO_DIR=$(realpath ${REPO_DIR})
 
@@ -241,111 +261,83 @@ REPO_DIR=$(realpath ${REPO_DIR})
 export O2DPG_ROOT=${REPO_DIR}
 
 
-###############
-# ANCHORED MC #
-###############
-# prepare our local test directory for PWG tests
-rm -rf ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null
-mkdir -p ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null
-pushd ${TEST_PARENT_DIR_ANCHORED} > /dev/null
-
-# global return code for PWGs
-ret_global_anchored=0
-if [[ "${changed_anchored_related}" != "" ]] ; then
-    echo "### Test anchored ###"
-    # Run an anchored test
-    test_anchored
-    ret_global_anchored=${?}
-    echo
-fi
-
-# return to where we came from
-popd > /dev/null
-
-########
-# PWGs #
-########
-# prepare our local test directory for PWG tests
-rm -rf ${TEST_PARENT_DIR_PWG} 2>/dev/null
-mkdir -p ${TEST_PARENT_DIR_PWG} 2>/dev/null
-pushd ${TEST_PARENT_DIR_PWG} > /dev/null
-
+##############################
+# PWG workflow shell scripts #
+##############################
 # global return code for PWGs
 ret_global_pwg=0
-if [[ "${changed_wf_bin}" != "" ]] ; then
-    # Run all the PWG related WF creations, hence overwrite what was collected by collect_changed_pwg_wf_files earlier
-    WF_FILES=$(get_all_workflows "MC/run/.*/")
-    echo
-fi
 
 # Test what we found
-if [[ "${WF_FILES}" != "" ]] ; then
-    echo "### Test PWG-related workflow creation ###"
+if [[ "${changed_workflows}" != "" ]] ; then
+    # prepare our local test directory for PWG tests
+    rm -rf ${TEST_PARENT_DIR_PWG} 2>/dev/null
+    mkdir -p ${TEST_PARENT_DIR_PWG} 2>/dev/null
+    pushd ${TEST_PARENT_DIR_PWG} > /dev/null
+
     echo
-    run_workflow_creation ${WF_FILES}
+    echo "==> START BLOCK: Test PWG-related workflow creation <=="
+    run_workflow_creation ${changed_workflows}
     ret_global_pwg=${?}
-    echo
+    [[ "${ret_global_pwg}" != "0" ]] && { echo "WARNING for workflows creations, some could not be built." ;  print_error_logs ./ ; }
+    echo "==> END BLOCK: Test PWG-related workflow creation <=="
+
+    # return to where we came from
+    popd > /dev/null
 fi
 
-# return to where we came from
-popd > /dev/null
 
-####################
-# sim workflow bin #
-####################
+####################################
+# sim workflow bin with AnalysisQC #
+####################################
 # prepare our local test directory for bin tests
-rm -rf ${TEST_PARENT_DIR_BIN} 2>/dev/null
-mkdir -p ${TEST_PARENT_DIR_BIN} 2>/dev/null
-pushd ${TEST_PARENT_DIR_BIN} > /dev/null
-
 # global return code for PWGs
-ret_global_bin=0
-if [[ "${changed_wf_bin}" != "" || "${changed_wf_bin_related}" != "" ]] ; then
-    echo "### Test bin-related workflow creation ###"
+ret_analysis_qc=0
+if [[ "${changed_analysis_qc}" != "" ]] ; then
+    rm -rf ${TEST_PARENT_DIR_BIN} 2>/dev/null
+    mkdir -p ${TEST_PARENT_DIR_BIN} 2>/dev/null
+    pushd ${TEST_PARENT_DIR_BIN} > /dev/null
+
     echo
+    echo "==> START BLOCK: Test running workflow with AnalysisQC <=="
+    # test command line interface
+    test_analysisqc_cli
+    ret_analysis_qc=${?}
     # Run all the bin test WF creations
-    run_workflow_creation $(get_all_workflows "MC/bin/tests") --execute
-    ret_global_bin=${?}
-    echo
+    [[ "${ret_analysis_qc}" == "0" ]] && { run_workflow_creation ${execute_workflows} --execute ; ret_analysis_qc=${?} ; }
+    [[ "${ret_analysis_qc}" != "0" ]] && { echo "ERROR for workflows execution and AnalysisQC." ;  print_error_logs ./ ; }
+    echo "==> END BLOCK: Test running workflow with AnalysisQC <=="
+
+    # return to where we came from
+    popd > /dev/null
 fi
 
-# return to where we came from
-popd > /dev/null
 
-# final printing of log files of failed tests
-# For PWG workflows, this triggers only a warning at the moment
-if [[ "${ret_global_pwg}" != "0" ]] ; then
-    echo
-    echo "#####################################"
-    echo "# WARNING for PWG-related workflows #"
-    echo "#####################################"
-    echo
-    print_error_logs ${TEST_PARENT_DIR_PWG}
-fi
+###############
+# ANCHORED MC #
+###############
+# global return code for PWGs
+ret_global_anchored=0
+if [[ "${changed_anchored}" != "" ]] ; then
+    # prepare our local test directory for PWG tests
+    rm -rf ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null
+    mkdir -p ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null
+    pushd ${TEST_PARENT_DIR_ANCHORED} > /dev/null
 
-# However, if a central test fails, exit code will be !=0
-if [[ "${ret_global_bin}" != "0" ]] ; then
-    echo
-    echo "###################################"
-    echo "# ERROR for bin-related workflows #"
-    echo "###################################"
     echo
-    print_error_logs ${TEST_PARENT_DIR_BIN}
-fi
+    echo "==> START BLOCK: Test anchored simulation"
+    # Run an anchored test
+    test_anchored
+    ret_global_anchored=${?}
+    [[ "${ret_global_anchored}" != "0" ]] && { echo "ERROR executing anchored simulation." ;  print_error_logs ./ ; }
+    echo "==> END BLOCK: Test anchored simulation"
 
-# However, if a central test fails, exit code will be !=0
-if [[ "${ret_global_anchored}" != "0" ]] ; then
-    echo
-    echo "##########################"
-    echo "# ERROR for anchored MCs #"
-    echo "##########################"
-    echo
-    print_error_logs ${TEST_PARENT_DIR_ANCHORED}
+    # return to where we came from
+    popd > /dev/null
 fi
 
-RET=$(( ret_global_bin + ret_global_anchored ))
+RET=$(( ret_analysis_qc + ret_global_anchored ))
 
 echo
-[[ "${RET}" != "0" ]] && echo "There were errors, please check!" || echo_green "All required workflow tests successful"
+[[ "${RET}" != "0" ]] && echo_red "There were errors, please check!" || echo_green "All required workflow tests successful"
 
 exit ${RET}