From 5208067ad76b66c007c903a7a7a995928ea8d526 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 25 Nov 2022 15:31:23 +0100 Subject: [PATCH 1/4] prune docker containers at the start of the loop otherwise the 1st job does not profit from this --- planemo_ci_actions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planemo_ci_actions.sh b/planemo_ci_actions.sh index 9ce2bbb..a805bd5 100755 --- a/planemo_ci_actions.sh +++ b/planemo_ci_actions.sh @@ -157,6 +157,7 @@ if [ "$MODE" == "test" ]; then mkdir -p json_output touch .tt_biocontainer_skip while read -r -a TOOL_GROUP; do + docker system prune --all --force --volumes || true # Check if any of the lines in .tt_biocontainer_skip is a substring of $TOOL_GROUP if echo "${TOOL_GROUP[@]}" | grep -qf .tt_biocontainer_skip; then PLANEMO_OPTIONS=() @@ -168,7 +169,6 @@ if [ "$MODE" == "test" ]; then fi json=$(mktemp -u -p json_output --suff .json) PIP_QUIET=1 planemo test "${PLANEMO_OPTIONS[@]}" "${PLANEMO_TEST_OPTIONS[@]}" --test_output_json "$json" "${TOOL_GROUP[@]}" "${ADDITIONAL_PLANEMO_OPTIONS[@]}" || true - docker system prune --all --force --volumes || true done < tool_list_chunk.txt if [ ! -s tool_list_chunk.txt ]; then From 88ae1d84e2a8dcde4254b1ed70d04bd9d71d84f0 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 17 Jul 2023 17:38:40 +0200 Subject: [PATCH 2/4] implement as infinite loop --- planemo_ci_actions.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/planemo_ci_actions.sh b/planemo_ci_actions.sh index a805bd5..d2ff293 100755 --- a/planemo_ci_actions.sh +++ b/planemo_ci_actions.sh @@ -1,6 +1,14 @@ #!/usr/bin/env bash set -exo pipefail +# function running an infinite loop pruning (unused) docker images +function background_prune_docker { + while 1; do + docker system prune --all --volumes + sleep 60 + done +} + PLANEMO_TEST_OPTIONS=("--database_connection" "$DATABASE_CONNECTION" "--galaxy_source" "https://github.com/$GALAXY_FORK/galaxy" "--galaxy_branch" "$GALAXY_BRANCH" "--galaxy_python_version" "$PYTHON_VERSION" --test_timeout "$TEST_TIMEOUT") PLANEMO_CONTAINER_DEPENDENCIES=("--biocontainers" "--no_dependency_resolution" "--no_conda_auto_init" "--docker_extra_volume" "./") PLANEMO_WORKFLOW_OPTIONS=("--tool_data_table" "/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml" "--tool_data_table" "/cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml" "--docker_extra_volume" "/cvmfs" --shed_tool_conf /tmp/shed_tool_conf.xml --shed_tool_path /tmp/shed_dir) @@ -152,12 +160,14 @@ if [ "$MODE" == "test" ]; then # show tools cat tool_list_chunk.txt - + + background_prune_docker & + PRUNE_PID=$! + # Test tools mkdir -p json_output touch .tt_biocontainer_skip while read -r -a TOOL_GROUP; do - docker system prune --all --force --volumes || true # Check if any of the lines in .tt_biocontainer_skip is a substring of $TOOL_GROUP if echo "${TOOL_GROUP[@]}" | grep -qf .tt_biocontainer_skip; then PLANEMO_OPTIONS=() @@ -179,6 +189,7 @@ if [ "$MODE" == "test" ]; then planemo test_reports tool_test_output.json --test_output tool_test_output.html mv tool_test_output.json tool_test_output.html upload/ + kill "$PRUNE_PID" fi # combine reports mode From 604a3c5c0911f0af69283055577e201f8cc3b19a Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 17 Jul 2023 18:02:24 +0200 Subject: [PATCH 3/4] force and redirect --- planemo_ci_actions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/planemo_ci_actions.sh b/planemo_ci_actions.sh index d2ff293..1c5d9fa 100755 --- a/planemo_ci_actions.sh +++ b/planemo_ci_actions.sh @@ -4,7 +4,7 @@ set -exo pipefail # function running an infinite loop pruning (unused) docker images function background_prune_docker { while 1; do - docker system prune --all --volumes + docker system prune --all --volumes --force sleep 60 done } @@ -161,7 +161,7 @@ if [ "$MODE" == "test" ]; then # show tools cat tool_list_chunk.txt - background_prune_docker & + background_prune_docker &> /dev/null & PRUNE_PID=$! # Test tools From 9302ac13c39439b2ab85cd38ab6b6d49ef83b7f3 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 17 Jul 2023 18:38:45 +0200 Subject: [PATCH 4/4] fix --- planemo_ci_actions.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/planemo_ci_actions.sh b/planemo_ci_actions.sh index 1c5d9fa..8adec31 100755 --- a/planemo_ci_actions.sh +++ b/planemo_ci_actions.sh @@ -3,8 +3,8 @@ set -exo pipefail # function running an infinite loop pruning (unused) docker images function background_prune_docker { - while 1; do - docker system prune --all --volumes --force + while true; do + docker system prune --all --volumes --force &> /dev/null sleep 60 done } @@ -161,7 +161,7 @@ if [ "$MODE" == "test" ]; then # show tools cat tool_list_chunk.txt - background_prune_docker &> /dev/null & + background_prune_docker & PRUNE_PID=$! # Test tools @@ -189,6 +189,7 @@ if [ "$MODE" == "test" ]; then planemo test_reports tool_test_output.json --test_output tool_test_output.html mv tool_test_output.json tool_test_output.html upload/ + kill "$PRUNE_PID" fi