From 0b67809d522e9bb38233caa9e9d554b347f02650 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 9 Aug 2022 13:53:13 -0500 Subject: [PATCH 001/196] Divided glidein_startup.sh in modules --- creation/web_base/glidein_cleanup.sh | 197 +++ creation/web_base/glidein_startup.sh | 1834 ++----------------------- creation/web_base/utils_crypto.sh | 86 ++ creation/web_base/utils_fetch.sh | 375 +++++ creation/web_base/utils_filesystem.sh | 253 ++++ creation/web_base/utils_http.sh | 158 +++ creation/web_base/utils_log.sh | 156 +++ creation/web_base/utils_params.sh | 268 ++++ creation/web_base/utils_signals.sh | 51 + creation/web_base/utils_tarballs.sh | 81 ++ creation/web_base/utils_xml.sh | 265 ++++ 11 files changed, 2024 insertions(+), 1700 deletions(-) create mode 100644 creation/web_base/glidein_cleanup.sh create mode 100644 creation/web_base/utils_crypto.sh create mode 100644 creation/web_base/utils_fetch.sh create mode 100644 creation/web_base/utils_filesystem.sh create mode 100644 creation/web_base/utils_http.sh create mode 100644 creation/web_base/utils_log.sh create mode 100644 creation/web_base/utils_params.sh create mode 100644 creation/web_base/utils_signals.sh create mode 100644 creation/web_base/utils_tarballs.sh create mode 100644 creation/web_base/utils_xml.sh diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh new file mode 100644 index 0000000000..a267e00313 --- /dev/null +++ b/creation/web_base/glidein_cleanup.sh @@ -0,0 +1,197 @@ +. utils_tarballs.sh +. utils_signals.sh +. utils_log.sh +. utils_xml.sh +. utils_filesystem.sh +. utils_params.sh +. utils_crypto.sh +. utils_http.sh +. utils_fetch.sh + +work_dir_created=0 +glide_local_tmp_dir_created=0 + +################################ +# Function used to clean up the glidein. +# It cleans-up, print out the message and exit +# It removes Glidein directories (work_dir, glide_local_tmp_dir) +# It uses GLIDEIN_DEBUG_OPTIONS, start_dir, work_dir_created, work_dir, glide_local_tmp_dir_created, glide_local_tmp_dir +# Arguments: +# 1: exit code +glidien_cleanup() { + if ! cd "${start_dir}"; then + log_warn "Cannot find ${start_dir} anymore, exiting but without cleanup" + else + if [[ ",${GLIDEIN_DEBUG_OPTIONS}," = *,nocleanup,* ]]; then + log_warn "Skipping cleanup, disabled via GLIDEIN_DEBUG_OPTIONS" + else + if [ "${work_dir_created}" -eq "1" ]; then + # rm -fR does not remove directories read only for the user + find "${work_dir}" -type d -exec chmod u+w {} \; + rm -fR "${work_dir}" + fi + if [ "${glide_local_tmp_dir_created}" -eq "1" ]; then + find "${glide_local_tmp_dir}" -type d -exec chmod u+w {} \; + rm -fR "${glide_local_tmp_dir}" + fi + fi + fi +} + +################################ +# Function used for early failures of the glidein, when we cannot assume we can write to disk at all +# too bad we end up with some repeated code, but difficult to do better +# Arguments: +# 1: error message +early_glidein_failure() { + result = "%s + + %s" + error_msg="$1" + log_warn "${error_msg}" + sleep "${sleep_time}" + # wait a bit in case of error, to reduce lost glideins + glidein_end_time="$(date +%s)" + printf "$result" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "WN_RESOURCE" "${error_msg}" + #result=" WN_RESOURCE + # ERROR + # + # ${error_msg} + # " + final_result="$(construct_xml "${result}")" + final_result_simple="$(basexml2simplexml "${final_result}")" + # have no global section + final_result_long="$(simplexml2longxml "${final_result_simple}" "")" + glidien_cleanup + print_tail 1 "${final_result_simple}" "${final_result_long}" + exit 1 +} + +################################ +# Function used for exiting the glidein, to be used when the most basic ops have been done +# too bad we end up with some repeated code, but difficult to do better +# Arguments: +# 1: exit code +glidein_exit() { + exit_code=$1 + # Removed lines about $lock_file (lock file for whole machine) not present elsewhere + gwms_process_scripts "$GWMS_DIR" cleanup "${glidein_config}" + global_result="" + if [ -f otr_outlist.list ]; then + global_result=$(cat otr_outlist.list) + chmod u+w otr_outlist.list + fi + ge_last_script_name=$(extract_parent_fname "${exit_code}") + result=$(extract_parent_xml_detail "${exit_code}") + final_result=$(construct_xml "${result}") + # augment with node info + final_result_simple=$(basexml2simplexml "${final_result}") + # Create a richer version, too + final_result_long=$(simplexml2longxml "${final_result_simple}" "${global_result}") + if [ "${exit_code}" -ne 0 ]; then + report_failed=$(grep -i "^GLIDEIN_Report_Failed " "${glidein_config}" | cut -d ' ' -f 2-) + + if [ -z "${report_failed}" ]; then + report_failed="NEVER" + fi + + factory_report_failed=$(grep -i "^GLIDEIN_Factory_Report_Failed " "${glidein_config}" | cut -d ' ' -f 2-) + + if [ -z "${factory_report_failed}" ]; then + factory_collector=$(grep -i "^GLIDEIN_Factory_Collector " "${glidein_config}" | cut -d ' ' -f 2-) + if [ -z "${factory_collector}" ]; then + # no point in enabling it if there are no collectors + factory_report_failed="NEVER" + else + factory_report_failed="ALIVEONLY" + fi + fi + do_report=0 + if [ "${report_failed}" != "NEVER" ] || [ "${factory_report_failed}" != "NEVER" ]; then + do_report=1 + fi + # wait a bit in case of error, to reduce lost glideins + let "dl=$(date +%s) + ${sleep_time}" + dlf=$(date --date="@${dl}") + add_config_line "GLIDEIN_ADVERTISE_ONLY" "1" + add_config_line "GLIDEIN_Failed" "True" + add_config_line "GLIDEIN_EXIT_CODE" "$1" + add_config_line "GLIDEIN_ToDie" "${dl}" + add_config_line "GLIDEIN_Expire" "${dl}" + add_config_line "GLIDEIN_LAST_SCRIPT" "${ge_last_script_name}" + add_config_line "GLIDEIN_ADVERTISE_TYPE" "Retiring" + + add_config_line "GLIDEIN_FAILURE_REASON" "Glidein failed while running ${ge_last_script_name}. Keeping node busy until ${dl} (${dlf})." + + condor_vars_file="$(grep -i "^CONDOR_VARS_FILE " "${glidein_config}" | cut -d ' ' -f 2-)" + if [ -n "${condor_vars_file}" ]; then + # if we are to advertise, this should be available... else, it does not matter anyhow + add_condor_vars_line "GLIDEIN_ADVERTISE_ONLY" "C" "True" "+" "Y" "Y" "-" + add_condor_vars_line "GLIDEIN_Failed" "C" "True" "+" "Y" "Y" "-" + add_condor_vars_line "GLIDEIN_EXIT_CODE" "I" "-" "+" "Y" "Y" "-" + add_condor_vars_line "GLIDEIN_ToDie" "I" "-" "+" "Y" "Y" "-" + add_condor_vars_line "GLIDEIN_Expire" "I" "-" "+" "Y" "Y" "-" + add_condor_vars_line "GLIDEIN_LAST_SCRIPT" "S" "-" "+" "Y" "Y" "-" + add_condor_vars_line "GLIDEIN_FAILURE_REASON" "S" "-" "+" "Y" "Y" "-" + fi + main_work_dir="$(get_work_dir main)" + + for ((t=$(date +%s); t < dl; t=$(date +%s))) + do + if [ -e "${main_work_dir}/${last_script}" ] && [ "${do_report}" = "1" ] ; then + # if the file exists, we should be able to talk to the collectors + # notify that things went badly and we are waiting + if [ "${factory_report_failed}" != "NEVER" ]; then + add_config_line "GLIDEIN_ADVERTISE_DESTINATION" "Factory" + log_warn "Notifying Factory of error" + "${main_work_dir}/${last_script}" glidein_config + fi + if [ "${report_failed}" != "NEVER" ]; then + add_config_line "GLIDEIN_ADVERTISE_DESTINATION" "VO" + log_warn "Notifying VO of error" + "${main_work_dir}/${last_script}" glidein_config + fi + fi + # sleep for about 5 mins... but randomize a bit + let "ds=250+${RANDOM}%100" + let "as=$(date +%s) + ${ds}" + if [ ${as} -gt ${dl} ]; then + # too long, shorten to the deadline + let "ds=${dl} - $(date +%s)" + fi + log_warn "Sleeping ${ds}" + sleep ${ds} + done + if [ -e "${main_work_dir}/${last_script}" ] && [ "${do_report}" = "1" ]; then + # notify that things went badly and we are going away + if [ "${factory_report_failed}" != "NEVER" ]; then + add_config_line "GLIDEIN_ADVERTISE_DESTINATION" "Factory" + if [ "${factory_report_failed}" = "ALIVEONLY" ]; then + add_config_line "GLIDEIN_ADVERTISE_TYPE" "INVALIDATE" + else + add_config_line "GLIDEIN_ADVERTISE_TYPE" "Killing" + add_config_line "GLIDEIN_FAILURE_REASON" "Glidein failed while running ${ge_last_script_name}. Terminating now. (${dl}) (${dlf})" + fi + "${main_work_dir}/${last_script}" glidein_config + log_warn "Last notification sent to Factory" + fi + if [ "${report_failed}" != "NEVER" ]; then + add_config_line "GLIDEIN_ADVERTISE_DESTINATION" "VO" + if [ "${report_failed}" = "ALIVEONLY" ]; then + add_config_line "GLIDEIN_ADVERTISE_TYPE" "INVALIDATE" + else + add_config_line "GLIDEIN_ADVERTISE_TYPE" "Killing" + add_config_line "GLIDEIN_FAILURE_REASON" "Glidein failed while running ${ge_last_script_name}. Terminating now. (${dl}) (${dlf})" + fi + "${main_work_dir}/${last_script}" glidein_config + log_warn "Last notification sent to VO" + fi + fi + fi + log_write "glidein_startup.sh" "text" "glidein is about to exit with retcode $1" "info" + send_logs_to_remote + glidien_cleanup + print_tail "$1" "${final_result_simple}" "${final_result_long}" + exit "$1" +} + diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 993dd8e38f..4eefc786cf 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1,120 +1,59 @@ #!/bin/bash +#*******************************************************************# +# glidein_startup.sh # +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # +# SPDX-License-Identifier: Apache-2.0 # +# File Version: # +#*******************************************************************# -# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC -# SPDX-License-Identifier: Apache-2.0 -# -# Project: -# glideinWMS -# -# File Version: -# - -# default IFS, to protect against unusual environment, better than "unset IFS" because works with restoring old one +################################ +# Default IFS, to protect against unusual environment +# better than "unset IFS" because works with restoring old one IFS=$' \t\n' global_args="$*" + # GWMS_STARTUP_SCRIPT=$0 GWMS_STARTUP_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" -GWMS_PATH="" + +################################ # Relative to the work directory (GWMS_DIR, gwms_lib_dir, gwms_bin_dir and gwms_exec_dir will be the absolute paths) # bin (utilities), lib (libraries), exec (aux scripts to be executed/sourced, e.g. pre-job) +GWMS_PATH="" GWMS_SUBDIR=".gwms.d" export LANG=C - +################################ # General options -GWMS_MULTIUSER_GLIDEIN= # Set GWMS_MULTIUSER_GLIDEIN if the Glidein may spawn processes (for jobs) as a different user. # This will prepare the glidein, e.g. setting to 777 the permission of TEMP directories # This should never happen only when using GlExec. Not in Singularity, not w/o sudo mechanisms. # Comment the following line if GlExec or similar will not be used +GWMS_MULTIUSER_GLIDEIN= #GWMS_MULTIUSER_GLIDEIN=true + # Default GWMS log server GWMS_LOGSERVER_ADDRESS='https://fermicloud152.fnal.gov/log' - -############################## -# Utility functions to allow the script to source functions and retrieve data stored as tarball at the end of the script itself - -get_data() { - # Retrieve the specified data, which is appended as tarball - # 1: selected file - sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar xz -O "$1" -} - -source_data() { - # Source the specified data, which is appended as tarball, without saving it - # 1: selected file - local data - data=$(get_data "$1") - [[ -n "$data" ]] && eval "$data" -} - -list_data() { - # Show a list of the payload tarballed files in this script - sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar tz -} - -extract_all_data() { - # Extract and source all the tarball files - local -a files - # change separator to split the output file list from 'tar tz' command - local IFS_OLD="${IFS}" - IFS=$'\n' - files=($(list_data)) - for f in "${files[@]}"; do - echo "Extracting file ${f}" - get_data "${f}" > "${f}" - echo "Sourcing file ${f}" - # source_data "${f}" - can source the file saved instead of re-extracting it - . "${f}" - done - IFS="${IFS_OLD}" -} - -################################ -# Extends 'trap' allowing to pass the signal name as argument to the handler -trap_with_arg() { - func="$1" ; shift - for sig ; do - # shellcheck disable=SC2064 - trap "${func} ${sig}" "${sig}" - done -} - -#function to handle passing signals to the child processes -# no need to re-raise sigint, caller does unconditional exit (https://www.cons.org/cracauer/sigint.html) -on_die() { - echo "Received kill signal... shutting down child processes (forwarding $1 signal)" 1>&2 - ON_DIE=1 - kill -s "$1" %1 -} - GWMS_MULTIGLIDEIN_CHILDS= -on_die_multi() { - echo "Multi-Glidein received signal... shutting down child glideins (forwarding $1 signal to ${GWMS_MULTIGLIDEIN_CHILDS})" 1>&2 - ON_DIE=1 - for i in ${GWMS_MULTIGLIDEIN_CHILDS}; do - kill -s "$1" "${i}" - done -} - -ignore_signal() { - echo "Ignoring SIGHUP signal... Use SIGTERM or SIGQUIT to kill processes" 1>&2 -} -warn() { - echo "WARN $(date)" "$@" 1>&2 -} - -logdebug() { - echo "DEBUG $(date)" "$@" 1>&2 -} +. utils_tarballs.sh +. utils_signals.sh +. utils_log.sh +. utils_xml.sh +. utils_filesystem.sh +. utils_params.sh +. utils_fetch.sh +. utils_crypto.sh +. utils_http.sh +. glidein_cleanup.sh -# Functions to start multiple glideins +################################ +# Function used to start multiple glideins +# Arguments: +# 1: prefix (of the files to skip) +# 2: directory copy_all() { - # 1:prefix (of the files to skip), 2:directory - # should it copy also hidden files? mkdir -p "$2" for f in *; do [[ -e "${f}" ]] || break # TODO: should this be a continue? @@ -124,17 +63,21 @@ copy_all() { cp -r "${f}" "$2"/ done } +# TODO: should it copy also hidden files? +################################ +# Function used to start all glideins +# Arguments: +# 1: number of glideins +# Important Variables: +# GLIDEIN_MULTIGLIDEIN_LAUNCHALL - if set in attrs, command to start all Glideins at once (multirestart 0) +# GLIDEIN_MULTIGLIDEIN_LAUNCHER - if set in attrs, command to start the individual Glideins do_start_all() { - # 1:number of glideins - # GLIDEIN_MULTIGLIDEIN_LAUNCHALL - if set in attrs, command to start all Glideins at once (multirestart 0) - # GLIDEIN_MULTIGLIDEIN_LAUNCHER - if set in attrs, command to start the individual Glideins local num_glideins initial_dir multiglidein_launchall multiglidein_launcher num_glideins=$1 initial_dir="$(pwd)" multiglidein_launchall=$(params_decode "$(params_get_simple GLIDEIN_MULTIGLIDEIN_LAUNCHALL "${params}")") multiglidein_launcher=$(params_decode "$(params_get_simple GLIDEIN_MULTIGLIDEIN_LAUNCHER "${params}")") - local startup_script="${GWMS_STARTUP_SCRIPT}" if [[ -n "${multiglidein_launchall}" ]]; then echo "Starting multi-glidein using launcher: ${multiglidein_launchall}" @@ -160,1042 +103,112 @@ do_start_all() { fi } -usage() { - echo "Usage: glidein_startup.sh " - echo "where is:" - echo " -factory : name of this factory" - echo " -name : name of this glidein" - echo " -entry : name of this glidein entry" - echo " -clientname : name of the requesting client" - echo " -clientgroup : group name of the requesting client" - echo " -web : base URL from where to fetch" - echo " -proxy : URL of the local proxy" - echo " -dir : directory ID (supports ., Condor, CONDOR, OSG, TMPDIR, AUTO)" - echo " -sign : signature of the signature file" - echo " -signtype : type of signature (only sha1 supported for now)" - echo " -signentry : signature of the entry signature file" - echo " -cluster : condorG ClusterId" - echo " -subcluster : condorG ProcId" - echo " -submitcredid : Credential ID of this condorG job" - echo " -schedd : condorG Schedd Name" - echo " -descript : description file name" - echo " -descriptentry : description file name for entry" - echo " -clientweb : base URL from where to fetch client files" - echo " -clientwebgroup : base URL from where to fetch client group files" - echo " -clientsign : signature of the client signature file" - echo " -clientsigntype : type of client signature (only sha1 supported for now)" - echo " -clientsigngroup : signature of the client group signature file" - echo " -clientdescript : client description file name" - echo " -clientdescriptgroup : client description file name for group" - echo " -slotslayout : how Condor will set up slots (fixed, partitionable)" - echo " -v : operation mode (std, nodebug, fast, check supported)" - echo " -multiglidein : spawn multiple () glideins (unless also multirestart is set)" - echo " -multirestart : started as one of multiple glideins (glidein number )" - echo " -param_* : user specified parameters" - exit 1 -} - - -# params will contain the full list of parameters -# -param_XXX YYY will become "XXX YYY" -# TODO: can use an array instead? -params="" - -while [ $# -gt 0 ] -do case "$1" in - -factory) glidein_factory="$2";; - -name) glidein_name="$2";; - -entry) glidein_entry="$2";; - -clientname) client_name="$2";; - -clientgroup) client_group="$2";; - -web) repository_url="$2";; - -proxy) proxy_url="$2";; - -dir) work_dir="$2";; - -sign) sign_id="$2";; - -signtype) sign_type="$2";; - -signentry) sign_entry_id="$2";; - -cluster) condorg_cluster="$2";; - -subcluster) condorg_subcluster="$2";; - -submitcredid) glidein_cred_id="$2";; - -schedd) condorg_schedd="$2";; - -descript) descript_file="$2";; - -descriptentry) descript_entry_file="$2";; - -clientweb) client_repository_url="$2";; - -clientwebgroup) client_repository_group_url="$2";; - -clientsign) client_sign_id="$2";; - -clientsigntype) client_sign_type="$2";; - -clientsigngroup) client_sign_group_id="$2";; - -clientdescript) client_descript_file="$2";; - -clientdescriptgroup) client_descript_group_file="$2";; - -slotslayout) slots_layout="$2";; - -v) operation_mode="$2";; - -multiglidein) multi_glidein="$2";; - -multirestart) multi_glidein_restart="$2";; - -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; - *) (warn "Unknown option $1"; usage) 1>&2; exit 1 -esac -shift 2 -done - -# make sure we have a valid slots_layout -if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then - slots_layout="fixed" -else - slots_layout="partitionable" -fi - -construct_xml() { - result="$1" - - glidein_end_time="$(date +%s)" - - echo " - - - ${start_dir} - - - $0 ${global_args} - $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) - $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z) - -${result} -" -} - - -extract_parent_fname() { - exitcode=$1 - - if [ -s otrx_output.xml ]; then - # file exists and is not 0 size - last_result=$(cat otrx_output.xml) - - if [ "${exitcode}" -eq 0 ]; then - echo "SUCCESS" - else - last_script_name=$(echo "${last_result}" |awk '/" - echo " OK" - # propagate metrics as well - echo "${last_result}" | grep '" - else - last_script_name=$(echo "${last_result}" |awk '//{fr=0;}{if (fr==1) print $0}//{fr=1;}') - my_reason=" Validation failed in ${last_script_name}. - -${last_script_reason}" - - echo " " - echo " ERROR - ${last_script_name}" - # propagate metrics as well (will include the failure metric) - echo "${last_result}" | grep '" - echo " -${my_reason} - " - fi - else - # create a minimal XML file, else - echo " " - if [ "${exitcode}" -eq 0 ]; then - echo " OK" - else - echo " ERROR" - echo " Unknown" - fi - echo " - - No detail. Could not find source XML file. - " - fi -} - -basexml2simplexml() { - final_result="$1" - - # augment with node info - echo "${final_result}" | awk 'BEGIN{fr=1;}{if (fr==1) print $0}//{fr=0;}' - - echo " ${client_name}" - echo " ${client_group}" - - echo " $(id -un)" - echo " $(uname -m)" - if [ -e '/etc/redhat-release' ]; then - echo " $(cat /etc/redhat-release)" - fi - echo " $(uname -n)" - - echo "${final_result}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=1;}' -} - -simplexml2longxml() { - final_result_simple="$1" - global_result="$2" - - echo "${final_result_simple}" | awk 'BEGIN{fr=1;}{if (fr==1) print $0}/' - echo ' ' - echo "${global_result}" | awk '{print " " $0}' - echo ' ' - echo ' ' - fi - - echo "${final_result_simple}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=0;}' - - echo " ${glidein_factory}" - echo " ${glidein_name}" - echo " ${glidein_entry}" - echo " ${condorg_cluster}" - echo " ${condorg_subcluster}" - echo " ${glidein_cred_id}" - echo " ${condorg_schedd}" - - echo "${final_result_simple}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=1;}' -} - -print_tail() { - exit_code=$1 - final_result_simple="$2" - final_result_long="$3" - - glidein_end_time=$(date +%s) - let total_time=${glidein_end_time}-${startup_time} - echo "=== Glidein ending $(date) (${glidein_end_time}) with code ${exit_code} after ${total_time} ===" - echo "" - echo "=== XML description of glidein activity ===" - echo "${final_result_simple}" | grep -v "" - echo "=== End XML description of glidein activity ===" - - echo "" 1>&2 - echo "=== Encoded XML description of glidein activity ===" 1>&2 - echo "${final_result_long}" | gzip --stdout - | b64uuencode 1>&2 - echo "=== End encoded XML description of glidein activity ===" 1>&2 -} - -#################################### -# Cleaup, print out message and exit -work_dir_created=0 -glide_local_tmp_dir_created=0 - - -glidien_cleanup() { - # Remove Glidein directories (work_dir, glide_local_tmp_dir) - # 1 - exit code - # Using GLIDEIN_DEBUG_OPTIONS, start_dir, work_dir_created, work_dir, - # glide_local_tmp_dir_created, glide_local_tmp_dir - if ! cd "${start_dir}"; then - warn "Cannot find ${start_dir} anymore, exiting but without cleanup" +################################ +# Function used to set the slots_layout +# make sure to have a valid slots_layout +set_slots_layout(){ + if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then + slots_layout="fixed" else - if [[ ",${GLIDEIN_DEBUG_OPTIONS}," = *,nocleanup,* ]]; then - warn "Skipping cleanup, disabled via GLIDEIN_DEBUG_OPTIONS" - else - if [ "${work_dir_created}" -eq "1" ]; then - # rm -fR does not remove directories read only for the user - find "${work_dir}" -type d -exec chmod u+w {} \; - rm -fR "${work_dir}" - fi - if [ "${glide_local_tmp_dir_created}" -eq "1" ]; then - find "${glide_local_tmp_dir}" -type d -exec chmod u+w {} \; - rm -fR "${glide_local_tmp_dir}" - fi - fi + slots_layout="partitionable" fi } -# use this for early failures, when we cannot assume we can write to disk at all -# too bad we end up with some repeated code, but difficult to do better -early_glidein_failure() { - error_msg="$1" - - warn "${error_msg}" - - sleep "${sleep_time}" - # wait a bit in case of error, to reduce lost glideins - - glidein_end_time="$(date +%s)" - result=" WN_RESOURCE - ERROR - - ${error_msg} - " - - final_result="$(construct_xml "${result}")" - final_result_simple="$(basexml2simplexml "${final_result}")" - # have no global section - final_result_long="$(simplexml2longxml "${final_result_simple}" "")" - - glidien_cleanup - - print_tail 1 "${final_result_simple}" "${final_result_long}" - - exit 1 +################################ +# Function used to generate the glidein UUID +generate_glidein_uuid(){ + if command -v uuidgen >/dev/null 2>&1; then + glidein_uuid="$(uuidgen)" + else + glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" + fi } +################################ +# Function used to spawn multiple glideins and wait, if needed +spawn_multiple_glideins(){ + if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then + # start multiple glideins + ON_DIE=0 + trap 'ignore_signal' SIGHUP + trap_with_arg 'on_die_multi' SIGTERM SIGINT SIGQUIT + do_start_all "${multi_glidein}" + # Wait for all glideins and exit 0 + # TODO: Summarize exit codes and status from all child glideins + echo "------ Multi-glidein parent waiting for child processes (${GWMS_MULTIGLIDEIN_CHILDS}) ----------" 1>&2 + wait + echo "------ Exiting multi-glidein parent ----------" 1>&2 + exit 0 + fi +} -# use this one once the most basic ops have been done -glidein_exit() { - # Removed lines about $lock_file (lock file for whole machine) not present elsewhere - - gwms_process_scripts "$GWMS_DIR" cleanup "${glidein_config}" - - global_result="" - if [ -f otr_outlist.list ]; then - global_result=$(cat otr_outlist.list) - chmod u+w otr_outlist.list - fi - - ge_last_script_name=$(extract_parent_fname "$1") - result=$(extract_parent_xml_detail "$1") - final_result=$(construct_xml "${result}") - - # augment with node info - final_result_simple=$(basexml2simplexml "${final_result}") - - # Create a richer version, too - final_result_long=$(simplexml2longxml "${final_result_simple}" "${global_result}") - - if [ "$1" -ne 0 ]; then - report_failed=$(grep -i "^GLIDEIN_Report_Failed " "${glidein_config}" | cut -d ' ' -f 2-) - - if [ -z "${report_failed}" ]; then - report_failed="NEVER" - fi - - factory_report_failed=$(grep -i "^GLIDEIN_Factory_Report_Failed " "${glidein_config}" | cut -d ' ' -f 2-) - - if [ -z "${factory_report_failed}" ]; then - factory_collector=$(grep -i "^GLIDEIN_Factory_Collector " "${glidein_config}" | cut -d ' ' -f 2-) - if [ -z "${factory_collector}" ]; then - # no point in enabling it if there are no collectors - factory_report_failed="NEVER" - else - factory_report_failed="ALIVEONLY" - fi - fi - - do_report=0 - if [ "${report_failed}" != "NEVER" ] || [ "${factory_report_failed}" != "NEVER" ]; then - do_report=1 - fi - - - # wait a bit in case of error, to reduce lost glideins - let "dl=$(date +%s) + ${sleep_time}" - dlf=$(date --date="@${dl}") - add_config_line "GLIDEIN_ADVERTISE_ONLY" "1" - add_config_line "GLIDEIN_Failed" "True" - add_config_line "GLIDEIN_EXIT_CODE" "$1" - add_config_line "GLIDEIN_ToDie" "${dl}" - add_config_line "GLIDEIN_Expire" "${dl}" - add_config_line "GLIDEIN_LAST_SCRIPT" "${ge_last_script_name}" - add_config_line "GLIDEIN_ADVERTISE_TYPE" "Retiring" - - add_config_line "GLIDEIN_FAILURE_REASON" "Glidein failed while running ${ge_last_script_name}. Keeping node busy until ${dl} (${dlf})." - - condor_vars_file="$(grep -i "^CONDOR_VARS_FILE " "${glidein_config}" | cut -d ' ' -f 2-)" - if [ -n "${condor_vars_file}" ]; then - # if we are to advertise, this should be available... else, it does not matter anyhow - add_condor_vars_line "GLIDEIN_ADVERTISE_ONLY" "C" "True" "+" "Y" "Y" "-" - add_condor_vars_line "GLIDEIN_Failed" "C" "True" "+" "Y" "Y" "-" - add_condor_vars_line "GLIDEIN_EXIT_CODE" "I" "-" "+" "Y" "Y" "-" - add_condor_vars_line "GLIDEIN_ToDie" "I" "-" "+" "Y" "Y" "-" - add_condor_vars_line "GLIDEIN_Expire" "I" "-" "+" "Y" "Y" "-" - add_condor_vars_line "GLIDEIN_LAST_SCRIPT" "S" "-" "+" "Y" "Y" "-" - add_condor_vars_line "GLIDEIN_FAILURE_REASON" "S" "-" "+" "Y" "Y" "-" - fi - main_work_dir="$(get_work_dir main)" - - for ((t=$(date +%s); t < dl; t=$(date +%s))) - do - if [ -e "${main_work_dir}/${last_script}" ] && [ "${do_report}" = "1" ] ; then - # if the file exists, we should be able to talk to the collectors - # notify that things went badly and we are waiting - if [ "${factory_report_failed}" != "NEVER" ]; then - add_config_line "GLIDEIN_ADVERTISE_DESTINATION" "Factory" - warn "Notifying Factory of error" - "${main_work_dir}/${last_script}" glidein_config - fi - if [ "${report_failed}" != "NEVER" ]; then - add_config_line "GLIDEIN_ADVERTISE_DESTINATION" "VO" - warn "Notifying VO of error" - "${main_work_dir}/${last_script}" glidein_config - fi - fi - - # sleep for about 5 mins... but randomize a bit - let "ds=250+${RANDOM}%100" - let "as=$(date +%s) + ${ds}" - if [ ${as} -gt ${dl} ]; then - # too long, shorten to the deadline - let "ds=${dl} - $(date +%s)" - fi - warn "Sleeping ${ds}" - sleep ${ds} - done - - if [ -e "${main_work_dir}/${last_script}" ] && [ "${do_report}" = "1" ]; then - # notify that things went badly and we are going away - if [ "${factory_report_failed}" != "NEVER" ]; then - add_config_line "GLIDEIN_ADVERTISE_DESTINATION" "Factory" - if [ "${factory_report_failed}" = "ALIVEONLY" ]; then - add_config_line "GLIDEIN_ADVERTISE_TYPE" "INVALIDATE" - else - add_config_line "GLIDEIN_ADVERTISE_TYPE" "Killing" - add_config_line "GLIDEIN_FAILURE_REASON" "Glidein failed while running ${ge_last_script_name}. Terminating now. (${dl}) (${dlf})" - fi - "${main_work_dir}/${last_script}" glidein_config - warn "Last notification sent to Factory" - fi - if [ "${report_failed}" != "NEVER" ]; then - add_config_line "GLIDEIN_ADVERTISE_DESTINATION" "VO" - if [ "${report_failed}" = "ALIVEONLY" ]; then - add_config_line "GLIDEIN_ADVERTISE_TYPE" "INVALIDATE" - else - add_config_line "GLIDEIN_ADVERTISE_TYPE" "Killing" - add_config_line "GLIDEIN_FAILURE_REASON" "Glidein failed while running ${ge_last_script_name}. Terminating now. (${dl}) (${dlf})" - fi - "${main_work_dir}/${last_script}" glidein_config - warn "Last notification sent to VO" - fi +######################################## +# Function used to setup OSG and/or Globus +setup_OSG_Globus(){ + if [ -r "${OSG_GRID}/setup.sh" ]; then + . "${OSG_GRID}/setup.sh" + else + if [ -r "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" ]; then + . "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" fi - fi - - log_write "glidein_startup.sh" "text" "glidein is about to exit with retcode $1" "info" - send_logs_to_remote - - glidien_cleanup - - print_tail "$1" "${final_result_simple}" "${final_result_long}" - - exit "$1" -} - -#################################################### -# automatically determine and setup work directories -automatic_work_dir() { - declare -a targets=("${_CONDOR_SCRATCH_DIR}" - "${OSG_WN_TMP}" - "${TG_NODE_SCRATCH}" - "${TG_CLUSTER_SCRATCH}" - "${SCRATCH}" - "${TMPDIR}" - "${TMP}" - "${PWD}" - ) - unset TMPDIR - - # 1 kB - disk_required=1000000 - - for d in "${targets[@]}"; do - - echo "Checking ${d} for potential use as work space... " 1>&2 - - # does the target exist? - if [ ! -e "${d}" ]; then - echo " Workdir: ${d} does not exist" 1>&2 - continue - fi - - # make sure there is enough available diskspace - free="$(df -kP "${d}" | awk '{if (NR==2) print $4}')" - if [ "x${free}" = "x" ] || [ "${free}" -lt ${disk_required} ]; then - echo " Workdir: not enough disk space available in ${d}" 1>&2 - continue - fi - - if touch "${d}/.dirtest.$$" >/dev/null 2>&1; then - echo " Workdir: ${d} selected" 1>&2 - rm -f "${d}/.dirtest.$$" >/dev/null 2>&1 - work_dir=${d} - return 0 - fi - echo " Workdir: not allowed to write to ${d}" 1>&2 - done - return 1 -} - -####################################### -# Parameters utility functions - -params_get_simple() { - # Retrieve a simple parameter (no special characters in its value) from the param list - # 1:param, 2:param_list (quoted string w/ spaces) - [[ ${2} = *\ ${1}\ * ]] || return - local retval="${2##*\ ${1}\ }" - echo "${retval%%\ *}" -} - -params_decode() { - echo "$1" | sed \ - -e 's/\.nbsp,/ /g' \ - -e 's/\.semicolon,/;/g' \ - -e 's/\.colon,/:/g' \ - -e 's/\.tilde,/~/g' \ - -e 's/\.not,/!/g' \ - -e 's/\.question,/?/g' \ - -e 's/\.star,/*/g' \ - -e 's/\.dollar,/$/g' \ - -e 's/\.comment,/#/g' \ - -e 's/\.sclose,/]/g' \ - -e 's/\.sopen,/[/g' \ - -e 's/\.gclose,/}/g' \ - -e 's/\.gopen,/{/g' \ - -e 's/\.close,/)/g' \ - -e 's/\.open,/(/g' \ - -e 's/\.gt,/>/g' \ - -e 's/\.lt,//g' \ - -e 's/\.lt,/ multi_glidein -tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") -[ -n "${tmp_par}" ] && multi_glidein=${tmp_par} - -case "${operation_mode}" in - nodebug) - sleep_time=1199 - set_debug=0;; - fast) - sleep_time=150 - set_debug=1;; - check) - sleep_time=150 - set -x - set_debug=2;; - *) - sleep_time=1199 - set_debug=1;; -esac - -if [ -z "${descript_file}" ]; then - warn "Missing descript fname." - usage -fi - -if [ -z "${descript_entry_file}" ]; then - warn "Missing descript fname for entry." - usage -fi - -if [ -z "${glidein_name}" ]; then - warn "Missing gliden name." - usage -fi - -if [ -z "${glidein_entry}" ]; then - warn "Missing glidein entry name." - usage -fi - - -if [ -z "${repository_url}" ]; then - warn "Missing Web URL." - usage -fi - -repository_entry_url="${repository_url}/entry_${glidein_entry}" - -if [ -z "${proxy_url}" ]; then - proxy_url="None" -fi - -if [ "${proxy_url}" = "OSG" ]; then - if [ -z "${OSG_SQUID_LOCATION}" ]; then - # if OSG does not define a Squid, then don't use any - proxy_url="None" - warn "OSG_SQUID_LOCATION undefined, not using any Squid URL" 1>&2 - else - proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" - fi -fi - -if [ -z "${sign_id}" ]; then - warn "Missing signature." - usage -fi - -if [ -z "${sign_entry_id}" ]; then - warn "Missing entry signature." - usage -fi - -if [ -z "${sign_type}" ]; then - sign_type="sha1" -fi - -if [ "${sign_type}" != "sha1" ]; then - warn "Unsupported signtype ${sign_type} found." - usage -fi - -if [ -n "${client_repository_url}" ]; then - # client data is optional, user url as a switch - if [ -z "${client_sign_type}" ]; then - client_sign_type="sha1" - fi - - if [ "${client_sign_type}" != "sha1" ]; then - warn "Unsupported clientsigntype ${client_sign_type} found." - usage - fi - - if [ -z "${client_descript_file}" ]; then - warn "Missing client descript fname." - usage - fi - - if [ -n "${client_repository_group_url}" ]; then - # client group data is optional, user url as a switch - if [ -z "${client_group}" ]; then - warn "Missing client group name." - usage fi - - if [ -z "${client_descript_group_file}" ]; then - warn "Missing client descript fname for group." - usage + + if [ -r "${GLOBUS_LOCATION}/etc/globus-user-env.sh" ]; then + . "${GLOBUS_LOCATION}/etc/globus-user-env.sh" + else + log_warn "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." + log_warn 'Continuing like nothing happened' fi - fi -fi - -md5wrapper() { - # $1 - file name - # $2 - option (quiet) - # Result returned on stdout - local ERROR_RESULT="???" - local ONLY_SUM - if [ "x$2" = "xquiet" ]; then - ONLY_SUM=yes - fi - local executable=md5sum - if which ${executable} 1>/dev/null 2>&1; then - [ -n "${ONLY_SUM}" ] && executable="md5sum \"$1\" | cut -d ' ' -f 1" || executable="md5sum \"$1\"" - else - executable=md5 - if ! which ${executable} 1>/dev/null 2>&1; then - echo "${ERROR_RESULT}" - warn "md5wrapper error: can't neither find md5sum nor md5" - return 1 - fi - [ -n "${ONLY_SUM}" ] && executable="md5 -q \"$1\"" || executable="md5 \"$1\"" - fi - local res - # Flagged by some checkers but OK - if ! res="$(eval "${executable}" 2>/dev/null)"; then - echo "${ERROR_RESULT}" - warn "md5wrapper error: can't calculate md5sum using ${executable}" - return 1 - fi - echo "${res}" -} - -# Generate directory ID -dir_id() { - # create an id to distinguish the directories when preserved - [[ ! ",${GLIDEIN_DEBUG_OPTIONS}," = *,nocleanup,* ]] && return - local dir_id="" - local tmp="${repository_url%%.*}" - tmp="${tmp#*//}" - dir_id="${tmp: -4}" - tmp="${client_repository_url%%.*}" - tmp="${tmp#*//}" - dir_id="${dir_id}${tmp: -4}" - [[ -z "${dir_id}" ]] && dir_id='debug' - echo "${dir_id}_" + fi } -# Generate glidein UUID -if command -v uuidgen >/dev/null 2>&1; then - glidein_uuid="$(uuidgen)" -else - glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" -fi - -startup_time="$(date +%s)" -echo "Starting glidein_startup.sh at $(date) (${startup_time})" - -echo "script_checksum = '$(md5wrapper "$0")'" -echo "debug_mode = '${operation_mode}'" -echo "condorg_cluster = '${condorg_cluster}'" -echo "condorg_subcluster= '${condorg_subcluster}'" -echo "condorg_schedd = '${condorg_schedd}'" -echo "glidein_uuid = '${glidein_uuid}'" -echo "glidein_credential_id = '${glidein_cred_id}'" -echo "glidein_factory = '${glidein_factory}'" -echo "glidein_name = '${glidein_name}'" -echo "glidein_entry = '${glidein_entry}'" -if [ -n "${client_name}" ]; then - # client name not required as it is not used for anything but debug info - echo "client_name = '${client_name}'" -fi -if [ -n "${client_group}" ]; then - echo "client_group = '${client_group}'" -fi -echo "multi_glidein/restart = '${multi_glidein}'/'${multi_glidein_restart}'" -echo "work_dir = '${work_dir}'" -echo "web_dir = '${repository_url}'" -echo "sign_type = '${sign_type}'" -echo "proxy_url = '${proxy_url}'" -echo "descript_fname = '${descript_file}'" -echo "descript_entry_fname = '${descript_entry_file}'" -echo "sign_id = '${sign_id}'" -echo "sign_entry_id = '${sign_entry_id}'" -if [ -n "${client_repository_url}" ]; then - echo "client_web_dir = '${client_repository_url}'" - echo "client_descript_fname = '${client_descript_file}'" - echo "client_sign_type = '${client_sign_type}'" - echo "client_sign_id = '${client_sign_id}'" - if [ -n "${client_repository_group_url}" ]; then - echo "client_web_group_dir = '${client_repository_group_url}'" - echo "client_descript_group_fname = '${client_descript_group_file}'" - echo "client_sign_group_id = '${client_sign_group_id}'" - fi -fi -echo -echo "Running on $(uname -n)" -echo "System: $(uname -a)" -if [ -e '/etc/redhat-release' ]; then - echo "Release: $(cat /etc/redhat-release 2>&1)" -fi -echo "As: $(id)" -echo "PID: $$" -echo - -if [ ${set_debug} -ne 0 ]; then - echo "------- Initial environment ---------------" 1>&2 - env 1>&2 - echo "------- =================== ---------------" 1>&2 -fi - -# Before anything else, spawn multiple glideins and wait, if asked to do so -if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then - # start multiple glideins - ON_DIE=0 - trap 'ignore_signal' SIGHUP - trap_with_arg 'on_die_multi' SIGTERM SIGINT SIGQUIT - do_start_all "${multi_glidein}" - # Wait for all glideins and exit 0 - # TODO: Summarize exit codes and status from all child glideins - echo "------ Multi-glidein parent waiting for child processes (${GWMS_MULTIGLIDEIN_CHILDS}) ----------" 1>&2 - wait - echo "------ Exiting multi-glidein parent ----------" 1>&2 - exit 0 -fi - ######################################## -# make sure nobody else can write my files -# In the Grid world I cannot trust anybody -if ! umask 0022; then - early_glidein_failure "Failed in umask 0022" -fi - -######################################## -# Setup OSG and/or Globus -if [ -r "${OSG_GRID}/setup.sh" ]; then - . "${OSG_GRID}/setup.sh" -else - if [ -r "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" ]; then - . "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" - fi -fi - -if [ -z "${GLOBUS_PATH}" ]; then - if [ -z "${GLOBUS_LOCATION}" ]; then - # if GLOBUS_LOCATION not defined, try to guess it - if [ -r "/opt/globus/etc/globus-user-env.sh" ]; then - GLOBUS_LOCATION=/opt/globus - elif [ -r "/osgroot/osgcore/globus/etc/globus-user-env.sh" ]; then - GLOBUS_LOCATION=/osgroot/osgcore/globus - else - warn "GLOBUS_LOCATION not defined and could not guess it." - warn "Looked in:" - warn ' /opt/globus/etc/globus-user-env.sh' - warn ' /osgroot/osgcore/globus/etc/globus-user-env.sh' - warn 'Continuing like nothing happened' - fi - fi - - if [ -r "${GLOBUS_LOCATION}/etc/globus-user-env.sh" ]; then - . "${GLOBUS_LOCATION}/etc/globus-user-env.sh" - else - warn "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." - warn 'Continuing like nothing happened' - fi -fi - -set_proxy_fullpath() { - # Set the X509_USER_PROXY path to full path to the file - if fullpath="$(readlink -f "${X509_USER_PROXY}")"; then - echo "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${fullpath}" 1>&2 - export X509_USER_PROXY="${fullpath}" - else - echo "Unable to get canonical path for X509_USER_PROXY, using ${X509_USER_PROXY}" 1>&2 - fi +# Function used to add $1 to GWMS_PATH and update PATH +add_to_path() { + logdebug "Adding to GWMS_PATH: $1" + local old_path=":${PATH%:}:" + old_path="${old_path//:$GWMS_PATH:/}" + local old_gwms_path=":${GWMS_PATH%:}:" + old_gwms_path="${old_gwms_path//:$1:/}" + old_gwms_path="${1%:}:${old_gwms_path#:}" + export GWMS_PATH="${old_gwms_path%:}" + old_path="${GWMS_PATH}:${old_path#:}" + export PATH="${old_path%:}" } - -[ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath - -num_gct=0 - -for tk in "$(pwd)/credential_"*".idtoken"; do - echo "Setting GLIDEIN_CONDOR_TOKEN to ${tk} " 1>&2 - num_gct=$(( num_gct + 1 )) - export GLIDEIN_CONDOR_TOKEN="${tk}" - fullpath="$(readlink -f "${tk}" )" - if [ $? -eq 0 ]; then - echo "Setting GLIDEIN_CONDOR_TOKEN ${tk} to canonical path ${fullpath}" 1>&2 - export GLIDEIN_CONDOR_TOKEN="${fullpath}" - else - echo "Unable to get canonical path for GLIDEIN_CONDOR_TOKEN ${tk}" 1>&2 - fi -done -if [ ! -f "${GLIDEIN_CONDOR_TOKEN}" ] ; then - token_err_msg="problem setting GLIDEIN_CONDOR_TOKEN" - token_err_msg="${token_err_msg} will attempt to recover, but condor IDTOKEN auth may fail" - echo "${token_err_msg}" - echo "${token_err_msg}" 1>&2 -fi -if [ ! "${num_gct}" -eq 1 ] ; then - token_err_msg="WARNING GLIDEIN_CONDOR_TOKEN set ${num_gct} times, should be 1 !" - token_err_msg="${token_err_msg} condor IDTOKEN auth may fail" - echo "${token_err_msg}" - echo "${token_err_msg}" 1>&2 -fi - -######################################## -# prepare and move to the work directory - -# Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . -# Empty $work_dir means PWD (same as ".") -# A custom path could be provided (no "*)" in case) -if [ -z "${work_dir}" ]; then - work_dir="$(pwd)" -else - case "${work_dir}" in - Condor|CONDOR) work_dir="${_CONDOR_SCRATCH_DIR}";; - OSG) work_dir="${OSG_WN_TMP}";; - TMPDIR) work_dir="${TMPDIR}";; - AUTO) automatic_work_dir;; - .) work_dir="$(pwd)";; - esac -fi - -if [ -z "${work_dir}" ]; then - early_glidein_failure "Unable to identify Startup dir for the glidein." -fi - -if [ ! -e "${work_dir}" ]; then - early_glidein_failure "Startup dir ${work_dir} does not exist." -fi - -start_dir="$(pwd)" -echo "Started in ${start_dir}" - -def_work_dir="${work_dir}/glide_$(dir_id)XXXXXX" -if ! work_dir="$(mktemp -d "${def_work_dir}")"; then - early_glidein_failure "Cannot create temp '${def_work_dir}'" -else - if ! cd "${work_dir}"; then - early_glidein_failure "Dir '${work_dir}' was created but cannot cd into it." - else - echo "Running in ${work_dir}" - fi -fi -work_dir_created=1 - -# GWMS_SUBDIR defined on top -GWMS_DIR="${work_dir}/$GWMS_SUBDIR" -if ! mkdir "$GWMS_DIR" ; then - early_glidein_failure "Cannot create '$GWMS_DIR'" -fi -gwms_lib_dir="${GWMS_DIR}/lib" -if ! mkdir -p "$gwms_lib_dir" ; then - early_glidein_failure "Cannot create '$gwms_lib_dir'" -fi -gwms_bin_dir="${GWMS_DIR}/bin" -if ! mkdir -p "$gwms_bin_dir" ; then - early_glidein_failure "Cannot create '$gwms_bin_dir'" -fi -gwms_exec_dir="${GWMS_DIR}/exec" -if ! mkdir -p "$gwms_exec_dir" ; then - early_glidein_failure "Cannot create '$gwms_exec_dir'" -else - for i in setup prejob postjob cleanup setup_singularity ; do - mkdir -p "$gwms_exec_dir"/$i - done -fi - -# mktemp makes it user readable by definition (ignores umask) -# TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed -if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then - if ! chmod a+rx "${work_dir}"; then - early_glidein_failure "Failed chmod '${work_dir}'" - fi -fi - -def_glide_local_tmp_dir="/tmp/glide_$(dir_id)$(id -u -n)_XXXXXX" -if ! glide_local_tmp_dir="$(mktemp -d "${def_glide_local_tmp_dir}")"; then - early_glidein_failure "Cannot create temp '${def_glide_local_tmp_dir}'" -fi -glide_local_tmp_dir_created=1 - -glide_tmp_dir="${work_dir}/tmp" -if ! mkdir "${glide_tmp_dir}"; then - early_glidein_failure "Cannot create '${glide_tmp_dir}'" -fi - -if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then - # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed - # the tmpdirs should be world writable - # This way it will work even if the user spawned by the glidein is different than the glidein user - # This happened in GlExec, outside user stays the same in Singularity - if ! chmod 1777 "${glide_local_tmp_dir}"; then - early_glidein_failure "Failed chmod '${glide_local_tmp_dir}'" - fi - - if ! chmod 1777 "${glide_tmp_dir}"; then - early_glidein_failure "Failed chmod '${glide_tmp_dir}'" - fi -fi - -short_main_dir=main -main_dir="${work_dir}/${short_main_dir}" -if ! mkdir "${main_dir}"; then - early_glidein_failure "Cannot create '${main_dir}'" -fi - -short_entry_dir=entry_${glidein_entry} -entry_dir="${work_dir}/${short_entry_dir}" -if ! mkdir "${entry_dir}"; then - early_glidein_failure "Cannot create '${entry_dir}'" -fi - -if [ -n "${client_repository_url}" ]; then - short_client_dir=client - client_dir="${work_dir}/${short_client_dir}" - if ! mkdir "$client_dir"; then - early_glidein_failure "Cannot create '${client_dir}'" - fi - - if [ -n "${client_repository_group_url}" ]; then - short_client_group_dir=client_group_${client_group} - client_group_dir="${work_dir}/${short_client_group_dir}" - if ! mkdir "${client_group_dir}"; then - early_glidein_failure "Cannot create '${client_group_dir}'" - fi - fi -fi - -# Move the token files from condor to glidein workspace -mv "${start_dir}/tokens.tgz" . -mv "${start_dir}/url_dirs.desc" . -for idtk in ${start_dir}/*.idtoken; do - if cp "${idtk}" . ; then - echo "copied idtoken ${idtk} to $(pwd)" - else - echo "failed to copy idtoken ${idtk} to $(pwd)" 1>&2 - fi -done -#if [ -e "${GLIDEIN_CONDOR_TOKEN}" ]; then -# mkdir -p ticket -# tname="$(basename ${GLIDEIN_CONDOR_TOKEN})" -# cp "${GLIDEIN_CONDOR_TOKEN}" "ticket/${tname}" -# export GLIDEIN_CONDOR_TOKEN="$(pwd)/ticket/${tname}" -#fi - -# Extract and source all the data contained at the end of this script as tarball -extract_all_data +#check return code +# add global, environment +# careful about subprocesses and enviornment parents +# careful to local as assignment +menu +set_slots_layout +parse_arguments +generate_glidein_uuid +print_header +spawn_multiple_glideins +check_umask +setup_OSG_Globus +set_proxy +prepare_workdir +extract_all_data # extract and source all the data contained at the end of this script as tarball + +# TODO(F): HERE wrapper_list="${PWD}/wrapper_list.lst" touch "${wrapper_list}" @@ -1274,585 +287,6 @@ log_init "${glidein_uuid}" "${work_dir}" rm -rf tokens.tgz url_dirs.desc tokens log_setup "${glidein_config}" -############################################ -# get the proper descript file based on id -# Arg: type (main/entry/client/client_group) -get_repository_url() { - case "$1" in - main) echo "${repository_url}";; - entry) echo "${repository_entry_url}";; - client) echo "${client_repository_url}";; - client_group) echo "${client_repository_group_url}";; - *) echo "[get_repository_url] Invalid id: $1" 1>&2 - return 1 - ;; - esac -} - -##################### -# Check signature -check_file_signature() { - cfs_id="$1" - cfs_fname="$2" - - cfs_work_dir="$(get_work_dir "${cfs_id}")" - - cfs_desc_fname="${cfs_work_dir}/${cfs_fname}" - cfs_signature="${cfs_work_dir}/signature.sha1" - - if [ "${check_signature}" -gt 0 ]; then # check_signature is global for simplicity - tmp_signname="${cfs_signature}_$$_$(date +%s)_${RANDOM}" - if ! grep " ${cfs_fname}$" "${cfs_signature}" > "${tmp_signname}"; then - rm -f "${tmp_signname}" - echo "No signature for ${cfs_desc_fname}." 1>&2 - else - (cd "${cfs_work_dir}" && sha1sum -c "${tmp_signname}") 1>&2 - cfs_rc=$? - if [ ${cfs_rc} -ne 0 ]; then - "${main_dir}"/error_augment.sh -init - "${main_dir}"/error_gen.sh -error "check_file_signature" "Corruption" "File $cfs_desc_fname is corrupted." "file" "${cfs_desc_fname}" "source_type" "${cfs_id}" - "${main_dir}"/error_augment.sh -process ${cfs_rc} "check_file_signature" "${PWD}" "sha1sum -c ${tmp_signname}" "$(date +%s)" "(date +%s)" - "${main_dir}"/error_augment.sh -concat - warn "File ${cfs_desc_fname} is corrupted." - rm -f "${tmp_signname}" - return 1 - fi - rm -f "${tmp_signname}" - echo "Signature OK for ${cfs_id}:${cfs_fname}." 1>&2 - fi - fi - return 0 -} - -##################### -# Untar support func - -get_untar_subdir() { - gus_id="$1" - gus_fname="$2" - - gus_prefix="$(get_prefix "${gus_id}")" - gus_config_cfg="${gus_prefix}UNTAR_CFG_FILE" - - gus_config_file="$(grep "^${gus_config_cfg} " glidein_config | cut -d ' ' -f 2-)" - if [ -z "${gus_config_file}" ]; then - warn "Error, cannot find '${gus_config_cfg}' in glidein_config." - glidein_exit 1 - fi - - gus_dir="$(grep -i "^${gus_fname} " "${gus_config_file}" | cut -s -f 2-)" - if [ -z "${gus_dir}" ]; then - warn "Error, untar dir for '${gus_fname}' cannot be empty." - glidein_exit 1 - fi - - echo "${gus_dir}" - return 0 -} - -##################### -# Periodic execution support function and global variable -add_startd_cron_counter=0 -add_periodic_script() { - # schedules a script for periodic execution using startd_cron - # parameters: wrapper full path, period, cwd, executable path (from cwd), - # config file path (from cwd), ID - # global variable: add_startd_cron_counter - #TODO: should it allow for variable number of parameters? - local include_fname=condor_config_startd_cron_include - local s_wrapper="$1" - local s_period_sec="${2}s" - local s_cwd="$3" - local s_fname="$4" - local s_config="$5" - local s_ffb_id="$6" - local s_cc_prefix="$7" - if [ ${add_startd_cron_counter} -eq 0 ]; then - # Make sure that no undesired file is there when called for first cron - rm -f ${include_fname} - fi - - let add_startd_cron_counter=add_startd_cron_counter+1 - local name_prefix=GLIDEIN_PS_ - local s_name="${name_prefix}${add_startd_cron_counter}" - - # Append the following to the startd configuration - # Instead of Periodic and Kill wait for completion: - # STARTD_CRON_DATE_MODE = WaitForExit - cat >> ${include_fname} << EOF -STARTD_CRON_JOBLIST = \$(STARTD_CRON_JOBLIST) ${s_name} -STARTD_CRON_${s_name}_MODE = Periodic -STARTD_CRON_${s_name}_KILL = True -STARTD_CRON_${s_name}_PERIOD = ${s_period_sec} -STARTD_CRON_${s_name}_EXECUTABLE = ${s_wrapper} -STARTD_CRON_${s_name}_ARGS = ${s_config} ${s_ffb_id} ${s_name} ${s_fname} ${s_cc_prefix} -STARTD_CRON_${s_name}_CWD = ${s_cwd} -STARTD_CRON_${s_name}_SLOTS = 1 -STARTD_CRON_${s_name}_JOB_LOAD = 0.01 -EOF - # NOPREFIX is a keyword for not setting the prefix for all condor attributes - [ "xNOPREFIX" != "x${s_cc_prefix}" ] && echo "STARTD_CRON_${s_name}_PREFIX = ${s_cc_prefix}" >> ${include_fname} - add_config_line "GLIDEIN_condor_config_startd_cron_include" "${include_fname}" - add_config_line "# --- Lines starting with ${s_cc_prefix} are from periodic scripts ---" -} - -##################### -# Fetch a single file -# -# Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) -fetch_file_regular() { - fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" -} - -fetch_file() { - # custom_scripts parameters format is set in the GWMS configuration (creation/lib) - # 1. ID - # 2. target fname - # 3. real fname - # 4. file type (regular, exec, exec:s, untar, nocache) - # 5. period (0 if not a periodic file) - # 6. periodic scripts prefix - # 7. config check TRUE,FALSE - # 8. config out TRUE,FALSE - # The above is the most recent list, below some adaptations for different versions - if [ $# -gt 8 ]; then - # For compatibility w/ future versions (add new parameters at the end) - echo "More then 8 arguments, considering the first 8 ($#/${ifs_str}): $*" 1>&2 - elif [ $# -ne 8 ]; then - if [ $# -eq 7 ]; then - #TODO: remove in version 3.3 - # For compatibility with past versions (old file list formats) - # 3.2.13 and older: prefix (par 6) added in #12705, 3.2.14? - # 3.2.10 and older: period (par 5) added: fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "GLIDEIN_PS_" "$6" "$7"; then - glidein_exit 1 - fi - return 0 - fi - if [ $# -eq 6 ]; then - # added to maintain compatibility with older (3.2.10) file list format - #TODO: remove in version 3.3 - if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6"; then - glidein_exit 1 - fi - return 0 - fi - local ifs_str - printf -v ifs_str '%q' "${IFS}" - warn "Not enough arguments in fetch_file, 8 expected ($#/${ifs_str}): $*" - glidein_exit 1 - fi - - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8"; then - glidein_exit 1 - fi - return 0 -} - -fetch_file_try() { - # Verifies if the file should be downloaded and acted upon (extracted, executed, ...) or not - # There are 2 mechanisms to control the download - # 1. tar files have the attribute "cond_attr" that is a name of a variable in glidein_config. - # if the named variable has value 1, then the file is downloaded. TRUE (default) means always download - # even if the mechanism is generic, there is no way to specify "cond_attr" for regular files in the configuration - # 2. if the file name starts with "gconditional_AAA_", the file is downloaded only if a variable GLIDEIN_USE_AAA - # exists in glidein_config and the value is not empty - # Both conditions are checked. If either one fails the file is not downloaded - fft_id="$1" - fft_target_fname="$2" - fft_real_fname="$3" - fft_file_type="$4" - fft_period="$5" - fft_cc_prefix="$6" - fft_config_check="$7" - fft_config_out="$8" - - if [[ "${fft_config_check}" != "TRUE" ]]; then - # TRUE is a special case, always downloaded and processed - local fft_get_ss - fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) - # Stop download and processing if the cond_attr variable is not defined or has a value different from 1 - [[ "${fft_get_ss}" != "1" ]] && return 0 - # TODO: what if fft_get_ss is not 1? nothing, still skip the file? - fi - - local fft_base_name fft_condition_attr fft_condition_attr_val - fft_base_name=$(basename "${fft_real_fname}") - if [[ "${fft_base_name}" = gconditional_* ]]; then - fft_condition_attr="${fft_base_name#gconditional_}" - fft_condition_attr="GLIDEIN_USE_${fft_condition_attr%%_*}" - fft_condition_attr_val=$(grep -i "^${fft_condition_attr} " glidein_config | cut -d ' ' -f 2-) - # if the variable fft_condition_attr is not defined or empty, do not download - [[ -z "${fft_condition_attr_val}" ]] && return 0 - fi - - fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" - # returning the exit code of fetch_file_base -} - -perform_wget() { - wget_args=("$@") - arg_len="${#wget_args[@]}" - ffb_url="${wget_args[0]}" - ffb_repository=$(dirname "${ffb_url}") - ffb_real_fname=$(basename "${ffb_url}") - proxy_url="None" - for ((i=0; i&1) - wget_retval=$? - else - wget_cmd=$(echo "wget" "${wget_args[@]}"| sed 's/"/\\\"/g') - wget_resp=$(wget "${wget_args[@]}" 2>&1) - wget_retval=$? - fi - - if [ ${wget_retval} -ne 0 ]; then - wget_version=$(wget --version 2>&1 | head -1) - warn "${wget_cmd} failed. version:${wget_version} exit code ${wget_retval} stderr: ${wget_resp}" - # cannot use error_*.sh helper functions - # may not have been loaded yet, and wget fails often - echo " - - ${PWD} - $(uname -a) - $(cat /etc/system-release) - ${wget_version} - - - ${wget_cmd} - $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - ERROR - Network - ${ffb_url} - ${proxy_url} - ${ffb_id} - - - Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp} - -" > otrb_output.xml - warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." - - if [ -f otr_outlist.list ]; then - chmod u+w otr_outlist.list - else - touch otr_outlist.list - fi - cat otrb_output.xml >> otr_outlist.list - echo "" > otrx_output.xml - cat otrb_output.xml >> otrx_output.xml - rm -f otrb_output.xml - chmod a-w otr_outlist.list - fi - return ${wget_retval} -} - -perform_curl() { - curl_args=("$@") - arg_len="${#curl_args[@]}" - ffb_url="${curl_args[0]}" - ffb_repository="$(dirname "${ffb_url}")" - ffb_real_fname="$(basename "${ffb_url}")" - for ((i=0; i&1)" - curl_retval=$? - if [ ${curl_retval} -eq 0 ] && [ ! -e "${ffb_tmp_outname}" ] ; then - touch "${ffb_tmp_outname}" - fi - - - if [ "${curl_retval}" -ne 0 ]; then - curl_version="$(curl --version 2>&1 | head -1)" - warn "${curl_cmd} failed. version:${curl_version} exit code ${curl_retval} stderr: ${curl_resp} " - # cannot use error_*.sh helper functions - # may not have been loaded yet, and wget fails often - echo " - - ${PWD} - $(uname -a) - $(cat /etc/system-release) - ${curl_version} - - - ${curl_cmd} - $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - ERROR - Network - ${ffb_url} - ${proxy_url} - ${ffb_id} - - - Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${curl_resp} - -" > otrb_output.xml - warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." - - if [ -f otr_outlist.list ]; then - chmod u+w otr_outlist.list - else - touch otr_outlist.list - fi - cat otrb_output.xml >> otr_outlist.list - echo "" > otrx_output.xml - cat otrb_output.xml >> otrx_output.xml - rm -f otrb_output.xml - chmod a-w otr_outlist.list - fi - return ${curl_retval} -} - -fetch_file_base() { - # Perform the file download and corresponding action (untar, execute, ...) - ffb_id="$1" - ffb_target_fname="$2" - ffb_real_fname="$3" - ffb_file_type="$4" - ffb_config_out="$5" - ffb_period=$6 - # condor cron prefix, used only for periodic executables - ffb_cc_prefix="$7" - - ffb_work_dir="$(get_work_dir "${ffb_id}")" - - ffb_repository="$(get_repository_url "${ffb_id}")" - - ffb_tmp_outname="${ffb_work_dir}/${ffb_real_fname}" - ffb_outname="${ffb_work_dir}/${ffb_target_fname}" - - # Create a dummy default in case something goes wrong - # cannot use error_*.sh helper functions - # may not have been loaded yet - have_dummy_otrx=1 - echo " - - - ${PWD} - - - Unknown - $(date +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - ERROR - Unknown - ${ffb_id} - - - An unknown error occurred. - -" > otrx_output.xml - user_agent="glidein/${glidein_entry}/${condorg_schedd}/${condorg_cluster}.${condorg_subcluster}/${client_name}" - ffb_url="${ffb_repository}/${ffb_real_fname}" - curl_version=$(curl --version | head -1 ) - wget_version=$(wget --version | head -1 ) - #old wget command: - #wget --user-agent="wget/glidein/$glidein_entry/$condorg_schedd/$condorg_cluster.$condorg_subcluster/$client_name" "$ffb_nocache_str" -q -O "$ffb_tmp_outname" "$ffb_repository/$ffb_real_fname" - #equivalent to: - #wget ${ffb_url} --user-agent=${user_agent} -q -O "${ffb_tmp_outname}" "${ffb_nocache_str}" - #with env http_proxy=$proxy_url set if proxy_url != "None" - # - #construct curl equivalent so we can try either - - wget_args=("${ffb_url}" "--user-agent" "wget/${user_agent}" "--quiet" "--output-document" "${ffb_tmp_outname}" ) - curl_args=("${ffb_url}" "--user-agent" "curl/${user_agent}" "--silent" "--show-error" "--output" "${ffb_tmp_outname}") - - if [ "${ffb_file_type}" = "nocache" ]; then - if [ "${curl_version}" != "" ]; then - curl_args+=("--header") - curl_args+=("'Cache-Control: no-cache'") - fi - if [ "${wget_version}" != "" ]; then - if wget --help | grep -q "\-\-no-cache "; then - wget_args+=("--no-cache") - elif wget --help |grep -q "\-\-cache="; then - wget_args+=("--cache=off") - else - warn "wget ${wget_version} cannot disable caching" - fi - fi - fi - - if [ "${proxy_url}" != "None" ];then - if [ "${curl_version}" != "" ]; then - curl_args+=("--proxy") - curl_args+=("${proxy_url}") - fi - if [ "${wget_version}" != "" ]; then - #these two arguments have to be last as coded, put any future - #wget args earlier in wget_args array - wget_args+=("--proxy") - wget_args+=("${proxy_url}") - fi - fi - - fetch_completed=1 - if [ ${fetch_completed} -ne 0 ] && [ "${wget_version}" != "" ]; then - perform_wget "${wget_args[@]}" - fetch_completed=$? - fi - if [ ${fetch_completed} -ne 0 ] && [ "${curl_version}" != "" ]; then - perform_curl "${curl_args[@]}" - fetch_completed=$? - fi - - if [ ${fetch_completed} -ne 0 ]; then - return ${fetch_completed} - fi - - # check signature - if ! check_file_signature "${ffb_id}" "${ffb_real_fname}"; then - # error already displayed inside the function - return 1 - fi - - # rename it to the correct final name, if needed - if [ "${ffb_tmp_outname}" != "${ffb_outname}" ]; then - if ! mv "${ffb_tmp_outname}" "${ffb_outname}"; then - warn "Failed to rename ${ffb_tmp_outname} into ${ffb_outname}" - return 1 - fi - fi - - # if executable, execute - if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then - if ! chmod u+x "${ffb_outname}"; then - warn "Error making '${ffb_outname}' executable" - return 1 - fi - if [ "${ffb_id}" = "main" ] && [ "${ffb_target_fname}" = "${last_script}" ]; then # last_script global for simplicity - echo "Skipping last script ${last_script}" 1>&2 - elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity - # TODO: temporary OR checking for cvmfs_umount.sh; to be removed after Bruno's ticket on cleanup [#25073] - echo "Skipping cleanup script ${ffb_outname} (${cleanup_script})" 1>&2 - cp "${ffb_outname}" "$gwms_exec_dir/cleanup/${ffb_target_fname}" - chmod a+x "${gwms_exec_dir}/cleanup/${ffb_target_fname}" - else - echo "Executing (flags:${ffb_file_type#exec}) ${ffb_outname}" - # have to do it here, as this will be run before any other script - chmod u+rx "${main_dir}"/error_augment.sh - - # the XML file will be overwritten now, and hopefully not an error situation - have_dummy_otrx=0 - "${main_dir}"/error_augment.sh -init - START=$(date +%s) - if [[ "${ffb_file_type}" = "exec:s" ]]; then - "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" - else - "${ffb_outname}" glidein_config "${ffb_id}" - fi - ret=$? - END=$(date +%s) - "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_outname} glidein_config" "${START}" "${END}" #generating test result document - "${main_dir}"/error_augment.sh -concat - if [ ${ret} -ne 0 ]; then - echo "=== Validation error in ${ffb_outname} ===" 1>&2 - warn "Error running '${ffb_outname}'" - < otrx_output.xml awk 'BEGIN{fr=0;}/<[/]detail>/{fr=0;}{if (fr==1) print $0}//{fr=1;}' 1>&2 - return 1 - else - # If ran successfully and periodic, schedule to execute with schedd_cron - echo "=== validation OK in ${ffb_outname} (${ffb_period}) ===" 1>&2 - if [ "${ffb_period}" -gt 0 ]; then - add_periodic_script "${main_dir}/script_wrapper.sh" "${ffb_period}" "${work_dir}" "${ffb_outname}" glidein_config "${ffb_id}" "${ffb_cc_prefix}" - fi - fi - fi - elif [ "${ffb_file_type}" = "wrapper" ]; then - echo "${ffb_outname}" >> "${wrapper_list}" - elif [ "${ffb_file_type}" = "untar" ]; then - ffb_short_untar_dir="$(get_untar_subdir "${ffb_id}" "${ffb_target_fname}")" - ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" - START=$(date +%s) - (mkdir "${ffb_untar_dir}" && cd "${ffb_untar_dir}" && tar -xmzf "${ffb_outname}") 1>&2 - ret=$? - if [ ${ret} -ne 0 ]; then - "${main_dir}"/error_augment.sh -init - "${main_dir}"/error_gen.sh -error "tar" "Corruption" "Error untarring '${ffb_outname}'" "file" "${ffb_outname}" "source_type" "${cfs_id}" - "${main_dir}"/error_augment.sh -process ${cfs_rc} "tar" "${PWD}" "mkdir ${ffb_untar_dir} && cd ${ffb_untar_dir} && tar -xmzf ${ffb_outname}" "${START}" "$(date +%s)" - "${main_dir}"/error_augment.sh -concat - warn "Error untarring '${ffb_outname}'" - return 1 - fi - fi - - if [ "${ffb_config_out}" != "FALSE" ]; then - ffb_prefix="$(get_prefix "${ffb_id}")" - if [ "${ffb_file_type}" = "untar" ]; then - # when untaring the original file is less interesting than the untar dir - if ! add_config_line "${ffb_prefix}${ffb_config_out}" "${ffb_untar_dir}"; then - glidein_exit 1 - fi - else - if ! add_config_line "${ffb_prefix}${ffb_config_out}" "${ffb_outname}"; then - glidein_exit 1 - fi - fi - fi - - if [ "${have_dummy_otrx}" -eq 1 ]; then - # no one should really look at this file, but just to avoid confusion - echo " - - - ${PWD} - - - Unknown - $(date +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - OK - -" > otrx_output.xml - fi - - return 0 -} - -# Adds $1 to GWMS_PATH and update PATH -add_to_path() { - logdebug "Adding to GWMS_PATH: $1" - local old_path=":${PATH%:}:" - old_path="${old_path//:$GWMS_PATH:/}" - local old_gwms_path=":${GWMS_PATH%:}:" - old_gwms_path="${old_gwms_path//:$1:/}" - old_gwms_path="${1%:}:${old_gwms_path#:}" - export GWMS_PATH="${old_gwms_path%:}" - old_path="${GWMS_PATH}:${old_path#:}" - export PATH="${old_path%:}" -} fixup_condor_dir() { # All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped @@ -1900,7 +334,7 @@ do gs_id_descript_file="$(get_descript_file ${gs_id})" fetch_file_regular "${gs_id}" "${gs_id_descript_file}" if ! signature_file_line="$(grep "^signature " "${gs_id_work_dir}/${gs_id_descript_file}")"; then - warn "No signature in description file ${gs_id_work_dir}/${gs_id_descript_file} (wc: $(wc < "${gs_id_work_dir}/${gs_id_descript_file}" 2>/dev/null))." + log_warn "No signature in description file ${gs_id_work_dir}/${gs_id_descript_file} (wc: $(wc < "${gs_id_work_dir}/${gs_id_descript_file}" 2>/dev/null))." glidein_exit 1 fi signature_file=$(echo "${signature_file_line}" | cut -s -f 2-) @@ -1910,7 +344,7 @@ do fetch_file_regular "${gs_id}" "${signature_file}" echo "${gs_id_signature} ${signature_file}" > "${gs_id_work_dir}/signature.sha1.test" if ! (cd "${gs_id_work_dir}" && sha1sum -c signature.sha1.test) 1>&2 ; then - warn "Corrupted signature file '${gs_id_work_dir}/${signature_file}'." + log_warn "Corrupted signature file '${gs_id_work_dir}/${signature_file}'." glidein_exit 1 fi # for simplicity use a fixed name for signature file @@ -1942,7 +376,7 @@ do gs_id_descript_file="$(get_descript_file ${gs_id})" if ! check_file_signature "${gs_id}" "${gs_id_descript_file}"; then gs_id_work_dir="$(get_work_dir ${gs_id})" - warn "Corrupted description file ${gs_id_work_dir}/${gs_id_descript_file}." + log_warn "Corrupted description file ${gs_id_work_dir}/${gs_id_descript_file}." glidein_exit 1 fi done @@ -1953,7 +387,7 @@ gs_id_work_dir="$(get_work_dir main)" gs_id_descript_file="$(get_descript_file main)" last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" if [ -z "${last_script}" ]; then - warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." + log_warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." glidein_exit 1 fi #cleanup_script="$(grep "^cleanup_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" @@ -1992,7 +426,7 @@ do continue fi fi - warn "No '${gs_file_list_id}' in description file ${gs_id_work_dir}/${gs_id_descript_file}." + log_warn "No '${gs_file_list_id}' in description file ${gs_id_work_dir}/${gs_id_descript_file}." glidein_exit 1 fi # space+tab separated file with multiple elements (was: awk '{print $2}', not safe for spaces in file name) @@ -2066,7 +500,7 @@ let last_script_time=${last_startup_end_time}-${last_startup_time} echo "=== Last script ended $(date) (${last_startup_end_time}) with code ${ret} after ${last_script_time} ===" echo if [ ${ret} -ne 0 ]; then - warn "Error running '${last_script}'" + log_warn "Error running '${last_script}'" fi #Things like periodic scripts might put messages here if they want them printed in the (stderr) logfile diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh new file mode 100644 index 0000000000..4eee0c768d --- /dev/null +++ b/creation/web_base/utils_crypto.sh @@ -0,0 +1,86 @@ +. utils_tarballs.sh +. utils_signals.sh +. utils_log.sh +. utils_xml.sh +. utils_filesystem.sh +. utils_params.sh +. utils_fetch.sh +. utils_http.sh +. glidein_cleanup.sh + +################################ +# Function used to calculate the md5 sum +# Arguments: +# 1: file name +# 2: option (quiet) +md5wrapper() { + local ERROR_RESULT="???" + local ONLY_SUM + if [ "x$2" = "xquiet" ]; then + ONLY_SUM=yes + fi + local executable=md5sum + if which ${executable} 1>/dev/null 2>&1; then + [ -n "${ONLY_SUM}" ] && executable="md5sum \"$1\" | cut -d ' ' -f 1" || executable="md5sum \"$1\"" + else + executable=md5 + if ! which ${executable} 1>/dev/null 2>&1; then + echo "${ERROR_RESULT}" + log_warn "md5wrapper error: can't neither find md5sum nor md5" + return 1 + fi + [ -n "${ONLY_SUM}" ] && executable="md5 -q \"$1\"" || executable="md5 \"$1\"" + fi + local res + # Flagged by some checkers but OK + if ! res="$(eval "${executable}" 2>/dev/null)"; then + echo "${ERROR_RESULT}" + log_warn "md5wrapper error: can't calculate md5sum using ${executable}" + return 1 + fi + echo "${res}" # result returned on stdout +} + +######################################## +# Function used to set the X509_USER_PROXY path to full path to the file +set_proxy_fullpath() { + if fullpath="$(readlink -f "${X509_USER_PROXY}")"; then + echo "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${fullpath}" 1>&2 + export X509_USER_PROXY="${fullpath}" + else + echo "Unable to get canonical path for X509_USER_PROXY, using ${X509_USER_PROXY}" 1>&2 + fi +} + +#TODO(F): check if all together is ok +######################################## +# Function used to set the tokens +set_proxy(){ + [ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath + num_gct=0 + + for tk in "$(pwd)/credential_"*".idtoken"; do + echo "Setting GLIDEIN_CONDOR_TOKEN to ${tk} " 1>&2 + num_gct=$(( num_gct + 1 )) + export GLIDEIN_CONDOR_TOKEN="${tk}" + fullpath="$(readlink -f "${tk}" )" + if [ $? -eq 0 ]; then + echo "Setting GLIDEIN_CONDOR_TOKEN ${tk} to canonical path ${fullpath}" 1>&2 + export GLIDEIN_CONDOR_TOKEN="${fullpath}" + else + echo "Unable to get canonical path for GLIDEIN_CONDOR_TOKEN ${tk}" 1>&2 + fi + done + if [ ! -f "${GLIDEIN_CONDOR_TOKEN}" ] ; then + token_err_msg="problem setting GLIDEIN_CONDOR_TOKEN" + token_err_msg="${token_err_msg} will attempt to recover, but condor IDTOKEN auth may fail" + echo "${token_err_msg}" + echo "${token_err_msg}" 1>&2 + fi + if [ ! "${num_gct}" -eq 1 ] ; then + token_err_msg="WARNING GLIDEIN_CONDOR_TOKEN set ${num_gct} times, should be 1 !" + token_err_msg="${token_err_msg} condor IDTOKEN auth may fail" + echo "${token_err_msg}" + echo "${token_err_msg}" 1>&2 + fi +} diff --git a/creation/web_base/utils_fetch.sh b/creation/web_base/utils_fetch.sh new file mode 100644 index 0000000000..13c3e5d746 --- /dev/null +++ b/creation/web_base/utils_fetch.sh @@ -0,0 +1,375 @@ +. utils_tarballs.sh +. utils_signals.sh +. utils_log.sh +. utils_xml.sh +. utils_filesystem.sh +. utils_params.sh +. utils_crypto.sh +. utils_http.sh +. glidein_cleanup.sh + +############################################ +# get the proper descript file based on id +# Arg: type (main/entry/client/client_group) +get_repository_url() { + case "$1" in + main) echo "${repository_url}";; + entry) echo "${repository_entry_url}";; + client) echo "${client_repository_url}";; + client_group) echo "${client_repository_group_url}";; + *) echo "[get_repository_url] Invalid id: $1" 1>&2 + return 1 + ;; + esac +} + +##################### +# Periodic execution support function and global variable +add_startd_cron_counter=0 +add_periodic_script() { + # schedules a script for periodic execution using startd_cron + # parameters: wrapper full path, period, cwd, executable path (from cwd), + # config file path (from cwd), ID + # global variable: add_startd_cron_counter + #TODO: should it allow for variable number of parameters? + local include_fname=condor_config_startd_cron_include + local s_wrapper="$1" + local s_period_sec="${2}s" + local s_cwd="$3" + local s_fname="$4" + local s_config="$5" + local s_ffb_id="$6" + local s_cc_prefix="$7" + if [ ${add_startd_cron_counter} -eq 0 ]; then + # Make sure that no undesired file is there when called for first cron + rm -f ${include_fname} + fi + + let add_startd_cron_counter=add_startd_cron_counter+1 + local name_prefix=GLIDEIN_PS_ + local s_name="${name_prefix}${add_startd_cron_counter}" + + # Append the following to the startd configuration + # Instead of Periodic and Kill wait for completion: + # STARTD_CRON_DATE_MODE = WaitForExit + cat >> ${include_fname} << EOF +STARTD_CRON_JOBLIST = \$(STARTD_CRON_JOBLIST) ${s_name} +STARTD_CRON_${s_name}_MODE = Periodic +STARTD_CRON_${s_name}_KILL = True +STARTD_CRON_${s_name}_PERIOD = ${s_period_sec} +STARTD_CRON_${s_name}_EXECUTABLE = ${s_wrapper} +STARTD_CRON_${s_name}_ARGS = ${s_config} ${s_ffb_id} ${s_name} ${s_fname} ${s_cc_prefix} +STARTD_CRON_${s_name}_CWD = ${s_cwd} +STARTD_CRON_${s_name}_SLOTS = 1 +STARTD_CRON_${s_name}_JOB_LOAD = 0.01 +EOF + # NOPREFIX is a keyword for not setting the prefix for all condor attributes + [ "xNOPREFIX" != "x${s_cc_prefix}" ] && echo "STARTD_CRON_${s_name}_PREFIX = ${s_cc_prefix}" >> ${include_fname} + add_config_line "GLIDEIN_condor_config_startd_cron_include" "${include_fname}" + add_config_line "# --- Lines starting with ${s_cc_prefix} are from periodic scripts ---" +} + +##################### +# Fetch a single file +# +# Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) +fetch_file_regular() { + fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" +} + +fetch_file() { + # custom_scripts parameters format is set in the GWMS configuration (creation/lib) + # 1. ID + # 2. target fname + # 3. real fname + # 4. file type (regular, exec, exec:s, untar, nocache) + # 5. period (0 if not a periodic file) + # 6. periodic scripts prefix + # 7. config check TRUE,FALSE + # 8. config out TRUE,FALSE + # The above is the most recent list, below some adaptations for different versions + if [ $# -gt 8 ]; then + # For compatibility w/ future versions (add new parameters at the end) + echo "More then 8 arguments, considering the first 8 ($#/${ifs_str}): $*" 1>&2 + elif [ $# -ne 8 ]; then + if [ $# -eq 7 ]; then + #TODO: remove in version 3.3 + # For compatibility with past versions (old file list formats) + # 3.2.13 and older: prefix (par 6) added in #12705, 3.2.14? + # 3.2.10 and older: period (par 5) added: fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" + if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "GLIDEIN_PS_" "$6" "$7"; then + glidein_exit 1 + fi + return 0 + fi + if [ $# -eq 6 ]; then + # added to maintain compatibility with older (3.2.10) file list format + #TODO: remove in version 3.3 + if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6"; then + glidein_exit 1 + fi + return 0 + fi + local ifs_str + printf -v ifs_str '%q' "${IFS}" + log_warn "Not enough arguments in fetch_file, 8 expected ($#/${ifs_str}): $*" + glidein_exit 1 + fi + + if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8"; then + glidein_exit 1 + fi + return 0 +} + +fetch_file_try() { + # Verifies if the file should be downloaded and acted upon (extracted, executed, ...) or not + # There are 2 mechanisms to control the download + # 1. tar files have the attribute "cond_attr" that is a name of a variable in glidein_config. + # if the named variable has value 1, then the file is downloaded. TRUE (default) means always download + # even if the mechanism is generic, there is no way to specify "cond_attr" for regular files in the configuration + # 2. if the file name starts with "gconditional_AAA_", the file is downloaded only if a variable GLIDEIN_USE_AAA + # exists in glidein_config and the value is not empty + # Both conditions are checked. If either one fails the file is not downloaded + fft_id="$1" + fft_target_fname="$2" + fft_real_fname="$3" + fft_file_type="$4" + fft_period="$5" + fft_cc_prefix="$6" + fft_config_check="$7" + fft_config_out="$8" + + if [[ "${fft_config_check}" != "TRUE" ]]; then + # TRUE is a special case, always downloaded and processed + local fft_get_ss + fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) + # Stop download and processing if the cond_attr variable is not defined or has a value different from 1 + [[ "${fft_get_ss}" != "1" ]] && return 0 + # TODO: what if fft_get_ss is not 1? nothing, still skip the file? + fi + + local fft_base_name fft_condition_attr fft_condition_attr_val + fft_base_name=$(basename "${fft_real_fname}") + if [[ "${fft_base_name}" = gconditional_* ]]; then + fft_condition_attr="${fft_base_name#gconditional_}" + fft_condition_attr="GLIDEIN_USE_${fft_condition_attr%%_*}" + fft_condition_attr_val=$(grep -i "^${fft_condition_attr} " glidein_config | cut -d ' ' -f 2-) + # if the variable fft_condition_attr is not defined or empty, do not download + [[ -z "${fft_condition_attr_val}" ]] && return 0 + fi + + fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" + # returning the exit code of fetch_file_base +} + + +fetch_file_base() { + # Perform the file download and corresponding action (untar, execute, ...) + ffb_id="$1" + ffb_target_fname="$2" + ffb_real_fname="$3" + ffb_file_type="$4" + ffb_config_out="$5" + ffb_period=$6 + # condor cron prefix, used only for periodic executables + ffb_cc_prefix="$7" + + ffb_work_dir="$(get_work_dir "${ffb_id}")" + + ffb_repository="$(get_repository_url "${ffb_id}")" + + ffb_tmp_outname="${ffb_work_dir}/${ffb_real_fname}" + ffb_outname="${ffb_work_dir}/${ffb_target_fname}" + + # Create a dummy default in case something goes wrong + # cannot use error_*.sh helper functions + # may not have been loaded yet + have_dummy_otrx=1 + echo " + + + ${PWD} + + + Unknown + $(date +%Y-%m-%dT%H:%M:%S%:z) + $(date +%Y-%m-%dT%H:%M:%S%:z) + + + ERROR + Unknown + ${ffb_id} + + + An unknown error occurred. + +" > otrx_output.xml + user_agent="glidein/${glidein_entry}/${condorg_schedd}/${condorg_cluster}.${condorg_subcluster}/${client_name}" + ffb_url="${ffb_repository}/${ffb_real_fname}" + curl_version=$(curl --version | head -1 ) + wget_version=$(wget --version | head -1 ) + #old wget command: + #wget --user-agent="wget/glidein/$glidein_entry/$condorg_schedd/$condorg_cluster.$condorg_subcluster/$client_name" "$ffb_nocache_str" -q -O "$ffb_tmp_outname" "$ffb_repository/$ffb_real_fname" + #equivalent to: + #wget ${ffb_url} --user-agent=${user_agent} -q -O "${ffb_tmp_outname}" "${ffb_nocache_str}" + #with env http_proxy=$proxy_url set if proxy_url != "None" + # + #construct curl equivalent so we can try either + + wget_args=("${ffb_url}" "--user-agent" "wget/${user_agent}" "--quiet" "--output-document" "${ffb_tmp_outname}" ) + curl_args=("${ffb_url}" "--user-agent" "curl/${user_agent}" "--silent" "--show-error" "--output" "${ffb_tmp_outname}") + + if [ "${ffb_file_type}" = "nocache" ]; then + if [ "${curl_version}" != "" ]; then + curl_args+=("--header") + curl_args+=("'Cache-Control: no-cache'") + fi + if [ "${wget_version}" != "" ]; then + if wget --help | grep -q "\-\-no-cache "; then + wget_args+=("--no-cache") + elif wget --help |grep -q "\-\-cache="; then + wget_args+=("--cache=off") + else + log_warn "wget ${wget_version} cannot disable caching" + fi + fi + fi + + if [ "${proxy_url}" != "None" ];then + if [ "${curl_version}" != "" ]; then + curl_args+=("--proxy") + curl_args+=("${proxy_url}") + fi + if [ "${wget_version}" != "" ]; then + #these two arguments have to be last as coded, put any future + #wget args earlier in wget_args array + wget_args+=("--proxy") + wget_args+=("${proxy_url}") + fi + fi + + fetch_completed=1 + if [ ${fetch_completed} -ne 0 ] && [ "${wget_version}" != "" ]; then + perform_wget "${wget_args[@]}" + fetch_completed=$? + fi + if [ ${fetch_completed} -ne 0 ] && [ "${curl_version}" != "" ]; then + perform_curl "${curl_args[@]}" + fetch_completed=$? + fi + + if [ ${fetch_completed} -ne 0 ]; then + return ${fetch_completed} + fi + + # check signature + if ! check_file_signature "${ffb_id}" "${ffb_real_fname}"; then + # error already displayed inside the function + return 1 + fi + + # rename it to the correct final name, if needed + if [ "${ffb_tmp_outname}" != "${ffb_outname}" ]; then + if ! mv "${ffb_tmp_outname}" "${ffb_outname}"; then + log_warn "Failed to rename ${ffb_tmp_outname} into ${ffb_outname}" + return 1 + fi + fi + + # if executable, execute + if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then + if ! chmod u+x "${ffb_outname}"; then + log_warn "Error making '${ffb_outname}' executable" + return 1 + fi + if [ "${ffb_id}" = "main" ] && [ "${ffb_target_fname}" = "${last_script}" ]; then # last_script global for simplicity + echo "Skipping last script ${last_script}" 1>&2 + elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity + # TODO: temporary OR checking for cvmfs_umount.sh; to be removed after Bruno's ticket on cleanup [#25073] + echo "Skipping cleanup script ${ffb_outname} (${cleanup_script})" 1>&2 + cp "${ffb_outname}" "$gwms_exec_dir/cleanup/${ffb_target_fname}" + chmod a+x "${gwms_exec_dir}/cleanup/${ffb_target_fname}" + else + echo "Executing (flags:${ffb_file_type#exec}) ${ffb_outname}" + # have to do it here, as this will be run before any other script + chmod u+rx "${main_dir}"/error_augment.sh + + # the XML file will be overwritten now, and hopefully not an error situation + have_dummy_otrx=0 + "${main_dir}"/error_augment.sh -init + START=$(date +%s) + if [[ "${ffb_file_type}" = "exec:s" ]]; then + "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" + else + "${ffb_outname}" glidein_config "${ffb_id}" + fi + ret=$? + END=$(date +%s) + "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_outname} glidein_config" "${START}" "${END}" #generating test result document + "${main_dir}"/error_augment.sh -concat + if [ ${ret} -ne 0 ]; then + echo "=== Validation error in ${ffb_outname} ===" 1>&2 + log_warn "Error running '${ffb_outname}'" + < otrx_output.xml awk 'BEGIN{fr=0;}/<[/]detail>/{fr=0;}{if (fr==1) print $0}//{fr=1;}' 1>&2 + return 1 + else + # If ran successfully and periodic, schedule to execute with schedd_cron + echo "=== validation OK in ${ffb_outname} (${ffb_period}) ===" 1>&2 + if [ "${ffb_period}" -gt 0 ]; then + add_periodic_script "${main_dir}/script_wrapper.sh" "${ffb_period}" "${work_dir}" "${ffb_outname}" glidein_config "${ffb_id}" "${ffb_cc_prefix}" + fi + fi + fi + elif [ "${ffb_file_type}" = "wrapper" ]; then + echo "${ffb_outname}" >> "${wrapper_list}" + elif [ "${ffb_file_type}" = "untar" ]; then + ffb_short_untar_dir="$(get_untar_subdir "${ffb_id}" "${ffb_target_fname}")" + ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" + START=$(date +%s) + (mkdir "${ffb_untar_dir}" && cd "${ffb_untar_dir}" && tar -xmzf "${ffb_outname}") 1>&2 + ret=$? + if [ ${ret} -ne 0 ]; then + "${main_dir}"/error_augment.sh -init + "${main_dir}"/error_gen.sh -error "tar" "Corruption" "Error untarring '${ffb_outname}'" "file" "${ffb_outname}" "source_type" "${cfs_id}" + "${main_dir}"/error_augment.sh -process ${cfs_rc} "tar" "${PWD}" "mkdir ${ffb_untar_dir} && cd ${ffb_untar_dir} && tar -xmzf ${ffb_outname}" "${START}" "$(date +%s)" + "${main_dir}"/error_augment.sh -concat + log_warn "Error untarring '${ffb_outname}'" + return 1 + fi + fi + + if [ "${ffb_config_out}" != "FALSE" ]; then + ffb_prefix="$(get_prefix "${ffb_id}")" + if [ "${ffb_file_type}" = "untar" ]; then + # when untaring the original file is less interesting than the untar dir + if ! add_config_line "${ffb_prefix}${ffb_config_out}" "${ffb_untar_dir}"; then + glidein_exit 1 + fi + else + if ! add_config_line "${ffb_prefix}${ffb_config_out}" "${ffb_outname}"; then + glidein_exit 1 + fi + fi + fi + + if [ "${have_dummy_otrx}" -eq 1 ]; then + # no one should really look at this file, but just to avoid confusion + echo " + + + ${PWD} + + + Unknown + $(date +%Y-%m-%dT%H:%M:%S%:z) + $(date +%Y-%m-%dT%H:%M:%S%:z) + + + OK + +" > otrx_output.xml + fi + + return 0 +} diff --git a/creation/web_base/utils_filesystem.sh b/creation/web_base/utils_filesystem.sh new file mode 100644 index 0000000000..69a32b1e3c --- /dev/null +++ b/creation/web_base/utils_filesystem.sh @@ -0,0 +1,253 @@ +. utils_tarballs.sh +. utils_signals.sh +. utils_log.sh +. utils_xml.sh +. utils_params.sh +. utils_crypto.sh +. utils_http.sh +. glidein_cleanup.sh +. utils_fetch.sh + +################################ +# Function used to automatically determine and setup work directories +automatic_work_dir() { + declare -a targets=("${_CONDOR_SCRATCH_DIR}" + "${OSG_WN_TMP}" + "${TG_NODE_SCRATCH}" + "${TG_CLUSTER_SCRATCH}" + "${SCRATCH}" + "${TMPDIR}" + "${TMP}" + "${PWD}" + ) + unset TMPDIR + + # 1 kB + disk_required=1000000 + + for d in "${targets[@]}"; do + + echo "Checking ${d} for potential use as work space... " 1>&2 + + # does the target exist? + if [ ! -e "${d}" ]; then + echo " Workdir: ${d} does not exist" 1>&2 + continue + fi + + # make sure there is enough available diskspace + free="$(df -kP "${d}" | awk '{if (NR==2) print $4}')" + if [ "x${free}" = "x" ] || [ "${free}" -lt ${disk_required} ]; then + echo " Workdir: not enough disk space available in ${d}" 1>&2 + continue + fi + + if touch "${d}/.dirtest.$$" >/dev/null 2>&1; then + echo " Workdir: ${d} selected" 1>&2 + rm -f "${d}/.dirtest.$$" >/dev/null 2>&1 + work_dir=${d} + return 0 + fi + echo " Workdir: not allowed to write to ${d}" 1>&2 + done + return 1 +} + +################################ +# Function used to generate the directory ID +# It creates an ID to distinguish the directories when preserved +dir_id() { + [[ ! ",${GLIDEIN_DEBUG_OPTIONS}," = *,nocleanup,* ]] && return + local dir_id="" + local tmp="${repository_url%%.*}" + tmp="${tmp#*//}" + dir_id="${tmp: -4}" + tmp="${client_repository_url%%.*}" + tmp="${tmp#*//}" + dir_id="${dir_id}${tmp: -4}" + [[ -z "${dir_id}" ]] && dir_id='debug' + echo "${dir_id}_" +} + +######################################## +# Function used to make sure nobody else can write my files +# In the Grid world I cannot trust anybody +check_umask(){ + if ! umask 0022; then + early_glidein_failure "Failed in umask 0022" + fi +} + +########################################### +# Function used to check the file signature +check_file_signature() { + cfs_id="$1" + cfs_fname="$2" + cfs_work_dir="$(get_work_dir "${cfs_id}")" + cfs_desc_fname="${cfs_work_dir}/${cfs_fname}" + cfs_signature="${cfs_work_dir}/signature.sha1" + if [ "${check_signature}" -gt 0 ]; then # check_signature is global for simplicity + tmp_signname="${cfs_signature}_$$_$(date +%s)_${RANDOM}" + if ! grep " ${cfs_fname}$" "${cfs_signature}" > "${tmp_signname}"; then + rm -f "${tmp_signname}" + echo "No signature for ${cfs_desc_fname}." 1>&2 + else + (cd "${cfs_work_dir}" && sha1sum -c "${tmp_signname}") 1>&2 + cfs_rc=$? + if [ ${cfs_rc} -ne 0 ]; then + "${main_dir}"/error_augment.sh -init + "${main_dir}"/error_gen.sh -error "check_file_signature" "Corruption" "File $cfs_desc_fname is corrupted." "file" "${cfs_desc_fname}" "source_type" "${cfs_id}" + "${main_dir}"/error_augment.sh -process ${cfs_rc} "check_file_signature" "${PWD}" "sha1sum -c ${tmp_signname}" "$(date +%s)" "(date +%s)" + "${main_dir}"/error_augment.sh -concat + log_warn "File ${cfs_desc_fname} is corrupted." + rm -f "${tmp_signname}" + return 1 + fi + rm -f "${tmp_signname}" + echo "Signature OK for ${cfs_id}:${cfs_fname}." 1>&2 + fi + fi + return 0 +} + +########################################### +# Function used to prepare and move to the work directory +# Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . +# Empty $work_dir means PWD (same as ".") +# A custom path could be provided (no "*)" in case) +prepare_workdir(){ + if [ -z "${work_dir}" ]; then + work_dir="$(pwd)" + else + case "${work_dir}" in + Condor|CONDOR) work_dir="${_CONDOR_SCRATCH_DIR}";; + OSG) work_dir="${OSG_WN_TMP}";; + TMPDIR) work_dir="${TMPDIR}";; + AUTO) automatic_work_dir;; + .) work_dir="$(pwd)";; + esac + fi + + if [ -z "${work_dir}" ]; then + early_glidein_failure "Unable to identify Startup dir for the glidein." + fi + + if [ ! -e "${work_dir}" ]; then + early_glidein_failure "Startup dir ${work_dir} does not exist." + fi + + start_dir="$(pwd)" + echo "Started in ${start_dir}" + + def_work_dir="${work_dir}/glide_$(dir_id)XXXXXX" + if ! work_dir="$(mktemp -d "${def_work_dir}")"; then + early_glidein_failure "Cannot create temp '${def_work_dir}'" + else + if ! cd "${work_dir}"; then + early_glidein_failure "Dir '${work_dir}' was created but cannot cd into it." + else + echo "Running in ${work_dir}" + fi + fi + work_dir_created=1 + + # GWMS_SUBDIR defined on top + GWMS_DIR="${work_dir}/$GWMS_SUBDIR" + if ! mkdir "$GWMS_DIR" ; then + early_glidein_failure "Cannot create '$GWMS_DIR'" + fi + gwms_lib_dir="${GWMS_DIR}/lib" + if ! mkdir -p "$gwms_lib_dir" ; then + early_glidein_failure "Cannot create '$gwms_lib_dir'" + fi + gwms_bin_dir="${GWMS_DIR}/bin" + if ! mkdir -p "$gwms_bin_dir" ; then + early_glidein_failure "Cannot create '$gwms_bin_dir'" + fi + gwms_exec_dir="${GWMS_DIR}/exec" + if ! mkdir -p "$gwms_exec_dir" ; then + early_glidein_failure "Cannot create '$gwms_exec_dir'" + else + for i in setup prejob postjob cleanup setup_singularity ; do + mkdir -p "$gwms_exec_dir"/$i + done + fi + + # mktemp makes it user readable by definition (ignores umask) + # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed + if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then + if ! chmod a+rx "${work_dir}"; then + early_glidein_failure "Failed chmod '${work_dir}'" + fi + fi + + def_glide_local_tmp_dir="/tmp/glide_$(dir_id)$(id -u -n)_XXXXXX" + if ! glide_local_tmp_dir="$(mktemp -d "${def_glide_local_tmp_dir}")"; then + early_glidein_failure "Cannot create temp '${def_glide_local_tmp_dir}'" + fi + glide_local_tmp_dir_created=1 + + glide_tmp_dir="${work_dir}/tmp" + if ! mkdir "${glide_tmp_dir}"; then + early_glidein_failure "Cannot create '${glide_tmp_dir}'" + fi + + if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then + # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed + # the tmpdirs should be world writable + # This way it will work even if the user spawned by the glidein is different than the glidein user + # This happened in GlExec, outside user stays the same in Singularity + if ! chmod 1777 "${glide_local_tmp_dir}"; then + early_glidein_failure "Failed chmod '${glide_local_tmp_dir}'" + fi + + if ! chmod 1777 "${glide_tmp_dir}"; then + early_glidein_failure "Failed chmod '${glide_tmp_dir}'" + fi + fi + + short_main_dir=main + main_dir="${work_dir}/${short_main_dir}" + if ! mkdir "${main_dir}"; then + early_glidein_failure "Cannot create '${main_dir}'" + fi + + short_entry_dir=entry_${glidein_entry} + entry_dir="${work_dir}/${short_entry_dir}" + if ! mkdir "${entry_dir}"; then + early_glidein_failure "Cannot create '${entry_dir}'" + fi + + if [ -n "${client_repository_url}" ]; then + short_client_dir=client + client_dir="${work_dir}/${short_client_dir}" + if ! mkdir "$client_dir"; then + early_glidein_failure "Cannot create '${client_dir}'" + fi + + if [ -n "${client_repository_group_url}" ]; then + short_client_group_dir=client_group_${client_group} + client_group_dir="${work_dir}/${short_client_group_dir}" + if ! mkdir "${client_group_dir}"; then + early_glidein_failure "Cannot create '${client_group_dir}'" + fi + fi + fi + + # Move the token files from condor to glidein workspace + mv "${start_dir}/tokens.tgz" . + mv "${start_dir}/url_dirs.desc" . + for idtk in ${start_dir}/*.idtoken; do + if cp "${idtk}" . ; then + echo "copied idtoken ${idtk} to $(pwd)" + else + echo "failed to copy idtoken ${idtk} to $(pwd)" 1>&2 + fi + done + #if [ -e "${GLIDEIN_CONDOR_TOKEN}" ]; then + # mkdir -p ticket + # tname="$(basename ${GLIDEIN_CONDOR_TOKEN})" + # cp "${GLIDEIN_CONDOR_TOKEN}" "ticket/${tname}" + # export GLIDEIN_CONDOR_TOKEN="$(pwd)/ticket/${tname}" + #fi +} diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh new file mode 100644 index 0000000000..e1bfb706c7 --- /dev/null +++ b/creation/web_base/utils_http.sh @@ -0,0 +1,158 @@ +. utils_tarballs.sh +. utils_signals.sh +. utils_log.sh +. utils_xml.sh +. utils_params.sh +. utils_crypto.sh +. glidein_cleanup.sh +. utils_fetch.sh + +################################ +# Function used to perform a wget request +# Arguments: +# @: wget args +perform_wget() { + wget_args=("$@") + arg_len="${#wget_args[@]}" + ffb_url="${wget_args[0]}" + ffb_repository=$(dirname "${ffb_url}") + ffb_real_fname=$(basename "${ffb_url}") + proxy_url="None" + for ((i=0; i&1) + wget_retval=$? + else + wget_cmd=$(echo "wget" "${wget_args[@]}"| sed 's/"/\\\"/g') + wget_resp=$(wget "${wget_args[@]}" 2>&1) + wget_retval=$? + fi + + if [ ${wget_retval} -ne 0 ]; then + wget_version=$(wget --version 2>&1 | head -1) + log_warn "${wget_cmd} failed. version:${wget_version} exit code ${wget_retval} stderr: ${wget_resp}" + # cannot use error_*.sh helper functions + # may not have been loaded yet, and wget fails often + # create_xml operatingenvironment { env cwd ${PWD} env uname $(uname -a) env release $(cat /etc/system-release) env wget_version ${wget_version} + echo " + + ${PWD} + $(uname -a) + $(cat /etc/system-release) + ${wget_version} + + + ${wget_cmd} + $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) + $(date +%Y-%m-%dT%H:%M:%S%:z) + + + ERROR + Network + ${ffb_url} + ${proxy_url} + ${ffb_id} + + + Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp} + +" > otrb_output.xml + log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." + + if [ -f otr_outlist.list ]; then + chmod u+w otr_outlist.list + else + touch otr_outlist.list + fi + cat otrb_output.xml >> otr_outlist.list + echo "" > otrx_output.xml + cat otrb_output.xml >> otrx_output.xml + rm -f otrb_output.xml + chmod a-w otr_outlist.list + fi + return ${wget_retval} +} + +################################ +# Function used to perform a curl request +# Arguments: +# @: curl args +perform_curl() { + curl_args=("$@") + arg_len="${#curl_args[@]}" + ffb_url="${curl_args[0]}" + ffb_repository="$(dirname "${ffb_url}")" + ffb_real_fname="$(basename "${ffb_url}")" + for ((i=0; i&1)" + curl_retval=$? + if [ ${curl_retval} -eq 0 ] && [ ! -e "${ffb_tmp_outname}" ] ; then + touch "${ffb_tmp_outname}" + fi + + + if [ "${curl_retval}" -ne 0 ]; then + curl_version="$(curl --version 2>&1 | head -1)" + log_warn "${curl_cmd} failed. version:${curl_version} exit code ${curl_retval} stderr: ${curl_resp} " + # cannot use error_*.sh helper functions + # may not have been loaded yet, and wget fails often + echo " + + ${PWD} + $(uname -a) + $(cat /etc/system-release) + ${curl_version} + + + ${curl_cmd} + $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) + $(date +%Y-%m-%dT%H:%M:%S%:z) + + + ERROR + Network + ${ffb_url} + ${proxy_url} + ${ffb_id} + + + Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${curl_resp} + +" > otrb_output.xml + log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." + + if [ -f otr_outlist.list ]; then + chmod u+w otr_outlist.list + else + touch otr_outlist.list + fi + cat otrb_output.xml >> otr_outlist.list + echo "" > otrx_output.xml + cat otrb_output.xml >> otrx_output.xml + rm -f otrb_output.xml + chmod a-w otr_outlist.list + fi + return ${curl_retval} +} diff --git a/creation/web_base/utils_log.sh b/creation/web_base/utils_log.sh new file mode 100644 index 0000000000..d4bec2e205 --- /dev/null +++ b/creation/web_base/utils_log.sh @@ -0,0 +1,156 @@ +. utils_tarballs.sh +. utils_signals.sh +. utils_xml.sh +. utils_params.sh +. utils_crypto.sh +. utils_http.sh +. glidein_cleanup.sh +. utils_fetch.sh + +################################ +# Function used to log warning statements +log_warn() { + echo "WARN $(date)" "$@" 1>&2 +} + +################################ +# Function used to log debug statements +log_debug() { + echo "DEBUG $(date)" "$@" 1>&2 +} + +print_header_line(){ + if [ $# -eq 1 ]; then + content=$1 + echo "=== ${content} ===" + elif [ $# -eq 2 && $2 -eq 2]; then + #TODO(F): Check + content=$1 + echo "=== ${content} ===" 1>&2 + fi +} + +################################ +# Function used to print the tail with the final results of the glideins +# Arguments: +# 1: exitcode +# 2: short version of the final results +# 3: long version of the final results +print_tail() { + exit_code=$1 + final_result_simple="$2" + final_result_long="$3" + glidein_end_time=$(date +%s) + let total_time=${glidein_end_time}-${startup_time} + print_header_line "Glidein ending $(date) (${glidein_end_time}) with code ${exit_code} after ${total_time}" + #echo "=== Glidein ending $(date) (${glidein_end_time}) with code ${exit_code} after ${total_time} ===" + echo "" + print_header_line "XML description of glidein activity" + #echo "=== XML description of glidein activity ===" + echo "${final_result_simple}" | grep -v "" + print_header_line "End XML description of glidein activity" + #echo "=== End XML description of glidein activity ===" + + echo "" 1>&2 + print_header_line "Encoded XML description of glidein activity" 2 + #echo "=== Encoded XML description of glidein activity ===" 1>&2 + echo "${final_result_long}" | gzip --stdout - | b64uuencode 1>&2 + print_header_line "End encoded XML description of glidein activity" 2 + #echo "=== End encoded XML description of glidein activity ===" 1>&2 +} + +################################ +# Function used to have information about the usage of the glidein_startup.sh script +usage() { + echo "Usage: glidein_startup.sh " + echo "where is:" + echo " -factory : name of this factory" + echo " -name : name of this glidein" + echo " -entry : name of this glidein entry" + echo " -clientname : name of the requesting client" + echo " -clientgroup : group name of the requesting client" + echo " -web : base URL from where to fetch" + echo " -proxy : URL of the local proxy" + echo " -dir : directory ID (supports ., Condor, CONDOR, OSG, TMPDIR, AUTO)" + echo " -sign : signature of the signature file" + echo " -signtype : type of signature (only sha1 supported for now)" + echo " -signentry : signature of the entry signature file" + echo " -cluster : condorG ClusterId" + echo " -subcluster : condorG ProcId" + echo " -submitcredid : Credential ID of this condorG job" + echo " -schedd : condorG Schedd Name" + echo " -descript : description file name" + echo " -descriptentry : description file name for entry" + echo " -clientweb : base URL from where to fetch client files" + echo " -clientwebgroup : base URL from where to fetch client group files" + echo " -clientsign : signature of the client signature file" + echo " -clientsigntype : type of client signature (only sha1 supported for now)" + echo " -clientsigngroup : signature of the client group signature file" + echo " -clientdescript : client description file name" + echo " -clientdescriptgroup : client description file name for group" + echo " -slotslayout : how Condor will set up slots (fixed, partitionable)" + echo " -v : operation mode (std, nodebug, fast, check supported)" + echo " -multiglidein : spawn multiple () glideins (unless also multirestart is set)" + echo " -multirestart : started as one of multiple glideins (glidein number )" + echo " -param_* : user specified parameters" + exit 1 #TODO(F): why? +} + +################################ +# Function used to print initial information header +print_header(){ + startup_time="$(date +%s)" + echo "Starting glidein_startup.sh at $(date) (${startup_time})" + echo "script_checksum = '$(md5wrapper "$0")'" + echo "debug_mode = '${operation_mode}'" + echo "condorg_cluster = '${condorg_cluster}'" + echo "condorg_subcluster= '${condorg_subcluster}'" + echo "condorg_schedd = '${condorg_schedd}'" + echo "glidein_uuid = '${glidein_uuid}'" + echo "glidein_credential_id = '${glidein_cred_id}'" + echo "glidein_factory = '${glidein_factory}'" + echo "glidein_name = '${glidein_name}'" + echo "glidein_entry = '${glidein_entry}'" + if [ -n "${client_name}" ]; then + # client name not required as it is not used for anything but debug info + echo "client_name = '${client_name}'" + fi + if [ -n "${client_group}" ]; then + echo "client_group = '${client_group}'" + fi + echo "multi_glidein/restart = '${multi_glidein}'/'${multi_glidein_restart}'" + echo "work_dir = '${work_dir}'" + echo "web_dir = '${repository_url}'" + echo "sign_type = '${sign_type}'" + echo "proxy_url = '${proxy_url}'" + echo "descript_fname = '${descript_file}'" + echo "descript_entry_fname = '${descript_entry_file}'" + echo "sign_id = '${sign_id}'" + echo "sign_entry_id = '${sign_entry_id}'" + if [ -n "${client_repository_url}" ]; then + echo "client_web_dir = '${client_repository_url}'" + echo "client_descript_fname = '${client_descript_file}'" + echo "client_sign_type = '${client_sign_type}'" + echo "client_sign_id = '${client_sign_id}'" + if [ -n "${client_repository_group_url}" ]; then + echo "client_web_group_dir = '${client_repository_group_url}'" + echo "client_descript_group_fname = '${client_descript_group_file}'" + echo "client_sign_group_id = '${client_sign_group_id}'" + fi + fi + echo + echo "Running on $(uname -n)" + echo "System: $(uname -a)" + if [ -e '/etc/redhat-release' ]; then + echo "Release: $(cat /etc/redhat-release 2>&1)" + fi + echo "As: $(id)" + echo "PID: $$" + echo + + if [ ${set_debug} -ne 0 ]; then + echo "------- Initial environment ---------------" 1>&2 + env 1>&2 + echo "------- =================== ---------------" 1>&2 + fi +} diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh new file mode 100644 index 0000000000..f62289cd62 --- /dev/null +++ b/creation/web_base/utils_params.sh @@ -0,0 +1,268 @@ +. utils_tarballs.sh +. utils_signals.sh +. utils_log.sh +. utils_xml.sh +. utils_crypto.sh +. utils_http.sh +. glidein_cleanup.sh +. utils_fetch.sh + +################################ +# Parameters utility functions + +################################ +# Function used to handle the list of parameters +# params will contain the full list of parameters +# -param_XXX YYY will become "XXX YYY" +menu() { + #TODO: can use an array instead? + params="" + while [ $# -gt 0 ] + do case "$1" in + -factory) glidein_factory="$2";; + -name) glidein_name="$2";; + -entry) glidein_entry="$2";; + -clientname) client_name="$2";; + -clientgroup) client_group="$2";; + -web) repository_url="$2";; + -proxy) proxy_url="$2";; + -dir) work_dir="$2";; + -sign) sign_id="$2";; + -signtype) sign_type="$2";; + -signentry) sign_entry_id="$2";; + -cluster) condorg_cluster="$2";; + -subcluster) condorg_subcluster="$2";; + -submitcredid) glidein_cred_id="$2";; + -schedd) condorg_schedd="$2";; + -descript) descript_file="$2";; + -descriptentry) descript_entry_file="$2";; + -clientweb) client_repository_url="$2";; + -clientwebgroup) client_repository_group_url="$2";; + -clientsign) client_sign_id="$2";; + -clientsigntype) client_sign_type="$2";; + -clientsigngroup) client_sign_group_id="$2";; + -clientdescript) client_descript_file="$2";; + -clientdescriptgroup) client_descript_group_file="$2";; + -slotslayout) slots_layout="$2";; + -v) operation_mode="$2";; + -multiglidein) multi_glidein="$2";; + -multirestart) multi_glidein_restart="$2";; + -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; + *) (log_warn "Unknown option $1"; usage) 1>&2; exit 1 + esac + shift 2 + done +} + +################################ +# Function used to retrieve a simple parameter (no special characters in its value) from the param list +# make sure to have a valid slots_layout +# Arguments: +# 1: param +# 2: param_list (quoted string w/ spaces) +params_get_simple() { + [[ ${2} = *\ ${1}\ * ]] || return + local retval="${2##*\ ${1}\ }" + echo "${retval%%\ *}" +} + +############################### +# Function used to decode the parameters +params_decode() { + echo "$1" | sed \ + -e 's/\.nbsp,/ /g' \ + -e 's/\.semicolon,/;/g' \ + -e 's/\.colon,/:/g' \ + -e 's/\.tilde,/~/g' \ + -e 's/\.not,/!/g' \ + -e 's/\.question,/?/g' \ + -e 's/\.star,/*/g' \ + -e 's/\.dollar,/$/g' \ + -e 's/\.comment,/#/g' \ + -e 's/\.sclose,/]/g' \ + -e 's/\.sopen,/[/g' \ + -e 's/\.gclose,/}/g' \ + -e 's/\.gopen,/{/g' \ + -e 's/\.close,/)/g' \ + -e 's/\.open,/(/g' \ + -e 's/\.gt,/>/g' \ + -e 's/\.lt,//g' \ + -e 's/\.lt,/ multi_glidein + tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") + [ -n "${tmp_par}" ] && multi_glidein=${tmp_par} + + case "${operation_mode}" in + nodebug) + sleep_time=1199 + set_debug=0;; + fast) + sleep_time=150 + set_debug=1;; + check) + sleep_time=150 + set -x + set_debug=2;; + *) + sleep_time=1199 + set_debug=1;; + esac + + if [ -z "${descript_file}" ]; then + log_warn "Missing descript fname." + usage + fi + + if [ -z "${descript_entry_file}" ]; then + log_warn "Missing descript fname for entry." + usage + fi + + if [ -z "${glidein_name}" ]; then + log_warn "Missing gliden name." + usage + fi + + if [ -z "${glidein_entry}" ]; then + log_warn "Missing glidein entry name." + usage + fi + + + if [ -z "${repository_url}" ]; then + log_warn "Missing Web URL." + usage + fi + + repository_entry_url="${repository_url}/entry_${glidein_entry}" + + if [ -z "${proxy_url}" ]; then + proxy_url="None" + fi + + if [ "${proxy_url}" = "OSG" ]; then + if [ -z "${OSG_SQUID_LOCATION}" ]; then + # if OSG does not define a Squid, then don't use any + proxy_url="None" + log_warn "OSG_SQUID_LOCATION undefined, not using any Squid URL" 1>&2 + else + proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" + fi + fi + + if [ -z "${sign_id}" ]; then + log_warn "Missing signature." + usage + fi + + if [ -z "${sign_entry_id}" ]; then + log_warn "Missing entry signature." + usage + fi + + if [ -z "${sign_type}" ]; then + sign_type="sha1" + fi + + if [ "${sign_type}" != "sha1" ]; then + log_warn "Unsupported signtype ${sign_type} found." + usage + fi + + if [ -n "${client_repository_url}" ]; then + # client data is optional, user url as a switch + if [ -z "${client_sign_type}" ]; then + client_sign_type="sha1" + fi + + if [ "${client_sign_type}" != "sha1" ]; then + log_warn "Unsupported clientsigntype ${client_sign_type} found." + usage + fi + + if [ -z "${client_descript_file}" ]; then + log_warn "Missing client descript fname." + usage + fi + + if [ -n "${client_repository_group_url}" ]; then + # client group data is optional, user url as a switch + if [ -z "${client_group}" ]; then + log_warn "Missing client group name." + usage + fi + + if [ -z "${client_descript_group_file}" ]; then + log_warn "Missing client descript fname for group." + usage + fi + fi + fi +} diff --git a/creation/web_base/utils_signals.sh b/creation/web_base/utils_signals.sh new file mode 100644 index 0000000000..abb7342e1f --- /dev/null +++ b/creation/web_base/utils_signals.sh @@ -0,0 +1,51 @@ +. utils_tarballs.sh +. utils_log.sh +. utils_xml.sh +. utils_params.sh +. utils_crypto.sh +. utils_http.sh +. glidein_cleanup.sh +. utils_fetch.sh + +################################ +# Extends 'trap' allowing to pass the signal name as argument to the handler +# Arguments: +# 1: handler +trap_with_arg() { + func="$1" ; shift + for sig ; do + # shellcheck disable=SC2064 + trap "${func} ${sig}" "${sig}" + done +} +# TODO: why do we need the shift? + +################################ +# Function that allows to pass signals to the children processes +# There is no need to re-raise sigint, caller does unconditional exit (https://www.cons.org/cracauer/sigint.html) +# Arguments: +# 1: signal +on_die() { + echo "Received kill signal... shutting down child processes (forwarding $1 signal)" 1>&2 + ON_DIE=1 + kill -s "$1" %1 +} + + +################################ +# Function that forwards signals to the children processes +# Arguments: +# 1: signal +on_die_multi() { + echo "Multi-Glidein received signal... shutting down child glideins (forwarding $1 signal to ${GWMS_MULTIGLIDEIN_CHILDS})" 1>&2 + ON_DIE=1 + for i in ${GWMS_MULTIGLIDEIN_CHILDS}; do + kill -s "$1" "${i}" + done +} + +################################ +# Function used to ignore signal SIGHUP +ignore_signal() { + echo "Ignoring SIGHUP signal... Use SIGTERM or SIGQUIT to kill processes" 1>&2 +} diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh new file mode 100644 index 0000000000..26287095d2 --- /dev/null +++ b/creation/web_base/utils_tarballs.sh @@ -0,0 +1,81 @@ +. utils_signals.sh +. utils_log.sh +. utils_xml.sh +. utils_params.sh +. utils_crypto.sh +. utils_http.sh +. glidein_cleanup.sh +. utils_fetch.sh + +############################## +# Utility functions to allow the script to source functions and retrieve data stored as tarball at the end of the script itself + +####################################### +# Retrieve the specified data, which is appended as tarball +# Arguments: +# 1: selected file +get_data() { + sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar xz -O "$1" +} + +####################################### +# Source the specified data, which is appended as tarball, without saving it +# Arguments: +# 1: selected file +source_data() { + local data + data=$(get_data "$1") + [[ -n "$data" ]] && eval "$data" +} + +####################################### +# Show a list of the payload tarballed files in this script +list_data() { + sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar tz +} + +####################################### +# Extract and source all the tarball files +extract_all_data() { + local -a files + # change separator to split the output file list from 'tar tz' command + local IFS_OLD="${IFS}" + IFS=$'\n' + files=($(list_data)) + for f in "${files[@]}"; do + echo "Extracting file ${f}" + get_data "${f}" > "${f}" + echo "Sourcing file ${f}" + # source_data "${f}" - can source the file saved instead of re-extracting it + . "${f}" + done + IFS="${IFS_OLD}" +} + +########################################### +# Untar support function +# Arguments: +# 1: id +# 2: filename +get_untar_subdir() { + gus_id="$1" + gus_fname="$2" + + gus_prefix="$(get_prefix "${gus_id}")" + gus_config_cfg="${gus_prefix}UNTAR_CFG_FILE" + + gus_config_file="$(grep "^${gus_config_cfg} " glidein_config | cut -d ' ' -f 2-)" + if [ -z "${gus_config_file}" ]; then + log_warn "Error, cannot find '${gus_config_cfg}' in glidein_config." + glidein_exit 1 + fi + + gus_dir="$(grep -i "^${gus_fname} " "${gus_config_file}" | cut -s -f 2-)" + if [ -z "${gus_dir}" ]; then + log_warn "Error, untar dir for '${gus_fname}' cannot be empty." + glidein_exit 1 + fi + + echo "${gus_dir}" + return 0 +} diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh new file mode 100644 index 0000000000..b6d38af5b1 --- /dev/null +++ b/creation/web_base/utils_xml.sh @@ -0,0 +1,265 @@ +. utils_tarballs.sh +. utils_signals.sh +. utils_log.sh +. utils_params.sh +. utils_crypto.sh +. utils_http.sh +. glidein_cleanup.sh +. utils_fetch.sh + +################################ +# Function used to create the xml content +# Arguments: +# 1: result +construct_xml() { + #TODO(F): Check + # OSG_test_result=" + # + # + # %s + # + # + # %s %s + # %s + # %s + # + # %s + # " + #printf "$OSG_test_result" "${start_dir}" "$0" "${global_args}" "$(date --date=@\"${startup_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "$result" + echo " + + + ${start_dir} + + + $0 ${global_args} + $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) + $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z) + + ${result} + " + result="$1" + glidein_end_time="$(date +%s)" +} + +################################ +# Function used to extract the parent xml fname +# Arguments: +# 1: exitcode +extract_parent_fname() { + exitcode=$1 + if [ -s otrx_output.xml ]; then # file exists and is not 0 size + last_result=$(cat otrx_output.xml) + if [ "${exitcode}" -eq 0 ]; then + echo "SUCCESS" + else + last_script_name=$(echo "${last_result}" |awk '/%s + # + # %s" + # correct_result_no_details=" + # %s + # " + # correct_result=" + # %s + # + # %s" + exitcode=$1 + glidein_end_time="$(date +%s)" + if [ -s otrx_output.xml ]; then + # file exists and is not 0 size + last_result="$(cat otrx_output.xml)" + if [ "${exitcode}" -eq 0 ]; then + #printf "$correct_result" "OK" + echo " " + echo " OK" + #propagate metrics as well + echo "${last_result}" | grep '" + else + last_script_name=$(echo "${last_result}" |awk '//{fr=0;}{if (fr==1) print $0}//{fr=1;}') + my_reason=" Validation failed in ${last_script_name}. ${last_script_reason}" "${last_script_name}" + #printf "$fail_result" "ERROR" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "${my_reason}" + echo " " + echo " ERROR + ${last_script_name}" + # propagate metrics as well (will include the failure metric) + echo "${last_result}" | grep '" + echo " ${my_reason}" + fi + else + # create a minimal XML file, else + echo " " + if [ "${exitcode}" -eq 0 ]; then + #printf "$correct_result" "OK" "No detail. Could not find source XML file." + echo " OK" + else + #printf "$fail_result" "ERROR" "No detail. Could not find source XML file." "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "Unknown" "No detail. Could not find source XML file." + echo " ERROR" + echo " Unknown" + fi + echo " + + No detail. Could not find source XML file. + " + fi +} + +################################ +# Function used to convert base xml to simple xml +# Arguments: +# 1: final result +basexml2simplexml() { + final_result="$1" + #env=" %s" + # augment with node info + echo "${final_result}" | awk 'BEGIN{fr=1;}{if (fr==1) print $0}//{fr=0;}' + #printf "$env" "client_name" "${client_name}" + #printf "$env" "client_group" "${client_group}" + #printf "$env" "user" "$(id -un)" + #printf "$env" "arch" "$(uname -m)" + echo " ${client_name}" + echo " ${client_group}" + echo " $(id -un)" + echo " $(uname -m)" + if [ -e '/etc/redhat-release' ]; then + #printf "$env" "os" "$(cat /etc/redhat-release)" + echo " $(cat /etc/redhat-release)" + fi + #printf "$env" "hostname" "$(uname -n)" + echo " $(uname -n)" + echo "${final_result}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=1;}' +} + +################################ +# Function used to convert simple xml to long xml +# Arguments: +# 1: simple final result +# 2: global result +simplexml2longxml() { + final_result_simple="$1" + global_result="$2" + #env=" %s" + #subtestlist = " + # + # %s + # ' + # " + echo "${final_result_simple}" | awk 'BEGIN{fr=1;}{if (fr==1) print $0}/' + echo ' ' + echo "${global_result}" | awk '{print " " $0}' + echo ' ' + echo ' ' + fi + echo "${final_result_simple}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=0;}' + #printf "$env" "glidein_factory" "${glidein_factory}" + #printf "$env" "glidein_name" "${glidein_name}" + #printf "$env" "glidein_entry" "${glidein_entry}" + #printf "$env" "condorg_cluster" "${condorg_cluster}" + #printf "$env" "condorg_subcluster" "${condorg_subcluster}" + #printf "$env" "glidein_credential_id" "${glidein_cred_id}" + #printf "$env" "condorg_schedd" "${condorg_schedd}" + echo " ${glidein_factory}" + echo " ${glidein_name}" + echo " ${glidein_entry}" + echo " ${condorg_cluster}" + echo " ${condorg_subcluster}" + echo " ${glidein_cred_id}" + echo " ${condorg_schedd}" + echo "${final_result_simple}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=1;}' +} + + +#look at err_gen.sh +# bigfiles.sh per costrutto finale source +create_xml(){ + xml="" + endxml="" + until [ $# -lt 1 ] + do + case "$1" in + OSG) xml+="" + if [ $3 == "{" ]; then + endxml=""+$endxml + shift 1 + else + xml+="" + fi + shift 2;; + OSGShort) xml+="" + if [ $2 == "{" ]; then + endxml=""+$endxml + shift 1 + else + xml+="" + fi + shift 1;; + operatingenvironment) xml += "" + if [ $2 == "{" ]; then + endxml=""+$endxml + shift 1 + else + xml+="" + fi + shift 1;; + env) xml+="$3" + shift 3;; + test) xml+="" + if [ $2 == "{" ]; then + endxml=""+$endxml + shift 1 + else + xml+="" + fi + shift 1;; + cmd) xml+="$2" + shift 2;; + tStart) xml+="$2" + shift 2;; + tEnd) xml+="$2" + shift 2;; + result) xml+="" + if [ $2 == "{" ]; then + endxml=""+$endxml + shift 1 + else + xml+="" + fi + shift 1;; + status) xml+="$2" + shift 2;; + metric) xml+="$4" + shift 4;; + detail) xml+="$2" + shift 2;; + "}") output=$(echo $endxml | cut -d'>' -f 1 | awk '{print $1">"}') + xml+=$output + endxml=$(echo $endxml | cut -d'<' -f 3 | awk '{print "<"$1}') + shift 1;; + *) echo "not available"; + shift 1;; + esac + done + echo "$xml" +} From 7b0f554992c07e5ed1e921d59ad1b80afc2651bd Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 9 Aug 2022 14:11:26 -0500 Subject: [PATCH 002/196] Added debug option --- creation/web_base/glidein_startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 4eefc786cf..cc964e0e86 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -xv #*******************************************************************# # glidein_startup.sh # # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # From 45692458bb66bb627c7b3cded8f25909086d7dc9 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 9 Aug 2022 14:22:12 -0500 Subject: [PATCH 003/196] Added manual installation script for testing --- creation/web_base/glidein_cleanup.sh | 10 ---------- creation/web_base/manual_installation_wget.sh | 16 ++++++++++++++++ creation/web_base/utils_crypto.sh | 10 ---------- creation/web_base/utils_fetch.sh | 10 ---------- creation/web_base/utils_filesystem.sh | 10 ---------- creation/web_base/utils_http.sh | 9 --------- creation/web_base/utils_log.sh | 9 --------- creation/web_base/utils_params.sh | 9 --------- creation/web_base/utils_signals.sh | 9 --------- creation/web_base/utils_tarballs.sh | 9 --------- creation/web_base/utils_xml.sh | 9 --------- 11 files changed, 16 insertions(+), 94 deletions(-) create mode 100644 creation/web_base/manual_installation_wget.sh diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index a267e00313..920437d00f 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -1,13 +1,3 @@ -. utils_tarballs.sh -. utils_signals.sh -. utils_log.sh -. utils_xml.sh -. utils_filesystem.sh -. utils_params.sh -. utils_crypto.sh -. utils_http.sh -. utils_fetch.sh - work_dir_created=0 glide_local_tmp_dir_created=0 diff --git a/creation/web_base/manual_installation_wget.sh b/creation/web_base/manual_installation_wget.sh new file mode 100644 index 0000000000..1d4c0955b8 --- /dev/null +++ b/creation/web_base/manual_installation_wget.sh @@ -0,0 +1,16 @@ +#!/bin/bash -xv + +wget -O glidein_startup_github.sh https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/glidein_startup.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/glidein_cleanup.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_crypto.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_fetch.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_log.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_http.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_filesystem.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_params.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_signals.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_tarballs.sh +wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_xml.sh +cat glidein_startup_github.sh term_file tar_utils.tar.gz > glidein_startup.sh +./manual_glidein_startup --wms-collector=fermicloud532.fnal.gov --client-name=fermicloud597-fnal-gov_OSG_gWMSFrontend.main --req-name=ITB_FC_CE2b@gfactory_instance@gfactory_service --cmd-out-file=glidein_startup_wrapper --glidein-startup=./glidein_startup.sh +./glidein_startup_wrapper diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index 4eee0c768d..7409b4101e 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -1,13 +1,3 @@ -. utils_tarballs.sh -. utils_signals.sh -. utils_log.sh -. utils_xml.sh -. utils_filesystem.sh -. utils_params.sh -. utils_fetch.sh -. utils_http.sh -. glidein_cleanup.sh - ################################ # Function used to calculate the md5 sum # Arguments: diff --git a/creation/web_base/utils_fetch.sh b/creation/web_base/utils_fetch.sh index 13c3e5d746..1004a19205 100644 --- a/creation/web_base/utils_fetch.sh +++ b/creation/web_base/utils_fetch.sh @@ -1,13 +1,3 @@ -. utils_tarballs.sh -. utils_signals.sh -. utils_log.sh -. utils_xml.sh -. utils_filesystem.sh -. utils_params.sh -. utils_crypto.sh -. utils_http.sh -. glidein_cleanup.sh - ############################################ # get the proper descript file based on id # Arg: type (main/entry/client/client_group) diff --git a/creation/web_base/utils_filesystem.sh b/creation/web_base/utils_filesystem.sh index 69a32b1e3c..8f4afa08b7 100644 --- a/creation/web_base/utils_filesystem.sh +++ b/creation/web_base/utils_filesystem.sh @@ -1,13 +1,3 @@ -. utils_tarballs.sh -. utils_signals.sh -. utils_log.sh -. utils_xml.sh -. utils_params.sh -. utils_crypto.sh -. utils_http.sh -. glidein_cleanup.sh -. utils_fetch.sh - ################################ # Function used to automatically determine and setup work directories automatic_work_dir() { diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index e1bfb706c7..3f9c930ff9 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -1,12 +1,3 @@ -. utils_tarballs.sh -. utils_signals.sh -. utils_log.sh -. utils_xml.sh -. utils_params.sh -. utils_crypto.sh -. glidein_cleanup.sh -. utils_fetch.sh - ################################ # Function used to perform a wget request # Arguments: diff --git a/creation/web_base/utils_log.sh b/creation/web_base/utils_log.sh index d4bec2e205..e0464e2e7d 100644 --- a/creation/web_base/utils_log.sh +++ b/creation/web_base/utils_log.sh @@ -1,12 +1,3 @@ -. utils_tarballs.sh -. utils_signals.sh -. utils_xml.sh -. utils_params.sh -. utils_crypto.sh -. utils_http.sh -. glidein_cleanup.sh -. utils_fetch.sh - ################################ # Function used to log warning statements log_warn() { diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index f62289cd62..c724f69bc6 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -1,12 +1,3 @@ -. utils_tarballs.sh -. utils_signals.sh -. utils_log.sh -. utils_xml.sh -. utils_crypto.sh -. utils_http.sh -. glidein_cleanup.sh -. utils_fetch.sh - ################################ # Parameters utility functions diff --git a/creation/web_base/utils_signals.sh b/creation/web_base/utils_signals.sh index abb7342e1f..1ca8cef651 100644 --- a/creation/web_base/utils_signals.sh +++ b/creation/web_base/utils_signals.sh @@ -1,12 +1,3 @@ -. utils_tarballs.sh -. utils_log.sh -. utils_xml.sh -. utils_params.sh -. utils_crypto.sh -. utils_http.sh -. glidein_cleanup.sh -. utils_fetch.sh - ################################ # Extends 'trap' allowing to pass the signal name as argument to the handler # Arguments: diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh index 26287095d2..8a2aa6f4cb 100644 --- a/creation/web_base/utils_tarballs.sh +++ b/creation/web_base/utils_tarballs.sh @@ -1,12 +1,3 @@ -. utils_signals.sh -. utils_log.sh -. utils_xml.sh -. utils_params.sh -. utils_crypto.sh -. utils_http.sh -. glidein_cleanup.sh -. utils_fetch.sh - ############################## # Utility functions to allow the script to source functions and retrieve data stored as tarball at the end of the script itself diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index b6d38af5b1..2a5c93d5f2 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -1,12 +1,3 @@ -. utils_tarballs.sh -. utils_signals.sh -. utils_log.sh -. utils_params.sh -. utils_crypto.sh -. utils_http.sh -. glidein_cleanup.sh -. utils_fetch.sh - ################################ # Function used to create the xml content # Arguments: From b9254e4e471f9097a8c9957b55708908a43cf3bc Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 9 Aug 2022 14:31:16 -0500 Subject: [PATCH 004/196] Modified menu --- creation/web_base/glidein_startup.sh | 59 +++++++++++++++---- creation/web_base/manual_installation_wget.sh | 4 ++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index cc964e0e86..e591fbbfd4 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -37,16 +37,16 @@ GWMS_MULTIUSER_GLIDEIN= GWMS_LOGSERVER_ADDRESS='https://fermicloud152.fnal.gov/log' GWMS_MULTIGLIDEIN_CHILDS= -. utils_tarballs.sh -. utils_signals.sh -. utils_log.sh -. utils_xml.sh -. utils_filesystem.sh -. utils_params.sh -. utils_fetch.sh -. utils_crypto.sh -. utils_http.sh -. glidein_cleanup.sh +source utils_tarballs.sh +source utils_signals.sh +source utils_log.sh +source utils_xml.sh +source utils_filesystem.sh +source utils_params.sh +source utils_fetch.sh +source utils_crypto.sh +source utils_http.sh +source glidein_cleanup.sh ################################ # Function used to start multiple glideins @@ -196,7 +196,44 @@ add_to_path() { # add global, environment # careful about subprocesses and enviornment parents # careful to local as assignment -menu +#TODO: can use an array instead? +params="" +while [ $# -gt 0 ] + do case "$1" in + -factory) glidein_factory="$2";; + -name) glidein_name="$2";; + -entry) glidein_entry="$2";; + -clientname) client_name="$2";; + -clientgroup) client_group="$2";; + -web) repository_url="$2";; + -proxy) proxy_url="$2";; + -dir) work_dir="$2";; + -sign) sign_id="$2";; + -signtype) sign_type="$2";; + -signentry) sign_entry_id="$2";; + -cluster) condorg_cluster="$2";; + -subcluster) condorg_subcluster="$2";; + -submitcredid) glidein_cred_id="$2";; + -schedd) condorg_schedd="$2";; + -descript) descript_file="$2";; + -descriptentry) descript_entry_file="$2";; + -clientweb) client_repository_url="$2";; + -clientwebgroup) client_repository_group_url="$2";; + -clientsign) client_sign_id="$2";; + -clientsigntype) client_sign_type="$2";; + -clientsigngroup) client_sign_group_id="$2";; + -clientdescript) client_descript_file="$2";; + -clientdescriptgroup) client_descript_group_file="$2";; + -slotslayout) slots_layout="$2";; + -v) operation_mode="$2";; + -multiglidein) multi_glidein="$2";; + -multirestart) multi_glidein_restart="$2";; + -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; + *) (log_warn "Unknown option $1"; usage) 1>&2; exit 1 + esac + shift 2 +done + set_slots_layout parse_arguments generate_glidein_uuid diff --git a/creation/web_base/manual_installation_wget.sh b/creation/web_base/manual_installation_wget.sh index 1d4c0955b8..feb9097be1 100644 --- a/creation/web_base/manual_installation_wget.sh +++ b/creation/web_base/manual_installation_wget.sh @@ -1,5 +1,9 @@ #!/bin/bash -xv +rm utils_* +rm glidein_startup_github.sh +rm glidein_cleanup.sh +rm glidein_startup.sh wget -O glidein_startup_github.sh https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/glidein_startup.sh wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/glidein_cleanup.sh wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_crypto.sh From 39265e53ae1f2ae244870810894b84d309e8195d Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 9 Aug 2022 14:35:29 -0500 Subject: [PATCH 005/196] Solved error printf --- creation/web_base/glidein_cleanup.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 920437d00f..0906d32cb5 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -34,20 +34,20 @@ glidien_cleanup() { # Arguments: # 1: error message early_glidein_failure() { - result = "%s - - %s" + #result = "%s + # + # %s" error_msg="$1" log_warn "${error_msg}" sleep "${sleep_time}" # wait a bit in case of error, to reduce lost glideins glidein_end_time="$(date +%s)" - printf "$result" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "WN_RESOURCE" "${error_msg}" - #result=" WN_RESOURCE - # ERROR - # - # ${error_msg} - # " + #printf "$result" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "WN_RESOURCE" "${error_msg}" + result=" WN_RESOURCE + ERROR + + ${error_msg} + " final_result="$(construct_xml "${result}")" final_result_simple="$(basexml2simplexml "${final_result}")" # have no global section From 31ce38caf45dc692762e6fc5d01ef7706bd570ba Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 9 Aug 2022 14:37:32 -0500 Subject: [PATCH 006/196] Modified manual installation --- creation/web_base/manual_installation_wget.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/creation/web_base/manual_installation_wget.sh b/creation/web_base/manual_installation_wget.sh index feb9097be1..7697bacb92 100644 --- a/creation/web_base/manual_installation_wget.sh +++ b/creation/web_base/manual_installation_wget.sh @@ -16,5 +16,6 @@ wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_tarballs.sh wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_xml.sh cat glidein_startup_github.sh term_file tar_utils.tar.gz > glidein_startup.sh +chmod +x glidein_startup.sh ./manual_glidein_startup --wms-collector=fermicloud532.fnal.gov --client-name=fermicloud597-fnal-gov_OSG_gWMSFrontend.main --req-name=ITB_FC_CE2b@gfactory_instance@gfactory_service --cmd-out-file=glidein_startup_wrapper --glidein-startup=./glidein_startup.sh ./glidein_startup_wrapper From e7d47d73fff2bb86bd9d9441a5f0c946b7d20a7c Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 10 Aug 2022 11:20:18 -0500 Subject: [PATCH 007/196] Updated functions' comments --- creation/web_base/glidein_cleanup.sh | 25 +++++ creation/web_base/glidein_startup.sh | 133 +++++++++++++++++--------- creation/web_base/utils_crypto.sh | 81 +++++++++------- creation/web_base/utils_fetch.sh | 130 +++++++++++++++++++------ creation/web_base/utils_filesystem.sh | 68 ++++++------- creation/web_base/utils_http.sh | 35 ++++++- creation/web_base/utils_log.sh | 21 +++- creation/web_base/utils_params.sh | 59 +++--------- creation/web_base/utils_tarballs.sh | 11 +++ creation/web_base/utils_xml.sh | 40 ++++++-- 10 files changed, 402 insertions(+), 201 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 0906d32cb5..c1b8a4ee82 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -1,3 +1,6 @@ +################################ +# Global variables of support to the glidein cleanup + work_dir_created=0 glide_local_tmp_dir_created=0 @@ -33,6 +36,13 @@ glidien_cleanup() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: error message +# Global: +# error_msg +# glidein_end_time +# result +# final_result +# final_result_simple +# final_result_long early_glidein_failure() { #result = "%s # @@ -62,6 +72,21 @@ early_glidein_failure() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: exit code +# Global: +# exit_code +# global_result +# ge_last_script_name +# result +# final_result +# final_result_simple +# final_result_long +# report_failed +# factory_report_failes +# factory_collector +# do_report +# dlf +# condor_vars_file +# main_work_dir glidein_exit() { exit_code=$1 # Removed lines about $lock_file (lock file for whole machine) not present elsewhere diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index e591fbbfd4..3147c27a52 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -69,6 +69,9 @@ copy_all() { # Function used to start all glideins # Arguments: # 1: number of glideins +# Global: +# GWMS_MULTIGLIDEIN_CHILDS +# g_dir # Important Variables: # GLIDEIN_MULTIGLIDEIN_LAUNCHALL - if set in attrs, command to start all Glideins at once (multirestart 0) # GLIDEIN_MULTIGLIDEIN_LAUNCHER - if set in attrs, command to start the individual Glideins @@ -103,29 +106,11 @@ do_start_all() { fi } -################################ -# Function used to set the slots_layout -# make sure to have a valid slots_layout -set_slots_layout(){ - if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then - slots_layout="fixed" - else - slots_layout="partitionable" - fi -} - -################################ -# Function used to generate the glidein UUID -generate_glidein_uuid(){ - if command -v uuidgen >/dev/null 2>&1; then - glidein_uuid="$(uuidgen)" - else - glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" - fi -} ################################ # Function used to spawn multiple glideins and wait, if needed +# Global: +# ON_DIE spawn_multiple_glideins(){ if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then # start multiple glideins @@ -144,6 +129,8 @@ spawn_multiple_glideins(){ ######################################## # Function used to setup OSG and/or Globus +# Global: +# GLOBUS_LOCATION setup_OSG_Globus(){ if [ -r "${OSG_GRID}/setup.sh" ]; then . "${OSG_GRID}/setup.sh" @@ -180,6 +167,9 @@ setup_OSG_Globus(){ ######################################## # Function used to add $1 to GWMS_PATH and update PATH +# Environment: +# GWMS_PATH +# PATH add_to_path() { logdebug "Adding to GWMS_PATH: $1" local old_path=":${PATH%:}:" @@ -192,10 +182,25 @@ add_to_path() { export PATH="${old_path%:}" } -#check return code -# add global, environment -# careful about subprocesses and enviornment parents -# careful to local as assignment +######################################## +# Function that removes the native condor tarballs directory to allow factory ops to use native condor tarballs +# All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped +# However the (not used anymore) gwms create_condor_tarball removes that dir +fixup_condor_dir() { + # Check if the condor dir has only one subdir, the one like "condor-9.0.11-1-x86_64_CentOS7-stripped" + # See https://stackoverflow.com/questions/32429333/how-to-test-if-a-linux-directory-contain-only-one-subdirectory-and-no-other-file + if [ $(find "${gs_id_work_dir}/condor" -maxdepth 1 -type d -printf 1 | wc -m) -eq 2 ]; then + echo "Fixing directory structure of condor tarball" + mv "${gs_id_work_dir}"/condor/condor*/* "${gs_id_work_dir}"/condor > /dev/null + else + echo "Condor tarball does not need to be fixed" + fi +} + +################################ +# Block of code used to handle the list of parameters +# params will contain the full list of parameters +# -param_XXX YYY will become "XXX YYY" #TODO: can use an array instead? params="" while [ $# -gt 0 ] @@ -234,18 +239,70 @@ while [ $# -gt 0 ] shift 2 done -set_slots_layout +################################ +# Code block used to set the slots_layout +# make sure to have a valid slots_layout +if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then + slots_layout="fixed" +else + slots_layout="partitionable" +fi + parse_arguments -generate_glidein_uuid + +################################ +# Code block used to generate the glidein UUID +if command -v uuidgen >/dev/null 2>&1; then + glidein_uuid="$(uuidgen)" +else + glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" +fi + print_header + spawn_multiple_glideins -check_umask + +######################################## +# Code block used to make sure nobody else can write my files +# In the Grid world I cannot trust anybody +if ! umask 0022; then + early_glidein_failure "Failed in umask 0022" +fi + setup_OSG_Globus -set_proxy + +######################################## +# Code block used to set the tokens +[ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath +num_gct=0 +for tk in "$(pwd)/credential_"*".idtoken"; do + echo "Setting GLIDEIN_CONDOR_TOKEN to ${tk} " 1>&2 + num_gct=$(( num_gct + 1 )) + export GLIDEIN_CONDOR_TOKEN="${tk}" + fullpath="$(readlink -f "${tk}" )" + if [ $? -eq 0 ]; then + echo "Setting GLIDEIN_CONDOR_TOKEN ${tk} to canonical path ${fullpath}" 1>&2 + export GLIDEIN_CONDOR_TOKEN="${fullpath}" + else + echo "Unable to get canonical path for GLIDEIN_CONDOR_TOKEN ${tk}" 1>&2 + fi +done +if [ ! -f "${GLIDEIN_CONDOR_TOKEN}" ] ; then + token_err_msg="problem setting GLIDEIN_CONDOR_TOKEN" + token_err_msg="${token_err_msg} will attempt to recover, but condor IDTOKEN auth may fail" + echo "${token_err_msg}" + echo "${token_err_msg}" 1>&2 +fi +if [ ! "${num_gct}" -eq 1 ] ; then + token_err_msg="WARNING GLIDEIN_CONDOR_TOKEN set ${num_gct} times, should be 1 !" + token_err_msg="${token_err_msg} condor IDTOKEN auth may fail" + echo "${token_err_msg}" + echo "${token_err_msg}" 1>&2 +fi + prepare_workdir -extract_all_data # extract and source all the data contained at the end of this script as tarball -# TODO(F): HERE +extract_all_data # extract and source all the data contained at the end of this script as tarball wrapper_list="${PWD}/wrapper_list.lst" touch "${wrapper_list}" @@ -324,22 +381,6 @@ log_init "${glidein_uuid}" "${work_dir}" rm -rf tokens.tgz url_dirs.desc tokens log_setup "${glidein_config}" - -fixup_condor_dir() { - # All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped - # However the (not used anymore) gwms create_condor_tarball removes that dir - # Here we remove that dir as well to allow factory ops to use native condor tarballs - - # Check if the condor dir has only one subdir, the one like "condor-9.0.11-1-x86_64_CentOS7-stripped" - # See https://stackoverflow.com/questions/32429333/how-to-test-if-a-linux-directory-contain-only-one-subdirectory-and-no-other-file - if [ $(find "${gs_id_work_dir}/condor" -maxdepth 1 -type d -printf 1 | wc -m) -eq 2 ]; then - echo "Fixing directory structure of condor tarball" - mv "${gs_id_work_dir}"/condor/condor*/* "${gs_id_work_dir}"/condor > /dev/null - else - echo "Condor tarball does not need to be fixed" - fi -} - echo "Downloading files from Factory and Frontend" log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index 7409b4101e..fb12d43527 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -3,6 +3,8 @@ # Arguments: # 1: file name # 2: option (quiet) +# Returns: +# 1 un case the md5sum cannot be calculated, or neither the md5sum nor the md5 can be found md5wrapper() { local ERROR_RESULT="???" local ONLY_SUM @@ -31,8 +33,55 @@ md5wrapper() { echo "${res}" # result returned on stdout } +########################################### +# Function used to check the file signature +# Arguments: +# 1: id +# 2: file name +# Globals: +# cfs_id +# cfs_fname +# cfs_work_dir +# cfs_desc_fname +# cfs_signature +# cfs_rc +# tmp_signname +# Returns: +# 1 in case of corrupted file +check_file_signature() { + cfs_id="$1" + cfs_fname="$2" + cfs_work_dir="$(get_work_dir "${cfs_id}")" + cfs_desc_fname="${cfs_work_dir}/${cfs_fname}" + cfs_signature="${cfs_work_dir}/signature.sha1" + if [ "${check_signature}" -gt 0 ]; then # check_signature is global for simplicity + tmp_signname="${cfs_signature}_$$_$(date +%s)_${RANDOM}" + if ! grep " ${cfs_fname}$" "${cfs_signature}" > "${tmp_signname}"; then + rm -f "${tmp_signname}" + echo "No signature for ${cfs_desc_fname}." 1>&2 + else + (cd "${cfs_work_dir}" && sha1sum -c "${tmp_signname}") 1>&2 + cfs_rc=$? + if [ ${cfs_rc} -ne 0 ]; then + "${main_dir}"/error_augment.sh -init + "${main_dir}"/error_gen.sh -error "check_file_signature" "Corruption" "File $cfs_desc_fname is corrupted." "file" "${cfs_desc_fname}" "source_type" "${cfs_id}" + "${main_dir}"/error_augment.sh -process ${cfs_rc} "check_file_signature" "${PWD}" "sha1sum -c ${tmp_signname}" "$(date +%s)" "(date +%s)" + "${main_dir}"/error_augment.sh -concat + log_warn "File ${cfs_desc_fname} is corrupted." + rm -f "${tmp_signname}" + return 1 + fi + rm -f "${tmp_signname}" + echo "Signature OK for ${cfs_id}:${cfs_fname}." 1>&2 + fi + fi + return 0 +} + ######################################## # Function used to set the X509_USER_PROXY path to full path to the file +# Environment variables exported: +# X509_USER_PROXY set_proxy_fullpath() { if fullpath="$(readlink -f "${X509_USER_PROXY}")"; then echo "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${fullpath}" 1>&2 @@ -42,35 +91,3 @@ set_proxy_fullpath() { fi } -#TODO(F): check if all together is ok -######################################## -# Function used to set the tokens -set_proxy(){ - [ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath - num_gct=0 - - for tk in "$(pwd)/credential_"*".idtoken"; do - echo "Setting GLIDEIN_CONDOR_TOKEN to ${tk} " 1>&2 - num_gct=$(( num_gct + 1 )) - export GLIDEIN_CONDOR_TOKEN="${tk}" - fullpath="$(readlink -f "${tk}" )" - if [ $? -eq 0 ]; then - echo "Setting GLIDEIN_CONDOR_TOKEN ${tk} to canonical path ${fullpath}" 1>&2 - export GLIDEIN_CONDOR_TOKEN="${fullpath}" - else - echo "Unable to get canonical path for GLIDEIN_CONDOR_TOKEN ${tk}" 1>&2 - fi - done - if [ ! -f "${GLIDEIN_CONDOR_TOKEN}" ] ; then - token_err_msg="problem setting GLIDEIN_CONDOR_TOKEN" - token_err_msg="${token_err_msg} will attempt to recover, but condor IDTOKEN auth may fail" - echo "${token_err_msg}" - echo "${token_err_msg}" 1>&2 - fi - if [ ! "${num_gct}" -eq 1 ] ; then - token_err_msg="WARNING GLIDEIN_CONDOR_TOKEN set ${num_gct} times, should be 1 !" - token_err_msg="${token_err_msg} condor IDTOKEN auth may fail" - echo "${token_err_msg}" - echo "${token_err_msg}" 1>&2 - fi -} diff --git a/creation/web_base/utils_fetch.sh b/creation/web_base/utils_fetch.sh index 1004a19205..4902cdf51d 100644 --- a/creation/web_base/utils_fetch.sh +++ b/creation/web_base/utils_fetch.sh @@ -1,6 +1,9 @@ ############################################ -# get the proper descript file based on id -# Arg: type (main/entry/client/client_group) +# Function that gets the proper descript file based on id +# Arguments: +# 1: type (main/entry/client/client_group) +# Returns: +# 1 in case of invalid ID get_repository_url() { case "$1" in main) echo "${repository_url}";; @@ -16,11 +19,19 @@ get_repository_url() { ##################### # Periodic execution support function and global variable add_startd_cron_counter=0 + +##################### +# Function that schedules a script for periodic execution using startd_cron +# Arguments: +# 1: wrapper full path +# 2: period +# 3: cwd +# 4: executable path (from cwd), +# 5: config file path (from cwd) +# 6: ID +# Globals: +# add_startd_cron_counter add_periodic_script() { - # schedules a script for periodic execution using startd_cron - # parameters: wrapper full path, period, cwd, executable path (from cwd), - # config file path (from cwd), ID - # global variable: add_startd_cron_counter #TODO: should it allow for variable number of parameters? local include_fname=condor_config_startd_cron_include local s_wrapper="$1" @@ -59,25 +70,31 @@ EOF add_config_line "# --- Lines starting with ${s_cc_prefix} are from periodic scripts ---" } -##################### -# Fetch a single file -# +############################################ +# Function that fetches a single regular file # Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) fetch_file_regular() { fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" } +############################################ +# Function that fetches a single file +# custom_scripts parameters format is set in the GWMS configuration (creation/lib) +# Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) +# Arguments: +# 1: ID +# 2: target fname +# 3: real fname +# 4: file type (regular, exec, exec:s, untar, nocache) +# 5: period (0 if not a periodic file) +# 6: periodic scripts prefix +# 7: config check TRUE,FALSE +# 8: config out TRUE,FALSE +# The above is the most recent list, below some adaptations for different versions +# Returns: +# 0 in case of success +# otherwise glidein_exit with 1 fetch_file() { - # custom_scripts parameters format is set in the GWMS configuration (creation/lib) - # 1. ID - # 2. target fname - # 3. real fname - # 4. file type (regular, exec, exec:s, untar, nocache) - # 5. period (0 if not a periodic file) - # 6. periodic scripts prefix - # 7. config check TRUE,FALSE - # 8. config out TRUE,FALSE - # The above is the most recent list, below some adaptations for different versions if [ $# -gt 8 ]; then # For compatibility w/ future versions (add new parameters at the end) echo "More then 8 arguments, considering the first 8 ($#/${ifs_str}): $*" 1>&2 @@ -112,15 +129,36 @@ fetch_file() { return 0 } +############################################ +# Function that verifies if the file should be downloaded and acted upon (extracted, executed, ...) or not +# There are 2 mechanisms to control the download +# 1. tar files have the attribute "cond_attr" that is a name of a variable in glidein_config. +# if the named variable has value 1, then the file is downloaded. TRUE (default) means always download +# even if the mechanism is generic, there is no way to specify "cond_attr" for regular files in the configuration +# 2. if the file name starts with "gconditional_AAA_", the file is downloaded only if a variable GLIDEIN_USE_AAA +# exists in glidein_config and the value is not empty +# Both conditions are checked. If either one fails the file is not downloaded +# Arguments: +# 1: ID +# 2: target fname +# 3: real fname +# 4: file type (regular, exec, exec:s, untar, nocache) +# 5: period (0 if not a periodic file) +# 6: periodic scripts prefix +# 7: config check TRUE,FALSE +# 8: config out TRUE,FALSE +# Globals: +# fft_id +# fft_target_fname +# fft_real_fname +# fft_file_type +# fft_period +# fft_cc_prefix +# fft_config_check +# fft_config_out +# Returns: +# 0 in case of success fetch_file_try() { - # Verifies if the file should be downloaded and acted upon (extracted, executed, ...) or not - # There are 2 mechanisms to control the download - # 1. tar files have the attribute "cond_attr" that is a name of a variable in glidein_config. - # if the named variable has value 1, then the file is downloaded. TRUE (default) means always download - # even if the mechanism is generic, there is no way to specify "cond_attr" for regular files in the configuration - # 2. if the file name starts with "gconditional_AAA_", the file is downloaded only if a variable GLIDEIN_USE_AAA - # exists in glidein_config and the value is not empty - # Both conditions are checked. If either one fails the file is not downloaded fft_id="$1" fft_target_fname="$2" fft_real_fname="$3" @@ -153,9 +191,43 @@ fetch_file_try() { # returning the exit code of fetch_file_base } - +############################################ +# Function that performs the file download and corresponding action (untar, execute, ...) +# Arguments: +# 1: ID +# 2: target fname +# 3: real fname +# 4: file type (regular, exec, exec:s, untar, nocache) +# 5: config out TRUE,FALSE +# 5: period (0 if not a periodic file) +# 6: periodic scripts prefix +# Globals: +# ffb_id +# ffb_target_fname +# ffb_real_fname +# ffb_file_type +# ffb_period +# ffb_cc_prefix +# ffb_config_out +# ffb_work_dir +# ffb_repository +# ffb_tmp_outname +# ffb_short_untar_dir +# ffb_untar_dir +# ffb_outname +# have_dummy_otrx +# user_agent +# ffb_url +# curl_version +# curl_args +# wget_version +# wget_args +# fetch_completed +# Returns: +# 1 in case the error is already displayed inside the function, +# in case of tarring, in case of failure in renaming it, +# of making it executable, or executing it fetch_file_base() { - # Perform the file download and corresponding action (untar, execute, ...) ffb_id="$1" ffb_target_fname="$2" ffb_real_fname="$3" diff --git a/creation/web_base/utils_filesystem.sh b/creation/web_base/utils_filesystem.sh index 8f4afa08b7..9446617d4f 100644 --- a/creation/web_base/utils_filesystem.sh +++ b/creation/web_base/utils_filesystem.sh @@ -1,5 +1,12 @@ ################################ # Function used to automatically determine and setup work directories +# Globals: +# targets +# disk_required +# free +# work_dir +# Returns: +# 1 in case you are not allowed to write automatic_work_dir() { declare -a targets=("${_CONDOR_SCRATCH_DIR}" "${OSG_WN_TMP}" @@ -59,52 +66,31 @@ dir_id() { echo "${dir_id}_" } -######################################## -# Function used to make sure nobody else can write my files -# In the Grid world I cannot trust anybody -check_umask(){ - if ! umask 0022; then - early_glidein_failure "Failed in umask 0022" - fi -} - -########################################### -# Function used to check the file signature -check_file_signature() { - cfs_id="$1" - cfs_fname="$2" - cfs_work_dir="$(get_work_dir "${cfs_id}")" - cfs_desc_fname="${cfs_work_dir}/${cfs_fname}" - cfs_signature="${cfs_work_dir}/signature.sha1" - if [ "${check_signature}" -gt 0 ]; then # check_signature is global for simplicity - tmp_signname="${cfs_signature}_$$_$(date +%s)_${RANDOM}" - if ! grep " ${cfs_fname}$" "${cfs_signature}" > "${tmp_signname}"; then - rm -f "${tmp_signname}" - echo "No signature for ${cfs_desc_fname}." 1>&2 - else - (cd "${cfs_work_dir}" && sha1sum -c "${tmp_signname}") 1>&2 - cfs_rc=$? - if [ ${cfs_rc} -ne 0 ]; then - "${main_dir}"/error_augment.sh -init - "${main_dir}"/error_gen.sh -error "check_file_signature" "Corruption" "File $cfs_desc_fname is corrupted." "file" "${cfs_desc_fname}" "source_type" "${cfs_id}" - "${main_dir}"/error_augment.sh -process ${cfs_rc} "check_file_signature" "${PWD}" "sha1sum -c ${tmp_signname}" "$(date +%s)" "(date +%s)" - "${main_dir}"/error_augment.sh -concat - log_warn "File ${cfs_desc_fname} is corrupted." - rm -f "${tmp_signname}" - return 1 - fi - rm -f "${tmp_signname}" - echo "Signature OK for ${cfs_id}:${cfs_fname}." 1>&2 - fi - fi - return 0 -} - ########################################### # Function used to prepare and move to the work directory # Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . # Empty $work_dir means PWD (same as ".") # A custom path could be provided (no "*)" in case) +# Globals: +# work_dir +# start_dir +# def_work_dir +# work_dir_created +# GWMS_DIR +# gwms_lib_dir +# gwms_bin_dir +# gwms_exec_dir +# def_glide_local_tmp_dir +# glide_local_tmp_dir_created +# glide_tmp_dir +# short_main_dir +# main_dir +# short_entry_dir +# entry_dir +# short_client_dir +# client_dir +# short_client_group_dir +# client_group_dir prepare_workdir(){ if [ -z "${work_dir}" ]; then work_dir="$(pwd)" diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 3f9c930ff9..dc1d011e83 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -2,6 +2,22 @@ # Function used to perform a wget request # Arguments: # @: wget args +# Globals: +# wget_args +# wget_cmd +# wget_resp +# wget_retval +# wget_version +# arg_len +# ffb_url +# ffb_repository +# ffb_real_fname +# ffb_tmp_outname +# proxy_url +# Returns: +# wget_retval +# Environment: +# http_proxy perform_wget() { wget_args=("$@") arg_len="${#wget_args[@]}" @@ -79,6 +95,22 @@ perform_wget() { # Function used to perform a curl request # Arguments: # @: curl args +# Globals: +# curl_args +# arg_len +# ffb_url +# ffb_repository +# ffb_real_fname +# ffb_tmp_outname +# proxy_url +# curl_cmd +# curl_resp +# curl_retval +# curl_version +# Returns: +# curl_retval +# Environment: +# http_proxy perform_curl() { curl_args=("$@") arg_len="${#curl_args[@]}" @@ -102,8 +134,7 @@ perform_curl() { if [ ${curl_retval} -eq 0 ] && [ ! -e "${ffb_tmp_outname}" ] ; then touch "${ffb_tmp_outname}" fi - - + if [ "${curl_retval}" -ne 0 ]; then curl_version="$(curl --version 2>&1 | head -1)" log_warn "${curl_cmd} failed. version:${curl_version} exit code ${curl_retval} stderr: ${curl_resp} " diff --git a/creation/web_base/utils_log.sh b/creation/web_base/utils_log.sh index e0464e2e7d..6f9b63ea64 100644 --- a/creation/web_base/utils_log.sh +++ b/creation/web_base/utils_log.sh @@ -1,15 +1,24 @@ ################################ # Function used to log warning statements +# Arguments: +# @: content to warn log_warn() { echo "WARN $(date)" "$@" 1>&2 } ################################ # Function used to log debug statements +# Arguments: +# @: content to debug log_debug() { echo "DEBUG $(date)" "$@" 1>&2 } +##################### +# Function used to prit a header line +# Arguments: +# 1: content of the header line +# 2 (optional): 2 if needs to write to stderr print_header_line(){ if [ $# -eq 1 ]; then content=$1 @@ -24,9 +33,15 @@ print_header_line(){ ################################ # Function used to print the tail with the final results of the glideins # Arguments: -# 1: exitcode +# 1: exit code # 2: short version of the final results # 3: long version of the final results +# Global: +# exit_code +# final_result_simple +# final_result_long +# glidein_end_time +# total_time print_tail() { exit_code=$1 final_result_simple="$2" @@ -52,6 +67,8 @@ print_tail() { ################################ # Function used to have information about the usage of the glidein_startup.sh script +# Returns: +# 1 in any case usage() { echo "Usage: glidein_startup.sh " echo "where is:" @@ -89,6 +106,8 @@ usage() { ################################ # Function used to print initial information header +# Global: +# startup_time print_header(){ startup_time="$(date +%s)" echo "Starting glidein_startup.sh at $(date) (${startup_time})" diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index c724f69bc6..4ebda12838 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -1,50 +1,6 @@ ################################ # Parameters utility functions -################################ -# Function used to handle the list of parameters -# params will contain the full list of parameters -# -param_XXX YYY will become "XXX YYY" -menu() { - #TODO: can use an array instead? - params="" - while [ $# -gt 0 ] - do case "$1" in - -factory) glidein_factory="$2";; - -name) glidein_name="$2";; - -entry) glidein_entry="$2";; - -clientname) client_name="$2";; - -clientgroup) client_group="$2";; - -web) repository_url="$2";; - -proxy) proxy_url="$2";; - -dir) work_dir="$2";; - -sign) sign_id="$2";; - -signtype) sign_type="$2";; - -signentry) sign_entry_id="$2";; - -cluster) condorg_cluster="$2";; - -subcluster) condorg_subcluster="$2";; - -submitcredid) glidein_cred_id="$2";; - -schedd) condorg_schedd="$2";; - -descript) descript_file="$2";; - -descriptentry) descript_entry_file="$2";; - -clientweb) client_repository_url="$2";; - -clientwebgroup) client_repository_group_url="$2";; - -clientsign) client_sign_id="$2";; - -clientsigntype) client_sign_type="$2";; - -clientsigngroup) client_sign_group_id="$2";; - -clientdescript) client_descript_file="$2";; - -clientdescriptgroup) client_descript_group_file="$2";; - -slotslayout) slots_layout="$2";; - -v) operation_mode="$2";; - -multiglidein) multi_glidein="$2";; - -multirestart) multi_glidein_restart="$2";; - -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; - *) (log_warn "Unknown option $1"; usage) 1>&2; exit 1 - esac - shift 2 - done -} - ################################ # Function used to retrieve a simple parameter (no special characters in its value) from the param list # make sure to have a valid slots_layout @@ -59,6 +15,8 @@ params_get_simple() { ############################### # Function used to decode the parameters +# Arguments: +# 1: param params_decode() { echo "$1" | sed \ -e 's/\.nbsp,/ /g' \ @@ -93,6 +51,14 @@ params_decode() { ############################### # Function used to put the parameters into the config file +# Arguments: +# @: parameters +# Global: +# param_list +# pfval +# Returns: +# 0 in case of success, +# otherwise glidein_exit with 1 params2file() { param_list="" while [ $# -gt 0 ] @@ -145,6 +111,11 @@ params2file() { ################################ # Function used to parse and verify arguments # It allows some parameters to change arguments +# Global: +# tmp_par +# repository_entry_url +# proxy_url +# client_sign_type parse_arguments(){ # multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh index 8a2aa6f4cb..8517aaccc5 100644 --- a/creation/web_base/utils_tarballs.sh +++ b/creation/web_base/utils_tarballs.sh @@ -27,6 +27,8 @@ list_data() { ####################################### # Extract and source all the tarball files +# Global: +# IFS extract_all_data() { local -a files # change separator to split the output file list from 'tar tz' command @@ -48,6 +50,15 @@ extract_all_data() { # Arguments: # 1: id # 2: filename +# Global: +# gus_id +# gus_fname +# gus_prefix +# gus_config_cfg +# gus_config_file +# gus_dir +# Returns: +# 0 in case of success, otherwise glidein_exit with 1 get_untar_subdir() { gus_id="$1" gus_fname="$2" diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 2a5c93d5f2..612c17b38e 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -2,6 +2,9 @@ # Function used to create the xml content # Arguments: # 1: result +# Global: +# result +# glidein_end_time construct_xml() { #TODO(F): Check # OSG_test_result=" @@ -17,6 +20,8 @@ construct_xml() { # %s # " #printf "$OSG_test_result" "${start_dir}" "$0" "${global_args}" "$(date --date=@\"${startup_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "$result" + #create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd ${start_dir} } test { cmd $0 ${global_args} tStart $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) tEnd $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)} } + create_xml OSG --id 0 { operatingenvironment { env --name cwd 0 } test { cmd 0 0 tStart 0 tEnd 0 } 0 } echo " @@ -36,8 +41,12 @@ construct_xml() { ################################ # Function used to extract the parent xml fname # Arguments: -# 1: exitcode -extract_parent_fname() { +# 1: exit code +# Global: +# last_result +# last_script_name +# exitcode +extract_parent_fname(){ exitcode=$1 if [ -s otrx_output.xml ]; then # file exists and is not 0 size last_result=$(cat otrx_output.xml) @@ -55,7 +64,14 @@ extract_parent_fname() { ################################ # Function used to extract the parent xml details # Arguments: -# 1: exitcode +# 1: exit code +# Global: +# exitcode +# glidein_end_time +# last_result +# last_script_name +# last_script_reason +# my_reason extract_parent_xml_detail() { # fail_result=" # %s @@ -116,6 +132,8 @@ extract_parent_xml_detail() { # Function used to convert base xml to simple xml # Arguments: # 1: final result +# Global: +# final_result basexml2simplexml() { final_result="$1" #env=" %s" @@ -143,6 +161,10 @@ basexml2simplexml() { # Arguments: # 1: simple final result # 2: global result +# Global: +# final_result_simple +# global_result +# content simplexml2longxml() { final_result_simple="$1" global_result="$2" @@ -181,9 +203,15 @@ simplexml2longxml() { echo "${final_result_simple}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=1;}' } - -#look at err_gen.sh -# bigfiles.sh per costrutto finale source +################################ +# Function used to create an xml file structure +# Arguments: +# @: tags, options, values +# Global: +# xml +# end_xml +# Returns: +# xml create_xml(){ xml="" endxml="" From bc9a8ee97476b9c5fc9a394040b90ce5417df4ef Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 10 Aug 2022 11:51:11 -0500 Subject: [PATCH 008/196] Fixed some comments --- creation/web_base/glidein_cleanup.sh | 7 +- creation/web_base/glidein_startup.sh | 165 ++++--- creation/web_base/utils_crypto.sh | 2 +- creation/web_base/utils_fetch.sh | 437 ------------------ creation/web_base/utils_http.sh | 435 +++++++++++++++++ .../web_base/{utils_log.sh => utils_io.sh} | 2 + creation/web_base/utils_signals.sh | 1 - creation/web_base/utils_xml.sh | 2 +- 8 files changed, 530 insertions(+), 521 deletions(-) delete mode 100644 creation/web_base/utils_fetch.sh rename creation/web_base/{utils_log.sh => utils_io.sh} (99%) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index c1b8a4ee82..4d7e734a78 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -1,6 +1,5 @@ ################################ -# Global variables of support to the glidein cleanup - +# Global variables of support to the glidein cleanup operations work_dir_created=0 glide_local_tmp_dir_created=0 @@ -44,15 +43,11 @@ glidien_cleanup() { # final_result_simple # final_result_long early_glidein_failure() { - #result = "%s - # - # %s" error_msg="$1" log_warn "${error_msg}" sleep "${sleep_time}" # wait a bit in case of error, to reduce lost glideins glidein_end_time="$(date +%s)" - #printf "$result" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "WN_RESOURCE" "${error_msg}" result=" WN_RESOURCE ERROR diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 3147c27a52..3755bbbb2f 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -39,11 +39,10 @@ GWMS_MULTIGLIDEIN_CHILDS= source utils_tarballs.sh source utils_signals.sh -source utils_log.sh +source utils_io.sh source utils_xml.sh source utils_filesystem.sh source utils_params.sh -source utils_fetch.sh source utils_crypto.sh source utils_http.sh source glidein_cleanup.sh @@ -51,7 +50,7 @@ source glidein_cleanup.sh ################################ # Function used to start multiple glideins # Arguments: -# 1: prefix (of the files to skip) +# 1: prefix of the files to skip # 2: directory copy_all() { mkdir -p "$2" @@ -197,6 +196,76 @@ fixup_condor_dir() { fi } +######################################## +# Function that creates the glidein configuration +# Global: +# glidein_config +create_glidein_config(){ + glidein_config="${PWD}/glidein_config" + if ! echo > "${glidein_config}"; then + early_glidein_failure "Could not create '${glidein_config}'" + fi + if ! { + echo "# --- glidein_startup vals ---" + echo "GLIDEIN_UUID ${glidein_uuid}" + echo "GLIDEIN_Factory ${glidein_factory}" + echo "GLIDEIN_Name ${glidein_name}" + echo "GLIDEIN_Entry_Name ${glidein_entry}" + + if [ -n "${client_name}" ]; then + # client name not required as it is not used for anything but debug info + echo "GLIDECLIENT_Name ${client_name}" + fi + if [ -n "${client_group}" ]; then + # client group not required as it is not used for anything but debug info + echo "GLIDECLIENT_Group ${client_group}" + fi + echo "GLIDEIN_CredentialIdentifier ${glidein_cred_id}" + echo "CONDORG_CLUSTER ${condorg_cluster}" + echo "CONDORG_SUBCLUSTER ${condorg_subcluster}" + echo "CONDORG_SCHEDD ${condorg_schedd}" + echo "DEBUG_MODE ${set_debug}" + echo "GLIDEIN_STARTUP_PID $$" + echo "GLIDEIN_START_DIR_ORIG ${start_dir}" + echo "GLIDEIN_WORKSPACE_ORIG $(pwd)" + echo "GLIDEIN_WORK_DIR ${main_dir}" + echo "GLIDEIN_ENTRY_WORK_DIR ${entry_dir}" + echo "TMP_DIR ${glide_tmp_dir}" + echo "GLIDEIN_LOCAL_TMP_DIR ${glide_local_tmp_dir}" + echo "PROXY_URL ${proxy_url}" + echo "DESCRIPTION_FILE ${descript_file}" + echo "DESCRIPTION_ENTRY_FILE ${descript_entry_file}" + echo "GLIDEIN_Signature ${sign_id}" + echo "GLIDEIN_Entry_Signature ${sign_entry_id}" + + if [ -n "${client_repository_url}" ]; then + echo "GLIDECLIENT_WORK_DIR ${client_dir}" + echo "GLIDECLIENT_DESCRIPTION_FILE ${client_descript_file}" + echo "GLIDECLIENT_Signature ${client_sign_id}" + if [ -n "${client_repository_group_url}" ]; then + echo "GLIDECLIENT_GROUP_WORK_DIR ${client_group_dir}" + echo "GLIDECLIENT_DESCRIPTION_GROUP_FILE ${client_descript_group_file}" + echo "GLIDECLIENT_Group_Signature ${client_sign_group_id}" + fi + fi + echo "B64UUENCODE_SOURCE ${PWD}/b64uuencode.source" + echo "ADD_CONFIG_LINE_SOURCE ${PWD}/add_config_line.source" + echo "GET_ID_SELECTORS_SOURCE ${PWD}/get_id_selectors.source" + echo "LOGGING_UTILS_SOURCE ${PWD}/logging_utils.source" + echo "GLIDEIN_PATHS_SOURCE ${PWD}/glidein_paths.source" + echo "WRAPPER_LIST ${wrapper_list}" + echo "SLOTS_LAYOUT ${slots_layout}" + # Add a line saying we are still initializing... + echo "GLIDEIN_INITIALIZED 0" + # ...but be optimist, and leave advertise_only for the actual error handling script + echo "GLIDEIN_ADVERTISE_ONLY 0" + echo "GLIDEIN_CONDOR_TOKEN ${GLIDEIN_CONDOR_TOKEN}" + echo "# --- User Parameters ---" + } >> "${glidein_config}"; then + early_glidein_failure "Failed in updating '${glidein_config}'" + fi +} + ################################ # Block of code used to handle the list of parameters # params will contain the full list of parameters @@ -248,6 +317,7 @@ else slots_layout="partitionable" fi +################################ parse_arguments ################################ @@ -258,17 +328,20 @@ else glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" fi -print_header +################################ +print_header "@" -spawn_multiple_glideins +################################ +spawn_multiple_glideins ######################################## # Code block used to make sure nobody else can write my files -# In the Grid world I cannot trust anybody +# in the Grid world I cannot trust anybody if ! umask 0022; then early_glidein_failure "Failed in umask 0022" fi +######################################## setup_OSG_Globus ######################################## @@ -300,77 +373,21 @@ if [ ! "${num_gct}" -eq 1 ] ; then echo "${token_err_msg}" 1>&2 fi +######################################## prepare_workdir -extract_all_data # extract and source all the data contained at the end of this script as tarball +######################################## +# extract and source all the data contained at the end of this script as tarball +extract_all_data +######################################## wrapper_list="${PWD}/wrapper_list.lst" touch "${wrapper_list}" -# create glidein_config -glidein_config="${PWD}/glidein_config" -if ! echo > "${glidein_config}"; then - early_glidein_failure "Could not create '${glidein_config}'" -fi -if ! { - echo "# --- glidein_startup vals ---" - echo "GLIDEIN_UUID ${glidein_uuid}" - echo "GLIDEIN_Factory ${glidein_factory}" - echo "GLIDEIN_Name ${glidein_name}" - echo "GLIDEIN_Entry_Name ${glidein_entry}" - - if [ -n "${client_name}" ]; then - # client name not required as it is not used for anything but debug info - echo "GLIDECLIENT_Name ${client_name}" - fi - if [ -n "${client_group}" ]; then - # client group not required as it is not used for anything but debug info - echo "GLIDECLIENT_Group ${client_group}" - fi - echo "GLIDEIN_CredentialIdentifier ${glidein_cred_id}" - echo "CONDORG_CLUSTER ${condorg_cluster}" - echo "CONDORG_SUBCLUSTER ${condorg_subcluster}" - echo "CONDORG_SCHEDD ${condorg_schedd}" - echo "DEBUG_MODE ${set_debug}" - echo "GLIDEIN_STARTUP_PID $$" - echo "GLIDEIN_START_DIR_ORIG ${start_dir}" - echo "GLIDEIN_WORKSPACE_ORIG $(pwd)" - echo "GLIDEIN_WORK_DIR ${main_dir}" - echo "GLIDEIN_ENTRY_WORK_DIR ${entry_dir}" - echo "TMP_DIR ${glide_tmp_dir}" - echo "GLIDEIN_LOCAL_TMP_DIR ${glide_local_tmp_dir}" - echo "PROXY_URL ${proxy_url}" - echo "DESCRIPTION_FILE ${descript_file}" - echo "DESCRIPTION_ENTRY_FILE ${descript_entry_file}" - echo "GLIDEIN_Signature ${sign_id}" - echo "GLIDEIN_Entry_Signature ${sign_entry_id}" - - if [ -n "${client_repository_url}" ]; then - echo "GLIDECLIENT_WORK_DIR ${client_dir}" - echo "GLIDECLIENT_DESCRIPTION_FILE ${client_descript_file}" - echo "GLIDECLIENT_Signature ${client_sign_id}" - if [ -n "${client_repository_group_url}" ]; then - echo "GLIDECLIENT_GROUP_WORK_DIR ${client_group_dir}" - echo "GLIDECLIENT_DESCRIPTION_GROUP_FILE ${client_descript_group_file}" - echo "GLIDECLIENT_Group_Signature ${client_sign_group_id}" - fi - fi - echo "B64UUENCODE_SOURCE ${PWD}/b64uuencode.source" - echo "ADD_CONFIG_LINE_SOURCE ${PWD}/add_config_line.source" - echo "GET_ID_SELECTORS_SOURCE ${PWD}/get_id_selectors.source" - echo "LOGGING_UTILS_SOURCE ${PWD}/logging_utils.source" - echo "GLIDEIN_PATHS_SOURCE ${PWD}/glidein_paths.source" - echo "WRAPPER_LIST ${wrapper_list}" - echo "SLOTS_LAYOUT ${slots_layout}" - # Add a line saying we are still initializing... - echo "GLIDEIN_INITIALIZED 0" - # ...but be optimist, and leave advertise_only for the actual error handling script - echo "GLIDEIN_ADVERTISE_ONLY 0" - echo "GLIDEIN_CONDOR_TOKEN ${GLIDEIN_CONDOR_TOKEN}" - echo "# --- User Parameters ---" -} >> "${glidein_config}"; then - early_glidein_failure "Failed in updating '${glidein_config}'" -fi +######################################## +create_glidein_config + +######################################## # shellcheck disable=SC2086 params2file ${params} @@ -380,13 +397,11 @@ log_init "${glidein_uuid}" "${work_dir}" # Remove these files, if they are still there rm -rf tokens.tgz url_dirs.desc tokens log_setup "${glidein_config}" - echo "Downloading files from Factory and Frontend" log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" ##################################### # Fetch descript and signature files - # disable signature check before I get the signature file itself # check_signature is global check_signature=0 @@ -543,12 +558,12 @@ do fi done +############################# fixup_condor_dir ############################## # Start the glidein main script add_config_line "GLIDEIN_INITIALIZED" "1" - log_write "glidein_startup.sh" "text" "Starting the glidein main script" "info" log_write "glidein_startup.sh" "file" "${glidein_config}" "debug" send_logs_to_remote # checkpoint @@ -573,7 +588,6 @@ fi last_startup_end_time=$(date +%s) "${main_dir}"/error_augment.sh -process ${ret} "${last_script}" "${PWD}" "${gs_id_work_dir}/${last_script} glidein_config" "${last_startup_time}" "${last_startup_end_time}" "${main_dir}"/error_augment.sh -concat - let last_script_time=${last_startup_end_time}-${last_startup_time} echo "=== Last script ended $(date) (${last_startup_end_time}) with code ${ret} after ${last_script_time} ===" echo @@ -581,6 +595,7 @@ if [ ${ret} -ne 0 ]; then log_warn "Error running '${last_script}'" fi +############################# #Things like periodic scripts might put messages here if they want them printed in the (stderr) logfile echo "=== Exit messages left by periodic scripts ===" 1>&2 if [ -f exit_message ]; then diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index fb12d43527..876d6eeacc 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -4,7 +4,7 @@ # 1: file name # 2: option (quiet) # Returns: -# 1 un case the md5sum cannot be calculated, or neither the md5sum nor the md5 can be found +# 1 in case the md5sum cannot be calculated, or neither the md5sum nor the md5 can be found md5wrapper() { local ERROR_RESULT="???" local ONLY_SUM diff --git a/creation/web_base/utils_fetch.sh b/creation/web_base/utils_fetch.sh deleted file mode 100644 index 4902cdf51d..0000000000 --- a/creation/web_base/utils_fetch.sh +++ /dev/null @@ -1,437 +0,0 @@ -############################################ -# Function that gets the proper descript file based on id -# Arguments: -# 1: type (main/entry/client/client_group) -# Returns: -# 1 in case of invalid ID -get_repository_url() { - case "$1" in - main) echo "${repository_url}";; - entry) echo "${repository_entry_url}";; - client) echo "${client_repository_url}";; - client_group) echo "${client_repository_group_url}";; - *) echo "[get_repository_url] Invalid id: $1" 1>&2 - return 1 - ;; - esac -} - -##################### -# Periodic execution support function and global variable -add_startd_cron_counter=0 - -##################### -# Function that schedules a script for periodic execution using startd_cron -# Arguments: -# 1: wrapper full path -# 2: period -# 3: cwd -# 4: executable path (from cwd), -# 5: config file path (from cwd) -# 6: ID -# Globals: -# add_startd_cron_counter -add_periodic_script() { - #TODO: should it allow for variable number of parameters? - local include_fname=condor_config_startd_cron_include - local s_wrapper="$1" - local s_period_sec="${2}s" - local s_cwd="$3" - local s_fname="$4" - local s_config="$5" - local s_ffb_id="$6" - local s_cc_prefix="$7" - if [ ${add_startd_cron_counter} -eq 0 ]; then - # Make sure that no undesired file is there when called for first cron - rm -f ${include_fname} - fi - - let add_startd_cron_counter=add_startd_cron_counter+1 - local name_prefix=GLIDEIN_PS_ - local s_name="${name_prefix}${add_startd_cron_counter}" - - # Append the following to the startd configuration - # Instead of Periodic and Kill wait for completion: - # STARTD_CRON_DATE_MODE = WaitForExit - cat >> ${include_fname} << EOF -STARTD_CRON_JOBLIST = \$(STARTD_CRON_JOBLIST) ${s_name} -STARTD_CRON_${s_name}_MODE = Periodic -STARTD_CRON_${s_name}_KILL = True -STARTD_CRON_${s_name}_PERIOD = ${s_period_sec} -STARTD_CRON_${s_name}_EXECUTABLE = ${s_wrapper} -STARTD_CRON_${s_name}_ARGS = ${s_config} ${s_ffb_id} ${s_name} ${s_fname} ${s_cc_prefix} -STARTD_CRON_${s_name}_CWD = ${s_cwd} -STARTD_CRON_${s_name}_SLOTS = 1 -STARTD_CRON_${s_name}_JOB_LOAD = 0.01 -EOF - # NOPREFIX is a keyword for not setting the prefix for all condor attributes - [ "xNOPREFIX" != "x${s_cc_prefix}" ] && echo "STARTD_CRON_${s_name}_PREFIX = ${s_cc_prefix}" >> ${include_fname} - add_config_line "GLIDEIN_condor_config_startd_cron_include" "${include_fname}" - add_config_line "# --- Lines starting with ${s_cc_prefix} are from periodic scripts ---" -} - -############################################ -# Function that fetches a single regular file -# Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) -fetch_file_regular() { - fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" -} - -############################################ -# Function that fetches a single file -# custom_scripts parameters format is set in the GWMS configuration (creation/lib) -# Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) -# Arguments: -# 1: ID -# 2: target fname -# 3: real fname -# 4: file type (regular, exec, exec:s, untar, nocache) -# 5: period (0 if not a periodic file) -# 6: periodic scripts prefix -# 7: config check TRUE,FALSE -# 8: config out TRUE,FALSE -# The above is the most recent list, below some adaptations for different versions -# Returns: -# 0 in case of success -# otherwise glidein_exit with 1 -fetch_file() { - if [ $# -gt 8 ]; then - # For compatibility w/ future versions (add new parameters at the end) - echo "More then 8 arguments, considering the first 8 ($#/${ifs_str}): $*" 1>&2 - elif [ $# -ne 8 ]; then - if [ $# -eq 7 ]; then - #TODO: remove in version 3.3 - # For compatibility with past versions (old file list formats) - # 3.2.13 and older: prefix (par 6) added in #12705, 3.2.14? - # 3.2.10 and older: period (par 5) added: fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "GLIDEIN_PS_" "$6" "$7"; then - glidein_exit 1 - fi - return 0 - fi - if [ $# -eq 6 ]; then - # added to maintain compatibility with older (3.2.10) file list format - #TODO: remove in version 3.3 - if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6"; then - glidein_exit 1 - fi - return 0 - fi - local ifs_str - printf -v ifs_str '%q' "${IFS}" - log_warn "Not enough arguments in fetch_file, 8 expected ($#/${ifs_str}): $*" - glidein_exit 1 - fi - - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8"; then - glidein_exit 1 - fi - return 0 -} - -############################################ -# Function that verifies if the file should be downloaded and acted upon (extracted, executed, ...) or not -# There are 2 mechanisms to control the download -# 1. tar files have the attribute "cond_attr" that is a name of a variable in glidein_config. -# if the named variable has value 1, then the file is downloaded. TRUE (default) means always download -# even if the mechanism is generic, there is no way to specify "cond_attr" for regular files in the configuration -# 2. if the file name starts with "gconditional_AAA_", the file is downloaded only if a variable GLIDEIN_USE_AAA -# exists in glidein_config and the value is not empty -# Both conditions are checked. If either one fails the file is not downloaded -# Arguments: -# 1: ID -# 2: target fname -# 3: real fname -# 4: file type (regular, exec, exec:s, untar, nocache) -# 5: period (0 if not a periodic file) -# 6: periodic scripts prefix -# 7: config check TRUE,FALSE -# 8: config out TRUE,FALSE -# Globals: -# fft_id -# fft_target_fname -# fft_real_fname -# fft_file_type -# fft_period -# fft_cc_prefix -# fft_config_check -# fft_config_out -# Returns: -# 0 in case of success -fetch_file_try() { - fft_id="$1" - fft_target_fname="$2" - fft_real_fname="$3" - fft_file_type="$4" - fft_period="$5" - fft_cc_prefix="$6" - fft_config_check="$7" - fft_config_out="$8" - - if [[ "${fft_config_check}" != "TRUE" ]]; then - # TRUE is a special case, always downloaded and processed - local fft_get_ss - fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) - # Stop download and processing if the cond_attr variable is not defined or has a value different from 1 - [[ "${fft_get_ss}" != "1" ]] && return 0 - # TODO: what if fft_get_ss is not 1? nothing, still skip the file? - fi - - local fft_base_name fft_condition_attr fft_condition_attr_val - fft_base_name=$(basename "${fft_real_fname}") - if [[ "${fft_base_name}" = gconditional_* ]]; then - fft_condition_attr="${fft_base_name#gconditional_}" - fft_condition_attr="GLIDEIN_USE_${fft_condition_attr%%_*}" - fft_condition_attr_val=$(grep -i "^${fft_condition_attr} " glidein_config | cut -d ' ' -f 2-) - # if the variable fft_condition_attr is not defined or empty, do not download - [[ -z "${fft_condition_attr_val}" ]] && return 0 - fi - - fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" - # returning the exit code of fetch_file_base -} - -############################################ -# Function that performs the file download and corresponding action (untar, execute, ...) -# Arguments: -# 1: ID -# 2: target fname -# 3: real fname -# 4: file type (regular, exec, exec:s, untar, nocache) -# 5: config out TRUE,FALSE -# 5: period (0 if not a periodic file) -# 6: periodic scripts prefix -# Globals: -# ffb_id -# ffb_target_fname -# ffb_real_fname -# ffb_file_type -# ffb_period -# ffb_cc_prefix -# ffb_config_out -# ffb_work_dir -# ffb_repository -# ffb_tmp_outname -# ffb_short_untar_dir -# ffb_untar_dir -# ffb_outname -# have_dummy_otrx -# user_agent -# ffb_url -# curl_version -# curl_args -# wget_version -# wget_args -# fetch_completed -# Returns: -# 1 in case the error is already displayed inside the function, -# in case of tarring, in case of failure in renaming it, -# of making it executable, or executing it -fetch_file_base() { - ffb_id="$1" - ffb_target_fname="$2" - ffb_real_fname="$3" - ffb_file_type="$4" - ffb_config_out="$5" - ffb_period=$6 - # condor cron prefix, used only for periodic executables - ffb_cc_prefix="$7" - - ffb_work_dir="$(get_work_dir "${ffb_id}")" - - ffb_repository="$(get_repository_url "${ffb_id}")" - - ffb_tmp_outname="${ffb_work_dir}/${ffb_real_fname}" - ffb_outname="${ffb_work_dir}/${ffb_target_fname}" - - # Create a dummy default in case something goes wrong - # cannot use error_*.sh helper functions - # may not have been loaded yet - have_dummy_otrx=1 - echo " - - - ${PWD} - - - Unknown - $(date +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - ERROR - Unknown - ${ffb_id} - - - An unknown error occurred. - -" > otrx_output.xml - user_agent="glidein/${glidein_entry}/${condorg_schedd}/${condorg_cluster}.${condorg_subcluster}/${client_name}" - ffb_url="${ffb_repository}/${ffb_real_fname}" - curl_version=$(curl --version | head -1 ) - wget_version=$(wget --version | head -1 ) - #old wget command: - #wget --user-agent="wget/glidein/$glidein_entry/$condorg_schedd/$condorg_cluster.$condorg_subcluster/$client_name" "$ffb_nocache_str" -q -O "$ffb_tmp_outname" "$ffb_repository/$ffb_real_fname" - #equivalent to: - #wget ${ffb_url} --user-agent=${user_agent} -q -O "${ffb_tmp_outname}" "${ffb_nocache_str}" - #with env http_proxy=$proxy_url set if proxy_url != "None" - # - #construct curl equivalent so we can try either - - wget_args=("${ffb_url}" "--user-agent" "wget/${user_agent}" "--quiet" "--output-document" "${ffb_tmp_outname}" ) - curl_args=("${ffb_url}" "--user-agent" "curl/${user_agent}" "--silent" "--show-error" "--output" "${ffb_tmp_outname}") - - if [ "${ffb_file_type}" = "nocache" ]; then - if [ "${curl_version}" != "" ]; then - curl_args+=("--header") - curl_args+=("'Cache-Control: no-cache'") - fi - if [ "${wget_version}" != "" ]; then - if wget --help | grep -q "\-\-no-cache "; then - wget_args+=("--no-cache") - elif wget --help |grep -q "\-\-cache="; then - wget_args+=("--cache=off") - else - log_warn "wget ${wget_version} cannot disable caching" - fi - fi - fi - - if [ "${proxy_url}" != "None" ];then - if [ "${curl_version}" != "" ]; then - curl_args+=("--proxy") - curl_args+=("${proxy_url}") - fi - if [ "${wget_version}" != "" ]; then - #these two arguments have to be last as coded, put any future - #wget args earlier in wget_args array - wget_args+=("--proxy") - wget_args+=("${proxy_url}") - fi - fi - - fetch_completed=1 - if [ ${fetch_completed} -ne 0 ] && [ "${wget_version}" != "" ]; then - perform_wget "${wget_args[@]}" - fetch_completed=$? - fi - if [ ${fetch_completed} -ne 0 ] && [ "${curl_version}" != "" ]; then - perform_curl "${curl_args[@]}" - fetch_completed=$? - fi - - if [ ${fetch_completed} -ne 0 ]; then - return ${fetch_completed} - fi - - # check signature - if ! check_file_signature "${ffb_id}" "${ffb_real_fname}"; then - # error already displayed inside the function - return 1 - fi - - # rename it to the correct final name, if needed - if [ "${ffb_tmp_outname}" != "${ffb_outname}" ]; then - if ! mv "${ffb_tmp_outname}" "${ffb_outname}"; then - log_warn "Failed to rename ${ffb_tmp_outname} into ${ffb_outname}" - return 1 - fi - fi - - # if executable, execute - if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then - if ! chmod u+x "${ffb_outname}"; then - log_warn "Error making '${ffb_outname}' executable" - return 1 - fi - if [ "${ffb_id}" = "main" ] && [ "${ffb_target_fname}" = "${last_script}" ]; then # last_script global for simplicity - echo "Skipping last script ${last_script}" 1>&2 - elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity - # TODO: temporary OR checking for cvmfs_umount.sh; to be removed after Bruno's ticket on cleanup [#25073] - echo "Skipping cleanup script ${ffb_outname} (${cleanup_script})" 1>&2 - cp "${ffb_outname}" "$gwms_exec_dir/cleanup/${ffb_target_fname}" - chmod a+x "${gwms_exec_dir}/cleanup/${ffb_target_fname}" - else - echo "Executing (flags:${ffb_file_type#exec}) ${ffb_outname}" - # have to do it here, as this will be run before any other script - chmod u+rx "${main_dir}"/error_augment.sh - - # the XML file will be overwritten now, and hopefully not an error situation - have_dummy_otrx=0 - "${main_dir}"/error_augment.sh -init - START=$(date +%s) - if [[ "${ffb_file_type}" = "exec:s" ]]; then - "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" - else - "${ffb_outname}" glidein_config "${ffb_id}" - fi - ret=$? - END=$(date +%s) - "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_outname} glidein_config" "${START}" "${END}" #generating test result document - "${main_dir}"/error_augment.sh -concat - if [ ${ret} -ne 0 ]; then - echo "=== Validation error in ${ffb_outname} ===" 1>&2 - log_warn "Error running '${ffb_outname}'" - < otrx_output.xml awk 'BEGIN{fr=0;}/<[/]detail>/{fr=0;}{if (fr==1) print $0}//{fr=1;}' 1>&2 - return 1 - else - # If ran successfully and periodic, schedule to execute with schedd_cron - echo "=== validation OK in ${ffb_outname} (${ffb_period}) ===" 1>&2 - if [ "${ffb_period}" -gt 0 ]; then - add_periodic_script "${main_dir}/script_wrapper.sh" "${ffb_period}" "${work_dir}" "${ffb_outname}" glidein_config "${ffb_id}" "${ffb_cc_prefix}" - fi - fi - fi - elif [ "${ffb_file_type}" = "wrapper" ]; then - echo "${ffb_outname}" >> "${wrapper_list}" - elif [ "${ffb_file_type}" = "untar" ]; then - ffb_short_untar_dir="$(get_untar_subdir "${ffb_id}" "${ffb_target_fname}")" - ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" - START=$(date +%s) - (mkdir "${ffb_untar_dir}" && cd "${ffb_untar_dir}" && tar -xmzf "${ffb_outname}") 1>&2 - ret=$? - if [ ${ret} -ne 0 ]; then - "${main_dir}"/error_augment.sh -init - "${main_dir}"/error_gen.sh -error "tar" "Corruption" "Error untarring '${ffb_outname}'" "file" "${ffb_outname}" "source_type" "${cfs_id}" - "${main_dir}"/error_augment.sh -process ${cfs_rc} "tar" "${PWD}" "mkdir ${ffb_untar_dir} && cd ${ffb_untar_dir} && tar -xmzf ${ffb_outname}" "${START}" "$(date +%s)" - "${main_dir}"/error_augment.sh -concat - log_warn "Error untarring '${ffb_outname}'" - return 1 - fi - fi - - if [ "${ffb_config_out}" != "FALSE" ]; then - ffb_prefix="$(get_prefix "${ffb_id}")" - if [ "${ffb_file_type}" = "untar" ]; then - # when untaring the original file is less interesting than the untar dir - if ! add_config_line "${ffb_prefix}${ffb_config_out}" "${ffb_untar_dir}"; then - glidein_exit 1 - fi - else - if ! add_config_line "${ffb_prefix}${ffb_config_out}" "${ffb_outname}"; then - glidein_exit 1 - fi - fi - fi - - if [ "${have_dummy_otrx}" -eq 1 ]; then - # no one should really look at this file, but just to avoid confusion - echo " - - - ${PWD} - - - Unknown - $(date +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - OK - -" > otrx_output.xml - fi - - return 0 -} diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index dc1d011e83..87542d201c 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -1,3 +1,438 @@ +############################################ +# Function that gets the proper descript file based on id +# Arguments: +# 1: type (main/entry/client/client_group) +# Returns: +# 1 in case of invalid ID +get_repository_url() { + case "$1" in + main) echo "${repository_url}";; + entry) echo "${repository_entry_url}";; + client) echo "${client_repository_url}";; + client_group) echo "${client_repository_group_url}";; + *) echo "[get_repository_url] Invalid id: $1" 1>&2 + return 1 + ;; + esac +} + +##################### +# Periodic execution support function and global variable +add_startd_cron_counter=0 + +##################### +# Function that schedules a script for periodic execution using startd_cron +# Arguments: +# 1: wrapper full path +# 2: period +# 3: cwd +# 4: executable path (from cwd), +# 5: config file path (from cwd) +# 6: ID +# Globals: +# add_startd_cron_counter +add_periodic_script() { + #TODO: should it allow for variable number of parameters? + local include_fname=condor_config_startd_cron_include + local s_wrapper="$1" + local s_period_sec="${2}s" + local s_cwd="$3" + local s_fname="$4" + local s_config="$5" + local s_ffb_id="$6" + local s_cc_prefix="$7" + if [ ${add_startd_cron_counter} -eq 0 ]; then + # Make sure that no undesired file is there when called for first cron + rm -f ${include_fname} + fi + + let add_startd_cron_counter=add_startd_cron_counter+1 + local name_prefix=GLIDEIN_PS_ + local s_name="${name_prefix}${add_startd_cron_counter}" + + # Append the following to the startd configuration + # Instead of Periodic and Kill wait for completion: + # STARTD_CRON_DATE_MODE = WaitForExit + cat >> ${include_fname} << EOF +STARTD_CRON_JOBLIST = \$(STARTD_CRON_JOBLIST) ${s_name} +STARTD_CRON_${s_name}_MODE = Periodic +STARTD_CRON_${s_name}_KILL = True +STARTD_CRON_${s_name}_PERIOD = ${s_period_sec} +STARTD_CRON_${s_name}_EXECUTABLE = ${s_wrapper} +STARTD_CRON_${s_name}_ARGS = ${s_config} ${s_ffb_id} ${s_name} ${s_fname} ${s_cc_prefix} +STARTD_CRON_${s_name}_CWD = ${s_cwd} +STARTD_CRON_${s_name}_SLOTS = 1 +STARTD_CRON_${s_name}_JOB_LOAD = 0.01 +EOF + # NOPREFIX is a keyword for not setting the prefix for all condor attributes + [ "xNOPREFIX" != "x${s_cc_prefix}" ] && echo "STARTD_CRON_${s_name}_PREFIX = ${s_cc_prefix}" >> ${include_fname} + add_config_line "GLIDEIN_condor_config_startd_cron_include" "${include_fname}" + add_config_line "# --- Lines starting with ${s_cc_prefix} are from periodic scripts ---" +} + +############################################ +# Function that fetches a single regular file +# Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) +fetch_file_regular() { + fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" +} + +############################################ +# Function that fetches a single file +# custom_scripts parameters format is set in the GWMS configuration (creation/lib) +# Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) +# Arguments: +# 1: ID +# 2: target fname +# 3: real fname +# 4: file type (regular, exec, exec:s, untar, nocache) +# 5: period (0 if not a periodic file) +# 6: periodic scripts prefix +# 7: config check TRUE,FALSE +# 8: config out TRUE,FALSE +# The above is the most recent list, below some adaptations for different versions +# Returns: +# 0 in case of success +# otherwise glidein_exit with 1 +fetch_file() { + if [ $# -gt 8 ]; then + # For compatibility w/ future versions (add new parameters at the end) + echo "More then 8 arguments, considering the first 8 ($#/${ifs_str}): $*" 1>&2 + elif [ $# -ne 8 ]; then + if [ $# -eq 7 ]; then + #TODO: remove in version 3.3 + # For compatibility with past versions (old file list formats) + # 3.2.13 and older: prefix (par 6) added in #12705, 3.2.14? + # 3.2.10 and older: period (par 5) added: fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" + if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "GLIDEIN_PS_" "$6" "$7"; then + glidein_exit 1 + fi + return 0 + fi + if [ $# -eq 6 ]; then + # added to maintain compatibility with older (3.2.10) file list format + #TODO: remove in version 3.3 + if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6"; then + glidein_exit 1 + fi + return 0 + fi + local ifs_str + printf -v ifs_str '%q' "${IFS}" + log_warn "Not enough arguments in fetch_file, 8 expected ($#/${ifs_str}): $*" + glidein_exit 1 + fi + + if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8"; then + glidein_exit 1 + fi + return 0 +} + +############################################ +# Function that verifies if the file should be downloaded and acted upon (extracted, executed, ...) or not +# There are 2 mechanisms to control the download +# 1. tar files have the attribute "cond_attr" that is a name of a variable in glidein_config. +# if the named variable has value 1, then the file is downloaded. TRUE (default) means always download +# even if the mechanism is generic, there is no way to specify "cond_attr" for regular files in the configuration +# 2. if the file name starts with "gconditional_AAA_", the file is downloaded only if a variable GLIDEIN_USE_AAA +# exists in glidein_config and the value is not empty +# Both conditions are checked. If either one fails the file is not downloaded +# Arguments: +# 1: ID +# 2: target fname +# 3: real fname +# 4: file type (regular, exec, exec:s, untar, nocache) +# 5: period (0 if not a periodic file) +# 6: periodic scripts prefix +# 7: config check TRUE,FALSE +# 8: config out TRUE,FALSE +# Globals: +# fft_id +# fft_target_fname +# fft_real_fname +# fft_file_type +# fft_period +# fft_cc_prefix +# fft_config_check +# fft_config_out +# Returns: +# 0 in case of success +fetch_file_try() { + fft_id="$1" + fft_target_fname="$2" + fft_real_fname="$3" + fft_file_type="$4" + fft_period="$5" + fft_cc_prefix="$6" + fft_config_check="$7" + fft_config_out="$8" + + if [[ "${fft_config_check}" != "TRUE" ]]; then + # TRUE is a special case, always downloaded and processed + local fft_get_ss + fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) + # Stop download and processing if the cond_attr variable is not defined or has a value different from 1 + [[ "${fft_get_ss}" != "1" ]] && return 0 + # TODO: what if fft_get_ss is not 1? nothing, still skip the file? + fi + + local fft_base_name fft_condition_attr fft_condition_attr_val + fft_base_name=$(basename "${fft_real_fname}") + if [[ "${fft_base_name}" = gconditional_* ]]; then + fft_condition_attr="${fft_base_name#gconditional_}" + fft_condition_attr="GLIDEIN_USE_${fft_condition_attr%%_*}" + fft_condition_attr_val=$(grep -i "^${fft_condition_attr} " glidein_config | cut -d ' ' -f 2-) + # if the variable fft_condition_attr is not defined or empty, do not download + [[ -z "${fft_condition_attr_val}" ]] && return 0 + fi + + fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" + # returning the exit code of fetch_file_base +} + +############################################ +# Function that performs the file download and corresponding action (untar, execute, ...) +# Arguments: +# 1: ID +# 2: target fname +# 3: real fname +# 4: file type (regular, exec, exec:s, untar, nocache) +# 5: config out TRUE,FALSE +# 5: period (0 if not a periodic file) +# 6: periodic scripts prefix +# Globals: +# ffb_id +# ffb_target_fname +# ffb_real_fname +# ffb_file_type +# ffb_period +# ffb_cc_prefix +# ffb_config_out +# ffb_work_dir +# ffb_repository +# ffb_tmp_outname +# ffb_short_untar_dir +# ffb_untar_dir +# ffb_outname +# have_dummy_otrx +# user_agent +# ffb_url +# curl_version +# curl_args +# wget_version +# wget_args +# fetch_completed +# Returns: +# 1 in case the error is already displayed inside the function, +# in case of tarring, in case of failure in renaming it, +# of making it executable, or executing it +fetch_file_base() { + ffb_id="$1" + ffb_target_fname="$2" + ffb_real_fname="$3" + ffb_file_type="$4" + ffb_config_out="$5" + ffb_period=$6 + # condor cron prefix, used only for periodic executables + ffb_cc_prefix="$7" + ffb_work_dir="$(get_work_dir "${ffb_id}")" + ffb_repository="$(get_repository_url "${ffb_id}")" + ffb_tmp_outname="${ffb_work_dir}/${ffb_real_fname}" + ffb_outname="${ffb_work_dir}/${ffb_target_fname}" + + # Create a dummy default in case something goes wrong + # cannot use error_*.sh helper functions + # may not have been loaded yet + have_dummy_otrx=1 + echo " + + + ${PWD} + + + Unknown + $(date +%Y-%m-%dT%H:%M:%S%:z) + $(date +%Y-%m-%dT%H:%M:%S%:z) + + + ERROR + Unknown + ${ffb_id} + + + An unknown error occurred. + +" > otrx_output.xml + user_agent="glidein/${glidein_entry}/${condorg_schedd}/${condorg_cluster}.${condorg_subcluster}/${client_name}" + ffb_url="${ffb_repository}/${ffb_real_fname}" + curl_version=$(curl --version | head -1 ) + wget_version=$(wget --version | head -1 ) + #old wget command: + #wget --user-agent="wget/glidein/$glidein_entry/$condorg_schedd/$condorg_cluster.$condorg_subcluster/$client_name" "$ffb_nocache_str" -q -O "$ffb_tmp_outname" "$ffb_repository/$ffb_real_fname" + #equivalent to: + #wget ${ffb_url} --user-agent=${user_agent} -q -O "${ffb_tmp_outname}" "${ffb_nocache_str}" + #with env http_proxy=$proxy_url set if proxy_url != "None" + # + #construct curl equivalent so we can try either + + wget_args=("${ffb_url}" "--user-agent" "wget/${user_agent}" "--quiet" "--output-document" "${ffb_tmp_outname}" ) + curl_args=("${ffb_url}" "--user-agent" "curl/${user_agent}" "--silent" "--show-error" "--output" "${ffb_tmp_outname}") + + if [ "${ffb_file_type}" = "nocache" ]; then + if [ "${curl_version}" != "" ]; then + curl_args+=("--header") + curl_args+=("'Cache-Control: no-cache'") + fi + if [ "${wget_version}" != "" ]; then + if wget --help | grep -q "\-\-no-cache "; then + wget_args+=("--no-cache") + elif wget --help |grep -q "\-\-cache="; then + wget_args+=("--cache=off") + else + log_warn "wget ${wget_version} cannot disable caching" + fi + fi + fi + + if [ "${proxy_url}" != "None" ];then + if [ "${curl_version}" != "" ]; then + curl_args+=("--proxy") + curl_args+=("${proxy_url}") + fi + if [ "${wget_version}" != "" ]; then + #these two arguments have to be last as coded, put any future + #wget args earlier in wget_args array + wget_args+=("--proxy") + wget_args+=("${proxy_url}") + fi + fi + + fetch_completed=1 + if [ ${fetch_completed} -ne 0 ] && [ "${wget_version}" != "" ]; then + perform_wget "${wget_args[@]}" + fetch_completed=$? + fi + if [ ${fetch_completed} -ne 0 ] && [ "${curl_version}" != "" ]; then + perform_curl "${curl_args[@]}" + fetch_completed=$? + fi + + if [ ${fetch_completed} -ne 0 ]; then + return ${fetch_completed} + fi + + # check signature + if ! check_file_signature "${ffb_id}" "${ffb_real_fname}"; then + # error already displayed inside the function + return 1 + fi + + # rename it to the correct final name, if needed + if [ "${ffb_tmp_outname}" != "${ffb_outname}" ]; then + if ! mv "${ffb_tmp_outname}" "${ffb_outname}"; then + log_warn "Failed to rename ${ffb_tmp_outname} into ${ffb_outname}" + return 1 + fi + fi + + # if executable, execute + if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then + if ! chmod u+x "${ffb_outname}"; then + log_warn "Error making '${ffb_outname}' executable" + return 1 + fi + if [ "${ffb_id}" = "main" ] && [ "${ffb_target_fname}" = "${last_script}" ]; then # last_script global for simplicity + echo "Skipping last script ${last_script}" 1>&2 + elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity + # TODO: temporary OR checking for cvmfs_umount.sh; to be removed after Bruno's ticket on cleanup [#25073] + echo "Skipping cleanup script ${ffb_outname} (${cleanup_script})" 1>&2 + cp "${ffb_outname}" "$gwms_exec_dir/cleanup/${ffb_target_fname}" + chmod a+x "${gwms_exec_dir}/cleanup/${ffb_target_fname}" + else + echo "Executing (flags:${ffb_file_type#exec}) ${ffb_outname}" + # have to do it here, as this will be run before any other script + chmod u+rx "${main_dir}"/error_augment.sh + + # the XML file will be overwritten now, and hopefully not an error situation + have_dummy_otrx=0 + "${main_dir}"/error_augment.sh -init + START=$(date +%s) + if [[ "${ffb_file_type}" = "exec:s" ]]; then + "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" + else + "${ffb_outname}" glidein_config "${ffb_id}" + fi + ret=$? + END=$(date +%s) + "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_outname} glidein_config" "${START}" "${END}" #generating test result document + "${main_dir}"/error_augment.sh -concat + if [ ${ret} -ne 0 ]; then + echo "=== Validation error in ${ffb_outname} ===" 1>&2 + log_warn "Error running '${ffb_outname}'" + < otrx_output.xml awk 'BEGIN{fr=0;}/<[/]detail>/{fr=0;}{if (fr==1) print $0}//{fr=1;}' 1>&2 + return 1 + else + # If ran successfully and periodic, schedule to execute with schedd_cron + echo "=== validation OK in ${ffb_outname} (${ffb_period}) ===" 1>&2 + if [ "${ffb_period}" -gt 0 ]; then + add_periodic_script "${main_dir}/script_wrapper.sh" "${ffb_period}" "${work_dir}" "${ffb_outname}" glidein_config "${ffb_id}" "${ffb_cc_prefix}" + fi + fi + fi + elif [ "${ffb_file_type}" = "wrapper" ]; then + echo "${ffb_outname}" >> "${wrapper_list}" + elif [ "${ffb_file_type}" = "untar" ]; then + ffb_short_untar_dir="$(get_untar_subdir "${ffb_id}" "${ffb_target_fname}")" + ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" + START=$(date +%s) + (mkdir "${ffb_untar_dir}" && cd "${ffb_untar_dir}" && tar -xmzf "${ffb_outname}") 1>&2 + ret=$? + if [ ${ret} -ne 0 ]; then + "${main_dir}"/error_augment.sh -init + "${main_dir}"/error_gen.sh -error "tar" "Corruption" "Error untarring '${ffb_outname}'" "file" "${ffb_outname}" "source_type" "${cfs_id}" + "${main_dir}"/error_augment.sh -process ${cfs_rc} "tar" "${PWD}" "mkdir ${ffb_untar_dir} && cd ${ffb_untar_dir} && tar -xmzf ${ffb_outname}" "${START}" "$(date +%s)" + "${main_dir}"/error_augment.sh -concat + log_warn "Error untarring '${ffb_outname}'" + return 1 + fi + fi + + if [ "${ffb_config_out}" != "FALSE" ]; then + ffb_prefix="$(get_prefix "${ffb_id}")" + if [ "${ffb_file_type}" = "untar" ]; then + # when untaring the original file is less interesting than the untar dir + if ! add_config_line "${ffb_prefix}${ffb_config_out}" "${ffb_untar_dir}"; then + glidein_exit 1 + fi + else + if ! add_config_line "${ffb_prefix}${ffb_config_out}" "${ffb_outname}"; then + glidein_exit 1 + fi + fi + fi + + if [ "${have_dummy_otrx}" -eq 1 ]; then + # no one should really look at this file, but just to avoid confusion + echo " + + + ${PWD} + + + Unknown + $(date +%Y-%m-%dT%H:%M:%S%:z) + $(date +%Y-%m-%dT%H:%M:%S%:z) + + + OK + +" > otrx_output.xml + fi + + return 0 +} + ################################ # Function used to perform a wget request # Arguments: diff --git a/creation/web_base/utils_log.sh b/creation/web_base/utils_io.sh similarity index 99% rename from creation/web_base/utils_log.sh rename to creation/web_base/utils_io.sh index 6f9b63ea64..fe08a3a0a9 100644 --- a/creation/web_base/utils_log.sh +++ b/creation/web_base/utils_io.sh @@ -106,6 +106,8 @@ usage() { ################################ # Function used to print initial information header +# Parameters: +# @: shell parameters # Global: # startup_time print_header(){ diff --git a/creation/web_base/utils_signals.sh b/creation/web_base/utils_signals.sh index 1ca8cef651..6b0553b36e 100644 --- a/creation/web_base/utils_signals.sh +++ b/creation/web_base/utils_signals.sh @@ -22,7 +22,6 @@ on_die() { kill -s "$1" %1 } - ################################ # Function that forwards signals to the children processes # Arguments: diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 612c17b38e..b01254734e 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -21,7 +21,7 @@ construct_xml() { # " #printf "$OSG_test_result" "${start_dir}" "$0" "${global_args}" "$(date --date=@\"${startup_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "$result" #create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd ${start_dir} } test { cmd $0 ${global_args} tStart $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) tEnd $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)} } - create_xml OSG --id 0 { operatingenvironment { env --name cwd 0 } test { cmd 0 0 tStart 0 tEnd 0 } 0 } + ##create_xml OSG --id 0 { operatingenvironment { env --name cwd 0 } test { cmd 0 0 tStart 0 tEnd 0 } 0 } echo " From 8d55fb0796634220d45e8081b0ce13494e381c34 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 10 Aug 2022 14:23:41 -0500 Subject: [PATCH 009/196] Marked some useless global variables as local --- creation/web_base/glidein_cleanup.sh | 30 +++-------- creation/web_base/glidein_startup.sh | 44 ++++++++++++--- creation/web_base/utils_crypto.sh | 6 ++- creation/web_base/utils_filesystem.sh | 6 ++- creation/web_base/utils_http.sh | 30 +++++++---- creation/web_base/utils_io.sh | 6 +-- creation/web_base/utils_params.sh | 3 +- creation/web_base/utils_tarballs.sh | 3 +- creation/web_base/utils_xml.sh | 77 ++++----------------------- 9 files changed, 89 insertions(+), 116 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 4d7e734a78..0c254ddc37 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -10,6 +10,8 @@ glide_local_tmp_dir_created=0 # It uses GLIDEIN_DEBUG_OPTIONS, start_dir, work_dir_created, work_dir, glide_local_tmp_dir_created, glide_local_tmp_dir # Arguments: # 1: exit code +# Outputs: +# warnings to stderr glidien_cleanup() { if ! cd "${start_dir}"; then log_warn "Cannot find ${start_dir} anymore, exiting but without cleanup" @@ -35,14 +37,11 @@ glidien_cleanup() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: error message -# Global: -# error_msg -# glidein_end_time -# result -# final_result -# final_result_simple -# final_result_long +# Outputs: +# warnings to stderr +# tail to stdout early_glidein_failure() { + local error_msg, glidein_end_time, result, final_result, final_result_simple, final_result_long error_msg="$1" log_warn "${error_msg}" sleep "${sleep_time}" @@ -67,22 +66,9 @@ early_glidein_failure() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: exit code -# Global: -# exit_code -# global_result -# ge_last_script_name -# result -# final_result -# final_result_simple -# final_result_long -# report_failed -# factory_report_failes -# factory_collector -# do_report -# dlf -# condor_vars_file -# main_work_dir glidein_exit() { + local exit_code, final_result, final_result_simple, final_result_long, global_result, ge_last_script_name, result, report_failed + local factory_report_failes, factory_collector, do_report, dlf, condor_vars_file, main_work_dir exit_code=$1 # Removed lines about $lock_file (lock file for whole machine) not present elsewhere gwms_process_scripts "$GWMS_DIR" cleanup "${glidein_config}" diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 3755bbbb2f..8d1e315893 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -3,7 +3,7 @@ # glidein_startup.sh # # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # # SPDX-License-Identifier: Apache-2.0 # -# File Version: # +# File Version: # #*******************************************************************# @@ -12,7 +12,7 @@ # better than "unset IFS" because works with restoring old one IFS=$' \t\n' -global_args="$*" +GLOBAL_ARGS="$*" # GWMS_STARTUP_SCRIPT=$0 GWMS_STARTUP_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" @@ -23,7 +23,6 @@ GWMS_STARTUP_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "$ GWMS_PATH="" GWMS_SUBDIR=".gwms.d" -export LANG=C ################################ # General options # Set GWMS_MULTIUSER_GLIDEIN if the Glidein may spawn processes (for jobs) as a different user. @@ -47,6 +46,8 @@ source utils_crypto.sh source utils_http.sh source glidein_cleanup.sh +export LANG=C + ################################ # Function used to start multiple glideins # Arguments: @@ -80,11 +81,12 @@ do_start_all() { initial_dir="$(pwd)" multiglidein_launchall=$(params_decode "$(params_get_simple GLIDEIN_MULTIGLIDEIN_LAUNCHALL "${params}")") multiglidein_launcher=$(params_decode "$(params_get_simple GLIDEIN_MULTIGLIDEIN_LAUNCHER "${params}")") - local startup_script="${GWMS_STARTUP_SCRIPT}" + local startup_script + startup_script="${GWMS_STARTUP_SCRIPT}" if [[ -n "${multiglidein_launchall}" ]]; then echo "Starting multi-glidein using launcher: ${multiglidein_launchall}" # shellcheck disable=SC2086 - ${multiglidein_launchall} "${startup_script}" -multirestart 0 ${global_args} & + ${multiglidein_launchall} "${startup_script}" -multirestart 0 ${GLOBAL_ARGS} & GWMS_MULTIGLIDEIN_CHILDS="${GWMS_MULTIGLIDEIN_CHILDS} $!" else if [[ "${initial_dir}" = "$(dirname "${startup_script}")" ]]; then @@ -97,7 +99,7 @@ do_start_all() { pushd "${g_dir}" || echo "Unable to cd in start directory" chmod +x "${startup_script}" # shellcheck disable=SC2086 - ${multiglidein_launcher} "${startup_script}" -multirestart "${i}" ${global_args} & + ${multiglidein_launcher} "${startup_script}" -multirestart "${i}" ${GLOBAL_ARGS} & GWMS_MULTIGLIDEIN_CHILDS="${GWMS_MULTIGLIDEIN_CHILDS} $!" popd || true done @@ -171,9 +173,11 @@ setup_OSG_Globus(){ # PATH add_to_path() { logdebug "Adding to GWMS_PATH: $1" - local old_path=":${PATH%:}:" + local old_path + old_path=":${PATH%:}:" old_path="${old_path//:$GWMS_PATH:/}" - local old_gwms_path=":${GWMS_PATH%:}:" + local old_gwms_path + old_gwms_path=":${GWMS_PATH%:}:" old_gwms_path="${old_gwms_path//:$1:/}" old_gwms_path="${1%:}:${old_gwms_path#:}" export GWMS_PATH="${old_gwms_path%:}" @@ -271,6 +275,30 @@ create_glidein_config(){ # params will contain the full list of parameters # -param_XXX YYY will become "XXX YYY" #TODO: can use an array instead? +# Transform long options to short ones +#for arg in "$@"; do +# shift +# case "$arg" in +# '-factory') set -- "$@" '-f' ;; +# '-name') set -- "$@" '-n' ;; +# '-entry') set -- "$@" '-e' ;; +# *) set -- "$@" "$arg" ;; +# esac +#done + +#OPTIND=1 +#while getopts "hn:rw" opt +#do +# case "Sopt" in +# 'h') print_usage; exit 0 ;; +# 'n') a=2 ;; +# 'r') rest=true ;; +# 'w') ws=true ;; +# '?') print_usage >&2; exit 1;; +# esac +#done +#shift $(expr $OPTIND- 1) + params="" while [ $# -gt 0 ] do case "$1" in diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index 876d6eeacc..28b7fe1393 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -6,12 +6,14 @@ # Returns: # 1 in case the md5sum cannot be calculated, or neither the md5sum nor the md5 can be found md5wrapper() { - local ERROR_RESULT="???" + local ERROR_RESULT + ERROR_RESULT="???" local ONLY_SUM if [ "x$2" = "xquiet" ]; then ONLY_SUM=yes fi - local executable=md5sum + local executable + executable=md5sum if which ${executable} 1>/dev/null 2>&1; then [ -n "${ONLY_SUM}" ] && executable="md5sum \"$1\" | cut -d ' ' -f 1" || executable="md5sum \"$1\"" else diff --git a/creation/web_base/utils_filesystem.sh b/creation/web_base/utils_filesystem.sh index 9446617d4f..3e15a9195c 100644 --- a/creation/web_base/utils_filesystem.sh +++ b/creation/web_base/utils_filesystem.sh @@ -55,8 +55,10 @@ automatic_work_dir() { # It creates an ID to distinguish the directories when preserved dir_id() { [[ ! ",${GLIDEIN_DEBUG_OPTIONS}," = *,nocleanup,* ]] && return - local dir_id="" - local tmp="${repository_url%%.*}" + local dir_id + dir_id="" + local tmp + tmp="${repository_url%%.*}" tmp="${tmp#*//}" dir_id="${tmp: -4}" tmp="${client_repository_url%%.*}" diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 87542d201c..b62cd84eec 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -33,22 +33,32 @@ add_startd_cron_counter=0 # add_startd_cron_counter add_periodic_script() { #TODO: should it allow for variable number of parameters? - local include_fname=condor_config_startd_cron_include - local s_wrapper="$1" - local s_period_sec="${2}s" - local s_cwd="$3" - local s_fname="$4" - local s_config="$5" - local s_ffb_id="$6" - local s_cc_prefix="$7" + local include_fname + local s_wrapper + local s_period_sec + local s_cwd + local s_fname + local s_config + local s_ffb_id + local s_cc_prefix + include_fname=condor_config_startd_cron_include + s_wrapper="$1" + s_period_sec="${2}s" + s_cwd="$3" + s_fname="$4" + s_config="$5" + s_ffb_id="$6" + s_cc_prefix="$7" if [ ${add_startd_cron_counter} -eq 0 ]; then # Make sure that no undesired file is there when called for first cron rm -f ${include_fname} fi let add_startd_cron_counter=add_startd_cron_counter+1 - local name_prefix=GLIDEIN_PS_ - local s_name="${name_prefix}${add_startd_cron_counter}" + local name_prefix + name_prefix=GLIDEIN_PS_ + local s_name + s_name="${name_prefix}${add_startd_cron_counter}" # Append the following to the startd configuration # Instead of Periodic and Kill wait for completion: diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index fe08a3a0a9..1a3db7dfa2 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -37,15 +37,13 @@ print_header_line(){ # 2: short version of the final results # 3: long version of the final results # Global: -# exit_code -# final_result_simple -# final_result_long -# glidein_end_time # total_time print_tail() { + local final_result_simple, final_result_long, exit_code exit_code=$1 final_result_simple="$2" final_result_long="$3" + local glidein_end_time glidein_end_time=$(date +%s) let total_time=${glidein_end_time}-${startup_time} print_header_line "Glidein ending $(date) (${glidein_end_time}) with code ${exit_code} after ${total_time}" diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index 4ebda12838..9875d14d84 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -9,7 +9,8 @@ # 2: param_list (quoted string w/ spaces) params_get_simple() { [[ ${2} = *\ ${1}\ * ]] || return - local retval="${2##*\ ${1}\ }" + local retval + retval="${2##*\ ${1}\ }" echo "${retval%%\ *}" } diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh index 8517aaccc5..70802d4a29 100644 --- a/creation/web_base/utils_tarballs.sh +++ b/creation/web_base/utils_tarballs.sh @@ -32,7 +32,8 @@ list_data() { extract_all_data() { local -a files # change separator to split the output file list from 'tar tz' command - local IFS_OLD="${IFS}" + local IFS_OLD + IFS_OLD="${IFS}" IFS=$'\n' files=($(list_data)) for f in "${files[@]}"; do diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index b01254734e..32117afc4a 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -4,38 +4,22 @@ # 1: result # Global: # result -# glidein_end_time -construct_xml() { - #TODO(F): Check - # OSG_test_result=" - # - # - # %s - # - # - # %s %s - # %s - # %s - # - # %s - # " - #printf "$OSG_test_result" "${start_dir}" "$0" "${global_args}" "$(date --date=@\"${startup_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "$result" - #create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd ${start_dir} } test { cmd $0 ${global_args} tStart $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) tEnd $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)} } - ##create_xml OSG --id 0 { operatingenvironment { env --name cwd 0 } test { cmd 0 0 tStart 0 tEnd 0 } 0 } +construct_xml() { + result="$1" + local glidein_end_time + glidein_end_time="$(date +%s)" echo " ${start_dir} - $0 ${global_args} + $0 ${GLOBAL_ARGS} $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z) ${result} " - result="$1" - glidein_end_time="$(date +%s)" } ################################ @@ -67,31 +51,18 @@ extract_parent_fname(){ # 1: exit code # Global: # exitcode -# glidein_end_time # last_result # last_script_name # last_script_reason # my_reason -extract_parent_xml_detail() { - # fail_result=" - # %s - # %s - # - # %s" - # correct_result_no_details=" - # %s - # " - # correct_result=" - # %s - # - # %s" +extract_parent_xml_detail() { exitcode=$1 + local glidein_end_time glidein_end_time="$(date +%s)" if [ -s otrx_output.xml ]; then # file exists and is not 0 size last_result="$(cat otrx_output.xml)" if [ "${exitcode}" -eq 0 ]; then - #printf "$correct_result" "OK" echo " " echo " OK" #propagate metrics as well @@ -101,7 +72,6 @@ extract_parent_xml_detail() { last_script_name=$(echo "${last_result}" |awk '//{fr=0;}{if (fr==1) print $0}//{fr=1;}') my_reason=" Validation failed in ${last_script_name}. ${last_script_reason}" "${last_script_name}" - #printf "$fail_result" "ERROR" "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "${my_reason}" echo " " echo " ERROR ${last_script_name}" @@ -114,10 +84,8 @@ extract_parent_xml_detail() { # create a minimal XML file, else echo " " if [ "${exitcode}" -eq 0 ]; then - #printf "$correct_result" "OK" "No detail. Could not find source XML file." echo " OK" else - #printf "$fail_result" "ERROR" "No detail. Could not find source XML file." "$(date --date=@\"${glidein_end_time}\" +%Y-%m-%dT%H:%M:%S%:z)" "Unknown" "No detail. Could not find source XML file." echo " ERROR" echo " Unknown" fi @@ -132,26 +100,18 @@ extract_parent_xml_detail() { # Function used to convert base xml to simple xml # Arguments: # 1: final result -# Global: -# final_result basexml2simplexml() { + local final_result final_result="$1" - #env=" %s" # augment with node info echo "${final_result}" | awk 'BEGIN{fr=1;}{if (fr==1) print $0}//{fr=0;}' - #printf "$env" "client_name" "${client_name}" - #printf "$env" "client_group" "${client_group}" - #printf "$env" "user" "$(id -un)" - #printf "$env" "arch" "$(uname -m)" echo " ${client_name}" echo " ${client_group}" echo " $(id -un)" echo " $(uname -m)" if [ -e '/etc/redhat-release' ]; then - #printf "$env" "os" "$(cat /etc/redhat-release)" echo " $(cat /etc/redhat-release)" fi - #printf "$env" "hostname" "$(uname -n)" echo " $(uname -n)" echo "${final_result}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=1;}' } @@ -162,23 +122,15 @@ basexml2simplexml() { # 1: simple final result # 2: global result # Global: -# final_result_simple -# global_result # content simplexml2longxml() { + local final_result_simple, global_result final_result_simple="$1" global_result="$2" - #env=" %s" - #subtestlist = " - # - # %s - # ' - # " echo "${final_result_simple}" | awk 'BEGIN{fr=1;}{if (fr==1) print $0}/' echo ' ' echo "${global_result}" | awk '{print " " $0}' @@ -186,13 +138,6 @@ simplexml2longxml() { echo ' ' fi echo "${final_result_simple}" | awk 'BEGIN{fr=0;}{if (fr==1) print $0}//{fr=0;}' - #printf "$env" "glidein_factory" "${glidein_factory}" - #printf "$env" "glidein_name" "${glidein_name}" - #printf "$env" "glidein_entry" "${glidein_entry}" - #printf "$env" "condorg_cluster" "${condorg_cluster}" - #printf "$env" "condorg_subcluster" "${condorg_subcluster}" - #printf "$env" "glidein_credential_id" "${glidein_cred_id}" - #printf "$env" "condorg_schedd" "${condorg_schedd}" echo " ${glidein_factory}" echo " ${glidein_name}" echo " ${glidein_entry}" @@ -234,7 +179,7 @@ create_xml(){ xml+="" fi shift 1;; - operatingenvironment) xml += "" + operatingenvironment) xml+="" if [ $2 == "{" ]; then endxml=""+$endxml shift 1 From fdb4a3555095ab7627678c2918ccbd69a54730b9 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 10 Aug 2022 15:58:20 -0500 Subject: [PATCH 010/196] Marked more global variables as local --- creation/web_base/glidein_cleanup.sh | 6 +- creation/web_base/glidein_startup.sh | 4 +- creation/web_base/utils_filesystem.sh | 3 +- creation/web_base/utils_io.sh | 1 + creation/web_base/utils_params.sh | 56 ++++----- creation/web_base/utils_xml.sh | 167 +++++++++++++++++--------- 6 files changed, 146 insertions(+), 91 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 0c254ddc37..7a5de4c157 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -41,7 +41,7 @@ glidien_cleanup() { # warnings to stderr # tail to stdout early_glidein_failure() { - local error_msg, glidein_end_time, result, final_result, final_result_simple, final_result_long + local error_msg glidein_end_time result final_result final_result_simple final_result_long error_msg="$1" log_warn "${error_msg}" sleep "${sleep_time}" @@ -67,8 +67,8 @@ early_glidein_failure() { # Arguments: # 1: exit code glidein_exit() { - local exit_code, final_result, final_result_simple, final_result_long, global_result, ge_last_script_name, result, report_failed - local factory_report_failes, factory_collector, do_report, dlf, condor_vars_file, main_work_dir + local exit_code final_result final_result_simple final_result_long global_result ge_last_script_name result report_failed + local factory_report_failes factory_collector do_report dlf condor_vars_file main_work_dir exit_code=$1 # Removed lines about $lock_file (lock file for whole machine) not present elsewhere gwms_process_scripts "$GWMS_DIR" cleanup "${glidein_config}" diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 8d1e315893..59ec2af452 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -71,12 +71,11 @@ copy_all() { # 1: number of glideins # Global: # GWMS_MULTIGLIDEIN_CHILDS -# g_dir # Important Variables: # GLIDEIN_MULTIGLIDEIN_LAUNCHALL - if set in attrs, command to start all Glideins at once (multirestart 0) # GLIDEIN_MULTIGLIDEIN_LAUNCHER - if set in attrs, command to start the individual Glideins do_start_all() { - local num_glideins initial_dir multiglidein_launchall multiglidein_launcher + local num_glideins initial_dir multiglidein_launchall multiglidein_launcher g_dir num_glideins=$1 initial_dir="$(pwd)" multiglidein_launchall=$(params_decode "$(params_get_simple GLIDEIN_MULTIGLIDEIN_LAUNCHALL "${params}")") @@ -107,6 +106,7 @@ do_start_all() { fi } +# TODO (HERE) ################################ # Function used to spawn multiple glideins and wait, if needed diff --git a/creation/web_base/utils_filesystem.sh b/creation/web_base/utils_filesystem.sh index 3e15a9195c..62a1c35cde 100644 --- a/creation/web_base/utils_filesystem.sh +++ b/creation/web_base/utils_filesystem.sh @@ -2,8 +2,6 @@ # Function used to automatically determine and setup work directories # Globals: # targets -# disk_required -# free # work_dir # Returns: # 1 in case you are not allowed to write @@ -19,6 +17,7 @@ automatic_work_dir() { ) unset TMPDIR + local disk_required free # 1 kB disk_required=1000000 diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index 1a3db7dfa2..a3a910427f 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -20,6 +20,7 @@ log_debug() { # 1: content of the header line # 2 (optional): 2 if needs to write to stderr print_header_line(){ + local content if [ $# -eq 1 ]; then content=$1 echo "=== ${content} ===" diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index 9875d14d84..5eaf051207 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -67,34 +67,34 @@ params2file() { # TODO: Use params_decode. For 3.4.8, not to introduce many changes now. Use params_converter # Note: using $() we escape blackslash with \\ like above. Using backticks would require \\\ pfval=$(echo "$2" | sed \ - -e 's/\.nbsp,/ /g' \ - -e 's/\.semicolon,/;/g' \ - -e 's/\.colon,/:/g' \ - -e 's/\.tilde,/~/g' \ - -e 's/\.not,/!/g' \ - -e 's/\.question,/?/g' \ - -e 's/\.star,/*/g' \ - -e 's/\.dollar,/$/g' \ - -e 's/\.comment,/#/g' \ - -e 's/\.sclose,/]/g' \ - -e 's/\.sopen,/[/g' \ - -e 's/\.gclose,/}/g' \ - -e 's/\.gopen,/{/g' \ - -e 's/\.close,/)/g' \ - -e 's/\.open,/(/g' \ - -e 's/\.gt,/>/g' \ - -e 's/\.lt,//g' \ + -e 's/\.lt,//{fr=1;}' } +################################ +# Function used as support to add tabs +# Global: +# xml +add_tabs(){ + for (( c=1; c<=tabs; c++ )) + do + xml+="\t" + done +} + ################################ # Function used to create an xml file structure # Arguments: @@ -159,71 +158,127 @@ simplexml2longxml() { # xml create_xml(){ xml="" - endxml="" + end_xml="" + declare -i tabs=0; until [ $# -lt 1 ] do + xml+="\n" case "$1" in - OSG) xml+="" - if [ $3 == "{" ]; then - endxml=""+$endxml + OSG) + add_tabs + xml+="" + if [ $2 == "{" ]; then + tabs+=1 + end_xml=""$end_xml shift 1 else xml+="" - fi - shift 2;; - OSGShort) xml+="" + fi;; + OSGShort) + add_tabs + xml+="" if [ $2 == "{" ]; then - endxml=""+$endxml + tabs+=1 + end_xml=""$end_xml shift 1 else xml+="" - fi - shift 1;; - operatingenvironment) xml+="" + fi;; + operatingenvironment) + add_tabs + xml+="" if [ $2 == "{" ]; then - endxml=""+$endxml + tabs+=1 + end_xml=""$end_xml shift 1 else xml+="" - fi + fi;; + env) + add_tabs + xml+="$3" - shift 3;; - test) xml+="" + test) + add_tabs + xml+="" if [ $2 == "{" ]; then - endxml=""+$endxml + tabs+=1 + end_xml=""$end_xml shift 1 else xml+="" - fi + fi;; + cmd) + add_tabs + xml+="$2" + shift 1;; + tStart) + add_tabs + xml+="$2" shift 1;; - cmd) xml+="$2" - shift 2;; - tStart) xml+="$2" - shift 2;; - tEnd) xml+="$2" - shift 2;; - result) xml+="" + tEnd) + add_tabs + xml+="$2" + shift 1;; + result) + add_tabs + xml+="" if [ $2 == "{" ]; then - endxml=""+$endxml + tabs+=1 + end_xml=""$end_xml shift 1 else xml+="" - fi - shift 1;; - status) xml+="$2" - shift 2;; - metric) xml+="$4" - shift 4;; - detail) xml+="$2" - shift 2;; - "}") output=$(echo $endxml | cut -d'>' -f 1 | awk '{print $1">"}') - xml+=$output - endxml=$(echo $endxml | cut -d'<' -f 3 | awk '{print "<"$1}') + fi;; + status) add_tabs + xml+="$2" + shift 1;; + metric) add_tabs + xml+="' -f 1 | awk '{print $1">"}') + tabs=tabs-1 + add_tabs + xml+=$output + end_xml=${end_xml#"$output"};; + *) add_tabs + xml+=$1;; esac + shift 1 done - echo "$xml" + echo -e "$xml" + # TODO(F) + #return xml } + From 11bc51336ddad35ab3c659b013699644baf22247 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 10 Aug 2022 16:12:25 -0500 Subject: [PATCH 011/196] Changed debug option --- creation/web_base/glidein_startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 59ec2af452..81ef54cd56 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1,4 +1,4 @@ -#!/bin/bash -xv +#!/bin/bash -x #*******************************************************************# # glidein_startup.sh # # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # From 44a88aa67227d3f08b4571cadc1ad01acf0a3d8a Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 10 Aug 2022 16:31:05 -0500 Subject: [PATCH 012/196] Modified print_header error handling --- creation/web_base/utils_io.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index a3a910427f..f69f3e19ae 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -112,7 +112,14 @@ usage() { print_header(){ startup_time="$(date +%s)" echo "Starting glidein_startup.sh at $(date) (${startup_time})" - echo "script_checksum = '$(md5wrapper "$0")'" + local md5wrapped + md5wrapped = $(md5wrapper "$0") + retVal=$? + if [ $retVal -ne 0 ]; then + echo "Error on the md5wrapper" + glidein_exit 1 #TODO(F): o solo exit? + fi + echo "script_checksum = '${md5wrapped}'" echo "debug_mode = '${operation_mode}'" echo "condorg_cluster = '${condorg_cluster}'" echo "condorg_subcluster= '${condorg_subcluster}'" From 82165504959103f13afff592b284e230e3dc2162 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 10 Aug 2022 16:32:02 -0500 Subject: [PATCH 013/196] Modified print_header error handling --- creation/web_base/utils_io.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index f69f3e19ae..de3829a041 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -113,7 +113,7 @@ print_header(){ startup_time="$(date +%s)" echo "Starting glidein_startup.sh at $(date) (${startup_time})" local md5wrapped - md5wrapped = $(md5wrapper "$0") + md5wrapped=$(md5wrapper "$0") retVal=$? if [ $retVal -ne 0 ]; then echo "Error on the md5wrapper" From 917c2f4ca1bebb355266d335ec0c3964cc9c6232 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 10 Aug 2022 16:43:00 -0500 Subject: [PATCH 014/196] Handled exit status --- creation/web_base/glidein_startup.sh | 2 +- creation/web_base/utils_http.sh | 2 ++ creation/web_base/utils_io.sh | 3 +-- creation/web_base/utils_xml.sh | 2 -- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 81ef54cd56..df28b7dbe4 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -331,7 +331,7 @@ while [ $# -gt 0 ] -multiglidein) multi_glidein="$2";; -multirestart) multi_glidein_restart="$2";; -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; - *) (log_warn "Unknown option $1"; usage) 1>&2; exit 1 + *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 esac shift 2 done diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index b62cd84eec..2793a164fc 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -168,6 +168,7 @@ fetch_file() { # fft_config_out # Returns: # 0 in case of success +# otherwise it returns the exit code of fetch_file_base fetch_file_try() { fft_id="$1" fft_target_fname="$2" @@ -198,6 +199,7 @@ fetch_file_try() { fi fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" + return $? # returning the exit code of fetch_file_base } diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index de3829a041..1786e05630 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -100,7 +100,6 @@ usage() { echo " -multiglidein : spawn multiple () glideins (unless also multirestart is set)" echo " -multirestart : started as one of multiple glideins (glidein number )" echo " -param_* : user specified parameters" - exit 1 #TODO(F): why? } ################################ @@ -113,7 +112,7 @@ print_header(){ startup_time="$(date +%s)" echo "Starting glidein_startup.sh at $(date) (${startup_time})" local md5wrapped - md5wrapped=$(md5wrapper "$0") + md5wrapped="$(md5wrapper "$0")" retVal=$? if [ $retVal -ne 0 ]; then echo "Error on the md5wrapper" diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 73a8406b30..2baeeb723e 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -154,8 +154,6 @@ add_tabs(){ # Global: # xml # end_xml -# Returns: -# xml create_xml(){ xml="" end_xml="" From 35b4872f140774d72607ee134afb255f39185fd0 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 10:42:48 -0500 Subject: [PATCH 015/196] Added first xml change --- creation/web_base/glidein_cleanup.sh | 5 ---- creation/web_base/glidein_startup.sh | 2 ++ creation/web_base/utils_http.sh | 1 + creation/web_base/utils_xml.sh | 34 ++++++++++++++++------------ 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 7a5de4c157..bc8714819b 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -10,8 +10,6 @@ glide_local_tmp_dir_created=0 # It uses GLIDEIN_DEBUG_OPTIONS, start_dir, work_dir_created, work_dir, glide_local_tmp_dir_created, glide_local_tmp_dir # Arguments: # 1: exit code -# Outputs: -# warnings to stderr glidien_cleanup() { if ! cd "${start_dir}"; then log_warn "Cannot find ${start_dir} anymore, exiting but without cleanup" @@ -37,9 +35,6 @@ glidien_cleanup() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: error message -# Outputs: -# warnings to stderr -# tail to stdout early_glidein_failure() { local error_msg glidein_end_time result final_result final_result_simple final_result_long error_msg="$1" diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index df28b7dbe4..4f178e41cc 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -502,6 +502,8 @@ do fi done +#TODO(F): qui + ################################################### # get last_script, as it is used by the fetch_file gs_id_work_dir="$(get_work_dir main)" diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 2793a164fc..c06976bcbf 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -379,6 +379,7 @@ fetch_file_base() { END=$(date +%s) "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_outname} glidein_config" "${START}" "${END}" #generating test result document "${main_dir}"/error_augment.sh -concat + # TODO(F): qui if [ ${ret} -ne 0 ]; then echo "=== Validation error in ${ffb_outname} ===" 1>&2 log_warn "Error running '${ffb_outname}'" diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 2baeeb723e..c463b1f8f1 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -7,18 +7,23 @@ construct_xml() { result="$1" local glidein_end_time glidein_end_time="$(date +%s)" - echo " - - - ${start_dir} - - - $0 ${GLOBAL_ARGS} - $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) - $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z) - - ${result} - " + cmd="$0 ${GLOBAL_ARGS}" + tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" + tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" + create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd ${start_dir} } test { cmd ${cmd} tStart ${tStart} tEnd ${tEnd} } ${result} } + echo -e $result + #echo " + # + # + # ${start_dir} + # + # + # $0 ${GLOBAL_ARGS} + # $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) + # $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z) + # + #${result} + #" } ################################ @@ -154,6 +159,7 @@ add_tabs(){ # Global: # xml # end_xml +# result create_xml(){ xml="" end_xml="" @@ -275,8 +281,8 @@ create_xml(){ esac shift 1 done - echo -e "$xml" - # TODO(F) + #echo -e "$xml" + result=$xml #return xml } From 9f5461c1a05704844a31070701737dcf477b8dc4 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 10:45:49 -0500 Subject: [PATCH 016/196] Added first xml change --- creation/web_base/utils_xml.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index c463b1f8f1..55c07dee91 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -10,7 +10,7 @@ construct_xml() { cmd="$0 ${GLOBAL_ARGS}" tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" - create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd ${start_dir} } test { cmd ${cmd} tStart ${tStart} tEnd ${tEnd} } ${result} } + create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd "${start_dir}" } test { cmd "${cmd}" tStart "${tStart}" tEnd "${tEnd}" } "${result}" } echo -e $result #echo " # From fe68b84303ca20c21ffb98dab0d2aba4e07dcac7 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 10:49:00 -0500 Subject: [PATCH 017/196] Added first xml change --- creation/web_base/utils_xml.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 55c07dee91..df26d24f73 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -10,7 +10,7 @@ construct_xml() { cmd="$0 ${GLOBAL_ARGS}" tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" - create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd "${start_dir}" } test { cmd "${cmd}" tStart "${tStart}" tEnd "${tEnd}" } "${result}" } + create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd "${start_dir}" } test { cmd '${cmd}' tStart "${tStart}" tEnd "${tEnd}" } "${result}" } echo -e $result #echo " # From f81636681c5e84e3d54654b600962964c93bd170 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 10:50:58 -0500 Subject: [PATCH 018/196] Added first xml change --- creation/web_base/utils_xml.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index df26d24f73..b002fbb8e2 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -7,10 +7,10 @@ construct_xml() { result="$1" local glidein_end_time glidein_end_time="$(date +%s)" - cmd="$0 ${GLOBAL_ARGS}" + cmd=$0 ${GLOBAL_ARGS} tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" - create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd "${start_dir}" } test { cmd '${cmd}' tStart "${tStart}" tEnd "${tEnd}" } "${result}" } + create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd "${start_dir}" } test { cmd "${cmd}" tStart "${tStart}" tEnd "${tEnd}" } "${result}" } echo -e $result #echo " # From 1e4af33c3fdd9494f6756dc4679747667d55f1c4 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 10:52:25 -0500 Subject: [PATCH 019/196] Added first xml change --- creation/web_base/utils_xml.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index b002fbb8e2..55c07dee91 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -7,7 +7,7 @@ construct_xml() { result="$1" local glidein_end_time glidein_end_time="$(date +%s)" - cmd=$0 ${GLOBAL_ARGS} + cmd="$0 ${GLOBAL_ARGS}" tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd "${start_dir}" } test { cmd "${cmd}" tStart "${tStart}" tEnd "${tEnd}" } "${result}" } From 0a2cf42c0434830dd6e9188af560f3abd7ad7169 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 11:23:19 -0500 Subject: [PATCH 020/196] Added xml changes --- creation/web_base/utils_xml.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 55c07dee91..a379da1fb4 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -11,7 +11,7 @@ construct_xml() { tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd "${start_dir}" } test { cmd "${cmd}" tStart "${tStart}" tEnd "${tEnd}" } "${result}" } - echo -e $result + #echo -e $result #echo " # # @@ -58,9 +58,12 @@ extract_parent_xml_detail() { # file exists and is not 0 size last_result="$(cat otrx_output.xml)" if [ "${exitcode}" -eq 0 ]; then + echo "my version" + create_xml result { status OK } echo " " echo " OK" #propagate metrics as well + echo "previous version" echo "${last_result}" | grep '" else @@ -148,7 +151,7 @@ simplexml2longxml() { add_tabs(){ for (( c=1; c<=tabs; c++ )) do - xml+="\t" + xml+=" " done } @@ -157,13 +160,14 @@ add_tabs(){ # Arguments: # @: tags, options, values # Global: +# spaces # xml # end_xml # result create_xml(){ xml="" end_xml="" - declare -i tabs=0; + declare -i spaces=0; until [ $# -lt 1 ] do xml+="\n" @@ -276,12 +280,11 @@ create_xml(){ add_tabs xml+=$output end_xml=${end_xml#"$output"};; - *) add_tabs - xml+=$1;; + *) xml+=$1;; esac shift 1 done - #echo -e "$xml" + echo -e "$xml" result=$xml #return xml } From 566df04b09dc8564f750f8cb5b410a538a0075cf Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 11:29:42 -0500 Subject: [PATCH 021/196] Added xml changes --- creation/web_base/glidein_cleanup.sh | 13 ++++++++----- creation/web_base/utils_xml.sh | 3 --- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index bc8714819b..c792a9148e 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -42,11 +42,14 @@ early_glidein_failure() { sleep "${sleep_time}" # wait a bit in case of error, to reduce lost glideins glidein_end_time="$(date +%s)" - result=" WN_RESOURCE - ERROR - - ${error_msg} - " + ts="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" + create_xml metric --name failure --ts "$ts" --uri local WN_RESOURCE status ERROR detail ${error_msg} + #result + #result=" WN_RESOURCE + # ERROR + # + # ${error_msg} + # " final_result="$(construct_xml "${result}")" final_result_simple="$(basexml2simplexml "${final_result}")" # have no global section diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index a379da1fb4..a13d61d0be 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -58,12 +58,9 @@ extract_parent_xml_detail() { # file exists and is not 0 size last_result="$(cat otrx_output.xml)" if [ "${exitcode}" -eq 0 ]; then - echo "my version" - create_xml result { status OK } echo " " echo " OK" #propagate metrics as well - echo "previous version" echo "${last_result}" | grep '" else From e669a0f4fc94ae02af1a3f34d8872b23e0eaca23 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 11:57:01 -0500 Subject: [PATCH 022/196] Added spaces support to create_xml method --- creation/web_base/glidein_cleanup.sh | 4 ++- creation/web_base/utils_xml.sh | 53 ++++++++++++++++------------ 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index c792a9148e..5e9df84e44 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -43,13 +43,15 @@ early_glidein_failure() { # wait a bit in case of error, to reduce lost glideins glidein_end_time="$(date +%s)" ts="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" - create_xml metric --name failure --ts "$ts" --uri local WN_RESOURCE status ERROR detail ${error_msg} + create_xml -s 4 metric --name failure --ts "$ts" --uri local WN_RESOURCE status ERROR detail ${error_msg} #result #result=" WN_RESOURCE # ERROR # # ${error_msg} # " + echo "testing xml" + echo -e $result final_result="$(construct_xml "${result}")" final_result_simple="$(basexml2simplexml "${final_result}")" # have no global section diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index a13d61d0be..1153e43d6d 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -142,11 +142,11 @@ simplexml2longxml() { } ################################ -# Function used as support to add tabs +# Function used as support to add spaces # Global: # xml -add_tabs(){ - for (( c=1; c<=tabs; c++ )) +add_spaces(){ + for (( c=1; c<=spaces; c++ )) do xml+=" " done @@ -162,15 +162,22 @@ add_tabs(){ # end_xml # result create_xml(){ - xml="" + xml = "" end_xml="" declare -i spaces=0; + if [[ $1 == "-s" ]] + then + spaces+=$2 + shift 2 + else + xml="" + fi until [ $# -lt 1 ] do xml+="\n" case "$1" in OSG) - add_tabs + add_spaces xml+="" if [ $2 == "{" ]; then - tabs+=1 + spaces+=1 end_xml=""$end_xml shift 1 else xml+="" fi;; OSGShort) - add_tabs + add_spaces xml+="" if [ $2 == "{" ]; then - tabs+=1 + spaces+=1 end_xml=""$end_xml shift 1 else xml+="" fi;; operatingenvironment) - add_tabs + add_spaces xml+="" if [ $2 == "{" ]; then - tabs+=1 + spaces+=1 end_xml=""$end_xml shift 1 else xml+="" fi;; env) - add_tabs + add_spaces xml+="' -f 1 | awk '{print $1">"}') - tabs=tabs-1 - add_tabs + spaces=spaces-1 + add_spaces xml+=$output end_xml=${end_xml#"$output"};; *) xml+=$1;; From 5adba357ee38c6cef107a54db85fd3ed1b9c72c3 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 12:06:00 -0500 Subject: [PATCH 023/196] Substitution of two methods with the call to the create_xml method --- creation/web_base/utils_http.sh | 75 ++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index c06976bcbf..945cc753a9 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -257,25 +257,30 @@ fetch_file_base() { # cannot use error_*.sh helper functions # may not have been loaded yet have_dummy_otrx=1 - echo " - - - ${PWD} - - - Unknown - $(date +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - ERROR - Unknown - ${ffb_id} - - - An unknown error occurred. - -" > otrx_output.xml + date=$(date +%Y-%m-%dT%H:%M:%S%:z) + create_xml OSG --id fetch_file_base { operatingenvironment { env --name cwd "${PWD}" } test { cmd Unknown tStart $date tEnd $date} result { status ERROR metric --name failure --ts $date --uri local Unknown metric --name source_type --ts $date --uri local ${ffb_id} } detail "An unknown error occurred."} + echo "breakpoint" + echo -e $result > otrx_output.xml + +# echo " +# +# +# ${PWD} +# +# +# Unknown +# $(date +%Y-%m-%dT%H:%M:%S%:z) +# $(date +%Y-%m-%dT%H:%M:%S%:z) +# +# +# ERROR +# Unknown +# ${ffb_id} +# +# +# An unknown error occurred. +# +#" > otrx_output.xml user_agent="glidein/${glidein_entry}/${condorg_schedd}/${condorg_cluster}.${condorg_subcluster}/${client_name}" ffb_url="${ffb_repository}/${ffb_real_fname}" curl_version=$(curl --version | head -1 ) @@ -427,20 +432,24 @@ fetch_file_base() { if [ "${have_dummy_otrx}" -eq 1 ]; then # no one should really look at this file, but just to avoid confusion - echo " - - - ${PWD} - - - Unknown - $(date +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - OK - -" > otrx_output.xml + date=$(date +%Y-%m-%dT%H:%M:%S%:z) + create_xml OSG --id fetch_file_base { operatingenvironment { env --name cwd "${PWD}" } test { cmd Unknown tStart "$date" tEnd "$date"} result { status OK } } + echo "breakpoint" + echo -e $result > otrx_output.xml +# echo " +# +# +# ${PWD} +# +# + # Unknown + # $(date +%Y-%m-%dT%H:%M:%S%:z) + # $(date +%Y-%m-%dT%H:%M:%S%:z) + # + # + # OK + # +#" > otrx_output.xml fi return 0 From 5ae02efa8e45432de8556419192a3791650037aa Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 13:30:44 -0500 Subject: [PATCH 024/196] Substitution of two methods with the call to the create_xml method --- creation/web_base/utils_http.sh | 123 +++++++++++++++++++------------- 1 file changed, 75 insertions(+), 48 deletions(-) diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 945cc753a9..2447293115 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -258,8 +258,9 @@ fetch_file_base() { # may not have been loaded yet have_dummy_otrx=1 date=$(date +%Y-%m-%dT%H:%M:%S%:z) - create_xml OSG --id fetch_file_base { operatingenvironment { env --name cwd "${PWD}" } test { cmd Unknown tStart $date tEnd $date} result { status ERROR metric --name failure --ts $date --uri local Unknown metric --name source_type --ts $date --uri local ${ffb_id} } detail "An unknown error occurred."} + create_xml OSG --id fetch_file_base { operatingenvironment { env --name cwd "${PWD}" } test { cmd Unknown tStart $date tEnd $date} result { status ERROR metric --name failure --ts $date --uri local Unknown metric --name source_type --ts $date --uri local ${ffb_id} } detail "An unknown error occurred." } echo "breakpoint" + echo -e $result echo -e $result > otrx_output.xml # echo " @@ -435,6 +436,7 @@ fetch_file_base() { date=$(date +%Y-%m-%dT%H:%M:%S%:z) create_xml OSG --id fetch_file_base { operatingenvironment { env --name cwd "${PWD}" } test { cmd Unknown tStart "$date" tEnd "$date"} result { status OK } } echo "breakpoint" + echo -e $result echo -e $result > otrx_output.xml # echo " # @@ -508,30 +510,42 @@ perform_wget() { log_warn "${wget_cmd} failed. version:${wget_version} exit code ${wget_retval} stderr: ${wget_resp}" # cannot use error_*.sh helper functions # may not have been loaded yet, and wget fails often - # create_xml operatingenvironment { env cwd ${PWD} env uname $(uname -a) env release $(cat /etc/system-release) env wget_version ${wget_version} - echo " - - ${PWD} - $(uname -a) - $(cat /etc/system-release) - ${wget_version} - - - ${wget_cmd} - $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - ERROR - Network - ${ffb_url} - ${proxy_url} - ${ffb_id} - - - Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp} - -" > otrb_output.xml + tStart="$(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z)" + tEnd="$(date +%Y-%m-%dT%H:%M:%S%:z)" + xml="" + create_xml OSG --id perform_wget { operatingenvironment { env --name cwd "${PWD}" env --name uname "$(uname -a)" env --name release "$(cat /etc/system-release)" env --name wget_version "${wget_version}" } + xml+=$result + create_xml -s 1 test { cmd ${wget_cmd} tStart ${tStart} tEnd ${tEnd} } result { status ERROR metric --name failure ${tStart} --uri local Network metric --name URL --ts ${tStart} --uri local ${ffb_url} metric --name http_proxy --ts ${tStart} --uri local ${proxy_url} metric --name source_type --ts ${tStart} --uri local ${ffb_id} } } + xml+=$result + create_xml -s 1 detail "Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp}" + xml+=$result + xml+="" + xml > otrb_output.xml + echo "breakpoint" + echo -e $xml + # echo " + # + # ${PWD} + # $(uname -a) + # $(cat /etc/system-release) + # ${wget_version} + # + # + # ${wget_cmd} + # $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) + # $(date +%Y-%m-%dT%H:%M:%S%:z) + # + # + # ERROR + # Network + # ${ffb_url} + # ${proxy_url} + # ${ffb_id} + # + # + #Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp} + # +#" > otrb_output.xml log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." if [ -f otr_outlist.list ]; then @@ -597,29 +611,42 @@ perform_curl() { log_warn "${curl_cmd} failed. version:${curl_version} exit code ${curl_retval} stderr: ${curl_resp} " # cannot use error_*.sh helper functions # may not have been loaded yet, and wget fails often - echo " - - ${PWD} - $(uname -a) - $(cat /etc/system-release) - ${curl_version} - - - ${curl_cmd} - $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) - $(date +%Y-%m-%dT%H:%M:%S%:z) - - - ERROR - Network - ${ffb_url} - ${proxy_url} - ${ffb_id} - - - Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${curl_resp} - -" > otrb_output.xml + tStart="$(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z)" + tEnd="$(date +%Y-%m-%dT%H:%M:%S%:z)" + xml="" + create_xml OSG --id perform_curl { operatingenvironment { env --name cwd "${PWD}" env --name uname "$(uname -a)" env --name release "$(cat /etc/system-release)" env --name curl_version "${curl_version}" } + xml+=$result + create_xml -s 1 test { cmd ${curl_cmd} tStart ${tStart} tEnd ${tEnd} } result { status ERROR metric --name failure ${tStart} --uri local Network metric --name URL --ts ${tStart} --uri local ${ffb_url} metric --name http_proxy --ts ${tStart} --uri local ${proxy_url} metric --name source_type --ts ${tStart} --uri local ${ffb_id} } } + xml+=$result + create_xml -s 1 detail "Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${curl_resp}" + xml+=$result + xml+="" + xml > otrb_output.xml + echo "breakpoint" + echo -e $xml + # echo " +# +# ${PWD} +# $(uname -a) +# $(cat /etc/system-release) +# ${curl_version} +# +# +# ${curl_cmd} +# $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) +# $(date +%Y-%m-%dT%H:%M:%S%:z) +# +# +# ERROR +# Network +# ${ffb_url} +# ${proxy_url} +# ${ffb_id} +# +# +# Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${curl_resp} +# +#" > otrb_output.xml log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." if [ -f otr_outlist.list ]; then From db70155ba64d4ecf4a5283c352ad9cae3aadc3e1 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 14:05:35 -0500 Subject: [PATCH 025/196] Substitution of the methods of utils_http with the call to the create_xml method --- creation/web_base/utils_http.sh | 62 ++++++++++++++++++--------------- creation/web_base/utils_xml.sh | 41 ++++++++++++++-------- 2 files changed, 59 insertions(+), 44 deletions(-) diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 2447293115..818aaaefa1 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -258,10 +258,9 @@ fetch_file_base() { # may not have been loaded yet have_dummy_otrx=1 date=$(date +%Y-%m-%dT%H:%M:%S%:z) - create_xml OSG --id fetch_file_base { operatingenvironment { env --name cwd "${PWD}" } test { cmd Unknown tStart $date tEnd $date} result { status ERROR metric --name failure --ts $date --uri local Unknown metric --name source_type --ts $date --uri local ${ffb_id} } detail "An unknown error occurred." } + create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { s ERROR m --name failure --ts "${date}" --uri local Unknown m --name source_type --ts "${date}" --uri local "${ffb_id}" } d "An unknown error occurred." } echo "breakpoint" - echo -e $result - echo -e $result > otrx_output.xml + echo -e "$result" > otrx_output.xml # echo " # @@ -433,11 +432,11 @@ fetch_file_base() { if [ "${have_dummy_otrx}" -eq 1 ]; then # no one should really look at this file, but just to avoid confusion - date=$(date +%Y-%m-%dT%H:%M:%S%:z) - create_xml OSG --id fetch_file_base { operatingenvironment { env --name cwd "${PWD}" } test { cmd Unknown tStart "$date" tEnd "$date"} result { status OK } } + date="$(date +%Y-%m-%dT%H:%M:%S%:z)" + create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { status OK } } echo "breakpoint" - echo -e $result - echo -e $result > otrx_output.xml + echo -e "$result" + echo -e "$result" > otrx_output.xml # echo " # # @@ -512,17 +511,19 @@ perform_wget() { # may not have been loaded yet, and wget fails often tStart="$(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date +%Y-%m-%dT%H:%M:%S%:z)" - xml="" - create_xml OSG --id perform_wget { operatingenvironment { env --name cwd "${PWD}" env --name uname "$(uname -a)" env --name release "$(cat /etc/system-release)" env --name wget_version "${wget_version}" } - xml+=$result - create_xml -s 1 test { cmd ${wget_cmd} tStart ${tStart} tEnd ${tEnd} } result { status ERROR metric --name failure ${tStart} --uri local Network metric --name URL --ts ${tStart} --uri local ${ffb_url} metric --name http_proxy --ts ${tStart} --uri local ${proxy_url} metric --name source_type --ts ${tStart} --uri local ${ffb_id} } } - xml+=$result - create_xml -s 1 detail "Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp}" - xml+=$result - xml+="" - xml > otrb_output.xml + xmlResult="" + create_xml OSG --id perform_wget { oe { e --name cwd "${PWD}" e --name uname "$(uname -a)" e --name release "$(cat /etc/system-release)" e --name wget_version "${wget_version}" } + xmlResult+=$result + create_xml -s 1 t { c "${wget_cmd}" tS "${tStart}" tE "${tEnd}" } r { s ERROR m --name failure --ts "${tStart}" --uri local Network m --name URL --ts "${tStart}" --uri local "${ffb_url}" m --name http_proxy --ts "${tStart}" --uri local "${proxy_url}" m --name source_type --ts "${tStart}" --uri local "${ffb_id}" } + xmlResult+=$result + create_xml -s 1 d "Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp}" + xmlResult+=$result + create_xml -t + xmlResult+=$result + xmlResult+="\n" echo "breakpoint" - echo -e $xml + echo -e "$xmlResult" + echo -e "$xmlResult" > otrb_output.xml # echo " # # ${PWD} @@ -613,17 +614,17 @@ perform_curl() { # may not have been loaded yet, and wget fails often tStart="$(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date +%Y-%m-%dT%H:%M:%S%:z)" - xml="" - create_xml OSG --id perform_curl { operatingenvironment { env --name cwd "${PWD}" env --name uname "$(uname -a)" env --name release "$(cat /etc/system-release)" env --name curl_version "${curl_version}" } - xml+=$result - create_xml -s 1 test { cmd ${curl_cmd} tStart ${tStart} tEnd ${tEnd} } result { status ERROR metric --name failure ${tStart} --uri local Network metric --name URL --ts ${tStart} --uri local ${ffb_url} metric --name http_proxy --ts ${tStart} --uri local ${proxy_url} metric --name source_type --ts ${tStart} --uri local ${ffb_id} } } - xml+=$result - create_xml -s 1 detail "Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${curl_resp}" - xml+=$result - xml+="" - xml > otrb_output.xml - echo "breakpoint" - echo -e $xml + xmlResult="" + create_xml OSG --id perform_curl { oe { e --name cwd "${PWD}" e --name uname "$(uname -a)" e --name release "$(cat /etc/system-release)" e --name curl_version "${curl_version}" } + xmlResult+=$result + create_xml -s 1 t { c "${curl_cmd}" tS "${tStart}" tE "${tEnd}" } r { s ERROR m --name failure --ts "${tStart}" --uri local Network m --name URL --ts "${tStart}" --uri local "${ffb_url}" m --name http_proxy --ts "${tStart}" --uri local "${proxy_url}" m --name source_type --ts "${tStart}" --uri local "${ffb_id}" } + xmlResult+=$result + create_xml -s 1 d "Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${curl_resp}" + xmlResult+=$result + create_xml -t + xmlResult+=$result + echo -e "$xmlResult" + echo -e "$xmlResult" > otrb_output.xml # echo " # # ${PWD} @@ -655,7 +656,10 @@ perform_curl() { touch otr_outlist.list fi cat otrb_output.xml >> otr_outlist.list - echo "" > otrx_output.xml + create_xml -h + echo $result > otrx_output.xml + echo "breakpoint" + echo -e otrx_output.xml cat otrb_output.xml >> otrx_output.xml rm -f otrb_output.xml chmod a-w otr_outlist.list diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 1153e43d6d..ed18c11f23 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -10,7 +10,7 @@ construct_xml() { cmd="$0 ${GLOBAL_ARGS}" tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" - create_xml OSG --id glidein_startup.sh { operatingenvironment { env --name cwd "${start_dir}" } test { cmd "${cmd}" tStart "${tStart}" tEnd "${tEnd}" } "${result}" } + create_xml OSG --id glidein_startup.sh { oe { e --name cwd "${start_dir}" } t { c "${cmd}" tS "${tStart}" tE "${tEnd}" } "${result}" } #echo -e $result #echo " # @@ -162,9 +162,19 @@ add_spaces(){ # end_xml # result create_xml(){ - xml = "" + xml="" end_xml="" declare -i spaces=0; + if [[ $1 == "-h" ]] + then + result="\n" + return 0 + fi + if [[ $1 == "-t" ]] + then + result="\n" + return 0 + fi if [[ $1 == "-s" ]] then spaces+=$2 @@ -176,7 +186,7 @@ create_xml(){ do xml+="\n" case "$1" in - OSG) + OSG|O) add_spaces xml+="' -f 1 | awk '{print $1">"}') @@ -287,8 +297,9 @@ create_xml(){ *) xml+=$1;; esac shift 1 + #echo $end_xml done - echo -e "$xml" + #echo -e "$xml" result=$xml #return xml } From 21e2aa3cf05e5191b18203f9c489f7602d420b55 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 14:13:09 -0500 Subject: [PATCH 026/196] Substitution of the methods of utils_http with the call to the create_xml method --- creation/web_base/utils_http.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 818aaaefa1..213b3a9a1d 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -259,7 +259,7 @@ fetch_file_base() { have_dummy_otrx=1 date=$(date +%Y-%m-%dT%H:%M:%S%:z) create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { s ERROR m --name failure --ts "${date}" --uri local Unknown m --name source_type --ts "${date}" --uri local "${ffb_id}" } d "An unknown error occurred." } - echo "breakpoint" + echo "breakpoint1" echo -e "$result" > otrx_output.xml # echo " @@ -434,7 +434,7 @@ fetch_file_base() { # no one should really look at this file, but just to avoid confusion date="$(date +%Y-%m-%dT%H:%M:%S%:z)" create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { status OK } } - echo "breakpoint" + echo "breakpoint2" echo -e "$result" echo -e "$result" > otrx_output.xml # echo " @@ -521,7 +521,7 @@ perform_wget() { create_xml -t xmlResult+=$result xmlResult+="\n" - echo "breakpoint" + echo "breakpoint3" echo -e "$xmlResult" echo -e "$xmlResult" > otrb_output.xml # echo " @@ -658,7 +658,7 @@ perform_curl() { cat otrb_output.xml >> otr_outlist.list create_xml -h echo $result > otrx_output.xml - echo "breakpoint" + echo "breakpoint4" echo -e otrx_output.xml cat otrb_output.xml >> otrx_output.xml rm -f otrb_output.xml From d1b8eadc8ca8b4c887624ed5b1ce03884c7314e9 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 14:14:12 -0500 Subject: [PATCH 027/196] Substitution of the methods of utils_http with the call to the create_xml method --- creation/web_base/utils_http.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 213b3a9a1d..2e1c7d6fde 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -260,6 +260,7 @@ fetch_file_base() { date=$(date +%Y-%m-%dT%H:%M:%S%:z) create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { s ERROR m --name failure --ts "${date}" --uri local Unknown m --name source_type --ts "${date}" --uri local "${ffb_id}" } d "An unknown error occurred." } echo "breakpoint1" + echo -e "$result" echo -e "$result" > otrx_output.xml # echo " @@ -659,6 +660,7 @@ perform_curl() { create_xml -h echo $result > otrx_output.xml echo "breakpoint4" + echo -e $result echo -e otrx_output.xml cat otrb_output.xml >> otrx_output.xml rm -f otrb_output.xml From 64f6c5af89db236d58a45ff5fafa5c9db36d0d99 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 14:20:15 -0500 Subject: [PATCH 028/196] Completed substituion of xml methods --- creation/web_base/utils_http.sh | 9 +++------ creation/web_base/utils_xml.sh | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 2e1c7d6fde..54aa67e6b6 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -520,8 +520,7 @@ perform_wget() { create_xml -s 1 d "Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp}" xmlResult+=$result create_xml -t - xmlResult+=$result - xmlResult+="\n" + xmlResult+=$result echo "breakpoint3" echo -e "$xmlResult" echo -e "$xmlResult" > otrb_output.xml @@ -556,7 +555,8 @@ perform_wget() { touch otr_outlist.list fi cat otrb_output.xml >> otr_outlist.list - echo "" > otrx_output.xml + create_xml -h + echo $result > otrx_output.xml cat otrb_output.xml >> otrx_output.xml rm -f otrb_output.xml chmod a-w otr_outlist.list @@ -659,9 +659,6 @@ perform_curl() { cat otrb_output.xml >> otr_outlist.list create_xml -h echo $result > otrx_output.xml - echo "breakpoint4" - echo -e $result - echo -e otrx_output.xml cat otrb_output.xml >> otrx_output.xml rm -f otrb_output.xml chmod a-w otr_outlist.list diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index ed18c11f23..647b40ae0c 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -167,7 +167,7 @@ create_xml(){ declare -i spaces=0; if [[ $1 == "-h" ]] then - result="\n" + result="" return 0 fi if [[ $1 == "-t" ]] From 880f7dd9ccf66b87f5571db8f47dd5ee80640293 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 14:21:42 -0500 Subject: [PATCH 029/196] Completed substituion of xml methods --- creation/web_base/utils_http.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_http.sh index 54aa67e6b6..779d38cf62 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_http.sh @@ -259,8 +259,6 @@ fetch_file_base() { have_dummy_otrx=1 date=$(date +%Y-%m-%dT%H:%M:%S%:z) create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { s ERROR m --name failure --ts "${date}" --uri local Unknown m --name source_type --ts "${date}" --uri local "${ffb_id}" } d "An unknown error occurred." } - echo "breakpoint1" - echo -e "$result" echo -e "$result" > otrx_output.xml # echo " @@ -435,8 +433,6 @@ fetch_file_base() { # no one should really look at this file, but just to avoid confusion date="$(date +%Y-%m-%dT%H:%M:%S%:z)" create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { status OK } } - echo "breakpoint2" - echo -e "$result" echo -e "$result" > otrx_output.xml # echo " # @@ -521,8 +517,6 @@ perform_wget() { xmlResult+=$result create_xml -t xmlResult+=$result - echo "breakpoint3" - echo -e "$xmlResult" echo -e "$xmlResult" > otrb_output.xml # echo " # @@ -624,7 +618,6 @@ perform_curl() { xmlResult+=$result create_xml -t xmlResult+=$result - echo -e "$xmlResult" echo -e "$xmlResult" > otrb_output.xml # echo " # From 49845aac927af54baf096f1f1372bbcd020ace06 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 14:46:43 -0500 Subject: [PATCH 030/196] Methods divided among glidein startup specific methods and general methods --- creation/web_base/glidein_startup.sh | 52 +-- ...s_filesystem.sh => utils_gs_filesystem.sh} | 0 .../{utils_http.sh => utils_gs_http.sh} | 80 ----- creation/web_base/utils_gs_io.sh | 302 ++++++++++++++++++ creation/web_base/utils_gs_signals.sh | 13 + creation/web_base/utils_gs_tarballs.sh | 36 +++ creation/web_base/utils_io.sh | 145 +-------- creation/web_base/utils_params.sh | 120 ------- creation/web_base/utils_signals.sh | 18 +- creation/web_base/utils_tarballs.sh | 37 --- creation/web_base/utils_xml.sh | 14 +- 11 files changed, 369 insertions(+), 448 deletions(-) rename creation/web_base/{utils_filesystem.sh => utils_gs_filesystem.sh} (100%) rename creation/web_base/{utils_http.sh => utils_gs_http.sh} (86%) create mode 100644 creation/web_base/utils_gs_io.sh create mode 100644 creation/web_base/utils_gs_signals.sh create mode 100644 creation/web_base/utils_gs_tarballs.sh diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 4f178e41cc..33564b668d 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -36,14 +36,18 @@ GWMS_MULTIUSER_GLIDEIN= GWMS_LOGSERVER_ADDRESS='https://fermicloud152.fnal.gov/log' GWMS_MULTIGLIDEIN_CHILDS= -source utils_tarballs.sh -source utils_signals.sh +# Include all source scripts +source utils_gs_filesystem.sh +source utils_gs_http.sh +source utils_gs_io.sh +source utils_gs_signals.sh +source utils_gs_tarballs.sh source utils_io.sh -source utils_xml.sh -source utils_filesystem.sh source utils_params.sh +source utils_signals.sh +source utils_tarballs.sh +source utils_xml.sh source utils_crypto.sh -source utils_http.sh source glidein_cleanup.sh export LANG=C @@ -299,42 +303,8 @@ create_glidein_config(){ #done #shift $(expr $OPTIND- 1) -params="" -while [ $# -gt 0 ] - do case "$1" in - -factory) glidein_factory="$2";; - -name) glidein_name="$2";; - -entry) glidein_entry="$2";; - -clientname) client_name="$2";; - -clientgroup) client_group="$2";; - -web) repository_url="$2";; - -proxy) proxy_url="$2";; - -dir) work_dir="$2";; - -sign) sign_id="$2";; - -signtype) sign_type="$2";; - -signentry) sign_entry_id="$2";; - -cluster) condorg_cluster="$2";; - -subcluster) condorg_subcluster="$2";; - -submitcredid) glidein_cred_id="$2";; - -schedd) condorg_schedd="$2";; - -descript) descript_file="$2";; - -descriptentry) descript_entry_file="$2";; - -clientweb) client_repository_url="$2";; - -clientwebgroup) client_repository_group_url="$2";; - -clientsign) client_sign_id="$2";; - -clientsigntype) client_sign_type="$2";; - -clientsigngroup) client_sign_group_id="$2";; - -clientdescript) client_descript_file="$2";; - -clientdescriptgroup) client_descript_group_file="$2";; - -slotslayout) slots_layout="$2";; - -v) operation_mode="$2";; - -multiglidein) multi_glidein="$2";; - -multirestart) multi_glidein_restart="$2";; - -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; - *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 - esac - shift 2 -done +################################ +parse_options ################################ # Code block used to set the slots_layout diff --git a/creation/web_base/utils_filesystem.sh b/creation/web_base/utils_gs_filesystem.sh similarity index 100% rename from creation/web_base/utils_filesystem.sh rename to creation/web_base/utils_gs_filesystem.sh diff --git a/creation/web_base/utils_http.sh b/creation/web_base/utils_gs_http.sh similarity index 86% rename from creation/web_base/utils_http.sh rename to creation/web_base/utils_gs_http.sh index 779d38cf62..015e0588a4 100644 --- a/creation/web_base/utils_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -260,26 +260,6 @@ fetch_file_base() { date=$(date +%Y-%m-%dT%H:%M:%S%:z) create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { s ERROR m --name failure --ts "${date}" --uri local Unknown m --name source_type --ts "${date}" --uri local "${ffb_id}" } d "An unknown error occurred." } echo -e "$result" > otrx_output.xml - -# echo " -# -# -# ${PWD} -# -# -# Unknown -# $(date +%Y-%m-%dT%H:%M:%S%:z) -# $(date +%Y-%m-%dT%H:%M:%S%:z) -# -# -# ERROR -# Unknown -# ${ffb_id} -# -# -# An unknown error occurred. -# -#" > otrx_output.xml user_agent="glidein/${glidein_entry}/${condorg_schedd}/${condorg_cluster}.${condorg_subcluster}/${client_name}" ffb_url="${ffb_repository}/${ffb_real_fname}" curl_version=$(curl --version | head -1 ) @@ -434,20 +414,6 @@ fetch_file_base() { date="$(date +%Y-%m-%dT%H:%M:%S%:z)" create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { status OK } } echo -e "$result" > otrx_output.xml -# echo " -# -# -# ${PWD} -# -# - # Unknown - # $(date +%Y-%m-%dT%H:%M:%S%:z) - # $(date +%Y-%m-%dT%H:%M:%S%:z) - # - # - # OK - # -#" > otrx_output.xml fi return 0 @@ -518,29 +484,6 @@ perform_wget() { create_xml -t xmlResult+=$result echo -e "$xmlResult" > otrb_output.xml - # echo " - # - # ${PWD} - # $(uname -a) - # $(cat /etc/system-release) - # ${wget_version} - # - # - # ${wget_cmd} - # $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) - # $(date +%Y-%m-%dT%H:%M:%S%:z) - # - # - # ERROR - # Network - # ${ffb_url} - # ${proxy_url} - # ${ffb_id} - # - # - #Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp} - # -#" > otrb_output.xml log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." if [ -f otr_outlist.list ]; then @@ -619,29 +562,6 @@ perform_curl() { create_xml -t xmlResult+=$result echo -e "$xmlResult" > otrb_output.xml - # echo " -# -# ${PWD} -# $(uname -a) -# $(cat /etc/system-release) -# ${curl_version} -# -# -# ${curl_cmd} -# $(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z) -# $(date +%Y-%m-%dT%H:%M:%S%:z) -# -# -# ERROR -# Network -# ${ffb_url} -# ${proxy_url} -# ${ffb_id} -# -# -# Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${curl_resp} -# -#" > otrb_output.xml log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." if [ -f otr_outlist.list ]; then diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh new file mode 100644 index 0000000000..05f371a173 --- /dev/null +++ b/creation/web_base/utils_gs_io.sh @@ -0,0 +1,302 @@ +################################ +# Function used to print the tail with the final results of the glideins +# Arguments: +# 1: exit code +# 2: short version of the final results +# 3: long version of the final results +# Global: +# total_time +print_tail() { + local final_result_simple, final_result_long, exit_code + exit_code=$1 + final_result_simple="$2" + final_result_long="$3" + local glidein_end_time + glidein_end_time=$(date +%s) + let total_time=${glidein_end_time}-${startup_time} + print_header_line "Glidein ending $(date) (${glidein_end_time}) with code ${exit_code} after ${total_time}" + echo "" + print_header_line "XML description of glidein activity" + echo "${final_result_simple}" | grep -v "" + print_header_line "End XML description of glidein activity" + + echo "" 1>&2 + print_header_line "Encoded XML description of glidein activity" 2 + echo "${final_result_long}" | gzip --stdout - | b64uuencode 1>&2 + print_header_line "End encoded XML description of glidein activity" 2 +} + +################################ +# Function used to have information about the usage of the glidein_startup.sh script +# Returns: +# 1 in any case +usage() { + echo "Usage: glidein_startup.sh " + echo "where is:" + echo " -factory : name of this factory" + echo " -name : name of this glidein" + echo " -entry : name of this glidein entry" + echo " -clientname : name of the requesting client" + echo " -clientgroup : group name of the requesting client" + echo " -web : base URL from where to fetch" + echo " -proxy : URL of the local proxy" + echo " -dir : directory ID (supports ., Condor, CONDOR, OSG, TMPDIR, AUTO)" + echo " -sign : signature of the signature file" + echo " -signtype : type of signature (only sha1 supported for now)" + echo " -signentry : signature of the entry signature file" + echo " -cluster : condorG ClusterId" + echo " -subcluster : condorG ProcId" + echo " -submitcredid : Credential ID of this condorG job" + echo " -schedd : condorG Schedd Name" + echo " -descript : description file name" + echo " -descriptentry : description file name for entry" + echo " -clientweb : base URL from where to fetch client files" + echo " -clientwebgroup : base URL from where to fetch client group files" + echo " -clientsign : signature of the client signature file" + echo " -clientsigntype : type of client signature (only sha1 supported for now)" + echo " -clientsigngroup : signature of the client group signature file" + echo " -clientdescript : client description file name" + echo " -clientdescriptgroup : client description file name for group" + echo " -slotslayout : how Condor will set up slots (fixed, partitionable)" + echo " -v : operation mode (std, nodebug, fast, check supported)" + echo " -multiglidein : spawn multiple () glideins (unless also multirestart is set)" + echo " -multirestart : started as one of multiple glideins (glidein number )" + echo " -param_* : user specified parameters" +} + +################################ +# Function used to print initial information header +# Parameters: +# @: shell parameters +# Global: +# startup_time +print_header(){ + startup_time="$(date +%s)" + echo "Starting glidein_startup.sh at $(date) (${startup_time})" + local md5wrapped + md5wrapped="$(md5wrapper "$0")" + retVal=$? + if [ $retVal -ne 0 ]; then + echo "Error on the md5wrapper" + glidein_exit 1 #TODO(F): o solo exit? + fi + echo "script_checksum = '${md5wrapped}'" + echo "debug_mode = '${operation_mode}'" + echo "condorg_cluster = '${condorg_cluster}'" + echo "condorg_subcluster= '${condorg_subcluster}'" + echo "condorg_schedd = '${condorg_schedd}'" + echo "glidein_uuid = '${glidein_uuid}'" + echo "glidein_credential_id = '${glidein_cred_id}'" + echo "glidein_factory = '${glidein_factory}'" + echo "glidein_name = '${glidein_name}'" + echo "glidein_entry = '${glidein_entry}'" + if [ -n "${client_name}" ]; then + # client name not required as it is not used for anything but debug info + echo "client_name = '${client_name}'" + fi + if [ -n "${client_group}" ]; then + echo "client_group = '${client_group}'" + fi + echo "multi_glidein/restart = '${multi_glidein}'/'${multi_glidein_restart}'" + echo "work_dir = '${work_dir}'" + echo "web_dir = '${repository_url}'" + echo "sign_type = '${sign_type}'" + echo "proxy_url = '${proxy_url}'" + echo "descript_fname = '${descript_file}'" + echo "descript_entry_fname = '${descript_entry_file}'" + echo "sign_id = '${sign_id}'" + echo "sign_entry_id = '${sign_entry_id}'" + if [ -n "${client_repository_url}" ]; then + echo "client_web_dir = '${client_repository_url}'" + echo "client_descript_fname = '${client_descript_file}'" + echo "client_sign_type = '${client_sign_type}'" + echo "client_sign_id = '${client_sign_id}'" + if [ -n "${client_repository_group_url}" ]; then + echo "client_web_group_dir = '${client_repository_group_url}'" + echo "client_descript_group_fname = '${client_descript_group_file}'" + echo "client_sign_group_id = '${client_sign_group_id}'" + fi + fi + echo + echo "Running on $(uname -n)" + echo "System: $(uname -a)" + if [ -e '/etc/redhat-release' ]; then + echo "Release: $(cat /etc/redhat-release 2>&1)" + fi + echo "As: $(id)" + echo "PID: $$" + echo + + if [ ${set_debug} -ne 0 ]; then + echo "------- Initial environment ---------------" 1>&2 + env 1>&2 + echo "------- =================== ---------------" 1>&2 + fi +} + +################################ +# Function used to parse the glidein startup options +# Parameters: +# @: shell parameters +# Global: +# params +# all other global variables to set +parse_options(){ + params="" + while [ $# -gt 0 ] + do case "$1" in + -factory) glidein_factory="$2";; + -name) glidein_name="$2";; + -entry) glidein_entry="$2";; + -clientname) client_name="$2";; + -clientgroup) client_group="$2";; + -web) repository_url="$2";; + -proxy) proxy_url="$2";; + -dir) work_dir="$2";; + -sign) sign_id="$2";; + -signtype) sign_type="$2";; + -signentry) sign_entry_id="$2";; + -cluster) condorg_cluster="$2";; + -subcluster) condorg_subcluster="$2";; + -submitcredid) glidein_cred_id="$2";; + -schedd) condorg_schedd="$2";; + -descript) descript_file="$2";; + -descriptentry) descript_entry_file="$2";; + -clientweb) client_repository_url="$2";; + -clientwebgroup) client_repository_group_url="$2";; + -clientsign) client_sign_id="$2";; + -clientsigntype) client_sign_type="$2";; + -clientsigngroup) client_sign_group_id="$2";; + -clientdescript) client_descript_file="$2";; + -clientdescriptgroup) client_descript_group_file="$2";; + -slotslayout) slots_layout="$2";; + -v) operation_mode="$2";; + -multiglidein) multi_glidein="$2";; + -multirestart) multi_glidein_restart="$2";; + -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; + *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 + esac + shift 2 + done +} + +################################ +# Function used to parse and verify arguments +# It allows some parameters to change arguments +# Global: +# tmp_par +# repository_entry_url +# proxy_url +# client_sign_type +parse_arguments(){ + # multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein + tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") + [ -n "${tmp_par}" ] && multi_glidein=${tmp_par} + + case "${operation_mode}" in + nodebug) + sleep_time=1199 + set_debug=0;; + fast) + sleep_time=150 + set_debug=1;; + check) + sleep_time=150 + set -x + set_debug=2;; + *) + sleep_time=1199 + set_debug=1;; + esac + + if [ -z "${descript_file}" ]; then + log_warn "Missing descript fname." + usage + fi + + if [ -z "${descript_entry_file}" ]; then + log_warn "Missing descript fname for entry." + usage + fi + + if [ -z "${glidein_name}" ]; then + log_warn "Missing gliden name." + usage + fi + + if [ -z "${glidein_entry}" ]; then + log_warn "Missing glidein entry name." + usage + fi + + + if [ -z "${repository_url}" ]; then + log_warn "Missing Web URL." + usage + fi + + repository_entry_url="${repository_url}/entry_${glidein_entry}" + + if [ -z "${proxy_url}" ]; then + proxy_url="None" + fi + + if [ "${proxy_url}" = "OSG" ]; then + if [ -z "${OSG_SQUID_LOCATION}" ]; then + # if OSG does not define a Squid, then don't use any + proxy_url="None" + log_warn "OSG_SQUID_LOCATION undefined, not using any Squid URL" 1>&2 + else + proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" + fi + fi + + if [ -z "${sign_id}" ]; then + log_warn "Missing signature." + usage + fi + + if [ -z "${sign_entry_id}" ]; then + log_warn "Missing entry signature." + usage + fi + + if [ -z "${sign_type}" ]; then + sign_type="sha1" + fi + + if [ "${sign_type}" != "sha1" ]; then + log_warn "Unsupported signtype ${sign_type} found." + usage + fi + + if [ -n "${client_repository_url}" ]; then + # client data is optional, user url as a switch + if [ -z "${client_sign_type}" ]; then + client_sign_type="sha1" + fi + + if [ "${client_sign_type}" != "sha1" ]; then + log_warn "Unsupported clientsigntype ${client_sign_type} found." + usage + fi + + if [ -z "${client_descript_file}" ]; then + log_warn "Missing client descript fname." + usage + fi + + if [ -n "${client_repository_group_url}" ]; then + # client group data is optional, user url as a switch + if [ -z "${client_group}" ]; then + log_warn "Missing client group name." + usage + fi + + if [ -z "${client_descript_group_file}" ]; then + log_warn "Missing client descript fname for group." + usage + fi + fi + fi +} diff --git a/creation/web_base/utils_gs_signals.sh b/creation/web_base/utils_gs_signals.sh new file mode 100644 index 0000000000..ffff9a9b09 --- /dev/null +++ b/creation/web_base/utils_gs_signals.sh @@ -0,0 +1,13 @@ +################################ +# Function that forwards signals to the children processes +# Arguments: +# 1: signal +# Globals: +# ON_DIE +on_die_multi() { + echo "Multi-Glidein received signal... shutting down child glideins (forwarding $1 signal to ${GWMS_MULTIGLIDEIN_CHILDS})" 1>&2 + ON_DIE=1 + for i in ${GWMS_MULTIGLIDEIN_CHILDS}; do + kill -s "$1" "${i}" + done +} diff --git a/creation/web_base/utils_gs_tarballs.sh b/creation/web_base/utils_gs_tarballs.sh new file mode 100644 index 0000000000..a5bdce8a95 --- /dev/null +++ b/creation/web_base/utils_gs_tarballs.sh @@ -0,0 +1,36 @@ +########################################### +# Untar support function +# Arguments: +# 1: id +# 2: filename +# Global: +# gus_id +# gus_fname +# gus_prefix +# gus_config_cfg +# gus_config_file +# gus_dir +# Returns: +# 0 in case of success, otherwise glidein_exit with 1 +get_untar_subdir() { + gus_id="$1" + gus_fname="$2" + + gus_prefix="$(get_prefix "${gus_id}")" + gus_config_cfg="${gus_prefix}UNTAR_CFG_FILE" + + gus_config_file="$(grep "^${gus_config_cfg} " glidein_config | cut -d ' ' -f 2-)" + if [ -z "${gus_config_file}" ]; then + log_warn "Error, cannot find '${gus_config_cfg}' in glidein_config." + glidein_exit 1 + fi + + gus_dir="$(grep -i "^${gus_fname} " "${gus_config_file}" | cut -s -f 2-)" + if [ -z "${gus_dir}" ]; then + log_warn "Error, untar dir for '${gus_fname}' cannot be empty." + glidein_exit 1 + fi + + echo "${gus_dir}" + return 0 +} diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index 1786e05630..4a4837fba2 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -24,150 +24,9 @@ print_header_line(){ if [ $# -eq 1 ]; then content=$1 echo "=== ${content} ===" - elif [ $# -eq 2 && $2 -eq 2]; then - #TODO(F): Check + elif [ $# -eq 2 -a $2 -eq 2 ] + then content=$1 echo "=== ${content} ===" 1>&2 fi } - -################################ -# Function used to print the tail with the final results of the glideins -# Arguments: -# 1: exit code -# 2: short version of the final results -# 3: long version of the final results -# Global: -# total_time -print_tail() { - local final_result_simple, final_result_long, exit_code - exit_code=$1 - final_result_simple="$2" - final_result_long="$3" - local glidein_end_time - glidein_end_time=$(date +%s) - let total_time=${glidein_end_time}-${startup_time} - print_header_line "Glidein ending $(date) (${glidein_end_time}) with code ${exit_code} after ${total_time}" - #echo "=== Glidein ending $(date) (${glidein_end_time}) with code ${exit_code} after ${total_time} ===" - echo "" - print_header_line "XML description of glidein activity" - #echo "=== XML description of glidein activity ===" - echo "${final_result_simple}" | grep -v "" - print_header_line "End XML description of glidein activity" - #echo "=== End XML description of glidein activity ===" - - echo "" 1>&2 - print_header_line "Encoded XML description of glidein activity" 2 - #echo "=== Encoded XML description of glidein activity ===" 1>&2 - echo "${final_result_long}" | gzip --stdout - | b64uuencode 1>&2 - print_header_line "End encoded XML description of glidein activity" 2 - #echo "=== End encoded XML description of glidein activity ===" 1>&2 -} - -################################ -# Function used to have information about the usage of the glidein_startup.sh script -# Returns: -# 1 in any case -usage() { - echo "Usage: glidein_startup.sh " - echo "where is:" - echo " -factory : name of this factory" - echo " -name : name of this glidein" - echo " -entry : name of this glidein entry" - echo " -clientname : name of the requesting client" - echo " -clientgroup : group name of the requesting client" - echo " -web : base URL from where to fetch" - echo " -proxy : URL of the local proxy" - echo " -dir : directory ID (supports ., Condor, CONDOR, OSG, TMPDIR, AUTO)" - echo " -sign : signature of the signature file" - echo " -signtype : type of signature (only sha1 supported for now)" - echo " -signentry : signature of the entry signature file" - echo " -cluster : condorG ClusterId" - echo " -subcluster : condorG ProcId" - echo " -submitcredid : Credential ID of this condorG job" - echo " -schedd : condorG Schedd Name" - echo " -descript : description file name" - echo " -descriptentry : description file name for entry" - echo " -clientweb : base URL from where to fetch client files" - echo " -clientwebgroup : base URL from where to fetch client group files" - echo " -clientsign : signature of the client signature file" - echo " -clientsigntype : type of client signature (only sha1 supported for now)" - echo " -clientsigngroup : signature of the client group signature file" - echo " -clientdescript : client description file name" - echo " -clientdescriptgroup : client description file name for group" - echo " -slotslayout : how Condor will set up slots (fixed, partitionable)" - echo " -v : operation mode (std, nodebug, fast, check supported)" - echo " -multiglidein : spawn multiple () glideins (unless also multirestart is set)" - echo " -multirestart : started as one of multiple glideins (glidein number )" - echo " -param_* : user specified parameters" -} - -################################ -# Function used to print initial information header -# Parameters: -# @: shell parameters -# Global: -# startup_time -print_header(){ - startup_time="$(date +%s)" - echo "Starting glidein_startup.sh at $(date) (${startup_time})" - local md5wrapped - md5wrapped="$(md5wrapper "$0")" - retVal=$? - if [ $retVal -ne 0 ]; then - echo "Error on the md5wrapper" - glidein_exit 1 #TODO(F): o solo exit? - fi - echo "script_checksum = '${md5wrapped}'" - echo "debug_mode = '${operation_mode}'" - echo "condorg_cluster = '${condorg_cluster}'" - echo "condorg_subcluster= '${condorg_subcluster}'" - echo "condorg_schedd = '${condorg_schedd}'" - echo "glidein_uuid = '${glidein_uuid}'" - echo "glidein_credential_id = '${glidein_cred_id}'" - echo "glidein_factory = '${glidein_factory}'" - echo "glidein_name = '${glidein_name}'" - echo "glidein_entry = '${glidein_entry}'" - if [ -n "${client_name}" ]; then - # client name not required as it is not used for anything but debug info - echo "client_name = '${client_name}'" - fi - if [ -n "${client_group}" ]; then - echo "client_group = '${client_group}'" - fi - echo "multi_glidein/restart = '${multi_glidein}'/'${multi_glidein_restart}'" - echo "work_dir = '${work_dir}'" - echo "web_dir = '${repository_url}'" - echo "sign_type = '${sign_type}'" - echo "proxy_url = '${proxy_url}'" - echo "descript_fname = '${descript_file}'" - echo "descript_entry_fname = '${descript_entry_file}'" - echo "sign_id = '${sign_id}'" - echo "sign_entry_id = '${sign_entry_id}'" - if [ -n "${client_repository_url}" ]; then - echo "client_web_dir = '${client_repository_url}'" - echo "client_descript_fname = '${client_descript_file}'" - echo "client_sign_type = '${client_sign_type}'" - echo "client_sign_id = '${client_sign_id}'" - if [ -n "${client_repository_group_url}" ]; then - echo "client_web_group_dir = '${client_repository_group_url}'" - echo "client_descript_group_fname = '${client_descript_group_file}'" - echo "client_sign_group_id = '${client_sign_group_id}'" - fi - fi - echo - echo "Running on $(uname -n)" - echo "System: $(uname -a)" - if [ -e '/etc/redhat-release' ]; then - echo "Release: $(cat /etc/redhat-release 2>&1)" - fi - echo "As: $(id)" - echo "PID: $$" - echo - - if [ ${set_debug} -ne 0 ]; then - echo "------- Initial environment ---------------" 1>&2 - env 1>&2 - echo "------- =================== ---------------" 1>&2 - fi -} diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index 5eaf051207..02bd1056c6 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -109,123 +109,3 @@ params2file() { return 0 } -################################ -# Function used to parse and verify arguments -# It allows some parameters to change arguments -# Global: -# tmp_par -# repository_entry_url -# proxy_url -# client_sign_type -parse_arguments(){ - # multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein - tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") - [ -n "${tmp_par}" ] && multi_glidein=${tmp_par} - - case "${operation_mode}" in - nodebug) - sleep_time=1199 - set_debug=0;; - fast) - sleep_time=150 - set_debug=1;; - check) - sleep_time=150 - set -x - set_debug=2;; - *) - sleep_time=1199 - set_debug=1;; - esac - - if [ -z "${descript_file}" ]; then - log_warn "Missing descript fname." - usage - fi - - if [ -z "${descript_entry_file}" ]; then - log_warn "Missing descript fname for entry." - usage - fi - - if [ -z "${glidein_name}" ]; then - log_warn "Missing gliden name." - usage - fi - - if [ -z "${glidein_entry}" ]; then - log_warn "Missing glidein entry name." - usage - fi - - - if [ -z "${repository_url}" ]; then - log_warn "Missing Web URL." - usage - fi - - repository_entry_url="${repository_url}/entry_${glidein_entry}" - - if [ -z "${proxy_url}" ]; then - proxy_url="None" - fi - - if [ "${proxy_url}" = "OSG" ]; then - if [ -z "${OSG_SQUID_LOCATION}" ]; then - # if OSG does not define a Squid, then don't use any - proxy_url="None" - log_warn "OSG_SQUID_LOCATION undefined, not using any Squid URL" 1>&2 - else - proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" - fi - fi - - if [ -z "${sign_id}" ]; then - log_warn "Missing signature." - usage - fi - - if [ -z "${sign_entry_id}" ]; then - log_warn "Missing entry signature." - usage - fi - - if [ -z "${sign_type}" ]; then - sign_type="sha1" - fi - - if [ "${sign_type}" != "sha1" ]; then - log_warn "Unsupported signtype ${sign_type} found." - usage - fi - - if [ -n "${client_repository_url}" ]; then - # client data is optional, user url as a switch - if [ -z "${client_sign_type}" ]; then - client_sign_type="sha1" - fi - - if [ "${client_sign_type}" != "sha1" ]; then - log_warn "Unsupported clientsigntype ${client_sign_type} found." - usage - fi - - if [ -z "${client_descript_file}" ]; then - log_warn "Missing client descript fname." - usage - fi - - if [ -n "${client_repository_group_url}" ]; then - # client group data is optional, user url as a switch - if [ -z "${client_group}" ]; then - log_warn "Missing client group name." - usage - fi - - if [ -z "${client_descript_group_file}" ]; then - log_warn "Missing client descript fname for group." - usage - fi - fi - fi -} diff --git a/creation/web_base/utils_signals.sh b/creation/web_base/utils_signals.sh index 6b0553b36e..83c884afd0 100644 --- a/creation/web_base/utils_signals.sh +++ b/creation/web_base/utils_signals.sh @@ -3,37 +3,27 @@ # Arguments: # 1: handler trap_with_arg() { - func="$1" ; shift + local func + func="$1" for sig ; do # shellcheck disable=SC2064 trap "${func} ${sig}" "${sig}" done } -# TODO: why do we need the shift? ################################ # Function that allows to pass signals to the children processes # There is no need to re-raise sigint, caller does unconditional exit (https://www.cons.org/cracauer/sigint.html) # Arguments: # 1: signal +# Global: +# ON_DIE on_die() { echo "Received kill signal... shutting down child processes (forwarding $1 signal)" 1>&2 ON_DIE=1 kill -s "$1" %1 } -################################ -# Function that forwards signals to the children processes -# Arguments: -# 1: signal -on_die_multi() { - echo "Multi-Glidein received signal... shutting down child glideins (forwarding $1 signal to ${GWMS_MULTIGLIDEIN_CHILDS})" 1>&2 - ON_DIE=1 - for i in ${GWMS_MULTIGLIDEIN_CHILDS}; do - kill -s "$1" "${i}" - done -} - ################################ # Function used to ignore signal SIGHUP ignore_signal() { diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh index 70802d4a29..0c356f5a85 100644 --- a/creation/web_base/utils_tarballs.sh +++ b/creation/web_base/utils_tarballs.sh @@ -45,40 +45,3 @@ extract_all_data() { done IFS="${IFS_OLD}" } - -########################################### -# Untar support function -# Arguments: -# 1: id -# 2: filename -# Global: -# gus_id -# gus_fname -# gus_prefix -# gus_config_cfg -# gus_config_file -# gus_dir -# Returns: -# 0 in case of success, otherwise glidein_exit with 1 -get_untar_subdir() { - gus_id="$1" - gus_fname="$2" - - gus_prefix="$(get_prefix "${gus_id}")" - gus_config_cfg="${gus_prefix}UNTAR_CFG_FILE" - - gus_config_file="$(grep "^${gus_config_cfg} " glidein_config | cut -d ' ' -f 2-)" - if [ -z "${gus_config_file}" ]; then - log_warn "Error, cannot find '${gus_config_cfg}' in glidein_config." - glidein_exit 1 - fi - - gus_dir="$(grep -i "^${gus_fname} " "${gus_config_file}" | cut -s -f 2-)" - if [ -z "${gus_dir}" ]; then - log_warn "Error, untar dir for '${gus_fname}' cannot be empty." - glidein_exit 1 - fi - - echo "${gus_dir}" - return 0 -} diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 647b40ae0c..de89592290 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -11,19 +11,7 @@ construct_xml() { tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" create_xml OSG --id glidein_startup.sh { oe { e --name cwd "${start_dir}" } t { c "${cmd}" tS "${tStart}" tE "${tEnd}" } "${result}" } - #echo -e $result - #echo " - # - # - # ${start_dir} - # - # - # $0 ${GLOBAL_ARGS} - # $(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z) - # $(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z) - # - #${result} - #" + echo -e "$result" } ################################ From 3857ec534e2fb09e11f7e292f2f9c41bb10345d1 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 15:00:13 -0500 Subject: [PATCH 031/196] Moved menu of options --- creation/web_base/glidein_startup.sh | 44 +++++++++++++++++++++++++- creation/web_base/utils_gs_io.sh | 46 ---------------------------- 2 files changed, 43 insertions(+), 47 deletions(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 33564b668d..f18aa819aa 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -304,7 +304,49 @@ create_glidein_config(){ #shift $(expr $OPTIND- 1) ################################ -parse_options +# Function used to parse the glidein startup options +# Parameters: +# @: shell parameters +# Global: +# params +# all other global variables to set + +params="" +while [ $# -gt 0 ] + do case "$1" in + -factory) glidein_factory="$2";; + -name) glidein_name="$2";; + -entry) glidein_entry="$2";; + -clientname) client_name="$2";; + -clientgroup) client_group="$2";; + -web) repository_url="$2";; + -proxy) proxy_url="$2";; + -dir) work_dir="$2";; + -sign) sign_id="$2";; + -signtype) sign_type="$2";; + -signentry) sign_entry_id="$2";; + -cluster) condorg_cluster="$2";; + -subcluster) condorg_subcluster="$2";; + -submitcredid) glidein_cred_id="$2";; + -schedd) condorg_schedd="$2";; + -descript) descript_file="$2";; + -descriptentry) descript_entry_file="$2";; + -clientweb) client_repository_url="$2";; + -clientwebgroup) client_repository_group_url="$2";; + -clientsign) client_sign_id="$2";; + -clientsigntype) client_sign_type="$2";; + -clientsigngroup) client_sign_group_id="$2";; + -clientdescript) client_descript_file="$2";; + -clientdescriptgroup) client_descript_group_file="$2";; + -slotslayout) slots_layout="$2";; + -v) operation_mode="$2";; + -multiglidein) multi_glidein="$2";; + -multirestart) multi_glidein_restart="$2";; + -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; + *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 + esac + shift 2 +done ################################ # Code block used to set the slots_layout diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index 05f371a173..33b6377ab2 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -134,52 +134,6 @@ print_header(){ fi } -################################ -# Function used to parse the glidein startup options -# Parameters: -# @: shell parameters -# Global: -# params -# all other global variables to set -parse_options(){ - params="" - while [ $# -gt 0 ] - do case "$1" in - -factory) glidein_factory="$2";; - -name) glidein_name="$2";; - -entry) glidein_entry="$2";; - -clientname) client_name="$2";; - -clientgroup) client_group="$2";; - -web) repository_url="$2";; - -proxy) proxy_url="$2";; - -dir) work_dir="$2";; - -sign) sign_id="$2";; - -signtype) sign_type="$2";; - -signentry) sign_entry_id="$2";; - -cluster) condorg_cluster="$2";; - -subcluster) condorg_subcluster="$2";; - -submitcredid) glidein_cred_id="$2";; - -schedd) condorg_schedd="$2";; - -descript) descript_file="$2";; - -descriptentry) descript_entry_file="$2";; - -clientweb) client_repository_url="$2";; - -clientwebgroup) client_repository_group_url="$2";; - -clientsign) client_sign_id="$2";; - -clientsigntype) client_sign_type="$2";; - -clientsigngroup) client_sign_group_id="$2";; - -clientdescript) client_descript_file="$2";; - -clientdescriptgroup) client_descript_group_file="$2";; - -slotslayout) slots_layout="$2";; - -v) operation_mode="$2";; - -multiglidein) multi_glidein="$2";; - -multirestart) multi_glidein_restart="$2";; - -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; - *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 - esac - shift 2 - done -} - ################################ # Function used to parse and verify arguments # It allows some parameters to change arguments From 4fee00054bb7342267aa4afd275c6d02edd4ff4d Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 15:01:39 -0500 Subject: [PATCH 032/196] Moved menu of options --- creation/web_base/glidein_startup.sh | 44 +------------------------- creation/web_base/utils_gs_io.sh | 46 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index f18aa819aa..bf8938701e 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -304,49 +304,7 @@ create_glidein_config(){ #shift $(expr $OPTIND- 1) ################################ -# Function used to parse the glidein startup options -# Parameters: -# @: shell parameters -# Global: -# params -# all other global variables to set - -params="" -while [ $# -gt 0 ] - do case "$1" in - -factory) glidein_factory="$2";; - -name) glidein_name="$2";; - -entry) glidein_entry="$2";; - -clientname) client_name="$2";; - -clientgroup) client_group="$2";; - -web) repository_url="$2";; - -proxy) proxy_url="$2";; - -dir) work_dir="$2";; - -sign) sign_id="$2";; - -signtype) sign_type="$2";; - -signentry) sign_entry_id="$2";; - -cluster) condorg_cluster="$2";; - -subcluster) condorg_subcluster="$2";; - -submitcredid) glidein_cred_id="$2";; - -schedd) condorg_schedd="$2";; - -descript) descript_file="$2";; - -descriptentry) descript_entry_file="$2";; - -clientweb) client_repository_url="$2";; - -clientwebgroup) client_repository_group_url="$2";; - -clientsign) client_sign_id="$2";; - -clientsigntype) client_sign_type="$2";; - -clientsigngroup) client_sign_group_id="$2";; - -clientdescript) client_descript_file="$2";; - -clientdescriptgroup) client_descript_group_file="$2";; - -slotslayout) slots_layout="$2";; - -v) operation_mode="$2";; - -multiglidein) multi_glidein="$2";; - -multirestart) multi_glidein_restart="$2";; - -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; - *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 - esac - shift 2 -done +parse_options $@ ################################ # Code block used to set the slots_layout diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index 33b6377ab2..05f371a173 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -134,6 +134,52 @@ print_header(){ fi } +################################ +# Function used to parse the glidein startup options +# Parameters: +# @: shell parameters +# Global: +# params +# all other global variables to set +parse_options(){ + params="" + while [ $# -gt 0 ] + do case "$1" in + -factory) glidein_factory="$2";; + -name) glidein_name="$2";; + -entry) glidein_entry="$2";; + -clientname) client_name="$2";; + -clientgroup) client_group="$2";; + -web) repository_url="$2";; + -proxy) proxy_url="$2";; + -dir) work_dir="$2";; + -sign) sign_id="$2";; + -signtype) sign_type="$2";; + -signentry) sign_entry_id="$2";; + -cluster) condorg_cluster="$2";; + -subcluster) condorg_subcluster="$2";; + -submitcredid) glidein_cred_id="$2";; + -schedd) condorg_schedd="$2";; + -descript) descript_file="$2";; + -descriptentry) descript_entry_file="$2";; + -clientweb) client_repository_url="$2";; + -clientwebgroup) client_repository_group_url="$2";; + -clientsign) client_sign_id="$2";; + -clientsigntype) client_sign_type="$2";; + -clientsigngroup) client_sign_group_id="$2";; + -clientdescript) client_descript_file="$2";; + -clientdescriptgroup) client_descript_group_file="$2";; + -slotslayout) slots_layout="$2";; + -v) operation_mode="$2";; + -multiglidein) multi_glidein="$2";; + -multirestart) multi_glidein_restart="$2";; + -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; + *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 + esac + shift 2 + done +} + ################################ # Function used to parse and verify arguments # It allows some parameters to change arguments From cf63cf94d4c35278252b6c0c4c80cc9633189001 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Aug 2022 20:14:39 +0000 Subject: [PATCH 033/196] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- creation/web_base/glidein_cleanup.sh | 5 +- creation/web_base/glidein_startup.sh | 20 ++++---- creation/web_base/manual_installation_wget.sh | 2 +- creation/web_base/utils_crypto.sh | 1 - creation/web_base/utils_gs_filesystem.sh | 30 ++++++------ creation/web_base/utils_gs_http.sh | 8 ++-- creation/web_base/utils_gs_io.sh | 46 +++++++++---------- creation/web_base/utils_io.sh | 6 +-- creation/web_base/utils_params.sh | 1 - creation/web_base/utils_xml.sh | 7 ++- 10 files changed, 61 insertions(+), 65 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 5e9df84e44..c869afd29e 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -44,7 +44,7 @@ early_glidein_failure() { glidein_end_time="$(date +%s)" ts="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" create_xml -s 4 metric --name failure --ts "$ts" --uri local WN_RESOURCE status ERROR detail ${error_msg} - #result + #result #result=" WN_RESOURCE # ERROR # @@ -90,7 +90,7 @@ glidein_exit() { if [ -z "${report_failed}" ]; then report_failed="NEVER" fi - + factory_report_failed=$(grep -i "^GLIDEIN_Factory_Report_Failed " "${glidein_config}" | cut -d ' ' -f 2-) if [ -z "${factory_report_failed}" ]; then @@ -190,4 +190,3 @@ glidein_exit() { print_tail "$1" "${final_result_simple}" "${final_result_long}" exit "$1" } - diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index bf8938701e..e1f49ebf86 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -130,7 +130,7 @@ spawn_multiple_glideins(){ echo "------ Exiting multi-glidein parent ----------" 1>&2 exit 0 fi -} +} ######################################## # Function used to setup OSG and/or Globus @@ -144,7 +144,7 @@ setup_OSG_Globus(){ . "${GLITE_LOCAL_CUSTOMIZATION_DIR}/cp_1.sh" fi fi - + if [ -z "${GLOBUS_PATH}" ]; then if [ -z "${GLOBUS_LOCATION}" ]; then # if GLOBUS_LOCATION not defined, try to guess it @@ -160,14 +160,14 @@ setup_OSG_Globus(){ log_warn 'Continuing like nothing happened' fi fi - + if [ -r "${GLOBUS_LOCATION}/etc/globus-user-env.sh" ]; then . "${GLOBUS_LOCATION}/etc/globus-user-env.sh" else log_warn "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." log_warn 'Continuing like nothing happened' fi - fi + fi } ######################################## @@ -219,7 +219,7 @@ create_glidein_config(){ echo "GLIDEIN_Factory ${glidein_factory}" echo "GLIDEIN_Name ${glidein_name}" echo "GLIDEIN_Entry_Name ${glidein_entry}" - + if [ -n "${client_name}" ]; then # client name not required as it is not used for anything but debug info echo "GLIDECLIENT_Name ${client_name}" @@ -245,7 +245,7 @@ create_glidein_config(){ echo "DESCRIPTION_ENTRY_FILE ${descript_entry_file}" echo "GLIDEIN_Signature ${sign_id}" echo "GLIDEIN_Entry_Signature ${sign_entry_id}" - + if [ -n "${client_repository_url}" ]; then echo "GLIDECLIENT_WORK_DIR ${client_dir}" echo "GLIDECLIENT_DESCRIPTION_FILE ${client_descript_file}" @@ -298,7 +298,7 @@ create_glidein_config(){ # 'n') a=2 ;; # 'r') rest=true ;; # 'w') ws=true ;; -# '?') print_usage >&2; exit 1;; +# '?') print_usage >&2; exit 1;; # esac #done #shift $(expr $OPTIND- 1) @@ -330,7 +330,7 @@ fi print_header "@" ################################ -spawn_multiple_glideins +spawn_multiple_glideins ######################################## # Code block used to make sure nobody else can write my files @@ -376,14 +376,14 @@ prepare_workdir ######################################## # extract and source all the data contained at the end of this script as tarball -extract_all_data +extract_all_data ######################################## wrapper_list="${PWD}/wrapper_list.lst" touch "${wrapper_list}" ######################################## -create_glidein_config +create_glidein_config ######################################## # shellcheck disable=SC2086 diff --git a/creation/web_base/manual_installation_wget.sh b/creation/web_base/manual_installation_wget.sh index 7697bacb92..742a4441d7 100644 --- a/creation/web_base/manual_installation_wget.sh +++ b/creation/web_base/manual_installation_wget.sh @@ -18,4 +18,4 @@ wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web cat glidein_startup_github.sh term_file tar_utils.tar.gz > glidein_startup.sh chmod +x glidein_startup.sh ./manual_glidein_startup --wms-collector=fermicloud532.fnal.gov --client-name=fermicloud597-fnal-gov_OSG_gWMSFrontend.main --req-name=ITB_FC_CE2b@gfactory_instance@gfactory_service --cmd-out-file=glidein_startup_wrapper --glidein-startup=./glidein_startup.sh -./glidein_startup_wrapper +./glidein_startup_wrapper diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index 28b7fe1393..2d81e9c3c7 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -92,4 +92,3 @@ set_proxy_fullpath() { echo "Unable to get canonical path for X509_USER_PROXY, using ${X509_USER_PROXY}" 1>&2 fi } - diff --git a/creation/web_base/utils_gs_filesystem.sh b/creation/web_base/utils_gs_filesystem.sh index 62a1c35cde..a7feda6691 100644 --- a/creation/web_base/utils_gs_filesystem.sh +++ b/creation/web_base/utils_gs_filesystem.sh @@ -104,18 +104,18 @@ prepare_workdir(){ .) work_dir="$(pwd)";; esac fi - + if [ -z "${work_dir}" ]; then early_glidein_failure "Unable to identify Startup dir for the glidein." fi - + if [ ! -e "${work_dir}" ]; then early_glidein_failure "Startup dir ${work_dir} does not exist." fi - + start_dir="$(pwd)" echo "Started in ${start_dir}" - + def_work_dir="${work_dir}/glide_$(dir_id)XXXXXX" if ! work_dir="$(mktemp -d "${def_work_dir}")"; then early_glidein_failure "Cannot create temp '${def_work_dir}'" @@ -127,7 +127,7 @@ prepare_workdir(){ fi fi work_dir_created=1 - + # GWMS_SUBDIR defined on top GWMS_DIR="${work_dir}/$GWMS_SUBDIR" if ! mkdir "$GWMS_DIR" ; then @@ -149,7 +149,7 @@ prepare_workdir(){ mkdir -p "$gwms_exec_dir"/$i done fi - + # mktemp makes it user readable by definition (ignores umask) # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then @@ -157,18 +157,18 @@ prepare_workdir(){ early_glidein_failure "Failed chmod '${work_dir}'" fi fi - + def_glide_local_tmp_dir="/tmp/glide_$(dir_id)$(id -u -n)_XXXXXX" if ! glide_local_tmp_dir="$(mktemp -d "${def_glide_local_tmp_dir}")"; then early_glidein_failure "Cannot create temp '${def_glide_local_tmp_dir}'" fi glide_local_tmp_dir_created=1 - + glide_tmp_dir="${work_dir}/tmp" if ! mkdir "${glide_tmp_dir}"; then early_glidein_failure "Cannot create '${glide_tmp_dir}'" fi - + if [ -n "${GWMS_MULTIUSER_GLIDEIN}" ]; then # TODO: MMSEC should this change to increase protection? Since GlExec is gone this should not be needed # the tmpdirs should be world writable @@ -177,31 +177,31 @@ prepare_workdir(){ if ! chmod 1777 "${glide_local_tmp_dir}"; then early_glidein_failure "Failed chmod '${glide_local_tmp_dir}'" fi - + if ! chmod 1777 "${glide_tmp_dir}"; then early_glidein_failure "Failed chmod '${glide_tmp_dir}'" fi fi - + short_main_dir=main main_dir="${work_dir}/${short_main_dir}" if ! mkdir "${main_dir}"; then early_glidein_failure "Cannot create '${main_dir}'" fi - + short_entry_dir=entry_${glidein_entry} entry_dir="${work_dir}/${short_entry_dir}" if ! mkdir "${entry_dir}"; then early_glidein_failure "Cannot create '${entry_dir}'" fi - + if [ -n "${client_repository_url}" ]; then short_client_dir=client client_dir="${work_dir}/${short_client_dir}" if ! mkdir "$client_dir"; then early_glidein_failure "Cannot create '${client_dir}'" fi - + if [ -n "${client_repository_group_url}" ]; then short_client_group_dir=client_group_${client_group} client_group_dir="${work_dir}/${short_client_group_dir}" @@ -210,7 +210,7 @@ prepare_workdir(){ fi fi fi - + # Move the token files from condor to glidein workspace mv "${start_dir}/tokens.tgz" . mv "${start_dir}/url_dirs.desc" . diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 015e0588a4..63600216b3 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -237,7 +237,7 @@ fetch_file_try() { # fetch_completed # Returns: # 1 in case the error is already displayed inside the function, -# in case of tarring, in case of failure in renaming it, +# in case of tarring, in case of failure in renaming it, # of making it executable, or executing it fetch_file_base() { ffb_id="$1" @@ -482,7 +482,7 @@ perform_wget() { create_xml -s 1 d "Failed to load file '${ffb_real_fname}' from '${ffb_repository}' using proxy '${proxy_url}'. ${wget_resp}" xmlResult+=$result create_xml -t - xmlResult+=$result + xmlResult+=$result echo -e "$xmlResult" > otrb_output.xml log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." @@ -544,7 +544,7 @@ perform_curl() { if [ ${curl_retval} -eq 0 ] && [ ! -e "${ffb_tmp_outname}" ] ; then touch "${ffb_tmp_outname}" fi - + if [ "${curl_retval}" -ne 0 ]; then curl_version="$(curl --version 2>&1 | head -1)" log_warn "${curl_cmd} failed. version:${curl_version} exit code ${curl_retval} stderr: ${curl_resp} " @@ -561,7 +561,7 @@ perform_curl() { xmlResult+=$result create_xml -t xmlResult+=$result - echo -e "$xmlResult" > otrb_output.xml + echo -e "$xmlResult" > otrb_output.xml log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." if [ -f otr_outlist.list ]; then diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index 05f371a173..a0aba564db 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -19,7 +19,7 @@ print_tail() { print_header_line "XML description of glidein activity" echo "${final_result_simple}" | grep -v "" print_header_line "End XML description of glidein activity" - + echo "" 1>&2 print_header_line "Encoded XML description of glidein activity" 2 echo "${final_result_long}" | gzip --stdout - | b64uuencode 1>&2 @@ -126,7 +126,7 @@ print_header(){ echo "As: $(id)" echo "PID: $$" echo - + if [ ${set_debug} -ne 0 ]; then echo "------- Initial environment ---------------" 1>&2 env 1>&2 @@ -177,7 +177,7 @@ parse_options(){ *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 esac shift 2 - done + done } ################################ @@ -192,7 +192,7 @@ parse_arguments(){ # multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") [ -n "${tmp_par}" ] && multi_glidein=${tmp_par} - + case "${operation_mode}" in nodebug) sleep_time=1199 @@ -208,39 +208,39 @@ parse_arguments(){ sleep_time=1199 set_debug=1;; esac - + if [ -z "${descript_file}" ]; then log_warn "Missing descript fname." usage fi - + if [ -z "${descript_entry_file}" ]; then log_warn "Missing descript fname for entry." usage fi - + if [ -z "${glidein_name}" ]; then log_warn "Missing gliden name." usage fi - + if [ -z "${glidein_entry}" ]; then log_warn "Missing glidein entry name." usage fi - - + + if [ -z "${repository_url}" ]; then log_warn "Missing Web URL." usage fi - + repository_entry_url="${repository_url}/entry_${glidein_entry}" - + if [ -z "${proxy_url}" ]; then proxy_url="None" fi - + if [ "${proxy_url}" = "OSG" ]; then if [ -z "${OSG_SQUID_LOCATION}" ]; then # if OSG does not define a Squid, then don't use any @@ -250,53 +250,53 @@ parse_arguments(){ proxy_url="$(echo "${OSG_SQUID_LOCATION}" | awk -F ':' '{if ($2 =="") {print $1 ":3128"} else {print $0}}')" fi fi - + if [ -z "${sign_id}" ]; then log_warn "Missing signature." usage fi - + if [ -z "${sign_entry_id}" ]; then log_warn "Missing entry signature." usage fi - + if [ -z "${sign_type}" ]; then sign_type="sha1" fi - + if [ "${sign_type}" != "sha1" ]; then log_warn "Unsupported signtype ${sign_type} found." usage fi - + if [ -n "${client_repository_url}" ]; then # client data is optional, user url as a switch if [ -z "${client_sign_type}" ]; then client_sign_type="sha1" fi - + if [ "${client_sign_type}" != "sha1" ]; then log_warn "Unsupported clientsigntype ${client_sign_type} found." usage fi - + if [ -z "${client_descript_file}" ]; then log_warn "Missing client descript fname." usage fi - + if [ -n "${client_repository_group_url}" ]; then # client group data is optional, user url as a switch if [ -z "${client_group}" ]; then log_warn "Missing client group name." usage fi - + if [ -z "${client_descript_group_file}" ]; then log_warn "Missing client descript fname for group." usage fi fi - fi + fi } diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index 4a4837fba2..bb761370ec 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -1,7 +1,7 @@ ################################ # Function used to log warning statements # Arguments: -# @: content to warn +# @: content to warn log_warn() { echo "WARN $(date)" "$@" 1>&2 } @@ -9,7 +9,7 @@ log_warn() { ################################ # Function used to log debug statements # Arguments: -# @: content to debug +# @: content to debug log_debug() { echo "DEBUG $(date)" "$@" 1>&2 } @@ -18,7 +18,7 @@ log_debug() { # Function used to prit a header line # Arguments: # 1: content of the header line -# 2 (optional): 2 if needs to write to stderr +# 2 (optional): 2 if needs to write to stderr print_header_line(){ local content if [ $# -eq 1 ]; then diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index 02bd1056c6..c1f097937e 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -108,4 +108,3 @@ params2file() { echo "PARAM_LIST ${param_list}" return 0 } - diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index de89592290..d903f2d726 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -2,7 +2,7 @@ # Function used to create the xml content # Arguments: # 1: result -construct_xml() { +construct_xml() { local result result="$1" local glidein_end_time @@ -33,12 +33,12 @@ extract_parent_fname(){ echo "Unknown" fi } - + ################################ # Function used to extract the parent xml details # Arguments: # 1: exit code -extract_parent_xml_detail() { +extract_parent_xml_detail() { local glidein_end_time exitcode last_result last_script_name last_script_reason my_reason exitcode=$1 glidein_end_time="$(date +%s)" @@ -291,4 +291,3 @@ create_xml(){ result=$xml #return xml } - From 5f76a4fb5c7fc973ddd5795c70d0c061ea5f0f28 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 11 Aug 2022 16:29:09 -0500 Subject: [PATCH 034/196] Header and license added to each script file --- creation/web_base/glidein_cleanup.sh | 10 ++++++++++ creation/web_base/glidein_startup.sh | 11 +++++++---- creation/web_base/utils_crypto.sh | 8 ++++++++ creation/web_base/utils_gs_filesystem.sh | 9 +++++++++ creation/web_base/utils_gs_http.sh | 10 ++++++++++ creation/web_base/utils_gs_io.sh | 9 +++++++++ creation/web_base/utils_gs_signals.sh | 9 +++++++++ creation/web_base/utils_gs_tarballs.sh | 9 +++++++++ creation/web_base/utils_io.sh | 8 ++++++++ creation/web_base/utils_params.sh | 8 ++++++++ creation/web_base/utils_signals.sh | 8 ++++++++ creation/web_base/utils_tarballs.sh | 8 ++++++++ creation/web_base/utils_xml.sh | 8 ++++++++ 13 files changed, 111 insertions(+), 4 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 5e9df84e44..b07e5d9f33 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -1,3 +1,13 @@ + +#*******************************************************************# +# glidein_cleanup.sh # +# Script containing the variables and functions used to support # +# the glidein cleanup, failure and exit operations # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Global variables of support to the glidein cleanup operations work_dir_created=0 diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index bf8938701e..85b00d168e 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1,11 +1,14 @@ #!/bin/bash -x + #*******************************************************************# # glidein_startup.sh # -# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # -# SPDX-License-Identifier: Apache-2.0 # -# File Version: # +# Script containing the variables and fuctions used as support # +# for the glidein startup. The script load the content of # +# glidein_cleanup and other utility functions # +# File Version: 1.0 # #*******************************************************************# - +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 ################################ # Default IFS, to protect against unusual environment diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index 28b7fe1393..dd94423256 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -1,3 +1,11 @@ +#*******************************************************************# +# utils_crypto.sh # +# This script contains cryptography utility functions # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Function used to calculate the md5 sum # Arguments: diff --git a/creation/web_base/utils_gs_filesystem.sh b/creation/web_base/utils_gs_filesystem.sh index 62a1c35cde..e880bd6422 100644 --- a/creation/web_base/utils_gs_filesystem.sh +++ b/creation/web_base/utils_gs_filesystem.sh @@ -1,3 +1,12 @@ +#*******************************************************************# +# utils_gs_filesystem.sh # +# This script contains filesystem utility functions for # +# the glidein_startup.sh script # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Function used to automatically determine and setup work directories # Globals: diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 015e0588a4..e90f6dccd6 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -1,3 +1,13 @@ +#*******************************************************************# +# utils_gs_http.sh # +# This script contains http utility functions for the # +# glidein_startup.sh script # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + + ############################################ # Function that gets the proper descript file based on id # Arguments: diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index 05f371a173..7ac730d8e6 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -1,3 +1,12 @@ +#*******************************************************************# +# utils_gs_io.sh # +# This script contains I/O utility functions for the # +# glidein_startup.sh script # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Function used to print the tail with the final results of the glideins # Arguments: diff --git a/creation/web_base/utils_gs_signals.sh b/creation/web_base/utils_gs_signals.sh index ffff9a9b09..85fa3c91dc 100644 --- a/creation/web_base/utils_gs_signals.sh +++ b/creation/web_base/utils_gs_signals.sh @@ -1,3 +1,12 @@ +#*******************************************************************# +# utils_gs_signals.sh # +# This script contains signals' utility functions for the # +# glidein_startup.sh script # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Function that forwards signals to the children processes # Arguments: diff --git a/creation/web_base/utils_gs_tarballs.sh b/creation/web_base/utils_gs_tarballs.sh index a5bdce8a95..0a876734c8 100644 --- a/creation/web_base/utils_gs_tarballs.sh +++ b/creation/web_base/utils_gs_tarballs.sh @@ -1,3 +1,12 @@ +#*******************************************************************# +# utils_gs_tarballs.sh # +# This script contains tarballs utility functions for the # +# glidein_startup.sh script # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ########################################### # Untar support function # Arguments: diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index 4a4837fba2..5ca0a712d6 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -1,3 +1,11 @@ +#*******************************************************************# +# utils_io.sh # +# This script contains I/O utility functions # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Function used to log warning statements # Arguments: diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index 02bd1056c6..57edf58aa2 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -1,3 +1,11 @@ +#*******************************************************************# +# utils_crypto.sh # +# This script contains parameters utility functions # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Parameters utility functions diff --git a/creation/web_base/utils_signals.sh b/creation/web_base/utils_signals.sh index 83c884afd0..e39965f179 100644 --- a/creation/web_base/utils_signals.sh +++ b/creation/web_base/utils_signals.sh @@ -1,3 +1,11 @@ +#*******************************************************************# +# utils_signals.sh # +# This script contains signals utility functions # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Extends 'trap' allowing to pass the signal name as argument to the handler # Arguments: diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh index 0c356f5a85..53ca4d8c97 100644 --- a/creation/web_base/utils_tarballs.sh +++ b/creation/web_base/utils_tarballs.sh @@ -1,3 +1,11 @@ +#*******************************************************************# +# utils_crypto.sh # +# This script contains tarballs utility functions # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ############################## # Utility functions to allow the script to source functions and retrieve data stored as tarball at the end of the script itself diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index de89592290..2c848831d5 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -1,3 +1,11 @@ +#*******************************************************************# +# utils_crypto.sh # +# This script contains xml utility functions # +# File Version: 1.0 # +#*******************************************************************# +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + ################################ # Function used to create the xml content # Arguments: From 1364e8cc1c96e393bcba6d9765452d1453a94a9d Mon Sep 17 00:00:00 2001 From: Franco Terranova <61695945+terranovaa@users.noreply.github.com> Date: Thu, 11 Aug 2022 16:39:16 -0500 Subject: [PATCH 035/196] Delete manual_installation_wget.sh --- creation/web_base/manual_installation_wget.sh | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 creation/web_base/manual_installation_wget.sh diff --git a/creation/web_base/manual_installation_wget.sh b/creation/web_base/manual_installation_wget.sh deleted file mode 100644 index 742a4441d7..0000000000 --- a/creation/web_base/manual_installation_wget.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -xv - -rm utils_* -rm glidein_startup_github.sh -rm glidein_cleanup.sh -rm glidein_startup.sh -wget -O glidein_startup_github.sh https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/glidein_startup.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/glidein_cleanup.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_crypto.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_fetch.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_log.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_http.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_filesystem.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_params.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_signals.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_tarballs.sh -wget https://raw.githubusercontent.com/terranovaa/glideinwms/master/creation/web_base/utils_xml.sh -cat glidein_startup_github.sh term_file tar_utils.tar.gz > glidein_startup.sh -chmod +x glidein_startup.sh -./manual_glidein_startup --wms-collector=fermicloud532.fnal.gov --client-name=fermicloud597-fnal-gov_OSG_gWMSFrontend.main --req-name=ITB_FC_CE2b@gfactory_instance@gfactory_service --cmd-out-file=glidein_startup_wrapper --glidein-startup=./glidein_startup.sh -./glidein_startup_wrapper From 40c6e21cd465d80fbe097981eac80a03b77bcc43 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 12 Aug 2022 09:44:42 -0500 Subject: [PATCH 036/196] Added more controls to the parse_option method and more detailed comments on I/O methods --- creation/web_base/utils_gs_io.sh | 4 ++++ creation/web_base/utils_io.sh | 8 ++++++-- creation/web_base/utils_xml.sh | 11 ++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index 2e76f3a1f5..2f0ca9da29 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -153,6 +153,9 @@ print_header(){ parse_options(){ params="" while [ $# -gt 0 ] + if [[ $2 == "-"* ]]; then + (log_warn "Wrong argument: $2 for option $1."; log_warn "You cannot set two consecutive options without specifying the option value!"; usage; exit 1) 1>&2; exit 1 + fi do case "$1" in -factory) glidein_factory="$2";; -name) glidein_name="$2";; @@ -183,6 +186,7 @@ parse_options(){ -multiglidein) multi_glidein="$2";; -multirestart) multi_glidein_restart="$2";; -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; + -h|--help) usage; exit 0;; *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 esac shift 2 diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index b2b99c5b35..4a7f79d609 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -23,15 +23,19 @@ log_debug() { } ##################### -# Function used to prit a header line +# Function used to print a header line, i.e. === HEADER === # Arguments: # 1: content of the header line -# 2 (optional): 2 if needs to write to stderr +# 2 (optional): 1 if needs to write to stdout, 2 if needs to write to stderr print_header_line(){ local content if [ $# -eq 1 ]; then content=$1 echo "=== ${content} ===" + elif [ $# -eq 2 -a $2 -eq 1 ] + then + content=$1 + echo "=== ${content} ===" elif [ $# -eq 2 -a $2 -eq 2 ] then content=$1 diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index b174d9c57d..09f1fb8829 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -54,6 +54,7 @@ extract_parent_xml_detail() { # file exists and is not 0 size last_result="$(cat otrx_output.xml)" if [ "${exitcode}" -eq 0 ]; then + #create_xml -s 2 result { status OK } echo " " echo " OK" #propagate metrics as well @@ -150,6 +151,13 @@ add_spaces(){ ################################ # Function used to create an xml file structure +# Call the function with the following syntax: +# create_xml TAG --option option_value .... tag_value TAG2 ..... +# use '{' to specify the start if an inner tag and use '}' to specify the end +# use create_xml -s SPACES ... in case you want to create a subpart of an xml file not starting from the beginning specifying +# the number of spaces needed at the beginning in order to have a good indentation +# use create_xml -t to require as output only the tail tag of the xml file, i.e. +# use create_xml -h to require as output only the header tag of the xml file, i.e. " # Arguments: # @: tags, options, values # Global: @@ -293,9 +301,6 @@ create_xml(){ *) xml+=$1;; esac shift 1 - #echo $end_xml done - #echo -e "$xml" result=$xml - #return xml } From e65687530d844752cad3e62dd3cde898b219bed3 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 12 Aug 2022 15:38:07 -0500 Subject: [PATCH 037/196] Added more controls to the parse_option method and more detailed comments on I/O methods --- creation/web_base/glidein_cleanup.sh | 14 ++++++-------- creation/web_base/glidein_startup.sh | 2 +- creation/web_base/utils_gs_io.sh | 2 +- creation/web_base/utils_xml.sh | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index cfe61d07a7..babda970f4 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -2,7 +2,7 @@ #*******************************************************************# # glidein_cleanup.sh # # Script containing the variables and functions used to support # -# the glidein cleanup, failure and exit operations # +# the glidein cleanup, failure and exit operations # # File Version: 1.0 # #*******************************************************************# # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC @@ -20,7 +20,7 @@ glide_local_tmp_dir_created=0 # It uses GLIDEIN_DEBUG_OPTIONS, start_dir, work_dir_created, work_dir, glide_local_tmp_dir_created, glide_local_tmp_dir # Arguments: # 1: exit code -glidien_cleanup() { +glidein_cleanup() { if ! cd "${start_dir}"; then log_warn "Cannot find ${start_dir} anymore, exiting but without cleanup" else @@ -60,13 +60,13 @@ early_glidein_failure() { # # ${error_msg} # " - echo "testing xml" - echo -e $result + #echo "testing xml" + #echo -e $result final_result="$(construct_xml "${result}")" final_result_simple="$(basexml2simplexml "${final_result}")" # have no global section final_result_long="$(simplexml2longxml "${final_result_simple}" "")" - glidien_cleanup + glidein_cleanup print_tail 1 "${final_result_simple}" "${final_result_long}" exit 1 } @@ -126,7 +126,6 @@ glidein_exit() { add_config_line "GLIDEIN_Expire" "${dl}" add_config_line "GLIDEIN_LAST_SCRIPT" "${ge_last_script_name}" add_config_line "GLIDEIN_ADVERTISE_TYPE" "Retiring" - add_config_line "GLIDEIN_FAILURE_REASON" "Glidein failed while running ${ge_last_script_name}. Keeping node busy until ${dl} (${dlf})." condor_vars_file="$(grep -i "^CONDOR_VARS_FILE " "${glidein_config}" | cut -d ' ' -f 2-)" @@ -141,7 +140,6 @@ glidein_exit() { add_condor_vars_line "GLIDEIN_FAILURE_REASON" "S" "-" "+" "Y" "Y" "-" fi main_work_dir="$(get_work_dir main)" - for ((t=$(date +%s); t < dl; t=$(date +%s))) do if [ -e "${main_work_dir}/${last_script}" ] && [ "${do_report}" = "1" ] ; then @@ -196,7 +194,7 @@ glidein_exit() { fi log_write "glidein_startup.sh" "text" "glidein is about to exit with retcode $1" "info" send_logs_to_remote - glidien_cleanup + glidein_cleanup print_tail "$1" "${final_result_simple}" "${final_result_long}" exit "$1" } diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index b251da7d3c..eff4a90669 100644 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -40,8 +40,8 @@ GWMS_LOGSERVER_ADDRESS='https://fermicloud152.fnal.gov/log' GWMS_MULTIGLIDEIN_CHILDS= # Include all source scripts -source utils_gs_filesystem.sh source utils_gs_http.sh +source utils_gs_filesystem.sh source utils_gs_io.sh source utils_gs_signals.sh source utils_gs_tarballs.sh diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index 2f0ca9da29..c47fe10177 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -1,7 +1,7 @@ #*******************************************************************# # utils_gs_io.sh # # This script contains I/O utility functions for the # -# glidein_startup.sh script # +# glidein_startup.sh script # # File Version: 1.0 # #*******************************************************************# # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 09f1fb8829..dcb4b6ad25 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -155,7 +155,7 @@ add_spaces(){ # create_xml TAG --option option_value .... tag_value TAG2 ..... # use '{' to specify the start if an inner tag and use '}' to specify the end # use create_xml -s SPACES ... in case you want to create a subpart of an xml file not starting from the beginning specifying -# the number of spaces needed at the beginning in order to have a good indentation +# the number of spaces needed at the beginning in order to have a good indentation # use create_xml -t to require as output only the tail tag of the xml file, i.e. # use create_xml -h to require as output only the header tag of the xml file, i.e. " # Arguments: From 8a4fe6dc0247b187c832ad91cf2c206737ec00cb Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 16 Aug 2022 15:58:06 -0500 Subject: [PATCH 038/196] Implemented the bats tests of the glidein_cleanup.sh, utils_gs_io.sh, utils_crypto.sh, utils_xml.sh, utils_signals.sh, utils_gs_filesystem.sh, utils_gs_http.sh, utils_io.sh, utils_signals.sh files --- .gitignore | 6 + creation/web_base/glidein_cleanup.sh | 12 +- creation/web_base/glidein_startup.sh | 612 +++++++++--------- creation/web_base/utils_crypto.sh | 4 + creation/web_base/utils_gs_filesystem.sh | 2 +- creation/web_base/utils_gs_http.sh | 2 +- creation/web_base/utils_gs_io.sh | 24 +- creation/web_base/utils_gs_tarballs.sh | 2 + creation/web_base/utils_tarballs.sh | 4 + creation/web_base/utils_xml.sh | 34 +- .../creation_web_base_glidein_cleanup.bats | 127 ++++ .../creation_web_base_glidein_startup.bats | 100 +++ test/bats/creation_web_base_utils_crypto.bats | 38 ++ ...creation_web_base_utils_gs_filesystem.bats | 102 +++ .../bats/creation_web_base_utils_gs_http.bats | 97 +++ test/bats/creation_web_base_utils_gs_io.bats | 33 + .../creation_web_base_utils_gs_tarballs.bats | 38 ++ test/bats/creation_web_base_utils_io.bats | 51 ++ .../bats/creation_web_base_utils_signals.bats | 45 ++ test/bats/creation_web_base_utils_xml.bats | 144 +++++ 20 files changed, 1131 insertions(+), 346 deletions(-) mode change 100644 => 100755 creation/web_base/glidein_startup.sh create mode 100755 test/bats/creation_web_base_glidein_cleanup.bats create mode 100755 test/bats/creation_web_base_glidein_startup.bats create mode 100755 test/bats/creation_web_base_utils_crypto.bats create mode 100755 test/bats/creation_web_base_utils_gs_filesystem.bats create mode 100755 test/bats/creation_web_base_utils_gs_http.bats create mode 100755 test/bats/creation_web_base_utils_gs_io.bats create mode 100755 test/bats/creation_web_base_utils_gs_tarballs.bats create mode 100755 test/bats/creation_web_base_utils_io.bats create mode 100755 test/bats/creation_web_base_utils_signals.bats create mode 100755 test/bats/creation_web_base_utils_xml.bats diff --git a/.gitignore b/.gitignore index 8eb5c8596c..bc16862fca 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,9 @@ bigfiles/* # Scripts were moved. The following 2 are here for back compatibility, can be removed in the future !bigfiles/pull-bigfiles.sh !bigfiles/push-bigfiles.sh + +.idea/* +node_modules/* +output/* +package-lock.json +package.json diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index babda970f4..612c72d0cf 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -46,7 +46,7 @@ glidein_cleanup() { # Arguments: # 1: error message early_glidein_failure() { - local error_msg glidein_end_time result final_result final_result_simple final_result_long + local error_msg glidein_end_time result final_result error_msg="$1" log_warn "${error_msg}" sleep "${sleep_time}" @@ -54,14 +54,6 @@ early_glidein_failure() { glidein_end_time="$(date +%s)" ts="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" create_xml -s 4 metric --name failure --ts "$ts" --uri local WN_RESOURCE status ERROR detail ${error_msg} - #result - #result=" WN_RESOURCE - # ERROR - # - # ${error_msg} - # " - #echo "testing xml" - #echo -e $result final_result="$(construct_xml "${result}")" final_result_simple="$(basexml2simplexml "${final_result}")" # have no global section @@ -117,7 +109,7 @@ glidein_exit() { do_report=1 fi # wait a bit in case of error, to reduce lost glideins - let "dl=$(date +%s) + ${sleep_time}" + let "dl=$(date +%s)+${sleep_time}" dlf=$(date --date="@${dl}") add_config_line "GLIDEIN_ADVERTISE_ONLY" "1" add_config_line "GLIDEIN_Failed" "True" diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh old mode 100644 new mode 100755 index eff4a90669..430c3f98ba --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash #*******************************************************************# # glidein_startup.sh # @@ -16,7 +16,6 @@ IFS=$' \t\n' GLOBAL_ARGS="$*" - # GWMS_STARTUP_SCRIPT=$0 GWMS_STARTUP_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" @@ -39,27 +38,30 @@ GWMS_MULTIUSER_GLIDEIN= GWMS_LOGSERVER_ADDRESS='https://fermicloud152.fnal.gov/log' GWMS_MULTIGLIDEIN_CHILDS= +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR=. + # Include all source scripts -source utils_gs_http.sh -source utils_gs_filesystem.sh -source utils_gs_io.sh -source utils_gs_signals.sh -source utils_gs_tarballs.sh -source utils_io.sh -source utils_params.sh -source utils_signals.sh -source utils_tarballs.sh -source utils_xml.sh -source utils_crypto.sh -source glidein_cleanup.sh +source "$GWMS_SOURCEDIR"/utils_gs_http.sh +source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh +source "$GWMS_SOURCEDIR"/utils_gs_io.sh +source "$GWMS_SOURCEDIR"/utils_gs_signals.sh +source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh +source "$GWMS_SOURCEDIR"/utils_io.sh +source "$GWMS_SOURCEDIR"/utils_params.sh +source "$GWMS_SOURCEDIR"/utils_signals.sh +source "$GWMS_SOURCEDIR"/utils_tarballs.sh +source "$GWMS_SOURCEDIR"/utils_xml.sh +source "$GWMS_SOURCEDIR"/utils_crypto.sh +source "$GWMS_SOURCEDIR"/glidein_cleanup.sh export LANG=C ################################ -# Function used to start multiple glideins +# Function used to copy all files from a directory to another +# (to support when needed to start multiple glideins) # Arguments: # 1: prefix of the files to skip -# 2: directory +# 2: destination directory copy_all() { mkdir -p "$2" for f in *; do @@ -113,7 +115,7 @@ do_start_all() { fi } -# TODO (HERE) +# TODO(F): bats tests HERE ################################ # Function used to spawn multiple glideins and wait, if needed @@ -277,337 +279,315 @@ create_glidein_config(){ fi } -################################ -# Block of code used to handle the list of parameters -# params will contain the full list of parameters -# -param_XXX YYY will become "XXX YYY" -#TODO: can use an array instead? -# Transform long options to short ones -#for arg in "$@"; do -# shift -# case "$arg" in -# '-factory') set -- "$@" '-f' ;; -# '-name') set -- "$@" '-n' ;; -# '-entry') set -- "$@" '-e' ;; -# *) set -- "$@" "$arg" ;; -# esac -#done - -#OPTIND=1 -#while getopts "hn:rw" opt -#do -# case "Sopt" in -# 'h') print_usage; exit 0 ;; -# 'n') a=2 ;; -# 'r') rest=true ;; -# 'w') ws=true ;; -# '?') print_usage >&2; exit 1;; -# esac -#done -#shift $(expr $OPTIND- 1) +_main(){ -################################ -parse_options $@ + ################################ + parse_options "$@" -################################ -# Code block used to set the slots_layout -# make sure to have a valid slots_layout -if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then - slots_layout="fixed" -else - slots_layout="partitionable" -fi - -################################ -parse_arguments + ################################ + # Code block used to set the slots_layout + # make sure to have a valid slots_layout + if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then + slots_layout="fixed" + else + slots_layout="partitionable" + fi -################################ -# Code block used to generate the glidein UUID -if command -v uuidgen >/dev/null 2>&1; then - glidein_uuid="$(uuidgen)" -else - glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" -fi + ################################ + parse_arguments -################################ -print_header "@" + ################################ + # Code block used to generate the glidein UUID + if command -v uuidgen >/dev/null 2>&1; then + glidein_uuid="$(uuidgen)" + else + glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" + fi -################################ -spawn_multiple_glideins + ################################ + print_header "$@" -######################################## -# Code block used to make sure nobody else can write my files -# in the Grid world I cannot trust anybody -if ! umask 0022; then - early_glidein_failure "Failed in umask 0022" -fi + ################################ + spawn_multiple_glideins -######################################## -setup_OSG_Globus + ######################################## + # Code block used to make sure nobody else can write my files + # in the Grid world I cannot trust anybody + if ! umask 0022; then + early_glidein_failure "Failed in umask 0022" + fi -######################################## -# Code block used to set the tokens -[ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath -num_gct=0 -for tk in "$(pwd)/credential_"*".idtoken"; do - echo "Setting GLIDEIN_CONDOR_TOKEN to ${tk} " 1>&2 - num_gct=$(( num_gct + 1 )) - export GLIDEIN_CONDOR_TOKEN="${tk}" - fullpath="$(readlink -f "${tk}" )" - if [ $? -eq 0 ]; then - echo "Setting GLIDEIN_CONDOR_TOKEN ${tk} to canonical path ${fullpath}" 1>&2 - export GLIDEIN_CONDOR_TOKEN="${fullpath}" - else - echo "Unable to get canonical path for GLIDEIN_CONDOR_TOKEN ${tk}" 1>&2 + ######################################## + setup_OSG_Globus + + ######################################## + # Code block used to set the tokens + [ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath + num_gct=0 + for tk in "$(pwd)/credential_"*".idtoken"; do + echo "Setting GLIDEIN_CONDOR_TOKEN to ${tk} " 1>&2 + num_gct=$(( num_gct + 1 )) + export GLIDEIN_CONDOR_TOKEN="${tk}" + fullpath="$(readlink -f "${tk}" )" + if [ $? -eq 0 ]; then + echo "Setting GLIDEIN_CONDOR_TOKEN ${tk} to canonical path ${fullpath}" 1>&2 + export GLIDEIN_CONDOR_TOKEN="${fullpath}" + else + echo "Unable to get canonical path for GLIDEIN_CONDOR_TOKEN ${tk}" 1>&2 + fi + done + if [ ! -f "${GLIDEIN_CONDOR_TOKEN}" ] ; then + token_err_msg="problem setting GLIDEIN_CONDOR_TOKEN" + token_err_msg="${token_err_msg} will attempt to recover, but condor IDTOKEN auth may fail" + echo "${token_err_msg}" + echo "${token_err_msg}" 1>&2 + fi + if [ ! "${num_gct}" -eq 1 ] ; then + token_err_msg="WARNING GLIDEIN_CONDOR_TOKEN set ${num_gct} times, should be 1 !" + token_err_msg="${token_err_msg} condor IDTOKEN auth may fail" + echo "${token_err_msg}" + echo "${token_err_msg}" 1>&2 fi -done -if [ ! -f "${GLIDEIN_CONDOR_TOKEN}" ] ; then - token_err_msg="problem setting GLIDEIN_CONDOR_TOKEN" - token_err_msg="${token_err_msg} will attempt to recover, but condor IDTOKEN auth may fail" - echo "${token_err_msg}" - echo "${token_err_msg}" 1>&2 -fi -if [ ! "${num_gct}" -eq 1 ] ; then - token_err_msg="WARNING GLIDEIN_CONDOR_TOKEN set ${num_gct} times, should be 1 !" - token_err_msg="${token_err_msg} condor IDTOKEN auth may fail" - echo "${token_err_msg}" - echo "${token_err_msg}" 1>&2 -fi -######################################## -prepare_workdir + ######################################## + prepare_workdir -######################################## -# extract and source all the data contained at the end of this script as tarball -extract_all_data + ######################################## + # extract and source all the data contained at the end of this script as tarball + extract_all_data -######################################## -wrapper_list="${PWD}/wrapper_list.lst" -touch "${wrapper_list}" + ######################################## + wrapper_list="${PWD}/wrapper_list.lst" + touch "${wrapper_list}" -######################################## -create_glidein_config + ######################################## + create_glidein_config -######################################## -# shellcheck disable=SC2086 -params2file ${params} - -############################################ -# Setup logging -log_init "${glidein_uuid}" "${work_dir}" -# Remove these files, if they are still there -rm -rf tokens.tgz url_dirs.desc tokens -log_setup "${glidein_config}" -echo "Downloading files from Factory and Frontend" -log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" - -##################################### -# Fetch descript and signature files -# disable signature check before I get the signature file itself -# check_signature is global -check_signature=0 - -for gs_id in main entry client client_group -do - if [ -z "${client_repository_url}" ]; then - if [ "${gs_id}" = "client" ]; then - # no client file when no cilent_repository - continue - fi - fi - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_id}" = "client_group" ]; then - # no client group file when no cilent_repository_group - continue - fi - fi - - gs_id_work_dir="$(get_work_dir ${gs_id})" - - # Fetch description file - gs_id_descript_file="$(get_descript_file ${gs_id})" - fetch_file_regular "${gs_id}" "${gs_id_descript_file}" - if ! signature_file_line="$(grep "^signature " "${gs_id_work_dir}/${gs_id_descript_file}")"; then - log_warn "No signature in description file ${gs_id_work_dir}/${gs_id_descript_file} (wc: $(wc < "${gs_id_work_dir}/${gs_id_descript_file}" 2>/dev/null))." - glidein_exit 1 - fi - signature_file=$(echo "${signature_file_line}" | cut -s -f 2-) - - # Fetch signature file - gs_id_signature="$(get_signature ${gs_id})" - fetch_file_regular "${gs_id}" "${signature_file}" - echo "${gs_id_signature} ${signature_file}" > "${gs_id_work_dir}/signature.sha1.test" - if ! (cd "${gs_id_work_dir}" && sha1sum -c signature.sha1.test) 1>&2 ; then - log_warn "Corrupted signature file '${gs_id_work_dir}/${signature_file}'." - glidein_exit 1 - fi - # for simplicity use a fixed name for signature file - mv "${gs_id_work_dir}/${signature_file}" "${gs_id_work_dir}/signature.sha1" -done - -# re-enable for everything else -check_signature=1 - -# Now verify the description was not tampered with -# doing it so late should be fine, since nobody should have been able -# to fake the signature file, even if it faked its name in -# the description file -for gs_id in main entry client client_group -do - if [ -z "${client_repository_url}" ]; then - if [ "${gs_id}" = "client" ]; then - # no client file when no cilent_repository - continue + ######################################## + # shellcheck disable=SC2086 + params2file ${params} + + ############################################ + # Setup logging + log_init "${glidein_uuid}" "${work_dir}" + # Remove these files, if they are still there + rm -rf tokens.tgz url_dirs.desc tokens + log_setup "${glidein_config}" + echo "Downloading files from Factory and Frontend" + log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" + + ##################################### + # Fetch descript and signature files + # disable signature check before I get the signature file itself + # check_signature is global + check_signature=0 + + for gs_id in main entry client client_group + do + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no cilent_repository + continue + fi fi - fi - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_id}" = "client_group" ]; then - # no client group file when no cilent_repository_group - continue + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no cilent_repository_group + continue + fi fi - fi - gs_id_descript_file="$(get_descript_file ${gs_id})" - if ! check_file_signature "${gs_id}" "${gs_id_descript_file}"; then gs_id_work_dir="$(get_work_dir ${gs_id})" - log_warn "Corrupted description file ${gs_id_work_dir}/${gs_id_descript_file}." - glidein_exit 1 - fi -done - -#TODO(F): qui - -################################################### -# get last_script, as it is used by the fetch_file -gs_id_work_dir="$(get_work_dir main)" -gs_id_descript_file="$(get_descript_file main)" -last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" -if [ -z "${last_script}" ]; then - log_warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." - glidein_exit 1 -fi -#cleanup_script="$(grep "^cleanup_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" -cleanup_script=$(grep "^GLIDEIN_CLEANUP_SCRIPT " "${glidein_config}" | cut -d ' ' -f 2-) + # Fetch description file + gs_id_descript_file="$(get_descript_file ${gs_id})" + fetch_file_regular "${gs_id}" "${gs_id_descript_file}" + if ! signature_file_line="$(grep "^signature " "${gs_id_work_dir}/${gs_id_descript_file}")"; then + log_warn "No signature in description file ${gs_id_work_dir}/${gs_id_descript_file} (wc: $(wc < "${gs_id_work_dir}/${gs_id_descript_file}" 2>/dev/null))." + glidein_exit 1 + fi + signature_file=$(echo "${signature_file_line}" | cut -s -f 2-) + + # Fetch signature file + gs_id_signature="$(get_signature ${gs_id})" + fetch_file_regular "${gs_id}" "${signature_file}" + echo "${gs_id_signature} ${signature_file}" > "${gs_id_work_dir}/signature.sha1.test" + if ! (cd "${gs_id_work_dir}" && sha1sum -c signature.sha1.test) 1>&2 ; then + log_warn "Corrupted signature file '${gs_id_work_dir}/${signature_file}'." + glidein_exit 1 + fi + # for simplicity use a fixed name for signature file + mv "${gs_id_work_dir}/${signature_file}" "${gs_id_work_dir}/signature.sha1" + done + + # re-enable for everything else + check_signature=1 + + # Now verify the description was not tampered with + # doing it so late should be fine, since nobody should have been able + # to fake the signature file, even if it faked its name in + # the description file + for gs_id in main entry client client_group + do + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no cilent_repository + continue + fi + fi + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no cilent_repository_group + continue + fi + fi -############################## -# Fetch all the other files -for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" -do - gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" - - if [ -z "${client_repository_url}" ]; then - if [ "${gs_id}" = "client" ]; then - # no client file when no client_repository - continue - fi - fi - if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_id}" = "client_group" ]; then - # no client group file when no client_repository_group - continue - fi + gs_id_descript_file="$(get_descript_file ${gs_id})" + if ! check_file_signature "${gs_id}" "${gs_id_descript_file}"; then + gs_id_work_dir="$(get_work_dir ${gs_id})" + log_warn "Corrupted description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 + fi + done + + #TODO(F): qui + + ################################################### + # get last_script, as it is used by the fetch_file + gs_id_work_dir="$(get_work_dir main)" + gs_id_descript_file="$(get_descript_file main)" + last_script="$(grep "^last_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" + if [ -z "${last_script}" ]; then + log_warn "last_script not in description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 fi + #cleanup_script="$(grep "^cleanup_script " "${gs_id_work_dir}/${gs_id_descript_file}" | cut -s -f 2-)" + cleanup_script=$(grep "^GLIDEIN_CLEANUP_SCRIPT " "${glidein_config}" | cut -d ' ' -f 2-) - gs_file_list_id="$(echo "${gs_file_id}" |awk '{print $2}')" - gs_id_work_dir="$(get_work_dir "${gs_id}")" - gs_id_descript_file="$(get_descript_file "${gs_id}")" + ############################## + # Fetch all the other files + for gs_file_id in "main file_list" "client preentry_file_list" "client_group preentry_file_list" "client aftergroup_preentry_file_list" "entry file_list" "main at_file_list" "client file_list" "client_group file_list" "client aftergroup_file_list" "main after_file_list" + do + gs_id="$(echo "${gs_file_id}" |awk '{print $1}')" - # extract list file name - if ! gs_file_list_line="$(grep "^${gs_file_list_id} " "${gs_id_work_dir}/${gs_id_descript_file}")"; then + if [ -z "${client_repository_url}" ]; then + if [ "${gs_id}" = "client" ]; then + # no client file when no client_repository + continue + fi + fi if [ -z "${client_repository_group_url}" ]; then - if [ "${gs_file_list_id:0:11}" = "aftergroup_" ]; then - # afterfile_.. files optional when no client_repository_group + if [ "${gs_id}" = "client_group" ]; then + # no client group file when no client_repository_group continue fi fi - log_warn "No '${gs_file_list_id}' in description file ${gs_id_work_dir}/${gs_id_descript_file}." - glidein_exit 1 - fi - # space+tab separated file with multiple elements (was: awk '{print $2}', not safe for spaces in file name) - gs_file_list="$(echo "${gs_file_list_line}" | cut -s -f 2 | sed -e 's/[[:space:]]*$//')" - # fetch list file - fetch_file_regular "${gs_id}" "${gs_file_list}" + gs_file_list_id="$(echo "${gs_file_id}" |awk '{print $2}')" - # Fetch files contained in list - # TODO: $file is actually a list, so it cannot be doublequoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here - # shellcheck disable=2086 - while read -r file - do - if [ "${file:0:1}" != "#" ]; then - fetch_file "${gs_id}" $file + gs_id_work_dir="$(get_work_dir "${gs_id}")" + gs_id_descript_file="$(get_descript_file "${gs_id}")" + + # extract list file name + if ! gs_file_list_line="$(grep "^${gs_file_list_id} " "${gs_id_work_dir}/${gs_id_descript_file}")"; then + if [ -z "${client_repository_group_url}" ]; then + if [ "${gs_file_list_id:0:11}" = "aftergroup_" ]; then + # afterfile_.. files optional when no client_repository_group + continue + fi + fi + log_warn "No '${gs_file_list_id}' in description file ${gs_id_work_dir}/${gs_id_descript_file}." + glidein_exit 1 fi - done < "${gs_id_work_dir}/${gs_file_list}" - - # Files to go into the GWMS_PATH - if [ "$gs_file_id" = "main at_file_list" ]; then - # setup here to make them available for other setup scripts - add_to_path "$gwms_bin_dir" - # all available now: gwms-python was in main,file_list; condor_chirp is in main,at_file_list - for file in "gwms-python" "condor_chirp" + # space+tab separated file with multiple elements (was: awk '{print $2}', not safe for spaces in file name) + gs_file_list="$(echo "${gs_file_list_line}" | cut -s -f 2 | sed -e 's/[[:space:]]*$//')" + + # fetch list file + fetch_file_regular "${gs_id}" "${gs_file_list}" + + # Fetch files contained in list + # TODO: $file is actually a list, so it cannot be doublequoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here + # shellcheck disable=2086 + while read -r file do - cp "${gs_id_work_dir}/$file" "$gwms_bin_dir"/ - done - cp -r "${gs_id_work_dir}/lib"/* "$gwms_lib_dir"/ - elif [ "$gs_file_id" = "main after_file_list" ]; then - # in case some library has been added/updated - rsync -ar "${gs_id_work_dir}/lib"/ "$gwms_lib_dir"/ - # new knowns binaries? add a loop like above: for file in ... - elif [[ "$gs_file_id" = client* ]]; then - # TODO: gwms25073 this is a workaround until there is an official designation for setup script fragments - [[ -e "${gs_id_work_dir}/setup_prejob.sh" ]] && { cp "${gs_id_work_dir}/setup_prejob.sh" "$gwms_exec_dir"/prejob/ ; chmod a-x "$gwms_exec_dir"/prejob/setup_prejob.sh ; } + if [ "${file:0:1}" != "#" ]; then + fetch_file "${gs_id}" $file + fi + done < "${gs_id_work_dir}/${gs_file_list}" + + # Files to go into the GWMS_PATH + if [ "$gs_file_id" = "main at_file_list" ]; then + # setup here to make them available for other setup scripts + add_to_path "$gwms_bin_dir" + # all available now: gwms-python was in main,file_list; condor_chirp is in main,at_file_list + for file in "gwms-python" "condor_chirp" + do + cp "${gs_id_work_dir}/$file" "$gwms_bin_dir"/ + done + cp -r "${gs_id_work_dir}/lib"/* "$gwms_lib_dir"/ + elif [ "$gs_file_id" = "main after_file_list" ]; then + # in case some library has been added/updated + rsync -ar "${gs_id_work_dir}/lib"/ "$gwms_lib_dir"/ + # new knowns binaries? add a loop like above: for file in ... + elif [[ "$gs_file_id" = client* ]]; then + # TODO: gwms25073 this is a workaround until there is an official designation for setup script fragments + [[ -e "${gs_id_work_dir}/setup_prejob.sh" ]] && { cp "${gs_id_work_dir}/setup_prejob.sh" "$gwms_exec_dir"/prejob/ ; chmod a-x "$gwms_exec_dir"/prejob/setup_prejob.sh ; } + fi + done + + ############################# + fixup_condor_dir + + ############################## + # Start the glidein main script + add_config_line "GLIDEIN_INITIALIZED" "1" + log_write "glidein_startup.sh" "text" "Starting the glidein main script" "info" + log_write "glidein_startup.sh" "file" "${glidein_config}" "debug" + send_logs_to_remote # checkpoint + echo "# --- Last Script values ---" >> glidein_config + last_startup_time=$(date +%s) + let validation_time=${last_startup_time}-${startup_time} + echo "=== Last script starting $(date) (${last_startup_time}) after validating for ${validation_time} ===" + echo + ON_DIE=0 + trap 'ignore_signal' SIGHUP + trap_with_arg 'on_die' SIGTERM SIGINT SIGQUIT + #trap 'on_die' TERM + #trap 'on_die' INT + gs_id_work_dir=$(get_work_dir main) + "${main_dir}"/error_augment.sh -init + "${gs_id_work_dir}/${last_script}" glidein_config & + wait $! + ret=$? + if [ ${ON_DIE} -eq 1 ]; then + ret=0 + fi + last_startup_end_time=$(date +%s) + "${main_dir}"/error_augment.sh -process ${ret} "${last_script}" "${PWD}" "${gs_id_work_dir}/${last_script} glidein_config" "${last_startup_time}" "${last_startup_end_time}" + "${main_dir}"/error_augment.sh -concat + let last_script_time=${last_startup_end_time}-${last_startup_time} + echo "=== Last script ended $(date) (${last_startup_end_time}) with code ${ret} after ${last_script_time} ===" + echo + if [ ${ret} -ne 0 ]; then + log_warn "Error running '${last_script}'" fi -done - -############################# -fixup_condor_dir - -############################## -# Start the glidein main script -add_config_line "GLIDEIN_INITIALIZED" "1" -log_write "glidein_startup.sh" "text" "Starting the glidein main script" "info" -log_write "glidein_startup.sh" "file" "${glidein_config}" "debug" -send_logs_to_remote # checkpoint -echo "# --- Last Script values ---" >> glidein_config -last_startup_time=$(date +%s) -let validation_time=${last_startup_time}-${startup_time} -echo "=== Last script starting $(date) (${last_startup_time}) after validating for ${validation_time} ===" -echo -ON_DIE=0 -trap 'ignore_signal' SIGHUP -trap_with_arg 'on_die' SIGTERM SIGINT SIGQUIT -#trap 'on_die' TERM -#trap 'on_die' INT -gs_id_work_dir=$(get_work_dir main) -"${main_dir}"/error_augment.sh -init -"${gs_id_work_dir}/${last_script}" glidein_config & -wait $! -ret=$? -if [ ${ON_DIE} -eq 1 ]; then - ret=0 -fi -last_startup_end_time=$(date +%s) -"${main_dir}"/error_augment.sh -process ${ret} "${last_script}" "${PWD}" "${gs_id_work_dir}/${last_script} glidein_config" "${last_startup_time}" "${last_startup_end_time}" -"${main_dir}"/error_augment.sh -concat -let last_script_time=${last_startup_end_time}-${last_startup_time} -echo "=== Last script ended $(date) (${last_startup_end_time}) with code ${ret} after ${last_script_time} ===" -echo -if [ ${ret} -ne 0 ]; then - log_warn "Error running '${last_script}'" -fi -############################# -#Things like periodic scripts might put messages here if they want them printed in the (stderr) logfile -echo "=== Exit messages left by periodic scripts ===" 1>&2 -if [ -f exit_message ]; then - cat exit_message 1>&2 -else - echo "No message left" 1>&2 -fi -echo 1>&2 + ############################# + #Things like periodic scripts might put messages here if they want them printed in the (stderr) logfile + echo "=== Exit messages left by periodic scripts ===" 1>&2 + if [ -f exit_message ]; then + cat exit_message 1>&2 + else + echo "No message left" 1>&2 + fi + echo 1>&2 + + ######################### + # clean up after I finish + glidein_exit ${ret} +} -######################### -# clean up after I finish -glidein_exit ${ret} +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + _main "$@" +fi diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index 23a2a21e3b..b76e570699 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -13,6 +13,8 @@ # 2: option (quiet) # Returns: # 1 in case the md5sum cannot be calculated, or neither the md5sum nor the md5 can be found +# Global: +# res md5wrapper() { local ERROR_RESULT ERROR_RESULT="???" @@ -56,6 +58,7 @@ md5wrapper() { # cfs_signature # cfs_rc # tmp_signname +# check_signature # Returns: # 1 in case of corrupted file check_file_signature() { @@ -93,6 +96,7 @@ check_file_signature() { # Environment variables exported: # X509_USER_PROXY set_proxy_fullpath() { + local fullpath if fullpath="$(readlink -f "${X509_USER_PROXY}")"; then echo "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${fullpath}" 1>&2 export X509_USER_PROXY="${fullpath}" diff --git a/creation/web_base/utils_gs_filesystem.sh b/creation/web_base/utils_gs_filesystem.sh index eaa0b4fa00..6b39a12943 100644 --- a/creation/web_base/utils_gs_filesystem.sh +++ b/creation/web_base/utils_gs_filesystem.sh @@ -10,7 +10,7 @@ ################################ # Function used to automatically determine and setup work directories # Globals: -# targets +# targets (_CONDOR_SCRATCH_DIR, OSG_WN_TMP, TG_NODE_SCRATCH, TG_CLUSTER_SCRATCH, SCRATCH, TMPDIR, TMP, PWD) # work_dir # Returns: # 1 in case you are not allowed to write diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 52ad9234ab..9e3d73458d 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -190,7 +190,7 @@ fetch_file_try() { fft_config_out="$8" if [[ "${fft_config_check}" != "TRUE" ]]; then - # TRUE is a special case, always downloaded and processed + # TRUE is a special case, always be downloaded and processed local fft_get_ss fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) # Stop download and processing if the cond_attr variable is not defined or has a value different from 1 diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index c47fe10177..8804aef98f 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -16,7 +16,7 @@ # Global: # total_time print_tail() { - local final_result_simple, final_result_long, exit_code + local final_result_simple final_result_long exit_code exit_code=$1 final_result_simple="$2" final_result_long="$3" @@ -187,7 +187,7 @@ parse_options(){ -multirestart) multi_glidein_restart="$2";; -param_*) params="$params $(echo "$1" | awk '{print substr($0,8)}') $2";; -h|--help) usage; exit 0;; - *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1 + *) (log_warn "Unknown option $1"; usage; exit 1) 1>&2; exit 1;; esac shift 2 done @@ -198,13 +198,31 @@ parse_options(){ # It allows some parameters to change arguments # Global: # tmp_par +# params +# repository_url +# client_repository_url # repository_entry_url # proxy_url # client_sign_type +# client_descript_file +# multi_glidein +# operation_mode +# descript_file +# descript_entry_file +# glidein_name +# glidein_entry +# sign_id +# sign_entry_id +# sign_type +# client_repository_group_url +# client_descript_group_file +# sleep_time +# set_debug +# OSG_SQUID_LOCATION parse_arguments(){ # multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein tmp_par=$(params_get_simple GLIDEIN_MULTIGLIDEIN "${params}") - [ -n "${tmp_par}" ] && multi_glidein=${tmp_par} + [ -n "${tmp_par}" ] && multi_glidein=${tmp_par} case "${operation_mode}" in nodebug) diff --git a/creation/web_base/utils_gs_tarballs.sh b/creation/web_base/utils_gs_tarballs.sh index 0a876734c8..89448e4f28 100644 --- a/creation/web_base/utils_gs_tarballs.sh +++ b/creation/web_base/utils_gs_tarballs.sh @@ -35,6 +35,8 @@ get_untar_subdir() { fi gus_dir="$(grep -i "^${gus_fname} " "${gus_config_file}" | cut -s -f 2-)" + echo "$(grep -i "^${gus_fname} " "${gus_config_file}")" + echo "${gus_dir}" if [ -z "${gus_dir}" ]; then log_warn "Error, untar dir for '${gus_fname}' cannot be empty." glidein_exit 1 diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh index 53ca4d8c97..479cef452e 100644 --- a/creation/web_base/utils_tarballs.sh +++ b/creation/web_base/utils_tarballs.sh @@ -13,6 +13,8 @@ # Retrieve the specified data, which is appended as tarball # Arguments: # 1: selected file +# Global: +# GWMS_STARTUP_SCRIPT get_data() { sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar xz -O "$1" } @@ -29,6 +31,8 @@ source_data() { ####################################### # Show a list of the payload tarballed files in this script +# Global: +# GWMS_STARTUP_SCRIPT list_data() { sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar tz } diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index dcb4b6ad25..d71ffad25b 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -23,7 +23,7 @@ construct_xml() { } ################################ -# Function used to extract the parent xml fname +# Function used to extract the parent xml fname (element inside the id of OSGTestResult) # Arguments: # 1: exit code extract_parent_fname(){ @@ -172,11 +172,13 @@ create_xml(){ if [[ $1 == "-h" ]] then result="" + echo -e "$result" return 0 fi if [[ $1 == "-t" ]] then - result="\n" + result="" + echo -e "$result" return 0 fi if [[ $1 == "-s" ]] @@ -185,15 +187,15 @@ create_xml(){ shift 2 else xml="" + xml+="\n" fi until [ $# -lt 1 ] do - xml+="\n" case "$1" in OSG|O) add_spaces xml+="" - if [ $2 == "{" ]; then + if [ "$2" == "{" ]; then spaces+=1 end_xml=""$end_xml shift 1 @@ -211,7 +213,7 @@ create_xml(){ OSGShort|OS) add_spaces xml+="" - if [ $2 == "{" ]; then + if [ "$2" == "{" ]; then spaces+=1 end_xml=""$end_xml shift 1 @@ -221,7 +223,7 @@ create_xml(){ operatingenvironment|oe) add_spaces xml+="" - if [ $2 == "{" ]; then + if [ "$2" == "{" ]; then spaces+=1 end_xml=""$end_xml shift 1 @@ -231,9 +233,9 @@ create_xml(){ env|e) add_spaces xml+="/dev/null 2>&1 && pwd )" + # make executables in src/ visible to PATH + PATH="$DIR/../src:$PATH" + source compat.bash + source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_signals.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_tarballs.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_io.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_params.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_signals.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_tarballs.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_xml.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_crypto.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_http.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_filesystem.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/creation/web_base/logging_utils.source + source "$GWMS_SOURCEDIR"/creation/web_base/glidein_cleanup.sh 2>&3 + source "$GWMS_SOURCEDIR"/creation/web_base/add_config_line.source + source "$GWMS_SOURCEDIR"/creation/web_base/glidein_paths.source + load 'mock_gwms_logs' +} + +setup_nameprint() { + if [ "${BATS_TEST_NUMBER}" = 1 ];then + echo "# --- TEST NAME IS $(basename "${BATS_TEST_FILENAME}")" >&3 + fi +} + +@test "glidien_cleanup" { + start_dir="random/stuff/" + run glidein_cleanup 0 + echo "$output" + assert_output --partial "Cannot find ${start_dir} anymore, exiting but without cleanup" + [ "$status" -eq 0 ] + start_dir="/tmp/start_dir/" + mkdir -p /tmp/start_dir/ + GLIDEIN_DEBUG_OPTIONS=",nocleanup," + run glidein_cleanup 0 + assert_output --partial "Skipping cleanup, disabled via GLIDEIN_DEBUG_OPTIONS" + [ "$status" -eq 0 ] + work_dir="/tmp/work_dir/" + mkdir -p /tmp/work_dir/ + GLIDEIN_DEBUG_OPTIONS="" + work_dir_created=1 + run glidein_cleanup 0 + echo "$output" >& 3 + [ "$output" == "" ] + [ "$status" -eq 0 ] + work_dir="/tmp/work_dir/" + mkdir -p /tmp/work_dir/ + glide_local_tmp_dir="/tmp/glide_local_tmp_dir/" + mkdir -p /tmp/glide_local_tmp_dir/ + glide_local_tmp_dir_created=1 + run glidein_cleanup 0 + echo "$output" >& 3 + [ "$output" == "" ] + [ "$status" -eq 0 ] +} + +@test "early_glidein_failure" { + message="random" + sleep_time=1 + let startup_time=$(date +%s) + run early_glidein_failure "${message}" + echo "$output" >&3 + assert_output --partial "WARN" + assert_output --partial "=== Glidein ending" + assert_output --partial "=== XML description of glidein activity ===" + assert_output --partial "=== End XML description of glidein activity ===" + assert_output --partial "=== Encoded XML description of glidein activity ===" + assert_output --partial "=== End encoded XML description of glidein activity ===" + [ "$status" -eq 1 ] +} + +@test "glidein_exit" { + if command -v uuidgen >/dev/null 2>&1; then + glidein_uuid="$(uuidgen)" + else + glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" + fi + work_dir="$(PWD)" + GWMS_SUBDIR=".gwms.d" + GWMS_DIR="${work_dir}/$GWMS_SUBDIR" + mkdir -p "$GWMS_DIR/exec/cleanup" + let startup_time=$(date +%s) + glidein_config="${PWD}/glidein_config" + echo "ADD_CONFIG_LINE_SOURCE ${PWD}/add_config_line.source" > glidein_config + touch "${PWD}/add_config_line.source" + touch glidein_config + log_init "${glidein_uuid}" "${work_dir}" + log_setup "${glidein_config}" + run glidein_exit 0 + assert_output --partial "=== Glidein ending" + assert_output --partial "=== XML description of glidein activity ===" + assert_output --partial "=== End XML description of glidein activity ===" + assert_output --partial "=== Encoded XML description of glidein activity ===" + assert_output --partial "=== End encoded XML description of glidein activity ===" + [ "$status" -eq 0 ] + run glidein_exit 1 + echo "$output" >&3 + assert_output --partial "=== Glidein ending" + [ "$status" -eq 1 ] + rm "${PWD}/add_config_line.source" + rm glidein_config + rm "glidein_config.history" + rm -rf "$GWMS_DIR" +} + +teardown() { + rm -rf /tmp/glide_local_tmp_dir/ + rm -rf /tmp/work_dir/ + rm -rf /tmp/start_dir/ +} diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats new file mode 100755 index 0000000000..3e55c8453f --- /dev/null +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -0,0 +1,100 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +#load 'helper' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + # get the containing directory of this file + # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, + # as those will point to the bats executable's location or the preprocessed file respectively + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + # make executables in src/ visible to PATH + PATH="$DIR/../src:$PATH" + source compat.bash + source "$GWMS_SOURCEDIR"/utils_gs_signals.sh + source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_params.sh + source "$GWMS_SOURCEDIR"/utils_signals.sh + source "$GWMS_SOURCEDIR"/utils_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_xml.sh + source "$GWMS_SOURCEDIR"/utils_crypto.sh + source "$GWMS_SOURCEDIR"/utils_gs_http.sh + source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh + source "$GWMS_SOURCEDIR"/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/logging_utils.source + source "$GWMS_SOURCEDIR"/glidein_cleanup.sh + source "$GWMS_SOURCEDIR"/glidein_startup.sh + source "$GWMS_SOURCEDIR"/add_config_line.source + source "$GWMS_SOURCEDIR"/glidein_paths.source + source ../../build/ci/utils.sh + #load 'mock_gwms_logs' +} + +setup_nameprint() { + if [ "${BATS_TEST_NUMBER}" = 1 ];then + echo "# --- TEST NAME IS $(basename "${BATS_TEST_FILENAME}")" >&3 + fi +} + +@test "copy_all" { + tmp_dir="/tmp/prova" + mkdir -p "$tmp_dir" + cd "$tmp_dir" + touch file1.txt + touch file2.txt + touch file3.txt + touch afile1.txt + target_dir="/tmp/prova2" + mkdir -p "$target_dir" + run copy_all "pfile" "${target_dir}" + [ "$output" == "" ] + [ -f "${target_dir}"/file1.txt ] + [ -f "${target_dir}"/file2.txt ] + [ -f "${target_dir}"/file3.txt ] + [ ! -f "${target_dir}"/pfile1.txt ] + [ "$status" -eq 0 ] + rm -rf "$tmp_dir" + rm -rf "$target_dir" +} + +@test "do_start_all" { + GWMS_SOURCEDIR="../../../creation/web_base" + export GWMS_SOURCEDIR + num_glideins=5 + run do_start_all ${num_glideins} + echo "$output" >& 3 + for i in ${num_glideins}; do + assert_output --partial "Starting glidein ${i} in glidein_dir${i}" + done + assert_output --partial "Started multiple glideins" + # TODO: Missing case of starting multi-glidein using launcher launchall +} + +@test "spawn_multiple_glideins" { + GWMS_SOURCEDIR="../../../creation/web_base" + export GWMS_SOURCEDIR + num_glideins=5 + run spawn_multiple_glideins + echo "$output" >& 3 + assert_output --partial "------ Multi-glidein parent waiting for child processes" + assert_output --partial "------ Exiting multi-glidein parent ----------" + [ "$status" -eq 0 ] + # TODO: Missing case of starting multi-glidein using launcher launchall +} + + +# TODO: FINISH + +@test "add_to_path" { + GWMS_PATH="" + run add_to_path "CIAO" + #assert_output --partial "------ Exiting multi-glidein parent ----------" + [ "$status" -eq 0 ] + # TODO: Missing case of starting multi-glidein using launcher launchall +} diff --git a/test/bats/creation_web_base_utils_crypto.bats b/test/bats/creation_web_base_utils_crypto.bats new file mode 100755 index 0000000000..c753e80556 --- /dev/null +++ b/test/bats/creation_web_base_utils_crypto.bats @@ -0,0 +1,38 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + source "$GWMS_SOURCEDIR"/utils_crypto.sh + source "$GWMS_SOURCEDIR"/utils_io.sh +} + +@test "md5wrapper" { + run md5wrapper "file" + assert_output --partial "???" + assert_output --partial "md5wrapper error: can't calculate md5sum using" + [ "$status" -eq 1 ] + touch "trial" + filename="trial" + run md5wrapper ${filename} + assert_output --regexp "^[0-9a-z]+ ${filename}" + rm "trial" + [ "$status" -eq 0 ] +} + +@test "set_proxy_fullpath" { + X509_USER_PROXY= + run set_proxy_fullpath + assert_output --partial "Unable to get canonical path for X509_USER_PROXY, using ${X509_USER_PROXY}" + mkdir -p "/tmp/trial" + touch "/tmp/trial/x509up_u" + X509_USER_PROXY="/tmp/trial/x509up_u" + run set_proxy_fullpath + assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path /private${X509_USER_PROXY}" + rm -rf "/tmp/trial/" +} diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats new file mode 100755 index 0000000000..f082172a15 --- /dev/null +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -0,0 +1,102 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh +} + +@test "automatic_work_dir" { + _CONDOR_SCRATCH_DIR="/tmp/condor_scratch/" + OSG_WN_TMP="/tmp/osg_wn/" + TG_NODE_SCRATCH="/tmp/tg_node/" + TG_CLUSTER_SCRATCH="/tmp/tg_cluster/" + SCRATCH="/tmp/scratch/" + TMPDIR="/tmp/tmpdir/" + TMP="/tmp/tmp/" + rm -rf ${_CONDOR_SCRATCH_DIR} + rm -rf ${OSG_WN_TMP} + rm -rf ${TG_NODE_SCRATCH} + rm -rf ${TG_CLUSTER_SCRATCH} + rm -rf ${SCRATCH} + rm -rf ${TMPDIR} + rm -rf ${TMP} + run automatic_work_dir + assert_output --partial "Workdir: ${_CONDOR_SCRATCH_DIR} does not exist" + assert_output --partial "Workdir: ${OSG_WN_TMP} does not exist" + assert_output --partial "Workdir: ${TG_NODE_SCRATCH} does not exist" + assert_output --partial "Workdir: ${TG_CLUSTER_SCRATCH} does not exist" + assert_output --partial "Workdir: ${SCRATCH} does not exist" + assert_output --partial "Workdir: ${TMPDIR} does not exist" + assert_output --partial "Workdir: ${TMP} does not exist" + assert_output --partial "Workdir: ${PWD} selected" + [ "$status" == 0 ] + mkdir ${_CONDOR_SCRATCH_DIR} + mkdir ${OSG_WN_TMP} + mkdir ${TG_NODE_SCRATCH} + mkdir ${TG_CLUSTER_SCRATCH} + mkdir ${SCRATCH} + mkdir ${TMPDIR} + mkdir ${TMP} + run automatic_work_dir + assert_output --partial "Workdir: ${_CONDOR_SCRATCH_DIR} selected" + [ "$status" == 0 ] + rm -rf ${_CONDOR_SCRATCH_DIR} + run automatic_work_dir + assert_output --partial "Workdir: ${OSG_WN_TMP} selected" + [ "$status" == 0 ] + mkdir ${_CONDOR_SCRATCH_DIR} + chmod 000 ${_CONDOR_SCRATCH_DIR} + run automatic_work_dir + chmod 777 ${_CONDOR_SCRATCH_DIR} + assert_output --partial "Workdir: not allowed to write to ${_CONDOR_SCRATCH_DIR}" + [ "$status" == 0 ] + rm -rf ${_CONDOR_SCRATCH_DIR} + rm -rf ${OSG_WN_TMP} + rm -rf ${TG_NODE_SCRATCH} + rm -rf ${TG_CLUSTER_SCRATCH} + rm -rf ${SCRATCH} + rm -rf ${TMPDIR} + rm -rf ${TMP} +} + +@test "dir_id" { + GLIDEIN_DEBUG_OPTIONS="" + repository_url="/tmp/repository/" + client_repository_url="/tmp/client_repository/" + run dir_id + [ "$output" == "" ] + [ "$status" == 0 ] + GLIDEIN_DEBUG_OPTIONS="nocleanup" + run dir_id + [ "$output" == "ory/ory/_" ] + [ "$status" == 0 ] +} + +# mock +early_glidein_failure() { + echo "$1" + exit 1 +} + +@test "prepare_workdir" { + work_dir="/tmp/workdir" + rm -rf ${work_dir} + run prepare_workdir + assert_output --partial "Startup dir ${work_dir} does not exist" + [ "$status" == 1 ] + mkdir ${work_dir} + GWMS_SUBDIR="subdir" + touch "tokens.tgz" + touch "url_dirs.desc" + touch "trial.idtoken" + run prepare_workdir + assert_output --partial "Started in ${pwd}" + assert_output --partial "Running in ${work_dir}" + assert_output --partial "copied idtoken" + [ "$status" == 0 ] +} diff --git a/test/bats/creation_web_base_utils_gs_http.bats b/test/bats/creation_web_base_utils_gs_http.bats new file mode 100755 index 0000000000..261581e770 --- /dev/null +++ b/test/bats/creation_web_base_utils_gs_http.bats @@ -0,0 +1,97 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + source "$GWMS_SOURCEDIR"/utils_gs_http.sh + source "$GWMS_SOURCEDIR"/utils_io.sh +} + +@test "get_repository_url" { + run get_repository_url main + [ "$status" -eq 0 ] + run get_repository_url entry + [ "$status" -eq 0 ] + run get_repository_url client + [ "$status" -eq 0 ] + run get_repository_url client_group + [ "$status" -eq 0 ] + id="id" + run get_repository_url id + [ "$status" -eq 1 ] + assert_output --partial "[get_repository_url] Invalid id: ${id}" +} + +#mock +glidein_exit(){ + return 1 +} + +@test "fetch_file" { + run fetch_file 1 2 3 4 5 6 7 8 9 + assert_output --partial "More then 8 arguments, considering the first 8" + run fetch_file 1 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" + run fetch_file 1 2 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" + run fetch_file 1 2 3 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" + run fetch_file 1 2 3 4 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" + run fetch_file 1 2 3 4 5 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" +} + +#mock +add_config_line(){ + echo "Added config line: $1" +} + + +@test "add_periodic_script" { + add_startd_cron_counter=0 + GLIDEIN_PS_=glidein_ps + wrapper="wrapper" + period="period" + cwd="cwd" + fname="fname" + config="config" + ffb_id="ffb_id" + prefix="NOPREFIX" + run add_periodic_script ${wrapper} ${period} ${cwd} ${fname} ${config} ${ffb_id} ${prefix} + assert_output --partial "Added config line: GLIDEIN_condor_config_startd_cron_include" + assert_output --partial "Added config line: # --- Lines starting with ${prefix} are from periodic scripts ---" + [ -f condor_config_startd_cron_include ] + let add_startd_cron_counter=add_startd_cron_counter+1 + grep -Fxq "STARTD_CRON_JOBLIST = \$(STARTD_CRON_JOBLIST) GLIDEIN_PS_${add_startd_cron_counter} + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_MODE = Periodic + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_KILL = True + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_PERIOD = periods + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_EXECUTABLE = wrapper + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_ARGS = config ffb_id GLIDEIN_PS_${add_startd_cron_counter} fname cc_prefix + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_CWD = cwd + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_SLOTS = 1 + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_JOB_LOAD = 0.01" condor_config_startd_cron_include + rm condor_config_startd_cron_include + prefix="cc_prefix" + run add_periodic_script ${wrapper} ${period} ${cwd} ${fname} ${config} ${ffb_id} ${prefix} + assert_output --partial "Added config line: GLIDEIN_condor_config_startd_cron_include" + assert_output --partial "Added config line: # --- Lines starting with ${prefix} are from periodic scripts ---" + [ -f condor_config_startd_cron_include ] + let add_startd_cron_counter=add_startd_cron_counter+1 + grep -Fxq "STARTD_CRON_JOBLIST = \$(STARTD_CRON_JOBLIST) GLIDEIN_PS_${add_startd_cron_counter} + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_MODE = Periodic + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_KILL = True + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_PERIOD = periods + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_EXECUTABLE = wrapper + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_ARGS = config ffb_id GLIDEIN_PS_${add_startd_cron_counter} fname cc_prefix + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_CWD = cwd + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_SLOTS = 1 + STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_JOB_LOAD = 0.01 + STARTD_CRON_GLIDEIN_PS_1_PREFIX = cc_prefix" condor_config_startd_cron_include + rm condor_config_startd_cron_include +} diff --git a/test/bats/creation_web_base_utils_gs_io.bats b/test/bats/creation_web_base_utils_gs_io.bats new file mode 100755 index 0000000000..dc6fccb189 --- /dev/null +++ b/test/bats/creation_web_base_utils_gs_io.bats @@ -0,0 +1,33 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +#load 'helper' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + source "$GWMS_SOURCEDIR"/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_crypto.sh +} + +@test "print_header" { + set_debug=1 + run print_header "" + assert_output --partial "Initial environment" + set_debug=0 + run print_header "" + ! assert_output --partial "Initial environment" +} + +@test "parse_options" { + run parse_options -name -factory + assert_output --partial "You cannot set two consecutive options without specifying the option value!" + [ "$status" -eq 1 ] + run parse_options -error + assert_output --partial "Unknown option" + [ "$status" -eq 1 ] +} diff --git a/test/bats/creation_web_base_utils_gs_tarballs.bats b/test/bats/creation_web_base_utils_gs_tarballs.bats new file mode 100755 index 0000000000..86a5281927 --- /dev/null +++ b/test/bats/creation_web_base_utils_gs_tarballs.bats @@ -0,0 +1,38 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/get_id_selectors.source +} + +#mock +glidein_exit(){ + echo "$1" + exit 1 +} + +@test "get_untar_subdir" { + id="main" + fname="file" + touch glidein_config + run get_untar_subdir ${id} ${fname} + assert_output --partial "Error, cannot find 'UNTAR_CFG_FILE' in glidein_config." + [ "$status" -eq 1 ] + echo "$output" >& 3 + file="trial" + touch ${file} + echo "UNTAR_CFG_FILE ${file}" > glidein_config + run get_untar_subdir ${id} ${fname} + echo "$output" >& 3 + assert_output --partial "Error, untar dir for" + [ "$status" -eq 1 ] + rm glidein_config + rm ${file} +} diff --git a/test/bats/creation_web_base_utils_io.bats b/test/bats/creation_web_base_utils_io.bats new file mode 100755 index 0000000000..884224c6f2 --- /dev/null +++ b/test/bats/creation_web_base_utils_io.bats @@ -0,0 +1,51 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +#load 'helper' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + # get the containing directory of this file + # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, + # as those will point to the bats executable's location or the preprocessed file respectively + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + # make executables in src/ visible to PATH + PATH="$DIR/../src:$PATH" + source compat.bash + source "$GWMS_SOURCEDIR"/utils_io.sh + #load 'mock_gwms_logs' +} + +setup_nameprint() { + if [ "${BATS_TEST_NUMBER}" = 1 ];then + echo "# --- TEST NAME IS $(basename "${BATS_TEST_FILENAME}")" >&3 + fi +} + +@test "log_warn" { + run log_warn "trial" + assert_output --partial "WARN" + [ "$status" == 0 ] +} + +@test "log_debug" { + run log_debug "trial" + assert_output --partial "DEBUG" + [ "$status" == 0 ] +} + +@test "print_header_line" { + run print_header_line "trial" + [ "$output" == "=== trial ===" ] + [ "$status" == 0 ] + run print_header_line "trial" 1 + [ "$output" == "=== trial ===" ] + [ "$status" == 0 ] + run print_header_line "trial" 2 + [ "$output" == "=== trial ===" ] + [ "$status" == 0 ] +} diff --git a/test/bats/creation_web_base_utils_signals.bats b/test/bats/creation_web_base_utils_signals.bats new file mode 100755 index 0000000000..77f5c36900 --- /dev/null +++ b/test/bats/creation_web_base_utils_signals.bats @@ -0,0 +1,45 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +#load 'helper' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + # get the containing directory of this file + # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, + # as those will point to the bats executable's location or the preprocessed file respectively + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + # make executables in src/ visible to PATH + PATH="$DIR/../src:$PATH" + source compat.bash + source "$GWMS_SOURCEDIR"/utils_gs_signals.sh + source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_params.sh + source "$GWMS_SOURCEDIR"/utils_signals.sh + source "$GWMS_SOURCEDIR"/utils_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_xml.sh + source "$GWMS_SOURCEDIR"/utils_crypto.sh + source "$GWMS_SOURCEDIR"/utils_gs_http.sh + source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh + source "$GWMS_SOURCEDIR"/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/logging_utils.source + source "$GWMS_SOURCEDIR"/glidein_cleanup.sh + source "$GWMS_SOURCEDIR"/glidein_startup.sh + source "$GWMS_SOURCEDIR"/add_config_line.source + source "$GWMS_SOURCEDIR"/glidein_paths.source + source ../../build/ci/utils.sh + #load 'mock_gwms_logs' +} + + +@test "trap_with_arg" { + run trap_with_arg 'on_die' SIGINT + # Todo: How to check if handler correctly assigned? + [ "$output" == "" ] + [ "status" == 0] +} diff --git a/test/bats/creation_web_base_utils_xml.bats b/test/bats/creation_web_base_utils_xml.bats new file mode 100755 index 0000000000..e3236fb673 --- /dev/null +++ b/test/bats/creation_web_base_utils_xml.bats @@ -0,0 +1,144 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +#load 'helper' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + # get the containing directory of this file + # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, + # as those will point to the bats executable's location or the preprocessed file respectively + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + # make executables in src/ visible to PATH + PATH="$DIR/../src:$PATH" + source compat.bash + source "$GWMS_SOURCEDIR"/utils_gs_signals.sh + source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_params.sh + source "$GWMS_SOURCEDIR"/utils_signals.sh + source "$GWMS_SOURCEDIR"/utils_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_xml.sh + source "$GWMS_SOURCEDIR"/utils_crypto.sh + source "$GWMS_SOURCEDIR"/utils_gs_http.sh + source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh + source "$GWMS_SOURCEDIR"/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/logging_utils.source + source "$GWMS_SOURCEDIR"/glidein_cleanup.sh + source "$GWMS_SOURCEDIR"/glidein_startup.sh + source "$GWMS_SOURCEDIR"/add_config_line.source + source "$GWMS_SOURCEDIR"/glidein_paths.source + source ../../build/ci/utils.sh + #load 'mock_gwms_logs' +} + +@test "construct_xml" { + run construct_xml "result" + echo "$output" >& 3 + # Todo: How to check if handler correctly assigned? + assert_output --partial '' + [ "$status" -eq 0 ] +} + +@test "extract_parent_fname" { + run extract_parent_fname 0 + assert_output --partial "Unknown" + [ "$status" -eq 0 ] + touch "otrx_output.xml" + echo "" > otrx_output.xml + run extract_parent_fname 0 + assert_output --partial "SUCCESS" + [ "$status" -eq 0 ] + run extract_parent_fname 1 + assert_output --partial "glidein_startup.sh" + [ "$status" -eq 0 ] +} + +@test "extract_parent_xml_detail" { + run extract_parent_xml_detail 0 + assert_output --partial "OK" + assert_output --partial "No detail. Could not find source XML file." + [ "$status" -eq 0 ] + run extract_parent_xml_detail 1 + assert_output --partial "ERROR" + assert_output --partial "No detail. Could not find source XML file." + [ "$status" -eq 0 ] + touch "otrx_output.xml" + echo "Content" > otrx_output.xml + run extract_parent_xml_detail 0 + assert_output --partial "OK" + assert_output --partial "Content" + [ "$status" -eq 0 ] + echo "" > otrx_output.xml + echo "Trial\n" >> otrx_output.xml + run extract_parent_xml_detail 1 + assert_output --partial "ERROR" + assert_output --partial "glidein_startup.sh" + [ "$status" -eq 0 ] +} + +@test "basexml2simplexml" { + argument="Content" + run basexml2simplexml ${argument} + assert_output --partial "${argument}" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + echo "$output" >& 3 + [ "$status" -eq 0 ] +} + +@test "simplexml2longxml" { + argument="Content" + run simplexml2longxml ${argument} + assert_output --partial "${argument}" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + echo "$output" >& 3 + [ "$status" -eq 0 ] +} + +@test "create_xml" { + run create_xml OSG { oe { e --name "Trial" "Trial" t { c "Trial" tS "Trial" tE "Trial" r { s "Trial" m --name "Trial" --ts "Trial" --uri "Trial" "Trial" } d "Trial" } "Trial" } } + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "Trial" + assert_output --partial "" + assert_output --partial "Trial" + assert_output --partial "Trial" + assert_output --partial "Trial" + assert_output --partial "" + assert_output --partial "Trial" + assert_output --partial "Trial" + assert_output --partial "Trial" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "Trial" + assert_output --partial "" + assert_output --partial "" + assert_output --partial "" + [ "$status" -eq 0 ] + run create_xml -h + assert_output --partial "" + run create_xml -t + assert_output --partial "" + run create_xml -s 5 OSG + assert_output --partial " " +} + +teardown() { + rm -f otrx_output.xml +} From 85421cff76424c4a5c4e9be1a9fa06783282dfa3 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 16 Aug 2022 16:52:23 -0500 Subject: [PATCH 039/196] Updated already existing bats test and implemented utils_gs_signals.sh and utils_params.sh bats tests --- creation/web_base/glidein_startup.sh | 18 ------- creation/web_base/utils_gs_filesystem.sh | 18 +++++++ creation/web_base/utils_gs_signals.sh | 1 + .../creation_web_base_glidein_startup.bats | 26 ---------- ...creation_web_base_utils_gs_filesystem.bats | 21 ++++++++ .../creation_web_base_utils_gs_signals.bats | 27 ++++++++++ test/bats/creation_web_base_utils_io.bats | 15 ------ test/bats/creation_web_base_utils_params.bats | 41 +++++++++++++++ .../bats/creation_web_base_utils_signals.bats | 50 ++++++++----------- test/bats/creation_web_base_utils_xml.bats | 27 ---------- 10 files changed, 128 insertions(+), 116 deletions(-) create mode 100755 test/bats/creation_web_base_utils_gs_signals.bats create mode 100755 test/bats/creation_web_base_utils_params.bats diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 430c3f98ba..fdabea1bc8 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -56,24 +56,6 @@ source "$GWMS_SOURCEDIR"/glidein_cleanup.sh export LANG=C -################################ -# Function used to copy all files from a directory to another -# (to support when needed to start multiple glideins) -# Arguments: -# 1: prefix of the files to skip -# 2: destination directory -copy_all() { - mkdir -p "$2" - for f in *; do - [[ -e "${f}" ]] || break # TODO: should this be a continue? - if [[ "${f}" = ${1}* ]]; then - continue - fi - cp -r "${f}" "$2"/ - done -} -# TODO: should it copy also hidden files? - ################################ # Function used to start all glideins # Arguments: diff --git a/creation/web_base/utils_gs_filesystem.sh b/creation/web_base/utils_gs_filesystem.sh index 6b39a12943..17a8326105 100644 --- a/creation/web_base/utils_gs_filesystem.sh +++ b/creation/web_base/utils_gs_filesystem.sh @@ -237,3 +237,21 @@ prepare_workdir(){ # export GLIDEIN_CONDOR_TOKEN="$(pwd)/ticket/${tname}" #fi } + +################################ +# Function used to copy all files from a directory to another +# (to support when needed to start multiple glideins) +# Arguments: +# 1: prefix of the files to skip +# 2: destination directory +copy_all() { + mkdir -p "$2" + for f in *; do + [[ -e "${f}" ]] || break # TODO: should this be a continue? + if [[ "${f}" = ${1}* ]]; then + continue + fi + cp -r "${f}" "$2"/ + done +} +# TODO: should it copy also hidden files? diff --git a/creation/web_base/utils_gs_signals.sh b/creation/web_base/utils_gs_signals.sh index 85fa3c91dc..5f6833d911 100644 --- a/creation/web_base/utils_gs_signals.sh +++ b/creation/web_base/utils_gs_signals.sh @@ -13,6 +13,7 @@ # 1: signal # Globals: # ON_DIE +# GWMS_MULTIGLIDEIN_CHILDS on_die_multi() { echo "Multi-Glidein received signal... shutting down child glideins (forwarding $1 signal to ${GWMS_MULTIGLIDEIN_CHILDS})" 1>&2 ON_DIE=1 diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 3e55c8453f..89c666be53 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -36,32 +36,6 @@ setup () { #load 'mock_gwms_logs' } -setup_nameprint() { - if [ "${BATS_TEST_NUMBER}" = 1 ];then - echo "# --- TEST NAME IS $(basename "${BATS_TEST_FILENAME}")" >&3 - fi -} - -@test "copy_all" { - tmp_dir="/tmp/prova" - mkdir -p "$tmp_dir" - cd "$tmp_dir" - touch file1.txt - touch file2.txt - touch file3.txt - touch afile1.txt - target_dir="/tmp/prova2" - mkdir -p "$target_dir" - run copy_all "pfile" "${target_dir}" - [ "$output" == "" ] - [ -f "${target_dir}"/file1.txt ] - [ -f "${target_dir}"/file2.txt ] - [ -f "${target_dir}"/file3.txt ] - [ ! -f "${target_dir}"/pfile1.txt ] - [ "$status" -eq 0 ] - rm -rf "$tmp_dir" - rm -rf "$target_dir" -} @test "do_start_all" { GWMS_SOURCEDIR="../../../creation/web_base" diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats index f082172a15..fd400e7c9d 100755 --- a/test/bats/creation_web_base_utils_gs_filesystem.bats +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -100,3 +100,24 @@ early_glidein_failure() { assert_output --partial "copied idtoken" [ "$status" == 0 ] } + +@test "copy_all" { + tmp_dir="/tmp/prova" + mkdir -p "$tmp_dir" + cd "$tmp_dir" + touch file1.txt + touch file2.txt + touch file3.txt + touch afile1.txt + target_dir="/tmp/prova2" + mkdir -p "$target_dir" + run copy_all "pfile" "${target_dir}" + [ "$output" == "" ] + [ -f "${target_dir}"/file1.txt ] + [ -f "${target_dir}"/file2.txt ] + [ -f "${target_dir}"/file3.txt ] + [ ! -f "${target_dir}"/pfile1.txt ] + [ "$status" -eq 0 ] + rm -rf "$tmp_dir" + rm -rf "$target_dir" +} diff --git a/test/bats/creation_web_base_utils_gs_signals.bats b/test/bats/creation_web_base_utils_gs_signals.bats new file mode 100755 index 0000000000..e337b138e3 --- /dev/null +++ b/test/bats/creation_web_base_utils_gs_signals.bats @@ -0,0 +1,27 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + source "$GWMS_SOURCEDIR"/utils_gs_signals.sh +} + +@test "on_die_multi" { + sleep 5 & + pid=$! + GWMS_MULTIGLIDEIN_CHILDS=${pid} + run on_die_multi "KILL" + echo "$output" >& 3 + if ! ps -p ${pid} > /dev/null + then + [ 0 -eq 0 ] + else + [ 1 -eq 0 ] + fi + [ "$status" == 0 ] + assert_output --partial "forwarding KILL signal to" +} diff --git a/test/bats/creation_web_base_utils_io.bats b/test/bats/creation_web_base_utils_io.bats index 884224c6f2..ed5d330637 100755 --- a/test/bats/creation_web_base_utils_io.bats +++ b/test/bats/creation_web_base_utils_io.bats @@ -4,27 +4,12 @@ load 'lib/bats-support/load' load 'lib/bats-assert/load' -#load 'helper' - [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" setup () { - # get the containing directory of this file - # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, - # as those will point to the bats executable's location or the preprocessed file respectively - DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" - # make executables in src/ visible to PATH - PATH="$DIR/../src:$PATH" - source compat.bash source "$GWMS_SOURCEDIR"/utils_io.sh - #load 'mock_gwms_logs' } -setup_nameprint() { - if [ "${BATS_TEST_NUMBER}" = 1 ];then - echo "# --- TEST NAME IS $(basename "${BATS_TEST_FILENAME}")" >&3 - fi -} @test "log_warn" { run log_warn "trial" diff --git a/test/bats/creation_web_base_utils_params.bats b/test/bats/creation_web_base_utils_params.bats new file mode 100755 index 0000000000..6c6a69a683 --- /dev/null +++ b/test/bats/creation_web_base_utils_params.bats @@ -0,0 +1,41 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + source "$GWMS_SOURCEDIR"/utils_params.sh + source "$GWMS_SOURCEDIR"/add_config_line.source +} + +@test "params_get_simple" { + run params_get_simple "param2" " param1 value1 param2 value2 " + [ "$output" == "value2" ] + run params_get_simple "param1" " param1 value1 param2 value2 " + [ "$output" == "value1" ] +} + +@test "params_decode" { + run params_decode "param2 .nbsp, .gt, .semicolon, .sclose, .comment, .minus, param3" + [ "$output" == "param2 > ; ] # - param3" ] +} + +#mock +glidein_exit(){ + echo "$1" + exit 1 +} + +@test "params2file" { + file="trial.txt" + glidein_config="glidein_config" + touch ${glidein_config} + run params2file ${file} "param2 .nbsp, .gt, .semicolon, .sclose, .comment, .minus, param3" + [ "$output" == "PARAM_LIST ${file}" ] + grep -Fxq "${file} param2 > ; ] # - param3" ${glidein_config} + rm ${glidein_config} + rm ${glidein_config}.history +} diff --git a/test/bats/creation_web_base_utils_signals.bats b/test/bats/creation_web_base_utils_signals.bats index 77f5c36900..0bfb0e5123 100755 --- a/test/bats/creation_web_base_utils_signals.bats +++ b/test/bats/creation_web_base_utils_signals.bats @@ -4,42 +4,32 @@ load 'lib/bats-support/load' load 'lib/bats-assert/load' -#load 'helper' - [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" setup () { - # get the containing directory of this file - # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, - # as those will point to the bats executable's location or the preprocessed file respectively - DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" - # make executables in src/ visible to PATH - PATH="$DIR/../src:$PATH" - source compat.bash - source "$GWMS_SOURCEDIR"/utils_gs_signals.sh - source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh - source "$GWMS_SOURCEDIR"/utils_io.sh - source "$GWMS_SOURCEDIR"/utils_params.sh source "$GWMS_SOURCEDIR"/utils_signals.sh - source "$GWMS_SOURCEDIR"/utils_tarballs.sh - source "$GWMS_SOURCEDIR"/utils_xml.sh - source "$GWMS_SOURCEDIR"/utils_crypto.sh - source "$GWMS_SOURCEDIR"/utils_gs_http.sh - source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh - source "$GWMS_SOURCEDIR"/utils_gs_io.sh - source "$GWMS_SOURCEDIR"/logging_utils.source - source "$GWMS_SOURCEDIR"/glidein_cleanup.sh - source "$GWMS_SOURCEDIR"/glidein_startup.sh - source "$GWMS_SOURCEDIR"/add_config_line.source - source "$GWMS_SOURCEDIR"/glidein_paths.source - source ../../build/ci/utils.sh - #load 'mock_gwms_logs' } - @test "trap_with_arg" { - run trap_with_arg 'on_die' SIGINT - # Todo: How to check if handler correctly assigned? + run trap 'ignore_signal' SIGTERM SIGINT SIGQUIT + echo "$output" >& 3 + # Todo: How to check if handler has been correctly assigned? [ "$output" == "" ] - [ "status" == 0] + [ "$status" == 0 ] +} + +@test "on_die" { + sleep 5 & + pid=$! + GWMS_MULTIGLIDEIN_CHILDS=${pid} + run on_die "KILL" + echo "$output" >& 3 + if ! ps -p ${pid} > /dev/null + then + [ 0 -eq 0 ] + else + [ 1 -eq 0 ] + fi + [ "$status" == 0 ] + assert_output --partial "forwarding KILL signal" } diff --git a/test/bats/creation_web_base_utils_xml.bats b/test/bats/creation_web_base_utils_xml.bats index e3236fb673..c4b9661053 100755 --- a/test/bats/creation_web_base_utils_xml.bats +++ b/test/bats/creation_web_base_utils_xml.bats @@ -4,42 +4,15 @@ load 'lib/bats-support/load' load 'lib/bats-assert/load' -#load 'helper' - [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" setup () { - # get the containing directory of this file - # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, - # as those will point to the bats executable's location or the preprocessed file respectively - DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" - # make executables in src/ visible to PATH - PATH="$DIR/../src:$PATH" - source compat.bash - source "$GWMS_SOURCEDIR"/utils_gs_signals.sh - source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh - source "$GWMS_SOURCEDIR"/utils_io.sh - source "$GWMS_SOURCEDIR"/utils_params.sh - source "$GWMS_SOURCEDIR"/utils_signals.sh - source "$GWMS_SOURCEDIR"/utils_tarballs.sh source "$GWMS_SOURCEDIR"/utils_xml.sh - source "$GWMS_SOURCEDIR"/utils_crypto.sh - source "$GWMS_SOURCEDIR"/utils_gs_http.sh - source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh - source "$GWMS_SOURCEDIR"/utils_gs_io.sh - source "$GWMS_SOURCEDIR"/logging_utils.source - source "$GWMS_SOURCEDIR"/glidein_cleanup.sh - source "$GWMS_SOURCEDIR"/glidein_startup.sh - source "$GWMS_SOURCEDIR"/add_config_line.source - source "$GWMS_SOURCEDIR"/glidein_paths.source - source ../../build/ci/utils.sh - #load 'mock_gwms_logs' } @test "construct_xml" { run construct_xml "result" echo "$output" >& 3 - # Todo: How to check if handler correctly assigned? assert_output --partial '' [ "$status" -eq 0 ] } From e744629f75b519fc25047470d2311e5ae5e3ec0f Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 16 Aug 2022 17:02:18 -0500 Subject: [PATCH 040/196] Removed redundant code from some bats tests --- .gitignore | 1 + .../creation_web_base_glidein_cleanup.bats | 24 ------------------- .../creation_web_base_glidein_startup.bats | 11 --------- test/bats/creation_web_base_utils_gs_io.bats | 2 -- 4 files changed, 1 insertion(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index bc16862fca..d600e7a187 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ node_modules/* output/* package-lock.json package.json +test/bats/logs/ diff --git a/test/bats/creation_web_base_glidein_cleanup.bats b/test/bats/creation_web_base_glidein_cleanup.bats index e96bf0bab6..e64cf0217f 100755 --- a/test/bats/creation_web_base_glidein_cleanup.bats +++ b/test/bats/creation_web_base_glidein_cleanup.bats @@ -4,40 +4,16 @@ load 'lib/bats-support/load' load 'lib/bats-assert/load' -#load 'helper' - [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR=../.. setup () { - # get the containing directory of this file - # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, - # as those will point to the bats executable's location or the preprocessed file respectively - DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" - # make executables in src/ visible to PATH - PATH="$DIR/../src:$PATH" - source compat.bash - source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_signals.sh - source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_tarballs.sh source "$GWMS_SOURCEDIR"/creation/web_base/utils_io.sh - source "$GWMS_SOURCEDIR"/creation/web_base/utils_params.sh - source "$GWMS_SOURCEDIR"/creation/web_base/utils_signals.sh - source "$GWMS_SOURCEDIR"/creation/web_base/utils_tarballs.sh source "$GWMS_SOURCEDIR"/creation/web_base/utils_xml.sh - source "$GWMS_SOURCEDIR"/creation/web_base/utils_crypto.sh - source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_http.sh - source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_filesystem.sh source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_io.sh source "$GWMS_SOURCEDIR"/creation/web_base/logging_utils.source source "$GWMS_SOURCEDIR"/creation/web_base/glidein_cleanup.sh 2>&3 source "$GWMS_SOURCEDIR"/creation/web_base/add_config_line.source source "$GWMS_SOURCEDIR"/creation/web_base/glidein_paths.source - load 'mock_gwms_logs' -} - -setup_nameprint() { - if [ "${BATS_TEST_NUMBER}" = 1 ];then - echo "# --- TEST NAME IS $(basename "${BATS_TEST_FILENAME}")" >&3 - fi } @test "glidien_cleanup" { diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 89c666be53..8cc03c1397 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -4,18 +4,9 @@ load 'lib/bats-support/load' load 'lib/bats-assert/load' -#load 'helper' - [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" setup () { - # get the containing directory of this file - # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, - # as those will point to the bats executable's location or the preprocessed file respectively - DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" - # make executables in src/ visible to PATH - PATH="$DIR/../src:$PATH" - source compat.bash source "$GWMS_SOURCEDIR"/utils_gs_signals.sh source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh source "$GWMS_SOURCEDIR"/utils_io.sh @@ -33,10 +24,8 @@ setup () { source "$GWMS_SOURCEDIR"/add_config_line.source source "$GWMS_SOURCEDIR"/glidein_paths.source source ../../build/ci/utils.sh - #load 'mock_gwms_logs' } - @test "do_start_all" { GWMS_SOURCEDIR="../../../creation/web_base" export GWMS_SOURCEDIR diff --git a/test/bats/creation_web_base_utils_gs_io.bats b/test/bats/creation_web_base_utils_gs_io.bats index dc6fccb189..4fb8df823c 100755 --- a/test/bats/creation_web_base_utils_gs_io.bats +++ b/test/bats/creation_web_base_utils_gs_io.bats @@ -4,8 +4,6 @@ load 'lib/bats-support/load' load 'lib/bats-assert/load' -#load 'helper' - [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" setup () { From 0a3b5311fa605a3061d4b243456e99d9ef2a5d81 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 17 Aug 2022 14:28:33 -0500 Subject: [PATCH 041/196] Creation of the remaining bats tests and refinement of existing bats tests --- creation/web_base/glidein_startup.sh | 62 +++--------- creation/web_base/utils_gs_filesystem.sh | 19 ++++ creation/web_base/utils_gs_signals.sh | 1 - creation/web_base/utils_gs_tarballs.sh | 15 +++ creation/web_base/utils_tarballs.sh | 2 + creation/web_base/utils_xml.sh | 4 +- .../creation_web_base_glidein_cleanup.bats | 86 ++++++++++++---- .../creation_web_base_glidein_startup.bats | 98 ++++++++++++++++--- test/bats/creation_web_base_utils_crypto.bats | 20 +++- ...creation_web_base_utils_gs_filesystem.bats | 62 +++++++----- .../bats/creation_web_base_utils_gs_http.bats | 52 ++++++---- test/bats/creation_web_base_utils_gs_io.bats | 4 + .../creation_web_base_utils_gs_signals.bats | 33 ++++++- .../creation_web_base_utils_gs_tarballs.bats | 43 ++++++-- test/bats/creation_web_base_utils_io.bats | 11 +-- test/bats/creation_web_base_utils_params.bats | 23 +++-- .../bats/creation_web_base_utils_signals.bats | 16 +-- test/bats/creation_web_base_utils_xml.bats | 25 ++++- 18 files changed, 408 insertions(+), 168 deletions(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index fdabea1bc8..e3855f618b 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -40,20 +40,6 @@ GWMS_MULTIGLIDEIN_CHILDS= [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR=. -# Include all source scripts -source "$GWMS_SOURCEDIR"/utils_gs_http.sh -source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh -source "$GWMS_SOURCEDIR"/utils_gs_io.sh -source "$GWMS_SOURCEDIR"/utils_gs_signals.sh -source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh -source "$GWMS_SOURCEDIR"/utils_io.sh -source "$GWMS_SOURCEDIR"/utils_params.sh -source "$GWMS_SOURCEDIR"/utils_signals.sh -source "$GWMS_SOURCEDIR"/utils_tarballs.sh -source "$GWMS_SOURCEDIR"/utils_xml.sh -source "$GWMS_SOURCEDIR"/utils_crypto.sh -source "$GWMS_SOURCEDIR"/glidein_cleanup.sh - export LANG=C ################################ @@ -74,6 +60,7 @@ do_start_all() { local startup_script startup_script="${GWMS_STARTUP_SCRIPT}" if [[ -n "${multiglidein_launchall}" ]]; then + echo "launchall" echo "Starting multi-glidein using launcher: ${multiglidein_launchall}" # shellcheck disable=SC2086 ${multiglidein_launchall} "${startup_script}" -multirestart 0 ${GLOBAL_ARGS} & @@ -157,40 +144,6 @@ setup_OSG_Globus(){ fi } -######################################## -# Function used to add $1 to GWMS_PATH and update PATH -# Environment: -# GWMS_PATH -# PATH -add_to_path() { - logdebug "Adding to GWMS_PATH: $1" - local old_path - old_path=":${PATH%:}:" - old_path="${old_path//:$GWMS_PATH:/}" - local old_gwms_path - old_gwms_path=":${GWMS_PATH%:}:" - old_gwms_path="${old_gwms_path//:$1:/}" - old_gwms_path="${1%:}:${old_gwms_path#:}" - export GWMS_PATH="${old_gwms_path%:}" - old_path="${GWMS_PATH}:${old_path#:}" - export PATH="${old_path%:}" -} - -######################################## -# Function that removes the native condor tarballs directory to allow factory ops to use native condor tarballs -# All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped -# However the (not used anymore) gwms create_condor_tarball removes that dir -fixup_condor_dir() { - # Check if the condor dir has only one subdir, the one like "condor-9.0.11-1-x86_64_CentOS7-stripped" - # See https://stackoverflow.com/questions/32429333/how-to-test-if-a-linux-directory-contain-only-one-subdirectory-and-no-other-file - if [ $(find "${gs_id_work_dir}/condor" -maxdepth 1 -type d -printf 1 | wc -m) -eq 2 ]; then - echo "Fixing directory structure of condor tarball" - mv "${gs_id_work_dir}"/condor/condor*/* "${gs_id_work_dir}"/condor > /dev/null - else - echo "Condor tarball does not need to be fixed" - fi -} - ######################################## # Function that creates the glidein configuration # Global: @@ -571,5 +524,18 @@ _main(){ } if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + # Include all source scripts + source "$GWMS_SOURCEDIR"/utils_gs_http.sh + source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh + source "$GWMS_SOURCEDIR"/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/utils_gs_signals.sh + source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_params.sh + source "$GWMS_SOURCEDIR"/utils_signals.sh + source "$GWMS_SOURCEDIR"/utils_tarballs.sh + source "$GWMS_SOURCEDIR"/utils_xml.sh + source "$GWMS_SOURCEDIR"/utils_crypto.sh + source "$GWMS_SOURCEDIR"/glidein_cleanup.sh _main "$@" fi diff --git a/creation/web_base/utils_gs_filesystem.sh b/creation/web_base/utils_gs_filesystem.sh index 17a8326105..10936cdad1 100644 --- a/creation/web_base/utils_gs_filesystem.sh +++ b/creation/web_base/utils_gs_filesystem.sh @@ -255,3 +255,22 @@ copy_all() { done } # TODO: should it copy also hidden files? + +######################################## +# Function used to add $1 to GWMS_PATH and update PATH +# Environment: +# GWMS_PATH +# PATH +add_to_path() { + logdebug "Adding to GWMS_PATH: $1" + local old_path + old_path=":${PATH%:}:" + old_path="${old_path//:$GWMS_PATH:/}" + local old_gwms_path + old_gwms_path=":${GWMS_PATH%:}:" + old_gwms_path="${old_gwms_path//:$1:/}" + old_gwms_path="${1%:}:${old_gwms_path#:}" + export GWMS_PATH="${old_gwms_path%:}" + old_path="${GWMS_PATH}:${old_path#:}" + export PATH="${old_path%:}" +} diff --git a/creation/web_base/utils_gs_signals.sh b/creation/web_base/utils_gs_signals.sh index 5f6833d911..85fa3c91dc 100644 --- a/creation/web_base/utils_gs_signals.sh +++ b/creation/web_base/utils_gs_signals.sh @@ -13,7 +13,6 @@ # 1: signal # Globals: # ON_DIE -# GWMS_MULTIGLIDEIN_CHILDS on_die_multi() { echo "Multi-Glidein received signal... shutting down child glideins (forwarding $1 signal to ${GWMS_MULTIGLIDEIN_CHILDS})" 1>&2 ON_DIE=1 diff --git a/creation/web_base/utils_gs_tarballs.sh b/creation/web_base/utils_gs_tarballs.sh index 89448e4f28..0e9c9b1759 100644 --- a/creation/web_base/utils_gs_tarballs.sh +++ b/creation/web_base/utils_gs_tarballs.sh @@ -45,3 +45,18 @@ get_untar_subdir() { echo "${gus_dir}" return 0 } + +######################################## +# Function that removes the native condor tarballs directory to allow factory ops to use native condor tarballs +# All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped +# However the (not used anymore) gwms create_condor_tarball removes that dir +fixup_condor_dir() { + # Check if the condor dir has only one subdir, the one like "condor-9.0.11-1-x86_64_CentOS7-stripped" + # See https://stackoverflow.com/questions/32429333/how-to-test-if-a-linux-directory-contain-only-one-subdirectory-and-no-other-file + if [ $(find "${gs_id_work_dir}/condor" -maxdepth 1 -type d -printf 1 | wc -m) -eq 2 ]; then + echo "Fixing directory structure of condor tarball" + mv "${gs_id_work_dir}"/condor/condor*/* "${gs_id_work_dir}"/condor > /dev/null + else + echo "Condor tarball does not need to be fixed" + fi +} diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh index 479cef452e..5fd2c0e516 100644 --- a/creation/web_base/utils_tarballs.sh +++ b/creation/web_base/utils_tarballs.sh @@ -57,3 +57,5 @@ extract_all_data() { done IFS="${IFS_OLD}" } + +#TODO: Bats test files need to be defined for these functions diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index d71ffad25b..836938cfd1 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -303,7 +303,9 @@ create_xml(){ *) xml+=$1;; esac shift 1 - xml+="\n" + if [ ! $# -eq 0 ]; then + xml+="\n" + fi done result=$xml echo -e "$result" diff --git a/test/bats/creation_web_base_glidein_cleanup.bats b/test/bats/creation_web_base_glidein_cleanup.bats index e64cf0217f..bdbd6910c9 100755 --- a/test/bats/creation_web_base_glidein_cleanup.bats +++ b/test/bats/creation_web_base_glidein_cleanup.bats @@ -14,47 +14,56 @@ setup () { source "$GWMS_SOURCEDIR"/creation/web_base/glidein_cleanup.sh 2>&3 source "$GWMS_SOURCEDIR"/creation/web_base/add_config_line.source source "$GWMS_SOURCEDIR"/creation/web_base/glidein_paths.source + source "$GWMS_SOURCEDIR"/creation/web_base/get_id_selectors.source } @test "glidien_cleanup" { + echo "Checking the case of not existing start_dir..." >& 3 start_dir="random/stuff/" run glidein_cleanup 0 - echo "$output" assert_output --partial "Cannot find ${start_dir} anymore, exiting but without cleanup" [ "$status" -eq 0 ] + echo "Checking the case of disabled cleanup..." >& 3 start_dir="/tmp/start_dir/" - mkdir -p /tmp/start_dir/ + mkdir -p "${start_dir}" GLIDEIN_DEBUG_OPTIONS=",nocleanup," run glidein_cleanup 0 assert_output --partial "Skipping cleanup, disabled via GLIDEIN_DEBUG_OPTIONS" [ "$status" -eq 0 ] + echo "Checking the case of work_dir created..." >& 3 work_dir="/tmp/work_dir/" - mkdir -p /tmp/work_dir/ + mkdir -p "${work_dir}" GLIDEIN_DEBUG_OPTIONS="" work_dir_created=1 run glidein_cleanup 0 - echo "$output" >& 3 - [ "$output" == "" ] + [ -z "$output" ] [ "$status" -eq 0 ] - work_dir="/tmp/work_dir/" - mkdir -p /tmp/work_dir/ + echo "Checking the case of glide_local_tmp_dir created..." >& 3 glide_local_tmp_dir="/tmp/glide_local_tmp_dir/" - mkdir -p /tmp/glide_local_tmp_dir/ + mkdir -p "${glide_local_tmp_dir}" + mkdir -p "${work_dir}" glide_local_tmp_dir_created=1 run glidein_cleanup 0 - echo "$output" >& 3 - [ "$output" == "" ] + [ -z "$output" ] [ "$status" -eq 0 ] } @test "early_glidein_failure" { - message="random" + echo "Checking the correctness of the xml structure..." >& 3 + glidein_cleanup(){ + echo "glidein_cleanup" + return 0 + } + message="message" sleep_time=1 let startup_time=$(date +%s) run early_glidein_failure "${message}" - echo "$output" >&3 assert_output --partial "WARN" + assert_output --partial "glidein_cleanup" assert_output --partial "=== Glidein ending" + assert_output --partial "ERROR" + assert_output --partial "" assert_output --partial "=== XML description of glidein activity ===" assert_output --partial "=== End XML description of glidein activity ===" assert_output --partial "=== Encoded XML description of glidein activity ===" @@ -63,6 +72,7 @@ setup () { } @test "glidein_exit" { + echo "Checking the correctness of the output and gliein_config file in case of exit call with 0 as argument..." >& 3 if command -v uuidgen >/dev/null 2>&1; then glidein_uuid="$(uuidgen)" else @@ -85,19 +95,55 @@ setup () { assert_output --partial "=== End XML description of glidein activity ===" assert_output --partial "=== Encoded XML description of glidein activity ===" assert_output --partial "=== End encoded XML description of glidein activity ===" + grep -iq "ADD_CONFIG_LINE_SOURCE" "${glidein_config}" + grep -iq "GLIDEIN_LOGDIR" "${glidein_config}" + grep -iq "GLIDEIN_STDOUT_LOGFILE" "${glidein_config}" + grep -iq "GLIDEIN_STDERR_LOGFILE" "${glidein_config}" + grep -iq "GLIDEIN_LOG_LOGFILE" "${glidein_config}" + grep -iq "GLIDEIN_LOG_RELATIVE_BASEPATH" "${glidein_config}" + grep -iq "CURL_VERSION" "${glidein_config}" + grep -iq "GLIDEIN_LOG_NO_SEND" "${glidein_config}" + grep -iq "GLIDEIN_LOG_INITIALIZED" "${glidein_config}" [ "$status" -eq 0 ] + echo "Checking the correctness of the output and gliein_config file in case of exit call with a value different than 0 as argument..." >& 3 run glidein_exit 1 - echo "$output" >&3 assert_output --partial "=== Glidein ending" + assert_output --partial "=== XML description of glidein activity ===" + assert_output --partial "=== End XML description of glidein activity ===" + assert_output --partial "=== Encoded XML description of glidein activity ===" + assert_output --partial "=== End encoded XML description of glidein activity ===" [ "$status" -eq 1 ] - rm "${PWD}/add_config_line.source" - rm glidein_config - rm "glidein_config.history" - rm -rf "$GWMS_DIR" + grep -iq "ADD_CONFIG_LINE_SOURCE" "${glidein_config}" + grep -iq "GLIDEIN_LOGDIR" "${glidein_config}" + grep -iq "GLIDEIN_STDOUT_LOGFILE" "${glidein_config}" + grep -iq "GLIDEIN_STDERR_LOGFILE" "${glidein_config}" + grep -iq "GLIDEIN_LOG_LOGFILE" "${glidein_config}" + grep -iq "GLIDEIN_LOG_RELATIVE_BASEPATH" "${glidein_config}" + grep -iq "CURL_VERSION" "${glidein_config}" + grep -iq "GLIDEIN_LOG_NO_SEND" "${glidein_config}" + grep -iq "GLIDEIN_LOG_INITIALIZED" "${glidein_config}" + grep -iq "GLIDEIN_ADVERTISE_ONLY" "${glidein_config}" + grep -iq "GLIDEIN_Failed" "${glidein_config}" + grep -iq "GLIDEIN_EXIT_CODE" "${glidein_config}" + grep -iq "GLIDEIN_ToDie" "${glidein_config}" + grep -iq "GLIDEIN_Expire" "${glidein_config}" + grep -iq "GLIDEIN_LAST_SCRIPT" "${glidein_config}" + grep -iq "GLIDEIN_ADVERTISE_TYPE" "${glidein_config}" + grep -iq "GLIDEIN_FAILURE_REASON" "${glidein_config}" + # TODO: Missing checks of the special cases of report failed } teardown() { - rm -rf /tmp/glide_local_tmp_dir/ - rm -rf /tmp/work_dir/ - rm -rf /tmp/start_dir/ + glide_local_tmp_dir="/tmp/glide_local_tmp_dir/" + work_dir="/tmp/work_dir/" + start_dir="/tmp/start_dir/" + rm -rf "${glide_local_tmp_dir}" + rm -rf "${work_dir}" + rm -rf "${start_dir}" + GWMS_SUBDIR=".gwms.d" + GWMS_DIR="${work_dir}/${GWMS_SUBDIR}" + rm "${PWD}/add_config_line.source" + rm "glidein_config" + rm "glidein_config.history" + rm -rf "${GWMS_DIR}" } diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 8cc03c1397..50a623fd86 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -19,7 +19,6 @@ setup () { source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh source "$GWMS_SOURCEDIR"/utils_gs_io.sh source "$GWMS_SOURCEDIR"/logging_utils.source - source "$GWMS_SOURCEDIR"/glidein_cleanup.sh source "$GWMS_SOURCEDIR"/glidein_startup.sh source "$GWMS_SOURCEDIR"/add_config_line.source source "$GWMS_SOURCEDIR"/glidein_paths.source @@ -27,37 +26,114 @@ setup () { } @test "do_start_all" { + echo "Testing the start of 5 glideins..." >& 3 + # exporting GWMS_SOURCEDIR in order to be visible to the children GWMS_SOURCEDIR="../../../creation/web_base" export GWMS_SOURCEDIR num_glideins=5 run do_start_all ${num_glideins} - echo "$output" >& 3 for i in ${num_glideins}; do assert_output --partial "Starting glidein ${i} in glidein_dir${i}" + [ -d "glidein_dir${i}" ] done assert_output --partial "Started multiple glideins" + [ "$status" -eq 0 ] # TODO: Missing case of starting multi-glidein using launcher launchall } @test "spawn_multiple_glideins" { + echo "Testing the spawning of multiple glideins..." >& 3 GWMS_SOURCEDIR="../../../creation/web_base" export GWMS_SOURCEDIR - num_glideins=5 + multi_glidein=2 + multi_glidein_restart="" run spawn_multiple_glideins - echo "$output" >& 3 assert_output --partial "------ Multi-glidein parent waiting for child processes" assert_output --partial "------ Exiting multi-glidein parent ----------" [ "$status" -eq 0 ] - # TODO: Missing case of starting multi-glidein using launcher launchall + echo "Testing the conditions to be chckedd to not spawn glideins..." >& 3 + multi_glidein="" + run spawn_multiple_glideins + [ "$output" == "" ] + [ "$status" -eq 0 ] + multi_glidein=2 + multi_glidein_restart="yes" + run spawn_multiple_glideins + [ "$output" == "" ] + [ "$status" -eq 0 ] } +@test "setup_OSG_Globus" { + echo "Testing the setup.sh presence with globus path present..." >& 3 + OSG_GRID=/tmp/setup_OSG + mkdir "${OSG_GRID}" + echo "echo setup" > "${OSG_GRID}"/setup.sh + GLOBUS_PATH="location" + run setup_OSG_Globus + [ "$output" == "setup" ] + [ "$status" -eq 0 ] + echo "Testing the cp_1.sh presence with globus path present..." >& 3 + rm "${OSG_GRID}"/setup.sh + GLITE_LOCAL_CUSTOMIZATION_DIR=/tmp/setup_OSG + echo "echo cp_1" > "${GLITE_LOCAL_CUSTOMIZATION_DIR}"/cp_1.sh + run setup_OSG_Globus + [ "$output" == "cp_1" ] + [ "$status" -eq 0 ] + echo "Testing the globus path absence and globus location presence with the globus-user-env.sh file..." >& 3 + GLOBUS_PATH="" + GLOBUS_LOCATION="/tmp/setup_OSG" + mkdir "${GLOBUS_LOCATION}"/etc + echo "echo globus-user-env" > "${GLOBUS_LOCATION}/etc/globus-user-env.sh" + run setup_OSG_Globus + [ "$status" -eq 0 ] + assert_output --partial "cp_1" + assert_output --partial "globus-user-env" + rm "${GLOBUS_LOCATION}/etc/globus-user-env.sh" + echo "Testing the globus path absence and globus location presence without the globus-user-env.sh file..." >& 3 + run setup_OSG_Globus + [ "$status" -eq 0 ] + assert_output --partial "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." + assert_output --partial "Continuing like nothing happened" + GLOBUS_LOCATION="" + echo "Testing the globus path and globus location absence..." >& 3 + run setup_OSG_Globus + assert_output --partial "GLOBUS_LOCATION not defined and could not guess it." + assert_output --partial "Looked in:" + assert_output --partial "/opt/globus/etc/globus-user-env.sh" + assert_output --partial "/osgroot/osgcore/globus/etc/globus-user-env.sh" + assert_output --partial "Continuing like nothing happened" + assert_output --partial "GLOBUS_PATH not defined and ${GLOBUS_LOCATION}/etc/globus-user-env.sh does not exist." + assert_output --partial "Continuing like nothing happened" + [ "$status" -eq 0 ] +} -# TODO: FINISH +#mock +early_glidein_failure() { + echo "$1" + exit 1 +} -@test "add_to_path" { - GWMS_PATH="" - run add_to_path "CIAO" - #assert_output --partial "------ Exiting multi-glidein parent ----------" +@test "create_glidein_config" { + echo "Testing the glidein_config file with no write permissions..." >& 3 + glidein_config="glidein_config" + touch ${glidein_config} + chmod 000 ${glidein_config} + run create_glidein_config + [ "$status" -eq 1 ] + echo "Testing the glidein_config file with write permissions..." >& 3 + assert_output --partial "${PWD}/${glidein_config}: Permission denied" + assert_output --partial "Could not create '${PWD}/${glidein_config}'" + chmod 777 ${glidein_config} + run create_glidein_config [ "$status" -eq 0 ] - # TODO: Missing case of starting multi-glidein using launcher launchall + grep -Fxq "# --- glidein_startup vals ---" ${glidein_config} + grep -Fxq "# --- User Parameters ---" ${glidein_config} +} + + +teardown() { + glidein_config="glidein_config" + rm -rf /tmp/setup_OSG + rm -f ${glidein_config} + rm -rf glidein_dir*/ } diff --git a/test/bats/creation_web_base_utils_crypto.bats b/test/bats/creation_web_base_utils_crypto.bats index c753e80556..dc57ee634d 100755 --- a/test/bats/creation_web_base_utils_crypto.bats +++ b/test/bats/creation_web_base_utils_crypto.bats @@ -4,7 +4,6 @@ load 'lib/bats-support/load' load 'lib/bats-assert/load' - [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" setup () { @@ -13,26 +12,37 @@ setup () { } @test "md5wrapper" { + echo "Testing the call to md5wrapper with non-valid file..." >& 3 run md5wrapper "file" assert_output --partial "???" assert_output --partial "md5wrapper error: can't calculate md5sum using" [ "$status" -eq 1 ] - touch "trial" - filename="trial" + echo "Testing the call to md5wrapper with valid file..." >& 3 + touch "/tmp/trial_file" + filename="/tmp/trial_file" run md5wrapper ${filename} assert_output --regexp "^[0-9a-z]+ ${filename}" - rm "trial" - [ "$status" -eq 0 ] + [ "$status" -eq 0 ] } @test "set_proxy_fullpath" { + echo "Testing the call to set_proxy_fullpath with non-valid user proxy..." >& 3 X509_USER_PROXY= run set_proxy_fullpath assert_output --partial "Unable to get canonical path for X509_USER_PROXY, using ${X509_USER_PROXY}" + [ "$status" -eq 0 ] + echo "Testing the call to set_proxy_fullpath with valid user proxy..." >& 3 mkdir -p "/tmp/trial" touch "/tmp/trial/x509up_u" X509_USER_PROXY="/tmp/trial/x509up_u" run set_proxy_fullpath assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path /private${X509_USER_PROXY}" + [ "$status" -eq 0 ] +} + +#TODO: check_file_signature test + +teardown(){ + rm -f "/tmp/trial_file" rm -rf "/tmp/trial/" } diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats index fd400e7c9d..1a2dcc2b7e 100755 --- a/test/bats/creation_web_base_utils_gs_filesystem.bats +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -8,6 +8,7 @@ load 'lib/bats-assert/load' setup () { source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh + source ../../build/ci/utils.sh } @test "automatic_work_dir" { @@ -18,13 +19,7 @@ setup () { SCRATCH="/tmp/scratch/" TMPDIR="/tmp/tmpdir/" TMP="/tmp/tmp/" - rm -rf ${_CONDOR_SCRATCH_DIR} - rm -rf ${OSG_WN_TMP} - rm -rf ${TG_NODE_SCRATCH} - rm -rf ${TG_CLUSTER_SCRATCH} - rm -rf ${SCRATCH} - rm -rf ${TMPDIR} - rm -rf ${TMP} + echo "Testing the correctness with non-existing work directories..." >& 3 run automatic_work_dir assert_output --partial "Workdir: ${_CONDOR_SCRATCH_DIR} does not exist" assert_output --partial "Workdir: ${OSG_WN_TMP} does not exist" @@ -34,7 +29,8 @@ setup () { assert_output --partial "Workdir: ${TMPDIR} does not exist" assert_output --partial "Workdir: ${TMP} does not exist" assert_output --partial "Workdir: ${PWD} selected" - [ "$status" == 0 ] + [ "$status" -eq 0 ] + echo "Testing the correctness with existing work directories..." >& 3 mkdir ${_CONDOR_SCRATCH_DIR} mkdir ${OSG_WN_TMP} mkdir ${TG_NODE_SCRATCH} @@ -45,36 +41,40 @@ setup () { run automatic_work_dir assert_output --partial "Workdir: ${_CONDOR_SCRATCH_DIR} selected" [ "$status" == 0 ] + echo "Testing the correctness with non-existing first target work directory..." >& 3 rm -rf ${_CONDOR_SCRATCH_DIR} run automatic_work_dir assert_output --partial "Workdir: ${OSG_WN_TMP} selected" [ "$status" == 0 ] + echo "Testing the correctness with non-writable work directory..." >& 3 mkdir ${_CONDOR_SCRATCH_DIR} chmod 000 ${_CONDOR_SCRATCH_DIR} run automatic_work_dir chmod 777 ${_CONDOR_SCRATCH_DIR} assert_output --partial "Workdir: not allowed to write to ${_CONDOR_SCRATCH_DIR}" - [ "$status" == 0 ] - rm -rf ${_CONDOR_SCRATCH_DIR} - rm -rf ${OSG_WN_TMP} - rm -rf ${TG_NODE_SCRATCH} - rm -rf ${TG_CLUSTER_SCRATCH} - rm -rf ${SCRATCH} - rm -rf ${TMPDIR} - rm -rf ${TMP} + [ "$status" -eq 0 ] + rm -rf "${_CONDOR_SCRATCH_DIR}" + rm -rf "${OSG_WN_TMP}" + rm -rf "${TG_NODE_SCRATCH}" + rm -rf "${TG_CLUSTER_SCRATCH}" + rm -rf "${SCRATCH}" + rm -rf "${TMPDIR}" + rm -rf "${TMP}" } @test "dir_id" { + echo "Testing the correctness with no debug options..." >& 3 GLIDEIN_DEBUG_OPTIONS="" repository_url="/tmp/repository/" client_repository_url="/tmp/client_repository/" run dir_id [ "$output" == "" ] - [ "$status" == 0 ] + [ "$status" -eq 0 ] + echo "Testing the correctness with 'nocleanup' debug option..." >& 3 GLIDEIN_DEBUG_OPTIONS="nocleanup" run dir_id [ "$output" == "ory/ory/_" ] - [ "$status" == 0 ] + [ "$status" -eq 0 ] } # mock @@ -84,12 +84,14 @@ early_glidein_failure() { } @test "prepare_workdir" { + rm -rf "${work_dir}" + echo "Testing the function with non-existing work directory..." >& 3 work_dir="/tmp/workdir" - rm -rf ${work_dir} run prepare_workdir assert_output --partial "Startup dir ${work_dir} does not exist" - [ "$status" == 1 ] - mkdir ${work_dir} + [ "$status" -eq 1 ] + echo "Testing the function with existing work directory..." >& 3 + mkdir "${work_dir}" GWMS_SUBDIR="subdir" touch "tokens.tgz" touch "url_dirs.desc" @@ -98,10 +100,13 @@ early_glidein_failure() { assert_output --partial "Started in ${pwd}" assert_output --partial "Running in ${work_dir}" assert_output --partial "copied idtoken" - [ "$status" == 0 ] + [ "$status" -eq 0 ] + rm "trial.idtoken" + rm -rf "${work_dir}" } @test "copy_all" { + echo "Testing the correctness with some trial files..." >& 3 tmp_dir="/tmp/prova" mkdir -p "$tmp_dir" cd "$tmp_dir" @@ -121,3 +126,16 @@ early_glidein_failure() { rm -rf "$tmp_dir" rm -rf "$target_dir" } + +@test "add_to_path" { + echo "Testing the addition of an element to the path..." >& 3 + GWMS_PATH="/tmp/gwms_path" + PATH="/tmp/path" + OLD_GWMS_PATH=${GWMS_PATH} + OLD_PATH=${PATH} + element="element" + add_to_path "${element}" + [ "${PATH}" == "${element}:${OLD_GWMS_PATH}:${OLD_PATH}" ] + [ "${GWMS_PATH}" == "${element}:${OLD_GWMS_PATH}" ] + PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" +} diff --git a/test/bats/creation_web_base_utils_gs_http.bats b/test/bats/creation_web_base_utils_gs_http.bats index 261581e770..eceb44a2f4 100755 --- a/test/bats/creation_web_base_utils_gs_http.bats +++ b/test/bats/creation_web_base_utils_gs_http.bats @@ -12,6 +12,7 @@ setup () { } @test "get_repository_url" { + echo "Testing the possible options of arguments..." >& 3 run get_repository_url main [ "$status" -eq 0 ] run get_repository_url entry @@ -20,32 +21,13 @@ setup () { [ "$status" -eq 0 ] run get_repository_url client_group [ "$status" -eq 0 ] + echo "Testing a possible wrong argument..." >& 3 id="id" run get_repository_url id [ "$status" -eq 1 ] assert_output --partial "[get_repository_url] Invalid id: ${id}" } -#mock -glidein_exit(){ - return 1 -} - -@test "fetch_file" { - run fetch_file 1 2 3 4 5 6 7 8 9 - assert_output --partial "More then 8 arguments, considering the first 8" - run fetch_file 1 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" - run fetch_file 1 2 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" - run fetch_file 1 2 3 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" - run fetch_file 1 2 3 4 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" - run fetch_file 1 2 3 4 5 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" -} - #mock add_config_line(){ echo "Added config line: $1" @@ -53,6 +35,7 @@ add_config_line(){ @test "add_periodic_script" { + echo "Testing the config line content in case of NOPREFIX..." >& 3 add_startd_cron_counter=0 GLIDEIN_PS_=glidein_ps wrapper="wrapper" @@ -77,6 +60,7 @@ add_config_line(){ STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_SLOTS = 1 STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_JOB_LOAD = 0.01" condor_config_startd_cron_include rm condor_config_startd_cron_include + echo "Testing the config line content in case of cc_prefix..." >& 3 prefix="cc_prefix" run add_periodic_script ${wrapper} ${period} ${cwd} ${fname} ${config} ${ffb_id} ${prefix} assert_output --partial "Added config line: GLIDEIN_condor_config_startd_cron_include" @@ -93,5 +77,31 @@ add_config_line(){ STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_SLOTS = 1 STARTD_CRON_GLIDEIN_PS_${add_startd_cron_counter}_JOB_LOAD = 0.01 STARTD_CRON_GLIDEIN_PS_1_PREFIX = cc_prefix" condor_config_startd_cron_include - rm condor_config_startd_cron_include +} + +#mock +glidein_exit(){ + return 1 +} + +@test "fetch_file" { + echo "Testing different possible numbers of arguments..." >& 3 + run fetch_file 1 2 3 4 5 6 7 8 9 + assert_output --partial "More then 8 arguments, considering the first 8" + run fetch_file 1 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" + run fetch_file 1 2 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" + run fetch_file 1 2 3 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" + run fetch_file 1 2 3 4 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" + run fetch_file 1 2 3 4 5 + assert_output --partial "Not enough arguments in fetch_file, 8 expected" +} + +#TODO: Test fetch_file_try, fetch_file_base, perform_wget, perform_curl + +teardown() { + rm -f condor_config_startd_cron_include } diff --git a/test/bats/creation_web_base_utils_gs_io.bats b/test/bats/creation_web_base_utils_gs_io.bats index 4fb8df823c..f39195fddb 100755 --- a/test/bats/creation_web_base_utils_gs_io.bats +++ b/test/bats/creation_web_base_utils_gs_io.bats @@ -13,18 +13,22 @@ setup () { } @test "print_header" { + echo "Testing the printing in case of debug set..." >& 3 set_debug=1 run print_header "" assert_output --partial "Initial environment" + echo "Testing the printing in case of debug not set..." >& 3 set_debug=0 run print_header "" ! assert_output --partial "Initial environment" } @test "parse_options" { + echo "Testing the option parsing in case of consequent options with no value..." >& 3 run parse_options -name -factory assert_output --partial "You cannot set two consecutive options without specifying the option value!" [ "$status" -eq 1 ] + echo "Testing the printing in case of unknown option..." >& 3 run parse_options -error assert_output --partial "Unknown option" [ "$status" -eq 1 ] diff --git a/test/bats/creation_web_base_utils_gs_signals.bats b/test/bats/creation_web_base_utils_gs_signals.bats index e337b138e3..b28d8c0650 100755 --- a/test/bats/creation_web_base_utils_gs_signals.bats +++ b/test/bats/creation_web_base_utils_gs_signals.bats @@ -11,17 +11,40 @@ setup () { } @test "on_die_multi" { + echo "Testing the function killing one child..." >& 3 sleep 5 & pid=$! - GWMS_MULTIGLIDEIN_CHILDS=${pid} + GWMS_MULTIGLIDEIN_CHILDS="${pid}" run on_die_multi "KILL" - echo "$output" >& 3 if ! ps -p ${pid} > /dev/null then - [ 0 -eq 0 ] + [ true ] else - [ 1 -eq 0 ] + [ false ] fi - [ "$status" == 0 ] + [ "$status" -eq 0 ] assert_output --partial "forwarding KILL signal to" + echo "Testing the function killing more than one children..." >& 3 + sleep 6 & + pid=$! + sleep 5 & + pid2=$! + GWMS_MULTIGLIDEIN_CHILDS="${pid} ${pid2}" + run on_die_multi "KILL" + if ! ps -p ${pid} > /dev/null; then + [ true ] + else + [ false ] + fi + if ! ps -p ${pid2} > /dev/null; then + [ true ] + else + [ false ] + fi + [ "$status" -eq 0 ] + assert_output --partial "forwarding KILL signal to" + echo "Testing the function killing no children..." >& 3 + GWMS_MULTIGLIDEIN_CHILDS= + run on_die_multi "KILL" + [ "$status" -eq 0 ] } diff --git a/test/bats/creation_web_base_utils_gs_tarballs.bats b/test/bats/creation_web_base_utils_gs_tarballs.bats index 86a5281927..db58d95107 100755 --- a/test/bats/creation_web_base_utils_gs_tarballs.bats +++ b/test/bats/creation_web_base_utils_gs_tarballs.bats @@ -19,20 +19,43 @@ glidein_exit(){ } @test "get_untar_subdir" { + echo "Testing the absence of 'UNTAR_CFG_FILE' in glidein_config..." >& 3 id="main" fname="file" - touch glidein_config - run get_untar_subdir ${id} ${fname} + glidein_config="glidein_config" + touch "${glidein_config}" + run get_untar_subdir "${id}" "${fname}" assert_output --partial "Error, cannot find 'UNTAR_CFG_FILE' in glidein_config." [ "$status" -eq 1 ] - echo "$output" >& 3 + echo "Testing the case of untar empty directory..." >& 3 file="trial" - touch ${file} - echo "UNTAR_CFG_FILE ${file}" > glidein_config - run get_untar_subdir ${id} ${fname} - echo "$output" >& 3 + touch "${file}" + echo "UNTAR_CFG_FILE ${file}" > "${glidein_config}" + run get_untar_subdir "${id}" "${fname}" assert_output --partial "Error, untar dir for" - [ "$status" -eq 1 ] - rm glidein_config - rm ${file} + [ "$status" -eq 1 ] + # TODO: Handle the case of correct directory +} + +@test "fixup_condor_dir" { + echo "Testing the case of condor tarballs that do not need to be fixed..." >& 3 + gs_id_work_dir="/tmp/gs/" + mkdir "${gs_id_work_dir}" + mkdir "${gs_id_work_dir}/condor" + mkdir "${gs_id_work_dir}/condor/condor_1/" + mkdir "${gs_id_work_dir}/condor/condor_2/" + mkdir "${gs_id_work_dir}/condor/condor_3/" + run fixup_condor_dir + assert_output --partial "Condor tarball does not need to be fixed" + echo "Testing the case of condor tarballs that needs to be fixed..." >& 3 + rm -rf "${gs_id_work_dir}/condor/condor_3/" + run fixup_condor_dir + # assert_output --partial "Fixing directory structure of condor tarball" + # printf not working +} + +teardown() { + rm -rf "${gs_id_work_dir}" + rm -f "${glidein_config}" + rm -f "${file}" } diff --git a/test/bats/creation_web_base_utils_io.bats b/test/bats/creation_web_base_utils_io.bats index ed5d330637..06e83b412d 100755 --- a/test/bats/creation_web_base_utils_io.bats +++ b/test/bats/creation_web_base_utils_io.bats @@ -10,27 +10,26 @@ setup () { source "$GWMS_SOURCEDIR"/utils_io.sh } - @test "log_warn" { run log_warn "trial" assert_output --partial "WARN" - [ "$status" == 0 ] + [ "$status" -eq 0 ] } @test "log_debug" { run log_debug "trial" assert_output --partial "DEBUG" - [ "$status" == 0 ] + [ "$status" -eq 0 ] } @test "print_header_line" { run print_header_line "trial" [ "$output" == "=== trial ===" ] - [ "$status" == 0 ] + [ "$status" -eq 0 ] run print_header_line "trial" 1 [ "$output" == "=== trial ===" ] - [ "$status" == 0 ] + [ "$status" -eq 0 ] run print_header_line "trial" 2 [ "$output" == "=== trial ===" ] - [ "$status" == 0 ] + [ "$status" -eq 0 ] } diff --git a/test/bats/creation_web_base_utils_params.bats b/test/bats/creation_web_base_utils_params.bats index 6c6a69a683..53fcee1a29 100755 --- a/test/bats/creation_web_base_utils_params.bats +++ b/test/bats/creation_web_base_utils_params.bats @@ -12,15 +12,20 @@ setup () { } @test "params_get_simple" { + echo "Testing the pickup of values associated to parameters..." >& 3 run params_get_simple "param2" " param1 value1 param2 value2 " [ "$output" == "value2" ] run params_get_simple "param1" " param1 value1 param2 value2 " [ "$output" == "value1" ] + echo "Testing the pickup of values associated to non-existing parameters..." >& 3 + run params_get_simple "param2" " " + [ "$output" == "" ] } @test "params_decode" { - run params_decode "param2 .nbsp, .gt, .semicolon, .sclose, .comment, .minus, param3" - [ "$output" == "param2 > ; ] # - param3" ] + echo "Testing the correctness of the decoding..." >& 3 + run params_decode "param2 .nbsp, .gt, .semicolon, .sclose, .comment, .minus, .tilde, .not, .question, .star, param3" + [ "$output" == "param2 > ; ] # - ~ ! ? * param3" ] } #mock @@ -30,12 +35,16 @@ glidein_exit(){ } @test "params2file" { + echo "Testing the correctness of the decoding and conversion to file..." >& 3 file="trial.txt" glidein_config="glidein_config" - touch ${glidein_config} - run params2file ${file} "param2 .nbsp, .gt, .semicolon, .sclose, .comment, .minus, param3" + touch "${glidein_config}" + run params2file "${file}" "param2 .nbsp, .gt, .semicolon, .sclose, .comment, .minus, param3" [ "$output" == "PARAM_LIST ${file}" ] - grep -Fxq "${file} param2 > ; ] # - param3" ${glidein_config} - rm ${glidein_config} - rm ${glidein_config}.history + grep -Fxq "${file} param2 > ; ] # - param3" "${glidein_config}" +} + +teardown() { + rm -f "${glidein_config}" + rm -f "${glidein_config}.history" } diff --git a/test/bats/creation_web_base_utils_signals.bats b/test/bats/creation_web_base_utils_signals.bats index 0bfb0e5123..017e9df8f0 100755 --- a/test/bats/creation_web_base_utils_signals.bats +++ b/test/bats/creation_web_base_utils_signals.bats @@ -11,25 +11,25 @@ setup () { } @test "trap_with_arg" { + echo "Testing the assignment of the handler to some signals..." >& 3 run trap 'ignore_signal' SIGTERM SIGINT SIGQUIT - echo "$output" >& 3 - # Todo: How to check if handler has been correctly assigned? + # Todo: How to check if the handler has been correctly assigned? [ "$output" == "" ] - [ "$status" == 0 ] + [ "$status" -eq 0 ] } @test "on_die" { + echo "Testing the launch of a signal to a process..." >& 3 sleep 5 & pid=$! - GWMS_MULTIGLIDEIN_CHILDS=${pid} + GWMS_MULTIGLIDEIN_CHILDS="${pid}" run on_die "KILL" - echo "$output" >& 3 if ! ps -p ${pid} > /dev/null then - [ 0 -eq 0 ] + [ true ] else - [ 1 -eq 0 ] + [ false] fi - [ "$status" == 0 ] + [ "$status" -eq 0 ] assert_output --partial "forwarding KILL signal" } diff --git a/test/bats/creation_web_base_utils_xml.bats b/test/bats/creation_web_base_utils_xml.bats index c4b9661053..75d0c243bf 100755 --- a/test/bats/creation_web_base_utils_xml.bats +++ b/test/bats/creation_web_base_utils_xml.bats @@ -11,41 +11,56 @@ setup () { } @test "construct_xml" { + echo "Testing the presence of xml fields in the output..." >& 3 run construct_xml "result" - echo "$output" >& 3 assert_output --partial '' + assert_output --partial '' + assert_output --partial '' + assert_output --partial '' + assert_output --partial '' + assert_output --partial '' + assert_output --partial '' + assert_output --partial '' + assert_output --partial '' [ "$status" -eq 0 ] } @test "extract_parent_fname" { + echo "Testing a non-valid parent to extract..." >& 3 run extract_parent_fname 0 assert_output --partial "Unknown" [ "$status" -eq 0 ] + echo "Testing a valid parent to extract with exitcode = 0 ..." >& 3 touch "otrx_output.xml" echo "" > otrx_output.xml run extract_parent_fname 0 assert_output --partial "SUCCESS" [ "$status" -eq 0 ] + echo "Testing a valid parent to extract with exitcode = 1 ..." >& 3 run extract_parent_fname 1 assert_output --partial "glidein_startup.sh" [ "$status" -eq 0 ] } @test "extract_parent_xml_detail" { + echo "Testing a call with non-valid XML file and exit code 0..." >& 3 run extract_parent_xml_detail 0 assert_output --partial "OK" assert_output --partial "No detail. Could not find source XML file." [ "$status" -eq 0 ] + echo "Testing a call with non-valid XML file and exit code 1.." >& 3 run extract_parent_xml_detail 1 assert_output --partial "ERROR" assert_output --partial "No detail. Could not find source XML file." [ "$status" -eq 0 ] + echo "Testing a call with valid XML file and exit code 0..." >& 3 touch "otrx_output.xml" echo "Content" > otrx_output.xml run extract_parent_xml_detail 0 assert_output --partial "OK" assert_output --partial "Content" [ "$status" -eq 0 ] + echo "Testing a call with non-valid XML file and exit code 1..." >& 3 echo "" > otrx_output.xml echo "Trial\n" >> otrx_output.xml run extract_parent_xml_detail 1 @@ -56,6 +71,7 @@ setup () { } @test "basexml2simplexml" { + echo "Testing the addition of env tags..." >& 3 argument="Content" run basexml2simplexml ${argument} assert_output --partial "${argument}" @@ -64,11 +80,11 @@ setup () { assert_output --partial "" assert_output --partial "" assert_output --partial "" - echo "$output" >& 3 [ "$status" -eq 0 ] } @test "simplexml2longxml" { + echo "Testing the addition of env tags..." >& 3 argument="Content" run simplexml2longxml ${argument} assert_output --partial "${argument}" @@ -79,11 +95,11 @@ setup () { assert_output --partial "" assert_output --partial "" assert_output --partial "" - echo "$output" >& 3 [ "$status" -eq 0 ] } @test "create_xml" { + echo "Testing the creation of an xml element with all tags..." >& 3 run create_xml OSG { oe { e --name "Trial" "Trial" t { c "Trial" tS "Trial" tE "Trial" r { s "Trial" m --name "Trial" --ts "Trial" --uri "Trial" "Trial" } d "Trial" } "Trial" } } assert_output --partial "" assert_output --partial "" @@ -104,10 +120,13 @@ setup () { assert_output --partial "" assert_output --partial "" [ "$status" -eq 0 ] + echo "Testing the creation of an xml header..." >& 3 run create_xml -h assert_output --partial "" + echo "Testing the creation of an xml tail..." >& 3 run create_xml -t assert_output --partial "" + echo "Testing the creation of an inner xml (with spaces)..." >& 3 run create_xml -s 5 OSG assert_output --partial " " } From 2122b075e908a07985ee841855b2e86292f3fd72 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 17 Aug 2022 14:57:38 -0500 Subject: [PATCH 042/196] Refinement of existing bats tests --- creation/web_base/utils_crypto.sh | 2 +- test/bats/creation_web_base_glidein_cleanup.bats | 6 +++--- test/bats/creation_web_base_glidein_startup.bats | 5 ----- test/bats/creation_web_base_utils_crypto.bats | 2 +- test/bats/creation_web_base_utils_gs_filesystem.bats | 3 --- 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index b76e570699..50ca1155b3 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -83,7 +83,7 @@ check_file_signature() { log_warn "File ${cfs_desc_fname} is corrupted." rm -f "${tmp_signname}" return 1 - fi + f rm -f "${tmp_signname}" echo "Signature OK for ${cfs_id}:${cfs_fname}." 1>&2 fi diff --git a/test/bats/creation_web_base_glidein_cleanup.bats b/test/bats/creation_web_base_glidein_cleanup.bats index bdbd6910c9..926c7fef7a 100755 --- a/test/bats/creation_web_base_glidein_cleanup.bats +++ b/test/bats/creation_web_base_glidein_cleanup.bats @@ -142,8 +142,8 @@ teardown() { rm -rf "${start_dir}" GWMS_SUBDIR=".gwms.d" GWMS_DIR="${work_dir}/${GWMS_SUBDIR}" - rm "${PWD}/add_config_line.source" - rm "glidein_config" - rm "glidein_config.history" + rm -f "${PWD}/add_config_line.source" + rm -f "glidein_config" + rm -f "glidein_config.history" rm -rf "${GWMS_DIR}" } diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 50a623fd86..5f9bf8b271 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -114,12 +114,7 @@ early_glidein_failure() { } @test "create_glidein_config" { - echo "Testing the glidein_config file with no write permissions..." >& 3 glidein_config="glidein_config" - touch ${glidein_config} - chmod 000 ${glidein_config} - run create_glidein_config - [ "$status" -eq 1 ] echo "Testing the glidein_config file with write permissions..." >& 3 assert_output --partial "${PWD}/${glidein_config}: Permission denied" assert_output --partial "Could not create '${PWD}/${glidein_config}'" diff --git a/test/bats/creation_web_base_utils_crypto.bats b/test/bats/creation_web_base_utils_crypto.bats index dc57ee634d..849f865379 100755 --- a/test/bats/creation_web_base_utils_crypto.bats +++ b/test/bats/creation_web_base_utils_crypto.bats @@ -36,7 +36,7 @@ setup () { touch "/tmp/trial/x509up_u" X509_USER_PROXY="/tmp/trial/x509up_u" run set_proxy_fullpath - assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path /private${X509_USER_PROXY}" + assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${X509_USER_PROXY}" [ "$status" -eq 0 ] } diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats index 1a2dcc2b7e..1f58f5d997 100755 --- a/test/bats/creation_web_base_utils_gs_filesystem.bats +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -48,10 +48,7 @@ setup () { [ "$status" == 0 ] echo "Testing the correctness with non-writable work directory..." >& 3 mkdir ${_CONDOR_SCRATCH_DIR} - chmod 000 ${_CONDOR_SCRATCH_DIR} run automatic_work_dir - chmod 777 ${_CONDOR_SCRATCH_DIR} - assert_output --partial "Workdir: not allowed to write to ${_CONDOR_SCRATCH_DIR}" [ "$status" -eq 0 ] rm -rf "${_CONDOR_SCRATCH_DIR}" rm -rf "${OSG_WN_TMP}" From e15bac02a247abc24afe6dcd18931bb15079ce22 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 17 Aug 2022 15:04:14 -0500 Subject: [PATCH 043/196] Refinement of existing bats tests --- .../bats/creation_web_base_glidein_cleanup.bats | 2 +- .../creation_web_base_utils_gs_filesystem.bats | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/test/bats/creation_web_base_glidein_cleanup.bats b/test/bats/creation_web_base_glidein_cleanup.bats index 926c7fef7a..39427a179e 100755 --- a/test/bats/creation_web_base_glidein_cleanup.bats +++ b/test/bats/creation_web_base_glidein_cleanup.bats @@ -78,7 +78,7 @@ setup () { else glidein_uuid="$(od -x -w32 -N32 /dev/urandom | awk 'NR==1{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}')" fi - work_dir="$(PWD)" + work_dir="$(pwd)" GWMS_SUBDIR=".gwms.d" GWMS_DIR="${work_dir}/$GWMS_SUBDIR" mkdir -p "$GWMS_DIR/exec/cleanup" diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats index 1f58f5d997..502c64f49b 100755 --- a/test/bats/creation_web_base_utils_gs_filesystem.bats +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -50,13 +50,6 @@ setup () { mkdir ${_CONDOR_SCRATCH_DIR} run automatic_work_dir [ "$status" -eq 0 ] - rm -rf "${_CONDOR_SCRATCH_DIR}" - rm -rf "${OSG_WN_TMP}" - rm -rf "${TG_NODE_SCRATCH}" - rm -rf "${TG_CLUSTER_SCRATCH}" - rm -rf "${SCRATCH}" - rm -rf "${TMPDIR}" - rm -rf "${TMP}" } @test "dir_id" { @@ -136,3 +129,13 @@ early_glidein_failure() { [ "${GWMS_PATH}" == "${element}:${OLD_GWMS_PATH}" ] PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" } + +teardown() { + rm -rf "${_CONDOR_SCRATCH_DIR}" + rm -rf "${OSG_WN_TMP}" + rm -rf "${TG_NODE_SCRATCH}" + rm -rf "${TG_CLUSTER_SCRATCH}" + rm -rf "${SCRATCH}" + rm -rf "${TMPDIR}" + rm -rf "${TMP}" +} From 676ce74a112e9bf09d342796d53696deb0c6174e Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 17 Aug 2022 15:08:01 -0500 Subject: [PATCH 044/196] Fixed missing end of if statement --- creation/web_base/utils_crypto.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index 50ca1155b3..dcd7bff710 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -86,6 +86,7 @@ check_file_signature() { f rm -f "${tmp_signname}" echo "Signature OK for ${cfs_id}:${cfs_fname}." 1>&2 + fi fi fi return 0 From a2dd1b179b92fcf1a04c07c85f373c71795c0957 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 17 Aug 2022 16:18:55 -0500 Subject: [PATCH 045/196] Changes that fixes some errors --- creation/web_base/glidein_startup.sh | 2 -- creation/web_base/utils_gs_io.sh | 3 +++ creation/web_base/utils_gs_tarballs.sh | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index e3855f618b..6c82648bce 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -84,7 +84,6 @@ do_start_all() { fi } -# TODO(F): bats tests HERE ################################ # Function used to spawn multiple glideins and wait, if needed @@ -385,7 +384,6 @@ _main(){ fi done - #TODO(F): qui ################################################### # get last_script, as it is used by the fetch_file diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index 8804aef98f..d70378f0bd 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -153,6 +153,9 @@ print_header(){ parse_options(){ params="" while [ $# -gt 0 ] + if [[ $1 != "-"* ]]; then + break + fi if [[ $2 == "-"* ]]; then (log_warn "Wrong argument: $2 for option $1."; log_warn "You cannot set two consecutive options without specifying the option value!"; usage; exit 1) 1>&2; exit 1 fi diff --git a/creation/web_base/utils_gs_tarballs.sh b/creation/web_base/utils_gs_tarballs.sh index 0e9c9b1759..6f5e33ec70 100644 --- a/creation/web_base/utils_gs_tarballs.sh +++ b/creation/web_base/utils_gs_tarballs.sh @@ -35,8 +35,6 @@ get_untar_subdir() { fi gus_dir="$(grep -i "^${gus_fname} " "${gus_config_file}" | cut -s -f 2-)" - echo "$(grep -i "^${gus_fname} " "${gus_config_file}")" - echo "${gus_dir}" if [ -z "${gus_dir}" ]; then log_warn "Error, untar dir for '${gus_fname}' cannot be empty." glidein_exit 1 From 26106d44f750aa7c232ad9c82809a1cbbcdcdad6 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 17 Aug 2022 16:48:49 -0500 Subject: [PATCH 046/196] Added the removal of the logs directory generated during the test --- test/bats/creation_web_base_glidein_cleanup.bats | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bats/creation_web_base_glidein_cleanup.bats b/test/bats/creation_web_base_glidein_cleanup.bats index 39427a179e..cfe0929017 100755 --- a/test/bats/creation_web_base_glidein_cleanup.bats +++ b/test/bats/creation_web_base_glidein_cleanup.bats @@ -146,4 +146,5 @@ teardown() { rm -f "glidein_config" rm -f "glidein_config.history" rm -rf "${GWMS_DIR}" + rm -rf "logs/" } From 4f999121f0920dc0cbe790e11da6322e62914c71 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Aug 2022 19:24:49 +0000 Subject: [PATCH 047/196] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/factory/troubleshooting.html | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/doc/factory/troubleshooting.html b/doc/factory/troubleshooting.html index e34f73ac59..c33a96e309 100644 --- a/doc/factory/troubleshooting.html +++ b/doc/factory/troubleshooting.html @@ -269,7 +269,8 @@

Security Overview

>my_identity="frontend_identity@myfactory.example.org" ... + > + ...
<security Security Overview >/DC=org/DC=doegrids/OU=Services/CN=glidein/myfrontend\.example\.org " ...
- <credential type="grid_proxy" - security_class="frontend" ...

+ <credential type="grid_proxy" + security_class="frontend" ... +

Factory config (on myfactory.example.org)
<frontend @@ -302,12 +304,18 @@

Security Overview

myfrontend.example.org and myfactory.example.org)
GSI " ^ - \/DC\=org\/DC\=doegrids\/OU\=Services\/CN\=glidein\/myfactory\.example\.org$ " + \/DC\=org\/DC\=doegrids\/OU\=Services\/CN\=glidein\/myfactory\.example\.org$ " factory_identity - \/DC\=org\/DC\=doegrids\/OU\=Services\/CN\=glidein\/myfrontend\.example\.org$ " + \/DC\=org\/DC\=doegrids\/OU\=Services\/CN\=glidein\/myfrontend\.example\.org$ " frontend_identity - + For a visual representation of the JWT configuration that must match, see the below: From 65f885da782f64ef76377f1b8c753d5eb7b97724 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 18 Aug 2022 14:57:34 -0500 Subject: [PATCH 048/196] Added the tests regarding the write permission --- test/bats/creation_web_base_glidein_startup.bats | 14 ++++++++++++-- .../creation_web_base_utils_gs_filesystem.bats | 13 +++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 5f9bf8b271..86ac665239 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -26,7 +26,8 @@ setup () { } @test "do_start_all" { - echo "Testing the start of 5 glideins..." >& 3 + skip + echo "Testing the spawning of 5 glideins..." >& 3 # exporting GWMS_SOURCEDIR in order to be visible to the children GWMS_SOURCEDIR="../../../creation/web_base" export GWMS_SOURCEDIR @@ -41,7 +42,8 @@ setup () { # TODO: Missing case of starting multi-glidein using launcher launchall } -@test "spawn_multiple_glideins" { +@test "spawn_multiple_glideins" { + skip echo "Testing the spawning of multiple glideins..." >& 3 GWMS_SOURCEDIR="../../../creation/web_base" export GWMS_SOURCEDIR @@ -115,6 +117,14 @@ early_glidein_failure() { @test "create_glidein_config" { glidein_config="glidein_config" + # if not a root user + if [ "$EUID" -ne 0 ]; then + echo "Testing the glidein_config file with no write permissions..." >& 3 + touch ${glidein_config} + chmod 000 ${glidein_config} + run create_glidein_config + [ "$status" -eq 1 ] + fi echo "Testing the glidein_config file with write permissions..." >& 3 assert_output --partial "${PWD}/${glidein_config}: Permission denied" assert_output --partial "Could not create '${PWD}/${glidein_config}'" diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats index 502c64f49b..251625893d 100755 --- a/test/bats/creation_web_base_utils_gs_filesystem.bats +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -46,10 +46,15 @@ setup () { run automatic_work_dir assert_output --partial "Workdir: ${OSG_WN_TMP} selected" [ "$status" == 0 ] - echo "Testing the correctness with non-writable work directory..." >& 3 - mkdir ${_CONDOR_SCRATCH_DIR} - run automatic_work_dir - [ "$status" -eq 0 ] + if [ "$EUID" -ne 0 ]; then + echo "Testing the correctness with non-writable work directory..." >& 3 + chmod 000 ${_CONDOR_SCRATCH_DIR} + run automatic_work_dir + assert_output --partial "Workdir: not allowed to write to ${_CONDOR_SCRATCH_DIR}" + chmod 777 ${_CONDOR_SCRATCH_DIR} + run automatic_work_dir + [ "$status" -eq 0 ] + fi } @test "dir_id" { From 6fe9d3488a43cc752929fbed5ce0b9db4167a42a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Aug 2022 19:58:23 +0000 Subject: [PATCH 049/196] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/bats/creation_web_base_glidein_startup.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 86ac665239..ed266b538c 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -42,7 +42,7 @@ setup () { # TODO: Missing case of starting multi-glidein using launcher launchall } -@test "spawn_multiple_glideins" { +@test "spawn_multiple_glideins" { skip echo "Testing the spawning of multiple glideins..." >& 3 GWMS_SOURCEDIR="../../../creation/web_base" From b52480e05f3aef9c4df7e40c9286750f30c70fe6 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 18 Aug 2022 15:06:41 -0500 Subject: [PATCH 050/196] Added the tests regarding the write permission --- test/bats/creation_web_base_glidein_startup.bats | 2 -- test/bats/creation_web_base_utils_gs_filesystem.bats | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 86ac665239..b0fb12fcad 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -26,7 +26,6 @@ setup () { } @test "do_start_all" { - skip echo "Testing the spawning of 5 glideins..." >& 3 # exporting GWMS_SOURCEDIR in order to be visible to the children GWMS_SOURCEDIR="../../../creation/web_base" @@ -43,7 +42,6 @@ setup () { } @test "spawn_multiple_glideins" { - skip echo "Testing the spawning of multiple glideins..." >& 3 GWMS_SOURCEDIR="../../../creation/web_base" export GWMS_SOURCEDIR diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats index 251625893d..6d09f59874 100755 --- a/test/bats/creation_web_base_utils_gs_filesystem.bats +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -47,6 +47,7 @@ setup () { assert_output --partial "Workdir: ${OSG_WN_TMP} selected" [ "$status" == 0 ] if [ "$EUID" -ne 0 ]; then + mkdir ${_CONDOR_SCRATCH_DIR} echo "Testing the correctness with non-writable work directory..." >& 3 chmod 000 ${_CONDOR_SCRATCH_DIR} run automatic_work_dir From 28e9f0accadcd6fc38f355d24f1098f878ea8018 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Aug 2022 20:08:08 +0000 Subject: [PATCH 051/196] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/bats/creation_web_base_glidein_startup.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index d24e616e09..5f6cb2336d 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -42,7 +42,7 @@ setup () { # TODO: Missing case of starting multi-glidein using launcher launchall } -@test "spawn_multiple_glideins" { +@test "spawn_multiple_glideins" { echo "Testing the spawning of multiple glideins..." >& 3 GWMS_SOURCEDIR="../../../creation/web_base" export GWMS_SOURCEDIR From a5c4804571ff2aac45092c3dc47301f80b3226ef Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 18 Aug 2022 15:47:30 -0500 Subject: [PATCH 052/196] Cleaned up code (removed some echos and fixed some local variables) --- creation/web_base/glidein_cleanup.sh | 22 +++++++++++++++++----- creation/web_base/glidein_startup.sh | 12 +++++++----- creation/web_base/utils_crypto.sh | 1 - creation/web_base/utils_gs_io.sh | 16 +++++++++++++--- creation/web_base/utils_xml.sh | 6 ++---- 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 612c72d0cf..74de7eb7bb 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -45,8 +45,12 @@ glidein_cleanup() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: error message +# Global: +# final_result +# final_result_simple +# final_result_long early_glidein_failure() { - local error_msg glidein_end_time result final_result + local error_msg glidein_end_time error_msg="$1" log_warn "${error_msg}" sleep "${sleep_time}" @@ -68,9 +72,17 @@ early_glidein_failure() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: exit code +# Global: +# final_result +# final_result_simple +# final_result_long +# global_result +# report_failed +# factory_report_failes +# ge_last_script_name +# do_report glidein_exit() { - local exit_code final_result final_result_simple final_result_long global_result ge_last_script_name result report_failed - local factory_report_failes factory_collector do_report dlf condor_vars_file main_work_dir + local exit_code factory_collector dlf condor_vars_file main_work_dir exit_code=$1 # Removed lines about $lock_file (lock file for whole machine) not present elsewhere gwms_process_scripts "$GWMS_DIR" cleanup "${glidein_config}" @@ -92,9 +104,9 @@ glidein_exit() { if [ -z "${report_failed}" ]; then report_failed="NEVER" fi - + factory_report_failed=$(grep -i "^GLIDEIN_Factory_Report_Failed " "${glidein_config}" | cut -d ' ' -f 2-) - + if [ -z "${factory_report_failed}" ]; then factory_collector=$(grep -i "^GLIDEIN_Factory_Collector " "${glidein_config}" | cut -d ' ' -f 2-) if [ -z "${factory_collector}" ]; then diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 6c82648bce..2119764069 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -60,7 +60,6 @@ do_start_all() { local startup_script startup_script="${GWMS_STARTUP_SCRIPT}" if [[ -n "${multiglidein_launchall}" ]]; then - echo "launchall" echo "Starting multi-glidein using launcher: ${multiglidein_launchall}" # shellcheck disable=SC2086 ${multiglidein_launchall} "${startup_script}" -multirestart 0 ${GLOBAL_ARGS} & @@ -258,6 +257,7 @@ _main(){ # Code block used to set the tokens [ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath num_gct=0 + for tk in "$(pwd)/credential_"*".idtoken"; do echo "Setting GLIDEIN_CONDOR_TOKEN to ${tk} " 1>&2 num_gct=$(( num_gct + 1 )) @@ -290,23 +290,25 @@ _main(){ # extract and source all the data contained at the end of this script as tarball extract_all_data - ######################################## + ##################################### wrapper_list="${PWD}/wrapper_list.lst" touch "${wrapper_list}" - ######################################## + ##################################### create_glidein_config - ######################################## + ##################################### # shellcheck disable=SC2086 params2file ${params} - ############################################ + ##################################### # Setup logging log_init "${glidein_uuid}" "${work_dir}" # Remove these files, if they are still there rm -rf tokens.tgz url_dirs.desc tokens log_setup "${glidein_config}" + + ##################################### echo "Downloading files from Factory and Frontend" log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index dcd7bff710..60d35cac26 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -35,7 +35,6 @@ md5wrapper() { fi [ -n "${ONLY_SUM}" ] && executable="md5 -q \"$1\"" || executable="md5 \"$1\"" fi - local res # Flagged by some checkers but OK if ! res="$(eval "${executable}" 2>/dev/null)"; then echo "${ERROR_RESULT}" diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index d70378f0bd..e70b421682 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -16,11 +16,10 @@ # Global: # total_time print_tail() { - local final_result_simple final_result_long exit_code + local final_result_simple final_result_long exit_code glidein_end_time exit_code=$1 final_result_simple="$2" final_result_long="$3" - local glidein_end_time glidein_end_time=$(date +%s) let total_time=${glidein_end_time}-${startup_time} print_header_line "Glidein ending $(date) (${glidein_end_time}) with code ${exit_code} after ${total_time}" @@ -28,7 +27,6 @@ print_tail() { print_header_line "XML description of glidein activity" echo "${final_result_simple}" | grep -v "" print_header_line "End XML description of glidein activity" - echo "" 1>&2 print_header_line "Encoded XML description of glidein activity" 2 echo "${final_result_long}" | gzip --stdout - | b64uuencode 1>&2 @@ -246,27 +244,32 @@ parse_arguments(){ if [ -z "${descript_file}" ]; then log_warn "Missing descript fname." usage + exit 1 fi if [ -z "${descript_entry_file}" ]; then log_warn "Missing descript fname for entry." usage + exit 1 fi if [ -z "${glidein_name}" ]; then log_warn "Missing gliden name." usage + exit 1 fi if [ -z "${glidein_entry}" ]; then log_warn "Missing glidein entry name." usage + exit 1 fi if [ -z "${repository_url}" ]; then log_warn "Missing Web URL." usage + exit 1 fi repository_entry_url="${repository_url}/entry_${glidein_entry}" @@ -288,11 +291,13 @@ parse_arguments(){ if [ -z "${sign_id}" ]; then log_warn "Missing signature." usage + exit 1 fi if [ -z "${sign_entry_id}" ]; then log_warn "Missing entry signature." usage + exit 1 fi if [ -z "${sign_type}" ]; then @@ -302,6 +307,7 @@ parse_arguments(){ if [ "${sign_type}" != "sha1" ]; then log_warn "Unsupported signtype ${sign_type} found." usage + exit 1 fi if [ -n "${client_repository_url}" ]; then @@ -313,11 +319,13 @@ parse_arguments(){ if [ "${client_sign_type}" != "sha1" ]; then log_warn "Unsupported clientsigntype ${client_sign_type} found." usage + exit 1 fi if [ -z "${client_descript_file}" ]; then log_warn "Missing client descript fname." usage + exit 1 fi if [ -n "${client_repository_group_url}" ]; then @@ -325,11 +333,13 @@ parse_arguments(){ if [ -z "${client_group}" ]; then log_warn "Missing client group name." usage + exit 1 fi if [ -z "${client_descript_group_file}" ]; then log_warn "Missing client descript fname for group." usage + exit 1 fi fi fi diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 836938cfd1..23daacda39 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -11,7 +11,7 @@ # Arguments: # 1: result construct_xml() { - local result + local result tStart tEnd cmd result="$1" local glidein_end_time glidein_end_time="$(date +%s)" @@ -19,7 +19,6 @@ construct_xml() { tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date --date=@"${glidein_end_time}" +%Y-%m-%dT%H:%M:%S%:z)" create_xml OSG --id glidein_startup.sh { oe { e --name cwd "${start_dir}" } t { c "${cmd}" tS "${tStart}" tE "${tEnd}" } "${result}" } - echo -e "$result" } ################################ @@ -54,7 +53,6 @@ extract_parent_xml_detail() { # file exists and is not 0 size last_result="$(cat otrx_output.xml)" if [ "${exitcode}" -eq 0 ]; then - #create_xml -s 2 result { status OK } echo " " echo " OK" #propagate metrics as well @@ -120,7 +118,6 @@ simplexml2longxml() { echo "${final_result_simple}" | awk 'BEGIN{fr=1;}{if (fr==1) print $0}/' echo ' ' echo "${global_result}" | awk '{print " " $0}' @@ -143,6 +140,7 @@ simplexml2longxml() { # Global: # xml add_spaces(){ + local c for (( c=1; c<=spaces; c++ )) do xml+=" " From 63d8eff5eb6d5c2157298038a1729143312715a1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Aug 2022 20:48:49 +0000 Subject: [PATCH 053/196] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- creation/web_base/glidein_cleanup.sh | 4 ++-- creation/web_base/glidein_startup.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 74de7eb7bb..010897cb13 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -104,9 +104,9 @@ glidein_exit() { if [ -z "${report_failed}" ]; then report_failed="NEVER" fi - + factory_report_failed=$(grep -i "^GLIDEIN_Factory_Report_Failed " "${glidein_config}" | cut -d ' ' -f 2-) - + if [ -z "${factory_report_failed}" ]; then factory_collector=$(grep -i "^GLIDEIN_Factory_Collector " "${glidein_config}" | cut -d ' ' -f 2-) if [ -z "${factory_collector}" ]; then diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 2119764069..b69da7a650 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -257,7 +257,7 @@ _main(){ # Code block used to set the tokens [ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath num_gct=0 - + for tk in "$(pwd)/credential_"*".idtoken"; do echo "Setting GLIDEIN_CONDOR_TOKEN to ${tk} " 1>&2 num_gct=$(( num_gct + 1 )) @@ -307,7 +307,7 @@ _main(){ # Remove these files, if they are still there rm -rf tokens.tgz url_dirs.desc tokens log_setup "${glidein_config}" - + ##################################### echo "Downloading files from Factory and Frontend" log_write "glidein_startup.sh" "text" "Downloading file from Factory and Frontend" "debug" From 42ab4cefbd5c6d4e470ca1ad07c8004eed42609b Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 18 Aug 2022 16:46:11 -0500 Subject: [PATCH 054/196] Cleaned up code (fixed the scope of some variables) --- creation/web_base/glidein_cleanup.sh | 5 +++-- creation/web_base/glidein_startup.sh | 5 ++--- creation/web_base/utils_gs_http.sh | 20 +++++++++++++++++--- creation/web_base/utils_gs_io.sh | 12 +----------- creation/web_base/utils_tarballs.sh | 4 ---- creation/web_base/utils_xml.sh | 7 +++---- 6 files changed, 26 insertions(+), 27 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 74de7eb7bb..777806d7fd 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -18,8 +18,6 @@ glide_local_tmp_dir_created=0 # It cleans-up, print out the message and exit # It removes Glidein directories (work_dir, glide_local_tmp_dir) # It uses GLIDEIN_DEBUG_OPTIONS, start_dir, work_dir_created, work_dir, glide_local_tmp_dir_created, glide_local_tmp_dir -# Arguments: -# 1: exit code glidein_cleanup() { if ! cd "${start_dir}"; then log_warn "Cannot find ${start_dir} anymore, exiting but without cleanup" @@ -73,6 +71,7 @@ early_glidein_failure() { # Arguments: # 1: exit code # Global: +# RESULT # final_result # final_result_simple # final_result_long @@ -81,6 +80,8 @@ early_glidein_failure() { # factory_report_failes # ge_last_script_name # do_report +# ds +# as glidein_exit() { local exit_code factory_collector dlf condor_vars_file main_work_dir exit_code=$1 diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 2119764069..595820b402 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -xv #*******************************************************************# # glidein_startup.sh # @@ -52,12 +52,11 @@ export LANG=C # GLIDEIN_MULTIGLIDEIN_LAUNCHALL - if set in attrs, command to start all Glideins at once (multirestart 0) # GLIDEIN_MULTIGLIDEIN_LAUNCHER - if set in attrs, command to start the individual Glideins do_start_all() { - local num_glideins initial_dir multiglidein_launchall multiglidein_launcher g_dir + local num_glideins initial_dir multiglidein_launchall multiglidein_launcher g_dir startup_script num_glideins=$1 initial_dir="$(pwd)" multiglidein_launchall=$(params_decode "$(params_get_simple GLIDEIN_MULTIGLIDEIN_LAUNCHALL "${params}")") multiglidein_launcher=$(params_decode "$(params_get_simple GLIDEIN_MULTIGLIDEIN_LAUNCHER "${params}")") - local startup_script startup_script="${GWMS_STARTUP_SCRIPT}" if [[ -n "${multiglidein_launchall}" ]]; then echo "Starting multi-glidein using launcher: ${multiglidein_launchall}" diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 9e3d73458d..47580a8345 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -93,6 +93,10 @@ EOF ############################################ # Function that fetches a single regular file # Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) +# Arguments: +# 1: ID +# 2: target fname +# 3: real fname fetch_file_regular() { fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" } @@ -176,6 +180,10 @@ fetch_file() { # fft_cc_prefix # fft_config_check # fft_config_out +# fft_get_ss +# fft_base_name +# fft_condition_attr +# fft_condition_attr_val # Returns: # 0 in case of success # otherwise it returns the exit code of fetch_file_base @@ -191,14 +199,12 @@ fetch_file_try() { if [[ "${fft_config_check}" != "TRUE" ]]; then # TRUE is a special case, always be downloaded and processed - local fft_get_ss fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) # Stop download and processing if the cond_attr variable is not defined or has a value different from 1 [[ "${fft_get_ss}" != "1" ]] && return 0 # TODO: what if fft_get_ss is not 1? nothing, still skip the file? fi - local fft_base_name fft_condition_attr fft_condition_attr_val fft_base_name=$(basename "${fft_real_fname}") if [[ "${fft_base_name}" = gconditional_* ]]; then fft_condition_attr="${fft_base_name#gconditional_}" @@ -237,6 +243,7 @@ fetch_file_try() { # ffb_short_untar_dir # ffb_untar_dir # ffb_outname +# ffb_prefix # have_dummy_otrx # user_agent # ffb_url @@ -245,6 +252,7 @@ fetch_file_try() { # wget_version # wget_args # fetch_completed +# ret # Returns: # 1 in case the error is already displayed inside the function, # in case of tarring, in case of failure in renaming it, @@ -370,6 +378,7 @@ fetch_file_base() { "${ffb_outname}" glidein_config "${ffb_id}" fi ret=$? + local END END=$(date +%s) "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_outname} glidein_config" "${START}" "${END}" #generating test result document "${main_dir}"/error_augment.sh -concat @@ -392,6 +401,7 @@ fetch_file_base() { elif [ "${ffb_file_type}" = "untar" ]; then ffb_short_untar_dir="$(get_untar_subdir "${ffb_id}" "${ffb_target_fname}")" ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" + local START START=$(date +%s) (mkdir "${ffb_untar_dir}" && cd "${ffb_untar_dir}" && tar -xmzf "${ffb_outname}") 1>&2 ret=$? @@ -421,6 +431,7 @@ fetch_file_base() { if [ "${have_dummy_otrx}" -eq 1 ]; then # no one should really look at this file, but just to avoid confusion + local date date="$(date +%Y-%m-%dT%H:%M:%S%:z)" create_xml OSG --id fetch_file_base { oe { e --name cwd "${PWD}" } t { c Unknown tStart "${date}" tEnd "${date}" } r { status OK } } echo -e "$result" > otrx_output.xml @@ -465,6 +476,7 @@ perform_wget() { proxy_url=${wget_args[${i}+1]} fi done + local START START=$(date +%s) if [ "${proxy_url}" != "None" ]; then wget_args=(${wget_args[@]:0:${arg_len}-2}) @@ -482,6 +494,7 @@ perform_wget() { log_warn "${wget_cmd} failed. version:${wget_version} exit code ${wget_retval} stderr: ${wget_resp}" # cannot use error_*.sh helper functions # may not have been loaded yet, and wget fails often + local tStart tEnd xmlResult tStart="$(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date +%Y-%m-%dT%H:%M:%S%:z)" xmlResult="" @@ -546,7 +559,7 @@ perform_curl() { proxy_url="${curl_args[${i}+1]}" fi done - + local START START="$(date +%s)" curl_cmd="$(echo "curl" "${curl_args[@]}" | sed 's/"/\\\"/g')" curl_resp="$(curl "${curl_args[@]}" 2>&1)" @@ -560,6 +573,7 @@ perform_curl() { log_warn "${curl_cmd} failed. version:${curl_version} exit code ${curl_retval} stderr: ${curl_resp} " # cannot use error_*.sh helper functions # may not have been loaded yet, and wget fails often + local tStart tEnd xmlResult tStart="$(date --date=@"${START}" +%Y-%m-%dT%H:%M:%S%:z)" tEnd="$(date +%Y-%m-%dT%H:%M:%S%:z)" xmlResult="" diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index e70b421682..68f45f23d8 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -77,6 +77,7 @@ usage() { # @: shell parameters # Global: # startup_time +# retVal print_header(){ startup_time="$(date +%s)" echo "Starting glidein_startup.sh at $(date) (${startup_time})" @@ -201,22 +202,11 @@ parse_options(){ # tmp_par # params # repository_url -# client_repository_url # repository_entry_url # proxy_url # client_sign_type -# client_descript_file # multi_glidein -# operation_mode -# descript_file -# descript_entry_file -# glidein_name -# glidein_entry -# sign_id -# sign_entry_id # sign_type -# client_repository_group_url -# client_descript_group_file # sleep_time # set_debug # OSG_SQUID_LOCATION diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh index 5fd2c0e516..fe752e11ca 100644 --- a/creation/web_base/utils_tarballs.sh +++ b/creation/web_base/utils_tarballs.sh @@ -13,8 +13,6 @@ # Retrieve the specified data, which is appended as tarball # Arguments: # 1: selected file -# Global: -# GWMS_STARTUP_SCRIPT get_data() { sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar xz -O "$1" } @@ -31,8 +29,6 @@ source_data() { ####################################### # Show a list of the payload tarballed files in this script -# Global: -# GWMS_STARTUP_SCRIPT list_data() { sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar tz } diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 23daacda39..4b67f667f3 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -11,9 +11,8 @@ # Arguments: # 1: result construct_xml() { - local result tStart tEnd cmd - result="$1" - local glidein_end_time + local result tStart tEnd cmd glidein_end_time + result=$1 glidein_end_time="$(date +%s)" cmd="$0 ${GLOBAL_ARGS}" tStart="$(date --date=@"${startup_time}" +%Y-%m-%dT%H:%M:%S%:z)" @@ -112,7 +111,7 @@ basexml2simplexml() { # 1: simple final result # 2: global result simplexml2longxml() { - local final_result_simple global_result content + local final_result_simple global_result final_result_simple="$1" global_result="$2" echo "${final_result_simple}" | awk 'BEGIN{fr=1;}{if (fr==1) print $0}/ Date: Mon, 22 Aug 2022 09:53:51 -0500 Subject: [PATCH 055/196] Updated paths of the test files to filenames in /tmp. Some files' position has not been modified since it is expected to be found in certain positions inside the scripts. --- .../creation_web_base_add_config_line.bats | 4 ++-- .../creation_web_base_glidein_cleanup.bats | 9 ++------- test/bats/creation_web_base_utils_crypto.bats | 4 ++-- .../creation_web_base_utils_gs_filesystem.bats | 18 ++++++++---------- .../creation_web_base_utils_gs_tarballs.bats | 5 +++-- test/bats/creation_web_base_utils_params.bats | 3 ++- test/bats/creation_web_base_utils_xml.bats | 16 +++++++++------- 7 files changed, 28 insertions(+), 31 deletions(-) diff --git a/test/bats/creation_web_base_add_config_line.bats b/test/bats/creation_web_base_add_config_line.bats index 397e67cc00..70f97136d4 100755 --- a/test/bats/creation_web_base_add_config_line.bats +++ b/test/bats/creation_web_base_add_config_line.bats @@ -23,6 +23,7 @@ no_teardown() { # executed after each test echo "teardown" >&3 #rm "$glidein_config" + rm -f "$glidein_config" "$glidein_config".lock "$glidein_config".pid } setup_nameprint() { @@ -60,8 +61,7 @@ setup_nameprint() { @test "Test lock and unlock" { - glidein_config=glidein_config_test.tmp - rm -f "$glidein_config" "$glidein_config".lock "$glidein_config".pid + glidein_config="/tmp/glidein_config_test.tmp" # Failure, file not existing run lock_file "$glidein_config" [ "$status" -eq 1 ] diff --git a/test/bats/creation_web_base_glidein_cleanup.bats b/test/bats/creation_web_base_glidein_cleanup.bats index cfe0929017..f59fcd7e77 100755 --- a/test/bats/creation_web_base_glidein_cleanup.bats +++ b/test/bats/creation_web_base_glidein_cleanup.bats @@ -80,8 +80,8 @@ setup () { fi work_dir="$(pwd)" GWMS_SUBDIR=".gwms.d" - GWMS_DIR="${work_dir}/$GWMS_SUBDIR" - mkdir -p "$GWMS_DIR/exec/cleanup" + GWMS_DIR="${work_dir}/${GWMS_SUBDIR}" + mkdir -p "${GWMS_DIR}/exec/cleanup" let startup_time=$(date +%s) glidein_config="${PWD}/glidein_config" echo "ADD_CONFIG_LINE_SOURCE ${PWD}/add_config_line.source" > glidein_config @@ -134,14 +134,9 @@ setup () { } teardown() { - glide_local_tmp_dir="/tmp/glide_local_tmp_dir/" - work_dir="/tmp/work_dir/" - start_dir="/tmp/start_dir/" rm -rf "${glide_local_tmp_dir}" rm -rf "${work_dir}" rm -rf "${start_dir}" - GWMS_SUBDIR=".gwms.d" - GWMS_DIR="${work_dir}/${GWMS_SUBDIR}" rm -f "${PWD}/add_config_line.source" rm -f "glidein_config" rm -f "glidein_config.history" diff --git a/test/bats/creation_web_base_utils_crypto.bats b/test/bats/creation_web_base_utils_crypto.bats index 849f865379..b6a9caf3cc 100755 --- a/test/bats/creation_web_base_utils_crypto.bats +++ b/test/bats/creation_web_base_utils_crypto.bats @@ -18,8 +18,8 @@ setup () { assert_output --partial "md5wrapper error: can't calculate md5sum using" [ "$status" -eq 1 ] echo "Testing the call to md5wrapper with valid file..." >& 3 - touch "/tmp/trial_file" filename="/tmp/trial_file" + touch ${filename} run md5wrapper ${filename} assert_output --regexp "^[0-9a-z]+ ${filename}" [ "$status" -eq 0 ] @@ -43,6 +43,6 @@ setup () { #TODO: check_file_signature test teardown(){ - rm -f "/tmp/trial_file" + rm -f "${filename}" rm -rf "/tmp/trial/" } diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats index 6d09f59874..46b5288614 100755 --- a/test/bats/creation_web_base_utils_gs_filesystem.bats +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -80,7 +80,6 @@ early_glidein_failure() { } @test "prepare_workdir" { - rm -rf "${work_dir}" echo "Testing the function with non-existing work directory..." >& 3 work_dir="/tmp/workdir" run prepare_workdir @@ -97,19 +96,16 @@ early_glidein_failure() { assert_output --partial "Running in ${work_dir}" assert_output --partial "copied idtoken" [ "$status" -eq 0 ] - rm "trial.idtoken" - rm -rf "${work_dir}" } @test "copy_all" { echo "Testing the correctness with some trial files..." >& 3 tmp_dir="/tmp/prova" mkdir -p "$tmp_dir" - cd "$tmp_dir" - touch file1.txt - touch file2.txt - touch file3.txt - touch afile1.txt + touch "${tmp_dir}/file1.txt" + touch "${tmp_dir}/file2.txt" + touch "${tmp_dir}/file3.txt" + touch "${tmp_dir}/afile1.txt" target_dir="/tmp/prova2" mkdir -p "$target_dir" run copy_all "pfile" "${target_dir}" @@ -119,8 +115,6 @@ early_glidein_failure() { [ -f "${target_dir}"/file3.txt ] [ ! -f "${target_dir}"/pfile1.txt ] [ "$status" -eq 0 ] - rm -rf "$tmp_dir" - rm -rf "$target_dir" } @test "add_to_path" { @@ -144,4 +138,8 @@ teardown() { rm -rf "${SCRATCH}" rm -rf "${TMPDIR}" rm -rf "${TMP}" + rm -rf "$tmp_dir" + rm -rf "$target_dir" + rm "trial.idtoken" + rm -rf "${work_dir}" } diff --git a/test/bats/creation_web_base_utils_gs_tarballs.bats b/test/bats/creation_web_base_utils_gs_tarballs.bats index db58d95107..b88314a118 100755 --- a/test/bats/creation_web_base_utils_gs_tarballs.bats +++ b/test/bats/creation_web_base_utils_gs_tarballs.bats @@ -21,14 +21,14 @@ glidein_exit(){ @test "get_untar_subdir" { echo "Testing the absence of 'UNTAR_CFG_FILE' in glidein_config..." >& 3 id="main" - fname="file" + fname="/tmp/file" glidein_config="glidein_config" touch "${glidein_config}" run get_untar_subdir "${id}" "${fname}" assert_output --partial "Error, cannot find 'UNTAR_CFG_FILE' in glidein_config." [ "$status" -eq 1 ] echo "Testing the case of untar empty directory..." >& 3 - file="trial" + file="/tmp/trial" touch "${file}" echo "UNTAR_CFG_FILE ${file}" > "${glidein_config}" run get_untar_subdir "${id}" "${fname}" @@ -58,4 +58,5 @@ teardown() { rm -rf "${gs_id_work_dir}" rm -f "${glidein_config}" rm -f "${file}" + rm -f "${fname}" } diff --git a/test/bats/creation_web_base_utils_params.bats b/test/bats/creation_web_base_utils_params.bats index 53fcee1a29..6f0f5f27aa 100755 --- a/test/bats/creation_web_base_utils_params.bats +++ b/test/bats/creation_web_base_utils_params.bats @@ -36,7 +36,7 @@ glidein_exit(){ @test "params2file" { echo "Testing the correctness of the decoding and conversion to file..." >& 3 - file="trial.txt" + file="/tmp/trial.txt" glidein_config="glidein_config" touch "${glidein_config}" run params2file "${file}" "param2 .nbsp, .gt, .semicolon, .sclose, .comment, .minus, param3" @@ -47,4 +47,5 @@ glidein_exit(){ teardown() { rm -f "${glidein_config}" rm -f "${glidein_config}.history" + rm -f "${file}" } diff --git a/test/bats/creation_web_base_utils_xml.bats b/test/bats/creation_web_base_utils_xml.bats index 75d0c243bf..cc29b15fc5 100755 --- a/test/bats/creation_web_base_utils_xml.bats +++ b/test/bats/creation_web_base_utils_xml.bats @@ -31,8 +31,9 @@ setup () { assert_output --partial "Unknown" [ "$status" -eq 0 ] echo "Testing a valid parent to extract with exitcode = 0 ..." >& 3 - touch "otrx_output.xml" - echo "" > otrx_output.xml + filename="otrx_output.xml" + touch $filename + echo "" > $filename run extract_parent_fname 0 assert_output --partial "SUCCESS" [ "$status" -eq 0 ] @@ -54,15 +55,16 @@ setup () { assert_output --partial "No detail. Could not find source XML file." [ "$status" -eq 0 ] echo "Testing a call with valid XML file and exit code 0..." >& 3 - touch "otrx_output.xml" - echo "Content" > otrx_output.xml + filename="otrx_output.xml" + touch $filename + echo "Content" > $filename run extract_parent_xml_detail 0 assert_output --partial "OK" assert_output --partial "Content" [ "$status" -eq 0 ] echo "Testing a call with non-valid XML file and exit code 1..." >& 3 - echo "" > otrx_output.xml - echo "Trial\n" >> otrx_output.xml + echo "" > $filename + echo "Trial\n" >> $filename run extract_parent_xml_detail 1 assert_output --partial "ERROR" assert_output --partial " Date: Wed, 24 Aug 2022 09:48:13 -0500 Subject: [PATCH 056/196] Re-committing for testing the timeout feature of the bats tests. --- creation/web_base/glidein_startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 800bcf5c90..6678f8ddcc 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -285,7 +285,7 @@ _main(){ ######################################## prepare_workdir - ######################################## + ###################################### # extract and source all the data contained at the end of this script as tarball extract_all_data From e4e91baf0d8db4099b773f56200ffa8479f79b44 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 24 Aug 2022 14:53:21 -0500 Subject: [PATCH 057/196] Applied some of the modifications requested. Modified comments structure, moved the license on top, moved utils_tarballs.sh functions to glidein_startup.sh, and renamed some functions. --- creation/web_base/glidein_cleanup.sh | 21 ++--- creation/web_base/glidein_startup.sh | 92 ++++++++++++++----- creation/web_base/utils_crypto.sh | 17 ++-- creation/web_base/utils_gs_filesystem.sh | 24 ++--- creation/web_base/utils_gs_http.sh | 29 +++--- creation/web_base/utils_gs_io.sh | 24 ++--- creation/web_base/utils_gs_signals.sh | 18 ++-- creation/web_base/utils_gs_tarballs.sh | 16 ++-- creation/web_base/utils_io.sh | 18 ++-- creation/web_base/utils_params.sh | 18 ++-- creation/web_base/utils_signals.sh | 22 +++-- creation/web_base/utils_tarballs.sh | 57 ------------ creation/web_base/utils_xml.sh | 27 +++--- .../creation_web_base_glidein_startup.bats | 1 - 14 files changed, 193 insertions(+), 191 deletions(-) delete mode 100644 creation/web_base/utils_tarballs.sh diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index eef14e8654..d1e6f4da7f 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -1,12 +1,13 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 #*******************************************************************# # glidein_cleanup.sh # -# Script containing the variables and functions used to support # -# the glidein cleanup, failure and exit operations # -# File Version: 1.0 # +# Script containing the variables and functions used to support # +# the glidein cleanup, failure and exit operations # #*******************************************************************# -# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC -# SPDX-License-Identifier: Apache-2.0 ################################ # Global variables of support to the glidein cleanup operations @@ -14,10 +15,8 @@ work_dir_created=0 glide_local_tmp_dir_created=0 ################################ -# Function used to clean up the glidein. -# It cleans-up, print out the message and exit -# It removes Glidein directories (work_dir, glide_local_tmp_dir) -# It uses GLIDEIN_DEBUG_OPTIONS, start_dir, work_dir_created, work_dir, glide_local_tmp_dir_created, glide_local_tmp_dir +# Clean-up the glidein, print out messages, and exit +# Remove glidein directories: work_dir, glide_local_tmp_dir glidein_cleanup() { if ! cd "${start_dir}"; then log_warn "Cannot find ${start_dir} anymore, exiting but without cleanup" @@ -39,7 +38,7 @@ glidein_cleanup() { } ################################ -# Function used for early failures of the glidein, when we cannot assume we can write to disk at all +# Early failures of the glidein, when we cannot assume we can write to disk at all # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: error message @@ -66,7 +65,7 @@ early_glidein_failure() { } ################################ -# Function used for exiting the glidein, to be used when the most basic ops have been done +# Exiting the glidein, to be used when the most basic ops have been done # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: exit code diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 6678f8ddcc..29267ea99a 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -1,15 +1,15 @@ #!/bin/bash -xv -#*******************************************************************# -# glidein_startup.sh # -# Script containing the variables and fuctions used as support # -# for the glidein startup. The script load the content of # -# glidein_cleanup and other utility functions # -# File Version: 1.0 # -#*******************************************************************# # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# glidein_startup.sh # +# Main Glidein script. Load all components from separate scripts, # +# start the Glidein up, invoke HTCondor startup, and cleanup # +# at the end # +#*******************************************************************# + ################################ # Default IFS, to protect against unusual environment # better than "unset IFS" because works with restoring old one @@ -18,11 +18,11 @@ IFS=$' \t\n' GLOBAL_ARGS="$*" # GWMS_STARTUP_SCRIPT=$0 GWMS_STARTUP_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" +GWMS_PATH="" ################################ # Relative to the work directory (GWMS_DIR, gwms_lib_dir, gwms_bin_dir and gwms_exec_dir will be the absolute paths) # bin (utilities), lib (libraries), exec (aux scripts to be executed/sourced, e.g. pre-job) -GWMS_PATH="" GWMS_SUBDIR=".gwms.d" ################################ @@ -38,12 +38,10 @@ GWMS_MULTIUSER_GLIDEIN= GWMS_LOGSERVER_ADDRESS='https://fermicloud152.fnal.gov/log' GWMS_MULTIGLIDEIN_CHILDS= -[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR=. - export LANG=C ################################ -# Function used to start all glideins +# Start all glideins # Arguments: # 1: number of glideins # Global: @@ -84,7 +82,7 @@ do_start_all() { ################################ -# Function used to spawn multiple glideins and wait, if needed +# Spawn multiple glideins and wait, if needed # Global: # ON_DIE spawn_multiple_glideins(){ @@ -104,7 +102,7 @@ spawn_multiple_glideins(){ } ######################################## -# Function used to setup OSG and/or Globus +# Setup OSG and/or Globus # Global: # GLOBUS_LOCATION setup_OSG_Globus(){ @@ -142,7 +140,7 @@ setup_OSG_Globus(){ } ######################################## -# Function that creates the glidein configuration +# Creates the glidein configuration # Global: # glidein_config create_glidein_config(){ @@ -211,14 +209,62 @@ create_glidein_config(){ fi } +############################## +# Utility functions to allow the script to source functions and retrieve data stored as tarball at the end of the script itself +#TODO: Bats test files need to be defined for these functions + +####################################### +# Retrieve the specified data, which is appended as tarball +# Arguments: +# 1: selected file +get_data() { + sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar xz -O "$1" +} + +####################################### +# Source the specified data, which is appended as tarball, without saving it +# Arguments: +# 1: selected file +source_data() { + local data + data=$(get_data "$1") + [[ -n "$data" ]] && eval "$data" +} + +####################################### +# Show a list of the payload tarballed files in this script +list_data() { + sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar tz +} + +####################################### +# Extract and source all the tarball files +# Global: +# IFS +extract_all_data() { + local -a files + # change separator to split the output file list from 'tar tz' command + local IFS_OLD + IFS_OLD="${IFS}" + IFS=$'\n' + files=($(list_data)) + for f in "${files[@]}"; do + echo "Extracting file ${f}" + get_data "${f}" > "${f}" + echo "Sourcing file ${f}" + # source_data "${f}" - can source the file saved instead of re-extracting it + . "${f}" + done + IFS="${IFS_OLD}" +} + _main(){ ################################ parse_options "$@" ################################ - # Code block used to set the slots_layout - # make sure to have a valid slots_layout + # Set the slots_layout, make sure to have a valid slots_layout if (echo "x${slots_layout}" | grep -i fixed) >/dev/null 2>&1 ; then slots_layout="fixed" else @@ -229,7 +275,7 @@ _main(){ parse_arguments ################################ - # Code block used to generate the glidein UUID + # Generate the glidein UUID if command -v uuidgen >/dev/null 2>&1; then glidein_uuid="$(uuidgen)" else @@ -243,8 +289,7 @@ _main(){ spawn_multiple_glideins ######################################## - # Code block used to make sure nobody else can write my files - # in the Grid world I cannot trust anybody + # Make sure nobody else can write my files. In the Grid world I cannot trust anybody. if ! umask 0022; then early_glidein_failure "Failed in umask 0022" fi @@ -253,7 +298,7 @@ _main(){ setup_OSG_Globus ######################################## - # Code block used to set the tokens + # Set the tokens [ -n "${X509_USER_PROXY}" ] && set_proxy_fullpath num_gct=0 @@ -301,9 +346,9 @@ _main(){ params2file ${params} ##################################### - # Setup logging + # setup logging log_init "${glidein_uuid}" "${work_dir}" - # Remove these files, if they are still there + # remove these files, if they are still there rm -rf tokens.tgz url_dirs.desc tokens log_setup "${glidein_config}" @@ -508,7 +553,7 @@ _main(){ fi ############################# - #Things like periodic scripts might put messages here if they want them printed in the (stderr) logfile + # Things like periodic scripts might put messages here if they want them printed in the (stderr) logfile echo "=== Exit messages left by periodic scripts ===" 1>&2 if [ -f exit_message ]; then cat exit_message 1>&2 @@ -532,7 +577,6 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then source "$GWMS_SOURCEDIR"/utils_io.sh source "$GWMS_SOURCEDIR"/utils_params.sh source "$GWMS_SOURCEDIR"/utils_signals.sh - source "$GWMS_SOURCEDIR"/utils_tarballs.sh source "$GWMS_SOURCEDIR"/utils_xml.sh source "$GWMS_SOURCEDIR"/utils_crypto.sh source "$GWMS_SOURCEDIR"/glidein_cleanup.sh diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index 60d35cac26..fa5d347127 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -1,13 +1,14 @@ -#*******************************************************************# -# utils_crypto.sh # -# This script contains cryptography utility functions # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_crypto.sh # +# This script contains cryptography utility functions # +#*******************************************************************# ################################ -# Function used to calculate the md5 sum +# Calculates the md5 sum # Arguments: # 1: file name # 2: option (quiet) @@ -45,7 +46,7 @@ md5wrapper() { } ########################################### -# Function used to check the file signature +# Checks the file signature # Arguments: # 1: id # 2: file name @@ -92,7 +93,7 @@ check_file_signature() { } ######################################## -# Function used to set the X509_USER_PROXY path to full path to the file +# Set the X509_USER_PROXY path to full path to the file # Environment variables exported: # X509_USER_PROXY set_proxy_fullpath() { diff --git a/creation/web_base/utils_gs_filesystem.sh b/creation/web_base/utils_gs_filesystem.sh index 10936cdad1..0c4d5523ac 100644 --- a/creation/web_base/utils_gs_filesystem.sh +++ b/creation/web_base/utils_gs_filesystem.sh @@ -1,14 +1,16 @@ -#*******************************************************************# -# utils_gs_filesystem.sh # -# This script contains filesystem utility functions for # -# the glidein_startup.sh script # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_gs_filesystem.sh # +# This script contains filesystem utility functions for the # +# glidein_startup.sh script # +#*******************************************************************# + ################################ -# Function used to automatically determine and setup work directories +# Automatically determine and setup work directories # Globals: # targets (_CONDOR_SCRATCH_DIR, OSG_WN_TMP, TG_NODE_SCRATCH, TG_CLUSTER_SCRATCH, SCRATCH, TMPDIR, TMP, PWD) # work_dir @@ -59,7 +61,7 @@ automatic_work_dir() { } ################################ -# Function used to generate the directory ID +# Generate the directory ID # It creates an ID to distinguish the directories when preserved dir_id() { [[ ! ",${GLIDEIN_DEBUG_OPTIONS}," = *,nocleanup,* ]] && return @@ -77,7 +79,7 @@ dir_id() { } ########################################### -# Function used to prepare and move to the work directory +# Prepare and move to the work directory # Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . # Empty $work_dir means PWD (same as ".") # A custom path could be provided (no "*)" in case) @@ -239,7 +241,7 @@ prepare_workdir(){ } ################################ -# Function used to copy all files from a directory to another +# Copy all files from a directory to another # (to support when needed to start multiple glideins) # Arguments: # 1: prefix of the files to skip @@ -257,7 +259,7 @@ copy_all() { # TODO: should it copy also hidden files? ######################################## -# Function used to add $1 to GWMS_PATH and update PATH +# Add $1 to GWMS_PATH and update PATH # Environment: # GWMS_PATH # PATH diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 47580a8345..180d99fef0 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -1,15 +1,16 @@ -#*******************************************************************# -# utils_gs_http.sh # -# This script contains http utility functions for the # -# glidein_startup.sh script # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_gs_http.sh # +# This script contains http utility functions for the # +# glidein_startup.sh script # +#*******************************************************************# ############################################ -# Function that gets the proper descript file based on id +# Gets the proper descript file based on id # Arguments: # 1: type (main/entry/client/client_group) # Returns: @@ -31,7 +32,7 @@ get_repository_url() { add_startd_cron_counter=0 ##################### -# Function that schedules a script for periodic execution using startd_cron +# Schedules a script for periodic execution using startd_cron # Arguments: # 1: wrapper full path # 2: period @@ -91,7 +92,7 @@ EOF } ############################################ -# Function that fetches a single regular file +# Fetches a single regular file # Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) # Arguments: # 1: ID @@ -102,7 +103,7 @@ fetch_file_regular() { } ############################################ -# Function that fetches a single file +# Fetches a single file # custom_scripts parameters format is set in the GWMS configuration (creation/lib) # Check cWDictFile/FileDictFile for the number and type of parameters (has to be consistent) # Arguments: @@ -154,7 +155,7 @@ fetch_file() { } ############################################ -# Function that verifies if the file should be downloaded and acted upon (extracted, executed, ...) or not +# Verifies if the file should be downloaded and acted upon (extracted, executed, ...) or not # There are 2 mechanisms to control the download # 1. tar files have the attribute "cond_attr" that is a name of a variable in glidein_config. # if the named variable has value 1, then the file is downloaded. TRUE (default) means always download @@ -220,7 +221,7 @@ fetch_file_try() { } ############################################ -# Function that performs the file download and corresponding action (untar, execute, ...) +# Performs the file download and corresponding action (untar, execute, ...) # Arguments: # 1: ID # 2: target fname @@ -441,7 +442,7 @@ fetch_file_base() { } ################################ -# Function used to perform a wget request +# Perform a wget request # Arguments: # @: wget args # Globals: @@ -525,7 +526,7 @@ perform_wget() { } ################################ -# Function used to perform a curl request +# Perform a curl request # Arguments: # @: curl args # Globals: diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index 68f45f23d8..d85de90a13 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -1,14 +1,16 @@ -#*******************************************************************# -# utils_gs_io.sh # -# This script contains I/O utility functions for the # -# glidein_startup.sh script # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_gs_io.sh # +# This script contains I/O utility functions for the # +# glidein_startup.sh script # +#*******************************************************************# + ################################ -# Function used to print the tail with the final results of the glideins +# Print the tail with the final results of the glideins # Arguments: # 1: exit code # 2: short version of the final results @@ -34,7 +36,7 @@ print_tail() { } ################################ -# Function used to have information about the usage of the glidein_startup.sh script +# Usage of the glidein_startup.sh script # Returns: # 1 in any case usage() { @@ -72,7 +74,7 @@ usage() { } ################################ -# Function used to print initial information header +# Print initial information header # Parameters: # @: shell parameters # Global: @@ -143,7 +145,7 @@ print_header(){ } ################################ -# Function used to parse the glidein startup options +# Parse the glidein startup options # Parameters: # @: shell parameters # Global: @@ -196,7 +198,7 @@ parse_options(){ } ################################ -# Function used to parse and verify arguments +# Parse and verify arguments # It allows some parameters to change arguments # Global: # tmp_par diff --git a/creation/web_base/utils_gs_signals.sh b/creation/web_base/utils_gs_signals.sh index 85fa3c91dc..90a4806a3d 100644 --- a/creation/web_base/utils_gs_signals.sh +++ b/creation/web_base/utils_gs_signals.sh @@ -1,19 +1,21 @@ -#*******************************************************************# -# utils_gs_signals.sh # -# This script contains signals' utility functions for the # -# glidein_startup.sh script # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_gs_signals.sh # +# This script contains signals' utility functions for the # +# glidein_startup.sh script # +#*******************************************************************# + ################################ -# Function that forwards signals to the children processes +# Forwards signals to the children processes # Arguments: # 1: signal # Globals: # ON_DIE -on_die_multi() { +signal_on_die_multi() { echo "Multi-Glidein received signal... shutting down child glideins (forwarding $1 signal to ${GWMS_MULTIGLIDEIN_CHILDS})" 1>&2 ON_DIE=1 for i in ${GWMS_MULTIGLIDEIN_CHILDS}; do diff --git a/creation/web_base/utils_gs_tarballs.sh b/creation/web_base/utils_gs_tarballs.sh index 6f5e33ec70..725e25368e 100644 --- a/creation/web_base/utils_gs_tarballs.sh +++ b/creation/web_base/utils_gs_tarballs.sh @@ -1,12 +1,14 @@ -#*******************************************************************# -# utils_gs_tarballs.sh # -# This script contains tarballs utility functions for the # -# glidein_startup.sh script # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_gs_tarballs.sh # +# This script contains tarballs utility functions for the # +# glidein_startup.sh script # +#*******************************************************************# + ########################################### # Untar support function # Arguments: @@ -45,7 +47,7 @@ get_untar_subdir() { } ######################################## -# Function that removes the native condor tarballs directory to allow factory ops to use native condor tarballs +# Removes the native condor tarballs directory to allow factory ops to use native condor tarballs # All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped # However the (not used anymore) gwms create_condor_tarball removes that dir fixup_condor_dir() { diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_io.sh index 4a7f79d609..edf01a0000 100644 --- a/creation/web_base/utils_io.sh +++ b/creation/web_base/utils_io.sh @@ -1,13 +1,15 @@ -#*******************************************************************# -# utils_io.sh # -# This script contains I/O utility functions # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_io.sh # +# This script contains I/O utility functions # +#*******************************************************************# + ################################ -# Function used to log warning statements +# Log warning statements # Arguments: # @: content to warn log_warn() { @@ -15,7 +17,7 @@ log_warn() { } ################################ -# Function used to log debug statements +# Log debug statements # Arguments: # @: content to debug log_debug() { @@ -23,7 +25,7 @@ log_debug() { } ##################### -# Function used to print a header line, i.e. === HEADER === +# Print a header line, i.e. === HEADER === # Arguments: # 1: content of the header line # 2 (optional): 1 if needs to write to stdout, 2 if needs to write to stderr diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index 3218b98656..55c9ae7a25 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -1,16 +1,18 @@ -#*******************************************************************# -# utils_crypto.sh # -# This script contains parameters utility functions # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_params.sh # +# This script contains parameters utility functions # +#*******************************************************************# + ################################ # Parameters utility functions ################################ -# Function used to retrieve a simple parameter (no special characters in its value) from the param list +# Retrieve a simple parameter (no special characters in its value) from the param list # make sure to have a valid slots_layout # Arguments: # 1: param @@ -23,7 +25,7 @@ params_get_simple() { } ############################### -# Function used to decode the parameters +# Decode the parameters # Arguments: # 1: param params_decode() { @@ -59,7 +61,7 @@ params_decode() { } ############################### -# Function used to put the parameters into the config file +# Put the parameters into the config file # Arguments: # @: parameters # Global: diff --git a/creation/web_base/utils_signals.sh b/creation/web_base/utils_signals.sh index e39965f179..f0aa736da8 100644 --- a/creation/web_base/utils_signals.sh +++ b/creation/web_base/utils_signals.sh @@ -1,16 +1,18 @@ -#*******************************************************************# -# utils_signals.sh # -# This script contains signals utility functions # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_signals.sh # +# This script contains signals utility functions # +#*******************************************************************# + ################################ # Extends 'trap' allowing to pass the signal name as argument to the handler # Arguments: # 1: handler -trap_with_arg() { +signal_trap_with_arg() { local func func="$1" for sig ; do @@ -20,20 +22,20 @@ trap_with_arg() { } ################################ -# Function that allows to pass signals to the children processes +# Propagate signals to the children processes # There is no need to re-raise sigint, caller does unconditional exit (https://www.cons.org/cracauer/sigint.html) # Arguments: # 1: signal # Global: # ON_DIE -on_die() { +signal_on_die() { echo "Received kill signal... shutting down child processes (forwarding $1 signal)" 1>&2 ON_DIE=1 kill -s "$1" %1 } ################################ -# Function used to ignore signal SIGHUP -ignore_signal() { +# Ignore signal SIGHUP +signal_ignore() { echo "Ignoring SIGHUP signal... Use SIGTERM or SIGQUIT to kill processes" 1>&2 } diff --git a/creation/web_base/utils_tarballs.sh b/creation/web_base/utils_tarballs.sh deleted file mode 100644 index fe752e11ca..0000000000 --- a/creation/web_base/utils_tarballs.sh +++ /dev/null @@ -1,57 +0,0 @@ -#*******************************************************************# -# utils_crypto.sh # -# This script contains tarballs utility functions # -# File Version: 1.0 # -#*******************************************************************# -# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC -# SPDX-License-Identifier: Apache-2.0 - -############################## -# Utility functions to allow the script to source functions and retrieve data stored as tarball at the end of the script itself - -####################################### -# Retrieve the specified data, which is appended as tarball -# Arguments: -# 1: selected file -get_data() { - sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar xz -O "$1" -} - -####################################### -# Source the specified data, which is appended as tarball, without saving it -# Arguments: -# 1: selected file -source_data() { - local data - data=$(get_data "$1") - [[ -n "$data" ]] && eval "$data" -} - -####################################### -# Show a list of the payload tarballed files in this script -list_data() { - sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar tz -} - -####################################### -# Extract and source all the tarball files -# Global: -# IFS -extract_all_data() { - local -a files - # change separator to split the output file list from 'tar tz' command - local IFS_OLD - IFS_OLD="${IFS}" - IFS=$'\n' - files=($(list_data)) - for f in "${files[@]}"; do - echo "Extracting file ${f}" - get_data "${f}" > "${f}" - echo "Sourcing file ${f}" - # source_data "${f}" - can source the file saved instead of re-extracting it - . "${f}" - done - IFS="${IFS_OLD}" -} - -#TODO: Bats test files need to be defined for these functions diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index 4b67f667f3..c6d857421d 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -1,13 +1,15 @@ -#*******************************************************************# -# utils_crypto.sh # -# This script contains xml utility functions # -# File Version: 1.0 # -#*******************************************************************# +#!/bin/bash + # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 +#*******************************************************************# +# utils_xml.sh # +# This script contains xml utility functions # +#*******************************************************************# + ################################ -# Function used to create the xml content +# Create the xml general content # Arguments: # 1: result construct_xml() { @@ -21,7 +23,7 @@ construct_xml() { } ################################ -# Function used to extract the parent xml fname (element inside the id of OSGTestResult) +# Extract the parent xml fname (element inside the id of OSGTestResult) # Arguments: # 1: exit code extract_parent_fname(){ @@ -41,7 +43,7 @@ extract_parent_fname(){ } ################################ -# Function used to extract the parent xml details +# Extract the parent xml details # Arguments: # 1: exit code extract_parent_xml_detail() { @@ -86,7 +88,7 @@ extract_parent_xml_detail() { } ################################ -# Function used to convert base xml to simple xml +# Convert base xml to simple xml # Arguments: # 1: final result basexml2simplexml() { @@ -106,7 +108,7 @@ basexml2simplexml() { } ################################ -# Function used to convert simple xml to long xml +# Convert simple xml to long xml # Arguments: # 1: simple final result # 2: global result @@ -135,7 +137,7 @@ simplexml2longxml() { } ################################ -# Function used as support to add spaces +# Add spaces, support to create_xml # Global: # xml add_spaces(){ @@ -147,8 +149,7 @@ add_spaces(){ } ################################ -# Function used to create an xml file structure -# Call the function with the following syntax: +# Create an xml file structure: # create_xml TAG --option option_value .... tag_value TAG2 ..... # use '{' to specify the start if an inner tag and use '}' to specify the end # use create_xml -s SPACES ... in case you want to create a subpart of an xml file not starting from the beginning specifying diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 5f6cb2336d..0183012c14 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -12,7 +12,6 @@ setup () { source "$GWMS_SOURCEDIR"/utils_io.sh source "$GWMS_SOURCEDIR"/utils_params.sh source "$GWMS_SOURCEDIR"/utils_signals.sh - source "$GWMS_SOURCEDIR"/utils_tarballs.sh source "$GWMS_SOURCEDIR"/utils_xml.sh source "$GWMS_SOURCEDIR"/utils_crypto.sh source "$GWMS_SOURCEDIR"/utils_gs_http.sh From 4c9d84daba5e8bc3ff15587afad6034dc83fa5e7 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 24 Aug 2022 15:59:35 -0500 Subject: [PATCH 058/196] Updated globals/used comments about functions. --- creation/web_base/glidein_cleanup.sh | 22 ++++++++- creation/web_base/glidein_startup.sh | 34 +++++++++++--- creation/web_base/utils_crypto.sh | 11 +++-- creation/web_base/utils_gs_filesystem.sh | 22 ++++++++- creation/web_base/utils_gs_http.sh | 32 ++++++++++--- creation/web_base/utils_gs_io.sh | 57 ++++++++++++++++++++---- creation/web_base/utils_gs_signals.sh | 4 +- creation/web_base/utils_gs_tarballs.sh | 4 +- creation/web_base/utils_params.sh | 2 +- creation/web_base/utils_signals.sh | 2 +- creation/web_base/utils_xml.sh | 11 ++++- 11 files changed, 168 insertions(+), 33 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index d1e6f4da7f..c8988a513e 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -17,6 +17,13 @@ glide_local_tmp_dir_created=0 ################################ # Clean-up the glidein, print out messages, and exit # Remove glidein directories: work_dir, glide_local_tmp_dir +# Used: +# start_dir +# GLIDEIN_DEBUG_OPTIONS +# work_dir_created +# work_dir +# glide_local_tmp_dir_created +# glide_local_tmp_dir glidein_cleanup() { if ! cd "${start_dir}"; then log_warn "Cannot find ${start_dir} anymore, exiting but without cleanup" @@ -42,10 +49,13 @@ glidein_cleanup() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: error message -# Global: +# Globals (r/w): # final_result # final_result_simple # final_result_long +# Used: +# sleep_time +# result early_glidein_failure() { local error_msg glidein_end_time error_msg="$1" @@ -69,7 +79,7 @@ early_glidein_failure() { # too bad we end up with some repeated code, but difficult to do better # Arguments: # 1: exit code -# Global: +# Globals (r/w): # RESULT # final_result # final_result_simple @@ -80,7 +90,15 @@ early_glidein_failure() { # ge_last_script_name # do_report # ds +# dlf # as +# Used: +# GWMS_DIR +# glidein_config +# result +# last_script +# dl +# sleep_time glidein_exit() { local exit_code factory_collector dlf condor_vars_file main_work_dir exit_code=$1 diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 29267ea99a..4e52aac387 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -44,11 +44,15 @@ export LANG=C # Start all glideins # Arguments: # 1: number of glideins -# Global: +# Globals (r/w): # GWMS_MULTIGLIDEIN_CHILDS -# Important Variables: +# Used: +# params +# GLOBAL_ARG +# Important variables: # GLIDEIN_MULTIGLIDEIN_LAUNCHALL - if set in attrs, command to start all Glideins at once (multirestart 0) # GLIDEIN_MULTIGLIDEIN_LAUNCHER - if set in attrs, command to start the individual Glideins + do_start_all() { local num_glideins initial_dir multiglidein_launchall multiglidein_launcher g_dir startup_script num_glideins=$1 @@ -83,8 +87,11 @@ do_start_all() { ################################ # Spawn multiple glideins and wait, if needed -# Global: +# Globals (r/w): # ON_DIE +# Used: +# multi_glidein +# multi_glidein_restart spawn_multiple_glideins(){ if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then # start multiple glideins @@ -103,8 +110,12 @@ spawn_multiple_glideins(){ ######################################## # Setup OSG and/or Globus -# Global: +# Globals (r/w): # GLOBUS_LOCATION +# Used: +# OSG_GRID +# GLITE_LOCAL_CUSTOMIZATION_DIR +# GLOBUS_PATH setup_OSG_Globus(){ if [ -r "${OSG_GRID}/setup.sh" ]; then . "${OSG_GRID}/setup.sh" @@ -141,8 +152,17 @@ setup_OSG_Globus(){ ######################################## # Creates the glidein configuration -# Global: +# Globals (r/w): # glidein_config +# Used: +# glidein_uuid, glidein_factory, glidein_name, glidein_entry, glidein_cred_id +# client_name, client_group, client_dir, client_descript_file, client_sign_id. client_repository_group_url +# client_group_dir, client_descript_group_file, client_sign_group_id +# condorg_cluster, condorg_schedd, condorg_subcluster +# set_debug, proxy_url, PWD, wrapper_list, slots_layout, GLIDEIN_CONDOR_TOKEN +# start_dir, main_dir, entry_dir, glide_tmp_dir, glide_local_tmp_dir +# descript_file, descript_entry_file +# sign_id, sign_entry_id create_glidein_config(){ glidein_config="${PWD}/glidein_config" if ! echo > "${glidein_config}"; then @@ -217,6 +237,8 @@ create_glidein_config(){ # Retrieve the specified data, which is appended as tarball # Arguments: # 1: selected file +# Used: +# GWMS_STARTUP_SCRIPT get_data() { sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar xz -O "$1" } @@ -233,6 +255,8 @@ source_data() { ####################################### # Show a list of the payload tarballed files in this script +# Used: +# GWMS_STARTUP_SCRIPT list_data() { sed '1,/^#EOF$/d' < "${GWMS_STARTUP_SCRIPT}" | tar tz } diff --git a/creation/web_base/utils_crypto.sh b/creation/web_base/utils_crypto.sh index fa5d347127..5eb7de0446 100644 --- a/creation/web_base/utils_crypto.sh +++ b/creation/web_base/utils_crypto.sh @@ -14,7 +14,7 @@ # 2: option (quiet) # Returns: # 1 in case the md5sum cannot be calculated, or neither the md5sum nor the md5 can be found -# Global: +# Globals (r/w): # res md5wrapper() { local ERROR_RESULT @@ -50,15 +50,18 @@ md5wrapper() { # Arguments: # 1: id # 2: file name -# Globals: +# Globals (r/w): # cfs_id # cfs_fname # cfs_work_dir # cfs_desc_fname # cfs_signature -# cfs_rc -# tmp_signname +# Used: # check_signature +# tmp_signname +# main_dir +# cfs_rc +# PWD # Returns: # 1 in case of corrupted file check_file_signature() { diff --git a/creation/web_base/utils_gs_filesystem.sh b/creation/web_base/utils_gs_filesystem.sh index 0c4d5523ac..3a6cc9744e 100644 --- a/creation/web_base/utils_gs_filesystem.sh +++ b/creation/web_base/utils_gs_filesystem.sh @@ -11,9 +11,18 @@ ################################ # Automatically determine and setup work directories -# Globals: +# Globals (r/w): # targets (_CONDOR_SCRATCH_DIR, OSG_WN_TMP, TG_NODE_SCRATCH, TG_CLUSTER_SCRATCH, SCRATCH, TMPDIR, TMP, PWD) # work_dir +# Used: +# _CONDOR_SCRATCH_DIR +# OSG_WN_TMP +# TG_NODE_SCRATCH +# TG_CLUSTER_SCRATCH +# SCRATCH +# TMP +# TMPDIR +# PWD # Returns: # 1 in case you are not allowed to write automatic_work_dir() { @@ -83,7 +92,7 @@ dir_id() { # Replace known keywords: Condor, CONDOR, OSG, TMPDIR, AUTO, . # Empty $work_dir means PWD (same as ".") # A custom path could be provided (no "*)" in case) -# Globals: +# Globals (r/w): # work_dir # start_dir # def_work_dir @@ -103,6 +112,15 @@ dir_id() { # client_dir # short_client_group_dir # client_group_dir +# Used: +# _CONDOR_SCRATCH_DIR +# OSG_WN_TMP +# TMPDIR +# GWMS_SUBDIR +# dir_id +# GWMS_MULTIUSER_GLIDEIN +# client_repository_url +# client_repository_group_url prepare_workdir(){ if [ -z "${work_dir}" ]; then work_dir="$(pwd)" diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 180d99fef0..bdd0b08214 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -13,6 +13,11 @@ # Gets the proper descript file based on id # Arguments: # 1: type (main/entry/client/client_group) +# Used: +# repository_url +# repository_entry_url +# client_repository_url +# client_repository_group_url # Returns: # 1 in case of invalid ID get_repository_url() { @@ -40,8 +45,10 @@ add_startd_cron_counter=0 # 4: executable path (from cwd), # 5: config file path (from cwd) # 6: ID -# Globals: +# Globals (r/w): # add_startd_cron_counter +# Used: +# condor_config_startd_cron_include add_periodic_script() { #TODO: should it allow for variable number of parameters? local include_fname @@ -116,6 +123,9 @@ fetch_file_regular() { # 7: config check TRUE,FALSE # 8: config out TRUE,FALSE # The above is the most recent list, below some adaptations for different versions +# Used: +# ifs_str +# IFS # Returns: # 0 in case of success # otherwise glidein_exit with 1 @@ -172,7 +182,7 @@ fetch_file() { # 6: periodic scripts prefix # 7: config check TRUE,FALSE # 8: config out TRUE,FALSE -# Globals: +# Globals (r/w): # fft_id # fft_target_fname # fft_real_fname @@ -184,7 +194,7 @@ fetch_file() { # fft_get_ss # fft_base_name # fft_condition_attr -# fft_condition_attr_val +# fft_condition_attr_val # Returns: # 0 in case of success # otherwise it returns the exit code of fetch_file_base @@ -254,6 +264,18 @@ fetch_file_try() { # wget_args # fetch_completed # ret +# Used: +# glidein_entry +# condorg_schedd +# condorg_cluster +# condorg_subcluster +# client_name +# ffb_nocache_str +# gwms_exec_dir +# main_dir +# work_dir +# cfs_id +# result # Returns: # 1 in case the error is already displayed inside the function, # in case of tarring, in case of failure in renaming it, @@ -445,7 +467,7 @@ fetch_file_base() { # Perform a wget request # Arguments: # @: wget args -# Globals: +# Globals (r/w): # wget_args # wget_cmd # wget_resp @@ -529,7 +551,7 @@ perform_wget() { # Perform a curl request # Arguments: # @: curl args -# Globals: +# Globals (r/w): # curl_args # arg_len # ffb_url diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_io.sh index d85de90a13..9b2e9e2538 100644 --- a/creation/web_base/utils_gs_io.sh +++ b/creation/web_base/utils_gs_io.sh @@ -15,7 +15,7 @@ # 1: exit code # 2: short version of the final results # 3: long version of the final results -# Global: +# Globals (r/w): # total_time print_tail() { local final_result_simple final_result_long exit_code glidein_end_time @@ -77,9 +77,39 @@ usage() { # Print initial information header # Parameters: # @: shell parameters -# Global: +# Globals(r/w): # startup_time # retVal +# Used: +# operation_mode +# condorg_cluster +# condorg_subcluster +# condorg_schedd +# glidein_uuid +# glidein_cred_id +# glidein_factory +# glidein_name +# glidein_entry +# client_name +# client_group +# client_descript_file +# client_descript_group_file +# client_repository_url +# client_sign_type +# client_sign_id +# client_sign_group_id +# client_repository_group_url +# multi_glidein +# multi_glidein_restart +# work_dir +# repository_url +# sign_type +# descript_file +# proxy_url +# descript_entry_file +# sign_id +# sign_entry_id +# set_debug print_header(){ startup_time="$(date +%s)" echo "Starting glidein_startup.sh at $(date) (${startup_time})" @@ -148,7 +178,7 @@ print_header(){ # Parse the glidein startup options # Parameters: # @: shell parameters -# Global: +# Globals (r/w): # params # all other global variables to set parse_options(){ @@ -200,17 +230,26 @@ parse_options(){ ################################ # Parse and verify arguments # It allows some parameters to change arguments -# Global: +# Globals (r/w): # tmp_par -# params -# repository_url +# multi_glidein +# sleep_time +# set_debug # repository_entry_url # proxy_url # client_sign_type -# multi_glidein # sign_type -# sleep_time -# set_debug +# Used: +# params +# operation_mode +# descript_file +# descript_entry_file +# glidein_name +# glidein_entry +# repository_url +# client_descript_group_file, client_repository_group_url, client_descript_file, client_repository_url +# sign_entry_id +# sign_id # OSG_SQUID_LOCATION parse_arguments(){ # multiglidein GLIDEIN_MULTIGLIDEIN -> multi_glidein diff --git a/creation/web_base/utils_gs_signals.sh b/creation/web_base/utils_gs_signals.sh index 90a4806a3d..735b7d8d98 100644 --- a/creation/web_base/utils_gs_signals.sh +++ b/creation/web_base/utils_gs_signals.sh @@ -13,8 +13,10 @@ # Forwards signals to the children processes # Arguments: # 1: signal -# Globals: +# Globals (r/w): # ON_DIE +# Used: +# GWMS_MULTIGLIDEIN_CHILDS signal_on_die_multi() { echo "Multi-Glidein received signal... shutting down child glideins (forwarding $1 signal to ${GWMS_MULTIGLIDEIN_CHILDS})" 1>&2 ON_DIE=1 diff --git a/creation/web_base/utils_gs_tarballs.sh b/creation/web_base/utils_gs_tarballs.sh index 725e25368e..476c695d2d 100644 --- a/creation/web_base/utils_gs_tarballs.sh +++ b/creation/web_base/utils_gs_tarballs.sh @@ -14,7 +14,7 @@ # Arguments: # 1: id # 2: filename -# Global: +# Globals (r/w): # gus_id # gus_fname # gus_prefix @@ -50,6 +50,8 @@ get_untar_subdir() { # Removes the native condor tarballs directory to allow factory ops to use native condor tarballs # All files in the native condor tarballs have a directory like condor-9.0.11-1-x86_64_CentOS7-stripped # However the (not used anymore) gwms create_condor_tarball removes that dir +# Used: +# gs_id_work_dir fixup_condor_dir() { # Check if the condor dir has only one subdir, the one like "condor-9.0.11-1-x86_64_CentOS7-stripped" # See https://stackoverflow.com/questions/32429333/how-to-test-if-a-linux-directory-contain-only-one-subdirectory-and-no-other-file diff --git a/creation/web_base/utils_params.sh b/creation/web_base/utils_params.sh index 55c9ae7a25..c300379439 100644 --- a/creation/web_base/utils_params.sh +++ b/creation/web_base/utils_params.sh @@ -64,7 +64,7 @@ params_decode() { # Put the parameters into the config file # Arguments: # @: parameters -# Global: +# Globals (r/w): # param_list # pfval # Returns: diff --git a/creation/web_base/utils_signals.sh b/creation/web_base/utils_signals.sh index f0aa736da8..eb3ee5d76f 100644 --- a/creation/web_base/utils_signals.sh +++ b/creation/web_base/utils_signals.sh @@ -26,7 +26,7 @@ signal_trap_with_arg() { # There is no need to re-raise sigint, caller does unconditional exit (https://www.cons.org/cracauer/sigint.html) # Arguments: # 1: signal -# Global: +# Globals (r/w): # ON_DIE signal_on_die() { echo "Received kill signal... shutting down child processes (forwarding $1 signal)" 1>&2 diff --git a/creation/web_base/utils_xml.sh b/creation/web_base/utils_xml.sh index c6d857421d..3fa4c6505e 100644 --- a/creation/web_base/utils_xml.sh +++ b/creation/web_base/utils_xml.sh @@ -12,6 +12,11 @@ # Create the xml general content # Arguments: # 1: result +# Used: +# GLOBAL_ARGS +# startup_time +# glidein_end_time +# start_dir construct_xml() { local result tStart tEnd cmd glidein_end_time result=$1 @@ -138,8 +143,10 @@ simplexml2longxml() { ################################ # Add spaces, support to create_xml -# Global: +# Globals (r/w): # xml +# Used: +# spaces add_spaces(){ local c for (( c=1; c<=spaces; c++ )) @@ -158,7 +165,7 @@ add_spaces(){ # use create_xml -h to require as output only the header tag of the xml file, i.e. " # Arguments: # @: tags, options, values -# Global: +# Globals (r/w): # spaces # xml # end_xml From 74ecac8cd280f38576621b8ba3d8082880647f46 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 25 Aug 2022 12:00:55 -0500 Subject: [PATCH 059/196] Modified some file names and solved some bugs in bats tests. --- creation/web_base/glidein_startup.sh | 16 +++++------ .../{utils_gs_io.sh => utils_gs_log.sh} | 0 .../web_base/{utils_io.sh => utils_log.sh} | 0 creation/web_base/utils_signals.sh | 1 + .../creation_web_base_glidein_cleanup.bats | 5 ++-- .../creation_web_base_glidein_startup.bats | 4 +-- test/bats/creation_web_base_utils_crypto.bats | 2 +- ...creation_web_base_utils_gs_filesystem.bats | 27 +++++++++---------- .../bats/creation_web_base_utils_gs_http.bats | 2 +- test/bats/creation_web_base_utils_gs_io.bats | 4 +-- .../creation_web_base_utils_gs_signals.bats | 8 +++--- .../creation_web_base_utils_gs_tarballs.bats | 2 +- test/bats/creation_web_base_utils_io.bats | 2 +- .../bats/creation_web_base_utils_signals.bats | 8 +++--- 14 files changed, 40 insertions(+), 41 deletions(-) rename creation/web_base/{utils_gs_io.sh => utils_gs_log.sh} (100%) rename creation/web_base/{utils_io.sh => utils_log.sh} (100%) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 4e52aac387..329310383d 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -96,8 +96,8 @@ spawn_multiple_glideins(){ if [[ -n "${multi_glidein}" ]] && [[ -z "${multi_glidein_restart}" ]] && [[ "${multi_glidein}" -gt 1 ]]; then # start multiple glideins ON_DIE=0 - trap 'ignore_signal' SIGHUP - trap_with_arg 'on_die_multi' SIGTERM SIGINT SIGQUIT + trap 'signal_ignore' SIGHUP + signal_trap_with_arg 'signal_on_die_multi' SIGTERM SIGINT SIGQUIT do_start_all "${multi_glidein}" # Wait for all glideins and exit 0 # TODO: Summarize exit codes and status from all child glideins @@ -554,10 +554,10 @@ _main(){ echo "=== Last script starting $(date) (${last_startup_time}) after validating for ${validation_time} ===" echo ON_DIE=0 - trap 'ignore_signal' SIGHUP - trap_with_arg 'on_die' SIGTERM SIGINT SIGQUIT - #trap 'on_die' TERM - #trap 'on_die' INT + trap 'signal_ignore' SIGHUP + signal_trap_with_arg 'signal_on_die' SIGTERM SIGINT SIGQUIT + #trap 'signal_on_die' TERM + #trap 'signal_on_die' INT gs_id_work_dir=$(get_work_dir main) "${main_dir}"/error_augment.sh -init "${gs_id_work_dir}/${last_script}" glidein_config & @@ -595,10 +595,10 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then # Include all source scripts source "$GWMS_SOURCEDIR"/utils_gs_http.sh source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh - source "$GWMS_SOURCEDIR"/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/utils_gs_log.sh source "$GWMS_SOURCEDIR"/utils_gs_signals.sh source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh - source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_log.sh source "$GWMS_SOURCEDIR"/utils_params.sh source "$GWMS_SOURCEDIR"/utils_signals.sh source "$GWMS_SOURCEDIR"/utils_xml.sh diff --git a/creation/web_base/utils_gs_io.sh b/creation/web_base/utils_gs_log.sh similarity index 100% rename from creation/web_base/utils_gs_io.sh rename to creation/web_base/utils_gs_log.sh diff --git a/creation/web_base/utils_io.sh b/creation/web_base/utils_log.sh similarity index 100% rename from creation/web_base/utils_io.sh rename to creation/web_base/utils_log.sh diff --git a/creation/web_base/utils_signals.sh b/creation/web_base/utils_signals.sh index eb3ee5d76f..534f224e1e 100644 --- a/creation/web_base/utils_signals.sh +++ b/creation/web_base/utils_signals.sh @@ -15,6 +15,7 @@ signal_trap_with_arg() { local func func="$1" + shift 1 for sig ; do # shellcheck disable=SC2064 trap "${func} ${sig}" "${sig}" diff --git a/test/bats/creation_web_base_glidein_cleanup.bats b/test/bats/creation_web_base_glidein_cleanup.bats index f59fcd7e77..9f1706d584 100755 --- a/test/bats/creation_web_base_glidein_cleanup.bats +++ b/test/bats/creation_web_base_glidein_cleanup.bats @@ -7,9 +7,9 @@ load 'lib/bats-assert/load' [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR=../.. setup () { - source "$GWMS_SOURCEDIR"/creation/web_base/utils_io.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_log.sh source "$GWMS_SOURCEDIR"/creation/web_base/utils_xml.sh - source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/creation/web_base/utils_gs_log.sh source "$GWMS_SOURCEDIR"/creation/web_base/logging_utils.source source "$GWMS_SOURCEDIR"/creation/web_base/glidein_cleanup.sh 2>&3 source "$GWMS_SOURCEDIR"/creation/web_base/add_config_line.source @@ -135,7 +135,6 @@ setup () { teardown() { rm -rf "${glide_local_tmp_dir}" - rm -rf "${work_dir}" rm -rf "${start_dir}" rm -f "${PWD}/add_config_line.source" rm -f "glidein_config" diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index 0183012c14..ef6ff135fe 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -9,14 +9,14 @@ load 'lib/bats-assert/load' setup () { source "$GWMS_SOURCEDIR"/utils_gs_signals.sh source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh - source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_log.sh source "$GWMS_SOURCEDIR"/utils_params.sh source "$GWMS_SOURCEDIR"/utils_signals.sh source "$GWMS_SOURCEDIR"/utils_xml.sh source "$GWMS_SOURCEDIR"/utils_crypto.sh source "$GWMS_SOURCEDIR"/utils_gs_http.sh source "$GWMS_SOURCEDIR"/utils_gs_filesystem.sh - source "$GWMS_SOURCEDIR"/utils_gs_io.sh + source "$GWMS_SOURCEDIR"/utils_gs_log.sh source "$GWMS_SOURCEDIR"/logging_utils.source source "$GWMS_SOURCEDIR"/glidein_startup.sh source "$GWMS_SOURCEDIR"/add_config_line.source diff --git a/test/bats/creation_web_base_utils_crypto.bats b/test/bats/creation_web_base_utils_crypto.bats index b6a9caf3cc..9271bc1090 100755 --- a/test/bats/creation_web_base_utils_crypto.bats +++ b/test/bats/creation_web_base_utils_crypto.bats @@ -8,7 +8,7 @@ load 'lib/bats-assert/load' setup () { source "$GWMS_SOURCEDIR"/utils_crypto.sh - source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_log.sh } @test "md5wrapper" { diff --git a/test/bats/creation_web_base_utils_gs_filesystem.bats b/test/bats/creation_web_base_utils_gs_filesystem.bats index 46b5288614..82b7349217 100755 --- a/test/bats/creation_web_base_utils_gs_filesystem.bats +++ b/test/bats/creation_web_base_utils_gs_filesystem.bats @@ -56,6 +56,13 @@ setup () { run automatic_work_dir [ "$status" -eq 0 ] fi + rm -rf "${_CONDOR_SCRATCH_DIR}" + rm -rf "${OSG_WN_TMP}" + rm -rf "${TG_NODE_SCRATCH}" + rm -rf "${TG_CLUSTER_SCRATCH}" + rm -rf "${SCRATCH}" + rm -rf "${TMPDIR}" + rm -rf "${TMP}" } @test "dir_id" { @@ -96,6 +103,8 @@ early_glidein_failure() { assert_output --partial "Running in ${work_dir}" assert_output --partial "copied idtoken" [ "$status" -eq 0 ] + rm "trial.idtoken" + rm -rf "${work_dir}" } @test "copy_all" { @@ -108,13 +117,17 @@ early_glidein_failure() { touch "${tmp_dir}/afile1.txt" target_dir="/tmp/prova2" mkdir -p "$target_dir" + cd ${tmp_dir} run copy_all "pfile" "${target_dir}" + cd - [ "$output" == "" ] [ -f "${target_dir}"/file1.txt ] [ -f "${target_dir}"/file2.txt ] [ -f "${target_dir}"/file3.txt ] [ ! -f "${target_dir}"/pfile1.txt ] [ "$status" -eq 0 ] + rm -rf "$tmp_dir" + rm -rf "$target_dir" } @test "add_to_path" { @@ -129,17 +142,3 @@ early_glidein_failure() { [ "${GWMS_PATH}" == "${element}:${OLD_GWMS_PATH}" ] PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" } - -teardown() { - rm -rf "${_CONDOR_SCRATCH_DIR}" - rm -rf "${OSG_WN_TMP}" - rm -rf "${TG_NODE_SCRATCH}" - rm -rf "${TG_CLUSTER_SCRATCH}" - rm -rf "${SCRATCH}" - rm -rf "${TMPDIR}" - rm -rf "${TMP}" - rm -rf "$tmp_dir" - rm -rf "$target_dir" - rm "trial.idtoken" - rm -rf "${work_dir}" -} diff --git a/test/bats/creation_web_base_utils_gs_http.bats b/test/bats/creation_web_base_utils_gs_http.bats index eceb44a2f4..f6a8343614 100755 --- a/test/bats/creation_web_base_utils_gs_http.bats +++ b/test/bats/creation_web_base_utils_gs_http.bats @@ -8,7 +8,7 @@ load 'lib/bats-assert/load' setup () { source "$GWMS_SOURCEDIR"/utils_gs_http.sh - source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_log.sh } @test "get_repository_url" { diff --git a/test/bats/creation_web_base_utils_gs_io.bats b/test/bats/creation_web_base_utils_gs_io.bats index f39195fddb..7d1ccbf61f 100755 --- a/test/bats/creation_web_base_utils_gs_io.bats +++ b/test/bats/creation_web_base_utils_gs_io.bats @@ -7,8 +7,8 @@ load 'lib/bats-assert/load' [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" setup () { - source "$GWMS_SOURCEDIR"/utils_gs_io.sh - source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_gs_log.sh + source "$GWMS_SOURCEDIR"/utils_log.sh source "$GWMS_SOURCEDIR"/utils_crypto.sh } diff --git a/test/bats/creation_web_base_utils_gs_signals.bats b/test/bats/creation_web_base_utils_gs_signals.bats index b28d8c0650..c6a9703505 100755 --- a/test/bats/creation_web_base_utils_gs_signals.bats +++ b/test/bats/creation_web_base_utils_gs_signals.bats @@ -10,12 +10,12 @@ setup () { source "$GWMS_SOURCEDIR"/utils_gs_signals.sh } -@test "on_die_multi" { +@test "signal_on_die_multi" { echo "Testing the function killing one child..." >& 3 sleep 5 & pid=$! GWMS_MULTIGLIDEIN_CHILDS="${pid}" - run on_die_multi "KILL" + run signal_on_die_multi "KILL" if ! ps -p ${pid} > /dev/null then [ true ] @@ -30,7 +30,7 @@ setup () { sleep 5 & pid2=$! GWMS_MULTIGLIDEIN_CHILDS="${pid} ${pid2}" - run on_die_multi "KILL" + run signal_on_die_multi "KILL" if ! ps -p ${pid} > /dev/null; then [ true ] else @@ -45,6 +45,6 @@ setup () { assert_output --partial "forwarding KILL signal to" echo "Testing the function killing no children..." >& 3 GWMS_MULTIGLIDEIN_CHILDS= - run on_die_multi "KILL" + run signal_on_die_multi "KILL" [ "$status" -eq 0 ] } diff --git a/test/bats/creation_web_base_utils_gs_tarballs.bats b/test/bats/creation_web_base_utils_gs_tarballs.bats index b88314a118..e42aa62745 100755 --- a/test/bats/creation_web_base_utils_gs_tarballs.bats +++ b/test/bats/creation_web_base_utils_gs_tarballs.bats @@ -8,7 +8,7 @@ load 'lib/bats-assert/load' setup () { source "$GWMS_SOURCEDIR"/utils_gs_tarballs.sh - source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_log.sh source "$GWMS_SOURCEDIR"/get_id_selectors.source } diff --git a/test/bats/creation_web_base_utils_io.bats b/test/bats/creation_web_base_utils_io.bats index 06e83b412d..f39e6434a1 100755 --- a/test/bats/creation_web_base_utils_io.bats +++ b/test/bats/creation_web_base_utils_io.bats @@ -7,7 +7,7 @@ load 'lib/bats-assert/load' [[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" setup () { - source "$GWMS_SOURCEDIR"/utils_io.sh + source "$GWMS_SOURCEDIR"/utils_log.sh } @test "log_warn" { diff --git a/test/bats/creation_web_base_utils_signals.bats b/test/bats/creation_web_base_utils_signals.bats index 017e9df8f0..9683bc406c 100755 --- a/test/bats/creation_web_base_utils_signals.bats +++ b/test/bats/creation_web_base_utils_signals.bats @@ -10,20 +10,20 @@ setup () { source "$GWMS_SOURCEDIR"/utils_signals.sh } -@test "trap_with_arg" { +@test "signal_trap_with_arg" { echo "Testing the assignment of the handler to some signals..." >& 3 - run trap 'ignore_signal' SIGTERM SIGINT SIGQUIT + run trap 'signal_ignore' SIGTERM SIGINT SIGQUIT # Todo: How to check if the handler has been correctly assigned? [ "$output" == "" ] [ "$status" -eq 0 ] } -@test "on_die" { +@test "signal_on_die" { echo "Testing the launch of a signal to a process..." >& 3 sleep 5 & pid=$! GWMS_MULTIGLIDEIN_CHILDS="${pid}" - run on_die "KILL" + run signal_on_die "KILL" if ! ps -p ${pid} > /dev/null then [ true ] From 8e411211950209e1f82f1401ad11c3b29afae41a Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 25 Aug 2022 12:01:09 -0500 Subject: [PATCH 060/196] Modified some file names and solved some bugs in bats tests. --- creation/web_base/utils_gs_http.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index bdd0b08214..245b0c7570 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -194,7 +194,7 @@ fetch_file() { # fft_get_ss # fft_base_name # fft_condition_attr -# fft_condition_attr_val +# fft_condition_attr_val # Returns: # 0 in case of success # otherwise it returns the exit code of fetch_file_base From f28d339e76935f504fe60547aa0fa94ee310fb09 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 31 Aug 2022 11:36:31 -0500 Subject: [PATCH 061/196] Added external files management modifications with the creation of support functions in utils_scripts.sh and changes on other scripts. --- creation/web_base/glidein_cleanup.sh | 6 + creation/web_base/glidein_startup.sh | 15 ++- creation/web_base/utils_gs_http.sh | 116 ++++++++---------- creation/web_base/utils_gs_tarballs.sh | 4 - creation/web_base/utils_scripts.sh | 156 +++++++++++++++++++++++++ 5 files changed, 226 insertions(+), 71 deletions(-) create mode 100644 creation/web_base/utils_scripts.sh diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index c8988a513e..3a9e4baebe 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -42,6 +42,8 @@ glidein_cleanup() { fi fi fi + + custom_scripts "cleanup" } ################################ @@ -103,6 +105,10 @@ glidein_exit() { local exit_code factory_collector dlf condor_vars_file main_work_dir exit_code=$1 # Removed lines about $lock_file (lock file for whole machine) not present elsewhere + + ############################## + failure_scripts $exit_code + gwms_process_scripts "$GWMS_DIR" cleanup "${glidein_config}" global_result="" if [ -f otr_outlist.list ]; then diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 329310383d..1b66d04fad 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -508,7 +508,7 @@ _main(){ # fetch list file fetch_file_regular "${gs_id}" "${gs_file_list}" - + # Fetch files contained in list # TODO: $file is actually a list, so it cannot be doublequoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here # shellcheck disable=2086 @@ -516,8 +516,15 @@ _main(){ do if [ "${file:0:1}" != "#" ]; then fetch_file "${gs_id}" $file + if [ "${ffb_file_type}" != "wrapper" ] && [ "${ffb_file_type}" != "untar" ]; then + add_entry "$ffb_outname" "$ffb_file_type" "$ffb_time" "$ffb_coordination" "$ffb_period" "$ffb_cc_prefix" "$gs_id" "$fft_tar_source" + fi fi done < "${gs_id_work_dir}/${gs_file_list}" + + #################################### + #TODO: or where + custom_scripts "startup" # Files to go into the GWMS_PATH if [ "$gs_file_id" = "main at_file_list" ]; then @@ -541,6 +548,9 @@ _main(){ ############################# fixup_condor_dir + + ############################## + custom_scripts "before_job" ############################## # Start the glidein main script @@ -585,6 +595,9 @@ _main(){ echo "No message left" 1>&2 fi echo 1>&2 + + ############################ + custom_scripts "after_job" ######################### # clean up after I finish diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 245b0c7570..a8553dfba3 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -106,7 +106,7 @@ EOF # 2: target fname # 3: real fname fetch_file_regular() { - fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" + fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" "no_time" "no_code" } ############################################ @@ -122,6 +122,9 @@ fetch_file_regular() { # 6: periodic scripts prefix # 7: config check TRUE,FALSE # 8: config out TRUE,FALSE +# 9: time (startup, cleanup, before_job, after_job, periodic, specific_code, failure:exit_code, or a comma-separated combination of them) +# 10: integer_code (internal representation of the coordination number of the scripts) +# 11: tar_source # The above is the most recent list, below some adaptations for different versions # Used: # ifs_str @@ -130,35 +133,42 @@ fetch_file_regular() { # 0 in case of success # otherwise glidein_exit with 1 fetch_file() { - if [ $# -gt 8 ]; then + if [ $# -gt 11 ]; then # For compatibility w/ future versions (add new parameters at the end) - echo "More then 8 arguments, considering the first 8 ($#/${ifs_str}): $*" 1>&2 - elif [ $# -ne 8 ]; then - if [ $# -eq 7 ]; then + echo "More then 11 arguments, considering the first 11 ($#/${ifs_str}): $*" 1>&2 + elif [ $# -ne 11 ]; then + if [ $# -eq 10 ]; then #TODO: remove in version 3.3 # For compatibility with past versions (old file list formats) # 3.2.13 and older: prefix (par 6) added in #12705, 3.2.14? # 3.2.10 and older: period (par 5) added: fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "GLIDEIN_PS_" "$6" "$7"; then + if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "GLIDEIN_PS_" "$6" "$7" "$8" "$9" "${10}"; then glidein_exit 1 fi return 0 fi - if [ $# -eq 6 ]; then + if [ $# -eq 9 ]; then # added to maintain compatibility with older (3.2.10) file list format #TODO: remove in version 3.3 - if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6"; then + if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" "$7" "$8" "$9"; then + glidein_exit 1 + fi + return 0 + fi + if [ $# -eq 8 ]; then + # added to maintain compatibility with older (3.2.10) file list format + #TODO: remove in version 3.3 + if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" "$7" "$8" "NULL"; then glidein_exit 1 fi return 0 fi local ifs_str printf -v ifs_str '%q' "${IFS}" - log_warn "Not enough arguments in fetch_file, 8 expected ($#/${ifs_str}): $*" + log_warn "Not enough arguments in fetch_file, 11 expected ($#/${ifs_str}): $*" glidein_exit 1 fi - - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8"; then + if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}"; then glidein_exit 1 fi return 0 @@ -182,6 +192,8 @@ fetch_file() { # 6: periodic scripts prefix # 7: config check TRUE,FALSE # 8: config out TRUE,FALSE +# 9: time (startup, cleanup, before_job, after_job, periodic, specific_code, failure:exit_code, or a comma-separated combination of them) +# 10: integer_code (internal representation of the coordination number of the scripts) # Globals (r/w): # fft_id # fft_target_fname @@ -191,6 +203,9 @@ fetch_file() { # fft_cc_prefix # fft_config_check # fft_config_out +# fft_time +# fft_coordination +# fft_tar_source # fft_get_ss # fft_base_name # fft_condition_attr @@ -207,6 +222,9 @@ fetch_file_try() { fft_cc_prefix="$6" fft_config_check="$7" fft_config_out="$8" + fft_time="$9" + fft_coordination="${10}" + fft_tar_source="${11}" if [[ "${fft_config_check}" != "TRUE" ]]; then # TRUE is a special case, always be downloaded and processed @@ -224,8 +242,12 @@ fetch_file_try() { # if the variable fft_condition_attr is not defined or empty, do not download [[ -z "${fft_condition_attr_val}" ]] && return 0 fi - - fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" + + if [[ "${fft_tar_source}" != NULL ]]; then + return 0 + fi + + fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" "${fft_time}" "${fft_coordination}" "${fft_tar_source}" return $? # returning the exit code of fetch_file_base } @@ -238,8 +260,10 @@ fetch_file_try() { # 3: real fname # 4: file type (regular, exec, exec:s, untar, nocache) # 5: config out TRUE,FALSE -# 5: period (0 if not a periodic file) -# 6: periodic scripts prefix +# 6: period (0 if not a periodic file) +# 7: periodic scripts prefix +# 8: time (startup, cleanup, before_job, after_job, periodic, specific_code, failure:exit_code, or a comma-separated combination of them) +# 9: integer_code (internal representation of the coordination number of the scripts) # Globals: # ffb_id # ffb_target_fname @@ -255,6 +279,8 @@ fetch_file_try() { # ffb_untar_dir # ffb_outname # ffb_prefix +# ffb_time +# ffb_coordination # have_dummy_otrx # user_agent # ffb_url @@ -289,6 +315,8 @@ fetch_file_base() { ffb_period=$6 # condor cron prefix, used only for periodic executables ffb_cc_prefix="$7" + ffb_time="$8" + ffb_coordination="$9" ffb_work_dir="$(get_work_dir "${ffb_id}")" ffb_repository="$(get_repository_url "${ffb_id}")" ffb_tmp_outname="${ffb_work_dir}/${ffb_real_fname}" @@ -372,59 +400,13 @@ fetch_file_base() { return 1 fi fi - - # if executable, execute - if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then - if ! chmod u+x "${ffb_outname}"; then - log_warn "Error making '${ffb_outname}' executable" - return 1 - fi - if [ "${ffb_id}" = "main" ] && [ "${ffb_target_fname}" = "${last_script}" ]; then # last_script global for simplicity - echo "Skipping last script ${last_script}" 1>&2 - elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity - # TODO: temporary OR checking for cvmfs_umount.sh; to be removed after Bruno's ticket on cleanup [#25073] - echo "Skipping cleanup script ${ffb_outname} (${cleanup_script})" 1>&2 - cp "${ffb_outname}" "$gwms_exec_dir/cleanup/${ffb_target_fname}" - chmod a+x "${gwms_exec_dir}/cleanup/${ffb_target_fname}" - else - echo "Executing (flags:${ffb_file_type#exec}) ${ffb_outname}" - # have to do it here, as this will be run before any other script - chmod u+rx "${main_dir}"/error_augment.sh - - # the XML file will be overwritten now, and hopefully not an error situation - have_dummy_otrx=0 - "${main_dir}"/error_augment.sh -init - START=$(date +%s) - if [[ "${ffb_file_type}" = "exec:s" ]]; then - "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" - else - "${ffb_outname}" glidein_config "${ffb_id}" - fi - ret=$? - local END - END=$(date +%s) - "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_outname} glidein_config" "${START}" "${END}" #generating test result document - "${main_dir}"/error_augment.sh -concat - # TODO(F): qui - if [ ${ret} -ne 0 ]; then - echo "=== Validation error in ${ffb_outname} ===" 1>&2 - log_warn "Error running '${ffb_outname}'" - < otrx_output.xml awk 'BEGIN{fr=0;}/<[/]detail>/{fr=0;}{if (fr==1) print $0}//{fr=1;}' 1>&2 - return 1 - else - # If ran successfully and periodic, schedule to execute with schedd_cron - echo "=== validation OK in ${ffb_outname} (${ffb_period}) ===" 1>&2 - if [ "${ffb_period}" -gt 0 ]; then - add_periodic_script "${main_dir}/script_wrapper.sh" "${ffb_period}" "${work_dir}" "${ffb_outname}" glidein_config "${ffb_id}" "${ffb_cc_prefix}" - fi - fi - fi - elif [ "${ffb_file_type}" = "wrapper" ]; then + + # handle wrapper or untar files + if [ "${ffb_file_type}" = "wrapper" ]; then echo "${ffb_outname}" >> "${wrapper_list}" elif [ "${ffb_file_type}" = "untar" ]; then ffb_short_untar_dir="$(get_untar_subdir "${ffb_id}" "${ffb_target_fname}")" ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" - local START START=$(date +%s) (mkdir "${ffb_untar_dir}" && cd "${ffb_untar_dir}" && tar -xmzf "${ffb_outname}") 1>&2 ret=$? @@ -433,10 +415,11 @@ fetch_file_base() { "${main_dir}"/error_gen.sh -error "tar" "Corruption" "Error untarring '${ffb_outname}'" "file" "${ffb_outname}" "source_type" "${cfs_id}" "${main_dir}"/error_augment.sh -process ${cfs_rc} "tar" "${PWD}" "mkdir ${ffb_untar_dir} && cd ${ffb_untar_dir} && tar -xmzf ${ffb_outname}" "${START}" "$(date +%s)" "${main_dir}"/error_augment.sh -concat - log_warn "Error untarring '${ffb_outname}'" + warn "Error untarring '${ffb_outname}'" return 1 fi fi + #TODO: modify untar if [ "${ffb_config_out}" != "FALSE" ]; then ffb_prefix="$(get_prefix "${ffb_id}")" @@ -451,7 +434,7 @@ fetch_file_base() { fi fi fi - + if [ "${have_dummy_otrx}" -eq 1 ]; then # no one should really look at this file, but just to avoid confusion local date @@ -463,6 +446,7 @@ fetch_file_base() { return 0 } + ################################ # Perform a wget request # Arguments: diff --git a/creation/web_base/utils_gs_tarballs.sh b/creation/web_base/utils_gs_tarballs.sh index 476c695d2d..b126ff76e2 100644 --- a/creation/web_base/utils_gs_tarballs.sh +++ b/creation/web_base/utils_gs_tarballs.sh @@ -26,22 +26,18 @@ get_untar_subdir() { gus_id="$1" gus_fname="$2" - gus_prefix="$(get_prefix "${gus_id}")" gus_config_cfg="${gus_prefix}UNTAR_CFG_FILE" - gus_config_file="$(grep "^${gus_config_cfg} " glidein_config | cut -d ' ' -f 2-)" if [ -z "${gus_config_file}" ]; then log_warn "Error, cannot find '${gus_config_cfg}' in glidein_config." glidein_exit 1 fi - gus_dir="$(grep -i "^${gus_fname} " "${gus_config_file}" | cut -s -f 2-)" if [ -z "${gus_dir}" ]; then log_warn "Error, untar dir for '${gus_fname}' cannot be empty." glidein_exit 1 fi - echo "${gus_dir}" return 0 } diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh new file mode 100644 index 0000000000..327d9a2277 --- /dev/null +++ b/creation/web_base/utils_scripts.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + +#*******************************************************************# +# utils_scripts.sh # +# This script contains scripts utility functions # +#*******************************************************************# + +################################ +# Add an entry to the descriptor file containing the ordered list of scripts to handle +# Arguments: +# 1: filename +# 2: type +# 3: time +# 4: integer_code for coordination +# 5: period +# 6: prefix +# 7: id +# 8: tar_source +add_entry(){ + local e_real_fname e_type e_time e_coordination descriptor_file + e_real_fname="$1" + e_type="$2" + e_time="$3" + e_coordination="$4" + e_period="$5" + e_prefix="$6" + e_id="$7" + e_tar_source="$8" + descriptor_file="testfile" + + if [ ! -f "$descriptor_file" ]; then + echo "# File: $descriptor_file" > "$descriptor_file" + echo "#" >> "$descriptor_file" + echo "# Time OrderedFilename RealFilename Type Period Prefix Id" >> "$descriptor_file" + echo "################################################################################################" >> "$descriptor_file" + fi + OLD_IFS=$IFS + IFS=', ' read -r -a array <<< $e_time + for time_entry in "${array[@]}" + do + if [[ $e_tar_source != NULL ]]; then + ffb_short_untar_dir="$(get_untar_subdir "${e_id}" "${e_tar_source}")" + ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" + e_complete_fname="${ffb_untar_dir}/$e_real_fname" + else + e_complete_fname="$e_real_fname" + fi + echo "${time_entry} ${e_coordination}_${e_real_fname} ${e_complete_fname} ${e_type} ${e_period} ${e_prefix} ${e_id}" >> "$descriptor_file" + done + IFS=$OLD_IFS +} + +################################ +# Extract from the descriptor file the ordered list of scripts to handle +# Arguments: +# 1: target_time +# Globals(r/w): +# list +extract_entry_files(){ + local target_time descriptor_file + list="" + target_time="$1" + descriptor_file="testfile" + grep ^$target_time $descriptor_file | sort > ${target_time}_descriptor_file +} + +################################ +# Perform the requested action associated to each script of the given time +# Arguments: +# 1: target_time +# Used: +# list +custom_scripts(){ + local target_time + target_time="$1" + extract_entry_files "$target_time" + # space+tab separated file with multiple elements + while read -r line + do + if [ "${file:0:1}" == "#" ]; then + continue + fi + read -ra arr -d '' <<<"$line" + ffb_target_fname=${arr[2]} + ffb_file_type=${arr[3]} + ffb_period=${arr[4]} + ffb_cc_prefix=${arr[5]} + ffb_id=${arr[6]} + # if executable, execute + if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then + if ! chmod u+x "${ffb_target_fname}"; then + warn "Error making '${ffb_target_fname}' executable" + return 1 + fi + #TODO:save main + if [ "${ffb_id}" = "main" ] && [ "${ffb_target_fname}" = "${last_script}" ]; then # last_script global for simplicity + echo "Skipping last script ${last_script}" 1>&2 + elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity + # TODO: temporary OR checking for cvmfs_umount.sh; to be removed after Bruno's ticket on cleanup [#25073] + echo "Skipping cleanup script ${ffb_target_fname} (${cleanup_script})" 1>&2 + cp "${ffb_target_fname}" "$gwms_exec_dir/cleanup/${ffb_target_fname}" + chmod a+x "${gwms_exec_dir}/cleanup/${ffb_target_fname}" + else + echo "Executing (flags:${ffb_file_type#exec}) ${ffb_target_fname}" + # have to do it here, as this will be run before any other script + chmod u+rx "${main_dir}"/error_augment.sh + + # the XML file will be overwritten now, and hopefully not an error situation + have_dummy_otrx=0 + "${main_dir}"/error_augment.sh -init + START=$(date +%s) + if [[ "${ffb_file_type}" = "exec:s" ]]; then + "${main_dir}/singularity_wrapper.sh" "${ffb_target_fname}" glidein_config "${ffb_id}" + else + "${ffb_target_fname}" glidein_config "${ffb_id}" + fi + ret=$? + END=$(date +%s) + "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_target_fname} glidein_config" "${START}" "${END}" #generating test result document + "${main_dir}"/error_augment.sh -concat + if [ ${ret} -ne 0 ]; then + echo "=== Validation error in ${ffb_target_fname} ===" 1>&2 + warn "Error running '${ffb_target_fname}'" + < otrx_output.xml awk 'BEGIN{fr=0;}/<[/]detail>/{fr=0;}{if (fr==1) print $0}//{fr=1;}' 1>&2 + return 1 + else + # If ran successfully and periodic, schedule to execute with schedd_cron + echo "=== validation OK in ${ffb_target_fname} (${ffb_period}) ===" 1>&2 + if [ "${ffb_period}" -gt 0 ]; then + add_periodic_script "${main_dir}/script_wrapper.sh" "${ffb_period}" "${work_dir}" "${ffb_target_fname}" glidein_config "${ffb_id}" "${ffb_cc_prefix}" + fi + fi + fi + #TODO: Handle executables in tarballs + elif [[ "${ffb_file_type}" = "source" || "${ffb_file_type}" = "library:shell" ]]; then + source "${ffb_target_fname}" + # TODO: what about other library types? + fi + done < ${target_time}_descriptor_file + rm -f ${target_time}_descriptor_file +} + +milestone_call(){ + local code + code=$1 + custom_scripts "milestone:"$code +} + +failure_call(){ + local exit_code + exit_code=$1 + custom_scripts "failure:"$exit_code +} From b21bae535e414a92d9b67c951e17504734b8a23c Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 09:55:40 -0500 Subject: [PATCH 062/196] Added external files management modifications with the modification of xml-related functions and with the redefinement of bash script. --- creation/lib/cWDictFile.py | 95 ++++++++++++++----- creation/lib/cWParamDict.py | 57 ++++++++--- creation/lib/cWParams.py | 41 ++++---- creation/lib/cgWParamDict.py | 1 + creation/lib/cgWParams.py | 1 + creation/lib/cvWConsts.py | 2 + creation/lib/cvWParams.py | 3 +- creation/lib/factory_defaults.xml | 8 +- .../frontend/default_singularity_wrapper.sh | 3 +- creation/web_base/glidein_startup.sh | 6 -- creation/web_base/utils_gs_http.sh | 49 +++++++++- creation/web_base/utils_scripts.sh | 20 +++- 12 files changed, 208 insertions(+), 78 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index e27e451432..82af4f39de 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -986,7 +986,7 @@ def save_files(self, allow_overwrite=False): except OSError as e: raise DictFileError("Error writing into file %s" % filepath) - +#TODO(F) class FileDictFile(SimpleFileDictFile): """Dictionary file for files (file list). Used for list of transferred files. @@ -997,18 +997,57 @@ class FileDictFile(SimpleFileDictFile): the key is used as key for the dictionary and the data (file content) is added reading the file. Here the attributes stored as tuple in the dictionary value: 1. real_fname, i.e file name - 2. cache/exec/... keyword identifying the file type: regular, nocache, exec (:s modifier to run in singularity), untar, wrapper - 3. period period in seconds at which an executable is re-invoked (only for periodic executables, 0 otherwise) - 4. prefix startd_cron variables prefix (default is GLIDEIN_PS_) - 5. cond_download has a special value of TRUE - 6. config_out has a special value of FALSE - 7. data - String containing the data extracted from the file (real_fname) (not in the serialized dictionary) + 2. cache/exec/... keyword identifying the file type: + - regular + - nocache + - exec (:s modifier to run in singularity) + - untar (if a modifier is present, it will be the name of the directory where the tarball will be unpacked, the name of the tarball will be used otherwise) + - wrapper + - source + - library (:shell in case of a shell library) + 3. prefix startd_cron variables prefix (default is GLIDEIN_PS_) + 4. time - phase of the lifecycle where to execute/source the script: + - startup + - pre_job + - after_job + - cleanup + - periodic (:period will specify the period in ms, 1000 by default) + - milestone (:code will specify the milestone code) + - failure (:exit_code will specify the failure code) + Time will be ignored for files when type is different than exec or source. + A combination of time phases can be used. + 5. priority - priority in the order of execution inside a time phase [0-99]. + Relevant only for executable, source and library files. + Order String code representation Associated integer code (internal representation) + Factory pre_entry e-[g-] 10 + Frontend pre_entry pre_group e-g- 20 + Frontend pre_entry group e-g= 30 + Frontend pre_entry after_group e-g+ 40 + entry e=[g-,g+,g=] 50 + Frontend after_entry pre_group e+g- 60 + Frontend after_entry group e+g= 70 + Frontend after_entry after_group e+g+ 80 + Factory after_entry e+[g+] 90 + A custom value inside the range [0-99] can be specified in order to alter the order of execution. + 6. cond_download has a special value of TRUE + 7. tar_source - Name of the tarball containing the file, if the file is contained in a tarball + 8. config_out - has a special value of FALSE + 9. cond_attr - Name of a configuration switch. (like "ENABLE_KRB5") + The operation requested on the file will be performed only if that parameter will be set to 1. + This parameter is going to influence the config_out element. + 10. absdir_outattr - Name of a variable name. (like "KRB5_SUBSYS_DIR") + The variable will be set to the absolute path of the file if not a tarball + or the directory where the tarball was unpacked in case it is a tarball, + this last thing if and only if the unpacking actually happened (else it will not be defined.) + ENTRY_ will be prepended if the directive occurs in an entry. + 11. data - String containing the data extracted from the file (real_fname) (not in the serialized dictionary) + For placeholders, the real_name is empty (and the tuple starts w/ an empty string). Placeholders cannot be serialized (saved into file). Empty strings would cause error when parsed back. """ - DATA_LENGTH = 7 # Length of value (attributes + data) - PLACEHOLDER_VALUE = ("", "", 0, "", "", "", "") # The tuple should be DATA_LENGTH long and have the correct values + DATA_LENGTH = 11 # Length of value (attributes + data) + PLACEHOLDER_VALUE = ("", "", "", "", "", 0, "NULL", "", "", "", "", "") # The tuple should be DATA_LENGTH long and have the correct values def add_placeholder(self, key, allow_overwrite=True): # using DictFile, no file content (FileDictFile or SimpleFileDictFile) @@ -1018,20 +1057,24 @@ def is_placeholder(self, key): return self[key][0] == "" # empty real_fname can only be a placeholder @staticmethod - def make_val_tuple(file_name, file_type, period=0, prefix="GLIDEIN_PS_", cond_download="TRUE", config_out="FALSE"): + def make_val_tuple(file_name, file_type, prefix="GLIDEIN_PS_", time, priority=0, cond_download="TRUE", tar_source="NULL",config_out="FALSE", cond_attr, absdir_outattr): """Make a tuple with the DATA_LENGTH-1 attributes in the correct order using the defaults :param file_name: name of the file (aka real_fname) - :param file_type: type of the file (regular, nocache, exec, untar, wrapper). 'exec allows modifiers like ':s' - :param period: period for periodic executables (ignored otherwise, default: 0) + :param file_type: type of the file (regular, nocache, exec, untar, wrapper, source, library). 'exec allows modifiers like ':s' :param prefix: prefix for periodic executables (ignored otherwise, default: GLIDEIN_PS_) + :param time: specific phase of the lifecycle where to execute/source the script (startup, pre_job, after_job, cleanup, periodic, milestone, failure) + :param priority: priority of execution inside the phase [0-99] :param cond_download: conditional download (default: 'TRUE') + :param tar_source: name of the tarball containing the file :param config_out: config out (default: 'FALSE') + :param cond_attr: conditional attribute + :param absdir_outattr: absolute path variable :return: tuple with the DATA_LENGTH-1 attributes See class definition for more information about the attributes """ # TODO: should it do some value checking? valid constant, int, ... - return file_name, file_type, period, prefix, cond_download, config_out # python constructs the tuple + return file_name, file_type, prefix, time, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr # python constructs the tuple @staticmethod def val_to_file_name(val): @@ -1066,7 +1109,7 @@ def add_from_str(self, key, val, data, allow_overwrite=False, allow_overwrite_pl Args: key (str): file ID - val (tuple): lists of 6 or 7 components (see class definition) + val (tuple): lists of 7 or 8 components (see class definition) data (str): string w/ data to add allow_overwrite (bool): if True the existing files can be replaced (default: False) allow_overwrite_placeholder (bool): if True, placeholder files can be replaced even if allow_overwrite @@ -1095,7 +1138,7 @@ def add( Args: key (str): file ID - val (tuple): lists of 6 or 7 components (see class definition) + val (tuple): lists of 7 or 8 components (see class definition) allow_overwrite (bool): if True the existing files can be replaced (default: False) allow_overwrite_placeholder (bool): if True, placeholder files can be replaced even if allow_overwrite is False (default: True) @@ -1117,7 +1160,7 @@ def add( try: int(val[2]) # to check if is integer. Period must be int or convertible to int except (ValueError, IndexError): - raise DictFileError("Values '%s' not (real_fname,cache/exec,period,prefix,cond_download,config_out)" % val) + raise DictFileError("Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" % val) if len(val) == self.DATA_LENGTH: # Alt: return self.add_from_str(key, val[:self.DATA_LENGTH-1], val[self.DATA_LENGTH-1], allow_overwrite) @@ -1127,14 +1170,14 @@ def add( # Maybe check also string length or possible values? if "\n" in val[-1]: raise DictFileError( - "Values '%s' not (real_fname,cache/exec,period,prefix,cond_download,config_out)" % val + "Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" % val ) return self.add_from_file(key, val, os.path.join(self.dir, self.val_to_file_name(val)), allow_overwrite) else: - raise DictFileError("Values '%s' not (real_fname,cache/exec,period,prefix,cond_download,config_out)" % val) + raise DictFileError("Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" % val) def format_val(self, key, want_comments): - return "{} \t{} \t{} \t{} \t{} \t{} \t{}".format( + return "{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{}".format( key, self.vals[key][0], self.vals[key][1], @@ -1142,6 +1185,10 @@ def format_val(self, key, want_comments): self.vals[key][3], self.vals[key][4], self.vals[key][5], + self.vals[key][6], + self.vals[key][7], + self.vals[key][8], + self.vals[key][9] ) def file_header(self, want_comments): @@ -1151,7 +1198,7 @@ def file_header(self, want_comments): + "\n" + ( "# %s \t%s \t%s \t%s \t%s \t%s \t%s\n" - % ("Outfile", "InFile ", "Cache/exec", "Period", "Prefix", "Condition", "ConfigOut") + % ("Outfile", "InFile ", "Cache/exec", "Prefix", "Time", "Priority","Condition", "TarSource", "ConfigOut", "CondAttr","AbsdirOutattr") ) + ("#" * 89) ) @@ -1175,15 +1222,11 @@ def parse_val(self, line): if len(arr) != self.DATA_LENGTH: # compatibility w/ old formats - # 3.2.13 (no prefix): key, fname, type, period, cond_download, config_out - # 3.2.10 (no period, prefix): key, fname, type, cond_download, config_out + # 3.2.13 (no prefix): key, fname, type, time, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr # TODO: remove in 3.3 or after a few version (will break upgrade) if len(arr) == self.DATA_LENGTH - 1: # For upgrade from 3.2.13 to 3.2.11 - return self.add(arr[0], [arr[1], arr[2], arr[3], "GLIDEIN_PS_", arr[4], arr[5]]) - elif len(arr) == self.DATA_LENGTH - 2: - # For upgrade from 3.2.10 or earlier - return self.add(arr[0], [arr[1], arr[2], 0, "GLIDEIN_PS_", arr[3], arr[4]]) + return self.add(arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9]]) raise RuntimeError( "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) ) diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index feec09c4c6..36d01f6824 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -63,16 +63,31 @@ def add_file_unparsed(user_file, dicts, is_factory): relfname = os.path.basename(absfname) # default is the final part of absfname if len(relfname) < 1: raise RuntimeError("Found a file element with an empty relfname: %s" % user_file) - + time = user_file.time is_const = is_true(user_file.const) - is_executable = is_true(user_file.executable) - is_wrapper = is_true(user_file.wrapper) - do_untar = is_true(user_file.untar) + is_executable = is_true(user_file.executable) | (user_file.type.startswith("exec")) + is_wrapper = is_true(user_file.wrapper) | (user_file.type=="wrapper") + is_source = (user_file.type=="source") + is_library = (user_file.type.startswith("library")) + is_periodic = (user_file.type.startswith("periodic")) + do_untar = is_true(user_file.untar) | (user_file.type.startswith("untar")) try: - period_value = int(user_file.period) + if user_file.is_periodic: + period_value = int(user_file.type.split(":")[1]) except (AttributeError, KeyError, ValueError, TypeError): period_value = 0 - + #TODO(F): what to do with period and after list + + priority=user_file.priority + if priority < 0 | priority > 99: + raise RuntimeError("Priority value out of the range [0,99]: %s" % user_file) + + # Extended for all categories + config_out = user_file.absdir_outattr + if config_out is None: + config_out = "FALSE" + cond_attr = user_file.cond_attr + if is_factory: # Factory (file_list, after_file_list) file_list_idx = "file_list" @@ -95,6 +110,10 @@ def add_file_unparsed(user_file, dicts, is_factory): if not is_executable: raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) + if is_executable | is_source | is_library: + if !(time.startswith("startup")) & !(time.startswith("cleanup")) & !(time.startswith("after_job")) & !(time.startswith("before_job")) & !(time.startswith("periodic:")) & !(time.startswith("milestone:")) & !(time.startswith("failure:"): + # we use startswith since we may have combination of time phases (e.g. startup, cleanup) + raise RuntimeError("The file does not have a valid time phase value: %s" % user_file) if is_executable: # a script if not is_const: raise RuntimeError("A file cannot be executable if it is not constant: %s" % user_file) @@ -131,14 +150,9 @@ def add_file_unparsed(user_file, dicts, is_factory): if not is_const: raise RuntimeError("A file cannot be untarred if it is not constant: %s" % user_file) - wnsubdir = user_file.untar_options.dir + wnsubdir = user_file.type.split(":")[1] if wnsubdir is None: - wnsubdir = relfname.split(".", 1)[0] # default is relfname up to the first . - - config_out = user_file.untar_options.absdir_outattr - if config_out is None: - config_out = "FALSE" - cond_attr = user_file.untar_options.cond_attr + wnsubdir = absfname # default is relfname up to the first dicts[file_list_idx].add_from_file( relfname, @@ -148,7 +162,22 @@ def add_file_unparsed(user_file, dicts, is_factory): absfname, ) dicts["untar_cfg"].add(relfname, wnsubdir) - + elif is_source: + if not is_const: + raise RuntimeError("A file cannot be sourced if it is not constant: %s" % user_file) + if do_untar: + raise RuntimeError("A tar file cannot be sourced: %s" % user_file) + if is_wrapper: + raise RuntimeError("A wrapper file cannot be an sourced: %s" % user_file) + elif is_library: + if not is_const: + raise RuntimeError("A file cannot be a library if it is not constant: %s" % user_file) + if do_untar: + raise RuntimeError("A tar file cannot be a library: %s" % user_file) + if is_wrapper: + raise RuntimeError("A wrapper file cannot be an a library: %s" % user_file) + if inside_tar: + #TODO(F): ? else: # not executable nor tarball => simple file if is_const: val = "regular" diff --git a/creation/lib/cWParams.py b/creation/lib/cWParams.py index ab5197348e..c04c09de9f 100644 --- a/creation/lib/cWParams.py +++ b/creation/lib/cWParams.py @@ -448,7 +448,7 @@ class CommonSubParams(SubParams): def extract_attr_val(self, attr_obj): return extract_attr_val(attr_obj) - +#TODO(F) class CommonParams(Params): # populate self.defaults def init_support_defaults(self): @@ -498,38 +498,39 @@ def init_support_defaults(self): None, ) self.file_defaults["untar"] = ("False", "Bool", "Do I need to untar it? ", None) - self.file_defaults["period"] = (0, "int", 'Re-run the executable every "period" seconds if > 0.', None) self.file_defaults["prefix"] = ("GLIDEIN_PS_", "string", "Prefix used for periodic jobs (STARTD_CRON).", None) self.file_defaults["type"] = ( None, "string", - 'File type (regular,run,source). Allows modifiers like ":singularity" to run in singularity.', + 'File type (regular,run,source,untar,library,wrapper). Allows modifiers like ":singularity" to run in singularity.', None, ) - # TODO: consider adding "time" setup, prejob, postjob, cleanup, periodic. setup & cleanup w/ qualifier :bebg-aeag before/after entry + before/after group og na (group positioning does not apply to factory files) - # to add check scripts around jobs: self.file_defaults["job_wrap"]=("no","pre|post|no",'Run the executable before (pre) or after (post) each job.',None) - - untar_defaults = CommentedOrderedDict() - untar_defaults["cond_attr"] = ( - "TRUE", - "attrname", - "If not the special value TRUE, the attribute name used at runtime to determine if the file should be untarred or not.", + self.file_defaults["priority"] = ( + 0, + "integer [0,99]", + "Priority value to alter the order of execution inside a lifecycle phase."", None, ) - untar_defaults["dir"] = ( - None, - "dirname", - "Subdirectory in which to untar. (defaults to relname up to first .)", - None, + + #TODO(F): ok specificare qualifier come secondo argomento? + self.file_defaults["time"] = ( + "no_time", "startup|before_job|after_job|cleanup|periodic:period|milestone:code|failure:code|no", 'Run the executable at startup, before (pre) or after (post) each job, at cleanup, at a milestone or in front of a failure.', None ) - untar_defaults["absdir_outattr"] = ( + self.file_defaults["tar_source"] = ( + NULL, "tarball", 'Specify the tar source where the file is contained.', None + ) + self.file_defaults["cond_attr"] = ( + "TRUE", + "attrname", + "If not the special value TRUE, the attribute name used at runtime to determine if the file should be untarred or not.", + None, + ) + self.file_defaults["absdir_outattr"] = ( None, "attrname", "Attribute to be set to the abs dir name where the tarball was unpacked. Will be defined only if untar effectively done. (Not defined if None)", None, - ) - self.file_defaults["untar_options"] = untar_defaults - + ) self.monitor_defaults = CommentedOrderedDict() self.monitor_defaults["javascriptRRD_dir"] = ( os.path.join(self.src_dir, "../../externals/flot"), diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 178d8132e3..7e821ced1c 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -902,6 +902,7 @@ def new_SubDicts(self, sub_name): # ############################################################ +#TODO(F): what to do,source, library ############################################# # Add a user file residing in the stage area # file as described by Params.file_defaults diff --git a/creation/lib/cgWParams.py b/creation/lib/cgWParams.py index fdc47d8d38..f956c2f011 100644 --- a/creation/lib/cgWParams.py +++ b/creation/lib/cgWParams.py @@ -474,6 +474,7 @@ def init_defaults(self): "Should this file be loaded after the entry ones?", None, ) + #TODO(F): what to do? self.defaults["entries"] = (OrderedDict(), "Dictionary of entries", "Each entry contains", self.entry_defaults) diff --git a/creation/lib/cvWConsts.py b/creation/lib/cvWConsts.py index 0c8ea3138e..77a5c58c72 100644 --- a/creation/lib/cvWConsts.py +++ b/creation/lib/cvWConsts.py @@ -20,6 +20,8 @@ AFTERGROUP_FILE_LISTFILE = "aftergroup_%s" % cWConsts.FILE_LISTFILE PREENTRY_FILE_LISTFILE = "preentry_%s" % cWConsts.FILE_LISTFILE AFTERGROUP_PREENTRY_FILE_LISTFILE = "aftergroup_%s" % PREENTRY_FILE_LISTFILE +#TODO(F): remain the same? + PARAMS_FILE = "params.cfg" ATTRS_FILE = "attrs.cfg" diff --git a/creation/lib/cvWParams.py b/creation/lib/cvWParams.py index d2d2491d0e..930f6decac 100644 --- a/creation/lib/cvWParams.py +++ b/creation/lib/cvWParams.py @@ -54,6 +54,7 @@ def init_defaults(self): # VO scripts should start after the factory has been set completely # but there could be exceptions + #TODO(F): remove this one? # Files/Validation/Custom scripts settings for frontend self.file_defaults["after_entry"] = ( "True", @@ -559,7 +560,7 @@ def init_defaults(self): "Should this file be loaded after the group ones?", None, ) - + #TODO(F):what to do here? global_config_defaults = cWParams.CommentedOrderedDict() global_config_defaults["ignore_down_entries"] = [ "False", diff --git a/creation/lib/factory_defaults.xml b/creation/lib/factory_defaults.xml index 403e4e63a0..d81e378d68 100644 --- a/creation/lib/factory_defaults.xml +++ b/creation/lib/factory_defaults.xml @@ -115,9 +115,7 @@ SPDX-License-Identifier: Apache-2.0 - - - + @@ -145,9 +143,7 @@ SPDX-License-Identifier: Apache-2.0 - - - + diff --git a/creation/web_base/frontend/default_singularity_wrapper.sh b/creation/web_base/frontend/default_singularity_wrapper.sh index 70fdf9a584..8bc69f5ec8 100644 --- a/creation/web_base/frontend/default_singularity_wrapper.sh +++ b/creation/web_base/frontend/default_singularity_wrapper.sh @@ -203,7 +203,8 @@ fi info_dbg "GWMS singularity wrapper, final setup." # Removed local code in favor of gwms_process_scripts from singularity_lib.sh and setup_prejob.sh (CodeRM1) -gwms_process_scripts "$GWMS_DIR" prejob "$glidein_config" +#gwms_process_scripts "$GWMS_DIR" prejob "$glidein_config" +custom_scripts "before_job" # TODO: This is OSG specific. Should there be something similar in GWMS? ############################### diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 1b66d04fad..860ee829da 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -522,9 +522,6 @@ _main(){ fi done < "${gs_id_work_dir}/${gs_file_list}" - #################################### - #TODO: or where - custom_scripts "startup" # Files to go into the GWMS_PATH if [ "$gs_file_id" = "main at_file_list" ]; then @@ -548,9 +545,6 @@ _main(){ ############################# fixup_condor_dir - - ############################## - custom_scripts "before_job" ############################## # Start the glidein main script diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index a8553dfba3..81b55427db 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -401,7 +401,54 @@ fetch_file_base() { fi fi - # handle wrapper or untar files + # if startup and executable, execute + if [[ "${ffb_time}" = "startup" ]]; then + if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then + if ! chmod u+x "${ffb_outname}"; then + warn "Error making '${ffb_outname}' executable" + return 1 + fi + if [ "${ffb_id}" = "main" ] && [ "${ffb_target_fname}" = "${last_script}" ]; then # last_script global for simplicity + echo "Skipping last script ${last_script}" 1>&2 + elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity + # TODO: temporary OR checking for cvmfs_umount.sh; to be removed after Bruno's ticket on cleanup [#25073] + echo "Skipping cleanup script ${ffb_outname} (${cleanup_script})" 1>&2 + cp "${ffb_outname}" "$gwms_exec_dir/cleanup/${ffb_target_fname}" + chmod a+x "${gwms_exec_dir}/cleanup/${ffb_target_fname}" + else + echo "Executing (flags:${ffb_file_type#exec}) ${ffb_outname}" + # have to do it here, as this will be run before any other script + chmod u+rx "${main_dir}"/error_augment.sh + + # the XML file will be overwritten now, and hopefully not an error situation + have_dummy_otrx=0 + "${main_dir}"/error_augment.sh -init + START=$(date +%s) + if [[ "${ffb_file_type}" = "exec:s" ]]; then + "${main_dir}/singularity_wrapper.sh" "${ffb_outname}" glidein_config "${ffb_id}" + else + "${ffb_outname}" glidein_config "${ffb_id}" + fi + ret=$? + END=$(date +%s) + "${main_dir}"/error_augment.sh -process ${ret} "${ffb_id}/${ffb_target_fname}" "${PWD}" "${ffb_outname} glidein_config" "${START}" "${END}" #generating test result document + "${main_dir}"/error_augment.sh -concat + if [ ${ret} -ne 0 ]; then + echo "=== Validation error in ${ffb_outname} ===" 1>&2 + warn "Error running '${ffb_outname}'" + < otrx_output.xml awk 'BEGIN{fr=0;}/<[/]detail>/{fr=0;}{if (fr==1) print $0}//{fr=1;}' 1>&2 + return 1 + else + # If ran successfully and periodic, schedule to execute with schedd_cron + echo "=== validation OK in ${ffb_outname} (${ffb_period}) ===" 1>&2 + if [ "${ffb_period}" -gt 0 ]; then + add_periodic_script "${main_dir}/script_wrapper.sh" "${ffb_period}" "${work_dir}" "${ffb_outname}" glidein_config "${ffb_id}" "${ffb_cc_prefix}" + fi + fi + fi + fi + fi + # handle wrapper or untar files regardless of the moment if [ "${ffb_file_type}" = "wrapper" ]; then echo "${ffb_outname}" >> "${wrapper_list}" elif [ "${ffb_file_type}" = "untar" ]; then diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 327d9a2277..2aef104b1c 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -45,6 +45,7 @@ add_entry(){ ffb_short_untar_dir="$(get_untar_subdir "${e_id}" "${e_tar_source}")" ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" e_complete_fname="${ffb_untar_dir}/$e_real_fname" + #TODO: check if filename is a path what happens else e_complete_fname="$e_real_fname" fi @@ -83,7 +84,7 @@ custom_scripts(){ if [ "${file:0:1}" == "#" ]; then continue fi - read -ra arr -d '' <<<"$line" + read -ra arr -d ' ' <<< "$line" ffb_target_fname=${arr[2]} ffb_file_type=${arr[3]} ffb_period=${arr[4]} @@ -95,7 +96,6 @@ custom_scripts(){ warn "Error making '${ffb_target_fname}' executable" return 1 fi - #TODO:save main if [ "${ffb_id}" = "main" ] && [ "${ffb_target_fname}" = "${last_script}" ]; then # last_script global for simplicity echo "Skipping last script ${last_script}" 1>&2 elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity @@ -134,7 +134,6 @@ custom_scripts(){ fi fi fi - #TODO: Handle executables in tarballs elif [[ "${ffb_file_type}" = "source" || "${ffb_file_type}" = "library:shell" ]]; then source "${ffb_target_fname}" # TODO: what about other library types? @@ -143,12 +142,27 @@ custom_scripts(){ rm -f ${target_time}_descriptor_file } +################################ +# Ask for the execution of scripts related to a milestone +# Arguments: +# 1: code +# Returns: +# 1 in case of a failure (code including space) milestone_call(){ local code code=$1 + # no spaces are allowed in the code + if [[ "$code" == *" "* ]]; then + return 1 + fi custom_scripts "milestone:"$code + return 0 } +################################ +# Ask for the execution of scripts related to a failure code +# Arguments: +# 1: exit code failure_call(){ local exit_code exit_code=$1 From 559bcb46a26ba5dd64a71d246dc036d6d661a200 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 10:05:39 -0500 Subject: [PATCH 063/196] Improved descriptor file handling --- creation/web_base/glidein_startup.sh | 2 +- creation/web_base/utils_scripts.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 860ee829da..3a159fa803 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -516,7 +516,7 @@ _main(){ do if [ "${file:0:1}" != "#" ]; then fetch_file "${gs_id}" $file - if [ "${ffb_file_type}" != "wrapper" ] && [ "${ffb_file_type}" != "untar" ]; then + if [ "${ffb_file_type}" != "wrapper" ] && [ "${ffb_file_type}" != "untar" ] && [ "${ffb_file_type}" != "regular" ]; then add_entry "$ffb_outname" "$ffb_file_type" "$ffb_time" "$ffb_coordination" "$ffb_period" "$ffb_cc_prefix" "$gs_id" "$fft_tar_source" fi fi diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 2aef104b1c..4a9738812b 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -91,7 +91,7 @@ custom_scripts(){ ffb_cc_prefix=${arr[5]} ffb_id=${arr[6]} # if executable, execute - if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then + if [[ "${ffb_file_type}" = "exec"* ]]; then if ! chmod u+x "${ffb_target_fname}"; then warn "Error making '${ffb_target_fname}' executable" return 1 From b22ed5d96e01d0f8fbee5e99c5ed142c7babe201 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 11:01:47 -0500 Subject: [PATCH 064/196] Pre-commit modifications --- creation/lib/cWDictFile.py | 95 ++++++++++++++++++++++------ creation/lib/cWParamDict.py | 53 +++++++++------- creation/lib/cWParams.py | 27 +++++--- creation/lib/cgWParamDict.py | 2 +- creation/lib/cgWParams.py | 2 +- creation/lib/cvWConsts.py | 2 +- creation/lib/cvWParams.py | 4 +- creation/web_base/glidein_cleanup.sh | 6 +- creation/web_base/glidein_startup.sh | 6 +- creation/web_base/utils_gs_http.sh | 10 +-- creation/web_base/utils_scripts.sh | 12 ++-- 11 files changed, 145 insertions(+), 74 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index 82af4f39de..ff8628326b 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -986,7 +986,8 @@ def save_files(self, allow_overwrite=False): except OSError as e: raise DictFileError("Error writing into file %s" % filepath) -#TODO(F) + +# TODO(F) class FileDictFile(SimpleFileDictFile): """Dictionary file for files (file list). Used for list of transferred files. @@ -997,9 +998,9 @@ class FileDictFile(SimpleFileDictFile): the key is used as key for the dictionary and the data (file content) is added reading the file. Here the attributes stored as tuple in the dictionary value: 1. real_fname, i.e file name - 2. cache/exec/... keyword identifying the file type: - - regular - - nocache + 2. cache/exec/... keyword identifying the file type: + - regular + - nocache - exec (:s modifier to run in singularity) - untar (if a modifier is present, it will be the name of the directory where the tarball will be unpacked, the name of the tarball will be used otherwise) - wrapper @@ -1016,7 +1017,7 @@ class FileDictFile(SimpleFileDictFile): - failure (:exit_code will specify the failure code) Time will be ignored for files when type is different than exec or source. A combination of time phases can be used. - 5. priority - priority in the order of execution inside a time phase [0-99]. + 5. priority - priority in the order of execution inside a time phase [0-99]. Relevant only for executable, source and library files. Order String code representation Associated integer code (internal representation) Factory pre_entry e-[g-] 10 @@ -1031,23 +1032,36 @@ class FileDictFile(SimpleFileDictFile): A custom value inside the range [0-99] can be specified in order to alter the order of execution. 6. cond_download has a special value of TRUE 7. tar_source - Name of the tarball containing the file, if the file is contained in a tarball - 8. config_out - has a special value of FALSE + 8. config_out - has a special value of FALSE 9. cond_attr - Name of a configuration switch. (like "ENABLE_KRB5") - The operation requested on the file will be performed only if that parameter will be set to 1. + The operation requested on the file will be performed only if that parameter will be set to 1. This parameter is going to influence the config_out element. 10. absdir_outattr - Name of a variable name. (like "KRB5_SUBSYS_DIR") The variable will be set to the absolute path of the file if not a tarball - or the directory where the tarball was unpacked in case it is a tarball, - this last thing if and only if the unpacking actually happened (else it will not be defined.) + or the directory where the tarball was unpacked in case it is a tarball, + this last thing if and only if the unpacking actually happened (else it will not be defined.) ENTRY_ will be prepended if the directive occurs in an entry. 11. data - String containing the data extracted from the file (real_fname) (not in the serialized dictionary) - + For placeholders, the real_name is empty (and the tuple starts w/ an empty string). Placeholders cannot be serialized (saved into file). Empty strings would cause error when parsed back. """ DATA_LENGTH = 11 # Length of value (attributes + data) - PLACEHOLDER_VALUE = ("", "", "", "", "", 0, "NULL", "", "", "", "", "") # The tuple should be DATA_LENGTH long and have the correct values + PLACEHOLDER_VALUE = ( + "", + "", + "", + "", + "", + 0, + "NULL", + "", + "", + "", + "", + "", + ) # The tuple should be DATA_LENGTH long and have the correct values def add_placeholder(self, key, allow_overwrite=True): # using DictFile, no file content (FileDictFile or SimpleFileDictFile) @@ -1057,7 +1071,18 @@ def is_placeholder(self, key): return self[key][0] == "" # empty real_fname can only be a placeholder @staticmethod - def make_val_tuple(file_name, file_type, prefix="GLIDEIN_PS_", time, priority=0, cond_download="TRUE", tar_source="NULL",config_out="FALSE", cond_attr, absdir_outattr): + def make_val_tuple( + file_name, + file_type, + prefix="GLIDEIN_PS_", + time="startup", + priority=0, + cond_download="TRUE", + tar_source="NULL", + config_out="FALSE", + cond_attr="NULL", + absdir_outattr="NULL", + ): """Make a tuple with the DATA_LENGTH-1 attributes in the correct order using the defaults :param file_name: name of the file (aka real_fname) @@ -1074,7 +1099,18 @@ def make_val_tuple(file_name, file_type, prefix="GLIDEIN_PS_", time, priority=0, See class definition for more information about the attributes """ # TODO: should it do some value checking? valid constant, int, ... - return file_name, file_type, prefix, time, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr # python constructs the tuple + return ( + file_name, + file_type, + prefix, + time, + priority, + cond_download, + tar_source, + config_out, + cond_attr, + absdir_outattr, + ) # python constructs the tuple @staticmethod def val_to_file_name(val): @@ -1160,7 +1196,10 @@ def add( try: int(val[2]) # to check if is integer. Period must be int or convertible to int except (ValueError, IndexError): - raise DictFileError("Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" % val) + raise DictFileError( + "Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" + % val + ) if len(val) == self.DATA_LENGTH: # Alt: return self.add_from_str(key, val[:self.DATA_LENGTH-1], val[self.DATA_LENGTH-1], allow_overwrite) @@ -1170,11 +1209,15 @@ def add( # Maybe check also string length or possible values? if "\n" in val[-1]: raise DictFileError( - "Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" % val + "Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" + % val ) return self.add_from_file(key, val, os.path.join(self.dir, self.val_to_file_name(val)), allow_overwrite) else: - raise DictFileError("Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" % val) + raise DictFileError( + "Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" + % val + ) def format_val(self, key, want_comments): return "{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{}".format( @@ -1188,7 +1231,7 @@ def format_val(self, key, want_comments): self.vals[key][6], self.vals[key][7], self.vals[key][8], - self.vals[key][9] + self.vals[key][9], ) def file_header(self, want_comments): @@ -1198,7 +1241,19 @@ def file_header(self, want_comments): + "\n" + ( "# %s \t%s \t%s \t%s \t%s \t%s \t%s\n" - % ("Outfile", "InFile ", "Cache/exec", "Prefix", "Time", "Priority","Condition", "TarSource", "ConfigOut", "CondAttr","AbsdirOutattr") + % ( + "Outfile", + "InFile ", + "Cache/exec", + "Prefix", + "Time", + "Priority", + "Condition", + "TarSource", + "ConfigOut", + "CondAttr", + "AbsdirOutattr", + ) ) + ("#" * 89) ) @@ -1226,7 +1281,9 @@ def parse_val(self, line): # TODO: remove in 3.3 or after a few version (will break upgrade) if len(arr) == self.DATA_LENGTH - 1: # For upgrade from 3.2.13 to 3.2.11 - return self.add(arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9]]) + return self.add( + arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9]] + ) raise RuntimeError( "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) ) diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index 36d01f6824..336c89164c 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -65,29 +65,29 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("Found a file element with an empty relfname: %s" % user_file) time = user_file.time is_const = is_true(user_file.const) - is_executable = is_true(user_file.executable) | (user_file.type.startswith("exec")) - is_wrapper = is_true(user_file.wrapper) | (user_file.type=="wrapper") - is_source = (user_file.type=="source") - is_library = (user_file.type.startswith("library")) - is_periodic = (user_file.type.startswith("periodic")) - do_untar = is_true(user_file.untar) | (user_file.type.startswith("untar")) + is_executable = is_true(user_file.executable) or (user_file.type.startswith("exec")) + is_wrapper = is_true(user_file.wrapper) or (user_file.type == "wrapper") + is_source = user_file.type == "source" + is_library = user_file.type.startswith("library") + is_periodic = user_file.type.startswith("periodic") + do_untar = is_true(user_file.untar) or (user_file.type.startswith("untar")) try: if user_file.is_periodic: period_value = int(user_file.type.split(":")[1]) except (AttributeError, KeyError, ValueError, TypeError): period_value = 0 - #TODO(F): what to do with period and after list - - priority=user_file.priority - if priority < 0 | priority > 99: + # TODO(F): what to do with period and after list + + priority = user_file.priority + if priority < 0 or priority > 99: raise RuntimeError("Priority value out of the range [0,99]: %s" % user_file) - + # Extended for all categories config_out = user_file.absdir_outattr if config_out is None: - config_out = "FALSE" + config_out = "FALSE" cond_attr = user_file.cond_attr - + if is_factory: # Factory (file_list, after_file_list) file_list_idx = "file_list" @@ -110,10 +110,18 @@ def add_file_unparsed(user_file, dicts, is_factory): if not is_executable: raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) - if is_executable | is_source | is_library: - if !(time.startswith("startup")) & !(time.startswith("cleanup")) & !(time.startswith("after_job")) & !(time.startswith("before_job")) & !(time.startswith("periodic:")) & !(time.startswith("milestone:")) & !(time.startswith("failure:"): + if is_executable or is_source or is_library: + if ( + not time.startswith("startup") + and not time.startswith("cleanup") + and not time.startswith("after_job") + and not time.startswith("before_job") + and not time.startswith("periodic:") + and not time.startswith("milestone:") + and not time.startswith("failure:") + ): # we use startswith since we may have combination of time phases (e.g. startup, cleanup) - raise RuntimeError("The file does not have a valid time phase value: %s" % user_file) + raise RuntimeError("The file does not have a valid time phase value: %s" % user_file) if is_executable: # a script if not is_const: raise RuntimeError("A file cannot be executable if it is not constant: %s" % user_file) @@ -152,7 +160,7 @@ def add_file_unparsed(user_file, dicts, is_factory): wnsubdir = user_file.type.split(":")[1] if wnsubdir is None: - wnsubdir = absfname # default is relfname up to the first + wnsubdir = absfname # default is relfname up to the first dicts[file_list_idx].add_from_file( relfname, @@ -163,11 +171,11 @@ def add_file_unparsed(user_file, dicts, is_factory): ) dicts["untar_cfg"].add(relfname, wnsubdir) elif is_source: - if not is_const: + if not is_const: raise RuntimeError("A file cannot be sourced if it is not constant: %s" % user_file) - if do_untar: + if do_untar: raise RuntimeError("A tar file cannot be sourced: %s" % user_file) - if is_wrapper: + if is_wrapper: raise RuntimeError("A wrapper file cannot be an sourced: %s" % user_file) elif is_library: if not is_const: @@ -176,9 +184,8 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("A tar file cannot be a library: %s" % user_file) if is_wrapper: raise RuntimeError("A wrapper file cannot be an a library: %s" % user_file) - if inside_tar: - #TODO(F): ? - else: # not executable nor tarball => simple file + else: + # not executable nor tarball => simple file if is_const: val = "regular" dicts[file_list_idx].add_from_file( diff --git a/creation/lib/cWParams.py b/creation/lib/cWParams.py index c04c09de9f..03dcda0d14 100644 --- a/creation/lib/cWParams.py +++ b/creation/lib/cWParams.py @@ -448,7 +448,8 @@ class CommonSubParams(SubParams): def extract_attr_val(self, attr_obj): return extract_attr_val(attr_obj) -#TODO(F) + +# TODO(F) class CommonParams(Params): # populate self.defaults def init_support_defaults(self): @@ -508,29 +509,35 @@ def init_support_defaults(self): self.file_defaults["priority"] = ( 0, "integer [0,99]", - "Priority value to alter the order of execution inside a lifecycle phase."", + "Priority value to alter the order of execution inside a lifecycle phase.", None, ) - - #TODO(F): ok specificare qualifier come secondo argomento? + + # TODO(F): ok specificare qualifier come secondo argomento? self.file_defaults["time"] = ( - "no_time", "startup|before_job|after_job|cleanup|periodic:period|milestone:code|failure:code|no", 'Run the executable at startup, before (pre) or after (post) each job, at cleanup, at a milestone or in front of a failure.', None + "no_time", + "startup|before_job|after_job|cleanup|periodic:period|milestone:code|failure:code|no", + "Run the executable at startup, before (pre) or after (post) each job, at cleanup, at a milestone or in front of a failure.", + None, ) self.file_defaults["tar_source"] = ( - NULL, "tarball", 'Specify the tar source where the file is contained.', None - ) + NULL, + "tarball", + "Specify the tar source where the file is contained.", + None, + ) self.file_defaults["cond_attr"] = ( "TRUE", "attrname", - "If not the special value TRUE, the attribute name used at runtime to determine if the file should be untarred or not.", - None, + "If not the special value TRUE, the attribute name used at runtime to determine if the file should be untarred or not.", + None, ) self.file_defaults["absdir_outattr"] = ( None, "attrname", "Attribute to be set to the abs dir name where the tarball was unpacked. Will be defined only if untar effectively done. (Not defined if None)", None, - ) + ) self.monitor_defaults = CommentedOrderedDict() self.monitor_defaults["javascriptRRD_dir"] = ( os.path.join(self.src_dir, "../../externals/flot"), diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 7e821ced1c..49067e734c 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -902,7 +902,7 @@ def new_SubDicts(self, sub_name): # ############################################################ -#TODO(F): what to do,source, library +# TODO(F): what to do,source, library ############################################# # Add a user file residing in the stage area # file as described by Params.file_defaults diff --git a/creation/lib/cgWParams.py b/creation/lib/cgWParams.py index f956c2f011..3eb9742a17 100644 --- a/creation/lib/cgWParams.py +++ b/creation/lib/cgWParams.py @@ -474,7 +474,7 @@ def init_defaults(self): "Should this file be loaded after the entry ones?", None, ) - #TODO(F): what to do? + # TODO(F): what to do? self.defaults["entries"] = (OrderedDict(), "Dictionary of entries", "Each entry contains", self.entry_defaults) diff --git a/creation/lib/cvWConsts.py b/creation/lib/cvWConsts.py index 77a5c58c72..4cbc90c50c 100644 --- a/creation/lib/cvWConsts.py +++ b/creation/lib/cvWConsts.py @@ -20,7 +20,7 @@ AFTERGROUP_FILE_LISTFILE = "aftergroup_%s" % cWConsts.FILE_LISTFILE PREENTRY_FILE_LISTFILE = "preentry_%s" % cWConsts.FILE_LISTFILE AFTERGROUP_PREENTRY_FILE_LISTFILE = "aftergroup_%s" % PREENTRY_FILE_LISTFILE -#TODO(F): remain the same? +# TODO(F): remain the same? PARAMS_FILE = "params.cfg" diff --git a/creation/lib/cvWParams.py b/creation/lib/cvWParams.py index 930f6decac..61a7403818 100644 --- a/creation/lib/cvWParams.py +++ b/creation/lib/cvWParams.py @@ -54,7 +54,7 @@ def init_defaults(self): # VO scripts should start after the factory has been set completely # but there could be exceptions - #TODO(F): remove this one? + # TODO(F): remove this one? # Files/Validation/Custom scripts settings for frontend self.file_defaults["after_entry"] = ( "True", @@ -560,7 +560,7 @@ def init_defaults(self): "Should this file be loaded after the group ones?", None, ) - #TODO(F):what to do here? + # TODO(F):what to do here? global_config_defaults = cWParams.CommentedOrderedDict() global_config_defaults["ignore_down_entries"] = [ "False", diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index 3a9e4baebe..cb9b8b9a54 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -42,7 +42,7 @@ glidein_cleanup() { fi fi fi - + custom_scripts "cleanup" } @@ -105,10 +105,10 @@ glidein_exit() { local exit_code factory_collector dlf condor_vars_file main_work_dir exit_code=$1 # Removed lines about $lock_file (lock file for whole machine) not present elsewhere - + ############################## failure_scripts $exit_code - + gwms_process_scripts "$GWMS_DIR" cleanup "${glidein_config}" global_result="" if [ -f otr_outlist.list ]; then diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 3a159fa803..00fbed18e7 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -508,7 +508,7 @@ _main(){ # fetch list file fetch_file_regular "${gs_id}" "${gs_file_list}" - + # Fetch files contained in list # TODO: $file is actually a list, so it cannot be doublequoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here # shellcheck disable=2086 @@ -521,7 +521,7 @@ _main(){ fi fi done < "${gs_id_work_dir}/${gs_file_list}" - + # Files to go into the GWMS_PATH if [ "$gs_file_id" = "main at_file_list" ]; then @@ -589,7 +589,7 @@ _main(){ echo "No message left" 1>&2 fi echo 1>&2 - + ############################ custom_scripts "after_job" diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 81b55427db..a19f91f79e 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -242,11 +242,11 @@ fetch_file_try() { # if the variable fft_condition_attr is not defined or empty, do not download [[ -z "${fft_condition_attr_val}" ]] && return 0 fi - + if [[ "${fft_tar_source}" != NULL ]]; then return 0 fi - + fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" "${fft_time}" "${fft_coordination}" "${fft_tar_source}" return $? # returning the exit code of fetch_file_base @@ -400,7 +400,7 @@ fetch_file_base() { return 1 fi fi - + # if startup and executable, execute if [[ "${ffb_time}" = "startup" ]]; then if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then @@ -419,7 +419,7 @@ fetch_file_base() { echo "Executing (flags:${ffb_file_type#exec}) ${ffb_outname}" # have to do it here, as this will be run before any other script chmod u+rx "${main_dir}"/error_augment.sh - + # the XML file will be overwritten now, and hopefully not an error situation have_dummy_otrx=0 "${main_dir}"/error_augment.sh -init @@ -481,7 +481,7 @@ fetch_file_base() { fi fi fi - + if [ "${have_dummy_otrx}" -eq 1 ]; then # no one should really look at this file, but just to avoid confusion local date diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 4a9738812b..6851ecfdfe 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -30,7 +30,7 @@ add_entry(){ e_id="$7" e_tar_source="$8" descriptor_file="testfile" - + if [ ! -f "$descriptor_file" ]; then echo "# File: $descriptor_file" > "$descriptor_file" echo "#" >> "$descriptor_file" @@ -41,12 +41,12 @@ add_entry(){ IFS=', ' read -r -a array <<< $e_time for time_entry in "${array[@]}" do - if [[ $e_tar_source != NULL ]]; then + if [[ $e_tar_source != NULL ]]; then ffb_short_untar_dir="$(get_untar_subdir "${e_id}" "${e_tar_source}")" ffb_untar_dir="${ffb_work_dir}/${ffb_short_untar_dir}" e_complete_fname="${ffb_untar_dir}/$e_real_fname" - #TODO: check if filename is a path what happens - else + #TODO: check if filename is a path what happens + else e_complete_fname="$e_real_fname" fi echo "${time_entry} ${e_coordination}_${e_real_fname} ${e_complete_fname} ${e_type} ${e_period} ${e_prefix} ${e_id}" >> "$descriptor_file" @@ -107,7 +107,7 @@ custom_scripts(){ echo "Executing (flags:${ffb_file_type#exec}) ${ffb_target_fname}" # have to do it here, as this will be run before any other script chmod u+rx "${main_dir}"/error_augment.sh - + # the XML file will be overwritten now, and hopefully not an error situation have_dummy_otrx=0 "${main_dir}"/error_augment.sh -init @@ -133,7 +133,7 @@ custom_scripts(){ add_periodic_script "${main_dir}/script_wrapper.sh" "${ffb_period}" "${work_dir}" "${ffb_target_fname}" glidein_config "${ffb_id}" "${ffb_cc_prefix}" fi fi - fi + fi elif [[ "${ffb_file_type}" = "source" || "${ffb_file_type}" = "library:shell" ]]; then source "${ffb_target_fname}" # TODO: what about other library types? From 7c573d35833306d08e6a727a2e8d5455fa15a97b Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 11:43:25 -0500 Subject: [PATCH 065/196] Modified existing bats tests in order to support the modifications --- creation/web_base/glidein_cleanup.sh | 2 +- creation/web_base/utils_gs_http.sh | 2 +- test/bats/creation_web_base_glidein_cleanup.bats | 5 +++++ test/bats/creation_web_base_glidein_startup.bats | 7 ++++--- test/bats/creation_web_base_utils_gs_http.bats | 13 +++++-------- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/creation/web_base/glidein_cleanup.sh b/creation/web_base/glidein_cleanup.sh index cb9b8b9a54..059b138696 100644 --- a/creation/web_base/glidein_cleanup.sh +++ b/creation/web_base/glidein_cleanup.sh @@ -25,6 +25,7 @@ glide_local_tmp_dir_created=0 # glide_local_tmp_dir_created # glide_local_tmp_dir glidein_cleanup() { + custom_scripts "cleanup" if ! cd "${start_dir}"; then log_warn "Cannot find ${start_dir} anymore, exiting but without cleanup" else @@ -43,7 +44,6 @@ glidein_cleanup() { fi fi - custom_scripts "cleanup" } ################################ diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index a19f91f79e..c9018e681b 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -106,7 +106,7 @@ EOF # 2: target fname # 3: real fname fetch_file_regular() { - fetch_file "$1" "$2" "$2" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" "no_time" "no_code" + fetch_file "$1" "$2" "$3" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" "no_time" "no_code" "NULL" } ############################################ diff --git a/test/bats/creation_web_base_glidein_cleanup.bats b/test/bats/creation_web_base_glidein_cleanup.bats index 9f1706d584..ba4de0ea3c 100755 --- a/test/bats/creation_web_base_glidein_cleanup.bats +++ b/test/bats/creation_web_base_glidein_cleanup.bats @@ -17,6 +17,11 @@ setup () { source "$GWMS_SOURCEDIR"/creation/web_base/get_id_selectors.source } +#mockup +custom_scripts(){ + return 0 +} + @test "glidien_cleanup" { echo "Checking the case of not existing start_dir..." >& 3 start_dir="random/stuff/" diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index ef6ff135fe..4c47f8ca34 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -115,19 +115,20 @@ early_glidein_failure() { @test "create_glidein_config" { glidein_config="glidein_config" + touch ${glidein_config} # if not a root user if [ "$EUID" -ne 0 ]; then echo "Testing the glidein_config file with no write permissions..." >& 3 - touch ${glidein_config} chmod 000 ${glidein_config} run create_glidein_config [ "$status" -eq 1 ] + assert_output --partial "${PWD}/${glidein_config}: Permission denied" + assert_output --partial "Could not create '${PWD}/${glidein_config}'" fi echo "Testing the glidein_config file with write permissions..." >& 3 - assert_output --partial "${PWD}/${glidein_config}: Permission denied" - assert_output --partial "Could not create '${PWD}/${glidein_config}'" chmod 777 ${glidein_config} run create_glidein_config + echo "$output" >& 3 [ "$status" -eq 0 ] grep -Fxq "# --- glidein_startup vals ---" ${glidein_config} grep -Fxq "# --- User Parameters ---" ${glidein_config} diff --git a/test/bats/creation_web_base_utils_gs_http.bats b/test/bats/creation_web_base_utils_gs_http.bats index f6a8343614..5f0a288e2b 100755 --- a/test/bats/creation_web_base_utils_gs_http.bats +++ b/test/bats/creation_web_base_utils_gs_http.bats @@ -86,18 +86,15 @@ glidein_exit(){ @test "fetch_file" { echo "Testing different possible numbers of arguments..." >& 3 - run fetch_file 1 2 3 4 5 6 7 8 9 - assert_output --partial "More then 8 arguments, considering the first 8" + run fetch_file 1 2 3 4 5 6 7 8 9 10 11 12 + assert_output --partial "More then 11 arguments, considering the first 11" run fetch_file 1 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" + assert_output --partial "Not enough arguments in fetch_file, 11 expected" run fetch_file 1 2 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" + assert_output --partial "Not enough arguments in fetch_file, 11 expected" run fetch_file 1 2 3 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" + assert_output --partial "Not enough arguments in fetch_file, 11 expected" run fetch_file 1 2 3 4 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" - run fetch_file 1 2 3 4 5 - assert_output --partial "Not enough arguments in fetch_file, 8 expected" } #TODO: Test fetch_file_try, fetch_file_base, perform_wget, perform_curl From 69bb8dc9b53002558535dcd9e9ff678dad060226 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 11:44:06 -0500 Subject: [PATCH 066/196] Corrected create_glidein_config bats test --- test/bats/creation_web_base_glidein_startup.bats | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/bats/creation_web_base_glidein_startup.bats b/test/bats/creation_web_base_glidein_startup.bats index ef6ff135fe..4c47f8ca34 100755 --- a/test/bats/creation_web_base_glidein_startup.bats +++ b/test/bats/creation_web_base_glidein_startup.bats @@ -115,19 +115,20 @@ early_glidein_failure() { @test "create_glidein_config" { glidein_config="glidein_config" + touch ${glidein_config} # if not a root user if [ "$EUID" -ne 0 ]; then echo "Testing the glidein_config file with no write permissions..." >& 3 - touch ${glidein_config} chmod 000 ${glidein_config} run create_glidein_config [ "$status" -eq 1 ] + assert_output --partial "${PWD}/${glidein_config}: Permission denied" + assert_output --partial "Could not create '${PWD}/${glidein_config}'" fi echo "Testing the glidein_config file with write permissions..." >& 3 - assert_output --partial "${PWD}/${glidein_config}: Permission denied" - assert_output --partial "Could not create '${PWD}/${glidein_config}'" chmod 777 ${glidein_config} run create_glidein_config + echo "$output" >& 3 [ "$status" -eq 0 ] grep -Fxq "# --- glidein_startup vals ---" ${glidein_config} grep -Fxq "# --- User Parameters ---" ${glidein_config} From 3b9e3952dc3ba47b732403864b3dea4597a42fdb Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 11:58:15 -0500 Subject: [PATCH 067/196] Added a subset of bats tests regarding the utils_scripts bash script --- creation/web_base/utils_scripts.sh | 10 ++++-- .../bats/creation_web_base_utils_scripts.bats | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100755 test/bats/creation_web_base_utils_scripts.bats diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 6851ecfdfe..2761742a1a 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -62,7 +62,6 @@ add_entry(){ # list extract_entry_files(){ local target_time descriptor_file - list="" target_time="$1" descriptor_file="testfile" grep ^$target_time $descriptor_file | sort > ${target_time}_descriptor_file @@ -152,7 +151,7 @@ milestone_call(){ local code code=$1 # no spaces are allowed in the code - if [[ "$code" == *" "* ]]; then + if [[ "$code" == *"\t"* ]]; then return 1 fi custom_scripts "milestone:"$code @@ -166,5 +165,12 @@ milestone_call(){ failure_call(){ local exit_code exit_code=$1 + # no spaces are allowed in the code + if [[ "$exit_code" == *"\t"* ]]; then + return 1 + fi custom_scripts "failure:"$exit_code + return 0 } + +#TODO: Bats tests for custom_scripts, extract_entry_files and add_entry not defined since they work on the descriptor_file statically defined diff --git a/test/bats/creation_web_base_utils_scripts.bats b/test/bats/creation_web_base_utils_scripts.bats new file mode 100755 index 0000000000..d667968416 --- /dev/null +++ b/test/bats/creation_web_base_utils_scripts.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 +load 'lib/bats-support/load' +load 'lib/bats-assert/load' + +[[ -z "$GWMS_SOURCEDIR" ]] && GWMS_SOURCEDIR="../../creation/web_base" + +setup () { + source "$GWMS_SOURCEDIR"/utils_scripts.sh +} + +@test "milestone_call" { + #mock + custom_scripts(){ + echo "$1" + } + run milestone_call "code\tcode" + [ "$status" -eq 1 ] + run milestone_call "code" + [ "$output" == "milestone:code" ] + [ "$status" -eq 0 ] +} + +@test "failure_call" { + #mock + custom_scripts(){ + echo "$1" + } + run failure_call "code\tcode" + [ "$status" -eq 1 ] + run failure_call "code" + [ "$output" == "failure:code" ] + [ "$status" -eq 0 ] +} From 7483af52986074a5a74b5e928816715cbd564852 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 14:09:53 -0500 Subject: [PATCH 068/196] Applied modifications to cWDictFile --- creation/lib/cWDictFile.py | 16 ++++++++++------ creation/web_base/utils_scripts.sh | 2 -- test/bats/creation_web_base_utils_scripts.bats | 2 ++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index ff8628326b..2797e84267 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -993,8 +993,8 @@ class FileDictFile(SimpleFileDictFile): It is using a dictionary (key, value) from DictFile, serialized to file. The key is the file ID - The value (line) on file has DATA_LENGTH (7) components: the key and the first DATA_LENGTH-1 attributes below. - The value in memory has DATA_LENGTH components (real_fname,cache/exec,period,prefix,cond_download,config_out, data), + The value (line) on file has DATA_LENGTH (11) components: the key and the first DATA_LENGTH-1 attributes below. + The value in memory has DATA_LENGTH components (real_fname, cache/exec, prefix, time, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr, data), the key is used as key for the dictionary and the data (file content) is added reading the file. Here the attributes stored as tuple in the dictionary value: 1. real_fname, i.e file name @@ -1053,14 +1053,13 @@ class FileDictFile(SimpleFileDictFile): "", "", "", - "", 0, - "NULL", - "", "", + "NULL", "", "", "", + "" ) # The tuple should be DATA_LENGTH long and have the correct values def add_placeholder(self, key, allow_overwrite=True): @@ -1240,7 +1239,7 @@ def file_header(self, want_comments): DictFile.file_header(self, want_comments) + "\n" + ( - "# %s \t%s \t%s \t%s \t%s \t%s \t%s\n" + "# %s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \n" % ( "Outfile", "InFile ", @@ -1284,6 +1283,11 @@ def parse_val(self, line): return self.add( arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9]] ) + if len(arr) == self.DATA_LENGTH - 2: + # (no prefix and tar_source): key, fname, type, time, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr + return self.add( + arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], "NULL", arr[7], arr[8]] + ) raise RuntimeError( "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) ) diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 2761742a1a..660adc77a7 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -172,5 +172,3 @@ failure_call(){ custom_scripts "failure:"$exit_code return 0 } - -#TODO: Bats tests for custom_scripts, extract_entry_files and add_entry not defined since they work on the descriptor_file statically defined diff --git a/test/bats/creation_web_base_utils_scripts.bats b/test/bats/creation_web_base_utils_scripts.bats index d667968416..dbe23d7d79 100755 --- a/test/bats/creation_web_base_utils_scripts.bats +++ b/test/bats/creation_web_base_utils_scripts.bats @@ -33,3 +33,5 @@ setup () { [ "$output" == "failure:code" ] [ "$status" -eq 0 ] } + +#TODO: Bats tests for custom_scripts, extract_entry_files and add_entry not defined since they work on the descriptor_file statically defined From 9ebbf93e7ea1336d3aeb7cb38599c972f6f88a0b Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 14:27:26 -0500 Subject: [PATCH 069/196] Modified some comments to test the pyunittest --- creation/lib/cgWParams.py | 1 - creation/lib/cvWParams.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/creation/lib/cgWParams.py b/creation/lib/cgWParams.py index 3eb9742a17..fdc47d8d38 100644 --- a/creation/lib/cgWParams.py +++ b/creation/lib/cgWParams.py @@ -474,7 +474,6 @@ def init_defaults(self): "Should this file be loaded after the entry ones?", None, ) - # TODO(F): what to do? self.defaults["entries"] = (OrderedDict(), "Dictionary of entries", "Each entry contains", self.entry_defaults) diff --git a/creation/lib/cvWParams.py b/creation/lib/cvWParams.py index 61a7403818..8ca36804bc 100644 --- a/creation/lib/cvWParams.py +++ b/creation/lib/cvWParams.py @@ -54,7 +54,6 @@ def init_defaults(self): # VO scripts should start after the factory has been set completely # but there could be exceptions - # TODO(F): remove this one? # Files/Validation/Custom scripts settings for frontend self.file_defaults["after_entry"] = ( "True", @@ -560,7 +559,6 @@ def init_defaults(self): "Should this file be loaded after the group ones?", None, ) - # TODO(F):what to do here? global_config_defaults = cWParams.CommentedOrderedDict() global_config_defaults["ignore_down_entries"] = [ "False", From 6fd9ad0c4d977cad86b9525c636fc0fe10e92cfe Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 2 Sep 2022 16:51:13 -0500 Subject: [PATCH 070/196] Applied modifications to the python scripts handling the xml files content --- creation/lib/cWDictFile.py | 53 ++++++++----- creation/lib/cWParamDict.py | 123 ++++++++++++++++++++--------- creation/lib/cgWParamDict.py | 3 +- creation/lib/cgWParams.py | 1 + creation/lib/cvWParamDict.py | 2 + creation/lib/cvWParams.py | 1 + creation/lib/xmlConfig.py | 31 +++++++- creation/web_base/utils_gs_http.sh | 80 ++++++++++--------- creation/web_base/utils_scripts.sh | 1 + 9 files changed, 194 insertions(+), 101 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index 2797e84267..5af68cf213 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -1017,7 +1017,8 @@ class FileDictFile(SimpleFileDictFile): - failure (:exit_code will specify the failure code) Time will be ignored for files when type is different than exec or source. A combination of time phases can be used. - 5. priority - priority in the order of execution inside a time phase [0-99]. + 5. period, 0 if not a periodic script + 6. priority - priority in the order of execution inside a time phase [0-99]. Relevant only for executable, source and library files. Order String code representation Associated integer code (internal representation) Factory pre_entry e-[g-] 10 @@ -1030,35 +1031,35 @@ class FileDictFile(SimpleFileDictFile): Frontend after_entry after_group e+g+ 80 Factory after_entry e+[g+] 90 A custom value inside the range [0-99] can be specified in order to alter the order of execution. - 6. cond_download has a special value of TRUE - 7. tar_source - Name of the tarball containing the file, if the file is contained in a tarball - 8. config_out - has a special value of FALSE - 9. cond_attr - Name of a configuration switch. (like "ENABLE_KRB5") + 7. cond_download has a special value of TRUE + 8. tar_source - Name of the tarball containing the file, if the file is contained in a tarball + 9. config_out - has a special value of FALSE + 10. cond_attr - Name of a configuration switch. (like "ENABLE_KRB5") The operation requested on the file will be performed only if that parameter will be set to 1. - This parameter is going to influence the config_out element. - 10. absdir_outattr - Name of a variable name. (like "KRB5_SUBSYS_DIR") + 11. absdir_outattr - Name of a variable name. (like "KRB5_SUBSYS_DIR") The variable will be set to the absolute path of the file if not a tarball or the directory where the tarball was unpacked in case it is a tarball, this last thing if and only if the unpacking actually happened (else it will not be defined.) ENTRY_ will be prepended if the directive occurs in an entry. - 11. data - String containing the data extracted from the file (real_fname) (not in the serialized dictionary) + 12. data - String containing the data extracted from the file (real_fname) (not in the serialized dictionary) For placeholders, the real_name is empty (and the tuple starts w/ an empty string). Placeholders cannot be serialized (saved into file). Empty strings would cause error when parsed back. """ - DATA_LENGTH = 11 # Length of value (attributes + data) + DATA_LENGTH = 12 # Length of value (attributes + data) PLACEHOLDER_VALUE = ( "", "", "", - "", + "no_time", 0, - "", + 0, + "TRUE", + "NULL", + "FALSE", + "NULL", "NULL", - "", - "", - "", "" ) # The tuple should be DATA_LENGTH long and have the correct values @@ -1074,7 +1075,8 @@ def make_val_tuple( file_name, file_type, prefix="GLIDEIN_PS_", - time="startup", + time="no_time", + period=0, priority=0, cond_download="TRUE", tar_source="NULL", @@ -1088,6 +1090,7 @@ def make_val_tuple( :param file_type: type of the file (regular, nocache, exec, untar, wrapper, source, library). 'exec allows modifiers like ':s' :param prefix: prefix for periodic executables (ignored otherwise, default: GLIDEIN_PS_) :param time: specific phase of the lifecycle where to execute/source the script (startup, pre_job, after_job, cleanup, periodic, milestone, failure) + :param period: period of the periodic execution, 0 if the executable is not periodic :param priority: priority of execution inside the phase [0-99] :param cond_download: conditional download (default: 'TRUE') :param tar_source: name of the tarball containing the file @@ -1103,6 +1106,7 @@ def make_val_tuple( file_type, prefix, time, + period, priority, cond_download, tar_source, @@ -1219,7 +1223,7 @@ def add( ) def format_val(self, key, want_comments): - return "{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{}".format( + return "{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{} \t{}".format( key, self.vals[key][0], self.vals[key][1], @@ -1231,6 +1235,7 @@ def format_val(self, key, want_comments): self.vals[key][7], self.vals[key][8], self.vals[key][9], + self.vals[key][10] ) def file_header(self, want_comments): @@ -1246,6 +1251,7 @@ def file_header(self, want_comments): "Cache/exec", "Prefix", "Time", + "Period" "Priority", "Condition", "TarSource", @@ -1276,17 +1282,22 @@ def parse_val(self, line): if len(arr) != self.DATA_LENGTH: # compatibility w/ old formats - # 3.2.13 (no prefix): key, fname, type, time, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr + # 3.2.13 (no prefix): key, fname, type, time, period, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr # TODO: remove in 3.3 or after a few version (will break upgrade) if len(arr) == self.DATA_LENGTH - 1: # For upgrade from 3.2.13 to 3.2.11 return self.add( - arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9]] + arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9], arr[10], arr[11]] ) - if len(arr) == self.DATA_LENGTH - 2: - # (no prefix and tar_source): key, fname, type, time, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr + elif len(arr) == self.DATA_LENGTH - 2: + # (no prefix, and period ): key, fname, type, time, priority, cond_download, config_out, cond_attr, absdir_outattr + return self.add( + arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], arr[7], arr[8], arr[9], arr[10]] + ) + elif len(arr) == self.DATA_LENGTH - 3: + # (no prefix, period and tar_source): key, fname, type, time, priority, cond_download, config_out, cond_attr, absdir_outattr return self.add( - arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], "NULL", arr[7], arr[8]] + arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], "NULL", arr[7], arr[8], arr[9]] ) raise RuntimeError( "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index 336c89164c..60169601b0 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -53,41 +53,70 @@ def add_file_unparsed(user_file, dicts, is_factory): :param is_factory: True if invoked for the factory (cgWParamDict.py), false for the frontend (cvWParamDict.py) :return: None (dictionaries are modified) """ - absfname = user_file.absfname if absfname is None: raise RuntimeError("Found a file element without an absname: %s" % user_file) - relfname = user_file.relfname if relfname is None: relfname = os.path.basename(absfname) # default is the final part of absfname if len(relfname) < 1: raise RuntimeError("Found a file element with an empty relfname: %s" % user_file) - time = user_file.time + is_const = is_true(user_file.const) is_executable = is_true(user_file.executable) or (user_file.type.startswith("exec")) is_wrapper = is_true(user_file.wrapper) or (user_file.type == "wrapper") - is_source = user_file.type == "source" - is_library = user_file.type.startswith("library") + is_source = is_true(user_file.source) or user_file.type.startswith("source") + is_library = is_true(user_file.library) or user_file.type.startswith("library") is_periodic = user_file.type.startswith("periodic") do_untar = is_true(user_file.untar) or (user_file.type.startswith("untar")) + + time = user_file.time + if is_executable or is_source or is_library: + if ( + not time.startswith("startup") + and not time.startswith("cleanup") + and not time.startswith("after_job") + and not time.startswith("before_job") + and not time.startswith("periodic:") + and not time.startswith("milestone:") + and not time.startswith("failure:") + and not time.startswith("no_time:") + ): + # we use startswith since we may have combination of time phases (e.g. startup, cleanup) + raise RuntimeError("The file does not have a valid time phase value: %s" % user_file) + try: if user_file.is_periodic: period_value = int(user_file.type.split(":")[1]) + else: + period_value = 0 except (AttributeError, KeyError, ValueError, TypeError): - period_value = 0 - # TODO(F): what to do with period and after list - + period_value = 1000 # default 1000ms + + if period_value > 0: # period has 0 as default (in dictionary definition). Should I still protect against it not being defined? + if not is_executable: + raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) + priority = user_file.priority if priority < 0 or priority > 99: - raise RuntimeError("Priority value out of the range [0,99]: %s" % user_file) - - # Extended for all categories - config_out = user_file.absdir_outattr + raise RuntimeError("Priority value out of the range [0,99]: %s" % user_file) + + prefix = user_file.prefix + + cond_download = user_file.cond_download + + tar_source = user_file.tar_source + if tar_source is None: + tar_source = "NULL" + + config_out = user_file.config_out if config_out is None: config_out = "FALSE" cond_attr = user_file.cond_attr - + + absdir_outattr = user_file.absdir_outattr + + # TODO(F): what to do with period and after list if is_factory: # Factory (file_list, after_file_list) file_list_idx = "file_list" @@ -104,24 +133,7 @@ def add_file_unparsed(user_file, dicts, is_factory): if "after_group" in user_file: if is_true(user_file.after_group): file_list_idx = "aftergroup_%s" % file_list_idx - - # period has 0 as default (in dictionary definition). Should I still protect against it not being defined? - if period_value > 0: - if not is_executable: - raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) - - if is_executable or is_source or is_library: - if ( - not time.startswith("startup") - and not time.startswith("cleanup") - and not time.startswith("after_job") - and not time.startswith("before_job") - and not time.startswith("periodic:") - and not time.startswith("milestone:") - and not time.startswith("failure:") - ): - # we use startswith since we may have combination of time phases (e.g. startup, cleanup) - raise RuntimeError("The file does not have a valid time phase value: %s" % user_file) + if is_executable: # a script if not is_const: raise RuntimeError("A file cannot be executable if it is not constant: %s" % user_file) @@ -129,6 +141,7 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("A tar file cannot be executable: %s" % user_file) if is_wrapper: raise RuntimeError("A wrapper file cannot be an executable: %s" % user_file) + file_type = "exec" if user_file.type: if user_file.type == "run:s" or user_file.type == "run:singularity": @@ -141,35 +154,50 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), file_type, user_file.period, user_file.prefix + cWConsts.insert_timestr(relfname), file_type, prefix, time, period, priority, cond_download, tar_source, config_out, cond_attr ), absfname, ) - elif is_wrapper: # a source-able script for the wrapper if not is_const: raise RuntimeError("A file cannot be a wrapper if it is not constant: %s" % user_file) if do_untar: raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file) + if is_source: + raise RuntimeError("A source file cannot be a wrapper: %s" % user_file) + if is_library: + raise RuntimeError("A library file cannot be a wrapper: %s" % user_file) + if is_periodic: + raise RuntimeError("A wrapper file cannot be periodic: %s" % user_file) + dicts[file_list_idx].add_from_file( - relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "wrapper"), absfname + relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname),"wrapper", tar_source=user_file.tar_source), absfname ) + elif do_untar: # a tarball if not is_const: raise RuntimeError("A file cannot be untarred if it is not constant: %s" % user_file) - + if is_periodic: + raise RuntimeError("A tar file cannot be periodic: %s" % user_file) + if is_library: + raise RuntimeError("A tar file cannot be a library: %s" % user_file) + if is_executable: + raise RuntimeError("A tar file cannot be executable: %s" % user_file) + if is_wrapper: + raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file) + wnsubdir = user_file.type.split(":")[1] if wnsubdir is None: wnsubdir = absfname # default is relfname up to the first - + dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), "untar", cond_download=cond_attr, config_out=config_out + cWConsts.insert_timestr(relfname), "untar", cond_download=cond_download, config_out=config_out, cond_attr=cond_attr, absdir_outattr=wnsubdir ), absfname, ) - dicts["untar_cfg"].add(relfname, wnsubdir) + #dicts["untar_cfg"].add(relfname, wnsubdir) elif is_source: if not is_const: raise RuntimeError("A file cannot be sourced if it is not constant: %s" % user_file) @@ -177,6 +205,15 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("A tar file cannot be sourced: %s" % user_file) if is_wrapper: raise RuntimeError("A wrapper file cannot be an sourced: %s" % user_file) + if is_periodic: + raise RuntimeError("A source file cannot be periodic: %s" % user_file) + if is_library: + raise RuntimeError("A source file cannot be a library: %s" % user_file) + + dicts[file_list_idx].add_from_file( + relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "source", tar_source=tar_source, time=time, priority=priority), absfname + ) + elif is_library: if not is_const: raise RuntimeError("A file cannot be a library if it is not constant: %s" % user_file) @@ -184,6 +221,14 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("A tar file cannot be a library: %s" % user_file) if is_wrapper: raise RuntimeError("A wrapper file cannot be an a library: %s" % user_file) + if is_periodic: + raise RuntimeError("A library file cannot be periodic: %s" % user_file) + + dicts[file_list_idx].add_from_file( + relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), user_file.type, tar_source=tar_source, time=time, priority=priority), absfname + ) + # user_file.type can be library:x + else: # not executable nor tarball => simple file if is_const: @@ -196,3 +241,5 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple(relfname, val), absfname ) # no timestamp in the name if it can be modified + + #TODO(F): config_out? diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 49067e734c..827ad1f5a2 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -902,7 +902,8 @@ def new_SubDicts(self, sub_name): # ############################################################ -# TODO(F): what to do,source, library +# TODO(F): what to do differently from the general one? + ############################################# # Add a user file residing in the stage area # file as described by Params.file_defaults diff --git a/creation/lib/cgWParams.py b/creation/lib/cgWParams.py index fdc47d8d38..3eb9742a17 100644 --- a/creation/lib/cgWParams.py +++ b/creation/lib/cgWParams.py @@ -474,6 +474,7 @@ def init_defaults(self): "Should this file be loaded after the entry ones?", None, ) + # TODO(F): what to do? self.defaults["entries"] = (OrderedDict(), "Dictionary of entries", "Each entry contains", self.entry_defaults) diff --git a/creation/lib/cvWParamDict.py b/creation/lib/cvWParamDict.py index 4474b6da2b..fa9dafbb97 100644 --- a/creation/lib/cvWParamDict.py +++ b/creation/lib/cvWParamDict.py @@ -283,6 +283,8 @@ def populate(self, params=None): os.path.join(params.src_dir, file_name), ) + #TODO(F): what to do here? + # Load initial system scripts # These should be executed before the other scripts for script_name in ("cat_consts.sh", "check_blacklist.sh"): diff --git a/creation/lib/cvWParams.py b/creation/lib/cvWParams.py index 8ca36804bc..a77befd4e8 100644 --- a/creation/lib/cvWParams.py +++ b/creation/lib/cvWParams.py @@ -61,6 +61,7 @@ def init_defaults(self): "Should this file be loaded after the factory entry ones?", None, ) + # TODO(F): remove this one? # Publishing attr specific to frontend self.attr_defaults["type"] = [ diff --git a/creation/lib/xmlConfig.py b/creation/lib/xmlConfig.py index 95d4641ba8..acfa12107a 100644 --- a/creation/lib/xmlConfig.py +++ b/creation/lib/xmlConfig.py @@ -285,24 +285,47 @@ def validate(self): self.check_boolean("executable") self.check_boolean("wrapper") self.check_boolean("untar") + self.check_boolean("source") + self.check_boolean("library") is_exec = eval(self["executable"]) is_wrapper = eval(self["wrapper"]) is_tar = eval(self["untar"]) - + is_source = eval(self["source"]) + is_library = eval(self["library"]) + try: period = int(self["period"]) except ValueError: raise RuntimeError(self.err_str("period must be an int")) - if is_exec + is_wrapper + is_tar > 1: - raise RuntimeError(self.err_str('must be exactly one of type "executable", "wrapper", or "untar"')) + time = self["time"] + if is_executable or is_source or is_library: + if ( + not time.startswith("startup") + and not time.startswith("cleanup") + and not time.startswith("after_job") + and not time.startswith("before_job") + and not time.startswith("periodic:") + and not time.startswith("milestone:") + and not time.startswith("failure:") + and not time.startswith("no_time:") + ): + # we use startswith since we may have combination of time phases (e.g. startup, cleanup) + raise RuntimeError(self.err_str('The file does not have a valid time phase value: %s'% user_file)) + + self.check_boolean("cond_download") + self.check_boolean("config_out") + + if is_exec + is_wrapper + is_tar + is_source + is_library> 1: + raise RuntimeError(self.err_str('must be exactly one of type "executable", "wrapper", "untar", "library" or "source"')) if (is_exec or is_wrapper or is_tar) and not eval(self["const"]): raise RuntimeError(self.err_str('type "executable", "wrapper", or "untar" requires const="True"')) if not is_exec and period > 0: raise RuntimeError(self.err_str('cannot have execution period if type is not "executable"')) - + + TAG_CLASS_MAPPING.update({"file": FileElement}) diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index c9018e681b..0f46c6edea 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -106,7 +106,7 @@ EOF # 2: target fname # 3: real fname fetch_file_regular() { - fetch_file "$1" "$2" "$3" "regular" 0 "GLIDEIN_PS_" "TRUE" "FALSE" "no_time" "no_code" "NULL" + fetch_file "$1" "$2" "$3" "regular" "GLIDEIN_PS_" "no_time" 0 0 "TRUE" "NULL" "FALSE" "NULL" "NULL" } ############################################ @@ -118,13 +118,15 @@ fetch_file_regular() { # 2: target fname # 3: real fname # 4: file type (regular, exec, exec:s, untar, nocache) -# 5: period (0 if not a periodic file) -# 6: periodic scripts prefix -# 7: config check TRUE,FALSE -# 8: config out TRUE,FALSE -# 9: time (startup, cleanup, before_job, after_job, periodic, specific_code, failure:exit_code, or a comma-separated combination of them) -# 10: integer_code (internal representation of the coordination number of the scripts) -# 11: tar_source +# 5: periodic scripts prefix +# 6: time (startup, cleanup, before_job, after_job, periodic, specific_code, failure:exit_code, or a comma-separated combination of them) +# 7: period (0 if not a periodic file) +# 8: priority/integer_code [0,99] (internal representation of the coordination number of the scripts) +# 9: config check TRUE,FALSE +# 10: tar_source +# 11: config out TRUE,FALSE +# 12: cond_attr +# 13: absdir_outattr # The above is the most recent list, below some adaptations for different versions # Used: # ifs_str @@ -133,32 +135,32 @@ fetch_file_regular() { # 0 in case of success # otherwise glidein_exit with 1 fetch_file() { - if [ $# -gt 11 ]; then + if [ $# -gt 13 ]; then # For compatibility w/ future versions (add new parameters at the end) echo "More then 11 arguments, considering the first 11 ($#/${ifs_str}): $*" 1>&2 - elif [ $# -ne 11 ]; then - if [ $# -eq 10 ]; then + elif [ $# -ne 13 ]; then + if [ $# -eq 12 ]; then #TODO: remove in version 3.3 # For compatibility with past versions (old file list formats) # 3.2.13 and older: prefix (par 6) added in #12705, 3.2.14? - # 3.2.10 and older: period (par 5) added: fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "GLIDEIN_PS_" "$6" "$7" "$8" "$9" "${10}"; then + # 3.2.10 and older: period (par 5) added + if ! fetch_file_try "$1" "$2" "$3" "$4" "GLIDEIN_PS_" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" "${12}"; then glidein_exit 1 fi return 0 fi - if [ $# -eq 9 ]; then + if [ $# -eq 11 ]; then # added to maintain compatibility with older (3.2.10) file list format #TODO: remove in version 3.3 - if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" "$7" "$8" "$9"; then + if ! fetch_file_try "$1" "$2" "$3" "$4" "GLIDEIN_PS_" "$5" 0 "$6" "$7" "$8" "$9" "${10}" "${11}"; then glidein_exit 1 fi return 0 fi - if [ $# -eq 8 ]; then + if [ $# -eq 10 ]; then # added to maintain compatibility with older (3.2.10) file list format #TODO: remove in version 3.3 - if ! fetch_file_try "$1" "$2" "$3" "$4" 0 "GLIDEIN_PS_" "$5" "$6" "$7" "$8" "NULL"; then + if ! fetch_file_try "$1" "$2" "$3" "$4" "GLIDEIN_PS_" "$5" 0 "$6" "$7" "NULL" "$8" "$9" "${10}"; then glidein_exit 1 fi return 0 @@ -168,7 +170,7 @@ fetch_file() { log_warn "Not enough arguments in fetch_file, 11 expected ($#/${ifs_str}): $*" glidein_exit 1 fi - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}"; then + if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" "${12}" "${13}"; then glidein_exit 1 fi return 0 @@ -188,12 +190,15 @@ fetch_file() { # 2: target fname # 3: real fname # 4: file type (regular, exec, exec:s, untar, nocache) -# 5: period (0 if not a periodic file) -# 6: periodic scripts prefix -# 7: config check TRUE,FALSE -# 8: config out TRUE,FALSE -# 9: time (startup, cleanup, before_job, after_job, periodic, specific_code, failure:exit_code, or a comma-separated combination of them) -# 10: integer_code (internal representation of the coordination number of the scripts) +# 5: periodic scripts prefix +# 6: time (startup, cleanup, before_job, after_job, periodic, specific_code, failure:exit_code, or a comma-separated combination of them) +# 7: period (0 if not a periodic file) +# 8: priority/integer_code [0,99] (internal representation of the coordination number of the scripts) +# 9: config check TRUE,FALSE +# 10: tar_source +# 11: config out TRUE,FALSE +# 12: cond_attr +# 13: absdir_outattr # Globals (r/w): # fft_id # fft_target_fname @@ -218,14 +223,20 @@ fetch_file_try() { fft_target_fname="$2" fft_real_fname="$3" fft_file_type="$4" - fft_period="$5" - fft_cc_prefix="$6" - fft_config_check="$7" - fft_config_out="$8" - fft_time="$9" - fft_coordination="${10}" - fft_tar_source="${11}" - + fft_cc_prefix="$5" + fft_time="$6" + fft_period="$7" + fft_coordination="$8" + fft_config_check="$9" + fft_tar_source="${10}" + fft_config_out="${11}" + fft_cond_attr="${12}" + fft_cond_attr="${13}" + + if [[ "${fft_tar_source}" != "NULL" ]]; then + return 0 + fi + if [[ "${fft_config_check}" != "TRUE" ]]; then # TRUE is a special case, always be downloaded and processed fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) @@ -243,10 +254,6 @@ fetch_file_try() { [[ -z "${fft_condition_attr_val}" ]] && return 0 fi - if [[ "${fft_tar_source}" != NULL ]]; then - return 0 - fi - fetch_file_base "${fft_id}" "${fft_target_fname}" "${fft_real_fname}" "${fft_file_type}" "${fft_config_out}" "${fft_period}" "${fft_cc_prefix}" "${fft_time}" "${fft_coordination}" "${fft_tar_source}" return $? # returning the exit code of fetch_file_base @@ -466,7 +473,6 @@ fetch_file_base() { return 1 fi fi - #TODO: modify untar if [ "${ffb_config_out}" != "FALSE" ]; then ffb_prefix="$(get_prefix "${ffb_id}")" diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 660adc77a7..74850e8577 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -172,3 +172,4 @@ failure_call(){ custom_scripts "failure:"$exit_code return 0 } + From 608ff2975b2a254ae7fc931f70b9b61836a6a3f7 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Tue, 6 Sep 2022 13:46:38 -0500 Subject: [PATCH 071/196] Applied modifications to the python scripts handling the xml files content --- creation/lib/cWConsts.py | 6 ++ creation/lib/cWDictFile.py | 36 ++++++--- creation/lib/cWParamDict.py | 117 ++++++++++++++++++----------- creation/lib/cWParams.py | 2 +- creation/lib/cvWParamDict.py | 4 +- creation/lib/cvWParams.py | 1 + creation/lib/xmlConfig.py | 39 +++++----- creation/web_base/utils_gs_http.sh | 6 +- creation/web_base/utils_scripts.sh | 1 - 9 files changed, 133 insertions(+), 79 deletions(-) diff --git a/creation/lib/cWConsts.py b/creation/lib/cWConsts.py index 076c684e31..861bc9ea65 100644 --- a/creation/lib/cWConsts.py +++ b/creation/lib/cWConsts.py @@ -77,3 +77,9 @@ def insert_timestr(instr): BLACKLIST_FILE = "nodes.blacklist" GRIDMAP_FILE = "grid-mapfile" + +# TODO(F) +# Using two digits for major/minor/patch version +# This is the first version (3.9.11) with this new format. +# Change this number only if the DictFIle format changes +DICT_FILE_VERSION = "031100" diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index 5af68cf213..0fcf97ea9e 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -438,6 +438,7 @@ def file_header(self, want_comments): """ if want_comments: return "# File: %s\n#" % self.fname + # TODO(F) versione else: return None @@ -1060,7 +1061,7 @@ class FileDictFile(SimpleFileDictFile): "FALSE", "NULL", "NULL", - "" + "", ) # The tuple should be DATA_LENGTH long and have the correct values def add_placeholder(self, key, allow_overwrite=True): @@ -1235,7 +1236,7 @@ def format_val(self, key, want_comments): self.vals[key][7], self.vals[key][8], self.vals[key][9], - self.vals[key][10] + self.vals[key][10], ) def file_header(self, want_comments): @@ -1251,8 +1252,7 @@ def file_header(self, want_comments): "Cache/exec", "Prefix", "Time", - "Period" - "Priority", + "Period" "Priority", "Condition", "TarSource", "ConfigOut", @@ -1284,21 +1284,39 @@ def parse_val(self, line): # compatibility w/ old formats # 3.2.13 (no prefix): key, fname, type, time, period, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr # TODO: remove in 3.3 or after a few version (will break upgrade) + + # TODO(F): parse versione if len(arr) == self.DATA_LENGTH - 1: # For upgrade from 3.2.13 to 3.2.11 return self.add( - arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9], arr[10], arr[11]] + arr[0], + [ + arr[1], + arr[2], + "GLIDEIN_PS_", + arr[3], + arr[4], + arr[5], + arr[6], + arr[7], + arr[8], + arr[9], + arr[10], + arr[11], + ], ) elif len(arr) == self.DATA_LENGTH - 2: # (no prefix, and period ): key, fname, type, time, priority, cond_download, config_out, cond_attr, absdir_outattr return self.add( - arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], arr[7], arr[8], arr[9], arr[10]] - ) + arr[0], + [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], arr[7], arr[8], arr[9], arr[10]], + ) elif len(arr) == self.DATA_LENGTH - 3: # (no prefix, period and tar_source): key, fname, type, time, priority, cond_download, config_out, cond_attr, absdir_outattr return self.add( - arr[0], [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], "NULL", arr[7], arr[8], arr[9]] - ) + arr[0], + [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], "NULL", arr[7], arr[8], arr[9]], + ) raise RuntimeError( "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) ) diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index 60169601b0..c179f6be2d 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -61,7 +61,7 @@ def add_file_unparsed(user_file, dicts, is_factory): relfname = os.path.basename(absfname) # default is the final part of absfname if len(relfname) < 1: raise RuntimeError("Found a file element with an empty relfname: %s" % user_file) - + is_const = is_true(user_file.const) is_executable = is_true(user_file.executable) or (user_file.type.startswith("exec")) is_wrapper = is_true(user_file.wrapper) or (user_file.type == "wrapper") @@ -69,7 +69,7 @@ def add_file_unparsed(user_file, dicts, is_factory): is_library = is_true(user_file.library) or user_file.type.startswith("library") is_periodic = user_file.type.startswith("periodic") do_untar = is_true(user_file.untar) or (user_file.type.startswith("untar")) - + time = user_file.time if is_executable or is_source or is_library: if ( @@ -84,39 +84,42 @@ def add_file_unparsed(user_file, dicts, is_factory): ): # we use startswith since we may have combination of time phases (e.g. startup, cleanup) raise RuntimeError("The file does not have a valid time phase value: %s" % user_file) - + try: if user_file.is_periodic: period_value = int(user_file.type.split(":")[1]) else: period_value = 0 except (AttributeError, KeyError, ValueError, TypeError): - period_value = 1000 # default 1000ms - - if period_value > 0: # period has 0 as default (in dictionary definition). Should I still protect against it not being defined? + period_value = 1000 # default 1000ms + + if ( + period_value > 0 + ): # period has 0 as default (in dictionary definition). Should I still protect against it not being defined? if not is_executable: raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) - + priority = user_file.priority if priority < 0 or priority > 99: - raise RuntimeError("Priority value out of the range [0,99]: %s" % user_file) - + raise RuntimeError("Priority value out of the range [0,99]: %s" % user_file) + prefix = user_file.prefix - + cond_download = user_file.cond_download - + tar_source = user_file.tar_source - if tar_source is None: - tar_source = "NULL" - + if tar_source is None: + tar_source = "NULL" + config_out = user_file.config_out if config_out is None: config_out = "FALSE" cond_attr = user_file.cond_attr - + absdir_outattr = user_file.absdir_outattr - - # TODO(F): what to do with period and after list + + # TODO(F): after list + if is_factory: # Factory (file_list, after_file_list) file_list_idx = "file_list" @@ -133,7 +136,7 @@ def add_file_unparsed(user_file, dicts, is_factory): if "after_group" in user_file: if is_true(user_file.after_group): file_list_idx = "aftergroup_%s" % file_list_idx - + if is_executable: # a script if not is_const: raise RuntimeError("A file cannot be executable if it is not constant: %s" % user_file) @@ -141,7 +144,7 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("A tar file cannot be executable: %s" % user_file) if is_wrapper: raise RuntimeError("A wrapper file cannot be an executable: %s" % user_file) - + file_type = "exec" if user_file.type: if user_file.type == "run:s" or user_file.type == "run:singularity": @@ -154,7 +157,16 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), file_type, prefix, time, period, priority, cond_download, tar_source, config_out, cond_attr + cWConsts.insert_timestr(relfname), + file_type, + prefix, + time, + period, + priority, + cond_download, + tar_source, + config_out, + cond_attr, ), absfname, ) @@ -164,40 +176,49 @@ def add_file_unparsed(user_file, dicts, is_factory): if do_untar: raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file) if is_source: - raise RuntimeError("A source file cannot be a wrapper: %s" % user_file) + raise RuntimeError("A source file cannot be a wrapper: %s" % user_file) if is_library: - raise RuntimeError("A library file cannot be a wrapper: %s" % user_file) + raise RuntimeError("A library file cannot be a wrapper: %s" % user_file) if is_periodic: - raise RuntimeError("A wrapper file cannot be periodic: %s" % user_file) - + raise RuntimeError("A wrapper file cannot be periodic: %s" % user_file) + dicts[file_list_idx].add_from_file( - relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname),"wrapper", tar_source=user_file.tar_source), absfname + relfname, + cWDictFile.FileDictFile.make_val_tuple( + cWConsts.insert_timestr(relfname), "wrapper", tar_source=user_file.tar_source + ), + absfname, ) - + elif do_untar: # a tarball if not is_const: raise RuntimeError("A file cannot be untarred if it is not constant: %s" % user_file) if is_periodic: - raise RuntimeError("A tar file cannot be periodic: %s" % user_file) + raise RuntimeError("A tar file cannot be periodic: %s" % user_file) if is_library: - raise RuntimeError("A tar file cannot be a library: %s" % user_file) + raise RuntimeError("A tar file cannot be a library: %s" % user_file) if is_executable: raise RuntimeError("A tar file cannot be executable: %s" % user_file) if is_wrapper: - raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file) - + raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file) + wnsubdir = user_file.type.split(":")[1] if wnsubdir is None: wnsubdir = absfname # default is relfname up to the first - + dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), "untar", cond_download=cond_download, config_out=config_out, cond_attr=cond_attr, absdir_outattr=wnsubdir + cWConsts.insert_timestr(relfname), + "untar", + cond_download=cond_download, + config_out=config_out, + cond_attr=cond_attr, + absdir_outattr=wnsubdir, ), absfname, ) - #dicts["untar_cfg"].add(relfname, wnsubdir) + # dicts["untar_cfg"].add(relfname, wnsubdir) elif is_source: if not is_const: raise RuntimeError("A file cannot be sourced if it is not constant: %s" % user_file) @@ -206,14 +227,18 @@ def add_file_unparsed(user_file, dicts, is_factory): if is_wrapper: raise RuntimeError("A wrapper file cannot be an sourced: %s" % user_file) if is_periodic: - raise RuntimeError("A source file cannot be periodic: %s" % user_file) + raise RuntimeError("A source file cannot be periodic: %s" % user_file) if is_library: - raise RuntimeError("A source file cannot be a library: %s" % user_file) - + raise RuntimeError("A source file cannot be a library: %s" % user_file) + dicts[file_list_idx].add_from_file( - relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "source", tar_source=tar_source, time=time, priority=priority), absfname + relfname, + cWDictFile.FileDictFile.make_val_tuple( + cWConsts.insert_timestr(relfname), "source", tar_source=tar_source, time=time, priority=priority + ), + absfname, ) - + elif is_library: if not is_const: raise RuntimeError("A file cannot be a library if it is not constant: %s" % user_file) @@ -222,13 +247,17 @@ def add_file_unparsed(user_file, dicts, is_factory): if is_wrapper: raise RuntimeError("A wrapper file cannot be an a library: %s" % user_file) if is_periodic: - raise RuntimeError("A library file cannot be periodic: %s" % user_file) - + raise RuntimeError("A library file cannot be periodic: %s" % user_file) + dicts[file_list_idx].add_from_file( - relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), user_file.type, tar_source=tar_source, time=time, priority=priority), absfname + relfname, + cWDictFile.FileDictFile.make_val_tuple( + cWConsts.insert_timestr(relfname), user_file.type, tar_source=tar_source, time=time, priority=priority + ), + absfname, ) # user_file.type can be library:x - + else: # not executable nor tarball => simple file if is_const: @@ -241,5 +270,5 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple(relfname, val), absfname ) # no timestamp in the name if it can be modified - - #TODO(F): config_out? + + # TODO(F): config_out? diff --git a/creation/lib/cWParams.py b/creation/lib/cWParams.py index 03dcda0d14..7c59dfdb53 100644 --- a/creation/lib/cWParams.py +++ b/creation/lib/cWParams.py @@ -521,7 +521,7 @@ def init_support_defaults(self): None, ) self.file_defaults["tar_source"] = ( - NULL, + "NULL", "tarball", "Specify the tar source where the file is contained.", None, diff --git a/creation/lib/cvWParamDict.py b/creation/lib/cvWParamDict.py index fa9dafbb97..2d61f0485a 100644 --- a/creation/lib/cvWParamDict.py +++ b/creation/lib/cvWParamDict.py @@ -283,8 +283,8 @@ def populate(self, params=None): os.path.join(params.src_dir, file_name), ) - #TODO(F): what to do here? - + # TODO(F): what to do here? + # Load initial system scripts # These should be executed before the other scripts for script_name in ("cat_consts.sh", "check_blacklist.sh"): diff --git a/creation/lib/cvWParams.py b/creation/lib/cvWParams.py index a77befd4e8..84d78c3587 100644 --- a/creation/lib/cvWParams.py +++ b/creation/lib/cvWParams.py @@ -62,6 +62,7 @@ def init_defaults(self): None, ) # TODO(F): remove this one? + # Gestisci anche lista, e porta a time con compatibilità in bash, qui no need # Publishing attr specific to frontend self.attr_defaults["type"] = [ diff --git a/creation/lib/xmlConfig.py b/creation/lib/xmlConfig.py index acfa12107a..28be83f695 100644 --- a/creation/lib/xmlConfig.py +++ b/creation/lib/xmlConfig.py @@ -293,7 +293,7 @@ def validate(self): is_tar = eval(self["untar"]) is_source = eval(self["source"]) is_library = eval(self["library"]) - + try: period = int(self["period"]) except ValueError: @@ -301,31 +301,32 @@ def validate(self): time = self["time"] if is_executable or is_source or is_library: - if ( - not time.startswith("startup") - and not time.startswith("cleanup") - and not time.startswith("after_job") - and not time.startswith("before_job") - and not time.startswith("periodic:") - and not time.startswith("milestone:") - and not time.startswith("failure:") - and not time.startswith("no_time:") - ): + if ( + not time.startswith("startup") + and not time.startswith("cleanup") + and not time.startswith("after_job") + and not time.startswith("before_job") + and not time.startswith("periodic:") + and not time.startswith("milestone:") + and not time.startswith("failure:") + and not time.startswith("no_time:") + ): # we use startswith since we may have combination of time phases (e.g. startup, cleanup) - raise RuntimeError(self.err_str('The file does not have a valid time phase value: %s'% user_file)) - + raise RuntimeError(self.err_str("The file does not have a valid time phase value: %s" % user_file)) + self.check_boolean("cond_download") - self.check_boolean("config_out") - - if is_exec + is_wrapper + is_tar + is_source + is_library> 1: - raise RuntimeError(self.err_str('must be exactly one of type "executable", "wrapper", "untar", "library" or "source"')) + self.check_boolean("config_out") + + if is_exec + is_wrapper + is_tar + is_source + is_library > 1: + raise RuntimeError( + self.err_str('must be exactly one of type "executable", "wrapper", "untar", "library" or "source"') + ) if (is_exec or is_wrapper or is_tar) and not eval(self["const"]): raise RuntimeError(self.err_str('type "executable", "wrapper", or "untar" requires const="True"')) if not is_exec and period > 0: raise RuntimeError(self.err_str('cannot have execution period if type is not "executable"')) - - + TAG_CLASS_MAPPING.update({"file": FileElement}) diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 0f46c6edea..bd90e329d7 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -225,18 +225,18 @@ fetch_file_try() { fft_file_type="$4" fft_cc_prefix="$5" fft_time="$6" - fft_period="$7" + fft_period="$7" fft_coordination="$8" fft_config_check="$9" fft_tar_source="${10}" fft_config_out="${11}" fft_cond_attr="${12}" fft_cond_attr="${13}" - + if [[ "${fft_tar_source}" != "NULL" ]]; then return 0 fi - + if [[ "${fft_config_check}" != "TRUE" ]]; then # TRUE is a special case, always be downloaded and processed fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 74850e8577..660adc77a7 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -172,4 +172,3 @@ failure_call(){ custom_scripts "failure:"$exit_code return 0 } - From 68d215068158210ece637d4a9ce3c6d778e1c4a0 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 7 Sep 2022 14:46:01 -0500 Subject: [PATCH 072/196] Added compatibility support with older versions, converter to the new version starting from the old one and solved some errors --- creation/conversion_xml.py | 89 +++++++++ creation/lib/cWConsts.py | 5 +- creation/lib/cWDictFile.py | 102 +++++----- creation/lib/cWParamDict.py | 11 +- creation/lib/cWParams.py | 2 - creation/lib/cgWParamDict.py | 177 +++++++++++++++--- creation/lib/cgWParams.py | 10 - creation/lib/cvWConsts.py | 2 - creation/lib/cvWParamDict.py | 2 - creation/lib/cvWParams.py | 8 - creation/lib/factory_defaults.xml | 6 +- creation/reconfig_frontend | 8 + creation/reconfig_glidein | 5 + creation/web_base/glidein_startup.sh | 14 +- creation/web_base/utils_gs_http.sh | 122 ++++++++---- creation/web_base/utils_scripts.sh | 33 ++-- .../bats/creation_web_base_utils_scripts.bats | 34 +++- 17 files changed, 478 insertions(+), 152 deletions(-) create mode 100644 creation/conversion_xml.py diff --git a/creation/conversion_xml.py b/creation/conversion_xml.py new file mode 100644 index 0000000000..0ac14a4950 --- /dev/null +++ b/creation/conversion_xml.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + +# Description: +# These functions check the version of xml files +# and perform the conversion if needed + +import xml.etree.ElementTree as ET + + +def convert(xml_file): + with open(xml_file, encoding="latin-1") as f: + tree = ET.parse(f) + for elem in tree.findall("file"): + print(elem) + try: + if "after_entry" in elem.attrib or "after_group" in elem.attrib: + if elem.attrib.get("after_entry") == "True": + if not "after_group" in elem.attrib: + elem.attrib["priority"] = "90" + elif elem.attrib.get("after_group") == "True": + elem.attrib["priority"] = "80" + else: + elem.attrib["priority"] = "60" + elif elem.attrib.get("after_entry") == "False": + if not "after_group" in elem.attrib: + elem.attrib["priority"] = "10" + elif elem.attrib.get("after_group") == "True": + elem.attrib["priority"] = "40" + else: + elem.attrib["priority"] = "20" + if "after_entry" in elem.attrib: + elem.attrib.pop("after_entry") + if "after_group" in elem.attrib: + elem.attrib.pop("after_group") + else: + continue + + if elem.attrib.get("executable") == "True": + elem.attrib.pop("executable") + elem.attrib["type"] = "exec" + + if "period" in elem.attrib: + elem.attrib["type"] = "periodic:" + elem.attrib.get("period") + elem.attrib.pop("period") + + if elem.attrib.get("wrapper") == "True": + elem.attrib.pop("wrapper") + elem.attrib["type"] = "wrapper" + + if elem.attrib.get("source") == "True": + elem.attrib.pop("source") + elem.attrib["type"] = "source" + + if elem.attrib.get("library") == "True": + elem.attrib.pop("library") + elem.attrib["type"] = "library" + + if elem.attrib.get("untar") == "True": + elem.attrib["type"] = "untar" + elem.attrib.pop("untar") + + if elem.attrib.get("type") == "untar": + for child in elem: + print(child) + if child.tag == "untar_options": + print("yes") + elem.attrib["cond_attr"] = child.attrib["cond_attr"] + elem.attrib["absdir_outattr"] = child.attrib["absdir_outattr"] + elem.attrib["type"] = "untar:" + child.attrib["dir"] + elem.remove(child) + + except AttributeError: + pass + tree.write("2" + xml_file, encoding="latin-1") + + +def is_old_version(xml_file): + with open(xml_file, encoding="latin-1") as f: + tree = ET.parse(f) + for elem in tree.findall("file"): + if "after_entry" in elem.attrib or "after_group" in elem.attrib or "period" in elem.attrib: + return True + for child in elem: + if child.tag == "untar_options": + return True + return False diff --git a/creation/lib/cWConsts.py b/creation/lib/cWConsts.py index 861bc9ea65..4c7990a259 100644 --- a/creation/lib/cWConsts.py +++ b/creation/lib/cWConsts.py @@ -78,8 +78,7 @@ def insert_timestr(instr): GRIDMAP_FILE = "grid-mapfile" -# TODO(F) # Using two digits for major/minor/patch version -# This is the first version (3.9.11) with this new format. -# Change this number only if the DictFIle format changes +# This is the first version (3.11.0) with this new format +# Change this number only if the DictFile format changes DICT_FILE_VERSION = "031100" diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index 0fcf97ea9e..efd54f084d 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -437,8 +437,7 @@ def file_header(self, want_comments): str: The file header, a comment containing the file name """ if want_comments: - return "# File: %s\n#" % self.fname - # TODO(F) versione + return f"# File: {self.fname}\n# Version: {cWConsts.DICT_FILE_VERSION}\n#" else: return None @@ -988,7 +987,6 @@ def save_files(self, allow_overwrite=False): raise DictFileError("Error writing into file %s" % filepath) -# TODO(F) class FileDictFile(SimpleFileDictFile): """Dictionary file for files (file list). Used for list of transferred files. @@ -1037,8 +1035,10 @@ class FileDictFile(SimpleFileDictFile): 9. config_out - has a special value of FALSE 10. cond_attr - Name of a configuration switch. (like "ENABLE_KRB5") The operation requested on the file will be performed only if that parameter will be set to 1. + If not set (set to NULL), the result will be considered as TRUE (i.e. download) + As result, a file will be downloaded if cond_download & $cond_attr == TRUE 11. absdir_outattr - Name of a variable name. (like "KRB5_SUBSYS_DIR") - The variable will be set to the absolute path of the file if not a tarball + The variable will be set to the absolute path of the file if the file is not a tarball or the directory where the tarball was unpacked in case it is a tarball, this last thing if and only if the unpacking actually happened (else it will not be defined.) ENTRY_ will be prepended if the directive occurs in an entry. @@ -1272,6 +1272,8 @@ def parse_val(self, line): :param line: string with the line content :return: tuple with DATA_LENGTH-1 values """ + if line[0].startswith("# Version:"): + version = line[0].split(":")[1] if not line or line[0] == "#": return # ignore empty lines and comments arr = line.split(None, self.DATA_LENGTH - 1) # split already eliminates multiple spaces (no need for strip) @@ -1280,47 +1282,63 @@ def parse_val(self, line): if len(arr[0]) == 0: return # empty key - if len(arr) != self.DATA_LENGTH: - # compatibility w/ old formats - # 3.2.13 (no prefix): key, fname, type, time, period, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr - # TODO: remove in 3.3 or after a few version (will break upgrade) - - # TODO(F): parse versione - if len(arr) == self.DATA_LENGTH - 1: - # For upgrade from 3.2.13 to 3.2.11 - return self.add( - arr[0], - [ - arr[1], - arr[2], - "GLIDEIN_PS_", - arr[3], - arr[4], - arr[5], - arr[6], - arr[7], - arr[8], - arr[9], - arr[10], - arr[11], - ], + # compatibility w/ old formats + # 3.2.13 (no prefix): key, fname, type, period, cond_download, config_out + if version <= "030213": + if len(arr) != self.DATA_LENGTH: + # TODO: remove in 3.3 or after a few version (will break upgrade) + if len(arr) == self.DATA_LENGTH - 1: + # For upgrade from 3.2.13 to 3.2.11 + return self.add( + arr[0], + [ + arr[1], + arr[2], + "GLIDEIN_PS_", + arr[3], + arr[4], + arr[5], + arr[6], + arr[7], + arr[8], + arr[9], + arr[10], + arr[11], + ], + ) + raise RuntimeError( + "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) ) - elif len(arr) == self.DATA_LENGTH - 2: - # (no prefix, and period ): key, fname, type, time, priority, cond_download, config_out, cond_attr, absdir_outattr - return self.add( - arr[0], - [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], arr[7], arr[8], arr[9], arr[10]], + # 3.2.10 (no period, prefix): key, fname, type, cond_download, config_out + elif version <= "030210": + if len(arr) != self.DATA_LENGTH: + if len(arr) == self.DATA_LENGTH - 2: + # For upgrade from 3.2.10 or earlier + return self.add( + arr[0], + [ + arr[1], + arr[2], + "GLIDEIN_PS_", + arr[3], + 0, + arr[4], + arr[5], + arr[6], + arr[7], + arr[8], + arr[9], + arr[10], + ], + ) + raise RuntimeError( + "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) ) - elif len(arr) == self.DATA_LENGTH - 3: - # (no prefix, period and tar_source): key, fname, type, time, priority, cond_download, config_out, cond_attr, absdir_outattr - return self.add( - arr[0], - [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], "NULL", arr[7], arr[8], arr[9]], + else: + if len(arr) != self.DATA_LENGTH: + raise RuntimeError( + "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) ) - raise RuntimeError( - "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) - ) - return self.add(arr[0], arr[1:]) def get_immutable_files(self): diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index c179f6be2d..7805f444aa 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -53,6 +53,7 @@ def add_file_unparsed(user_file, dicts, is_factory): :param is_factory: True if invoked for the factory (cgWParamDict.py), false for the frontend (cvWParamDict.py) :return: None (dictionaries are modified) """ + absfname = user_file.absfname if absfname is None: raise RuntimeError("Found a file element without an absname: %s" % user_file) @@ -118,13 +119,11 @@ def add_file_unparsed(user_file, dicts, is_factory): absdir_outattr = user_file.absdir_outattr - # TODO(F): after list - if is_factory: # Factory (file_list, after_file_list) file_list_idx = "file_list" - if "after_entry" in user_file: - if is_true(user_file.after_entry): # eval(user_file.after_entry,{},{}): + if "priority" in user_file: + if user_file.priority >= 60: # eval(user_file.after_entry,{},{}): file_list_idx = "after_file_list" else: # Frontend (preentry_file_list, file_list, aftergroup_preentry_file_list, aftergroup_file_list) @@ -204,7 +203,7 @@ def add_file_unparsed(user_file, dicts, is_factory): wnsubdir = user_file.type.split(":")[1] if wnsubdir is None: - wnsubdir = absfname # default is relfname up to the first + wnsubdir = relfname # default is relfname up to the first dicts[file_list_idx].add_from_file( relfname, @@ -270,5 +269,3 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple(relfname, val), absfname ) # no timestamp in the name if it can be modified - - # TODO(F): config_out? diff --git a/creation/lib/cWParams.py b/creation/lib/cWParams.py index 7c59dfdb53..20bf68db63 100644 --- a/creation/lib/cWParams.py +++ b/creation/lib/cWParams.py @@ -449,7 +449,6 @@ def extract_attr_val(self, attr_obj): return extract_attr_val(attr_obj) -# TODO(F) class CommonParams(Params): # populate self.defaults def init_support_defaults(self): @@ -513,7 +512,6 @@ def init_support_defaults(self): None, ) - # TODO(F): ok specificare qualifier come secondo argomento? self.file_defaults["time"] = ( "no_time", "startup|before_job|after_job|cleanup|periodic:period|milestone:code|failure:code|no", diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 827ad1f5a2..66aacc6b8c 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -902,8 +902,6 @@ def new_SubDicts(self, sub_name): # ############################################################ -# TODO(F): what to do differently from the general one? - ############################################# # Add a user file residing in the stage area # file as described by Params.file_defaults @@ -913,6 +911,8 @@ def new_SubDicts(self, sub_name): # the common cWParamDict version # # is_factory is just a dummy placeholder to make the transition easier later + + def add_file_unparsed(user_file, dicts, is_factory): absfname = user_file["absfname"] @@ -922,13 +922,61 @@ def add_file_unparsed(user_file, dicts, is_factory): relfname = user_file["relfname"] is_const = eval(user_file["const"]) - is_executable = eval(user_file["executable"]) - is_wrapper = eval(user_file["wrapper"]) - do_untar = eval(user_file["untar"]) + is_executable = eval(user_file["executable"]) or (user_file.type.startswith("exec")) + is_wrapper = eval(user_file["wrapper"]) or (user_file.type == "wrapper") + do_untar = eval(user_file["untar"]) or (user_file.type.startswith("untar")) + is_const = is_true(user_file.const) + is_source = eval(user_file["source"]) or user_file.type.startswith("source") + is_library = eval(user_file["library"]) or user_file.type.startswith("library") + is_periodic = user_file.type.startswith("periodic") + + time = user_file.time + if is_executable or is_source or is_library: + if ( + not time.startswith("startup") + and not time.startswith("cleanup") + and not time.startswith("after_job") + and not time.startswith("before_job") + and not time.startswith("periodic:") + and not time.startswith("milestone:") + and not time.startswith("failure:") + and not time.startswith("no_time:") + ): + # we use startswith since we may have combination of time phases (e.g. startup, cleanup) + raise RuntimeError("The file does not have a valid time phase value: %s" % user_file) + + try: + if user_file.is_periodic: + period_value = int(user_file.type.split(":")[1]) + else: + period_value = 0 + except (AttributeError, KeyError, ValueError, TypeError): + period_value = 1000 # default 1000ms + + if ( + period_value > 0 + ): # period has 0 as default (in dictionary definition). Should I still protect against it not being defined? + if not is_executable: + raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) + + priority = user_file.priority + if priority < 0 or priority > 99: + raise RuntimeError("Priority value out of the range [0,99]: %s" % user_file) - prefix = user_file["prefix"] + prefix = user_file.prefix - period_value = int(user_file["period"]) + cond_download = user_file.cond_download + + tar_source = user_file.tar_source + if tar_source is None: + tar_source = "NULL" + + config_out = user_file.config_out + if config_out is None: + config_out = "FALSE" + cond_attr = user_file.cond_attr + + absdir_outattr = user_file.absdir_outattr file_list_idx = "file_list" if "after_entry" in user_file: @@ -936,36 +984,121 @@ def add_file_unparsed(user_file, dicts, is_factory): file_list_idx = "after_file_list" if is_executable: # a script + if not is_const: + raise RuntimeError("A file cannot be executable if it is not constant: %s" % user_file) + if do_untar: + raise RuntimeError("A tar file cannot be executable: %s" % user_file) + if is_wrapper: + raise RuntimeError("A wrapper file cannot be an executable: %s" % user_file) + file_type = "exec" + if user_file.type: + if user_file.type == "run:s" or user_file.type == "run:singularity": + if file_list_idx.endswith("preentry_file_list"): + raise RuntimeError("An executable cannot use singularity before the entry setup: %s" % user_file) + file_type = "exec:s" + else: + if not user_file.type.startswith("run"): + raise RuntimeError("An executable file type must start with 'run': $s" % user_file) dicts[file_list_idx].add_from_file( relfname, - cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "exec", period_value, prefix), + cWDictFile.FileDictFile.make_val_tuple( + cWConsts.insert_timestr(relfname), + file_type, + prefix, + time, + period, + priority, + cond_download, + tar_source, + config_out, + cond_attr, + ), absfname, ) elif is_wrapper: # a source-able script for the wrapper + if not is_const: + raise RuntimeError("A file cannot be a wrapper if it is not constant: %s" % user_file) + if do_untar: + raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file) + if is_source: + raise RuntimeError("A source file cannot be a wrapper: %s" % user_file) + if is_library: + raise RuntimeError("A library file cannot be a wrapper: %s" % user_file) + if is_periodic: + raise RuntimeError("A wrapper file cannot be periodic: %s" % user_file) + dicts[file_list_idx].add_from_file( - relfname, cWDictFile.FileDictFile.make_val_tuple(cWConsts.insert_timestr(relfname), "wrapper"), absfname + relfname, + cWDictFile.FileDictFile.make_val_tuple( + cWConsts.insert_timestr(relfname), "wrapper", tar_source=user_file.tar_source + ), + absfname, ) elif do_untar: # a tarball - untar_opts = user_file.get_child("untar_options") - if "dir" in untar_opts: - wnsubdir = untar_opts["dir"] - else: - wnsubdir = relfname.split(".", 1)[0] # default is relfname up to the first . - - if "absdir_outattr" in untar_opts: - config_out = untar_opts["absdir_outattr"] - else: - config_out = "FALSE" - cond_attr = untar_opts["cond_attr"] + if not is_const: + raise RuntimeError("A file cannot be untarred if it is not constant: %s" % user_file) + if is_periodic: + raise RuntimeError("A tar file cannot be periodic: %s" % user_file) + if is_library: + raise RuntimeError("A tar file cannot be a library: %s" % user_file) + if is_executable: + raise RuntimeError("A tar file cannot be executable: %s" % user_file) + if is_wrapper: + raise RuntimeError("A tar file cannot be a wrapper: %s" % user_file) + + wnsubdir = user_file.type.split(":")[1] + if wnsubdir is None: + wnsubdir = relfname # default is relfname up to the first dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), "untar", cond_download=cond_attr, config_out=config_out + cWConsts.insert_timestr(relfname), + "untar", + cond_download=cond_download, + config_out=config_out, + cond_attr=cond_attr, + absdir_outattr=wnsubdir, + ), + absfname, + ) + # dicts["untar_cfg"].add(relfname, wnsubdir) + elif is_source: + if not is_const: + raise RuntimeError("A file cannot be sourced if it is not constant: %s" % user_file) + if do_untar: + raise RuntimeError("A tar file cannot be sourced: %s" % user_file) + if is_wrapper: + raise RuntimeError("A wrapper file cannot be an sourced: %s" % user_file) + if is_periodic: + raise RuntimeError("A source file cannot be periodic: %s" % user_file) + if is_library: + raise RuntimeError("A source file cannot be a library: %s" % user_file) + dicts[file_list_idx].add_from_file( + relfname, + cWDictFile.FileDictFile.make_val_tuple( + cWConsts.insert_timestr(relfname), "source", tar_source=tar_source, time=time, priority=priority ), absfname, ) - dicts["untar_cfg"].add(relfname, wnsubdir) + elif is_library: + if not is_const: + raise RuntimeError("A file cannot be a library if it is not constant: %s" % user_file) + if do_untar: + raise RuntimeError("A tar file cannot be a library: %s" % user_file) + if is_wrapper: + raise RuntimeError("A wrapper file cannot be an a library: %s" % user_file) + if is_periodic: + raise RuntimeError("A library file cannot be periodic: %s" % user_file) + dicts[file_list_idx].add_from_file( + relfname, + cWDictFile.FileDictFile.make_val_tuple( + cWConsts.insert_timestr(relfname), user_file.type, tar_source=tar_source, time=time, priority=priority + ), + absfname, + ) + # user_file.type can be library:x + else: # not executable nor tarball => simple file if is_const: val = "regular" diff --git a/creation/lib/cgWParams.py b/creation/lib/cgWParams.py index 3eb9742a17..a261bf7d47 100644 --- a/creation/lib/cgWParams.py +++ b/creation/lib/cgWParams.py @@ -467,17 +467,7 @@ def init_defaults(self): self.defaults["attrs"] = sub_defaults["attrs"] self.defaults["files"] = copy.deepcopy(sub_defaults["files"]) - # ordering is specific to global section of factory - self.defaults["files"][3]["after_entry"] = ( - "False", - "Bool", - "Should this file be loaded after the entry ones?", - None, - ) - # TODO(F): what to do? - self.defaults["entries"] = (OrderedDict(), "Dictionary of entries", "Each entry contains", self.entry_defaults) - return # return name of top element diff --git a/creation/lib/cvWConsts.py b/creation/lib/cvWConsts.py index 4cbc90c50c..0c8ea3138e 100644 --- a/creation/lib/cvWConsts.py +++ b/creation/lib/cvWConsts.py @@ -20,8 +20,6 @@ AFTERGROUP_FILE_LISTFILE = "aftergroup_%s" % cWConsts.FILE_LISTFILE PREENTRY_FILE_LISTFILE = "preentry_%s" % cWConsts.FILE_LISTFILE AFTERGROUP_PREENTRY_FILE_LISTFILE = "aftergroup_%s" % PREENTRY_FILE_LISTFILE -# TODO(F): remain the same? - PARAMS_FILE = "params.cfg" ATTRS_FILE = "attrs.cfg" diff --git a/creation/lib/cvWParamDict.py b/creation/lib/cvWParamDict.py index 2d61f0485a..4474b6da2b 100644 --- a/creation/lib/cvWParamDict.py +++ b/creation/lib/cvWParamDict.py @@ -283,8 +283,6 @@ def populate(self, params=None): os.path.join(params.src_dir, file_name), ) - # TODO(F): what to do here? - # Load initial system scripts # These should be executed before the other scripts for script_name in ("cat_consts.sh", "check_blacklist.sh"): diff --git a/creation/lib/cvWParams.py b/creation/lib/cvWParams.py index 84d78c3587..7d8c0cec72 100644 --- a/creation/lib/cvWParams.py +++ b/creation/lib/cvWParams.py @@ -55,14 +55,6 @@ def init_defaults(self): # but there could be exceptions # Files/Validation/Custom scripts settings for frontend - self.file_defaults["after_entry"] = ( - "True", - "Bool", - "Should this file be loaded after the factory entry ones?", - None, - ) - # TODO(F): remove this one? - # Gestisci anche lista, e porta a time con compatibilità in bash, qui no need # Publishing attr specific to frontend self.attr_defaults["type"] = [ diff --git a/creation/lib/factory_defaults.xml b/creation/lib/factory_defaults.xml index d81e378d68..624991f528 100644 --- a/creation/lib/factory_defaults.xml +++ b/creation/lib/factory_defaults.xml @@ -115,8 +115,8 @@ SPDX-License-Identifier: Apache-2.0 - - + + @@ -143,7 +143,7 @@ SPDX-License-Identifier: Apache-2.0 - + diff --git a/creation/reconfig_frontend b/creation/reconfig_frontend index 4e517b45d9..01ae7a0fb5 100755 --- a/creation/reconfig_frontend +++ b/creation/reconfig_frontend @@ -17,6 +17,8 @@ import signal import sys import tempfile +import conversion_xml + from glideinwms.creation.lib import ( check_config_frontend, check_python3_expr, @@ -130,6 +132,7 @@ def main(params, old_params, update_scripts, update_def_cfg): ############################################################ if __name__ == "__main__": + usage = "usage: reconfig_frontend { -force_name name -writeback yes|no -update_scripts yes|no -xml xml -update_def_cfg yes|no [-xslt_plugin_dir xdir] | -help }" argv = sys.argv @@ -142,6 +145,11 @@ if __name__ == "__main__": "NOTE: Executing reconfig_frontend as user 'root' is not allowed. Use the frontend user instead. For rpm based installations, use the 'service gwms-frontend ' command to perform gwms-frontend operations" ) + # TODO(F): read xml file and check if compatible + # if is_old_version("frontend.xml"): + # print2("The xml file seems to be outdated. Consider to convert the structure to version 3.11.0") + # sys.exit(1) + force_name = None writeback = "no" update_scripts = "no" diff --git a/creation/reconfig_glidein b/creation/reconfig_glidein index 413297b32a..c7c9c01e09 100755 --- a/creation/reconfig_glidein +++ b/creation/reconfig_glidein @@ -211,6 +211,7 @@ def main(conf, update_scripts, update_def_cfg, comment=""): ############################################################ if __name__ == "__main__": + exit_code = 0 usage = "usage: reconfig_glidein { -force_name name -writeback yes|no -update_scripts yes|no -xml xml -update_def_cfg yes|no [-force_delete] [-xslt_plugin_dir xdir] | -help }" argv = sys.argv @@ -224,6 +225,10 @@ if __name__ == "__main__": "NOTE: Executing reconfig_glidein as user 'root' is not allowed. Use the factory user instead. For rpm based installations, use the 'service gwms-factory ' command to perform gwms-factory operations" ) + # if is_old_version("glideinWMS.xml"): + # print2("The xml file seems to be outdated. Consider to convert the structure to version 3.11.0") + # sys.exit(1) + force_name = None writeback = "no" update_scripts = "no" diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index 00fbed18e7..f894630c8c 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -514,15 +514,23 @@ _main(){ # shellcheck disable=2086 while read -r file do + # read version number + if [ "${file:0:10}" == "# Version:" ]; then + OLD_IFS=$IFS + IFS=':' read -ra ARR <<< "${file}" + version="${ARR[1]}" + IFS=$OLD_IFS + fi + # for each non-comment/non-header line if [ "${file:0:1}" != "#" ]; then fetch_file "${gs_id}" $file - if [ "${ffb_file_type}" != "wrapper" ] && [ "${ffb_file_type}" != "untar" ] && [ "${ffb_file_type}" != "regular" ]; then - add_entry "$ffb_outname" "$ffb_file_type" "$ffb_time" "$ffb_coordination" "$ffb_period" "$ffb_cc_prefix" "$gs_id" "$fft_tar_source" + # wrappers, tarballs and regular files just need to be downloaded + if [ "${ff_file_type}" != "wrapper" ] && [ "${ff_file_type}" != "untar" ] && [ "${ff_file_type}" != "regular" ]; then + add_entry "$ff_real_fname" "$ff_file_type" "$ff_time" "$ff_coordination" "$ff_period" "$ff_cc_prefix" "$gs_id" "$ff_tar_source" fi fi done < "${gs_id_work_dir}/${gs_file_list}" - # Files to go into the GWMS_PATH if [ "$gs_file_id" = "main at_file_list" ]; then # setup here to make them available for other setup scripts diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index bd90e329d7..44a7730375 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -128,49 +128,96 @@ fetch_file_regular() { # 12: cond_attr # 13: absdir_outattr # The above is the most recent list, below some adaptations for different versions +# Globals (r/w): +# ff_id +# ff_target_fname +# ff_real_fname +# ff_file_type +# ff_time +# ff_period +# ff_cc_prefix +# ff_coordination +# ff_config_check +# ff_config_out +# ff_tar_source +# ff_cond_attr +# ff_absdir # Used: # ifs_str # IFS +# version # Returns: # 0 in case of success # otherwise glidein_exit with 1 fetch_file() { + ff_id="$1" + ff_target_fname="$2" + ff_real_fname="$3" + ff_file_type="$4" + # for compatibility reasons, check the version and act opportunately + if [[ $version < "031100" ]]; then + # Different list of arguments in case of old format + # 1. ID + # 2. target fname + # 3. real fname + # 4. file type (regular, exec, exec:s, untar, nocache) + # 5. period (0 if not a periodic file) + # 6. periodic scripts prefix + # 7. config check TRUE,FALSE + # 8. config out TRUE,FALSE + ff_time="startup" + ff_period="$5" + ff_cc_prefix="$6" + ff_coordination=0 + ff_config_check="$7" + ff_config_out="$8" + ff_tar_source="NULL" + ff_cond_attr="NULL" + ff_absdir="NULL" + else + ff_cc_prefix="$5" + ff_time="$6" + ff_period="$7" + ff_coordination="$8" + ff_config_check="$9" + ff_tar_source="${10}" + ff_config_out="${11}" + ff_cond_attr="${12}" + ff_absdir="${13}" + fi + # From version 3.11.0 we need 13 arguments if [ $# -gt 13 ]; then # For compatibility w/ future versions (add new parameters at the end) - echo "More then 11 arguments, considering the first 11 ($#/${ifs_str}): $*" 1>&2 - elif [ $# -ne 13 ]; then - if [ $# -eq 12 ]; then - #TODO: remove in version 3.3 - # For compatibility with past versions (old file list formats) - # 3.2.13 and older: prefix (par 6) added in #12705, 3.2.14? - # 3.2.10 and older: period (par 5) added - if ! fetch_file_try "$1" "$2" "$3" "$4" "GLIDEIN_PS_" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" "${12}"; then - glidein_exit 1 - fi - return 0 - fi - if [ $# -eq 11 ]; then - # added to maintain compatibility with older (3.2.10) file list format - #TODO: remove in version 3.3 - if ! fetch_file_try "$1" "$2" "$3" "$4" "GLIDEIN_PS_" "$5" 0 "$6" "$7" "$8" "$9" "${10}" "${11}"; then - glidein_exit 1 + echo "More then 13 arguments, considering the first 13 ($#/${ifs_str}): $*" 1>&2 + fi + # TODO: consider to remove considering the current versions + if [[ $version < "031100" ]]; then + if [ $# -lt 8 ]; then + if [ $# -eq 7 ]; then + #TODO: remove in version 3.3 + # For compatibility with past versions (old file list formats) + # 3.2.13 and older: prefix (par 6) added in #12705, 3.2.14? + # 3.2.10 and older: period (par 5) added + if ! fetch_file_try "$ff_id" "$ff_target_fname" "$ff_real_fname" "$ff_file_type" "GLIDEIN_PS_" "$ff_time" "$ff_period" "$ff_coordination" "$ff_config_check" "$ff_tar_source" "$ff_config_out" "$ff_cond_attr" "$ff_absdir"; then + glidein_exit 1 + fi + return 0 fi - return 0 - fi - if [ $# -eq 10 ]; then - # added to maintain compatibility with older (3.2.10) file list format - #TODO: remove in version 3.3 - if ! fetch_file_try "$1" "$2" "$3" "$4" "GLIDEIN_PS_" "$5" 0 "$6" "$7" "NULL" "$8" "$9" "${10}"; then - glidein_exit 1 + if [ $# -eq 6 ]; then + # added to maintain compatibility with older (3.2.10) file list format + #TODO: remove in version 3.3 + if ! fetch_file_try "$ff_id" "$ff_target_fname" "$ff_real_fname" "$ff_file_type" "GLIDEIN_PS_" "$ff_time" 0 "$ff_coordination" "$ff_config_check" "$ff_tar_source" "$ff_config_out" "$ff_cond_attr" "$ff_absdir"; then + glidein_exit 1 + fi + return 0 fi - return 0 + local ifs_str + printf -v ifs_str '%q' "${IFS}" + log_warn "Not enough arguments in fetch_file, 8 or 13 expected ($#/${ifs_str}): $*" + glidein_exit 1 fi - local ifs_str - printf -v ifs_str '%q' "${IFS}" - log_warn "Not enough arguments in fetch_file, 11 expected ($#/${ifs_str}): $*" - glidein_exit 1 fi - if ! fetch_file_try "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" "${12}" "${13}"; then + if ! fetch_file_try "$ff_id" "$ff_target_fname" "$ff_real_fname" "$ff_file_type" "$ff_cc_prefix" "$ff_time" "$ff_period" "$ff_coordination" "$ff_config_check" "$ff_tar_source" "$ff_config_out" "$ff_cond_attr" "$ff_absdir"; then glidein_exit 1 fi return 0 @@ -231,13 +278,24 @@ fetch_file_try() { fft_tar_source="${10}" fft_config_out="${11}" fft_cond_attr="${12}" - fft_cond_attr="${13}" + fft_absdir="${13}" + + # absdir_outattr - Name of a variable name. (like "KRB5_SUBSYS_DIR") + # The variable will be set to the absolute path of the file if the file is not a tarball + # or the directory where the tarball was unpacked in case it is a tarball, + # this last thing if and only if the unpacking actually happened (else it will not be defined.) + + if [[ "${fft_absdir}" != "NULL" ]]; then + eval ${fft_absdir}="fft_real_fname" + fi + # if contained in a tarball, do not download if [[ "${fft_tar_source}" != "NULL" ]]; then return 0 fi - if [[ "${fft_config_check}" != "TRUE" ]]; then + # if fft_config_check is not TRUE or the variable pointed by fft_cond_attr (if any) is not TRUE, do not download (and operator between the two) + if [[ "${fft_config_check}" != "TRUE" || ("${fft_cond_attr}" != "NULL" && "${!fft_cond_attr}" != "TRUE") ]]; then # TRUE is a special case, always be downloaded and processed fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) # Stop download and processing if the cond_attr variable is not defined or has a value different from 1 diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 660adc77a7..d904345268 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -19,17 +19,19 @@ # 6: prefix # 7: id # 8: tar_source +# Used: +# version add_entry(){ - local e_real_fname e_type e_time e_coordination descriptor_file - e_real_fname="$1" - e_type="$2" - e_time="$3" - e_coordination="$4" - e_period="$5" - e_prefix="$6" - e_id="$7" - e_tar_source="$8" - descriptor_file="testfile" + local e_real_fname e_type e_time e_coordination e_period e_prefix e_id e_tar_source descriptor_file + e_real_fname="$2" + e_type="$3" + e_time="$4" + e_coordination="$5" + e_period="$6" + e_prefix="$7" + e_id="$8" + e_tar_source="$9" + descriptor_file=$gwms_exec_dir/"descriptor_file.txt" if [ ! -f "$descriptor_file" ]; then echo "# File: $descriptor_file" > "$descriptor_file" @@ -37,6 +39,7 @@ add_entry(){ echo "# Time OrderedFilename RealFilename Type Period Prefix Id" >> "$descriptor_file" echo "################################################################################################" >> "$descriptor_file" fi + # From version 3.11.0 OLD_IFS=$IFS IFS=', ' read -r -a array <<< $e_time for time_entry in "${array[@]}" @@ -50,7 +53,7 @@ add_entry(){ e_complete_fname="$e_real_fname" fi echo "${time_entry} ${e_coordination}_${e_real_fname} ${e_complete_fname} ${e_type} ${e_period} ${e_prefix} ${e_id}" >> "$descriptor_file" - done + done IFS=$OLD_IFS } @@ -63,8 +66,8 @@ add_entry(){ extract_entry_files(){ local target_time descriptor_file target_time="$1" - descriptor_file="testfile" - grep ^$target_time $descriptor_file | sort > ${target_time}_descriptor_file + descriptor_file=$gwms_exec_dir/"descriptor_file.txt" + grep ^$target_time $descriptor_file | sort > $gwms_exec_dir/${target_time}_descriptor_file.txt } ################################ @@ -137,8 +140,8 @@ custom_scripts(){ source "${ffb_target_fname}" # TODO: what about other library types? fi - done < ${target_time}_descriptor_file - rm -f ${target_time}_descriptor_file + done < $gwms_exec_dir/${target_time}_descriptor_file.txt + rm -f $gwms_exec_dir/${target_time}_descriptor_file.txt } ################################ diff --git a/test/bats/creation_web_base_utils_scripts.bats b/test/bats/creation_web_base_utils_scripts.bats index dbe23d7d79..90f4b77b6d 100755 --- a/test/bats/creation_web_base_utils_scripts.bats +++ b/test/bats/creation_web_base_utils_scripts.bats @@ -34,4 +34,36 @@ setup () { [ "$status" -eq 0 ] } -#TODO: Bats tests for custom_scripts, extract_entry_files and add_entry not defined since they work on the descriptor_file statically defined +@test "add_entry" { + gwms_exec_dir="/tmp" + get_untar_subdir(){ + echo "tmp" + } + run add_entry name type time coordination period prefix id tar_source + cat $gwms_exec_dir/descriptor_file.txt >& 3 + grep -q "Time OrderedFilename RealFilename Type Period Prefix Id" $gwms_exec_dir/descriptor_file.txt + grep -q "time coordination_name /tmp/name type period prefix id" $gwms_exec_dir/descriptor_file.txt + rm -f $gwms_exec_dir/descriptor_file.txt + run add_entry name type time,time2 coordination period prefix id tar_source + cat $gwms_exec_dir/descriptor_file.txt >& 3 + grep -q "Time OrderedFilename RealFilename Type Period Prefix Id" $gwms_exec_dir/descriptor_file.txt + grep -q "time coordination_name /tmp/name type period prefix id" $gwms_exec_dir/descriptor_file.txt + grep -q "time2 coordination_name /tmp/name type period prefix id" $gwms_exec_dir/descriptor_file.txt + rm -f $gwms_exec_dir/descriptor_file.txt +} + + +@test "extract_entry_files" { + gwms_exec_dir="/tmp" + get_untar_subdir(){ + echo "tmp" + } + run add_entry name type time2 coordination period prefix id tar_source + run add_entry name type time3 coordination period prefix id tar_source + run add_entry name type time coordination period prefix id tar_source + cat $gwms_exec_dir/descriptor_file.txt >& 3 + run extract_entry_files "time" + grep -q "time coordination_name /tmp/name type period prefix id" $gwms_exec_dir/time_descriptor_file.txt +} + +#TODO: Bats tests for custom_scripts not defined due to complexity From d3a9f15b6ce1cb8291c01e37ed8ff91c468f0daf Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 7 Sep 2022 15:07:44 -0500 Subject: [PATCH 073/196] Solved some bugs of pyunittest --- creation/lib/cWDictFile.py | 99 +++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 55 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index efd54f084d..a61e7c724a 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -24,6 +24,8 @@ from glideinwms.lib.defaults import BINARY_ENCODING from glideinwms.lib.util import chmod +from . import cWConsts + ######################################## # # File dictionary classes @@ -1272,8 +1274,6 @@ def parse_val(self, line): :param line: string with the line content :return: tuple with DATA_LENGTH-1 values """ - if line[0].startswith("# Version:"): - version = line[0].split(":")[1] if not line or line[0] == "#": return # ignore empty lines and comments arr = line.split(None, self.DATA_LENGTH - 1) # split already eliminates multiple spaces (no need for strip) @@ -1284,61 +1284,50 @@ def parse_val(self, line): # compatibility w/ old formats # 3.2.13 (no prefix): key, fname, type, period, cond_download, config_out - if version <= "030213": - if len(arr) != self.DATA_LENGTH: - # TODO: remove in 3.3 or after a few version (will break upgrade) - if len(arr) == self.DATA_LENGTH - 1: - # For upgrade from 3.2.13 to 3.2.11 - return self.add( - arr[0], - [ - arr[1], - arr[2], - "GLIDEIN_PS_", - arr[3], - arr[4], - arr[5], - arr[6], - arr[7], - arr[8], - arr[9], - arr[10], - arr[11], - ], - ) - raise RuntimeError( - "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) + if len(arr) != self.DATA_LENGTH: + # TODO: remove in 3.3 or after a few version (will break upgrade) + if len(arr) == self.DATA_LENGTH - 1: + # For upgrade from 3.2.13 to 3.2.11 + return self.add( + arr[0], + [ + arr[1], + arr[2], + "GLIDEIN_PS_", + arr[3], + arr[4], + arr[5], + arr[6], + arr[7], + arr[8], + arr[9], + arr[10], + arr[11], + ], ) - # 3.2.10 (no period, prefix): key, fname, type, cond_download, config_out - elif version <= "030210": - if len(arr) != self.DATA_LENGTH: - if len(arr) == self.DATA_LENGTH - 2: - # For upgrade from 3.2.10 or earlier - return self.add( - arr[0], - [ - arr[1], - arr[2], - "GLIDEIN_PS_", - arr[3], - 0, - arr[4], - arr[5], - arr[6], - arr[7], - arr[8], - arr[9], - arr[10], - ], - ) - raise RuntimeError( - "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) - ) - else: - if len(arr) != self.DATA_LENGTH: - raise RuntimeError( - "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) + # 3.2.10 (no period, prefix): key, fname, type, cond_download, config_out + elif len(arr) == self.DATA_LENGTH - 2: + # For upgrade from 3.2.10 or earlier + return self.add( + arr[0], + [ + arr[1], + arr[2], + "GLIDEIN_PS_", + arr[3], + 0, + arr[4], + arr[5], + arr[6], + arr[7], + arr[8], + arr[9], + arr[10], + ], ) + raise RuntimeError( + "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) + ) return self.add(arr[0], arr[1:]) def get_immutable_files(self): From 7df0080d07bfe53d0c019dda2e69008bec098b94 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 7 Sep 2022 16:11:18 -0500 Subject: [PATCH 074/196] Solved some bugs of pyunittest --- unittests/test_creation_lib_cWDictFile_DictFile.py | 3 ++- unittests/test_creation_lib_cgWDictFile.py | 1 + unittests/test_creation_lib_cgWParams.py | 1 + unittests/test_creation_lib_cvWParamDict.py | 1 + unittests/test_creation_lib_cvWParams.py | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/unittests/test_creation_lib_cWDictFile_DictFile.py b/unittests/test_creation_lib_cWDictFile_DictFile.py index 178ab262db..9d1b262dbc 100755 --- a/unittests/test_creation_lib_cWDictFile_DictFile.py +++ b/unittests/test_creation_lib_cWDictFile_DictFile.py @@ -25,6 +25,7 @@ try: from glideinwms.creation.lib.cWDictFile import DictFile + import * from glideinwms.creation.lib.cWConsts except ImportError as err: raise TestImportError(str(err)) @@ -79,7 +80,7 @@ def test_file_footer(self): def test_file_header(self): self.assertEqual(None, self.dict_file.file_header(want_comments=False)) - self.assertEqual("# File: %s\n#" % self.dict_file.fname, self.dict_file.file_header(want_comments=True)) + self.assertEqual("# File: %s\n# Version: %s\n#" % self.dict_file.fname, cWConsts.DICT_FILE_VERSION, self.dict_file.file_header(want_comments=True)) def test_format_val(self): key = "GLIDEIN_Expose_Grid_Env" diff --git a/unittests/test_creation_lib_cgWDictFile.py b/unittests/test_creation_lib_cgWDictFile.py index 404804b603..5a0004e02a 100755 --- a/unittests/test_creation_lib_cgWDictFile.py +++ b/unittests/test_creation_lib_cgWDictFile.py @@ -416,6 +416,7 @@ def test_get_main_dicts(self): assert False def test_load(self): + #TODO: Change it self.gmdicts.load() def test_reuse(self): diff --git a/unittests/test_creation_lib_cgWParams.py b/unittests/test_creation_lib_cgWParams.py index d914e97ad4..2d088e86e1 100755 --- a/unittests/test_creation_lib_cgWParams.py +++ b/unittests/test_creation_lib_cgWParams.py @@ -61,6 +61,7 @@ def test_get_xml_format(self): def test_get_xml(self): self.assertTrue(len(self.glidein_params.get_xml().__repr__()) > 0) + #TODO(F): what? def test_get_description(self): self.assertTrue(len(self.glidein_params.get_description().__repr__()) > 0) diff --git a/unittests/test_creation_lib_cvWParamDict.py b/unittests/test_creation_lib_cvWParamDict.py index 58fe9ae185..96f734019a 100755 --- a/unittests/test_creation_lib_cvWParamDict.py +++ b/unittests/test_creation_lib_cvWParamDict.py @@ -84,6 +84,7 @@ def test_find_parend_dir(self): pd = self.fed.find_parent_dir("fixtures/frontend", "index.html") self.assertEqual(pd, "fixtures/frontend/web-base/frontend") + #TODO(F) def test_save_monitor(self): self.fed.save_monitor() diff --git a/unittests/test_creation_lib_cvWParams.py b/unittests/test_creation_lib_cvWParams.py index 1e61682a21..566246b33f 100755 --- a/unittests/test_creation_lib_cvWParams.py +++ b/unittests/test_creation_lib_cvWParams.py @@ -99,6 +99,7 @@ def test_get_xml_format(self): def test_get_xml(self): self.assertTrue(len(self.v_o_frontend_params.get_xml().__repr__()) > 0) + #TODO(F) def test_get_description(self): self.assertTrue(len(self.v_o_frontend_params.get_description().__repr__()) > 0) From 3309f573cb3039e063f29b2a9fcb32651c7a8e5a Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 7 Sep 2022 16:40:38 -0500 Subject: [PATCH 075/196] Solved some bugs that blocked bats tests --- creation/web_base/utils_gs_http.sh | 6 ++++-- creation/web_base/utils_scripts.sh | 16 ++++++++-------- test/bats/creation_web_base_utils_crypto.bats | 2 +- test/bats/creation_web_base_utils_gs_http.bats | 11 ++++++----- test/bats/creation_web_base_utils_scripts.bats | 3 --- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 44a7730375..a95a1ac6da 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -192,7 +192,6 @@ fetch_file() { fi # TODO: consider to remove considering the current versions if [[ $version < "031100" ]]; then - if [ $# -lt 8 ]; then if [ $# -eq 7 ]; then #TODO: remove in version 3.3 # For compatibility with past versions (old file list formats) @@ -213,8 +212,11 @@ fetch_file() { fi local ifs_str printf -v ifs_str '%q' "${IFS}" - log_warn "Not enough arguments in fetch_file, 8 or 13 expected ($#/${ifs_str}): $*" + log_warn "Not enough arguments in fetch_file, at least 8 expected ($#/${ifs_str}): $*" glidein_exit 1 + else + if [ $# -lt 13 ]; then + log_warn "Not enough arguments in fetch_file, 13 expected ($#/${ifs_str}): $*" fi fi if ! fetch_file_try "$ff_id" "$ff_target_fname" "$ff_real_fname" "$ff_file_type" "$ff_cc_prefix" "$ff_time" "$ff_period" "$ff_coordination" "$ff_config_check" "$ff_tar_source" "$ff_config_out" "$ff_cond_attr" "$ff_absdir"; then diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index d904345268..5875fc0620 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -23,14 +23,14 @@ # version add_entry(){ local e_real_fname e_type e_time e_coordination e_period e_prefix e_id e_tar_source descriptor_file - e_real_fname="$2" - e_type="$3" - e_time="$4" - e_coordination="$5" - e_period="$6" - e_prefix="$7" - e_id="$8" - e_tar_source="$9" + e_real_fname="$1" + e_type="$2" + e_time="$3" + e_coordination="$4" + e_period="$5" + e_prefix="$6" + e_id="$7" + e_tar_source="$8" descriptor_file=$gwms_exec_dir/"descriptor_file.txt" if [ ! -f "$descriptor_file" ]; then diff --git a/test/bats/creation_web_base_utils_crypto.bats b/test/bats/creation_web_base_utils_crypto.bats index 9271bc1090..6b03c9bc65 100755 --- a/test/bats/creation_web_base_utils_crypto.bats +++ b/test/bats/creation_web_base_utils_crypto.bats @@ -36,7 +36,7 @@ setup () { touch "/tmp/trial/x509up_u" X509_USER_PROXY="/tmp/trial/x509up_u" run set_proxy_fullpath - assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${X509_USER_PROXY}" + assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${X509_USER_PROXY}" || assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path /private${X509_USER_PROXY}" [ "$status" -eq 0 ] } diff --git a/test/bats/creation_web_base_utils_gs_http.bats b/test/bats/creation_web_base_utils_gs_http.bats index 5f0a288e2b..00b73ab904 100755 --- a/test/bats/creation_web_base_utils_gs_http.bats +++ b/test/bats/creation_web_base_utils_gs_http.bats @@ -86,14 +86,15 @@ glidein_exit(){ @test "fetch_file" { echo "Testing different possible numbers of arguments..." >& 3 - run fetch_file 1 2 3 4 5 6 7 8 9 10 11 12 - assert_output --partial "More then 11 arguments, considering the first 11" + run fetch_file 1 2 3 4 5 6 7 8 9 10 11 12 13 "NULL" + assert_output --partial "More then 13 arguments, considering the first 13" + version="031100" run fetch_file 1 - assert_output --partial "Not enough arguments in fetch_file, 11 expected" + assert_output --partial "Not enough arguments in fetch_file, 13 expected" run fetch_file 1 2 - assert_output --partial "Not enough arguments in fetch_file, 11 expected" + assert_output --partial "Not enough arguments in fetch_file, 13 expected" run fetch_file 1 2 3 - assert_output --partial "Not enough arguments in fetch_file, 11 expected" + assert_output --partial "Not enough arguments in fetch_file, 13 expected" run fetch_file 1 2 3 4 } diff --git a/test/bats/creation_web_base_utils_scripts.bats b/test/bats/creation_web_base_utils_scripts.bats index 90f4b77b6d..657ef8cf07 100755 --- a/test/bats/creation_web_base_utils_scripts.bats +++ b/test/bats/creation_web_base_utils_scripts.bats @@ -40,12 +40,10 @@ setup () { echo "tmp" } run add_entry name type time coordination period prefix id tar_source - cat $gwms_exec_dir/descriptor_file.txt >& 3 grep -q "Time OrderedFilename RealFilename Type Period Prefix Id" $gwms_exec_dir/descriptor_file.txt grep -q "time coordination_name /tmp/name type period prefix id" $gwms_exec_dir/descriptor_file.txt rm -f $gwms_exec_dir/descriptor_file.txt run add_entry name type time,time2 coordination period prefix id tar_source - cat $gwms_exec_dir/descriptor_file.txt >& 3 grep -q "Time OrderedFilename RealFilename Type Period Prefix Id" $gwms_exec_dir/descriptor_file.txt grep -q "time coordination_name /tmp/name type period prefix id" $gwms_exec_dir/descriptor_file.txt grep -q "time2 coordination_name /tmp/name type period prefix id" $gwms_exec_dir/descriptor_file.txt @@ -61,7 +59,6 @@ setup () { run add_entry name type time2 coordination period prefix id tar_source run add_entry name type time3 coordination period prefix id tar_source run add_entry name type time coordination period prefix id tar_source - cat $gwms_exec_dir/descriptor_file.txt >& 3 run extract_entry_files "time" grep -q "time coordination_name /tmp/name type period prefix id" $gwms_exec_dir/time_descriptor_file.txt } From ab0bc251830df10a675850e94a9970c9b47c6cf0 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Wed, 7 Sep 2022 16:56:21 -0500 Subject: [PATCH 076/196] Solved pre-commit reported issues --- test/bats/creation_web_base_utils_crypto.bats | 2 +- unittests/test_creation_lib_cWDictFile_DictFile.py | 8 ++++++-- unittests/test_creation_lib_cgWDictFile.py | 2 +- unittests/test_creation_lib_cgWParams.py | 2 +- unittests/test_creation_lib_cvWParamDict.py | 2 +- unittests/test_creation_lib_cvWParams.py | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/test/bats/creation_web_base_utils_crypto.bats b/test/bats/creation_web_base_utils_crypto.bats index 6b03c9bc65..60462f5ba5 100755 --- a/test/bats/creation_web_base_utils_crypto.bats +++ b/test/bats/creation_web_base_utils_crypto.bats @@ -36,7 +36,7 @@ setup () { touch "/tmp/trial/x509up_u" X509_USER_PROXY="/tmp/trial/x509up_u" run set_proxy_fullpath - assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${X509_USER_PROXY}" || assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path /private${X509_USER_PROXY}" + assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path ${X509_USER_PROXY}" || assert_output --partial "Setting X509_USER_PROXY ${X509_USER_PROXY} to canonical path /private${X509_USER_PROXY}" [ "$status" -eq 0 ] } diff --git a/unittests/test_creation_lib_cWDictFile_DictFile.py b/unittests/test_creation_lib_cWDictFile_DictFile.py index 9d1b262dbc..30edf88f3b 100755 --- a/unittests/test_creation_lib_cWDictFile_DictFile.py +++ b/unittests/test_creation_lib_cWDictFile_DictFile.py @@ -24,8 +24,8 @@ from glideinwms.unittests.unittest_utils import create_temp_file, TestImportError try: + from glideinwms.creation.lib.cWConsts import * from glideinwms.creation.lib.cWDictFile import DictFile - import * from glideinwms.creation.lib.cWConsts except ImportError as err: raise TestImportError(str(err)) @@ -80,7 +80,11 @@ def test_file_footer(self): def test_file_header(self): self.assertEqual(None, self.dict_file.file_header(want_comments=False)) - self.assertEqual("# File: %s\n# Version: %s\n#" % self.dict_file.fname, cWConsts.DICT_FILE_VERSION, self.dict_file.file_header(want_comments=True)) + self.assertEqual( + "# File: %s\n# Version: %s\n#" % self.dict_file.fname, + cWConsts.DICT_FILE_VERSION, + self.dict_file.file_header(want_comments=True), + ) def test_format_val(self): key = "GLIDEIN_Expose_Grid_Env" diff --git a/unittests/test_creation_lib_cgWDictFile.py b/unittests/test_creation_lib_cgWDictFile.py index 5a0004e02a..3c4893640c 100755 --- a/unittests/test_creation_lib_cgWDictFile.py +++ b/unittests/test_creation_lib_cgWDictFile.py @@ -416,7 +416,7 @@ def test_get_main_dicts(self): assert False def test_load(self): - #TODO: Change it + # TODO: Change it self.gmdicts.load() def test_reuse(self): diff --git a/unittests/test_creation_lib_cgWParams.py b/unittests/test_creation_lib_cgWParams.py index 2d088e86e1..19e812b0c7 100755 --- a/unittests/test_creation_lib_cgWParams.py +++ b/unittests/test_creation_lib_cgWParams.py @@ -61,7 +61,7 @@ def test_get_xml_format(self): def test_get_xml(self): self.assertTrue(len(self.glidein_params.get_xml().__repr__()) > 0) - #TODO(F): what? + # TODO(F): what? def test_get_description(self): self.assertTrue(len(self.glidein_params.get_description().__repr__()) > 0) diff --git a/unittests/test_creation_lib_cvWParamDict.py b/unittests/test_creation_lib_cvWParamDict.py index 96f734019a..55308cbc84 100755 --- a/unittests/test_creation_lib_cvWParamDict.py +++ b/unittests/test_creation_lib_cvWParamDict.py @@ -84,7 +84,7 @@ def test_find_parend_dir(self): pd = self.fed.find_parent_dir("fixtures/frontend", "index.html") self.assertEqual(pd, "fixtures/frontend/web-base/frontend") - #TODO(F) + # TODO(F) def test_save_monitor(self): self.fed.save_monitor() diff --git a/unittests/test_creation_lib_cvWParams.py b/unittests/test_creation_lib_cvWParams.py index 566246b33f..549d387fe3 100755 --- a/unittests/test_creation_lib_cvWParams.py +++ b/unittests/test_creation_lib_cvWParams.py @@ -99,7 +99,7 @@ def test_get_xml_format(self): def test_get_xml(self): self.assertTrue(len(self.v_o_frontend_params.get_xml().__repr__()) > 0) - #TODO(F) + # TODO(F) def test_get_description(self): self.assertTrue(len(self.v_o_frontend_params.get_description().__repr__()) > 0) From 4fcceeeafcb1eaf739541856d7a6ddae14272fe7 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 8 Sep 2022 09:49:18 -0500 Subject: [PATCH 077/196] Solved pre-commit reported issues --- unittests/test_creation_lib_cWDictFile_DictFile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unittests/test_creation_lib_cWDictFile_DictFile.py b/unittests/test_creation_lib_cWDictFile_DictFile.py index 30edf88f3b..e46dbeb757 100755 --- a/unittests/test_creation_lib_cWDictFile_DictFile.py +++ b/unittests/test_creation_lib_cWDictFile_DictFile.py @@ -81,8 +81,7 @@ def test_file_footer(self): def test_file_header(self): self.assertEqual(None, self.dict_file.file_header(want_comments=False)) self.assertEqual( - "# File: %s\n# Version: %s\n#" % self.dict_file.fname, - cWConsts.DICT_FILE_VERSION, + f"# File: {self.dict_file.fname}\n# Version: {cWConsts.DICT_FILE_VERSION}\n#", self.dict_file.file_header(want_comments=True), ) From cdd329533d76dc4c820c2c8737d7b78cb656148e Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 8 Sep 2022 09:52:32 -0500 Subject: [PATCH 078/196] Solved pre-commit reported issues --- unittests/test_creation_lib_cWDictFile_DictFile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unittests/test_creation_lib_cWDictFile_DictFile.py b/unittests/test_creation_lib_cWDictFile_DictFile.py index e46dbeb757..877f2fb7bc 100755 --- a/unittests/test_creation_lib_cWDictFile_DictFile.py +++ b/unittests/test_creation_lib_cWDictFile_DictFile.py @@ -24,7 +24,7 @@ from glideinwms.unittests.unittest_utils import create_temp_file, TestImportError try: - from glideinwms.creation.lib.cWConsts import * + from glideinwms.creation.lib.cWConsts import DICT_FILE_VERSION from glideinwms.creation.lib.cWDictFile import DictFile except ImportError as err: raise TestImportError(str(err)) @@ -81,7 +81,7 @@ def test_file_footer(self): def test_file_header(self): self.assertEqual(None, self.dict_file.file_header(want_comments=False)) self.assertEqual( - f"# File: {self.dict_file.fname}\n# Version: {cWConsts.DICT_FILE_VERSION}\n#", + f"# File: {self.dict_file.fname}\n# Version: {DICT_FILE_VERSION}\n#", self.dict_file.file_header(want_comments=True), ) From c072989f0c9fdf22d45a06935fe7671c248d1e94 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 8 Sep 2022 10:04:05 -0500 Subject: [PATCH 079/196] Solved pyunittest error --- unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py b/unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py index 160535cfc6..9efc7f5da4 100755 --- a/unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py +++ b/unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py @@ -25,6 +25,7 @@ from glideinwms.unittests.unittest_utils import create_temp_file, TestImportError try: + from glideinwms.creation.lib.cWConsts import DICT_FILE_VERSION from glideinwms.creation.lib.cWDictFile import DictFileTwoKeys except ImportError as err: raise TestImportError(str(err)) @@ -77,7 +78,10 @@ def test_file_footer(self): def test_file_header(self): self.assertEqual(None, self.dict_file.file_header(want_comments=False)) - self.assertEqual("# File: %s\n#" % self.dict_file.fname, self.dict_file.file_header(want_comments=True)) + self.assertEqual( + f"# File: {self.dict_file.fname}\n# Version: {DICT_FILE_VERSION}\n#", + self.dict_file.file_header(want_comments=True), + ) def test_format_val(self): key = "GLIDECLIENT_Group_Start" From 712f701483fa8f07f11d30fa2ba8aeee315e0b17 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Thu, 8 Sep 2022 11:04:41 -0500 Subject: [PATCH 080/196] Added some missing modifications --- creation/conversion_xml.py | 3 -- creation/lib/cWDictFile.py | 32 ++-------------- creation/lib/cWParamDict.py | 55 ++++++++++++++++++++-------- creation/lib/cWParams.py | 20 +++++++++- creation/lib/cgWParamDict.py | 50 ++++++++++++++++++++----- creation/lib/xmlConfig.py | 10 ++--- creation/web_base/glidein_startup.sh | 7 +++- creation/web_base/utils_gs_http.sh | 19 +++++----- creation/web_base/utils_scripts.sh | 11 +++--- 9 files changed, 128 insertions(+), 79 deletions(-) diff --git a/creation/conversion_xml.py b/creation/conversion_xml.py index 0ac14a4950..44c760b197 100644 --- a/creation/conversion_xml.py +++ b/creation/conversion_xml.py @@ -14,7 +14,6 @@ def convert(xml_file): with open(xml_file, encoding="latin-1") as f: tree = ET.parse(f) for elem in tree.findall("file"): - print(elem) try: if "after_entry" in elem.attrib or "after_group" in elem.attrib: if elem.attrib.get("after_entry") == "True": @@ -64,9 +63,7 @@ def convert(xml_file): if elem.attrib.get("type") == "untar": for child in elem: - print(child) if child.tag == "untar_options": - print("yes") elem.attrib["cond_attr"] = child.attrib["cond_attr"] elem.attrib["absdir_outattr"] = child.attrib["absdir_outattr"] elem.attrib["type"] = "untar:" + child.attrib["dir"] diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index a61e7c724a..cc9ce89db4 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -1080,7 +1080,7 @@ def make_val_tuple( prefix="GLIDEIN_PS_", time="no_time", period=0, - priority=0, + priority=0, # executed first cond_download="TRUE", tar_source="NULL", config_out="FALSE", @@ -1290,40 +1290,14 @@ def parse_val(self, line): # For upgrade from 3.2.13 to 3.2.11 return self.add( arr[0], - [ - arr[1], - arr[2], - "GLIDEIN_PS_", - arr[3], - arr[4], - arr[5], - arr[6], - arr[7], - arr[8], - arr[9], - arr[10], - arr[11], - ], + [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9], arr[10]], ) # 3.2.10 (no period, prefix): key, fname, type, cond_download, config_out elif len(arr) == self.DATA_LENGTH - 2: # For upgrade from 3.2.10 or earlier return self.add( arr[0], - [ - arr[1], - arr[2], - "GLIDEIN_PS_", - arr[3], - 0, - arr[4], - arr[5], - arr[6], - arr[7], - arr[8], - arr[9], - arr[10], - ], + [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], arr[7], arr[8], arr[9]], ) raise RuntimeError( "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index 7805f444aa..786ea82409 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -123,18 +123,16 @@ def add_file_unparsed(user_file, dicts, is_factory): # Factory (file_list, after_file_list) file_list_idx = "file_list" if "priority" in user_file: - if user_file.priority >= 60: # eval(user_file.after_entry,{},{}): + if user_file.priority >= 60: file_list_idx = "after_file_list" else: # Frontend (preentry_file_list, file_list, aftergroup_preentry_file_list, aftergroup_file_list) file_list_idx = "preentry_file_list" - if "after_entry" in user_file: - if is_true(user_file.after_entry): - file_list_idx = "file_list" - - if "after_group" in user_file: - if is_true(user_file.after_group): + if "priority" in user_file: + if user_file.priority >= 80: file_list_idx = "aftergroup_%s" % file_list_idx + elif user_file.priority >= 60: + file_list_idx = "file_list" if is_executable: # a script if not is_const: @@ -146,13 +144,18 @@ def add_file_unparsed(user_file, dicts, is_factory): file_type = "exec" if user_file.type: - if user_file.type == "run:s" or user_file.type == "run:singularity": + if ( + user_file.type == "exec:s" + or user_file.type == "exec:singularity" + or user_file.type == "run:s" + or user_file.type == "run:singularity" + ): if file_list_idx.endswith("preentry_file_list"): raise RuntimeError("An executable cannot use singularity before the entry setup: %s" % user_file) file_type = "exec:s" else: - if not user_file.type.startswith("run"): - raise RuntimeError("An executable file type must start with 'run': $s" % user_file) + if not user_file.type.startswith("run") and not user_file.type.startswith("exec"): + raise RuntimeError("An executable file type must start with 'run' or 'exec': $s" % user_file) dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( @@ -166,6 +169,7 @@ def add_file_unparsed(user_file, dicts, is_factory): tar_source, config_out, cond_attr, + absdir_outattr, ), absfname, ) @@ -180,11 +184,16 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("A library file cannot be a wrapper: %s" % user_file) if is_periodic: raise RuntimeError("A wrapper file cannot be periodic: %s" % user_file) - dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), "wrapper", tar_source=user_file.tar_source + cWConsts.insert_timestr(relfname), + "wrapper", + tar_source=user_file.tar_source, + cond_download=cond_download, + config_out=config_out, + cond_attr=cond_attr, + absdir_outattr=absdir_outattr, ), absfname, ) @@ -213,7 +222,7 @@ def add_file_unparsed(user_file, dicts, is_factory): cond_download=cond_download, config_out=config_out, cond_attr=cond_attr, - absdir_outattr=wnsubdir, + absdir_outattr=absdir_outattr, ), absfname, ) @@ -233,7 +242,15 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), "source", tar_source=tar_source, time=time, priority=priority + cWConsts.insert_timestr(relfname), + "source", + tar_source=tar_source, + time=time, + priority=priority, + cond_download=cond_download, + config_out=config_out, + cond_attr=cond_attr, + absdir_outattr=absdir_outattr, ), absfname, ) @@ -251,7 +268,15 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), user_file.type, tar_source=tar_source, time=time, priority=priority + cWConsts.insert_timestr(relfname), + user_file.type, + tar_source=tar_source, + time=time, + priority=priority, + cond_download=cond_download, + config_out=config_out, + cond_attr=cond_attr, + absdir_outattr=absdir_outattr, ), absfname, ) diff --git a/creation/lib/cWParams.py b/creation/lib/cWParams.py index 20bf68db63..6df47f60fb 100644 --- a/creation/lib/cWParams.py +++ b/creation/lib/cWParams.py @@ -497,6 +497,18 @@ def init_support_defaults(self): "Is this a wrapper script that needs to be sourced in the glidein job wrapper?", None, ) + self.file_defaults["source"] = ( + "False", + "Bool", + "Is this a source script that needs to be sourced in the glidein?", + None, + ) + self.file_defaults["library"] = ( + "False", + "Bool", + "Is this a library script that needs to be sourced in the glidein?", + None, + ) self.file_defaults["untar"] = ("False", "Bool", "Do I need to untar it? ", None) self.file_defaults["prefix"] = ("GLIDEIN_PS_", "string", "Prefix used for periodic jobs (STARTD_CRON).", None) self.file_defaults["type"] = ( @@ -524,8 +536,14 @@ def init_support_defaults(self): "Specify the tar source where the file is contained.", None, ) - self.file_defaults["cond_attr"] = ( + self.file_defaults["cond_download"] = ( "TRUE", + "boolean", + "Download the file if set to true", + None, + ) + self.file_defaults["cond_attr"] = ( + "NULL", "attrname", "If not the special value TRUE, the attribute name used at runtime to determine if the file should be untarred or not.", None, diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 66aacc6b8c..932a7e669f 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -979,8 +979,8 @@ def add_file_unparsed(user_file, dicts, is_factory): absdir_outattr = user_file.absdir_outattr file_list_idx = "file_list" - if "after_entry" in user_file: - if eval(user_file["after_entry"]): + if "priority" in user_file: + if user_file.priority >= 60: file_list_idx = "after_file_list" if is_executable: # a script @@ -992,13 +992,18 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("A wrapper file cannot be an executable: %s" % user_file) file_type = "exec" if user_file.type: - if user_file.type == "run:s" or user_file.type == "run:singularity": + if ( + user_file.type == "exec:s" + or user_file.type == "exec:singularity" + or user_file.type == "run:s" + or user_file.type == "run:singularity" + ): if file_list_idx.endswith("preentry_file_list"): raise RuntimeError("An executable cannot use singularity before the entry setup: %s" % user_file) file_type = "exec:s" else: - if not user_file.type.startswith("run"): - raise RuntimeError("An executable file type must start with 'run': $s" % user_file) + if not user_file.type.startswith("run") and not user_file.type.startswith("exec"): + raise RuntimeError("An executable file type must start with 'run' or 'exec': $s" % user_file) dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( @@ -1012,6 +1017,7 @@ def add_file_unparsed(user_file, dicts, is_factory): tar_source, config_out, cond_attr, + absdir_outattr, ), absfname, ) @@ -1030,7 +1036,13 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), "wrapper", tar_source=user_file.tar_source + cWConsts.insert_timestr(relfname), + "wrapper", + tar_source=user_file.tar_source, + cond_download=cond_download, + config_out=config_out, + cond_attr=cond_attr, + absdir_outattr=absdir_outattr, ), absfname, ) @@ -1049,6 +1061,7 @@ def add_file_unparsed(user_file, dicts, is_factory): wnsubdir = user_file.type.split(":")[1] if wnsubdir is None: wnsubdir = relfname # default is relfname up to the first + # TODO(F): should it be absfname instead? dicts[file_list_idx].add_from_file( relfname, @@ -1058,7 +1071,7 @@ def add_file_unparsed(user_file, dicts, is_factory): cond_download=cond_download, config_out=config_out, cond_attr=cond_attr, - absdir_outattr=wnsubdir, + absdir_outattr=absdir_outattr, ), absfname, ) @@ -1074,10 +1087,19 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("A source file cannot be periodic: %s" % user_file) if is_library: raise RuntimeError("A source file cannot be a library: %s" % user_file) + dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), "source", tar_source=tar_source, time=time, priority=priority + cWConsts.insert_timestr(relfname), + "source", + tar_source=tar_source, + time=time, + priority=priority, + cond_download=cond_download, + config_out=config_out, + cond_attr=cond_attr, + absdir_outattr=absdir_outattr, ), absfname, ) @@ -1093,13 +1115,21 @@ def add_file_unparsed(user_file, dicts, is_factory): dicts[file_list_idx].add_from_file( relfname, cWDictFile.FileDictFile.make_val_tuple( - cWConsts.insert_timestr(relfname), user_file.type, tar_source=tar_source, time=time, priority=priority + cWConsts.insert_timestr(relfname), + user_file.type, + tar_source=tar_source, + time=time, + priority=priority, + cond_download=cond_download, + config_out=config_out, + cond_attr=cond_attr, + absdir_outattr=absdir_outattr, ), absfname, ) # user_file.type can be library:x - else: # not executable nor tarball => simple file + else: # simple file if is_const: val = "regular" dicts[file_list_idx].add_from_file( diff --git a/creation/lib/xmlConfig.py b/creation/lib/xmlConfig.py index 28be83f695..6c45f2bad0 100644 --- a/creation/lib/xmlConfig.py +++ b/creation/lib/xmlConfig.py @@ -288,11 +288,11 @@ def validate(self): self.check_boolean("source") self.check_boolean("library") - is_exec = eval(self["executable"]) - is_wrapper = eval(self["wrapper"]) - is_tar = eval(self["untar"]) - is_source = eval(self["source"]) - is_library = eval(self["library"]) + is_exec = eval(self["executable"]) or self["type"] == "exec" + is_wrapper = eval(self["wrapper"]) or self["type"] == "wrapper" + is_tar = eval(self["untar"]) or self["type"] == "untar" + is_source = eval(self["source"]) or self["type"] == "source" + is_library = eval(self["library"]) or self["type"] == "library" try: period = int(self["period"]) diff --git a/creation/web_base/glidein_startup.sh b/creation/web_base/glidein_startup.sh index f894630c8c..4e991d08c4 100755 --- a/creation/web_base/glidein_startup.sh +++ b/creation/web_base/glidein_startup.sh @@ -512,9 +512,13 @@ _main(){ # Fetch files contained in list # TODO: $file is actually a list, so it cannot be doublequoted (expanding here is needed). Can it be made more robust for linters? for now, just suppress the sc warning here # shellcheck disable=2086 + + # by default, old version + version="030900" + while read -r file do - # read version number + # read version number if present if [ "${file:0:10}" == "# Version:" ]; then OLD_IFS=$IFS IFS=':' read -ra ARR <<< "${file}" @@ -525,6 +529,7 @@ _main(){ if [ "${file:0:1}" != "#" ]; then fetch_file "${gs_id}" $file # wrappers, tarballs and regular files just need to be downloaded + # tar files are unpacked but not used after, wrapper files are added to wrapper_list if [ "${ff_file_type}" != "wrapper" ] && [ "${ff_file_type}" != "untar" ] && [ "${ff_file_type}" != "regular" ]; then add_entry "$ff_real_fname" "$ff_file_type" "$ff_time" "$ff_coordination" "$ff_period" "$ff_cc_prefix" "$gs_id" "$ff_tar_source" fi diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index a95a1ac6da..1998dd3103 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -190,7 +190,6 @@ fetch_file() { # For compatibility w/ future versions (add new parameters at the end) echo "More then 13 arguments, considering the first 13 ($#/${ifs_str}): $*" 1>&2 fi - # TODO: consider to remove considering the current versions if [[ $version < "031100" ]]; then if [ $# -eq 7 ]; then #TODO: remove in version 3.3 @@ -285,10 +284,8 @@ fetch_file_try() { # absdir_outattr - Name of a variable name. (like "KRB5_SUBSYS_DIR") # The variable will be set to the absolute path of the file if the file is not a tarball # or the directory where the tarball was unpacked in case it is a tarball, - # this last thing if and only if the unpacking actually happened (else it will not be defined.) - if [[ "${fft_absdir}" != "NULL" ]]; then - eval ${fft_absdir}="fft_real_fname" + eval ${fft_absdir}="$fft_real_fname" fi # if contained in a tarball, do not download @@ -296,8 +293,10 @@ fetch_file_try() { return 0 fi - # if fft_config_check is not TRUE or the variable pointed by fft_cond_attr (if any) is not TRUE, do not download (and operator between the two) - if [[ "${fft_config_check}" != "TRUE" || ("${fft_cond_attr}" != "NULL" && "${!fft_cond_attr}" != "TRUE") ]]; then + # if fft_config_check is not TRUE and the variable pointed by fft_cond_attr (if any) is not TRUE, do not download (AND operator between the two) + # this operator has been chosen instead of the OR one, since most of the times fft_cond_attr may not be set (default NULL) + # so the file would not be downloaded if the OR operator would have been chosen + if [[ "${fft_config_check}" != "TRUE" && ("${fft_cond_attr}" != "NULL" && "${!fft_cond_attr}" != "TRUE") ]]; then # TRUE is a special case, always be downloaded and processed fft_get_ss=$(grep -i "^${fft_config_check} " glidein_config | cut -d ' ' -f 2-) # Stop download and processing if the cond_attr variable is not defined or has a value different from 1 @@ -407,7 +406,6 @@ fetch_file_base() { #with env http_proxy=$proxy_url set if proxy_url != "None" # #construct curl equivalent so we can try either - wget_args=("${ffb_url}" "--user-agent" "wget/${user_agent}" "--quiet" "--output-document" "${ffb_tmp_outname}" ) curl_args=("${ffb_url}" "--user-agent" "curl/${user_agent}" "--silent" "--show-error" "--output" "${ffb_tmp_outname}") @@ -468,7 +466,8 @@ fetch_file_base() { fi fi - # if startup and executable, execute + # if time = startup and executable or source or library file, execute/source the file + # we perform this operation here since it may alter the need to download another file, so we execute before downloading other files if [[ "${ffb_time}" = "startup" ]]; then if [[ "${ffb_file_type}" = "exec" || "${ffb_file_type}" = "exec:"* ]]; then if ! chmod u+x "${ffb_outname}"; then @@ -513,6 +512,9 @@ fetch_file_base() { fi fi fi + elif [[ "${ffb_file_type}" = "source" || "${ffb_file_type}" = "library:shell" ]]; then + source "${ffb_target_fname}" + # TODO: what about other library types? fi fi # handle wrapper or untar files regardless of the moment @@ -707,7 +709,6 @@ perform_curl() { xmlResult+=$result echo -e "$xmlResult" > otrb_output.xml log_warn "Failed to load file '${ffb_real_fname}' from '${ffb_repository}'." - if [ -f otr_outlist.list ]; then chmod u+w otr_outlist.list else diff --git a/creation/web_base/utils_scripts.sh b/creation/web_base/utils_scripts.sh index 5875fc0620..85a1bd4e3d 100644 --- a/creation/web_base/utils_scripts.sh +++ b/creation/web_base/utils_scripts.sh @@ -31,7 +31,7 @@ add_entry(){ e_prefix="$6" e_id="$7" e_tar_source="$8" - descriptor_file=$gwms_exec_dir/"descriptor_file.txt" + descriptor_file=${gwms_exec_dir}/"descriptor_file.txt" if [ ! -f "$descriptor_file" ]; then echo "# File: $descriptor_file" > "$descriptor_file" @@ -39,7 +39,6 @@ add_entry(){ echo "# Time OrderedFilename RealFilename Type Period Prefix Id" >> "$descriptor_file" echo "################################################################################################" >> "$descriptor_file" fi - # From version 3.11.0 OLD_IFS=$IFS IFS=', ' read -r -a array <<< $e_time for time_entry in "${array[@]}" @@ -66,8 +65,8 @@ add_entry(){ extract_entry_files(){ local target_time descriptor_file target_time="$1" - descriptor_file=$gwms_exec_dir/"descriptor_file.txt" - grep ^$target_time $descriptor_file | sort > $gwms_exec_dir/${target_time}_descriptor_file.txt + descriptor_file=${gwms_exec_dir}/"descriptor_file.txt" + grep ^$target_time $descriptor_file | sort > "${gwms_exec_dir}/${target_time}_descriptor_file.txt" } ################################ @@ -140,8 +139,8 @@ custom_scripts(){ source "${ffb_target_fname}" # TODO: what about other library types? fi - done < $gwms_exec_dir/${target_time}_descriptor_file.txt - rm -f $gwms_exec_dir/${target_time}_descriptor_file.txt + done < "${gwms_exec_dir}/${target_time}_descriptor_file.txt" + rm -f "${gwms_exec_dir}/${target_time}_descriptor_file.txt" } ################################ From a83fd9f94c72e96d9b33a4c64ed13b9ed2ae1a98 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 9 Sep 2022 10:15:11 -0500 Subject: [PATCH 081/196] Added the control to invite to convert if boolean variables are present in xml files and performed some modifications to make the tests work --- creation/conversion_xml.py | 86 ------------ creation/lib/cWDictFile.py | 43 +++--- creation/lib/cWParamDict.py | 14 +- creation/lib/cWParams.py | 25 ---- creation/lib/cgWParamDict.py | 12 +- creation/lib/conversion_xml.py | 152 +++++++++++++++++++++ creation/lib/xmlConfig.py | 23 ++-- creation/reconfig_frontend | 13 +- creation/reconfig_glidein | 11 +- creation/web_base/utils_gs_http.sh | 2 + unittests/fixtures/frontend/frontend.xml | 6 +- unittests/fixtures/frontend/frontend39.xml | 99 ++++++++++++++ 12 files changed, 311 insertions(+), 175 deletions(-) delete mode 100644 creation/conversion_xml.py create mode 100644 creation/lib/conversion_xml.py create mode 100644 unittests/fixtures/frontend/frontend39.xml diff --git a/creation/conversion_xml.py b/creation/conversion_xml.py deleted file mode 100644 index 44c760b197..0000000000 --- a/creation/conversion_xml.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC -# SPDX-License-Identifier: Apache-2.0 - -# Description: -# These functions check the version of xml files -# and perform the conversion if needed - -import xml.etree.ElementTree as ET - - -def convert(xml_file): - with open(xml_file, encoding="latin-1") as f: - tree = ET.parse(f) - for elem in tree.findall("file"): - try: - if "after_entry" in elem.attrib or "after_group" in elem.attrib: - if elem.attrib.get("after_entry") == "True": - if not "after_group" in elem.attrib: - elem.attrib["priority"] = "90" - elif elem.attrib.get("after_group") == "True": - elem.attrib["priority"] = "80" - else: - elem.attrib["priority"] = "60" - elif elem.attrib.get("after_entry") == "False": - if not "after_group" in elem.attrib: - elem.attrib["priority"] = "10" - elif elem.attrib.get("after_group") == "True": - elem.attrib["priority"] = "40" - else: - elem.attrib["priority"] = "20" - if "after_entry" in elem.attrib: - elem.attrib.pop("after_entry") - if "after_group" in elem.attrib: - elem.attrib.pop("after_group") - else: - continue - - if elem.attrib.get("executable") == "True": - elem.attrib.pop("executable") - elem.attrib["type"] = "exec" - - if "period" in elem.attrib: - elem.attrib["type"] = "periodic:" + elem.attrib.get("period") - elem.attrib.pop("period") - - if elem.attrib.get("wrapper") == "True": - elem.attrib.pop("wrapper") - elem.attrib["type"] = "wrapper" - - if elem.attrib.get("source") == "True": - elem.attrib.pop("source") - elem.attrib["type"] = "source" - - if elem.attrib.get("library") == "True": - elem.attrib.pop("library") - elem.attrib["type"] = "library" - - if elem.attrib.get("untar") == "True": - elem.attrib["type"] = "untar" - elem.attrib.pop("untar") - - if elem.attrib.get("type") == "untar": - for child in elem: - if child.tag == "untar_options": - elem.attrib["cond_attr"] = child.attrib["cond_attr"] - elem.attrib["absdir_outattr"] = child.attrib["absdir_outattr"] - elem.attrib["type"] = "untar:" + child.attrib["dir"] - elem.remove(child) - - except AttributeError: - pass - tree.write("2" + xml_file, encoding="latin-1") - - -def is_old_version(xml_file): - with open(xml_file, encoding="latin-1") as f: - tree = ET.parse(f) - for elem in tree.findall("file"): - if "after_entry" in elem.attrib or "after_group" in elem.attrib or "period" in elem.attrib: - return True - for child in elem: - if child.tag == "untar_options": - return True - return False diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index cc9ce89db4..14f8192bd9 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -994,7 +994,7 @@ class FileDictFile(SimpleFileDictFile): It is using a dictionary (key, value) from DictFile, serialized to file. The key is the file ID - The value (line) on file has DATA_LENGTH (11) components: the key and the first DATA_LENGTH-1 attributes below. + The value (line) on file has DATA_LENGTH (12) components: the key and the first DATA_LENGTH-1 attributes below. The value in memory has DATA_LENGTH components (real_fname, cache/exec, prefix, time, priority, cond_download, tar_source, config_out, cond_attr, absdir_outattr, data), the key is used as key for the dictionary and the data (file content) is added reading the file. Here the attributes stored as tuple in the dictionary value: @@ -1051,6 +1051,7 @@ class FileDictFile(SimpleFileDictFile): """ DATA_LENGTH = 12 # Length of value (attributes + data) + OLD_DATA_LENGTH = 7 PLACEHOLDER_VALUE = ( "", "", @@ -1247,14 +1248,15 @@ def file_header(self, want_comments): DictFile.file_header(self, want_comments) + "\n" + ( - "# %s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \n" + "# %s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \t%s \n" % ( "Outfile", "InFile ", "Cache/exec", "Prefix", "Time", - "Period" "Priority", + "Period", + "Priority", "Condition", "TarSource", "ConfigOut", @@ -1282,27 +1284,22 @@ def parse_val(self, line): if len(arr[0]) == 0: return # empty key - # compatibility w/ old formats - # 3.2.13 (no prefix): key, fname, type, period, cond_download, config_out - if len(arr) != self.DATA_LENGTH: - # TODO: remove in 3.3 or after a few version (will break upgrade) - if len(arr) == self.DATA_LENGTH - 1: - # For upgrade from 3.2.13 to 3.2.11 - return self.add( - arr[0], - [arr[1], arr[2], "GLIDEIN_PS_", arr[3], arr[4], arr[5], arr[6], arr[7], arr[8], arr[9], arr[10]], - ) - # 3.2.10 (no period, prefix): key, fname, type, cond_download, config_out - elif len(arr) == self.DATA_LENGTH - 2: - # For upgrade from 3.2.10 or earlier - return self.add( - arr[0], - [arr[1], arr[2], "GLIDEIN_PS_", arr[3], 0, arr[4], arr[5], arr[6], arr[7], arr[8], arr[9]], - ) - raise RuntimeError( - "Not a valid file line (expected %i, found %i elements): '%s'" % (self.DATA_LENGTH, len(arr), line) + if len(arr) == self.DATA_LENGTH: + return self.add(arr[0], arr[1:]) + + # for compatibility with old format + if len(arr) == self.OLD_DATA_LENGTH: + # old format: file, type, period, prefix, condition, configout + return self.add( + arr[0], + [arr[1], arr[2], arr[4], "startup", arr[3], 0, arr[5], "NULL", arr[6], "NULL", "NULL", "NULL"], ) - return self.add(arr[0], arr[1:]) + + # length error + raise RuntimeError( + "Not a valid file line (expected %i or %i, found %i elements): '%s'" + % (self.DATA_LENGTH, self.OLD_DATA_LENGTH, len(arr), line) + ) def get_immutable_files(self): mkeys = [] diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index 786ea82409..f54bfee736 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -64,12 +64,12 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("Found a file element with an empty relfname: %s" % user_file) is_const = is_true(user_file.const) - is_executable = is_true(user_file.executable) or (user_file.type.startswith("exec")) - is_wrapper = is_true(user_file.wrapper) or (user_file.type == "wrapper") - is_source = is_true(user_file.source) or user_file.type.startswith("source") - is_library = is_true(user_file.library) or user_file.type.startswith("library") - is_periodic = user_file.type.startswith("periodic") - do_untar = is_true(user_file.untar) or (user_file.type.startswith("untar")) + is_executable = user_file.type.startswith("exec") or user_file.type.startswith("run") + is_wrapper = user_file.type == "wrapper" + is_source = user_file.type.startswith("source") + is_library = user_file.type.startswith("library") + is_periodic = user_file.time.startswith("periodic") + do_untar = user_file.type.startswith("untar") time = user_file.time if is_executable or is_source or is_library: @@ -88,7 +88,7 @@ def add_file_unparsed(user_file, dicts, is_factory): try: if user_file.is_periodic: - period_value = int(user_file.type.split(":")[1]) + period_value = int(user_file.time.split(":")[1]) else: period_value = 0 except (AttributeError, KeyError, ValueError, TypeError): diff --git a/creation/lib/cWParams.py b/creation/lib/cWParams.py index 6df47f60fb..4218572913 100644 --- a/creation/lib/cWParams.py +++ b/creation/lib/cWParams.py @@ -485,31 +485,6 @@ def init_support_defaults(self): "Will the file be constant? If True, the file will be signed. If False, it can be modified at any time and will not be cached.", None, ) - self.file_defaults["executable"] = ( - "False", - "Bool", - "Is this an executable that needs to be run in the glidein?", - None, - ) - self.file_defaults["wrapper"] = ( - "False", - "Bool", - "Is this a wrapper script that needs to be sourced in the glidein job wrapper?", - None, - ) - self.file_defaults["source"] = ( - "False", - "Bool", - "Is this a source script that needs to be sourced in the glidein?", - None, - ) - self.file_defaults["library"] = ( - "False", - "Bool", - "Is this a library script that needs to be sourced in the glidein?", - None, - ) - self.file_defaults["untar"] = ("False", "Bool", "Do I need to untar it? ", None) self.file_defaults["prefix"] = ("GLIDEIN_PS_", "string", "Prefix used for periodic jobs (STARTD_CRON).", None) self.file_defaults["type"] = ( None, diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 932a7e669f..5d07e7423a 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -922,13 +922,13 @@ def add_file_unparsed(user_file, dicts, is_factory): relfname = user_file["relfname"] is_const = eval(user_file["const"]) - is_executable = eval(user_file["executable"]) or (user_file.type.startswith("exec")) - is_wrapper = eval(user_file["wrapper"]) or (user_file.type == "wrapper") - do_untar = eval(user_file["untar"]) or (user_file.type.startswith("untar")) + is_executable = user_file.type.startswith("exec") or user_file.type.startswith("run") + is_wrapper = user_file.type == "wrapper" + do_untar = user_file.type.startswith("untar") is_const = is_true(user_file.const) - is_source = eval(user_file["source"]) or user_file.type.startswith("source") - is_library = eval(user_file["library"]) or user_file.type.startswith("library") - is_periodic = user_file.type.startswith("periodic") + is_source = user_file.type.startswith("source") + is_library = user_file.type.startswith("library") + is_periodic = user_file.time.startswith("periodic") time = user_file.time if is_executable or is_source or is_library: diff --git a/creation/lib/conversion_xml.py b/creation/lib/conversion_xml.py new file mode 100644 index 0000000000..d64606f863 --- /dev/null +++ b/creation/lib/conversion_xml.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC +# SPDX-License-Identifier: Apache-2.0 + +# Description: +# These functions check the version of xml files +# and perform the conversion if needed + +import sys +import xml.etree.ElementTree as ET + + +def convert(type, input, output): + with open(input, encoding="latin-1") as f: + tree = ET.parse(f) + parent_map = {c: p for p in tree.iter() for c in p} + for elem in tree.findall(".//file"): + try: + print(elem) + parent = parent_map[elem] + if parent.tag != "xml": + grand_parent = parent_map[parent] + else: + grand_parent = parent + print(grand_parent) + if grand_parent.tag == "entry": + print("entry") + elem.attrib["priority"] = "50" + elif grand_parent.tag == "group": + if "after_entry" in elem.attrib: + if elem.attrib.get("after_entry") == "True": + print("group ae") + elem.attrib["priority"] = "70" + else: + print("group be") + elem.attrib["priority"] = "30" + elif "after_entry" in elem.attrib or "after_group" in elem.attrib: + if elem.attrib.get("after_entry") == "True": + if type == "factory": + elem.attrib["priority"] = "90" + elif elem.attrib.get("after_group") == "True": + elem.attrib["priority"] = "80" + else: + elem.attrib["priority"] = "60" + else: + if type == "factory": + elem.attrib["priority"] = "10" + elif elem.attrib.get("after_group") == "True": + elem.attrib["priority"] = "40" + else: + elem.attrib["priority"] = "20" + + if "after_entry" in elem.attrib: + elem.attrib.pop("after_entry") + if "after_group" in elem.attrib: + elem.attrib.pop("after_group") + else: + continue + + if elem.attrib.get("executable") == "True": + elem.attrib.pop("executable") + elem.attrib["type"] = "exec" + + if "period" in elem.attrib: + elem.attrib["type"] = "executable" + elem.attrib["time"] = "periodic:" + elem.attrib.get("period") + elem.attrib.pop("period") + + if elem.attrib.get("wrapper") == "True": + elem.attrib.pop("wrapper") + elem.attrib["type"] = "wrapper" + + if elem.attrib.get("source") == "True": + elem.attrib.pop("source") + elem.attrib["type"] = "source" + + if elem.attrib.get("library") == "True": + elem.attrib.pop("library") + elem.attrib["type"] = "library" + + if elem.attrib.get("untar") == "True": + elem.attrib["type"] = "untar" + elem.attrib.pop("untar") + + if elem.attrib.get("type") == "untar": + for child in elem: + if child.tag == "untar_options": + elem.attrib["cond_attr"] = child.attrib["cond_attr"] + elem.attrib["absdir_outattr"] = child.attrib["absdir_outattr"] + elem.attrib["type"] = "untar:" + child.attrib["dir"] + elem.remove(child) + + except AttributeError: + pass + if output == "stdout": + tree.write(sys.stdout.buffer) + else: + tree.write(output, encoding="latin-1") + + +def is_old_version(xml_file): + with open(xml_file, encoding="latin-1") as f: + tree = ET.parse(f) + for elem in tree.findall("file"): + if ( + "after_entry" in elem.attrib + or "after_group" in elem.attrib + or "period" in elem.attrib + or "untar" in elem.attrib + or "wrapper" in elem.attrib + or "executable" in elem.attrib + ): + return True + for child in elem: + if child.tag == "untar_options": + return True + return False + + +def main(type, input, output="stdout"): + if is_old_version(input): + convert(type, input, output) + + +def usage(): + print("Usage: python conversion_xml.py ") + print(" : frontend/factory") + print(" : input filename") + print(" : (optional) output filename, stdout if not specified") + print(" <-i> : (optional) overwrite input file if present") + + +if __name__ == "__main__": + if len(sys.argv) - 1 == 3: + type = sys.argv[1] + input = sys.argv[2] + output = sys.argv[3] + print(type, input, output) + main(type, input, output) + elif len(sys.argv) - 1 == 2: + type = sys.argv[1] + input = sys.argv[2] + main(type, input) + elif (len(sys.argv) - 1 == 4 or len(sys.argv) - 1 == 3) and sys.argv[len(sys.argv) - 1] == "-i": + type = sys.argv[1] + input = sys.argv[2] + main(type, input, input) + else: + usage() + exit(1) + sys.exit(0) diff --git a/creation/lib/xmlConfig.py b/creation/lib/xmlConfig.py index 6c45f2bad0..56e5a92479 100644 --- a/creation/lib/xmlConfig.py +++ b/creation/lib/xmlConfig.py @@ -282,20 +282,17 @@ def validate(self): raise RuntimeError(self.err_str("relfname cannot be empty")) self.check_boolean("const") - self.check_boolean("executable") - self.check_boolean("wrapper") - self.check_boolean("untar") - self.check_boolean("source") - self.check_boolean("library") - - is_exec = eval(self["executable"]) or self["type"] == "exec" - is_wrapper = eval(self["wrapper"]) or self["type"] == "wrapper" - is_tar = eval(self["untar"]) or self["type"] == "untar" - is_source = eval(self["source"]) or self["type"] == "source" - is_library = eval(self["library"]) or self["type"] == "library" + + is_exec = self["type"].startswith("exec") or self["type"].startswith("run") + is_wrapper = self["type"] == "wrapper" + is_tar = self["type"] == "untar" + is_source = self["type"].startswith("source") + is_library = self["type"] == "library" + is_periodic = self["time"].startswith("periodic") try: - period = int(self["period"]) + if is_periodic: + period = int(self["time"].split(":")[1]) except ValueError: raise RuntimeError(self.err_str("period must be an int")) @@ -324,7 +321,7 @@ def validate(self): if (is_exec or is_wrapper or is_tar) and not eval(self["const"]): raise RuntimeError(self.err_str('type "executable", "wrapper", or "untar" requires const="True"')) - if not is_exec and period > 0: + if not is_exec and is_periodic: raise RuntimeError(self.err_str('cannot have execution period if type is not "executable"')) diff --git a/creation/reconfig_frontend b/creation/reconfig_frontend index 01ae7a0fb5..931d3b7e4e 100755 --- a/creation/reconfig_frontend +++ b/creation/reconfig_frontend @@ -17,11 +17,10 @@ import signal import sys import tempfile -import conversion_xml - from glideinwms.creation.lib import ( check_config_frontend, check_python3_expr, + conversion_xml, cvWConsts, cvWCreate, cvWDictFile, @@ -145,11 +144,6 @@ if __name__ == "__main__": "NOTE: Executing reconfig_frontend as user 'root' is not allowed. Use the frontend user instead. For rpm based installations, use the 'service gwms-frontend ' command to perform gwms-frontend operations" ) - # TODO(F): read xml file and check if compatible - # if is_old_version("frontend.xml"): - # print2("The xml file seems to be outdated. Consider to convert the structure to version 3.11.0") - # sys.exit(1) - force_name = None writeback = "no" update_scripts = "no" @@ -228,6 +222,11 @@ if __name__ == "__main__": print2(e) sys.exit(1) + # check the version of the xml file + if conversion_xml.is_old_version(xml): + print2("The xml file seems to be outdated. Consider to convert the structure to version 3.11.0") + sys.exit(1) + if force_name is not None: if params.frontend_name != force_name: print2(usage) diff --git a/creation/reconfig_glidein b/creation/reconfig_glidein index c7c9c01e09..7d3031170a 100755 --- a/creation/reconfig_glidein +++ b/creation/reconfig_glidein @@ -20,7 +20,7 @@ import tempfile # from glideinwms.creation.lib import cWConsts # from glideinwms.creation.lib import cgWParams # from glideinwms.creation.lib import cgWDictFile -from glideinwms.creation.lib import cgWConsts, cgWCreate, cgWParamDict, factoryXmlConfig, xslt +from glideinwms.creation.lib import cgWConsts, cgWCreate, cgWParamDict, conversion_xml, factoryXmlConfig, xslt from glideinwms.factory import glideFactoryConfig, glideFactoryLib, glideFactoryMonitorAggregator from glideinwms.lib import logSupport from glideinwms.lib.util import handle_hooks, HOOK_POST_RECONFIG_DIRNAME, HOOK_PRE_RECONFIG_DIRNAME @@ -225,10 +225,6 @@ if __name__ == "__main__": "NOTE: Executing reconfig_glidein as user 'root' is not allowed. Use the factory user instead. For rpm based installations, use the 'service gwms-factory ' command to perform gwms-factory operations" ) - # if is_old_version("glideinWMS.xml"): - # print2("The xml file seems to be outdated. Consider to convert the structure to version 3.11.0") - # sys.exit(1) - force_name = None writeback = "no" update_scripts = "no" @@ -313,6 +309,11 @@ if __name__ == "__main__": print2(e) exit_code = 1 + # check the version of the xml file + if conversion_xml.is_old_version(xml): + print2("The xml file seems to be outdated. Consider to convert the structure to version 3.11.0") + sys.exit(1) + handle_hooks(os.path.dirname(os.path.abspath(xml)), HOOK_POST_RECONFIG_DIRNAME) if sighupreload: diff --git a/creation/web_base/utils_gs_http.sh b/creation/web_base/utils_gs_http.sh index 1998dd3103..a7fb574825 100644 --- a/creation/web_base/utils_gs_http.sh +++ b/creation/web_base/utils_gs_http.sh @@ -478,6 +478,8 @@ fetch_file_base() { echo "Skipping last script ${last_script}" 1>&2 elif [[ "${ffb_target_fname}" = "cvmfs_umount.sh" ]] || [[ -n "${cleanup_script}" && "${ffb_target_fname}" = "${cleanup_script}" ]]; then # cleanup_script global for simplicity # TODO: temporary OR checking for cvmfs_umount.sh; to be removed after Bruno's ticket on cleanup [#25073] + # set time to cleanup for the cleanup script + ffb_time="cleanup" echo "Skipping cleanup script ${ffb_outname} (${cleanup_script})" 1>&2 cp "${ffb_outname}" "$gwms_exec_dir/cleanup/${ffb_target_fname}" chmod a+x "${gwms_exec_dir}/cleanup/${ffb_target_fname}" diff --git a/unittests/fixtures/frontend/frontend.xml b/unittests/fixtures/frontend/frontend.xml index 97444abf2f..4a518ad465 100644 --- a/unittests/fixtures/frontend/frontend.xml +++ b/unittests/fixtures/frontend/frontend.xml @@ -92,8 +92,8 @@ - - - + + + diff --git a/unittests/fixtures/frontend/frontend39.xml b/unittests/fixtures/frontend/frontend39.xml new file mode 100644 index 0000000000..97444abf2f --- /dev/null +++ b/unittests/fixtures/frontend/frontend39.xml @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 6d5d91f3470b5d8cfcb08f75b4458eacc5d99d0f Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 9 Sep 2022 10:24:19 -0500 Subject: [PATCH 082/196] Performed some modifications to make the tests work --- creation/lib/cWDictFile.py | 2 ++ creation/lib/cWParamDict.py | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index 14f8192bd9..8ffc6218f0 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -1220,6 +1220,8 @@ def add( % val ) return self.add_from_file(key, val, os.path.join(self.dir, self.val_to_file_name(val)), allow_overwrite) + elif len(val) == self.OLD_DATA_LENGTH; + return DictFile.add(self, key, tuple(val), allow_overwrite) else: raise DictFileError( "Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index f54bfee736..109bff1c33 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -94,10 +94,7 @@ def add_file_unparsed(user_file, dicts, is_factory): except (AttributeError, KeyError, ValueError, TypeError): period_value = 1000 # default 1000ms - if ( - period_value > 0 - ): # period has 0 as default (in dictionary definition). Should I still protect against it not being defined? - if not is_executable: + if is_periodic and not is_executable: raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) priority = user_file.priority From c634017b6f853308239884f41d43fb4d57042fa2 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 9 Sep 2022 10:26:37 -0500 Subject: [PATCH 083/196] Performed some modifications to make the tests work --- creation/lib/cWDictFile.py | 2 +- creation/lib/cWParamDict.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index 8ffc6218f0..f93e487f2a 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -1220,7 +1220,7 @@ def add( % val ) return self.add_from_file(key, val, os.path.join(self.dir, self.val_to_file_name(val)), allow_overwrite) - elif len(val) == self.OLD_DATA_LENGTH; + elif len(val) == self.OLD_DATA_LENGTH: return DictFile.add(self, key, tuple(val), allow_overwrite) else: raise DictFileError( diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index 109bff1c33..84eafc96c8 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -95,7 +95,7 @@ def add_file_unparsed(user_file, dicts, is_factory): period_value = 1000 # default 1000ms if is_periodic and not is_executable: - raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) + raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) priority = user_file.priority if priority < 0 or priority > 99: From 7a57835754cc79e3b16d0505b42c83060df9c077 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 9 Sep 2022 10:49:48 -0500 Subject: [PATCH 084/196] Performed some modifications to make the tests work --- creation/lib/cWDictFile.py | 10 ---------- creation/lib/cWParamDict.py | 26 +++++++++++++++----------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index f93e487f2a..4f9f3809ad 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -1198,16 +1198,6 @@ def add( # since the other functions from base class know nothing about placeholders, need to force overwrite allow_overwrite = True - # This will help identify calls not migrated to the new format - # TODO: check parameters!! - try: - int(val[2]) # to check if is integer. Period must be int or convertible to int - except (ValueError, IndexError): - raise DictFileError( - "Values '%s' not (real_fname,cache/exec,prefix,time,priority,cond_download,tar_source,config_out,cond_attr,absdir_outattr)" - % val - ) - if len(val) == self.DATA_LENGTH: # Alt: return self.add_from_str(key, val[:self.DATA_LENGTH-1], val[self.DATA_LENGTH-1], allow_overwrite) return DictFile.add(self, key, tuple(val), allow_overwrite) diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index 84eafc96c8..092e54d15a 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -40,7 +40,7 @@ def has_file_wrapper_params(file_params): # file_params is the list in a files section (global o group): each one is a file specification # If there is one wrapper return true for user_file in file_params: - if is_true(user_file.wrapper): + if user_file.type == "wrapper": return True return False @@ -81,23 +81,23 @@ def add_file_unparsed(user_file, dicts, is_factory): and not time.startswith("periodic:") and not time.startswith("milestone:") and not time.startswith("failure:") - and not time.startswith("no_time:") + and not time.startswith("no_time") ): # we use startswith since we may have combination of time phases (e.g. startup, cleanup) raise RuntimeError("The file does not have a valid time phase value: %s" % user_file) try: if user_file.is_periodic: - period_value = int(user_file.time.split(":")[1]) + period = int(user_file.time.split(":")[1]) else: - period_value = 0 + period = 0 except (AttributeError, KeyError, ValueError, TypeError): - period_value = 1000 # default 1000ms + period = 1000 # default 1000ms if is_periodic and not is_executable: raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) - priority = user_file.priority + priority = int(user_file.priority) if priority < 0 or priority > 99: raise RuntimeError("Priority value out of the range [0,99]: %s" % user_file) @@ -109,9 +109,13 @@ def add_file_unparsed(user_file, dicts, is_factory): if tar_source is None: tar_source = "NULL" - config_out = user_file.config_out - if config_out is None: + try: + config_out = user_file.config_out + if config_out is None: + config_out = "FALSE" + except (AttributeError, KeyError, ValueError, TypeError): config_out = "FALSE" + cond_attr = user_file.cond_attr absdir_outattr = user_file.absdir_outattr @@ -120,15 +124,15 @@ def add_file_unparsed(user_file, dicts, is_factory): # Factory (file_list, after_file_list) file_list_idx = "file_list" if "priority" in user_file: - if user_file.priority >= 60: + if priority >= 60: file_list_idx = "after_file_list" else: # Frontend (preentry_file_list, file_list, aftergroup_preentry_file_list, aftergroup_file_list) file_list_idx = "preentry_file_list" if "priority" in user_file: - if user_file.priority >= 80: + if priority >= 80: file_list_idx = "aftergroup_%s" % file_list_idx - elif user_file.priority >= 60: + elif priority >= 60: file_list_idx = "file_list" if is_executable: # a script From 90afd5858efb4b9b3eb9d305b53ca3065157c3f8 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 9 Sep 2022 12:09:37 -0500 Subject: [PATCH 085/196] Performed some modifications to make the tests work --- creation/lib/cgWParamDict.py | 10 +++++----- creation/lib/xmlConfig.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 5d07e7423a..96638c06bf 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -26,7 +26,7 @@ # why this is commented out for now # # from cWParamDict import is_true, add_file_unparsed - +from cWParamDict import is_true # from factoryXmlConfig import EntrySetElement @@ -947,14 +947,14 @@ def add_file_unparsed(user_file, dicts, is_factory): try: if user_file.is_periodic: - period_value = int(user_file.type.split(":")[1]) + period = int(user_file.type.split(":")[1]) else: - period_value = 0 + period = 0 except (AttributeError, KeyError, ValueError, TypeError): - period_value = 1000 # default 1000ms + period = 1000 # default 1000ms if ( - period_value > 0 + is_periodic ): # period has 0 as default (in dictionary definition). Should I still protect against it not being defined? if not is_executable: raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) diff --git a/creation/lib/xmlConfig.py b/creation/lib/xmlConfig.py index 56e5a92479..8b28f7644e 100644 --- a/creation/lib/xmlConfig.py +++ b/creation/lib/xmlConfig.py @@ -283,7 +283,7 @@ def validate(self): self.check_boolean("const") - is_exec = self["type"].startswith("exec") or self["type"].startswith("run") + is_executable = self["type"].startswith("exec") or self["type"].startswith("run") is_wrapper = self["type"] == "wrapper" is_tar = self["type"] == "untar" is_source = self["type"].startswith("source") @@ -309,7 +309,7 @@ def validate(self): and not time.startswith("no_time:") ): # we use startswith since we may have combination of time phases (e.g. startup, cleanup) - raise RuntimeError(self.err_str("The file does not have a valid time phase value: %s" % user_file)) + raise RuntimeError(self.err_str("The file does not have a valid time phase value.")) self.check_boolean("cond_download") self.check_boolean("config_out") From 3f28cdade4b4f79fe032e72b05ed7dabd84bc92b Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 9 Sep 2022 12:12:47 -0500 Subject: [PATCH 086/196] Performed some modifications to make the tests work --- creation/lib/cgWParamDict.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 96638c06bf..8f93e8bdce 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -26,9 +26,11 @@ # why this is commented out for now # # from cWParamDict import is_true, add_file_unparsed -from cWParamDict import is_true # from factoryXmlConfig import EntrySetElement +def is_true(s): + """Case insensitive string parsing helper. Return True for true (case insensitive matching), False otherwise.""" + return type(s) == str and s.lower() == "true" class UnconfiguredScheddError(Exception): def __init__(self, schedd): From 1cd915e82feeb77e670dc20673dae95b12d68b08 Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 9 Sep 2022 13:46:52 -0500 Subject: [PATCH 087/196] Performed some modifications to make the tests work --- creation/lib/xmlConfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/creation/lib/xmlConfig.py b/creation/lib/xmlConfig.py index 8b28f7644e..c162959d45 100644 --- a/creation/lib/xmlConfig.py +++ b/creation/lib/xmlConfig.py @@ -314,14 +314,14 @@ def validate(self): self.check_boolean("cond_download") self.check_boolean("config_out") - if is_exec + is_wrapper + is_tar + is_source + is_library > 1: + if is_executable + is_wrapper + is_tar + is_source + is_library > 1: raise RuntimeError( self.err_str('must be exactly one of type "executable", "wrapper", "untar", "library" or "source"') ) - if (is_exec or is_wrapper or is_tar) and not eval(self["const"]): + if (is_executable or is_wrapper or is_tar) and not eval(self["const"]): raise RuntimeError(self.err_str('type "executable", "wrapper", or "untar" requires const="True"')) - if not is_exec and is_periodic: + if not is_executable and is_periodic: raise RuntimeError(self.err_str('cannot have execution period if type is not "executable"')) From 72d0aafaa6d21c235e4ae546291921d22db7b319 Mon Sep 17 00:00:00 2001 From: Marco Mambelli Date: Fri, 9 Sep 2022 12:09:37 -0500 Subject: [PATCH 088/196] Fixed python3 check return value in case of exception and minor pep8 improvements --- creation/lib/check_python3_expr.py | 37 +++++++++++++++--------------- creation/reconfig_frontend | 13 +++++++---- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/creation/lib/check_python3_expr.py b/creation/lib/check_python3_expr.py index ba0f184ad2..6dd552f1f8 100755 --- a/creation/lib/check_python3_expr.py +++ b/creation/lib/check_python3_expr.py @@ -3,16 +3,8 @@ # SPDX-FileCopyrightText: 2009 Fermi Research Alliance, LLC # SPDX-License-Identifier: Apache-2.0 -# -# Project: -# glideinWMS -# # Description: # Validate expressions in frontend.xml for compatibility with Python 3 -# -# Author: -# Bruno Coimbra -# import argparse import ast @@ -58,7 +50,9 @@ def check_types(expression, factory_attrs, job_attrs): """Validates the types of match_attrs in a match_expr. Args: - code (str): Code to validate. + expression (str): Code to validate. + factory_attrs: + job_attrs: Returns: str: None if code is valid. Error message if the code is invalid. @@ -68,8 +62,8 @@ def check_types(expression, factory_attrs, job_attrs): default_value = {"string": "", "int": 0, "real": 0.0, "bool": False, "Expr": ""} try: - job = {attr: default_value[type] for (attr, type) in job_attrs} - glidein = {"attrs": {attr: default_value[type] for (attr, type) in factory_attrs}} + job = {attr: default_value[a_type] for (attr, a_type) in job_attrs} + glidein = {"attrs": {attr: default_value[a_type] for (attr, a_type) in factory_attrs}} except KeyError as e: return f"Invalid match_attr type: {e.args[0]}" @@ -87,8 +81,8 @@ def check_2to3(code, patch=False, refactoring_tool=rt): """Evaluates an expression using 2to3 and returns refactoring suggestions. Args: - expr (str): Expression to evaluate. - diff (bool): If True, returns a patch with the suggested changes. + code (str): Code to evaluate. + patch (bool): If True, returns a patch with the suggested changes. refactoring_tool (RefactoringTool): Used to by 2to3 to evaluate the expression. Returns: @@ -188,6 +182,8 @@ def main(config_file, enforce_2to3=False, silent=False, refactoring_tool=rt): Args: config_file (str): Path to the frontend configuration file. enforce_2to3 (bool, optional): Treats 2to3 suggestions as errors. Defaults to False. + silent (bool, optional): + refactoring_tool (RefactoringTool): Used to by 2to3 to evaluate the expression Returns: bool: True if the file is valid and False otherwise. @@ -208,9 +204,9 @@ def main(config_file, enforce_2to3=False, silent=False, refactoring_tool=rt): try: tree = ET.parse(config_file) except OSError: - return "Config file not readable: %s" % config_file + return False, "Config file not readable: %s" % config_file except: - return "Error parsing config file: %s" % config_file + return False, "Error parsing config file: %s" % config_file # Recursively finds all elements in the XML for element in findall_path(tree.getroot(), "match"): @@ -258,6 +254,7 @@ def main(config_file, enforce_2to3=False, silent=False, refactoring_tool=rt): location = f"{element.parent.data.tag} {element_name(element.parent.data)}" _log(f'\n\nEvaluating policy file "{path}"\n', silent) _log(f"at {location}\n", silent) + result = {} try: text = open(path).read() except FileNotFoundError as e: @@ -267,7 +264,6 @@ def main(config_file, enforce_2to3=False, silent=False, refactoring_tool=rt): result["error"] = error passed = False continue - result = {} result["type"] = "policy_file" result["value"] = path result["location"] = location @@ -313,10 +309,13 @@ def main(config_file, enforce_2to3=False, silent=False, refactoring_tool=rt): else: config_file = CONFIG_FILE - passed, _ = main(config_file, args.enforce_2to3, args.silent) - if passed: + main_passed, res = main(config_file, args.enforce_2to3, args.silent) + if main_passed: _log("\n\nPassed (configuration compatible with python3)\n", args.silent) exit(0) else: - _log("\n\nFailed (invalid python3 in configuration)\n", args.silent) + if isinstance(res, list): + _log("\n\nFailed (invalid python3 in configuration)\n", args.silent) + else: + _log("\n\nFailed (invalid configuration: %s)\n" % res, args.silent) exit(1) diff --git a/creation/reconfig_frontend b/creation/reconfig_frontend index 4e517b45d9..6d1fceda75 100755 --- a/creation/reconfig_frontend +++ b/creation/reconfig_frontend @@ -197,11 +197,14 @@ if __name__ == "__main__": if p3_compatible: print2("The configuration is compatible with Python 3") else: - print2("The configuration is not compatible with Python 3") - for element in report: - if not element["valid"]: - print2(f"...{element['type']} at {element['location']}:") - print2(f"\t{element['error']}") + if isinstance(report, list): + print2("The configuration is not compatible with Python 3") + for element in report: + if not element["valid"]: + print2(f"...{element['type']} at {element['location']}:") + print2(f"\t{element['error']}") + else: + print2(f"Bad configuration file: {report}") sys.exit(1) try: From 578d1267e5c68dc573de2b5ae8b60486b54f25ff Mon Sep 17 00:00:00 2001 From: terranovaa Date: Fri, 9 Sep 2022 14:50:28 -0500 Subject: [PATCH 089/196] Modified HTML website with new modifications --- creation/lib/cWDictFile.py | 10 +- creation/lib/cWParamDict.py | 16 ++- creation/lib/cgWParamDict.py | 16 ++- creation/lib/xmlConfig.py | 1 - doc/factory/configuration.html | 168 ++++++++++++++++++++------------ doc/factory/custom_scripts.html | 60 +++++------- doc/frontend/configuration.html | 28 +++--- 7 files changed, 172 insertions(+), 127 deletions(-) diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index 4f9f3809ad..97098414a8 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -1002,18 +1002,18 @@ class FileDictFile(SimpleFileDictFile): 2. cache/exec/... keyword identifying the file type: - regular - nocache - - exec (:s modifier to run in singularity) + - executable (:s modifier to run in singularity) - untar (if a modifier is present, it will be the name of the directory where the tarball will be unpacked, the name of the tarball will be used otherwise) - wrapper - source - library (:shell in case of a shell library) 3. prefix startd_cron variables prefix (default is GLIDEIN_PS_) - 4. time - phase of the lifecycle where to execute/source the script: + 4. time - phase of the lifecycle when to execute/source the scripts: - startup - pre_job - after_job - cleanup - - periodic (:period will specify the period in ms, 1000 by default) + - periodic (:period will specify the period in seconds, 60 by default) - milestone (:code will specify the milestone code) - failure (:exit_code will specify the failure code) Time will be ignored for files when type is different than exec or source. @@ -1033,12 +1033,12 @@ class FileDictFile(SimpleFileDictFile): Factory after_entry e+[g+] 90 A custom value inside the range [0-99] can be specified in order to alter the order of execution. 7. cond_download has a special value of TRUE - 8. tar_source - Name of the tarball containing the file, if the file is contained in a tarball + 8. tar_source - Name of the tarball containing the file, if the file is contained in a tarball, NULL otherwise (default) 9. config_out - has a special value of FALSE 10. cond_attr - Name of a configuration switch. (like "ENABLE_KRB5") The operation requested on the file will be performed only if that parameter will be set to 1. If not set (set to NULL), the result will be considered as TRUE (i.e. download) - As result, a file will be downloaded if cond_download & $cond_attr == TRUE + As result, a file will be downloaded if cond_download == TRUE || $cond_attr == TRUE 11. absdir_outattr - Name of a variable name. (like "KRB5_SUBSYS_DIR") The variable will be set to the absolute path of the file if the file is not a tarball or the directory where the tarball was unpacked in case it is a tarball, diff --git a/creation/lib/cWParamDict.py b/creation/lib/cWParamDict.py index 092e54d15a..2ae8da38d6 100644 --- a/creation/lib/cWParamDict.py +++ b/creation/lib/cWParamDict.py @@ -64,7 +64,9 @@ def add_file_unparsed(user_file, dicts, is_factory): raise RuntimeError("Found a file element with an empty relfname: %s" % user_file) is_const = is_true(user_file.const) - is_executable = user_file.type.startswith("exec") or user_file.type.startswith("run") + is_executable = ( + user_file.type.startswith("exec") or user_file.type.startswith("run") or user_file.type.startswith("executable") + ) is_wrapper = user_file.type == "wrapper" is_source = user_file.type.startswith("source") is_library = user_file.type.startswith("library") @@ -92,7 +94,7 @@ def add_file_unparsed(user_file, dicts, is_factory): else: period = 0 except (AttributeError, KeyError, ValueError, TypeError): - period = 1000 # default 1000ms + period = 60 # default 60s if is_periodic and not is_executable: raise RuntimeError("A file cannot have an execution period if it is not executable: %s" % user_file) @@ -112,7 +114,7 @@ def add_file_unparsed(user_file, dicts, is_factory): try: config_out = user_file.config_out if config_out is None: - config_out = "FALSE" + config_out = "FALSE" except (AttributeError, KeyError, ValueError, TypeError): config_out = "FALSE" @@ -150,12 +152,18 @@ def add_file_unparsed(user_file, dicts, is_factory): or user_file.type == "exec:singularity" or user_file.type == "run:s" or user_file.type == "run:singularity" + or user_file.type == "executable:s" + or user_file.type == "executable:singularity" ): if file_list_idx.endswith("preentry_file_list"): raise RuntimeError("An executable cannot use singularity before the entry setup: %s" % user_file) file_type = "exec:s" else: - if not user_file.type.startswith("run") and not user_file.type.startswith("exec"): + if ( + not user_file.type.startswith("run") + and not user_file.type.startswith("exec") + and not user_file.type.startswith("executable") + ): raise RuntimeError("An executable file type must start with 'run' or 'exec': $s" % user_file) dicts[file_list_idx].add_from_file( relfname, diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 8f93e8bdce..467e7fa072 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -28,10 +28,12 @@ # from cWParamDict import is_true, add_file_unparsed # from factoryXmlConfig import EntrySetElement + def is_true(s): """Case insensitive string parsing helper. Return True for true (case insensitive matching), False otherwise.""" return type(s) == str and s.lower() == "true" + class UnconfiguredScheddError(Exception): def __init__(self, schedd): self.schedd = schedd @@ -924,7 +926,9 @@ def add_file_unparsed(user_file, dicts, is_factory): relfname = user_file["relfname"] is_const = eval(user_file["const"]) - is_executable = user_file.type.startswith("exec") or user_file.type.startswith("run") + is_executable = ( + user_file.type.startswith("exec") or user_file.type.startswith("run") or user_file.type.startswith("executable") + ) is_wrapper = user_file.type == "wrapper" do_untar = user_file.type.startswith("untar") is_const = is_true(user_file.const) @@ -953,7 +957,7 @@ def add_file_unparsed(user_file, dicts, is_factory): else: period = 0 except (AttributeError, KeyError, ValueError, TypeError): - period = 1000 # default 1000ms + period = 60 # default 60s if ( is_periodic @@ -999,12 +1003,18 @@ def add_file_unparsed(user_file, dicts, is_factory): or user_file.type == "exec:singularity" or user_file.type == "run:s" or user_file.type == "run:singularity" + or user_file.type == "eecutable:s" + or user_file.type == "executable:s" ): if file_list_idx.endswith("preentry_file_list"): raise RuntimeError("An executable cannot use singularity before the entry setup: %s" % user_file) file_type = "exec:s" else: - if not user_file.type.startswith("run") and not user_file.type.startswith("exec"): + if ( + not user_file.type.startswith("run") + and not user_file.type.startswith("exec") + and not user_file.type.startswith("executable") + ): raise RuntimeError("An executable file type must start with 'run' or 'exec': $s" % user_file) dicts[file_list_idx].add_from_file( relfname, diff --git a/creation/lib/xmlConfig.py b/creation/lib/xmlConfig.py index c162959d45..803692aeda 100644 --- a/creation/lib/xmlConfig.py +++ b/creation/lib/xmlConfig.py @@ -327,7 +327,6 @@ def validate(self): TAG_CLASS_MAPPING.update({"file": FileElement}) - ####################### # # Module functions diff --git a/doc/factory/configuration.html b/doc/factory/configuration.html index ac42d32428..b44027886a 100644 --- a/doc/factory/configuration.html +++ b/doc/factory/configuration.html @@ -1,6 +1,7 @@ @@ -363,9 +364,8 @@

Example Configuration

<files>
<file absfname="/usr/conf/sethome.source" - after_entry="False" const="True" executable="False" - untar="False" wrapper="True" /><file absfname="/usr/conf/sethome.source" const="True" + type="wrapper" priority="value" time="phase" />
</files>
@@ -946,9 +946,8 @@

Global arguments

<glidein><files><file absfname="script name" executable="True" period="period in - seconds" prefix="cron prefix" - after_entry="True" type="file type" + >" type="executable" time="phase" + prefix="cron prefix" priority="value" comment="comment" />
@@ -2277,7 +2276,7 @@

Multiple Collectors

multiple collectors. (You should now see a line like: "<file absfname="web_base/collector_setup.sh" - executable="True"/>" after reconfiguring).

@@ -2619,8 +2618,9 @@

additional files. Both , Factory and Frontend configuration allow to specify lists of files either for all their glideins, for a specific entry (Factory), or job group (Frontend). These files can be scripts - (executables), regular files or tarballs and depending on the options - can be downloaded at different times (see + (executables), source files, libraries, wrapper, regular files or + tarballs and depending on the options can be downloaded at different + times (see the custom scripts document for more detils).

@@ -2659,83 +2659,110 @@

-

executable

+

type

- True if the file is a script (executable, see example below), - default is False + Keyword identifying the file type. Can have qualifiers depending + on the type after a colon (":"). - regular - nocache - + executable/exec/run (:s/:singularity modifier to run in + singularity) - untar (if a qualifier is present, it will be the + name of the directory where the tarball will be unpacked, the + name of the tarball will be used otherwise) - wrapper - source - + library (:shell in case of a shell library)

-

wrapper

+

const

- True if the file is a user wrapper (see example below), default - is False + If False the file is not constant (i.e. changes may happen + without a reconfiguration of the Factory and the file cannot be + checksummed), default is True

-

untar

+

relfname

- True if the file is a tarball that needs to be expanded (see - example below), default is False + Path to save the file, relative to the glidein main directory

-

type

+

time

- File type, must be one of the valid types (regular, run, source, - wrapper, untar). Can have qualifiers depending on the type after - a colon (":"). Valid qualifier for run is "singularity" - causes setup scripts to run in Singularity. Currently - type is ignored for non executables and the only values - used are "run" or "run:singularity". Default is empty + Phase of the lifecycle when to execute/source the scripts: - + startup - pre_job - after_job - cleanup - periodic (:period will + specify the period in seconds, 60 by default) - milestone (:code + will specify the milestone code) - failure (:exit_code will + specify the failure code) Time will be ignored for files when + type is different than exec or source. A combination of time + phases can be used.

-

const

+

prefix

- If False the file is not constant (i.e. changes may happen - without a reconfiguration of the Factory and the file cannot be - checksummed), default is True + STARTD_CRON prefix, it is prepended to all HTCondor variables + generated by the script (see documentation). This is ignored + from anything different from a periodic executable script. The + default value is GLIDEIN_PS_, use NOPREFIX to + have no prefix. Be aware that if you choose not to have a prefix + your HTCondor variables may collide with other scripts or + variables. Variables used by the wrapper, like GLIDEIN_PS_LAST, + will keep using the prefix.

-

relfname

+

priority

- Path to save the file, relative to the glidein main directory + priority - priority in the order of execution inside a time + phase [0-99]. Relevant only for executable, source and library + files. Order String code representation Associated integer code + (internal representation) Factory pre_entry e-[g-] 10 Frontend + pre_entry pre_group e-g- 20 Frontend pre_entry group e-g= 30 + Frontend pre_entry after_group e-g+ 40 entry e=[g-,g+,g=] 50 + Frontend after_entry pre_group e+g- 60 Frontend after_entry + group e+g= 70 Frontend after_entry after_group e+g+ 80 Factory + after_entry e+[g+] 90 A custom value inside the range [0-99] can + be specified in order to alter the order of execution. (see + the custom scripts document + for more details).

-

period

+

comment

+ +

Arbitrary comment string

+ + + +

after_entry

- If period>0 is the period (in seconds) of the executable . - Default is 0 (non periodic script). This is ignored for non - executable scripts. (see + If True, the script is executed after the entry scripts. Default + is False for the Factory, default is True for the Frontend. (see the custom scripts document for more)

@@ -2743,50 +2770,60 @@

-

prefix

+

cond_download

- STARTD_CRON prefix, it is prepended to all HTCondor variables - generated by the script (see documentation). This is ignored - from anything different from a periodic executable script. The - default value is GLIDEIN_PS_, use NOPREFIX to - have no prefix. Be aware that if you choose not to have a prefix - your HTCondor variables may collide with other scripts or - variables. Variables used by the wrapper, like GLIDEIN_PS_LAST, - will keep using the prefix. + Conditional boolean variable conditioning the download of the + file (True if file has to be downloaded).

-

comment

+

tar_source

+ + +

+ Name of the tarball containing the file, if the file is + contained in a tarball, NULL otherwise (default) +

-

Arbitrary comment string

-

after_entry

+

config_out

+ + +

has a special value of FALSE

+ + + + +

cond_attr

- If True, the script is executed after the entry scripts. Default - is False for the Factory, default is True for the Frontend. (see - the custom scripts document - for more) + cond_attr - Name of a configuration switch. (like "ENABLE_KRB5") + The operation requested on the file will be performed only if + that parameter will be set to 1. If not set (set to NULL), the + result will be considered as TRUE (i.e. download) As result, a + file will be downloaded if cond_download == TRUE OR $cond_attr + == TRUE

-

after_group

+

absdir_outattr

- If True, the Frontend script is executed after the group - scripts. Default is False. Not considered in the Factory. (see - the custom scripts document - for more) + Name of a variable name. (like "KRB5_SUBSYS_DIR") The variable + will be set to the absolute path of the file if the file is not + a tarball or the directory where the tarball was unpacked in + case it is a tarball, this last thing if and only if the + unpacking actually happened (else it will not be defined.)

@@ -2800,7 +2837,7 @@

 [<entries><entry>]
 <files>
  <file absfname="script name" - executable="True" prefix="cron_prefix" + type="executable" prefix="cron_prefix" comment="comment"/>

@@ -2818,7 +2855,7 @@

 [<entries><entry>]
 <files>
   <file absfname="script name" - wrapper="True" comment="comment"/> + type="wrapper" comment="comment"/>

Path to the wrapper custom script. The script will be copied in @@ -2836,7 +2873,7 @@

 <files>
   <file absfname="local file name" relfname="target file name" - const="Bool" executable="False" + const="Bool" comment="comment"/>

@@ -2859,10 +2896,10 @@

 <files>
  <file absfname="local file name" - untar="True" comment="comment">
-     <untar_options - cond_attr="conf_sw" dir="dir name" absdir_outattr="attr name"/> + type="untar:dir_name" comment="comment" + cond_attr="conf_sw" absdir_outattr="attr name" >

Sometimes it is useful to transfer a whole set of files, or even @@ -2885,8 +2922,9 @@

TRUE can be used to always untar it.
  • - dir: Name of the subdirectory to untar it in. (like - "krb5") + dir_name: Name of the subdirectory to untar it in. (like + "krb5", if not specified it will have the tarball + name)
  • absdir_outattr: Name of a variable name. (like diff --git a/doc/factory/custom_scripts.html b/doc/factory/custom_scripts.html index bd65455b4d..9f58bae662 100644 --- a/doc/factory/custom_scripts.html +++ b/doc/factory/custom_scripts.html @@ -142,7 +142,7 @@

    Script inclusion

    configuration file as being executable (see the attributes description):

    -
    executable="True"
    +
    type="executable"

    By default the files listed are non executable, so an administrator needs explicitly list the executable ones. @@ -152,7 +152,7 @@

    Script inclusion

    Periodic scripts are invoked also later, repeatedly according to the period specified (in seconds):

    -
    period="3600"
    +
    time="periodic:period"

    Periodic script can know if they are run at setup or later by looking at the GLIDEIN_PERIODIC_SCRIPT environment variable, which is set only @@ -533,7 +533,10 @@

    Loading order

    user scripts are the ones listed in the Factory and Frontend configuration files. First all files are downloaded, then the scripts are executed. Both the download and the first invocation follow the - same order in six distinct stages: + same order in distinct stages according to the priority value (ranging + in [0,99]). Priority integer values are associated to each phase. + Intermediate values among multiples of 10 can be used if needed to + have a more flexible choice on the moment of execution.

    -
  • - v3_6_5 released on October 5, 2020 (Manual, - Installation instructions)
    -
      -
    • - Added GitHub actions to test the code (Python unit tests, BARS, - pylint, pycodestyle) -
      • -
    • - Bug fix: Fix for paths in Singularity not substituted correctly - when the execute directory has a link or bind mount -
      • -
    • - Bug fix: Fixed shared_port_port to adapt to HTCondor 8.8.10 - changes -
      • -
    • - Bug fix: Fixed overrides in OSG_autoconf using depcopy, use - https for gfdiff -
      • -
    • - Bug fix: Fixed credential dir in manual_glidein_submit removing - hardcoding -
      • -
    -
    • -
  • - v3_6_4 released on September 17, 2020 (Manual, - Installation instructions)
    -
      -
    • Improved OSG_autoconf
      • -
    • Dropped Python 2.6 support
      • -
    • Added support for RHEL 8 worker nodes
      • -
    • - Harmonized, improved and extended CI tests (same CLI, support - for Python 3, ...) -
      • -
    • - Added tool to verify if the code for a ticket has been merged -
      • -
    • - Bug fix: Fix for condor_chirp not finding htchirp in the - PYTHONPATH -
      • -
    • - Bug fix: Fixed unquoted SINGULARITY_PATH causing HTCondor not to - start -
      • -
    -
    • -
  • - v3_6_3 released on July 16, 2020 (Manual, - Installation instructions,Tarball)
    -
      -
    • Add Python condor_chirp to PATH
      • -
    • Dropped tarball installation support
      • -
    • - Add Python condor_chirp and other GWMS tools to PATH in - Singularity -
      • -
    • - Removed "Group:System Environment/Daemons" from glideinwms.spec - in OSG repo -
      • -
    • Added new API documentation using sphinx
      • -
    • - Added a runner to run the different tests with consistent CLI -
      • -
    • Added new RHEL 8 Singularity image
      • -
    • - Updated the singularity wrapper with new features form the OSG - wrapper -
      • -
    • - Clean PYTHONPATH and setup GWMS environment in Singularity - containers -
      • -
    • - Improved the environment handling of the Singularity wrapper -
      • -
    • - Bug fix: Upgrade the factory documentation not to ask to install - the switchboard -
      • -
    • - Bug fix: The Python version of condor_chirp now works with - Singularity jobs -
      • -
    • Bug fix: Enable RFC proxies by default for direct proxies
      • -
    • Bug fix: Fix OSG_autoconf and gfdiff
      • -
    • - Bug fix: Added COMPLETED to the known list of "GridJobStatus"es -
      • -
    • Bug fix: Add a knob to set entry limits in OSG_autoconf
      • -
    • - Bug fix: Stop pilots even after the downtime timestamp has been - reached when draining pilots -
      • -
    • - Bug fix: Glideins are unrecoverable unless listed as recoverable -
      • -
    • - Bug fix: User job invocation via singularity eating up -e or -n -
      • -
    -
    • -
  • - v3_6_2 released on March 26, 2020 (Manual, - Installation instructions,Tarball)
    -
      -
    • - Add a portable condor_chirp for user jobs running in the - glideins -
      • -
    • Automate the generation of factory configuration
      • -
    • - Improve control of setup of module and spack in Singularity - using MODULE_USE -
      • -
    • Adding GPU monitor as default for pilots
      • -
    • Reduce number of queries from Frontend to User collector
      • -
    • - Make lgwms-logcat.sh work with non standard factory instance - names -
      • -
    • - Bug fix: Pilot using an expired proxy when authenticating to the - collector -
      • -
    • - Bug fix: CI issues (coverage broken in SL6, script selection - affected by file name) -
      • -
    • - Bug fix: Incorrect CERTIFICATE_MAPFILE used when using HTCondor - python binding (failed schedd authentication) -
      • -
    • Bug fix: Fix release manager relative imports
      • -
    • Bug fix: Broken submission to GCE and AWS
      • -
    • Bug fix: Correct how entry_ls detect an rpm installation
      • -
    • - Bug fix: Adjust size of ClusterId and ProcessId fields for the - condor_q custom format -
      • -
    • Bug fix: Total of frontend info is 0 in Factory monitring
      • -
    -
    • -
  • - v3_6_1 released on Novemver 13, 2019 (Manual, - Installation instructions,Tarball)
    -
      -
    • Added compatibility w/ HTCondor 8.8.x in OSG 3.5
      • -
    • Monitoring pages use https if available
      • -
    • Improved search and testing of Singularity binary
      • -
    • Unset LD_LIBRARY_PATH and PATH for jobs in Singularity
      • -
    • Updated documentation links and Google search
      • -
    • Improved CI testing
      • -
    • - Stop considering held limits when counting maximum jobs in - Factory -
      • -
    • - Bug fix: Fix Fatcory tools (entry_rm, entry_q and entry_ls) to - be more verbose with single user Factory -
      • -
    • - Bug fix: Removed hardcoded CVMFS requirement for Singularity -
      • -
    • - Bug fix: Improve diagnostic messages when rsa.key file is - corrupted -
      • -
    • - Bug fix: Improve control of the worker node draining, avoiding - to stop accepting jobs as soon as the draining file is present -
      • -
    -
    • -
  • - v3_6 released on September 25, 2019 (Manual, - Installation instructions,Tarball)
    -
      -
    • - Renaming of v3_5_1 (released on 9/18) to respect the odd/even - numbering -
      • -
    • Includes all 3.4.6 features
      • -
    • Updated SW and docs for the change in OSG factories
      • -
    • - Updated all the obsolete links to HTCondor manual in GlideinWMS - website -
      • -
    • Set up an ITB Frontend for GWMS and FIFE testing
      • -
    • - Updated gitattributes to resolve conflicts w/ checksum files -
      • -
    • Added editorconfig and default encoding
      • -
    • GlideinWMS code will use now Google docstring format
      • -
    • - Advertise if a Glidein can use privileged or unprivileged - Singularity -
      • -
    • - Check if single user factory migration script has been run - before startup -
      • -
    • Bug fix: pip errors in nightly CI
      • -
    • Bug fix: Unittest failing at times on SL7
      • -
    • Bug fix: Factory could start also w/ a GT2 entry enabled
      • -
    • - NOTE: v3_6 is fully interoperable w/ v3_5 and v3_4. I.e. You can - mix v3_6_x, v3_5_x and v3_4_x Factories and Frontends in your - system -
      • -
    • - NOTE-FACTORY: The v3.6 Factory (like v3.5) is using a single - user. This is a major change. If you are upgrading, after the - RPM upgrade you will need to:\ 1. stop Factory and HTCondor, 2. - Migrate the HTCondor status running the fact_chown script, 3. - Run upgrade and restart the HTCondor and the Factory. For - detailed instructions see - http://glideinwms.fnal.gov/doc.dev/factory/configuration.html#single_user -
      • -
    -
    • +
  • + v3_6_5 released on October 5, 2020 (Manual, + Installation instructions)
    +
      +
    • Added GitHub actions to test the code (Python unit tests, BARS, pylint, pycodestyle)
      • +
    • Bug fix: Fix for paths in Singularity not substituted correctly when the execute directory has a link or bind mount
      • +
    • Bug fix: Fixed shared_port_port to adapt to HTCondor 8.8.10 changes
      • +
    • Bug fix: Fixed overrides in OSG_autoconf using depcopy, use https for gfdiff
      • +
    • Bug fix: Fixed credential dir in manual_glidein_submit removing hardcoding
      • +
    +
    • +
  • + v3_6_4 released on September 17, 2020 (Manual, + Installation instructions)
    +
      +
    • Improved OSG_autoconf
      • +
    • Dropped Python 2.6 support
      • +
    • Added support for RHEL 8 worker nodes
      • +
    • Harmonized, improved and extended CI tests (same CLI, support for Python 3, ...)
      • +
    • Added tool to verify if the code for a ticket has been merged
      • +
    • Bug fix: Fix for condor_chirp not finding htchirp in the PYTHONPATH
      • +
    • Bug fix: Fixed unquoted SINGULARITY_PATH causing HTCondor not to start
      • +
    +
    • +
  • + v3_6_3 released on July 16, 2020 (Manual, + Installation instructions,Tarball)
    +
      +
    • Add Python condor_chirp to PATH
      • +
    • Dropped tarball installation support
      • +
    • Add Python condor_chirp and other GWMS tools to PATH in Singularity
      • +
    • Removed "Group:System Environment/Daemons" from glideinwms.spec in OSG repo
      • +
    • Added new API documentation using sphinx
      • +
    • Added a runner to run the different tests with consistent CLI
      • +
    • Added new RHEL 8 Singularity image
      • +
    • Updated the singularity wrapper with new features form the OSG wrapper
      • +
    • Clean PYTHONPATH and setup GWMS environment in Singularity containers
      • +
    • Improved the environment handling of the Singularity wrapper
      • +
    • Bug fix: Upgrade the factory documentation not to ask to install the switchboard
      • +
    • Bug fix: The Python version of condor_chirp now works with Singularity jobs
      • +
    • Bug fix: Enable RFC proxies by default for direct proxies
      • +
    • Bug fix: Fix OSG_autoconf and gfdiff
      • +
    • Bug fix: Added COMPLETED to the known list of "GridJobStatus"es
      • +
    • Bug fix: Add a knob to set entry limits in OSG_autoconf
      • +
    • Bug fix: Stop pilots even after the downtime timestamp has been reached when draining pilots
      • +
    • Bug fix: Glideins are unrecoverable unless listed as recoverable
      • +
    • Bug fix: User job invocation via singularity eating up -e or -n
      • +
    +
    • +
  • + v3_6_2 released on March 26, 2020 (Manual, + Installation instructions,Tarball)
    +
      +
    • Add a portable condor_chirp for user jobs running in the glideins
      • +
    • Automate the generation of factory configuration
      • +
    • Improve control of setup of module and spack in Singularity using MODULE_USE
      • +
    • Adding GPU monitor as default for pilots
      • +
    • Reduce number of queries from Frontend to User collector
      • +
    • Make lgwms-logcat.sh work with non standard factory instance names
      • +
    • Bug fix: Pilot using an expired proxy when authenticating to the collector
      • +
    • Bug fix: CI issues (coverage broken in SL6, script selection affected by file name)
      • +
    • Bug fix: Incorrect CERTIFICATE_MAPFILE used when using HTCondor python binding (failed schedd authentication)
      • +
    • Bug fix: Fix release manager relative imports
      • +
    • Bug fix: Broken submission to GCE and AWS
      • +
    • Bug fix: Correct how entry_ls detect an rpm installation
      • +
    • Bug fix: Adjust size of ClusterId and ProcessId fields for the condor_q custom format
      • +
    • Bug fix: Total of frontend info is 0 in Factory monitring
      • +
    +
    • +
  • + v3_6_1 released on Novemver 13, 2019 (Manual, + Installation instructions,Tarball)
    +
      +
    • Added compatibility w/ HTCondor 8.8.x in OSG 3.5
      • +
    • Monitoring pages use https if available
      • +
    • Improved search and testing of Singularity binary
      • +
    • Unset LD_LIBRARY_PATH and PATH for jobs in Singularity
      • +
    • Updated documentation links and Google search
      • +
    • Improved CI testing
      • +
    • Stop considering held limits when counting maximum jobs in Factory
      • +
    • Bug fix: Fix Fatcory tools (entry_rm, entry_q and entry_ls) to be more verbose with single user Factory
      • +
    • Bug fix: Removed hardcoded CVMFS requirement for Singularity
      • +
    • Bug fix: Improve diagnostic messages when rsa.key file is corrupted
      • +
    • Bug fix: Improve control of the worker node draining, avoiding to stop accepting jobs as soon as the draining file is present
      • +
    +
    • +
  • + v3_6 released on September 25, 2019 (Manual, + Installation instructions,Tarball)
    +
      +
    • Renaming of v3_5_1 (released on 9/18) to respect the odd/even numbering
      • +
    • Includes all 3.4.6 features
      • +
    • Updated SW and docs for the change in OSG factories
      • +
    • Updated all the obsolete links to HTCondor manual in GlideinWMS website
      • +
    • Set up an ITB Frontend for GWMS and FIFE testing
      • +
    • Updated gitattributes to resolve conflicts w/ checksum files
      • +
    • Added editorconfig and default encoding
      • +
    • GlideinWMS code will use now Google docstring format
      • +
    • Advertise if a Glidein can use privileged or unprivileged Singularity
      • +
    • Check if single user factory migration script has been run before startup
      • +
    • Bug fix: pip errors in nightly CI
      • +
    • Bug fix: Unittest failing at times on SL7
      • +
    • Bug fix: Factory could start also w/ a GT2 entry enabled
      • +
    • NOTE: v3_6 is fully interoperable w/ v3_5 and v3_4. I.e. You can mix v3_6_x, v3_5_x and v3_4_x Factories and Frontends in your system
      • +
    • NOTE-FACTORY: The v3.6 Factory (like v3.5) is using a single user. This is a major change. If you are upgrading, after the RPM upgrade you will need to:\ 1. stop Factory and HTCondor, 2. Migrate the HTCondor status running the fact_chown script, 3. Run upgrade and restart the HTCondor and the Factory. For detailed instructions see http://glideinwms.fnal.gov/doc.dev/factory/configuration.html#single_user
      • +
    +
  • v3_4_6 released on August 14, 2019 (Development Series