Feature: Reduce build log verbosity

images/kubectl-build-deploy-dind/build-deploy.sh

@@ -3,6 +3,8 @@ set -x
 set -eo pipefail
 set -o noglob
 
+set +x # reduce noise in build logs
+
 # print out the build-deploy-tool version information
 echo "##############################################"
 build-deploy-tool version
@@ -14,13 +16,11 @@ NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
 REGISTRY_REPOSITORY=$NAMESPACE
 LAGOON_VERSION=$(cat /lagoon/version)
 
-set +x # reduce noise in build logs
 if [ ! -z "$LAGOON_PROJECT_VARIABLES" ]; then
   INTERNAL_REGISTRY_URL=$(jq --argjson data "$LAGOON_PROJECT_VARIABLES" -n -r '$data | .[] | select(.scope == "internal_container_registry") | select(.name == "INTERNAL_REGISTRY_URL") | .value' | sed -e 's#^http://##' | sed -e 's#^https://##')
   INTERNAL_REGISTRY_USERNAME=$(jq --argjson data "$LAGOON_PROJECT_VARIABLES" -n -r '$data | .[] | select(.scope == "internal_container_registry") | select(.name == "INTERNAL_REGISTRY_USERNAME") | .value')
   INTERNAL_REGISTRY_PASSWORD=$(jq --argjson data "$LAGOON_PROJECT_VARIABLES" -n -r '$data | .[] | select(.scope == "internal_container_registry") | select(.name == "INTERNAL_REGISTRY_PASSWORD") | .value')
 fi
-set -x
 
 if [ "$CI" == "true" ]; then
   CI_OVERRIDE_IMAGE_REPO=172.17.0.1:5000/lagoon
@@ -56,7 +56,6 @@ PRIVATE_REGISTRY_URLS=()
 PRIVATE_DOCKER_HUB_REGISTRY=0
 PRIVATE_EXTERNAL_REGISTRY=0
 
-set +x # reduce noise in build logs
 DEPLOYER_TOKEN=$(cat /var/run/secrets/lagoon/deployer/token)
 
 kubectl config set-credentials lagoon/kubernetes.default.svc --token="${DEPLOYER_TOKEN}"
@@ -147,6 +146,6 @@ do
     fi
   fi
 done
-set -x
 
 .  /kubectl-build-deploy/build-deploy-docker-compose.sh
+set -x

images/kubectl-build-deploy-dind/scripts/exec-backup-generation.sh

@@ -73,9 +73,7 @@ if [[ "${CAPABILITIES[@]}" =~ "backup.appuio.ch/v1alpha1/Schedule" ]]; then
       HELM_CUSTOM_BAAS_BACKUP_ACCESS_KEY=${BAAS_CUSTOM_BACKUP_ACCESS_KEY}
       HELM_CUSTOM_BAAS_BACKUP_SECRET_KEY=${BAAS_CUSTOM_BACKUP_SECRET_KEY}
     else
-      set +x
       kubectl --insecure-skip-tls-verify -n ${NAMESPACE} delete secret baas-custom-backup-credentials --ignore-not-found
-      set -x
     fi
   fi
 
@@ -90,30 +88,24 @@ if [[ "${CAPABILITIES[@]}" =~ "backup.appuio.ch/v1alpha1/Schedule" ]]; then
       HELM_CUSTOM_BAAS_RESTORE_ACCESS_KEY=${BAAS_CUSTOM_RESTORE_ACCESS_KEY}
       HELM_CUSTOM_BAAS_RESTORE_SECRET_KEY=${BAAS_CUSTOM_RESTORE_SECRET_KEY}
     else
-      set +x
       kubectl --insecure-skip-tls-verify -n ${NAMESPACE} delete secret baas-custom-restore-credentials --ignore-not-found
-      set -x
     fi
   fi
 
   if ! kubectl --insecure-skip-tls-verify -n ${NAMESPACE} get secret baas-repo-pw &> /dev/null; then
     # Create baas-repo-pw secret based on the project secret
-    set +x
     kubectl --insecure-skip-tls-verify -n ${NAMESPACE} create secret generic baas-repo-pw --from-literal=repo-pw=$(echo -n "$PROJECT_SECRET-BAAS-REPO-PW" | sha256sum | cut -d " " -f 1)
-    set -x
   fi
 
   TEMPLATE_PARAMETERS=()
 
-  set +x # reduce noise in build logs
   # Check for custom baas bucket name
   if [ ! -z "$LAGOON_PROJECT_VARIABLES" ]; then
     BAAS_BUCKET_NAME=$(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_BUCKET_NAME") | "\(.value)"')
   fi
   if [ -z $BAAS_BUCKET_NAME ]; then
     BAAS_BUCKET_NAME=baas-${PROJECT}
   fi
-  set -x
 
   # Pull in .lagoon.yml variables
   PRODUCTION_MONTHLY_BACKUP_RETENTION=$(cat .lagoon.yml | shyaml get-value backup-retention.production.monthly "")
@@ -337,4 +329,4 @@ if [[ "${CAPABILITIES[@]}" =~ "backup.appuio.ch/v1alpha1/Schedule" ]]; then
     --set customRestoreLocation.secretKey="${BAAS_CUSTOM_RESTORE_SECRET_KEY}" \
     --set customBackupLocation.accessKey="${BAAS_CUSTOM_BACKUP_ACCESS_KEY}" \
     --set customBackupLocation.secretKey="${BAAS_CUSTOM_BACKUP_SECRET_KEY}" "${HELM_ARGUMENTS[@]}" > $YAML_FOLDER/k8up-lagoon-backup-schedule.yaml
-fi
+fi

images/kubectl-build-deploy-dind/scripts/exec-build.sh

@@ -1,6 +1,4 @@
 #!/bin/bash
 
 # try to pull the last pushed image so we can use it for --cache-from during the build
-set +x
 docker build --network=host "${BUILD_ARGS[@]}" -t $TEMPORARY_IMAGE_NAME -f $BUILD_CONTEXT/$DOCKERFILE $BUILD_CONTEXT
-set -x

images/kubectl-build-deploy-dind/scripts/exec-generate-insights-configmap.sh

@@ -7,7 +7,6 @@ SBOM_CONFIGMAP="lagoon-insights-sbom-${IMAGE_NAME}"
 IMAGE_INSPECT_CONFIGMAP="lagoon-insights-image-${IMAGE_NAME}"
 IMAGE_INSPECT_OUTPUT_FILE="${TMP_DIR}/${IMAGE_NAME}.image-inspect.json.gz"
 
-set +x
 echo "Running image inspect on: ${IMAGE_FULL}"
 
 skopeo inspect --retry-times 5 docker://${IMAGE_FULL} --tls-verify=false | gzip > ${IMAGE_INSPECT_OUTPUT_FILE}

images/kubectl-build-deploy-dind/scripts/exec-kubectl-mariadb-dbaas.sh

@@ -16,7 +16,7 @@ else
     exit 1
 fi
 done
-set +x
+
 # Grab the details from the consumer spec
 DB_HOST=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary)
 DB_USER=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username)
@@ -39,5 +39,3 @@ if DB_READREPLICA_HOSTS=$(kubectl -n ${NAMESPACE} get mariadbconsumer/${SERVICE_
         configmap lagoon-env \
         -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_READREPLICA_HOSTS\":\"${DB_READREPLICA_HOSTS}\"}}"
 fi
-
-set -x

images/kubectl-build-deploy-dind/scripts/exec-kubectl-mongodb-dbaas.sh

@@ -16,7 +16,7 @@ else
     exit 1
 fi
 done
-set +x
+
 # Grab the details from the consumer spec
 DB_HOST=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary)
 DB_USER=$(kubectl -n ${NAMESPACE} get mongodbconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username)
@@ -32,5 +32,3 @@ kubectl patch \
     -n ${NAMESPACE} \
     configmap lagoon-env \
     -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_HOST\":\"${DB_HOST}\", \"${SERVICE_NAME_UPPERCASE}_USERNAME\":\"${DB_USER}\", \"${SERVICE_NAME_UPPERCASE}_PASSWORD\":\"${DB_PASSWORD}\", \"${SERVICE_NAME_UPPERCASE}_DATABASE\":\"${DB_NAME}\", \"${SERVICE_NAME_UPPERCASE}_PORT\":\"${DB_PORT}\", \"${SERVICE_NAME_UPPERCASE}_AUTHSOURCE\":\"${DB_AUTHSOURCE}\", \"${SERVICE_NAME_UPPERCASE}_AUTHMECHANISM\":\"${DB_AUTHMECHANISM}\", \"${SERVICE_NAME_UPPERCASE}_AUTHTLS\":\"${DB_AUTHTLS}\" }}"
-
-set -x

images/kubectl-build-deploy-dind/scripts/exec-kubectl-postgres-dbaas.sh

@@ -16,7 +16,7 @@ else
     exit 1
 fi
 done
-set +x
+
 # Grab the details from the consumer spec
 DB_HOST=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.services.primary)
 DB_USER=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVICE_NAME} -o yaml | shyaml get-value spec.consumer.username)
@@ -39,5 +39,3 @@ if DB_READREPLICA_HOSTS=$(kubectl -n ${NAMESPACE} get postgresqlconsumer/${SERVI
         configmap lagoon-env \
         -p "{\"data\":{\"${SERVICE_NAME_UPPERCASE}_READREPLICA_HOSTS\":\"${DB_READREPLICA_HOSTS}\"}}"
 fi
-
-set -x

images/kubectl-build-deploy-dind/scripts/exec-monitor-deploy.sh

@@ -4,7 +4,7 @@
 # in case this rollout fails, we show the logs of the new containers to the user as they might contain information about why
 # the rollout has failed
 stream_logs_deployment() {
-  set +x
+  # set +x
   # load the version of the new pods
   LATEST_POD_TEMPLATE_HASH=$(kubectl get replicaset -l app.kubernetes.io/instance=${SERVICE_NAME} --sort-by=.metadata.creationTimestamp -o=json | jq -r '.items[-1].metadata.labels."pod-template-hash"')
   mkdir -p /tmp/kubectl-build-deploy/logs/container/${SERVICE_NAME}
@@ -25,9 +25,9 @@ stream_logs_deployment() {
 
     # If we are here, this means the pods have all stopped (probably because they failed), we just restart
   done
+  # set -x
 }
 
-set +x # reduce noise in build logs
 # start background logs streaming
 stream_logs_deployment &
 STREAM_LOGS_PID=$!
@@ -68,5 +68,4 @@ if [[ $ret -ne 0 ]]; then
 fi
 
 # stop all running stream logs
-pkill -P $STREAM_LOGS_PID || true
-set -x
+pkill -P $STREAM_LOGS_PID 2>/dev/null || true

images/kubectl-build-deploy-dind/scripts/exec-push-parallel.sh

@@ -2,5 +2,6 @@
 
 docker tag ${TEMPORARY_IMAGE_NAME} ${REGISTRY}/${PROJECT}/${ENVIRONMENT}/${IMAGE_NAME}:${IMAGE_TAG:-latest}
 
-echo "docker push ${REGISTRY}/${PROJECT}/${ENVIRONMENT}/${IMAGE_NAME}:${IMAGE_TAG:-latest}" >> /kubectl-build-deploy/lagoon/push
+# only show final output as push steps are not required
+echo "docker push ${REGISTRY}/${PROJECT}/${ENVIRONMENT}/${IMAGE_NAME}:${IMAGE_TAG:-latest} | grep -v -E 'Waiting|Layer already exists|Preparing'" >> /kubectl-build-deploy/lagoon/push