Skip to content

Commit

Permalink
refactor: split build and push to independent steps
Browse files Browse the repository at this point in the history
  • Loading branch information
@shreddedbacon
shreddedbacon committed Nov 5, 2024
1 parent 72bcaf0 commit 1aea8a2
Showing 1 changed file with 82 additions and 52 deletions.
134 changes: 82 additions & 52 deletions legacy/build-deploy-docker-compose.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,8 +616,6 @@ done

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${buildStartTime}" "${currentStepEnd}" "${NAMESPACE}" "registryLogin" "Container Registry Login" "false"
previousStepEnd=${currentStepEnd}
beginBuildStep "Image Builds" "buildingImages"


##############################################
Expand Down Expand Up @@ -653,19 +651,23 @@ if [[ "$BUILD_TYPE" == "pullrequest" || "$BUILD_TYPE" == "branch" ]]; then
echo "Pulling Image: ${FPI}"
docker pull "${FPI}"
done

HAS_PULLED_IMAGES=false
# now we loop through the images in the build data and determine if they need to be pulled or build
for IMAGE_BUILD_DATA in $(echo "$ENVIRONMENT_IMAGE_BUILD_DATA" | jq -c '.images[]')
do
SERVICE_NAME=$(echo "$IMAGE_BUILD_DATA" | jq -r '.name // false')
SERVICE_NAME_TITLE=$(echo "$SERVICE_NAME" | tr '[:upper:]' '[:lower:]' | tr -d '-')
DOCKERFILE=$(echo "$IMAGE_BUILD_DATA" | jq -r '.imageBuild.dockerFile // false')
# if there is no dockerfile, then this image needs to be pulled from somewhere else
if [ $DOCKERFILE == "false" ]; then
PULL_IMAGE=$(echo "$IMAGE_BUILD_DATA" | jq -r '.imageBuild.pullImage // false')
if [ "$PULL_IMAGE" != "false" ]; then
IMAGES_PULL["${SERVICE_NAME}"]="${PULL_IMAGE}"
HAS_PULLED_IMAGES=true
fi
else
previousStepEnd=${currentStepEnd}
beginBuildStep "Building Image ${SERVICE_NAME}" "buildingImage${SERVICE_NAME_TITLE}"
# otherwise extract build information from the image build data payload
# this is a temporary image name to use for the build, it is based on the namespace and service, this can probably be deprecated and the images could just be
# built with the name they are meant to be. only 1 build can run at a time within a namespace
Expand Down Expand Up @@ -698,28 +700,50 @@ if [[ "$BUILD_TYPE" == "pullrequest" || "$BUILD_TYPE" == "branch" ]]; then

# adding the build image to the list of arguments passed into the next image builds
SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]')
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "imageBuild${SERVICE_NAME_TITLE}Complete" "Building Image ${SERVICE_NAME}" "false"
fi
done
fi

# print information about built image sizes
function printBytes {
local -i bytes=$1;
echo "$(( (bytes + 1000000)/1000000 ))MB"
}
if [[ "${IMAGES_BUILD[@]}" ]]; then
echo "##############################################"
echo "Built image sizes:"
echo "##############################################"
if [ $HAS_PULLED_IMAGES == "true" ]; then
previousStepEnd=${currentStepEnd}
beginBuildStep "Pulled Images" "pulledImageInfo"
echo "The following images will be pulled and then pushed directly with no building required"
for IMAGE_NAME in "${!IMAGES_PULL[@]}"
do
PULL_IMAGE="${IMAGES_PULL[${IMAGE_NAME}]}" #extract the pull image name from the images to pull list
echo "- ${IMAGE_NAME}: ${PULL_IMAGE}"
done
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "pulledImageInfoComplete" "Pulled Images" "false"
fi

previousStepEnd=${currentStepEnd}
beginBuildStep "Image Build Stats" "imageBuildStats"

# print information about built image sizes
function printBytes {
local -i bytes=$1;
echo "$(( (bytes + 1000000)/1000000 ))MB"
}
if [[ "${IMAGES_BUILD[@]}" ]]; then
echo "##############################################"
echo "Built image sizes:"
echo "##############################################"
fi
for IMAGE_NAME in "${!IMAGES_BUILD[@]}"
do
TEMPORARY_IMAGE_NAME="${IMAGES_BUILD[${IMAGE_NAME}]}"
echo -e "- image: ${TEMPORARY_IMAGE_NAME}\t\t$(printBytes $(docker inspect ${TEMPORARY_IMAGE_NAME} | jq -r '.[0].Size'))"
done

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "imageBuildStatsComplete" "Image Build Stats" "false"
fi
if [[ "$BUILD_TYPE" == "promote" ]]; then
echo "No images built for promote environments"
fi
for IMAGE_NAME in "${!IMAGES_BUILD[@]}"
do
TEMPORARY_IMAGE_NAME="${IMAGES_BUILD[${IMAGE_NAME}]}"
echo -e "Image ${TEMPORARY_IMAGE_NAME}\t\t$(printBytes $(docker inspect ${TEMPORARY_IMAGE_NAME} | jq -r '.[0].Size'))"
done

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "imageBuildComplete" "Image Builds" "false"
previousStepEnd=${currentStepEnd}
beginBuildStep "Service Configuration Phase 1" "serviceConfigurationPhase1"

Expand Down Expand Up @@ -1212,8 +1236,6 @@ done

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "updateConfigmapComplete" "Update Configmap" "false"
previousStepEnd=${currentStepEnd}
beginBuildStep "Image Push to Registry" "pushingImages"

##############################################
### REDEPLOY DEPLOYMENTS IF CONFIG MAP CHANGES
Expand All @@ -1237,33 +1259,7 @@ declare -A DEPRECATED_IMAGE_SUGGESTION

# pullrequest/branch start
if [ "$BUILD_TYPE" == "pullrequest" ] || [ "$BUILD_TYPE" == "branch" ]; then

# All images that should be pulled are copied to the harbor registry
for IMAGE_NAME in "${!IMAGES_PULL[@]}"
do
PULL_IMAGE="${IMAGES_PULL[${IMAGE_NAME}]}" #extract the pull image name from the images to pull list
PUSH_IMAGE="${IMAGES_PUSH[${IMAGE_NAME}]}" #extract the push image name from the images to push list

# Try to handle private registries first

# the external pull image name is all calculated in the build-deploy tool now, it knows how to calculate it
# from being a promote image, or an image from an imagecache or from some other registry entirely
skopeo copy --retry-times 5 --dest-tls-verify=false docker://${PULL_IMAGE} docker://${PUSH_IMAGE}

# store the resulting image hash
SKOPEO_INSPECT=$(skopeo inspect --retry-times 5 docker://${PUSH_IMAGE} --tls-verify=false)
IMAGE_HASHES[${IMAGE_NAME}]=$(echo "${SKOPEO_INSPECT}" | jq ".Name + \"@\" + .Digest" -r)

# check if the pull through image is deprecated
DEPRECATED_STATUS=$(echo "${SKOPEO_INSPECT}" | jq -r '.Labels."sh.lagoon.image.deprecated.status" // false')
if [ "${DEPRECATED_STATUS}" != false ]; then
DEPRECATED_IMAGE_WARNINGS="true"
DEPRECATED_IMAGE_NAME[${IMAGE_NAME}]=${PULL_IMAGE#$IMAGECACHE_REGISTRY}
DEPRECATED_IMAGE_STATUS[${IMAGE_NAME}]=$DEPRECATED_STATUS
DEPRECATED_IMAGE_SUGGESTION[${IMAGE_NAME}]=$(echo "${SKOPEO_INSPECT}" | jq -r '.Labels."sh.lagoon.image.deprecated.suggested" | sub("docker.io\/";"")? // false')
fi
done

# calculate the images required to be pushed to the registry
for IMAGE_NAME in "${!IMAGES_BUILD[@]}"
do
PUSH_IMAGE="${IMAGES_PUSH[${IMAGE_NAME}]}" #extract the push image name from the images to push list
Expand All @@ -1286,6 +1282,8 @@ if [ "$BUILD_TYPE" == "pullrequest" ] || [ "$BUILD_TYPE" == "branch" ]; then
fi
done

previousStepEnd=${currentStepEnd}
beginBuildStep "Pushing Images" "pushingImages"
# If we have images to push to the registry, let's do so
if [ -f /kubectl-build-deploy/lagoon/push ]; then
parallel --retries 4 < /kubectl-build-deploy/lagoon/push
Expand All @@ -1298,25 +1296,57 @@ if [ "$BUILD_TYPE" == "pullrequest" ] || [ "$BUILD_TYPE" == "branch" ]; then
JQ_QUERY=(jq -r ".[]|select(test(\"${REGISTRY}/${PROJECT}/${ENVIRONMENT}/${IMAGE_NAME}@\"))")
IMAGE_HASHES[${IMAGE_NAME}]=$(docker inspect ${PUSH_IMAGE} --format '{{json .RepoDigests}}' | "${JQ_QUERY[@]}")
done
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "pushingImagesComplete" "Pushing Images" "false"

# All images that should be pulled are copied to the harbor registry
for IMAGE_NAME in "${!IMAGES_PULL[@]}"
do
PULL_IMAGE="${IMAGES_PULL[${IMAGE_NAME}]}" #extract the pull image name from the images to pull list
PUSH_IMAGE="${IMAGES_PUSH[${IMAGE_NAME}]}" #extract the push image name from the images to push list

# Try to handle private registries first
previousStepEnd=${currentStepEnd}
beginBuildStep "Pushing Pulled Image ${IMAGE_NAME}" "pushingImage${IMAGE_NAME}"
# the external pull image name is all calculated in the build-deploy tool now, it knows how to calculate it
# from being a promote image, or an image from an imagecache or from some other registry entirely
skopeo copy --retry-times 5 --dest-tls-verify=false docker://${PULL_IMAGE} docker://${PUSH_IMAGE}

# store the resulting image hash
SKOPEO_INSPECT=$(skopeo inspect --retry-times 5 docker://${PUSH_IMAGE} --tls-verify=false)
IMAGE_HASHES[${IMAGE_NAME}]=$(echo "${SKOPEO_INSPECT}" | jq ".Name + \"@\" + .Digest" -r)

# check if the pull through image is deprecated
DEPRECATED_STATUS=$(echo "${SKOPEO_INSPECT}" | jq -r '.Labels."sh.lagoon.image.deprecated.status" // false')
if [ "${DEPRECATED_STATUS}" != false ]; then
DEPRECATED_IMAGE_WARNINGS="true"
DEPRECATED_IMAGE_NAME[${IMAGE_NAME}]=${PULL_IMAGE#$IMAGECACHE_REGISTRY}
DEPRECATED_IMAGE_STATUS[${IMAGE_NAME}]=$DEPRECATED_STATUS
DEPRECATED_IMAGE_SUGGESTION[${IMAGE_NAME}]=$(echo "${SKOPEO_INSPECT}" | jq -r '.Labels."sh.lagoon.image.deprecated.suggested" | sub("docker.io\/";"")? // false')
fi
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "pushingImage${IMAGE_NAME}Complete" "Pushing Pulled Image ${IMAGE_NAME}" "false"
done

# pullrequest/branch end
# promote start
elif [ "$BUILD_TYPE" == "promote" ]; then

for IMAGE_NAME in "${IMAGES[@]}"
do
previousStepEnd=${currentStepEnd}
beginBuildStep "Pushing Pulled Promote Image ${IMAGE_NAME}" "pushingImage${IMAGE_NAME}"
PUSH_IMAGE="${IMAGES_PUSH[${IMAGE_NAME}]}" #extract the push image name from the images to push list
PROMOTE_IMAGE="${IMAGES_PROMOTE[${IMAGE_NAME}]}" #extract the push image name from the images to push list
skopeo copy --retry-times 5 --src-tls-verify=false --dest-tls-verify=false docker://${PROMOTE_IMAGE} docker://${PUSH_IMAGE}

IMAGE_HASHES[${IMAGE_NAME}]=$(skopeo inspect --retry-times 5 docker://${PUSH_IMAGE} --tls-verify=false | jq ".Name + \"@\" + .Digest" -r)
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "pushingImage${IMAGE_NAME}Complete" "Pushing Pulled Promote Image ${IMAGE_NAME}" "false"
done
# promote end
fi

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "imagePushComplete" "Image Push to Registry" "false"

##############################################
### Check for deprecated images
##############################################
Expand Down

0 comments on commit 1aea8a2

Please sign in to comment.