Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: split build and push to independent steps #390

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

refactor: split build and push to independent steps #390

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 82 additions & 52 deletions legacy/build-deploy-docker-compose.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -616,8 +616,6 @@ done

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${buildStartTime}" "${currentStepEnd}" "${NAMESPACE}" "registryLogin" "Container Registry Login" "false"
previousStepEnd=${currentStepEnd}
beginBuildStep "Image Builds" "buildingImages"


##############################################
Expand Down Expand Up @@ -653,19 +651,23 @@ if [[ "$BUILD_TYPE" == "pullrequest" || "$BUILD_TYPE" == "branch" ]]; then
echo "Pulling Image: ${FPI}"
docker pull "${FPI}"
done

HAS_PULLED_IMAGES=false
# now we loop through the images in the build data and determine if they need to be pulled or build
for IMAGE_BUILD_DATA in $(echo "$ENVIRONMENT_IMAGE_BUILD_DATA" | jq -c '.images[]')
do
SERVICE_NAME=$(echo "$IMAGE_BUILD_DATA" | jq -r '.name // false')
SERVICE_NAME_TITLE=$(echo "$SERVICE_NAME" | tr '[:upper:]' '[:lower:]' | tr -d '-')
DOCKERFILE=$(echo "$IMAGE_BUILD_DATA" | jq -r '.imageBuild.dockerFile // false')
# if there is no dockerfile, then this image needs to be pulled from somewhere else
if [ $DOCKERFILE == "false" ]; then
PULL_IMAGE=$(echo "$IMAGE_BUILD_DATA" | jq -r '.imageBuild.pullImage // false')
if [ "$PULL_IMAGE" != "false" ]; then
IMAGES_PULL["${SERVICE_NAME}"]="${PULL_IMAGE}"
HAS_PULLED_IMAGES=true
fi
else
previousStepEnd=${currentStepEnd}
beginBuildStep "Building Image ${SERVICE_NAME}" "buildingImage${SERVICE_NAME_TITLE}"
# otherwise extract build information from the image build data payload
# this is a temporary image name to use for the build, it is based on the namespace and service, this can probably be deprecated and the images could just be
# built with the name they are meant to be. only 1 build can run at a time within a namespace
Expand Down Expand Up @@ -698,28 +700,50 @@ if [[ "$BUILD_TYPE" == "pullrequest" || "$BUILD_TYPE" == "branch" ]]; then

# adding the build image to the list of arguments passed into the next image builds
SERVICE_NAME_UPPERCASE=$(echo "$SERVICE_NAME" | tr '[:lower:]' '[:upper:]')
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "imageBuild${SERVICE_NAME_TITLE}Complete" "Building Image ${SERVICE_NAME}" "false"
fi
done
fi

# print information about built image sizes
function printBytes {
local -i bytes=$1;
echo "$(( (bytes + 1000000)/1000000 ))MB"
}
if [[ "${IMAGES_BUILD[@]}" ]]; then
echo "##############################################"
echo "Built image sizes:"
echo "##############################################"
if [ $HAS_PULLED_IMAGES == "true" ]; then
previousStepEnd=${currentStepEnd}
beginBuildStep "Pulled Images" "pulledImageInfo"
echo "The following images will be pulled and then pushed directly with no building required"
for IMAGE_NAME in "${!IMAGES_PULL[@]}"
do
PULL_IMAGE="${IMAGES_PULL[${IMAGE_NAME}]}" #extract the pull image name from the images to pull list
echo "- ${IMAGE_NAME}: ${PULL_IMAGE}"
done
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "pulledImageInfoComplete" "Pulled Images" "false"
fi

previousStepEnd=${currentStepEnd}
beginBuildStep "Image Build Stats" "imageBuildStats"

# print information about built image sizes
function printBytes {
local -i bytes=$1;
echo "$(( (bytes + 1000000)/1000000 ))MB"
}
if [[ "${IMAGES_BUILD[@]}" ]]; then
echo "##############################################"
echo "Built image sizes:"
echo "##############################################"
fi
for IMAGE_NAME in "${!IMAGES_BUILD[@]}"
do
TEMPORARY_IMAGE_NAME="${IMAGES_BUILD[${IMAGE_NAME}]}"
echo -e "- image: ${TEMPORARY_IMAGE_NAME}\t\t$(printBytes $(docker inspect ${TEMPORARY_IMAGE_NAME} | jq -r '.[0].Size'))"
done

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "imageBuildStatsComplete" "Image Build Stats" "false"
fi
if [[ "$BUILD_TYPE" == "promote" ]]; then
echo "No images built for promote environments"
fi
for IMAGE_NAME in "${!IMAGES_BUILD[@]}"
do
TEMPORARY_IMAGE_NAME="${IMAGES_BUILD[${IMAGE_NAME}]}"
echo -e "Image ${TEMPORARY_IMAGE_NAME}\t\t$(printBytes $(docker inspect ${TEMPORARY_IMAGE_NAME} | jq -r '.[0].Size'))"
done

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "imageBuildComplete" "Image Builds" "false"
previousStepEnd=${currentStepEnd}
beginBuildStep "Service Configuration Phase 1" "serviceConfigurationPhase1"

Expand Down Expand Up @@ -1212,8 +1236,6 @@ done

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "updateConfigmapComplete" "Update Configmap" "false"
previousStepEnd=${currentStepEnd}
beginBuildStep "Image Push to Registry" "pushingImages"

##############################################
### REDEPLOY DEPLOYMENTS IF CONFIG MAP CHANGES
Expand All @@ -1237,33 +1259,7 @@ declare -A DEPRECATED_IMAGE_SUGGESTION

# pullrequest/branch start
if [ "$BUILD_TYPE" == "pullrequest" ] || [ "$BUILD_TYPE" == "branch" ]; then

# All images that should be pulled are copied to the harbor registry
for IMAGE_NAME in "${!IMAGES_PULL[@]}"
do
PULL_IMAGE="${IMAGES_PULL[${IMAGE_NAME}]}" #extract the pull image name from the images to pull list
PUSH_IMAGE="${IMAGES_PUSH[${IMAGE_NAME}]}" #extract the push image name from the images to push list

# Try to handle private registries first

# the external pull image name is all calculated in the build-deploy tool now, it knows how to calculate it
# from being a promote image, or an image from an imagecache or from some other registry entirely
skopeo copy --retry-times 5 --dest-tls-verify=false docker://${PULL_IMAGE} docker://${PUSH_IMAGE}

# store the resulting image hash
SKOPEO_INSPECT=$(skopeo inspect --retry-times 5 docker://${PUSH_IMAGE} --tls-verify=false)
IMAGE_HASHES[${IMAGE_NAME}]=$(echo "${SKOPEO_INSPECT}" | jq ".Name + \"@\" + .Digest" -r)

# check if the pull through image is deprecated
DEPRECATED_STATUS=$(echo "${SKOPEO_INSPECT}" | jq -r '.Labels."sh.lagoon.image.deprecated.status" // false')
if [ "${DEPRECATED_STATUS}" != false ]; then
DEPRECATED_IMAGE_WARNINGS="true"
DEPRECATED_IMAGE_NAME[${IMAGE_NAME}]=${PULL_IMAGE#$IMAGECACHE_REGISTRY}
DEPRECATED_IMAGE_STATUS[${IMAGE_NAME}]=$DEPRECATED_STATUS
DEPRECATED_IMAGE_SUGGESTION[${IMAGE_NAME}]=$(echo "${SKOPEO_INSPECT}" | jq -r '.Labels."sh.lagoon.image.deprecated.suggested" | sub("docker.io\/";"")? // false')
fi
done

# calculate the images required to be pushed to the registry
for IMAGE_NAME in "${!IMAGES_BUILD[@]}"
do
PUSH_IMAGE="${IMAGES_PUSH[${IMAGE_NAME}]}" #extract the push image name from the images to push list
Expand All @@ -1286,6 +1282,8 @@ if [ "$BUILD_TYPE" == "pullrequest" ] || [ "$BUILD_TYPE" == "branch" ]; then
fi
done

previousStepEnd=${currentStepEnd}
beginBuildStep "Pushing Images" "pushingImages"
# If we have images to push to the registry, let's do so
if [ -f /kubectl-build-deploy/lagoon/push ]; then
parallel --retries 4 < /kubectl-build-deploy/lagoon/push
Expand All @@ -1298,25 +1296,57 @@ if [ "$BUILD_TYPE" == "pullrequest" ] || [ "$BUILD_TYPE" == "branch" ]; then
JQ_QUERY=(jq -r ".[]|select(test(\"${REGISTRY}/${PROJECT}/${ENVIRONMENT}/${IMAGE_NAME}@\"))")
IMAGE_HASHES[${IMAGE_NAME}]=$(docker inspect ${PUSH_IMAGE} --format '{{json .RepoDigests}}' | "${JQ_QUERY[@]}")
done
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "pushingImagesComplete" "Pushing Images" "false"

# All images that should be pulled are copied to the harbor registry
for IMAGE_NAME in "${!IMAGES_PULL[@]}"
do
PULL_IMAGE="${IMAGES_PULL[${IMAGE_NAME}]}" #extract the pull image name from the images to pull list
PUSH_IMAGE="${IMAGES_PUSH[${IMAGE_NAME}]}" #extract the push image name from the images to push list

# Try to handle private registries first
previousStepEnd=${currentStepEnd}
beginBuildStep "Pushing Pulled Image ${IMAGE_NAME}" "pushingImage${IMAGE_NAME}"
# the external pull image name is all calculated in the build-deploy tool now, it knows how to calculate it
# from being a promote image, or an image from an imagecache or from some other registry entirely
skopeo copy --retry-times 5 --dest-tls-verify=false docker://${PULL_IMAGE} docker://${PUSH_IMAGE}

# store the resulting image hash
SKOPEO_INSPECT=$(skopeo inspect --retry-times 5 docker://${PUSH_IMAGE} --tls-verify=false)
IMAGE_HASHES[${IMAGE_NAME}]=$(echo "${SKOPEO_INSPECT}" | jq ".Name + \"@\" + .Digest" -r)

# check if the pull through image is deprecated
DEPRECATED_STATUS=$(echo "${SKOPEO_INSPECT}" | jq -r '.Labels."sh.lagoon.image.deprecated.status" // false')
if [ "${DEPRECATED_STATUS}" != false ]; then
DEPRECATED_IMAGE_WARNINGS="true"
DEPRECATED_IMAGE_NAME[${IMAGE_NAME}]=${PULL_IMAGE#$IMAGECACHE_REGISTRY}
DEPRECATED_IMAGE_STATUS[${IMAGE_NAME}]=$DEPRECATED_STATUS
DEPRECATED_IMAGE_SUGGESTION[${IMAGE_NAME}]=$(echo "${SKOPEO_INSPECT}" | jq -r '.Labels."sh.lagoon.image.deprecated.suggested" | sub("docker.io\/";"")? // false')
fi
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "pushingImage${IMAGE_NAME}Complete" "Pushing Pulled Image ${IMAGE_NAME}" "false"
done

# pullrequest/branch end
# promote start
elif [ "$BUILD_TYPE" == "promote" ]; then

for IMAGE_NAME in "${IMAGES[@]}"
do
previousStepEnd=${currentStepEnd}
beginBuildStep "Pushing Pulled Promote Image ${IMAGE_NAME}" "pushingImage${IMAGE_NAME}"
PUSH_IMAGE="${IMAGES_PUSH[${IMAGE_NAME}]}" #extract the push image name from the images to push list
PROMOTE_IMAGE="${IMAGES_PROMOTE[${IMAGE_NAME}]}" #extract the push image name from the images to push list
skopeo copy --retry-times 5 --src-tls-verify=false --dest-tls-verify=false docker://${PROMOTE_IMAGE} docker://${PUSH_IMAGE}

IMAGE_HASHES[${IMAGE_NAME}]=$(skopeo inspect --retry-times 5 docker://${PUSH_IMAGE} --tls-verify=false | jq ".Name + \"@\" + .Digest" -r)
currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "pushingImage${IMAGE_NAME}Complete" "Pushing Pulled Promote Image ${IMAGE_NAME}" "false"
done
# promote end
fi

currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")"
patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "imagePushComplete" "Image Push to Registry" "false"

##############################################
### Check for deprecated images
##############################################
Expand Down
Loading