skopeo-copy: Allow multiple images to be copied

openshift-pipelines · Oct 1, 2024 · fba6974 · fba6974
1 parent 9d12f07
commit fba6974
Show file tree

Hide file tree

Showing 9 changed files with 157 additions and 43 deletions.
diff --git a/Makefile b/Makefile
@@ -66,7 +66,7 @@ E2E_S2I_LANGUAGE ?= python
 E2E_S2I_IMAGE_SCRIPTS_URL ?= image:///usr/libexec/s2i
 
 # s2i end-to-end test pipeline params adding env variables as a comma-separated string
-E2E_S2I_PARAMS_ENV_VARS ?= 
+E2E_S2I_PARAMS_ENV_VARS ?=
 
 # path to the github actions testing workflows
 ACT_WORKFLOWS ?= ./.github/workflows/test.yaml
@@ -146,7 +146,7 @@ helm-package: clean
 
 # prepares the buildah end-to-end tests, installs the required resources
 .PHONY: prepare-e2e-buildah
-prepare-e2e-buildah: 
+prepare-e2e-buildah:
 	kubectl apply -f $(E2E_BUILDAH_TASK_CONTAINERFILE_STUB)
 
 # runs bats-core against the pre-determined tests

diff --git a/docs/task-skopeo-copy.md b/docs/task-skopeo-copy.md
@@ -39,22 +39,66 @@ kubectl create secret docker-registry imagestreams \
 
 Then make sure the Secret is linked with the Service-Account running the `TaskRun`/`PipelineRun`.
 
+In case multiple images need to be copied, or one image under multiple names/tags, a `url.txt` file can be provided:
+
+```text
+docker://docker.io/library/busybox:latest docker://image-registry.openshift-image-registry.svc.cluster.local:5000/my-project/busybox:latest
+docker://docker.io/library/busybox:1 docker://image-registry.openshift-image-registry.svc.cluster.local:5000/my-project/busybox:1
+```
+
+
+This file has to be present at the root and mounted under the workspace `images_url`, for example using a ConfigMap:
+```yaml
+---
+apiVersion: tekton.dev/v1beta1
+kind: TaskRun
+metadata: {}
+spec:
+  taskRef:
+    name: skopeo-copy
+  workspaces:
+    - name: images_url
+      configmap:
+        name: configmap-images
+```
+
+Referenced config map:
+
+```yaml
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: configmap-images
+data:
+  url.txt: |
+    docker://docker.io/library/busybox:latest docker://image-registry.openshift-image-registry.svc.cluster.local:5000/my-project/busybox:latest
+    docker://docker.io/library/busybox:1 docker://image-registry.openshift-image-registry.svc.cluster.local:5000/my-project/busybox:1
+
+```
+
+## Workspace
+
+| Name         | Optional | Description                                                                                                                                                                                |
+|:-------------|:--------:|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `images_url` |  `true`  | For storing image urls in case of more than one image to copy. It must have a url.txt file at the root path containing a source and a destination image separated by a space on each line. |
+
 ## Params
 
-| Param         | Type                       | Default                      | Description                |
-| :------------ | :------------------------: | :--------------------------- | :------------------------- |
-| `SOURCE_IMAGE_URL` | `string` | (required) | Fully qualified source container image name, including tag, to be copied into `DESTINATION_IMAGE_URL` param. |
-| `DESTINATION_IMAGE_URL` | `string` | (required) | Fully qualified destination container image name, including tag. |
-| `SRC_TLS_VERIFY` | `string` | `true` | Sets the TLS verification flags for the source registry, `true` is recommended. |
-| `DEST_TLS_VERIFY` | `string` | `true` | Sets the TLS verification flags for the destination registry, `true` is recommended. |
-| `VERBOSE` | `string` | `false` | Shows a more verbose (debug) output. |
+| Param         | Type                       | Default                      | Description                                                                                                                                                          |
+| :------------ | :------------------------: | :---------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `SOURCE_IMAGE_URL` | `string` | "" | Fully qualified source container image name, including tag, to be copied into `DESTINATION_IMAGE_URL` param. If more than one image needs to be copied, leave empty. |
+| `DESTINATION_IMAGE_URL` | `string` | "" | Fully qualified destination container image name, including tag. If more than one image needs to be copied, leave empty.                                                                                                    |
+| `SRC_TLS_VERIFY` | `string` | `true` | Sets the TLS verification flags for the source registry, `true` is recommended.                                                                                      |
+| `DEST_TLS_VERIFY` | `string` | `true` | Sets the TLS verification flags for the destination registry, `true` is recommended.                                                                                 |
+| `VERBOSE` | `string` | `false` | Shows a more verbose (debug) output.                                                                                                                                 |
 
 ## Results
 
-| Result        | Description                |
-| :------------ | :------------------------- |
-| `SOURCE_DIGEST` | Source image SHA256 digest. |
-| `DESTINATION_DIGEST` | Destination image SHA256 digest. |
+| Result        | Description                                                                                                                                                                        |
+| :------------ |:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `SOURCE_DIGEST` | If a single image is copied it contains a source image SHA256 digest. When copying multiple images it contains SHA256 digests of all source images separated by a space.           |
+| `DESTINATION_DIGEST` | If a single image is copied it contains a destination image SHA256 digest. When copying multiple images it contains SHA256 digests of all destination images separated by a space. |
 
 [tektonPipelineAuth]: https://tekton.dev/docs/pipelines/auth/#configuring-docker-authentication-for-docker
 [containersSkopeo]: https://github.com/containers/skopeo
diff --git a/scripts/common.sh b/scripts/common.sh
@@ -16,7 +16,7 @@ function phase() {
     echo "---> Phase: ${*}..."
 }
 
-# assert local variables are exporeted on the environment
+# assert local variables are exported on the environment
 function exported_or_fail() {
     declare -a _required_vars="${@}"
 

diff --git a/scripts/skopeo-common.sh b/scripts/skopeo-common.sh
@@ -17,11 +17,9 @@ declare -rx RESULTS_DESTINATION_DIGEST_PATH="${RESULTS_DESTINATION_DIGEST_PATH:-
 #
 
 exported_or_fail \
-    PARAMS_SOURCE_IMAGE_URL \
-    PARAMS_DESTINATION_IMAGE_URL \
     RESULTS_SOURCE_DIGEST_PATH \
     RESULTS_DESTINATION_DIGEST_PATH
-     
+
 
 #
 # Skopeo Authentication

diff --git a/scripts/skopeo-copy.sh b/scripts/skopeo-copy.sh
@@ -9,32 +9,30 @@ source "$(dirname "${BASH_SOURCE[0]}")/skopeo-common.sh"
 # Ensure the /tekton/home/.docker directory exists
 mkdir -p /workspace/home/.docker
 
-phase "Copying '${PARAMS_SOURCE_IMAGE_URL}' into '${PARAMS_DESTINATION_IMAGE_URL}'"
-
 set -x
 
 if [ -n "${PARAMS_SOURCE_IMAGE_URL}" ] && [ -n "${PARAMS_DESTINATION_IMAGE_URL}" ]; then
+    phase "Copying '${PARAMS_SOURCE_IMAGE_URL}' into '${PARAMS_DESTINATION_IMAGE_URL}'"
     skopeo copy ${SKOPEO_DEBUG_FLAG} \
         --src-tls-verify="${PARAMS_SRC_TLS_VERIFY}" \
         --dest-tls-verify="${PARAMS_DEST_TLS_VERIFY}" \
         "${PARAMS_SOURCE_IMAGE_URL}" \
         "${PARAMS_DESTINATION_IMAGE_URL}"
-else
+elif [ "${WORKSPACES_IMAGES_URL_BOUND}" == "true" ]; then
+    phase "Copying using url.txt file"
     # Function to copy multiple images.
     copyimages() {
         filename="${WORKSPACES_IMAGES_URL_PATH}/url.txt"
+        [[ ! -f "${filename}" ]] && fail "url.txt file not found at: '${filename}'"
         while IFS= read -r line || [ -n "$line" ]
         do
-            cmd=""
-            for url in $line
-            do
-                cmd="$cmd $url"
-            done
-            read -ra SOURCE <<<"${cmd}"
-            skopeo copy "${SOURCE[@]}" --src-tls-verify="${PARAMS_SRC_TLS_VERIFY}" --dest-tls-verify="${PARAMS_DEST_TLS_VERIFY}"
-            echo "$cmd"
+            read -ra SOURCE <<<"${line}"
+            skopeo copy "${SOURCE[@]}" ${SKOPEO_DEBUG_FLAG} --src-tls-verify="${PARAMS_SRC_TLS_VERIFY}" --dest-tls-verify="${PARAMS_DEST_TLS_VERIFY}"
+            echo "$line"
         done < "$filename"
     }
 
     copyimages
+else
+  fail "Neither Source/Destination image URL parameters nor workspace images_url provided"
 fi
diff --git a/scripts/skopeo-results.sh b/scripts/skopeo-results.sh
@@ -15,13 +15,31 @@ function skopeo_inspect() {
         "${image}"
 }
 
-phase "Extracting '${PARAMS_SOURCE_IMAGE_URL}' source image digest"
-source_digest="$(skopeo_inspect "${PARAMS_SOURCE_IMAGE_URL}" "${PARAMS_SRC_TLS_VERIFY}")"
-phase "Source image digest '${source_digest}'"
+if [ -n "${PARAMS_SOURCE_IMAGE_URL}" ] && [ -n "${PARAMS_DESTINATION_IMAGE_URL}" ]; then
+    phase "Extracting '${PARAMS_SOURCE_IMAGE_URL}' source image digest"
+    source_digest="$(skopeo_inspect "${PARAMS_SOURCE_IMAGE_URL}" "${PARAMS_SRC_TLS_VERIFY}")"
+    phase "Source image digest '${source_digest}'"
 
-phase "Extracting '${PARAMS_DESTINATION_IMAGE_URL}' destination image digest"
-destination_digest="$(skopeo_inspect "${PARAMS_DESTINATION_IMAGE_URL}" "${PARAMS_DEST_TLS_VERIFY}")"
-phase "Destination image digest '${destination_digest}'"
+    phase "Extracting '${PARAMS_DESTINATION_IMAGE_URL}' destination image digest"
+    destination_digest="$(skopeo_inspect "${PARAMS_DESTINATION_IMAGE_URL}" "${PARAMS_DEST_TLS_VERIFY}")"
+    phase "Destination image digest '${destination_digest}'"
+else
+    phase "Extracting source and destination image digests for images from url.txt file"
+    filename="${WORKSPACES_IMAGES_URL_PATH}/url.txt"
+    source_digest=""
+    destination_digest=""
+    while IFS= read -r line || [ -n "$line" ]
+    do
+        read -ra SOURCE <<<"${line}"
+        source_digest="$source_digest $(skopeo_inspect ${SOURCE[0]} ${PARAMS_SRC_TLS_VERIFY})"
+        destination_digest="$destination_digest $(skopeo_inspect ${SOURCE[1]} ${PARAMS_DEST_TLS_VERIFY})"
+    done < "$filename"
+    # Remove whitespace from the start
+    source_digest="${source_digest# }"
+    destination_digest="${destination_digest# }"
+    phase "Source image digests '${source_digest}'"
+    phase "Destination image digests '${destination_digest}'"
+fi
 
 printf "%s" "${source_digest}" > "${RESULTS_SOURCE_DIGEST_PATH}"
 printf "%s" "${destination_digest}" > "${RESULTS_DESTINATION_DIGEST_PATH}"
diff --git a/templates/task-skopeo-copy.yaml b/templates/task-skopeo-copy.yaml
@@ -21,15 +21,18 @@ spec:
     - name: images_url
       optional: true
       description: |
-        For storing image urls in case we have more than one image to copy.
+        For storing image urls in case of more than one image to copy. It must have a url.txt file at the root path
+        containing a source and a destination image separated by a space on each line.
   params:
     - name: SOURCE_IMAGE_URL
       type: string
+      default: ""
       description: |
         Fully qualified source container image name, including tag, to be copied
         into `DESTINATION_IMAGE_URL` param.
     - name: DESTINATION_IMAGE_URL
       type: string
+      default: ""
       description: |
         Fully qualified destination container image name, including tag.
     - name: SRC_TLS_VERIFY
@@ -61,7 +64,7 @@ spec:
   volumes:
     - name: scripts-dir
       emptyDir: {}
-    
+
   stepTemplate:
     env:
 {{- $variables := list

diff --git a/test/e2e/e2e-skopeo-copy.bats b/test/e2e/e2e-skopeo-copy.bats
@@ -32,12 +32,9 @@ declare -rx E2E_SC_PARAMS_DESTINATION_IMAGE_URL="${E2E_SC_PARAMS_DESTINATION_IMA
         --showlog
     assert_success
 
-    # waiting a few seconds before asserting results
-	sleep 30
-
-    # assering the taskrun status, making sure all steps have been successful
+    # asserting the taskrun status, making sure all steps have been successful
     assert_tekton_resource "taskrun" --partial 'All Steps have completed executing'
-    # asserting the latest taskrun instacne to inspect the resources against a regular expression
+    # asserting the latest taskrun instance to inspect the resources against a regular expression
     assert_tekton_resource "taskrun" --regexp $'\S+\n?DESTINATION_DIGEST=\S+\nSOURCE_DIGEST=\S+'
 }
 
@@ -68,8 +65,54 @@ declare -rx E2E_SC_PARAMS_DESTINATION_IMAGE_URL="${E2E_SC_PARAMS_DESTINATION_IMA
         --showlog
     assert_success
 
-    # waiting a few seconds before asserting results
-	sleep 100
+    # assering the taskrun status, making sure all steps have been successful
+    assert_tekton_resource "taskrun" --partial 'All Steps have completed executing'
+    # asserting the latest taskrun instacne to inspect the resources against a regular expression
+    assert_tekton_resource "taskrun" --regexp $'\S+\n?DESTINATION_DIGEST=\S+\nSOURCE_DIGEST=\S+'
+}
+
+# Testing the skopeo-copy task, copying multiple images using url.txt file, verbose = true
+@test "[e2e] skopeo-copy task copying multiple images from source to destination registry with verbose set to true" {
+    # asserting all required configuration is informed
+	[ -n "${E2E_SC_PARAMS_SOURCE_IMAGE_URL}" ]
+    [ -n "${E2E_SC_PARAMS_DESTINATION_IMAGE_URL}" ]
+    [ -n "${E2E_PARAMS_SRC_TLS_VERIFY}" ]
+    [ -n "${E2E_PARAMS_DEST_TLS_VERIFY}" ]
+
+    E2E_SC_CONFIG_MAP_TEMPLATE_PATH="./test/e2e/resources/configmap-images-template.yaml"
+    E2E_SC_CONFIG_MAP_PATH="${BASE_DIR}/configmap-images.yaml"
+
+    # cleaning up all the existing resources before starting a new taskrun, the test assertion
+	  # will describe the objects on the current namespace
+    run kubectl delete taskrun --all
+    assert_success
+
+    # copying the config map template to the bats temporary directory
+    run cp "${E2E_SC_CONFIG_MAP_TEMPLATE_PATH}" "${E2E_SC_CONFIG_MAP_PATH}"
+    assert_success
+
+    # filter the template with source and destination images
+    run sed -i "s|E2E_SC_PARAMS_SOURCE_IMAGE_URL|${E2E_SC_PARAMS_SOURCE_IMAGE_URL}|" "${E2E_SC_CONFIG_MAP_PATH}"
+    assert_success
+    run sed -i "s|E2E_SC_PARAMS_DESTINATION_IMAGE_URL|${E2E_SC_PARAMS_DESTINATION_IMAGE_URL}|" "${E2E_SC_CONFIG_MAP_PATH}"
+    assert_success
+
+    # create a new config map
+    run kubectl apply -f "${E2E_SC_CONFIG_MAP_PATH}"
+    assert_success
+
+    #
+    # E2E TaskRun
+    #
+
+    run tkn task start skopeo-copy \
+        --param="SRC_TLS_VERIFY=${E2E_PARAMS_SRC_TLS_VERIFY}" \
+        --param="DEST_TLS_VERIFY=${E2E_PARAMS_DEST_TLS_VERIFY}" \
+        --param="VERBOSE=true" \
+        --workspace name=images_url,config=configmap-images \
+        --use-param-defaults \
+        --showlog
+    assert_success
 
     # assering the taskrun status, making sure all steps have been successful
     assert_tekton_resource "taskrun" --partial 'All Steps have completed executing'

diff --git a/test/e2e/resources/configmap-images-template.yaml b/test/e2e/resources/configmap-images-template.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: configmap-images
+data:
+  url.txt: |
+    E2E_SC_PARAMS_SOURCE_IMAGE_URL E2E_SC_PARAMS_DESTINATION_IMAGE_URL
+    E2E_SC_PARAMS_SOURCE_IMAGE_URL E2E_SC_PARAMS_DESTINATION_IMAGE_URL
+    E2E_SC_PARAMS_SOURCE_IMAGE_URL E2E_SC_PARAMS_DESTINATION_IMAGE_URL