From 086475dad448e6e2ae3082fbe092f4664727be92 Mon Sep 17 00:00:00 2001
From: Paul Fouquet <pfouquet@linz.govt.nz>
Date: Wed, 22 May 2024 16:09:49 +1200
Subject: [PATCH] refactor: allow stac-validate-parallel to be used as
 templateRef

---
 workflows/cron/README.md                      |  2 +-
 .../cron/cron-stac-validate-all-data.yaml     | 18 ++++++++--
 workflows/stac/stac-validate-parallel.yaml    | 35 ++++++++++++++++---
 3 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/workflows/cron/README.md b/workflows/cron/README.md
index 86c2bd89f..4ecf0bbb5 100644
--- a/workflows/cron/README.md
+++ b/workflows/cron/README.md
@@ -22,4 +22,4 @@ It also validate that the [STAC assets](https://github.com/radiantearth/stac-spe
 
 > **_NOTE:_** Due to the parallelism design, this workflow does not validate the root parent `catalog.json` in order to validate each `collection.json` separately. This is not an issue as the `catalog.json` does not contain any `asset` and is already validated by the [cron-stac-validata](#cron-stac-validate) job.
 
-- schedule: **TBD**
+- schedule: **every 1st of the month**
diff --git a/workflows/cron/cron-stac-validate-all-data.yaml b/workflows/cron/cron-stac-validate-all-data.yaml
index 2926cb4b2..d139342fe 100644
--- a/workflows/cron/cron-stac-validate-all-data.yaml
+++ b/workflows/cron/cron-stac-validate-all-data.yaml
@@ -2,7 +2,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: CronWorkflow
 metadata:
-  name: test-cron-stac-validate-all-data
+  name: cron-stac-validate-all-data
   labels:
     linz.govt.nz/category: stac
 spec:
@@ -17,6 +17,10 @@ spec:
     entrypoint: main
     arguments:
       parameters:
+        - name: version_argo_tasks
+          value: 'v4'
+        - name: 'include'
+          value: 'collection.json$'
         - name: checksum_assets
           value: 'true'
         - name: 'checksum_links'
@@ -32,8 +36,12 @@ spec:
                 template: main
               arguments:
                 parameters:
+                  - name: version_argo_tasks
+                    value: '{{workflow.parameters.version_argo_tasks}}'
                   - name: 'uri'
-                    value: 's3://nz-imageryfails/'
+                    value: 's3://nz-imagery/'
+                  - name: include
+                    value: '{{workflow.parameters.include}}'
                   - name: 'checksum_assets'
                     value: '{{workflow.parameters.checksum_assets}}'
                   - name: 'checksum_links'
@@ -44,8 +52,12 @@ spec:
                 template: main
               arguments:
                 parameters:
+                  - name: version_argo_tasks
+                    value: '{{workflow.parameters.version_argo_tasks}}'
                   - name: 'uri'
-                    value: 's3://nz-elevationfails/'
+                    value: 's3://nz-elevation/'
+                  - name: include
+                    value: '{{workflow.parameters.include}}'
                   - name: 'checksum_assets'
                     value: '{{workflow.parameters.checksum_assets}}'
                   - name: 'checksum_links'
diff --git a/workflows/stac/stac-validate-parallel.yaml b/workflows/stac/stac-validate-parallel.yaml
index d0d75b57b..ab949ba7d 100644
--- a/workflows/stac/stac-validate-parallel.yaml
+++ b/workflows/stac/stac-validate-parallel.yaml
@@ -38,12 +38,32 @@ spec:
       image: ''
   templates:
     - name: main
+      inputs:
+        parameters:
+          - name: version_argo_tasks
+            value: '{{workflow.parameters.version_argo_tasks}}'
+          - name: include
+            value: '{{workflow.parameters.include}}'
+          - name: uri
+            value: '{{workflow.parameters.uri}}'
+          - name: checksum_assets
+            value: '{{workflow.parameters.checksum_assets}}'
+          - name: checksum_links
+            value: '{{workflow.parameters.checksum_links}}'
       retryStrategy:
         limit: '0' # avoid retrying any of the following task as `tpl-at-stac-validate` already retries its own tasks.
       dag:
         tasks:
           - name: aws-list-collections
             template: aws-list-collections
+            arguments:
+              parameters:
+                - name: version_argo_tasks
+                  value: '{{inputs.parameters.version_argo_tasks}}'
+                - name: include
+                  value: '{{inputs.parameters.include}}'
+                - name: uri
+                  value: '{{inputs.parameters.uri}}'
           - name: stac-validate-collections
             templateRef:
               name: tpl-at-stac-validate
@@ -53,16 +73,21 @@ spec:
                 - name: uri
                   value: '{{item}}'
                 - name: checksum_assets
-                  value: '{{workflow.parameters.checksum_assets}}'
+                  value: '{{inputs.parameters.checksum_assets}}'
                 - name: checksum_links
-                  value: '{{workflow.parameters.checksum_links}}'
+                  value: '{{inputs.parameters.checksum_links}}'
             depends: aws-list-collections
             withParam: '{{tasks.aws-list-collections.outputs.parameters.files}}'
     - name: aws-list-collections
+      inputs:
+        parameters:
+          - name: version_argo_tasks
+          - name: include
+          - name: uri
       retryStrategy:
         limit: '2' # force retrying this specific task
       container:
-        image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}'
+        image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version_argo_tasks)}}'
         command: [node, /app/index.js]
         env:
           - name: AWS_ROLE_CONFIG_PATH
@@ -72,12 +97,12 @@ spec:
             'list',
             '--verbose',
             '--include',
-            '{{=sprig.trim(workflow.parameters.include)}}',
+            '{{=sprig.trim(inputs.parameters.include)}}',
             '--group',
             '1',
             '--output',
             '/tmp/file_list.json',
-            '{{=sprig.trim(workflow.parameters.uri)}}',
+            '{{=sprig.trim(inputs.parameters.uri)}}',
           ]
       outputs:
         parameters: