wip: cron workflows to validate ODR buckets STAC

linz · May 13, 2024 · 955b3f4 · 955b3f4
1 parent 71b3396
commit 955b3f4
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 8 deletions.
diff --git a/workflows/cron/cron-stac-validate-nz-elevation.yaml b/workflows/cron/cron-stac-validate-nz-elevation.yaml
@@ -6,7 +6,7 @@ metadata:
   labels:
     linz.govt.nz/category: stac
 spec:
-  schedule: '* * * * *'
+  schedule: '0 07 * * *' # 7 AM every day
   timezone: 'NZ'
   startingDeadlineSeconds: 3600 # Allow 1 hour delay if the workflow-controller clashes during the starting time.
   concurrencyPolicy: 'Allow'
@@ -15,12 +15,39 @@ spec:
   suspend: false
   workflowSpec:
     workflowTemplateRef:
-      name: stac-validate-parallel
+      name: tpl-at-stac-validate
+    entrypoint: main
     arguments:
       parameters:
         - name: 'uri'
           value: 's3://nz-elevation/'
-        - name: 'checksum'
+        - name: 'checksum_assets'
           value: 'false'
         - name: 'checksum_links'
           value: 'true'
+---
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+  name: cron-stac-validate-nz-elevation-all-checksums
+  labels:
+    linz.govt.nz/category: stac
+spec:
+  schedule: '0 07 1 * *' # 7 AM every 1st of the month 
+  timezone: 'NZ'
+  startingDeadlineSeconds: 3600 # Allow 1 hour delay if the workflow-controller clashes during the starting time.
+  concurrencyPolicy: 'Allow'
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  suspend: false
+  workflowSpec:
+    workflowTemplateRef:
+      name: stac-validate-parallel
+    arguments:
+      parameters:
+        - name: 'uri'
+          value: 's3://nz-elevation/'
+        - name: 'checksum_assets'
+          value: 'true'
+        - name: 'checksum_links'
+          value: 'true'
diff --git a/workflows/cron/cron-stac-validate-nz-imagery.yaml b/workflows/cron/cron-stac-validate-nz-imagery.yaml
@@ -6,7 +6,7 @@ metadata:
   labels:
     linz.govt.nz/category: stac
 spec:
-  schedule: '* * * * *'
+  schedule: '0 07 * * *' # 7 AM every day
   timezone: 'NZ'
   startingDeadlineSeconds: 3600 # Allow 1 hour delay if the workflow-controller clashes during the starting time.
   concurrencyPolicy: 'Allow'
@@ -15,12 +15,39 @@ spec:
   suspend: false
   workflowSpec:
     workflowTemplateRef:
-      name: stac-validate-parallel
+      name: tpl-at-stac-validate
+    entrypoint: main
     arguments:
       parameters:
         - name: 'uri'
           value: 's3://nz-imagery/'
-        - name: 'checksum'
+        - name: 'checksum_assets'
           value: 'false'
         - name: 'checksum_links'
           value: 'true'
+---
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+  name: cron-stac-validate-nz-imagery-all-checksums
+  labels:
+    linz.govt.nz/category: stac
+spec:
+  schedule: '0 07 1 * *' # 7 AM every 1st of the month 
+  timezone: 'NZ'
+  startingDeadlineSeconds: 3600 # Allow 1 hour delay if the workflow-controller clashes during the starting time.
+  concurrencyPolicy: 'Allow'
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  suspend: false
+  workflowSpec:
+    workflowTemplateRef:
+      name: stac-validate-parallel
+    arguments:
+      parameters:
+        - name: 'uri'
+          value: 's3://nz-imagery/'
+        - name: 'checksum_assets'
+          value: 'true'
+        - name: 'checksum_links'
+          value: 'true'
diff --git a/workflows/stac/stac-validate-parallel.yaml b/workflows/stac/stac-validate-parallel.yaml
@@ -3,7 +3,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: WorkflowTemplate
 metadata:
-  name: stac-validate-parallel
+  name: test-stac-validate-parallel
   labels:
     linz.govt.nz/category: stac
 spec:
@@ -21,7 +21,7 @@ spec:
       - name: include
         value: 'collection.json$'
       - name: checksum_assets
-        description: 'Validate the file:checksum of each asset if it exists'
+        description: 'Validate asset checksums.'
         value: 'false'
         enum:
           - 'true'
@@ -32,6 +32,12 @@ spec:
         enum:
           - 'false'
           - 'true'
+      - name: checksum_links
+        description: 'Validate links checksums.'
+        value: 'false'
+        enum:
+          - 'false'
+          - 'true'
   templateDefaults:
     container:
       imagePullPolicy: Always