From d3db2c3755647b20c90edd8b41a863058ce62c7b Mon Sep 17 00:00:00 2001
From: Wentao Kuang <wkuang@linz.govt.nz>
Date: Mon, 25 Mar 2024 14:58:51 +1300
Subject: [PATCH] feat(basemaps): Update the ETL to support the new version
 with individual layer. BM-977 (#493)

#### Motivation

The new ETL version contains a filename which can used for individual
layer processing, and default to topographic.

#### Modification

- Update the ETL workflow to v1 which is the current latest version
- Add the filename parameter to workflow
- Rename the `cron-vector-etl` workflow to `cron-vector-etl-topographic`
workflow
- Update the ETL entry command to `node /app/index.js` which been
removed from the ETL container.


#### Checklist
- [ ] Tests updated - No test for workflow, already running the workflow
in argo for testing.
- [ ] Docs updated - No doc for vector etl to update
- [x] Issue linked in Title
---
 workflows/basemaps/vector-etl.yaml  | 11 ++++++++---
 workflows/cron/cron-vector-etl.yaml |  4 +++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/workflows/basemaps/vector-etl.yaml b/workflows/basemaps/vector-etl.yaml
index b68b6a177..368cae8ec 100644
--- a/workflows/basemaps/vector-etl.yaml
+++ b/workflows/basemaps/vector-etl.yaml
@@ -18,7 +18,7 @@ spec:
       
       - name: version_basemaps_etl
         description: Version of the Basemaps ETL eks container to use
-        value: 'v1.0.0'
+        value: 'v1'
 
       - name: target
         description: S3 Bucket to use for storing the output
@@ -27,6 +27,10 @@ spec:
           - 'linz-basemaps'
           - 'linz-basemaps-staging'
 
+      - name: filename
+        description: output filename in s3
+        value: 'topographic'
+
       - name: create_pull_request
         description: Should a pull request be created in linz/basemaps-config
         value: 'true'
@@ -71,13 +75,14 @@ spec:
           requests:
             memory: 7.8Gi
             cpu: 15000m
-        command: [node, index.cjs]
+        command: [node, /app/index.js]
         env:
           - name: AWS_ROLE_CONFIG_PATH
             value: s3://linz-bucket-config/config.basemaps.json
         args:
-          - '--all'
+          - "{{= workflow.parameters.filename == 'topographic'? '--all' : '--layer=' + workflow.parameters.filename }}"
           - '--target={{ workflow.parameters.target }}'
+          - '--filename={{ workflow.parameters.filename }}'
           - '--output=/tmp/'
           - '--commit'
       outputs:
diff --git a/workflows/cron/cron-vector-etl.yaml b/workflows/cron/cron-vector-etl.yaml
index 52af9e7b0..8f9fea8eb 100644
--- a/workflows/cron/cron-vector-etl.yaml
+++ b/workflows/cron/cron-vector-etl.yaml
@@ -2,7 +2,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: CronWorkflow
 metadata:
-  name: cron-vector-etl
+  name: cron-vector-etl-topographic
   namespace: argo
   labels:
     linz.govt.nz/category: basemaps
@@ -20,5 +20,7 @@ spec:
       name: basemaps-vector-etl
     arguments:
       parameters:
+        - filename:
+          value: 'topographic'
         - name: 'retry'
           value: '2'