Merge branch 'master' into feat/stac-format-tde-759

.eslintrc.cjs

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require('@linzjs/style/.eslintrc.cjs'),
+};

.github/pull_request_template.md

@@ -0,0 +1,15 @@
+#### Motivation
+
+_What does this change aim to achieve?_
+
+#### Modification
+
+_Why is this change being made? What implications or other considerations are there?_
+
+#### Checklist
+
+_If not applicable, provide explanation of why._
+
+- [ ] Tests updated
+- [ ] Docs updated
+- [ ] Issue linked in Title

.github/workflows/main.yml

@@ -14,69 +14,72 @@ jobs:
       - name: Use Node.js 18.x
         uses: actions/[email protected]
         with:
-          node-version: "18.x"
+          node-version: '18.x'
 
       - name: Install
-        run: yarn
+        run: npm install --ci
 
       - name: format
-        run: yarn format
+        run: npm run lint -- --fix=false # ensure eslint is not configured to --fix
 
-      - name: Check formatting
-        run: git diff --exit-code
-
-      - name: Setup kubectl
-        uses: azure/setup-kubectl@v3
-        with:
-          version: "latest"
-
-      - name: AWS Configure
-        uses: aws-actions/[email protected]
-        with:
-          aws-region: ap-southeast-2
-          mask-aws-account-id: true
-          role-to-assume: ${{ secrets.AWS_CI_ROLE }}
-
-      - name: Login to EKS
-        run: |
-          aws eks update-kubeconfig --name Workflow --region ap-southeast-2 --role-arn ${{ secrets.AWS_EKS_ROLE }}
-
-      - name: Check EKS connection
-        run: |
-          kubectl get nodes
-
-      - name: Install Argo
-        run: |
-          curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.4.0-rc2/argo-linux-amd64.gz
-          gunzip argo-linux-amd64.gz
-          chmod +x argo-linux-amd64
-          ./argo-linux-amd64 version
-
-      - name: Lint workflows
-        run: |
-          ./argo-linux-amd64 lint templates/ -n argo
-          ./argo-linux-amd64 lint workflows/ -n argo
-
-      - name: Deploy workflows
-        if: github.ref == 'refs/heads/master'
-        run: |
-          # Deploy templates first
-          kubectl apply -f templates/argo-tasks/ --namespace argo
-
-          # Find all workflows that have kind "WorkflowTemplate"
-          WORKFLOWS=$(grep '^kind: WorkflowTemplate$' -R workflows/ -H | cut -d ':' -f1)
-
-          # For each workflow attempt to deploy it using kubectl
-          for wf in $WORKFLOWS; do
-              kubectl apply -f $wf --namespace argo
-          done
-
-      - name: Deploy config files
-        if: github.ref == 'refs/heads/master'
-        run: |
-          kubectl apply -f config/*.yml --namespace argo
-      # Re-lint the workflows after deployment because their dependencies is test with deployement.
-      - name: Re-Lint workflows
-        run: |
-          ./argo-linux-amd64 lint templates/ -n argo
-          ./argo-linux-amd64 lint workflows/ -n argo
+      - name: format
+        run: npm run format -- --fix=false # ensure eslint is not configured to --fix
+
+
+      # - name: Setup kubectl
+      #   uses: azure/setup-kubectl@v3
+      #   with:
+      #     version: 'latest'
+
+      # - name: AWS Configure
+      #   uses: aws-actions/[email protected]
+      #   with:
+      #     aws-region: ap-southeast-2
+      #     mask-aws-account-id: true
+      #     role-to-assume: ${{ secrets.AWS_CI_ROLE }}
+
+      # - name: Login to EKS
+      #   run: |
+      #     aws eks update-kubeconfig --name Workflow --region ap-southeast-2 --role-arn ${{ secrets.AWS_EKS_ROLE }}
+
+      # - name: Check EKS connection
+      #   run: |
+      #     kubectl get nodes
+
+      # - name: Install Argo
+      #   run: |
+      #     curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.4.0-rc2/argo-linux-amd64.gz
+      #     gunzip argo-linux-amd64.gz
+      #     chmod +x argo-linux-amd64
+      #     ./argo-linux-amd64 version
+
+      # - name: Lint workflows
+      #   run: |
+      #     ./argo-linux-amd64 lint templates/ -n argo
+      #     ./argo-linux-amd64 lint workflows/ -n argo
+
+      # - name: Deploy workflows
+      #   if: github.ref == 'refs/heads/master'
+      #   run: |
+      #     # Deploy templates first
+      #     kubectl apply -f templates/argo-tasks/ --namespace argo
+
+      #     # Find all workflows that have kind "WorkflowTemplate"
+      #     WORKFLOWS=$(grep '^kind: WorkflowTemplate$' -R workflows/ -H | cut -d ':' -f1)
+
+      #     # For each workflow attempt to deploy it using kubectl
+      #     for wf in $WORKFLOWS; do
+      #         kubectl apply -f $wf --namespace argo
+      #     done
+
+      # - name: Deploy config files
+      #   if: github.ref == 'refs/heads/master'
+      #   run: |
+      #     npx cdk8s synth
+      #     kubectl apply -f dist/
+
+      # # Re-lint the workflows after deployment because their dependencies is test with deployement.
+      # - name: Re-Lint workflows
+      #   run: |
+      #     ./argo-linux-amd64 lint templates/ -n argo
+      #     ./argo-linux-amd64 lint workflows/ -n argo

.github/workflows/pull-request.yml

@@ -0,0 +1,16 @@
+name: Pull Request lint
+
+on:
+  pull_request:
+    types: ['opened', 'edited', 'reopened', 'synchronize']
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: linz/action-pull-request-lint@v1
+        with:
+          conventional: 'error' # require conventional pull request title (default: "error" options: "error", "warn", "off")
+
+          jira: 'warn' # Require JIRA ticket references (default: "warn", options: "error", "warn", "off")
+          jira-projects: 'BM,TDE' # optional list of jira projects

.gitignore

@@ -1 +1,4 @@
-node_modules/
+node_modules/
+dist/
+cdk.out/
+cdk.context.json

.prettierrc.cjs

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require('@linzjs/style/.prettierrc.cjs'),
+};

CONFIGURATION.md

@@ -78,12 +78,7 @@ container:
   env:
     - name: AWS_ROLE_CONFIG_PATH
       value: s3://linz-bucket-config/config.json
-  args:
-    [
-      "stac-validate",
-      "--recursive",
-      "{{inputs.parameters.location}}flat/collection.json",
-    ]
+  args: ['stac-validate', '--recursive', '{{inputs.parameters.location}}flat/collection.json']
 ```
 
 ## <a name='WorkflowPerformanceSettings'></a>Workflow Performance Settings
@@ -122,9 +117,9 @@ Two example workflow tasks requesting specific resources (memory and CPU) from t
 ```yaml
 - name: standardise-validate
   retryStrategy:
-    limit: "2"
+    limit: '2'
   nodeSelector:
-    karpenter.sh/capacity-type: "spot"
+    karpenter.sh/capacity-type: 'spot'
   inputs:
     parameters:
       - name: file
@@ -138,13 +133,13 @@ Two example workflow tasks requesting specific resources (memory and CPU) from t
         ephemeral-storage: 3Gi
     volumeMounts:
       - name: ephemeral
-        mountPath: "/tmp"
+        mountPath: '/tmp'
 ```
 
 ```yaml
 - name: copy
   retryStrategy:
-    limit: "2"
+    limit: '2'
   inputs:
     parameters:
       - name: file
@@ -173,7 +168,7 @@ _Note: the Argo Workflows flowchart UI refers to pods as “nodes”._
 In the standardising workflow, the `aws-list` task creates an output of a list of lists. Each `standardise-validate` task pod receives a list of files to process as an input.
 
 ```yaml
-withParam: "{{tasks.aws-list.outputs.parameters.files}}"
+withParam: '{{tasks.aws-list.outputs.parameters.files}}'
 ```
 
 For example, `aws-list` output with a `group` size of 4 set:

README.md

@@ -1,6 +1,6 @@
 # Topo Workflows
 
-Topo workflows are run on a AWS EKS Cluster using [Argo Workflows](https://argoproj.github.io/argo-workflows/)
+Topo workflows are run on a AWS EKS Cluster using [Argo Workflows](https://argoproj.github.io/argo-workflows/). The detailed configuration is available in [this repo](./config/).
 
 To get setup you need access to the Argo user role inside the EKS cluster, you will need to contact someone from Topo Data Engineering to get access, all Imagery maintainers will already have access.
 
@@ -33,12 +33,7 @@ Then to setup the cluster, only the first time using the cluster you need to run
 You will need a AWS CLI > 2.7.x
 
 ```bash
-
-# For Imagery maintainers you will already have the correct role so no role arn is needed.
-aws eks update-kubeconfig --name Workflow --region ap-southeast-2
-
-# For AWS Admin users you will need to find the correct EKS role to use
-aws eks update-kubeconfig --name Workflow --region ap-southeast-2 --role-arn arn:aws:iam::...
+aws eks update-kubeconfig --name Workflows --region ap-southeast-2
 ```
 
 to validate the cluster is connected,
@@ -54,7 +49,7 @@ ip-255-100-39-100.ap-southeast-2.compute.internal   Ready    <none>   7d   v1.21
 to make the cli access easier you can set the default namespace to `argo`
 
 ```bash
-k config set-context --current  --namespace=argo
+k config set-context --current --namespace=argo
 ```
 
 ## Submitting a job

cdk.json

@@ -0,0 +1 @@
+{ "app": "npx tsx config/cdk.ts" }

cdk8s.yaml

@@ -0,0 +1,5 @@
+app: npx tsx config/cdk8s.ts
+language: typescript
+imports:
+  - https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.sh_provisioners.yaml
+  - https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml

config/README.md

@@ -0,0 +1,73 @@
+# Topo-Workflows Infrastructure
+
+The infrastructure running the workflows is mainly based on a Kubernetes (EKS) cluster and Argo Workflows. It is currently run on AWS.
+Generally all Kubernetes resources are defined with cdk8s and anything that needs AWS interactions such as service accounts are defined with CDK.
+
+## EKS Cluster / AWS CDK
+
+The EKS Cluster base configuration is defined in `./cdk.ts` using [`aws-cdk`](https://aws.amazon.com/cdk/).
+
+## Kubernetes resources / CDK8s
+
+The additional components (or Kubernetes resources) running on the EKS cluster are defined in `./cdk8s` using [`cdk8s`](https://cdk8s.io/).
+
+Main entry point: [app](./cdk8s.ts)
+
+- Argo - Argo workflows for use with [linz/topo-workflows](https://github.com/linz/topo-workflows)
+- Karpenter
+
+### Argo Workflows
+
+#### Semaphores
+
+ConfigMap that list the synchronization limits for parallel execution of the workflows.
+
+### Karpenter
+
+TODO
+
+### Generate code
+
+Generate code from Helm:
+It is possible to generate a specific Helm construct for the component if their chart includes a `value.schema.json`. This is useful to provide typing hints when specifying their configuration (<https://github.com/cdk8s-team/cdk8s/blob/master/docs/cli/import.md#values-schema>)
+
+To generate the Helm Construct for a specific Chart, follow the instructions [here](https://github.com/cdk8s-team/cdk8s/blob/master/docs/cli/import.md#values-schema):
+
+Specify the output for the imports:
+
+`--output config/imports/`
+
+However, some of the component Helm charts do not have a `values.schema.json`. For those we won't generate any code and use the default `Helm` construct:
+
+- aws-for-fluent-bit (<https://github.com/aws/eks-charts/issues/1011>)
+- Karpenter
+
+## Usage (for test)
+
+Ensure all dependencies are installed
+
+```shell
+npm install
+```
+
+Login to AWS
+
+Generate the kubernetes configuration yaml into `dist/`
+
+```shell
+npx cdk8s synth
+```
+
+Apply the generated yaml files
+
+```shell
+kubectl apply -f dist/
+```
+
+## Deployment
+
+The deployment of the K8s config is managed by GithubActions in [main](../.github/workflows/main.yml).
+
+## Troubleshoot
+
+- [DNS](../docs/dns.configuration.md)

config/cdk.ts

@@ -0,0 +1,16 @@
+import { App } from 'aws-cdk-lib';
+
+import { CLUSTER_NAME } from './constants';
+import { LinzEksCluster } from './eks/cluster';
+
+const app = new App();
+
+async function main(): Promise<void> {
+  new LinzEksCluster(app, CLUSTER_NAME, {
+    env: { region: 'ap-southeast-2', account: process.env.CDK_DEFAULT_ACCOUNT },
+  });
+
+  app.synth();
+}
+
+main();