From 0877929d31acfffbf2753ad3fce00f43fcc9ceb1 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Fri, 5 Apr 2024 12:12:56 +1300 Subject: [PATCH 01/28] feat: add node local dns cache (#511) #### Motivation We are seeing a large amount of DNS failures with timeouts and `EAI_AGAIN` responses, we believe it might be due to the DNS servers getting overwhelmed when the cluster is scalling rapidly up. By adding a DNS cache to every node there is a huge reduction of DNS requests made over the wire to the primary coredns servers. #### Modification Adds a DNS cache to every node inside the cluster. --------- Co-authored-by: paulfouquet <86932794+paulfouquet@users.noreply.github.com> --- docs/dns.configuration.md | 22 +- .../components/node.local.dns.md | 42 ++++ infra/cdk8s.ts | 45 ++-- infra/charts/kube-system.coredns.ts | 7 +- infra/charts/kube-system.node.local.dns.ts | 207 ++++++++++++++++++ infra/constants.ts | 7 + infra/util/cloud.formation.ts | 8 + 7 files changed, 315 insertions(+), 23 deletions(-) create mode 100644 docs/infrastructure/components/node.local.dns.md create mode 100644 infra/charts/kube-system.node.local.dns.ts diff --git a/docs/dns.configuration.md b/docs/dns.configuration.md index d65338bc5..2a36a9ca6 100644 --- a/docs/dns.configuration.md +++ b/docs/dns.configuration.md @@ -12,10 +12,16 @@ Start a shell on the container k exec -n :namespace -it :podName -- /bin/bash ``` -Install basic dns utils `dig` `ping` `wget` and `curl` +Install basic networking utils `dig`, `ping`, `ping6`, `wget`, `nslookup`, and `curl` ```bash -apt install dnsutils iptools-ping wget curl +apt update && apt install -y dnsutils iputils-ping wget curl +``` + +Other useful tools may include `tracepath`, `traceroute` and `mtr` + +```bash +apt update && apt install -y iputils-tracepath mtr traceroute ``` ### Name resolution @@ -69,18 +75,24 @@ Depending on the container you may have access to scripting languages. #### NodeJS -file: index.mjs +create a new file `index.mjs` ```javascript fetch('https://google.com').then((c) => console.log(c)); import * as dns from 'dns/promises'; -await dns.resolve('google.com', 'A'); -await dns.resolve('google.com', 'AAAA'); +console.log(await dns.resolve('google.com', 'A')); +console.log(await dns.resolve('google.com', 'AAAA')); ``` +Run the file + ```bash node --version node index.mjs ``` + +## Node Local DNS + +A local DNS cache is running on every node, [node-local-dns](./infrastructure/components/node.local.dns.md) if any DNS issues occur it is recommended to turn the DNS cache off as a first step for debugging diff --git a/docs/infrastructure/components/node.local.dns.md b/docs/infrastructure/components/node.local.dns.md new file mode 100644 index 000000000..188d66eb8 --- /dev/null +++ b/docs/infrastructure/components/node.local.dns.md @@ -0,0 +1,42 @@ +# Node Local DNS + +When large [argo](./argo.workflows.md) jobs are submitted the kubernetes cluster can sometimes scale up very quickly which can overwhelm the coredns DNS resolvers that are running on the primary nodes. + +To prevent the overload a DNS cache is installed on every new node when it starts. + +It is based off https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/ and it greatly reduces the load on the primary DNS servers. + +## Debugging DNS problems with node local DNS + +If DNS problems occur while node local dns is running, it is recommended to turn it off using the `UseNodeLocalDns = false` constant in `infra/constants.ts` + +## Watching DNS requests + +By default the DNS cache will log any external DNS requests it is resolving (anything that is not ending with `.cluster.local`) since there can be a large number of dns cache pods the following command will tail the logs from + +``` +kubectl logs -n kube-system --all-containers=true -f daemonset/node-local-dns --since=1m --timestamps=true --prefix=true +``` + +### Structured logs + +`coredns` does not provide a simple way of constructing a structured log from the DNS request, it does provide a template system which can be used to craft a JSON log line, if the log line is in structured format like JSON it can be more easily processed into something like elasticsearch for additional debugging. + +For the current log format see `CoreFileJsonLogFormat` and below is a example log request + +```json +{ + "remoteIp": "[2406:da1c:afb:bc0b:d0e3::6]", + "remotePort": 43621, + "protocol": "udp", + "queryId": "14962", + "queryType": "A", + "queryClass": "IN", + "queryName": "logs.ap-southeast-2.amazonaws.com.", + "querySize": 51, + "dnsSecOk": "false", + "responseCode": "NOERROR", + "responseFlags": "qr,rd,ra", + "responseSize": 443 +} +``` diff --git a/infra/cdk8s.ts b/infra/cdk8s.ts index 9cb8ede06..a569e2436 100644 --- a/infra/cdk8s.ts +++ b/infra/cdk8s.ts @@ -7,32 +7,45 @@ import { EventExporter } from './charts/event.exporter.js'; import { FluentBit } from './charts/fluentbit.js'; import { Karpenter, KarpenterProvisioner } from './charts/karpenter.js'; import { CoreDns } from './charts/kube-system.coredns.js'; -import { CfnOutputKeys, ClusterName, ScratchBucketName, validateKeys } from './constants.js'; -import { getCfnOutputs } from './util/cloud.formation.js'; +import { NodeLocalDns } from './charts/kube-system.node.local.dns.js'; +import { CfnOutputKeys, ClusterName, ScratchBucketName, UseNodeLocalDns, validateKeys } from './constants.js'; +import { describeCluster, getCfnOutputs } from './util/cloud.formation.js'; import { fetchSsmParameters } from './util/ssm.js'; const app = new App(); async function main(): Promise { // Get cloudformation outputs - const cfnOutputs = await getCfnOutputs(ClusterName); + const [cfnOutputs, ssmConfig, clusterConfig] = await Promise.all([ + getCfnOutputs(ClusterName), + fetchSsmParameters({ + // Config for Cloudflared to access argo-server + tunnelId: '/eks/cloudflared/argo/tunnelId', + tunnelSecret: '/eks/cloudflared/argo/tunnelSecret', + tunnelName: '/eks/cloudflared/argo/tunnelName', + accountId: '/eks/cloudflared/argo/accountId', + + // Personal access token to gain access to linz-li-bot github user + githubPat: '/eks/github/linz-li-bot/pat', + + // Argo Database connection password + argoDbPassword: '/eks/argo/postgres/password', + }), + describeCluster(ClusterName), + ]); validateKeys(cfnOutputs); - const ssmConfig = await fetchSsmParameters({ - // Config for Cloudflared to access argo-server - tunnelId: '/eks/cloudflared/argo/tunnelId', - tunnelSecret: '/eks/cloudflared/argo/tunnelSecret', - tunnelName: '/eks/cloudflared/argo/tunnelName', - accountId: '/eks/cloudflared/argo/accountId', - - // Personal access token to gain access to linz-li-bot github user - githubPat: '/eks/github/linz-li-bot/pat', + const coredns = new CoreDns(app, 'dns', {}); - // Argo Database connection password - argoDbPassword: '/eks/argo/postgres/password', - }); + // Node localDNS is very expermential in this cluster, it can and will break DNS resolution + // If there are any issues with DNS, NodeLocalDNS should be disabled first. + if (UseNodeLocalDns) { + const ipv6Cidr = clusterConfig.kubernetesNetworkConfig?.serviceIpv6Cidr; + if (ipv6Cidr == null) throw new Error('Unable to use node-local-dns without ipv6Cidr'); + const nodeLocal = new NodeLocalDns(app, 'node-local-dns', { serviceIpv6Cidr: ipv6Cidr }); + nodeLocal.addDependency(coredns); + } - const coredns = new CoreDns(app, 'dns', {}); const fluentbit = new FluentBit(app, 'fluentbit', { saName: cfnOutputs[CfnOutputKeys.FluentBitServiceAccountName], clusterName: ClusterName, diff --git a/infra/charts/kube-system.coredns.ts b/infra/charts/kube-system.coredns.ts index 27542e484..860f4b00e 100644 --- a/infra/charts/kube-system.coredns.ts +++ b/infra/charts/kube-system.coredns.ts @@ -4,6 +4,9 @@ import { Construct } from 'constructs'; import { applyDefaultLabels } from '../util/labels.js'; +/** Configure CoreDNS to output a JSON object for its log files */ +export const CoreFileJsonLogFormat = `{"remoteIp":"{remote}","remotePort":{port},"protocol":"{proto}","queryId":"{>id}","queryType":"{type}","queryClass":"{class}","queryName":"{name}","querySize":{size},"dnsSecOk":"{>do}","responseCode":"{rcode}","responseFlags":"{>rflags}","responseSize":{rsize}}`; + /** * This cluster is setup as dual ipv4/ipv6 where ipv4 is used for external traffic * and ipv6 for internal traffic. @@ -36,7 +39,7 @@ export class CoreDns extends Chart { // FIXME: is there a better way of handling config files inside of cdk8s Corefile: ` cluster.local:53 { - log + log . ${CoreFileJsonLogFormat} errors health kubernetes cluster.local in-addr.arpa ip6.arpa { @@ -53,7 +56,7 @@ cluster.local:53 { } .:53 { - log + log . ${CoreFileJsonLogFormat} errors health template ANY AAAA { diff --git a/infra/charts/kube-system.node.local.dns.ts b/infra/charts/kube-system.node.local.dns.ts new file mode 100644 index 000000000..c19ccd3ce --- /dev/null +++ b/infra/charts/kube-system.node.local.dns.ts @@ -0,0 +1,207 @@ +import { ApiObject, Chart, ChartProps, JsonPatch, Size } from 'cdk8s'; +import * as kplus from 'cdk8s-plus-27'; +import { Construct } from 'constructs'; + +import { applyDefaultLabels } from '../util/labels.js'; +import { CoreFileJsonLogFormat } from './kube-system.coredns.js'; + +export interface NodeLocalDnsProps extends ChartProps { + /** cluster networking configuration */ + serviceIpv6Cidr: string; + + /** + * bind the node-local-dns to a top level suffix on the {@link serviceIpv6Cidr} + * + * @defaultValue "ffaa" + */ + bindAddressSuffix?: string; +} + +export class NodeLocalDns extends Chart { + constructor(scope: Construct, id: string, props: NodeLocalDnsProps) { + super(scope, id, { + ...applyDefaultLabels(props, 'node-local-dns', 'v1', 'kube-dns', 'kube-dns'), + namespace: 'kube-system', + }); + + const bindAddressSuffix = props.bindAddressSuffix ?? 'ffaa'; + + const serviceBaseAddress = props.serviceIpv6Cidr.slice(0, props.serviceIpv6Cidr.lastIndexOf('/')); + + const bindAddress = serviceBaseAddress + bindAddressSuffix; + const upstreamDns = serviceBaseAddress + 'a'; // TODO is this always `::a` ? + + const dnsUpstream = new kplus.Service(this, 'kube-dns-upstream', { + metadata: { + name: 'kube-dns-upstream', + labels: { + 'kubernetes.io/cluster-service': 'true', + 'kubernetes.io/name': 'KubeDNSUpstream', + 'k8s-app': 'kube-dns', + }, + }, + ports: [ + { name: 'dns', port: 53, protocol: kplus.Protocol.UDP, targetPort: 53 }, + { name: 'dns-tcp', port: 53, protocol: kplus.Protocol.TCP, targetPort: 53 }, + ], + selector: kplus.Pods.select(this, 'kube-dns-upstream-pods', { labels: { 'k8s-app': 'kube-dns' } }), + }); + + const configMap = new kplus.ConfigMap(this, 'node-local-dns-config', { + metadata: { name: 'node-local-dns' }, + data: { + Corefile: generateCorefile({ bindAddress: bindAddress, upstreamAddress: upstreamDns }), + }, + }); + + const ds = new kplus.DaemonSet(this, 'node-local-dns-daemon', { + metadata: { + name: 'node-local-dns', + labels: { 'kubernetes.io/cluster-service': 'true' }, + }, + + serviceAccount: new kplus.ServiceAccount(this, 'node-local-dns-sa', { + metadata: { + name: 'node-local-dns', + labels: { 'kubernetes.io/cluster-service': 'true' }, + }, + }), + securityContext: { ensureNonRoot: false }, + dns: { policy: kplus.DnsPolicy.DEFAULT }, + hostNetwork: true, + podMetadata: {}, + containers: [ + { + name: 'node-cache', + securityContext: { + ensureNonRoot: false, + allowPrivilegeEscalation: true, + readOnlyRootFilesystem: false, + privileged: true, + }, + image: 'registry.k8s.io/dns/k8s-dns-node-cache:1.22.28', + resources: { cpu: { request: kplus.Cpu.millis(25) }, memory: { request: Size.mebibytes(5) } }, // { cpu: 25m, 5Mi} ContainerResources + args: [ + '-localip', + [bindAddress, upstreamDns].join(','), + '-conf', + '/etc/Corefile', + '-upstreamsvc', + dnsUpstream.name, + ], + ports: [ + { name: 'dns', protocol: kplus.Protocol.UDP, number: 53 }, + // { name: 'dns-tcp', protocol: kplus.Protocol.TCP, number: 53 }, // TODO this is broken, see JSONPatch + { name: 'metrics', protocol: kplus.Protocol.TCP, number: 9253 }, + ], + liveness: kplus.Probe.fromHttpGet('/health', { port: 8080 /* TODO: host: bindAddress, see JSONPatch*/ }), + volumeMounts: [ + { + path: '/etc/coredns', + volume: kplus.Volume.fromConfigMap(this, 'config-volume', configMap, { + name: 'node-local-dns', + items: { Corefile: { path: 'Corefile.base' } }, + }), + }, + { + path: '/run/xtables.lock', + readOnly: false, + volume: kplus.Volume.fromHostPath(this, 'iptables', 'xtables-lock', { + path: '/run/xtables.lock', + type: kplus.HostPathVolumeType.FILE_OR_CREATE, + }), + }, + ], + }, + ], + }); + + // ESCAPE hatches to manually overwrite a few configuration options that could not be configured with cdk8s + const apiDs = ApiObject.of(ds); + apiDs.addJsonPatch( + // httpGet.host is missing from kplus.Probe.fromHttpGet + JsonPatch.add('/spec/template/spec/containers/0/livenessProbe/httpGet/host', bindAddress), + // TODO where is this defined + JsonPatch.add('/spec/template/spec/priorityClassName', 'system-node-critical'), + // unable to add two ports with the same number even though they are different protocols + JsonPatch.add('/spec/template/spec/containers/0/ports/1', { + containerPort: 53, + name: 'dns-tcp', + protocol: 'TCP', + }), + // Unable to set the security context + JsonPatch.replace('/spec/template/spec/containers/0/securityContext', { capabilities: { add: ['NET_ADMIN'] } }), + + // Force the tolerations on to the pod + JsonPatch.add('/spec/template/spec/tolerations', [ + { key: 'CriticalAddonsOnly', operator: 'Exists' }, + { effect: 'NoExecute', operator: 'Exists' }, + { effect: 'NoSchedule', operator: 'Exists' }, + ]), + ); + } +} + +/** + * Generate a node-local-dns cache Corefile + * + * Taken from https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml#L56 + * @param ctx addresses to replace + * @returns coredns's CoreFile configuration for node local dns + */ +function generateCorefile(ctx: { bindAddress: string; upstreamAddress: string }): string { + // __PILLAR___ keys are replaced automatically by the node local dns pod + return `cluster.local:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind ${ctx.bindAddress} ${ctx.upstreamAddress} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + health [${ctx.bindAddress}]:8080 + } + + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind ${ctx.bindAddress} ${ctx.upstreamAddress} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + } + + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind ${ctx.bindAddress} ${ctx.upstreamAddress} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + } + + .:53 { + log . ${CoreFileJsonLogFormat} + errors + cache 30 + reload + loop + template ANY AAAA { + rcode NOERROR + } + bind ${ctx.bindAddress} ${ctx.upstreamAddress} + forward . __PILLAR__UPSTREAM__SERVERS__ + prometheus :9253 + }`; +} diff --git a/infra/constants.ts b/infra/constants.ts index 62acb98b0..077a16a9b 100644 --- a/infra/constants.ts +++ b/infra/constants.ts @@ -9,6 +9,13 @@ export const ArgoDbName = 'argo'; /* Argo Database user */ export const ArgoDbUser = 'argo_user'; +/** + * Should NodeLocal DNS be enabled for the cluster + * + * @see ./charts/kube-system.coredns.ts + */ +export const UseNodeLocalDns = true; + /* CloudFormation Output to access from CDK8s */ export const CfnOutputKeys = { ClusterEndpoint: 'ClusterEndpoint', diff --git a/infra/util/cloud.formation.ts b/infra/util/cloud.formation.ts index 04cb10dde..cadae0c04 100644 --- a/infra/util/cloud.formation.ts +++ b/infra/util/cloud.formation.ts @@ -1,4 +1,5 @@ import { CloudFormation } from '@aws-sdk/client-cloudformation'; +import { Cluster, EKS } from '@aws-sdk/client-eks'; export async function getCfnOutputs(stackName: string): Promise> { const cfn = new CloudFormation(); @@ -12,3 +13,10 @@ export async function getCfnOutputs(stackName: string): Promise { + const eks = new EKS(); + const describe = await eks.describeCluster({ name: clusterName }); + if (describe.cluster == null) throw new Error('Cluster not found: ' + clusterName); + return describe.cluster; +} From a4c5239e1258f22cfa638d54990877239c746eb5 Mon Sep 17 00:00:00 2001 From: Rebecca Clarke Date: Fri, 5 Apr 2024 12:17:56 +1300 Subject: [PATCH 02/28] feat: change default value for exclude argument (#514) The national 10m satellite imagery mosaic has had a name change since the source imagery was published to the ODR. Previously it was named `nz_satellite_2021-2022_10m_RGB` in the basemaps aerial config It is now named `new-zealand_2022-2023_10m`. This new name structure will continue going forward. --- workflows/basemaps/mapsheet-json.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/basemaps/mapsheet-json.yaml b/workflows/basemaps/mapsheet-json.yaml index d567c053a..32dfeb650 100644 --- a/workflows/basemaps/mapsheet-json.yaml +++ b/workflows/basemaps/mapsheet-json.yaml @@ -22,7 +22,7 @@ spec: - name: include value: '' - name: exclude - value: 'nz-satellite' + value: 'new-zealand' templateDefaults: container: imagePullPolicy: Always From 423b24a51d7bdfd2a38655c922500cb73ec4ce26 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Mon, 8 Apr 2024 11:17:04 +1200 Subject: [PATCH 03/28] feat(basemaps): Cron workflow for update nz roads addressing vector layer. BM-977 (#510) #### Motivation Another cron etl workflow for update the [53382-nz-roads-addressing](https://data.linz.govt.nz/layer/53382-nz-roads-addressing/) data. #### Modification - Add new cron workflow running 6am Wednesday for update nz road addressing vector data. - Upgrade the argo task to v3 to support individual vector tileset in create-pr. #### Checklist - [ ] Tests updated - no tests - [ ] Docs updated - no doc - [x] Issue linked in Title --- workflows/basemaps/vector-etl.yaml | 2 +- .../cron-vector-etl-roads-addressing.yaml | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 workflows/cron/cron-vector-etl-roads-addressing.yaml diff --git a/workflows/basemaps/vector-etl.yaml b/workflows/basemaps/vector-etl.yaml index 368cae8ec..609344f56 100644 --- a/workflows/basemaps/vector-etl.yaml +++ b/workflows/basemaps/vector-etl.yaml @@ -14,7 +14,7 @@ spec: parameters: - name: version_argo_tasks description: Version of the Argo Tasks CLI docker container to use - value: 'v2' + value: 'v3' - name: version_basemaps_etl description: Version of the Basemaps ETL eks container to use diff --git a/workflows/cron/cron-vector-etl-roads-addressing.yaml b/workflows/cron/cron-vector-etl-roads-addressing.yaml new file mode 100644 index 000000000..6a2fce4c7 --- /dev/null +++ b/workflows/cron/cron-vector-etl-roads-addressing.yaml @@ -0,0 +1,26 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: cron-vector-etl-roads-addressing + namespace: argo + labels: + linz.govt.nz/category: basemaps + linz.govt.nz/data-type: vector +spec: + schedule: '0 06 * * 3' # 6 AM every Wednesday + timezone: 'NZ' + startingDeadlineSeconds: 3600 # Allow 1 hour delay if the workflow-controller clashes during the starting time. + concurrencyPolicy: 'Allow' + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + suspend: false + workflowSpec: + workflowTemplateRef: + name: basemaps-vector-etl + arguments: + parameters: + - filename: + value: '53382-nz-roads-addressing' + - name: 'retry' + value: '2' From 920976e49c19c990a42c67846f2bcf7f3ccc9dad Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 9 Apr 2024 12:44:01 +1200 Subject: [PATCH 04/28] feat(basemaps): Upgrade the basemaps cli version in the workflow (#518) #### Motivation Upgrade the cli version to the latest releases. #### Modification - basemaps cli to v7 - argo tasks cli to v3 - create-overview remains as v6 which is disabled in v7 until we re-implement in cogify. #### Checklist - [ ] Tests updated - no test - [ ] Docs updated - no doc - [ ] Issue linked in Title - no ticket --- workflows/basemaps/imagery-import-cogify.yml | 4 ++-- workflows/basemaps/mapsheet-json.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index a8e938265..7de5006fb 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -30,11 +30,11 @@ spec: parameters: - name: version_basemaps_cli description: Version of the basemaps CLI docker container to use - value: v6 + value: v7 - name: version_argo_tasks description: Version of the basemaps CLI docker container to use - value: v2 + value: v3 - name: ticket description: Ticket ID e.g. 'AIP-55' diff --git a/workflows/basemaps/mapsheet-json.yaml b/workflows/basemaps/mapsheet-json.yaml index 32dfeb650..5662aabc5 100644 --- a/workflows/basemaps/mapsheet-json.yaml +++ b/workflows/basemaps/mapsheet-json.yaml @@ -14,7 +14,7 @@ spec: arguments: parameters: - name: version_argo_tasks - value: 'v2' + value: 'v3' - name: layer value: '104687' - name: config From 2100855885b6ec0d2edcd7a47a4780b76f126d1f Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:13:19 +1200 Subject: [PATCH 05/28] fix: argo controller disabling watch configmap wrong env var name TDE-1016 (#520) #### Motivation https://github.com/linz/topo-workflows/pull/498 has been using the wrong environment variable name to disable watching configmaps, so the issue is not fixed. #### Modification Use the correct [environment variable name](https://argo-workflows.readthedocs.io/en/stable/environment-variables/) #### Checklist - [ ] Tests updated - [ ] Docs updated - [x] Issue linked in Title --- infra/charts/argo.workflows.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/charts/argo.workflows.ts b/infra/charts/argo.workflows.ts index 52dd2fdd5..0b5c36d07 100644 --- a/infra/charts/argo.workflows.ts +++ b/infra/charts/argo.workflows.ts @@ -130,7 +130,7 @@ export class ArgoWorkflows extends Chart { workflowNamespaces: ['argo'], extraArgs: [], // FIXME: workaround for https://github.com/argoproj/argo-workflows/issues/11657 - extraEnv: [{ name: 'WATCH_CONFIGMAPS', value: 'false' }], + extraEnv: [{ name: 'WATCH_CONTROLLER_SEMAPHORE_CONFIGMAPS', value: 'false' }], persistence, replicas: 2, workflowDefaults: { From 46219652b52c9543ad1df8f4b5db5fb8641b89c8 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 9 Apr 2024 16:27:38 +1200 Subject: [PATCH 06/28] fix: rename node-cache to node-local-dns (#519) #### Motivation Our analytics use the container name for DNS query logs, `node-cache` is not easily mappable to `node-local-dns` #### Modification Switches the container name of node-local-dns to "node-local-dns" #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title --- infra/charts/kube-system.node.local.dns.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/charts/kube-system.node.local.dns.ts b/infra/charts/kube-system.node.local.dns.ts index c19ccd3ce..deb233f86 100644 --- a/infra/charts/kube-system.node.local.dns.ts +++ b/infra/charts/kube-system.node.local.dns.ts @@ -72,7 +72,7 @@ export class NodeLocalDns extends Chart { podMetadata: {}, containers: [ { - name: 'node-cache', + name: 'node-local-dns', securityContext: { ensureNonRoot: false, allowPrivilegeEscalation: true, From c7df1230c9d8e193e914c1fceb26e3a6df73ce0f Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 10 Apr 2024 08:50:28 +1200 Subject: [PATCH 07/28] fix(cron): Add the missing parameter name for the cron workflow (#522) #### Motivation Parameter name is defined incorrectly in the cron workflow. #### Modification fix the name in two cron workflow #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - no test - [ ] Docs updated - no doc - [ ] Issue linked in Title - no ticket --- workflows/cron/cron-vector-etl-roads-addressing.yaml | 2 +- workflows/cron/cron-vector-etl.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/cron/cron-vector-etl-roads-addressing.yaml b/workflows/cron/cron-vector-etl-roads-addressing.yaml index 6a2fce4c7..850ff49ac 100644 --- a/workflows/cron/cron-vector-etl-roads-addressing.yaml +++ b/workflows/cron/cron-vector-etl-roads-addressing.yaml @@ -20,7 +20,7 @@ spec: name: basemaps-vector-etl arguments: parameters: - - filename: + - name: 'filename' value: '53382-nz-roads-addressing' - name: 'retry' value: '2' diff --git a/workflows/cron/cron-vector-etl.yaml b/workflows/cron/cron-vector-etl.yaml index 8f9fea8eb..5aff27e8d 100644 --- a/workflows/cron/cron-vector-etl.yaml +++ b/workflows/cron/cron-vector-etl.yaml @@ -20,7 +20,7 @@ spec: name: basemaps-vector-etl arguments: parameters: - - filename: + - name: 'filename' value: 'topographic' - name: 'retry' value: '2' From ebec9b7ff6b1f90a4589d17f466cca3dcbde0225 Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Fri, 12 Apr 2024 09:35:22 +1200 Subject: [PATCH 08/28] feat: default retryStrategy TDE-1112 (#506) #### Motivation We want every task within our workflows to retry (twice) automatically and on a different node/host (network issues) when they fail. #### Modification Configure the `retryStrategy` at the `workflowDefaults` level. Notes: - the `nodeAntiAffinity` to prioritize retrying on a different node/host is not working due to an issue in the Argo Workflows system. A [PR is opened](https://github.com/argoproj/argo-workflows/pull/12701) in the Argo repo - this change will make every task retrying on failure. Some of our tasks (for example, `stac-validate`, `tileindex-validate`) are expected to fail if the system invalidate something. They will still retry twice in that case. This would be handle in a separate PR. #### Checklist - [ ] Tests updated NA - [x] Docs updated - [x] Issue linked in Title --- docs/retry.md | 29 ++++++++++++++++++++ infra/charts/argo.workflows.ts | 2 ++ templates/argo-tasks/copy.yml | 2 -- templates/argo-tasks/push-to-github.yml | 2 -- workflows/basemaps/create-config.yaml | 2 -- workflows/basemaps/create-overview-all.yaml | 2 -- workflows/basemaps/create-overview.yaml | 2 -- workflows/basemaps/imagery-import-cogify.yml | 8 ------ workflows/raster/standardising.yaml | 8 ------ workflows/stac/stac-validate.yaml | 2 -- workflows/test/flatten.yaml | 2 -- workflows/util/create-thumbnails.yaml | 2 -- 12 files changed, 31 insertions(+), 32 deletions(-) create mode 100644 docs/retry.md diff --git a/docs/retry.md b/docs/retry.md new file mode 100644 index 000000000..16b476d57 --- /dev/null +++ b/docs/retry.md @@ -0,0 +1,29 @@ +# Default retryStrategy + +The default [`retryStrategy`](https://argo-workflows.readthedocs.io/en/stable/fields/#retrystrategy) is defined at the `workflowDefaults` level in the [Argo Workflow chart configuration](https://github.com/linz/topo-workflows/blob/master/infra/charts/argo.workflows.ts). This will be apply to every step/tasks by default. + +## Overriding + +To override the default `retryStrategy`, it can be done at the workflow or template level by defining a specific `retryStrategy`. + +## Avoiding retry + +For example, to avoid the default `retryStrategy` and make sure the task does not retry: + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: my-wf- +spec: + entrypoint: main + templates: + - name: main + retryStrategy: + expression: 'false' + container: + image: python:alpine3.6 + command: ['python'] + source: | + # Do something that fails ... +``` diff --git a/infra/charts/argo.workflows.ts b/infra/charts/argo.workflows.ts index 0b5c36d07..a4720a108 100644 --- a/infra/charts/argo.workflows.ts +++ b/infra/charts/argo.workflows.ts @@ -147,6 +147,8 @@ export class ArgoWorkflows extends Chart { }, ], parallelism: 3, + // FIXME: `nodeAntiAffinity` - to retry on different node - is not working yet (https://github.com/argoproj/argo-workflows/pull/12701) + retryStrategy: { limit: 2, affinity: { nodeAntiAffinity: {} } }, }, }, }, diff --git a/templates/argo-tasks/copy.yml b/templates/argo-tasks/copy.yml index 8cadec150..246141961 100644 --- a/templates/argo-tasks/copy.yml +++ b/templates/argo-tasks/copy.yml @@ -14,8 +14,6 @@ spec: entrypoint: main templates: - name: main - retryStrategy: - limit: '2' inputs: parameters: - name: file diff --git a/templates/argo-tasks/push-to-github.yml b/templates/argo-tasks/push-to-github.yml index ec0bd54e4..29d7eb3e0 100644 --- a/templates/argo-tasks/push-to-github.yml +++ b/templates/argo-tasks/push-to-github.yml @@ -13,8 +13,6 @@ spec: entrypoint: main templates: - name: main - retryStrategy: - limit: '2' inputs: parameters: - name: source diff --git a/workflows/basemaps/create-config.yaml b/workflows/basemaps/create-config.yaml index d0f97a920..c35546a0b 100644 --- a/workflows/basemaps/create-config.yaml +++ b/workflows/basemaps/create-config.yaml @@ -30,8 +30,6 @@ spec: - name: location value: '{{workflow.parameters.location}}' - name: create-config - retryStrategy: - limit: '2' inputs: parameters: - name: location diff --git a/workflows/basemaps/create-overview-all.yaml b/workflows/basemaps/create-overview-all.yaml index 05136adc8..30d7cb53f 100644 --- a/workflows/basemaps/create-overview-all.yaml +++ b/workflows/basemaps/create-overview-all.yaml @@ -51,8 +51,6 @@ spec: path: '/tmp/imagery.json' - name: create-overview - retryStrategy: - limit: '2' inputs: parameters: - name: source diff --git a/workflows/basemaps/create-overview.yaml b/workflows/basemaps/create-overview.yaml index b5cc0a5c0..f1c9110e9 100644 --- a/workflows/basemaps/create-overview.yaml +++ b/workflows/basemaps/create-overview.yaml @@ -35,8 +35,6 @@ spec: - name: output value: '{{workflow.parameters.output}}' - name: create-overview - retryStrategy: - limit: '2' inputs: parameters: - name: source diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index 7de5006fb..f3b14fc3b 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -239,8 +239,6 @@ spec: # Generate a tile covering for input imagery - name: create-covering - retryStrategy: - limit: '2' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: @@ -284,8 +282,6 @@ spec: # Actually create COGs using gdal_translate on a large spot instances - name: create-cog - retryStrategy: - limit: '2' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: @@ -312,8 +308,6 @@ spec: # Create a basemaps configuration file to view the imagery - name: create-config - retryStrategy: - limit: '2' inputs: parameters: - name: path @@ -340,8 +334,6 @@ spec: # create additional overviews for any COGs found in the path - name: create-overview - retryStrategy: - limit: '2' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index f9f4abaca..2ac87be0c 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -441,8 +441,6 @@ spec: path: '/tmp/collection-id' - name: standardise-validate - retryStrategy: - limit: '2' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: @@ -489,8 +487,6 @@ spec: - '{{=sprig.trim(workflow.parameters.gsd)}}' - name: create-collection - retryStrategy: - limit: '2' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: @@ -582,8 +578,6 @@ spec: path: '/tmp/key' - name: create-overview - retryStrategy: - limit: '2' inputs: parameters: - name: location @@ -608,8 +602,6 @@ spec: ] - name: create-config - retryStrategy: - limit: '2' inputs: parameters: - name: location diff --git a/workflows/stac/stac-validate.yaml b/workflows/stac/stac-validate.yaml index 6be391533..23c46b88d 100644 --- a/workflows/stac/stac-validate.yaml +++ b/workflows/stac/stac-validate.yaml @@ -68,8 +68,6 @@ spec: valueFrom: path: /tmp/file_list.json - name: stac-validate-collections - retryStrategy: - limit: '2' inputs: parameters: - name: file diff --git a/workflows/test/flatten.yaml b/workflows/test/flatten.yaml index 967c63552..4ecfdcc08 100644 --- a/workflows/test/flatten.yaml +++ b/workflows/test/flatten.yaml @@ -80,8 +80,6 @@ spec: valueFrom: path: /tmp/file_list.json - name: flatten-copy - retryStrategy: - limit: '2' inputs: parameters: - name: file diff --git a/workflows/util/create-thumbnails.yaml b/workflows/util/create-thumbnails.yaml index bcc49dc90..be1736722 100644 --- a/workflows/util/create-thumbnails.yaml +++ b/workflows/util/create-thumbnails.yaml @@ -75,8 +75,6 @@ spec: path: /tmp/file_list.json - name: thumbnails - retryStrategy: - limit: '2' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: From ac5877c2044a4266e21e30434598967624bbbd2b Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Tue, 16 Apr 2024 08:40:08 +1200 Subject: [PATCH 09/28] chore: upgrade argo jsonschema to 3.5.5 (#531) #### Motivation The jsonschema version should be consistent with the Argo version deployed. #### Modification Upgrade jsonschema version as the yaml-language-server of the workflow files. #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title --- templates/argo-tasks/copy.yml | 2 +- templates/argo-tasks/create-manifest.yml | 2 +- templates/argo-tasks/generate-path.yml | 2 +- templates/argo-tasks/group.yml | 2 +- templates/argo-tasks/push-to-github.yml | 2 +- templates/argo-tasks/tile-index-validate.yml | 2 +- workflows/basemaps/create-config.yaml | 2 +- workflows/basemaps/create-overview-all.yaml | 2 +- workflows/basemaps/create-overview.yaml | 2 +- workflows/basemaps/imagery-import-cogify.yml | 2 +- workflows/basemaps/mapsheet-json.yaml | 2 +- workflows/basemaps/vector-etl.yaml | 2 +- workflows/cron/cron-vector-etl-roads-addressing.yaml | 2 +- workflows/cron/cron-vector-etl.yaml | 2 +- workflows/raster/copy.yaml | 2 +- workflows/raster/publish-odr.yaml | 2 +- workflows/raster/standardising.yaml | 2 +- workflows/raster/tests.yaml | 2 +- workflows/stac/stac-validate.yaml | 3 ++- workflows/test/env.yaml | 2 +- workflows/test/flatten.yaml | 2 +- workflows/test/generate-path.yaml | 2 +- workflows/test/list.arm.yaml | 2 +- workflows/test/list.yaml | 2 +- workflows/test/sleep.yml | 2 +- workflows/util/create-thumbnails.yaml | 2 +- 26 files changed, 27 insertions(+), 26 deletions(-) diff --git a/templates/argo-tasks/copy.yml b/templates/argo-tasks/copy.yml index 246141961..b935eb264 100644 --- a/templates/argo-tasks/copy.yml +++ b/templates/argo-tasks/copy.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/templates/argo-tasks/create-manifest.yml b/templates/argo-tasks/create-manifest.yml index 995fc760f..cfaa953ff 100644 --- a/templates/argo-tasks/create-manifest.yml +++ b/templates/argo-tasks/create-manifest.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/templates/argo-tasks/generate-path.yml b/templates/argo-tasks/generate-path.yml index a5ff7374b..81ac5e3a0 100644 --- a/templates/argo-tasks/generate-path.yml +++ b/templates/argo-tasks/generate-path.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/templates/argo-tasks/group.yml b/templates/argo-tasks/group.yml index 5464d30e9..1291d5eea 100644 --- a/templates/argo-tasks/group.yml +++ b/templates/argo-tasks/group.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/templates/argo-tasks/push-to-github.yml b/templates/argo-tasks/push-to-github.yml index 29d7eb3e0..200641ae1 100644 --- a/templates/argo-tasks/push-to-github.yml +++ b/templates/argo-tasks/push-to-github.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: diff --git a/templates/argo-tasks/tile-index-validate.yml b/templates/argo-tasks/tile-index-validate.yml index 517138036..a1b5cb5a7 100644 --- a/templates/argo-tasks/tile-index-validate.yml +++ b/templates/argo-tasks/tile-index-validate.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/basemaps/create-config.yaml b/workflows/basemaps/create-config.yaml index c35546a0b..ed9e50b5a 100644 --- a/workflows/basemaps/create-config.yaml +++ b/workflows/basemaps/create-config.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/basemaps/create-overview-all.yaml b/workflows/basemaps/create-overview-all.yaml index 30d7cb53f..82195512a 100644 --- a/workflows/basemaps/create-overview-all.yaml +++ b/workflows/basemaps/create-overview-all.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: Workflow diff --git a/workflows/basemaps/create-overview.yaml b/workflows/basemaps/create-overview.yaml index f1c9110e9..5a0784a35 100644 --- a/workflows/basemaps/create-overview.yaml +++ b/workflows/basemaps/create-overview.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: Workflow diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index f3b14fc3b..77d52eb2a 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/basemaps/mapsheet-json.yaml b/workflows/basemaps/mapsheet-json.yaml index 5662aabc5..0b1df6518 100644 --- a/workflows/basemaps/mapsheet-json.yaml +++ b/workflows/basemaps/mapsheet-json.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/basemaps/vector-etl.yaml b/workflows/basemaps/vector-etl.yaml index 609344f56..b93047e4b 100644 --- a/workflows/basemaps/vector-etl.yaml +++ b/workflows/basemaps/vector-etl.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/cron/cron-vector-etl-roads-addressing.yaml b/workflows/cron/cron-vector-etl-roads-addressing.yaml index 850ff49ac..4e4fd52ca 100644 --- a/workflows/cron/cron-vector-etl-roads-addressing.yaml +++ b/workflows/cron/cron-vector-etl-roads-addressing.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: CronWorkflow metadata: diff --git a/workflows/cron/cron-vector-etl.yaml b/workflows/cron/cron-vector-etl.yaml index 5aff27e8d..219712f16 100644 --- a/workflows/cron/cron-vector-etl.yaml +++ b/workflows/cron/cron-vector-etl.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: CronWorkflow metadata: diff --git a/workflows/raster/copy.yaml b/workflows/raster/copy.yaml index 5e1956a79..9266bf451 100644 --- a/workflows/raster/copy.yaml +++ b/workflows/raster/copy.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/raster/publish-odr.yaml b/workflows/raster/publish-odr.yaml index c8188e9b5..6ef52fac9 100644 --- a/workflows/raster/publish-odr.yaml +++ b/workflows/raster/publish-odr.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 2ac87be0c..077b0142a 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/raster/tests.yaml b/workflows/raster/tests.yaml index 3288aea4b..5408d0b29 100644 --- a/workflows/raster/tests.yaml +++ b/workflows/raster/tests.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: Workflow diff --git a/workflows/stac/stac-validate.yaml b/workflows/stac/stac-validate.yaml index 23c46b88d..39ac3dd4c 100644 --- a/workflows/stac/stac-validate.yaml +++ b/workflows/stac/stac-validate.yaml @@ -1,4 +1,5 @@ ---- +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json + apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: diff --git a/workflows/test/env.yaml b/workflows/test/env.yaml index 74375e353..c4709d1d6 100644 --- a/workflows/test/env.yaml +++ b/workflows/test/env.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: Workflow diff --git a/workflows/test/flatten.yaml b/workflows/test/flatten.yaml index 4ecfdcc08..00f9c4eac 100644 --- a/workflows/test/flatten.yaml +++ b/workflows/test/flatten.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/test/generate-path.yaml b/workflows/test/generate-path.yaml index 64919a6ad..373270272 100644 --- a/workflows/test/generate-path.yaml +++ b/workflows/test/generate-path.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate diff --git a/workflows/test/list.arm.yaml b/workflows/test/list.arm.yaml index c426cf394..7b4bc241a 100644 --- a/workflows/test/list.arm.yaml +++ b/workflows/test/list.arm.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json # Example of using ARM + Spot instances for processing apiVersion: argoproj.io/v1alpha1 diff --git a/workflows/test/list.yaml b/workflows/test/list.yaml index e0b93c296..89d0813d6 100644 --- a/workflows/test/list.yaml +++ b/workflows/test/list.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: Workflow diff --git a/workflows/test/sleep.yml b/workflows/test/sleep.yml index 92bbdd253..a8b3a537a 100644 --- a/workflows/test/sleep.yml +++ b/workflows/test/sleep.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: Workflow diff --git a/workflows/util/create-thumbnails.yaml b/workflows/util/create-thumbnails.yaml index be1736722..d1d8bfe16 100644 --- a/workflows/util/create-thumbnails.yaml +++ b/workflows/util/create-thumbnails.yaml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.4.13/api/jsonschema/schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate From 3923381dda1765af524fa28aa803b6ce42324b94 Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Tue, 16 Apr 2024 13:45:43 +1200 Subject: [PATCH 10/28] fix: workflow-controller is not picking up ConfigMap changes TDE-1016 TDE-1112 (#534) #### Motivation [This change](https://github.com/linz/topo-workflows/pull/520) introduced the fact that Argo Workflow Controller is not picking up changes in ConfigMap, [as specified for the added `WATCH_CONTROLLER_SEMAPHORE_CONFIGMAPS` environment variable](https://argo-workflows.readthedocs.io/en/stable/environment-variables/#controller). An initial test showed that the `semaphore` changes were picked up, but, unfortunately, no further test on the controller ConfigMap has been done. #### Modification The workaround is to restart the controller at each kubernetes config deployment to make sure changes are picked up. This does not cause downtime. #### Checklist - [ ] Tests updated - [x] Docs updated - [x] Issue linked in Title --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1caac12e6..01ac6e0f8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,6 +84,9 @@ jobs: if: steps.get-infra-changes.outputs.run_infra == 'true' run: | kubectl apply -f dist/ + # FIXME since `WATCH_CONTROLLER_SEMAPHORE_CONFIGMAPS=false` we need to restart argo-workflow-controller + # to make sure ConfigMap changes are taken into account + kubectl rollout restart deployment argo-workflows-workflow-controller -n argo - name: Install Argo run: | From 882408c4a90e94f4db1d48455385b22f1ef3f934 Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:37:06 +1200 Subject: [PATCH 11/28] refactor: remove metadata.namespace from WorkflowTemplates and CronWorkflows TDE-1149 (#536) #### Motivation The `metadata.namespace` is overriden when the WorkflowTemplates and CronWorkflows are deployed (using `--namespace`). Having this data is redundant but also create issue while linting locally (`argo lint --offline`) the WorkflowTemplates with `templateRefs` when namespaces do not match. #### Modification Remove the default namespace in `metadata.namespace` from all WorkflowTemplate and CronWorkflow files #### Checklist - [ ] Tests updated N/A - [ ] Docs updated N/A - [x] Issue linked in Title --- workflows/basemaps/create-config.yaml | 1 - workflows/basemaps/imagery-import-cogify.yml | 1 - workflows/basemaps/mapsheet-json.yaml | 1 - workflows/basemaps/vector-etl.yaml | 1 - workflows/cron/cron-vector-etl-roads-addressing.yaml | 1 - workflows/cron/cron-vector-etl.yaml | 1 - workflows/raster/copy.yaml | 1 - workflows/raster/publish-odr.yaml | 1 - workflows/raster/standardising.yaml | 1 - workflows/stac/stac-validate.yaml | 1 - workflows/test/flatten.yaml | 1 - workflows/test/generate-path.yaml | 1 - workflows/util/create-thumbnails.yaml | 1 - 13 files changed, 13 deletions(-) diff --git a/workflows/basemaps/create-config.yaml b/workflows/basemaps/create-config.yaml index ed9e50b5a..9c7e8747b 100644 --- a/workflows/basemaps/create-config.yaml +++ b/workflows/basemaps/create-config.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: basemaps-config-create - namespace: argo labels: linz.govt.nz/category: basemaps spec: diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index 77d52eb2a..d8bd44964 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: basemaps-imagery-import-cogify - namespace: argo labels: linz.govt.nz/category: basemaps linz.govt.nz/data-type: raster diff --git a/workflows/basemaps/mapsheet-json.yaml b/workflows/basemaps/mapsheet-json.yaml index 0b1df6518..3204f314e 100644 --- a/workflows/basemaps/mapsheet-json.yaml +++ b/workflows/basemaps/mapsheet-json.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: basemaps-create-mapsheet-json - namespace: argo labels: linz.govt.nz/category: basemaps linz.govt.nz/data-type: raster diff --git a/workflows/basemaps/vector-etl.yaml b/workflows/basemaps/vector-etl.yaml index b93047e4b..51f086541 100644 --- a/workflows/basemaps/vector-etl.yaml +++ b/workflows/basemaps/vector-etl.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: basemaps-vector-etl - namespace: argo labels: linz.govt.nz/category: basemaps linz.govt.nz/data-type: vector diff --git a/workflows/cron/cron-vector-etl-roads-addressing.yaml b/workflows/cron/cron-vector-etl-roads-addressing.yaml index 4e4fd52ca..5a87c350b 100644 --- a/workflows/cron/cron-vector-etl-roads-addressing.yaml +++ b/workflows/cron/cron-vector-etl-roads-addressing.yaml @@ -3,7 +3,6 @@ apiVersion: argoproj.io/v1alpha1 kind: CronWorkflow metadata: name: cron-vector-etl-roads-addressing - namespace: argo labels: linz.govt.nz/category: basemaps linz.govt.nz/data-type: vector diff --git a/workflows/cron/cron-vector-etl.yaml b/workflows/cron/cron-vector-etl.yaml index 219712f16..cc5e0feea 100644 --- a/workflows/cron/cron-vector-etl.yaml +++ b/workflows/cron/cron-vector-etl.yaml @@ -3,7 +3,6 @@ apiVersion: argoproj.io/v1alpha1 kind: CronWorkflow metadata: name: cron-vector-etl-topographic - namespace: argo labels: linz.govt.nz/category: basemaps linz.govt.nz/data-type: vector diff --git a/workflows/raster/copy.yaml b/workflows/raster/copy.yaml index 9266bf451..27b48808f 100644 --- a/workflows/raster/copy.yaml +++ b/workflows/raster/copy.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: copy - namespace: argo labels: linz.govt.nz/category: raster linz.govt.nz/data-type: raster diff --git a/workflows/raster/publish-odr.yaml b/workflows/raster/publish-odr.yaml index 6ef52fac9..927d2cd24 100644 --- a/workflows/raster/publish-odr.yaml +++ b/workflows/raster/publish-odr.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: publish-odr - namespace: argo labels: linz.govt.nz/category: raster linz.govt.nz/data-type: raster diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 077b0142a..af23bbbd7 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: imagery-standardising - namespace: argo labels: linz.govt.nz/category: raster linz.govt.nz/data-type: raster diff --git a/workflows/stac/stac-validate.yaml b/workflows/stac/stac-validate.yaml index 39ac3dd4c..36171db1a 100644 --- a/workflows/stac/stac-validate.yaml +++ b/workflows/stac/stac-validate.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: stac-validate - namespace: argo labels: linz.govt.nz/category: stac spec: diff --git a/workflows/test/flatten.yaml b/workflows/test/flatten.yaml index 00f9c4eac..300e8adc7 100644 --- a/workflows/test/flatten.yaml +++ b/workflows/test/flatten.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: test-flatten - namespace: argo labels: linz.govt.nz/category: test spec: diff --git a/workflows/test/generate-path.yaml b/workflows/test/generate-path.yaml index 373270272..14030189e 100644 --- a/workflows/test/generate-path.yaml +++ b/workflows/test/generate-path.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: test-generate-target - namespace: argo labels: linz.govt.nz/category: test spec: diff --git a/workflows/util/create-thumbnails.yaml b/workflows/util/create-thumbnails.yaml index d1d8bfe16..2f7ecb3d4 100644 --- a/workflows/util/create-thumbnails.yaml +++ b/workflows/util/create-thumbnails.yaml @@ -4,7 +4,6 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: name: create-thumbnails - namespace: argo labels: linz.govt.nz/category: util linz.govt.nz/data-type: raster From e24bb0814744a682715b8876ca766a4a2fc9248d Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:37:29 +1200 Subject: [PATCH 12/28] fix: WorkflowTemplates linting fails TDE-1143 TDE-1149 (#537) #### Motivation An enum default value should be one of the enum values. `argo lint` fails on this. #### Modification Add the default value in the enum values. #### Checklist - [ ] Tests updated N/A - [ ] Docs updated N/A - [x] Issue linked in Title --- workflows/raster/publish-odr.yaml | 1 + workflows/raster/standardising.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/workflows/raster/publish-odr.yaml b/workflows/raster/publish-odr.yaml index 927d2cd24..98dc1615e 100644 --- a/workflows/raster/publish-odr.yaml +++ b/workflows/raster/publish-odr.yaml @@ -61,6 +61,7 @@ spec: enum: - 'nz-elevation' - 'nz-imagery' + - '' - name: copy_option value: '--no-clobber' enum: diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index af23bbbd7..8d0b8e8d6 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -278,6 +278,7 @@ spec: enum: - 'nz-imagery' - 'nz-elevation' + - '' - name: copy_option value: '--no-clobber' enum: From 9a242803396aa352ac129131859fb467b05de2aa Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Thu, 18 Apr 2024 09:18:41 +1200 Subject: [PATCH 13/28] refactor: store default region in const (#508) #### Motivation It is easier to maintain the region of our AWS stack if we have to modify it in a single place. #### Modification Store the region in a constant. #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated N/A - [x] Docs updated - [ ] Issue linked in Title N/A --- infra/cdk.ts | 4 ++-- infra/charts/argo.workflows.ts | 4 ++-- infra/charts/fluentbit.ts | 5 +++-- infra/constants.ts | 14 ++++++++------ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/infra/cdk.ts b/infra/cdk.ts index 15ffad9ea..b2aa6cb39 100644 --- a/infra/cdk.ts +++ b/infra/cdk.ts @@ -1,6 +1,6 @@ import { App } from 'aws-cdk-lib'; -import { ClusterName } from './constants.js'; +import { ClusterName, DefaultRegion } from './constants.js'; import { tryGetContextArns } from './eks/arn.js'; import { LinzEksCluster } from './eks/cluster.js'; import { fetchSsmParameters } from './util/ssm.js'; @@ -22,7 +22,7 @@ async function main(): Promise { } new LinzEksCluster(app, ClusterName, { - env: { region: 'ap-southeast-2', account: accountId }, + env: { region: DefaultRegion, account: accountId }, maintainerRoleArns, slackChannelConfigurationName: slackSsmConfig.slackChannelConfigurationName, slackWorkspaceId: slackSsmConfig.slackWorkspaceId, diff --git a/infra/charts/argo.workflows.ts b/infra/charts/argo.workflows.ts index a4720a108..60790143e 100644 --- a/infra/charts/argo.workflows.ts +++ b/infra/charts/argo.workflows.ts @@ -2,7 +2,7 @@ import { Chart, ChartProps, Duration, Helm } from 'cdk8s'; import { Secret } from 'cdk8s-plus-27'; import { Construct } from 'constructs'; -import { ArgoDbName, ArgoDbUser } from '../constants.js'; +import { ArgoDbName, ArgoDbUser, DefaultRegion } from '../constants.js'; import { applyDefaultLabels } from '../util/labels.js'; export interface ArgoWorkflowsProps { @@ -65,7 +65,7 @@ export class ArgoWorkflows extends Chart { bucket: props.tempBucketName, keyFormat: '{{workflow.creationTimestamp.Y}}-{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}-{{workflow.name}}/{{pod.name}}', - region: 'ap-southeast-2', + region: DefaultRegion, endpoint: 's3.amazonaws.com', useSDKCreds: true, insecure: false, diff --git a/infra/charts/fluentbit.ts b/infra/charts/fluentbit.ts index 9121eda42..956b7f4e3 100644 --- a/infra/charts/fluentbit.ts +++ b/infra/charts/fluentbit.ts @@ -1,6 +1,7 @@ import { Chart, ChartProps, Helm } from 'cdk8s'; import { Construct } from 'constructs'; +import { DefaultRegion } from '../constants.js'; import { applyDefaultLabels } from '../util/labels.js'; /** @@ -73,9 +74,9 @@ HC_Period 5 serviceAccount: { name: props.saName, create: false }, cloudWatchLogs: { enabled: true, - region: 'ap-southeast-2', + region: DefaultRegion, /** Specify Cloudwatch endpoint to add a trailing `.` to force FQDN DNS request */ - endpoint: 'logs.ap-southeast-2.amazonaws.com.', + endpoint: `logs.${DefaultRegion}.amazonaws.com.`, autoCreateGroup: true, logRetentionDays: 30, logGroupName: `/aws/eks/${props.clusterName}/logs`, diff --git a/infra/constants.ts b/infra/constants.ts index 077a16a9b..bf6e530a9 100644 --- a/infra/constants.ts +++ b/infra/constants.ts @@ -1,13 +1,15 @@ -/* Cluster name */ +/** Cluster name */ export const ClusterName = 'Workflows'; -/* LINZ conventional name for Argo Workflows artifact bucket */ +/** LINZ conventional name for Argo Workflows artifact bucket */ export const ScratchBucketName = `linz-${ClusterName.toLowerCase()}-scratch`; -/* Argo Database Instance name */ +/** Argo Database Instance name */ export const ArgoDbInstanceName = 'ArgoDb'; -/* Argo Database name */ +/** Argo Database name */ export const ArgoDbName = 'argo'; -/* Argo Database user */ +/** Argo Database user */ export const ArgoDbUser = 'argo_user'; +/** AWS default region for our stack */ +export const DefaultRegion = 'ap-southeast-2'; /** * Should NodeLocal DNS be enabled for the cluster @@ -16,7 +18,7 @@ export const ArgoDbUser = 'argo_user'; */ export const UseNodeLocalDns = true; -/* CloudFormation Output to access from CDK8s */ +/** CloudFormation Output to access from CDK8s */ export const CfnOutputKeys = { ClusterEndpoint: 'ClusterEndpoint', From bb05a6a526981fe10d97fde379eba1528e266032 Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:09:27 +1200 Subject: [PATCH 14/28] ci: use --offline mode to lint workflows TDE-1149 TDE-1143 (#540) #### Motivation The current process of linting the workflows is complex as we need to publish them on the kubernetes cluster. Using an offline mode makes things simpler and quicker, removing the connection to the cluster. #### Modification Use the `--offline` mode to run `argo lint`, rather than linting from the resources deployed on the cluster. #### Checklist - [ ] Tests updated - [ ] Docs updated - [X] Issue linked in Title --- .github/workflows/main.yml | 67 +++++++------------------------------- 1 file changed, 11 insertions(+), 56 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 01ac6e0f8..f4e259a6b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,6 +13,17 @@ jobs: - name: Run actionlint to check workflow files run: docker run --volume="${PWD}:/repo" --workdir=/repo actionlint -color + - name: Install Argo + run: | + curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.5.5/argo-linux-amd64.gz + gunzip argo-linux-amd64.gz + chmod +x argo-linux-amd64 + ./argo-linux-amd64 version + + - name: Lint workflows + run: | + ./argo-linux-amd64 lint --offline templates/ workflows/ + deploy-prod: runs-on: ubuntu-latest concurrency: deploy-prod-${{ github.ref }} @@ -84,62 +95,6 @@ jobs: if: steps.get-infra-changes.outputs.run_infra == 'true' run: | kubectl apply -f dist/ - # FIXME since `WATCH_CONTROLLER_SEMAPHORE_CONFIGMAPS=false` we need to restart argo-workflow-controller - # to make sure ConfigMap changes are taken into account - kubectl rollout restart deployment argo-workflows-workflow-controller -n argo - - - name: Install Argo - run: | - curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.4.0-rc2/argo-linux-amd64.gz - gunzip argo-linux-amd64.gz - chmod +x argo-linux-amd64 - ./argo-linux-amd64 version - - - name: Lint workflows - if: github.ref != 'refs/heads/master' - run: | - # Create test namespace - kubectl create namespace "$GITHUB_SHA" - - # Create copy of Workflows files to change their namespaces - mkdir test - cp -r workflows/ test/workflows/ - - # Deploy templates in the test namespace - # Note: the templates have no default namespace so no need to modify them - kubectl apply -f templates/argo-tasks/ --namespace "$GITHUB_SHA" - - # Find all workflows that have kind "WorkflowTemplate" - WORKFLOWS=$(grep -R -H '^kind: WorkflowTemplate$' test/workflows/ | cut -d ':' -f1) - # For each workflow template attempt to deploy it using kubectl - for wf in $WORKFLOWS; do - # Change namespace in files - sed -i "/^\([[:space:]]*namespace: \).*/s//\1$GITHUB_SHA/" "$wf" - kubectl apply -f "$wf" --namespace "$GITHUB_SHA" - done - - # Find all cron workflows that have kind "CronWorkflow" - CRON_WORKFLOWS=$(grep -R -H '^kind: CronWorkflow$' test/workflows/ | cut -d ':' -f1) - # For each cron workflow attempt to deploy it using kubectl - for cwf in $CRON_WORKFLOWS; do - # Change namespace in files - sed -i "/^\([[:space:]]*namespace: \).*/s//\1$GITHUB_SHA/" "$cwf" - kubectl apply -f "$cwf" --namespace "$GITHUB_SHA" - done - - # Finally lint the templates - ./argo-linux-amd64 lint templates/ -n "$GITHUB_SHA" - ./argo-linux-amd64 lint test/workflows/ -n "$GITHUB_SHA" - - - name: Delete Test namespace - if: always() - run: | - # Delete the test namespace - stderr_tmp="$(mktemp --directory)/stderr" - if ! kubectl delete namespaces "$GITHUB_SHA" 2> >(tee "$stderr_tmp" >&2) - then - grep -q 'Error from server (NotFound): namespaces ".*" not found' "$stderr_tmp" - fi - name: Deploy workflows if: github.ref == 'refs/heads/master' From 5f578a60db391e95e17ea1eef495076990617d3e Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Thu, 18 Apr 2024 12:36:03 +1200 Subject: [PATCH 15/28] fix: enable Basemaps preview for Elevation data TDE-1151 BM-932 (#543) #### Motivation Enable the Basemaps configuration for `dem_lerc`, as Basemaps now supports previewing Elevation data. #### Modification Remove exclusion for `dem_lerc` preset. #### Checklist - [ ] Tests updated N/A - [x] Docs updated - [x] Issue linked in Title --- workflows/raster/README.md | 4 ++-- workflows/raster/standardising.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/workflows/raster/README.md b/workflows/raster/README.md index 984ca2c62..c0ec484bb 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -121,9 +121,9 @@ graph TD; get-location-->standardise-validate; tileindex-validate-->standardise-validate; standardise-validate-->create-collection; - standardise-validate-.->|compression != dem_lerc|create-overview; + standardise-validate-->create-overview; create-collection-->stac-validate-.->|publish_to_odr == true|publish-odr; - create-overview-.->create-config-.->|publish_to_odr == true|publish-odr; + create-overview-.->|compression != dem_lerc|create-config-.->|publish_to_odr == true|publish-odr; ``` ### [collection-id-setup](./standardising.yaml) diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 8d0b8e8d6..15d34d359 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -386,7 +386,7 @@ spec: depends: 'standardise-validate' - name: create-config - when: "'{{workflow.parameters.target_epsg}}' =~ '2193|3857' && '{{workflow.parameters.compression}}' != 'dem_lerc'" + when: "'{{workflow.parameters.target_epsg}}' =~ '2193|3857'" arguments: parameters: - name: location From 43af2ea100e13f2d55c8b802020bd8f6afb3730a Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Thu, 18 Apr 2024 15:17:23 +1200 Subject: [PATCH 16/28] refactor: templating stac-validate workflow TDE-1136 (#530) #### Motivation `stac-validate` is used in many places, it should be easier to maintain and reuse if it was a "template". #### Modification Create a `stac-validate` template so it can be re-use by workflows #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [x] Docs updated - [x] Issue linked in Title --- templates/argo-tasks/README.md | 26 ++++++++- templates/argo-tasks/stac-validate.yml | 55 +++++++++++++++++++ workflows/raster/standardising.yaml | 21 ++----- workflows/stac/README.md | 23 +------- ...idate.yaml => stac-validate-parallel.yaml} | 35 +++--------- 5 files changed, 96 insertions(+), 64 deletions(-) create mode 100644 templates/argo-tasks/stac-validate.yml rename workflows/stac/{stac-validate.yaml => stac-validate-parallel.yaml} (70%) diff --git a/templates/argo-tasks/README.md b/templates/argo-tasks/README.md index ad11000d9..cc44fd635 100644 --- a/templates/argo-tasks/README.md +++ b/templates/argo-tasks/README.md @@ -178,7 +178,7 @@ See https://github.com/linz/argo-tasks#stac-github-import Template to build ODR target paths using collection metadata. See https://github.com/linz/argo-tasks#generate-paths -## Template Usage +### Template Usage ```yaml name: generate-path @@ -194,3 +194,27 @@ arguments: - name: source value: '{{inputs.parameters.source}}' ``` + +## argo-tasks/stac-validate + +Template to validate STAC Collections and Items against [STAC](https://stacspec.org/) schemas and STAC Extension schemas. +See (https://github.com/linz/argo-tasks#stac-validate) + +### Template Usage + +```yaml +- name: stac-validate + templateRef: + name: tpl-at-stac-validate + template: main + arguments: + parameters: + - name: uri + value: 's3://my-bucket/path/collection.json' + - name: checksum + value: '{{workflow.parameters.checksum}}' + - name: recursive + value: '{{workflow.parameters.recursive}}' + - name: concurrency + value: '20' +``` diff --git a/templates/argo-tasks/stac-validate.yml b/templates/argo-tasks/stac-validate.yml new file mode 100644 index 000000000..77b8d6ba4 --- /dev/null +++ b/templates/argo-tasks/stac-validate.yml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json + +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + # Template from linz/argo-tasks + # see https://github.com/linz/argo-tasks?tab=readme-ov-file#stac-validate + name: tpl-at-stac-validate +spec: + templateDefaults: + container: + imagePullPolicy: Always + image: '' + entrypoint: main + templates: + - name: main + inputs: + parameters: + - name: uri + description: STAC file uri to validate + default: '' + + - name: recursive + description: Follow and validate STAC links + default: 'true' + + - name: concurrency + description: Number of requests to run concurrently + default: '50' + + - name: checksum + description: Validate the file:checksum if it exists + default: 'false' + + - name: version + description: container version to use + default: 'v3' + + container: + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version)}}' + resources: + requests: + cpu: 15000m + memory: 7.8Gi + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + - 'stac' + - 'validate' + - '--concurrency={{inputs.parameters.concurrency}}' + - '--recursive={{inputs.parameters.recursive}}' + - '--checksum={{inputs.parameters.checksum}}' + - '{{inputs.parameters.uri}}' diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 15d34d359..0cb4b4ad6 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -276,6 +276,7 @@ spec: - name: target_bucket_name value: '' enum: + - '' - 'nz-imagery' - 'nz-elevation' - '' @@ -362,11 +363,13 @@ spec: depends: 'standardise-validate' - name: stac-validate - template: stac-validate + templateRef: + name: tpl-at-stac-validate + template: main arguments: parameters: - - name: location - value: '{{tasks.get-location.outputs.parameters.location}}' + - name: uri + value: '{{tasks.get-location.outputs.parameters.location}}flat/collection.json' artifacts: - name: stac-result raw: @@ -542,18 +545,6 @@ spec: - '--concurrency' - '25' - - name: stac-validate - inputs: - parameters: - - name: location - container: - image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' - command: [node, /app/index.js] - env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json - args: ['stac', 'validate', '--recursive', '{{inputs.parameters.location}}flat/collection.json'] - - name: get-location script: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' diff --git a/workflows/stac/README.md b/workflows/stac/README.md index 464c78da0..ddb980333 100644 --- a/workflows/stac/README.md +++ b/workflows/stac/README.md @@ -1,25 +1,6 @@ -# Contents +# stac-validate-parallel -- [stac-validate](#stac-validate) - -# stac-validate - -Validate STAC Collections and Items against [STAC](https://stacspec.org/) schemas and STAC Extension schemas. -Uses the [argo-tasks](https://github.com/linz/argo-tasks#stac-validate) container `stac-validate` command. - -## Workflow Input Parameters - -| Parameter | Type | Default | Description | -| --------- | ----- | --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | -| uri | str | s3://linz-imagery-staging/test/stac-validate/ | The full AWS S3 URI (path) to the STAC file(s) | -| include | regex | `collection.json$` | Regular expression to match object path(s) or name(s) from within the source path to include in STAC validation. | -| checksum | enum | false | Set to "true" to validate the checksums of linked asset files. | - -The `--recursive` flag is specified inside the STAC Validate WorkflowTemplate. Linked STAC items linked to from a STAC collection will also be validated. - -The STAC Validate Workflow will validate each collection (and linked items/assets) in a separate pod so that multiple collections can be processed in parallel. - -Access permissions are controlled by the [Bucket Sharing Config](https://github.com/linz/topo-aws-infrastructure/blob/master/src/stacks/bucket.sharing.ts) which gives Argo Workflows access to the S3 buckets we use. +This Workflow will validate each collection (and linked items/assets) in a separate pod so that multiple collections can be processed in parallel, using the `tpl-at-stac-validate` template. ## Workflow Outputs diff --git a/workflows/stac/stac-validate.yaml b/workflows/stac/stac-validate-parallel.yaml similarity index 70% rename from workflows/stac/stac-validate.yaml rename to workflows/stac/stac-validate-parallel.yaml index 36171db1a..7895d390b 100644 --- a/workflows/stac/stac-validate.yaml +++ b/workflows/stac/stac-validate-parallel.yaml @@ -3,7 +3,7 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - name: stac-validate + name: stac-validate-parallel labels: linz.govt.nz/category: stac spec: @@ -29,6 +29,7 @@ spec: templateDefaults: container: imagePullPolicy: Always + image: '' templates: - name: main dag: @@ -36,11 +37,15 @@ spec: - name: aws-list-collections template: aws-list-collections - name: stac-validate-collections - template: stac-validate-collections + templateRef: + name: tpl-at-stac-validate + template: main arguments: parameters: - - name: file + - name: uri value: '{{item}}' + - name: checksum + value: '{{workflow.parameters.checksum}}' depends: aws-list-collections withParam: '{{tasks.aws-list-collections.outputs.parameters.files}}' - name: aws-list-collections @@ -67,27 +72,3 @@ spec: - name: files valueFrom: path: /tmp/file_list.json - - name: stac-validate-collections - inputs: - parameters: - - name: file - container: - image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' - resources: - requests: - cpu: 15000m - memory: 7.8Gi - command: [node, /app/index.js] - env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json - args: - [ - 'stac', - 'validate', - '--concurrency', - '50', - '--recursive', - '--checksum={{workflow.parameters.checksum}}', - '{{inputs.parameters.file}}', - ] From 8022c8daf6c69bcb828d00e57dafef1228d88771 Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Fri, 19 Apr 2024 09:56:25 +1200 Subject: [PATCH 17/28] fix: node nodeAntiAffinity seems to break karpenter TDE-1112 BM-1018 (#545) #### Motivation We've noticed that karpenter is erroring since https://github.com/linz/topo-workflows/pull/506. Errors makes us thinking that it could be related. Since the retry on a new node using `nodeAntiAffinity` [is not working](https://github.com/argoproj/argo-workflows/pull/12701), it's a good idea to remove it to see if it solved `karpenter` issues. #### Modification Remove the `affinity` in the `retryStrategy`. #### Checklist - [ ] Tests updated - [x] Docs updated - [x] Issue linked in Title --- infra/charts/argo.workflows.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/charts/argo.workflows.ts b/infra/charts/argo.workflows.ts index 60790143e..0ab49a876 100644 --- a/infra/charts/argo.workflows.ts +++ b/infra/charts/argo.workflows.ts @@ -147,8 +147,10 @@ export class ArgoWorkflows extends Chart { }, ], parallelism: 3, - // FIXME: `nodeAntiAffinity` - to retry on different node - is not working yet (https://github.com/argoproj/argo-workflows/pull/12701) - retryStrategy: { limit: 2, affinity: { nodeAntiAffinity: {} } }, + /** TODO: `nodeAntiAffinity` - to retry on different node - is not working yet (https://github.com/argoproj/argo-workflows/pull/12701) + * `affinity: { nodeAntiAffinity: {} }` seems to break `karpenter`, need more investigation + */ + retryStrategy: { limit: 2 }, }, }, }, From 48ec91615ce7b0eb81f55e6dd182c8c2025dcbda Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Fri, 19 Apr 2024 10:56:10 +1200 Subject: [PATCH 18/28] fix: pass compression as preset to tileindex-validate TDE-895 TDE-1151 (#546) #### Motivation We only want to check `webp` datasets are 8-bit so Argo Tasks tileindex-validate needs to know the dataset type. #### Modification Pass the `compression` type of the dataset to Argo Tasks tileindex-validate as `preset` for validation #### Checklist - [ ] Tests updated N/A - [x] Docs updated - [x] Issue linked in Title --- templates/argo-tasks/tile-index-validate.yml | 5 +++++ workflows/raster/README.md | 2 +- workflows/raster/standardising.yaml | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/templates/argo-tasks/tile-index-validate.yml b/templates/argo-tasks/tile-index-validate.yml index a1b5cb5a7..2556a9b51 100644 --- a/templates/argo-tasks/tile-index-validate.yml +++ b/templates/argo-tasks/tile-index-validate.yml @@ -39,6 +39,10 @@ spec: description: Output tile configuration for retiling default: 'false' + - name: preset + description: Compression type of dataset e.g. webp, lzw, dem_lerc + default: 'webp' + - name: version description: container version to use default: 'v3' @@ -78,6 +82,7 @@ spec: - '--scale={{= inputs.parameters.scale }}' - '--validate={{= inputs.parameters.validate }}' - '--retile={{= inputs.parameters.retile }}' + - '--preset={{= inputs.parameters.compression }}' - "{{= sprig.empty(inputs.parameters.source_epsg) ? '' : '--source-epsg=' + inputs.parameters.source_epsg }}" - "{{= sprig.empty(inputs.parameters.include) ? '' : '--include=' + inputs.parameters.include }}" - '{{= sprig.trim(inputs.parameters.source) }}' diff --git a/workflows/raster/README.md b/workflows/raster/README.md index c0ec484bb..e48ef9063 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -133,7 +133,7 @@ If no input collection ID is provided a ULID is generated and used as the collec ### [tileindex-validate](https://github.com/linz/argo-tasks/blob/master/src/commands/tileindex-validate/) -Lists tiffs from source input, validates they match a LINZ Mapsheet tile index and asserts that there will be no duplicates. +Lists tiffs from source input, validates they match a LINZ Mapsheet tile index and asserts that there will be no duplicates. Checks `webp` files are 8-bit. ### [standardise-validate](https://github.com/linz/topo-imagery/blob/master/scripts/standardise_validate.py) diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 0cb4b4ad6..562f42c0d 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -318,6 +318,8 @@ spec: value: '{{= workflow.parameters.validate}}' - name: retile value: '{{= workflow.parameters.retile}}' + - name: preset + value: '{{= workflow.parameters.compression}}' - name: version value: '{{= workflow.parameters.version_argo_tasks}}' From 62168e3551358e4804aaf71adcc407712eeaad44 Mon Sep 17 00:00:00 2001 From: Alice Fage Date: Fri, 19 Apr 2024 11:04:46 +1200 Subject: [PATCH 19/28] fix: preset parameter name TDE-895 TDE-1151 (#547) #### Motivation Mistake in parameter name, renamed to `preset` --- templates/argo-tasks/tile-index-validate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/argo-tasks/tile-index-validate.yml b/templates/argo-tasks/tile-index-validate.yml index 2556a9b51..3fe143d34 100644 --- a/templates/argo-tasks/tile-index-validate.yml +++ b/templates/argo-tasks/tile-index-validate.yml @@ -82,7 +82,7 @@ spec: - '--scale={{= inputs.parameters.scale }}' - '--validate={{= inputs.parameters.validate }}' - '--retile={{= inputs.parameters.retile }}' - - '--preset={{= inputs.parameters.compression }}' + - '--preset={{= inputs.parameters.preset }}' - "{{= sprig.empty(inputs.parameters.source_epsg) ? '' : '--source-epsg=' + inputs.parameters.source_epsg }}" - "{{= sprig.empty(inputs.parameters.include) ? '' : '--include=' + inputs.parameters.include }}" - '{{= sprig.trim(inputs.parameters.source) }}' From 6c16f6b3594b08cc3a009a08ed77b304c3e030e6 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Fri, 19 Apr 2024 11:37:56 +1200 Subject: [PATCH 20/28] build: upgrade basemaps cli to v7 for DEM support BM-932 (#548) #### Motivation Basemaps need to be on v7+ for DEM support #### Modification Upgrades basmeaps cli from v6 to v7 Locks create-overview to v6 until overview creation is redone in v7 BM-944 #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title --- workflows/basemaps/create-config.yaml | 2 +- workflows/raster/standardising.yaml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/workflows/basemaps/create-config.yaml b/workflows/basemaps/create-config.yaml index 9c7e8747b..baae7ecfd 100644 --- a/workflows/basemaps/create-config.yaml +++ b/workflows/basemaps/create-config.yaml @@ -11,7 +11,7 @@ spec: arguments: parameters: - name: version_basemaps_cli - value: 'v6' + value: 'v7' - name: location value: 's3://bucket/path/to' templateDefaults: diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 562f42c0d..1a9612ee5 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -28,7 +28,7 @@ spec: - name: version_argo_tasks value: 'v3' - name: version_basemaps_cli - value: 'v6' + value: 'v7' - name: version_topo_imagery value: 'v4' - name: ticket @@ -575,7 +575,8 @@ spec: parameters: - name: location container: - image: 'ghcr.io/linz/basemaps/cli:{{=sprig.trim(workflow.parameters.version_basemaps_cli)}}' + # Basemaps v7+ has removed overview creation + image: 'ghcr.io/linz/basemaps/cli:v6' resources: requests: cpu: 3000m From 4c06ed6914bc699734342eed96c879be751f8516 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 23 Apr 2024 13:43:54 +1200 Subject: [PATCH 21/28] feat(basemaps): Upgrade vector elt include title in the stac. BM-1017 (#550) #### Motivation The new version of vector etl container to which include the fix of road addressing big tiles, and include the title into stac file. #### Modification - include title in the etl cmd #### Checklist - [ ] Tests updated - [ ] Docs updated - [x] Issue linked in Title --- workflows/basemaps/vector-etl.yaml | 5 +++++ workflows/cron/cron-vector-etl-roads-addressing.yaml | 2 ++ workflows/cron/cron-vector-etl.yaml | 2 ++ 3 files changed, 9 insertions(+) diff --git a/workflows/basemaps/vector-etl.yaml b/workflows/basemaps/vector-etl.yaml index 51f086541..da09679c6 100644 --- a/workflows/basemaps/vector-etl.yaml +++ b/workflows/basemaps/vector-etl.yaml @@ -30,6 +30,10 @@ spec: description: output filename in s3 value: 'topographic' + - name: title + description: title for the output data, which will populate in stac file + value: 'Topographic' + - name: create_pull_request description: Should a pull request be created in linz/basemaps-config value: 'true' @@ -82,6 +86,7 @@ spec: - "{{= workflow.parameters.filename == 'topographic'? '--all' : '--layer=' + workflow.parameters.filename }}" - '--target={{ workflow.parameters.target }}' - '--filename={{ workflow.parameters.filename }}' + - '--title={{ workflow.parameters.title }}' - '--output=/tmp/' - '--commit' outputs: diff --git a/workflows/cron/cron-vector-etl-roads-addressing.yaml b/workflows/cron/cron-vector-etl-roads-addressing.yaml index 5a87c350b..718a61e3e 100644 --- a/workflows/cron/cron-vector-etl-roads-addressing.yaml +++ b/workflows/cron/cron-vector-etl-roads-addressing.yaml @@ -21,5 +21,7 @@ spec: parameters: - name: 'filename' value: '53382-nz-roads-addressing' + - name: 'title' + value: 'NZ Roads (Addressing)' - name: 'retry' value: '2' diff --git a/workflows/cron/cron-vector-etl.yaml b/workflows/cron/cron-vector-etl.yaml index cc5e0feea..5ea37f868 100644 --- a/workflows/cron/cron-vector-etl.yaml +++ b/workflows/cron/cron-vector-etl.yaml @@ -21,5 +21,7 @@ spec: parameters: - name: 'filename' value: 'topographic' + - name: 'title' + value: 'Topographic' - name: 'retry' value: '2' From 2df071cff36c901a46b8766373f1791e67ff8932 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Tue, 23 Apr 2024 13:57:04 +1200 Subject: [PATCH 22/28] feat: include ticket number in basemaps pull requests TDE-1146 (#535) #### Motivation It is helpful to have all pull requests linked to the ticket, since we now have the ticket number in the workflow we should pass it into anything interacting with github #### Modification Adds ticket number to basemaps `create-pr` action. This pull request needs https://github.com/linz/argo-tasks/pull/962 to be merged first. #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title --- workflows/basemaps/imagery-import-cogify.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index d8bd44964..e61fd1384 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -371,10 +371,9 @@ spec: name: github-linz-li-bot-pat key: pat args: - [ - 'bmc', - 'create-pr', - '--target={{inputs.parameters.target}}', - "--individual={{= workflow.parameters.create_pull_request == 'individual'? 'true' : 'false' }}", - '--category={{workflow.parameters.category}}', - ] + - 'bmc' + - 'create-pr' + - '--target={{inputs.parameters.target}}' + - '--ticket={{workflow.parameters.ticket}}' + - "--individual={{= workflow.parameters.create_pull_request == 'individual'? 'true' : 'false' }}" + - '--category={{workflow.parameters.category}}' From 7b6c360ccddee7589a38a13213454ad8ba79c816 Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Tue, 7 May 2024 21:33:45 +0000 Subject: [PATCH 23/28] feat: Add option to create capture area TDE-1159 (#566) #### Motivation Give the user control of whether we create capture areas for datasets. This is a workaround for some footprints resulting in unusably complex capture areas. #### Modification New `create_capture_area` parameter, passed through as `--create-footprints`. Depends on . #### Checklist - [ ] Tests updated (N/A) - [x] Docs updated - [x] Issue linked in Title --- workflows/raster/README.md | 62 +++++++++++++++-------------- workflows/raster/standardising.yaml | 8 ++++ 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/workflows/raster/README.md b/workflows/raster/README.md index e48ef9063..6eb8522ea 100644 --- a/workflows/raster/README.md +++ b/workflows/raster/README.md @@ -15,36 +15,37 @@ Publishing to the AWS Registry of Open Data is an optional step [publish-odr](#P ## Workflow Input Parameters -| Parameter | Type | Default | Description | -| ---------------------- | ----- | ------------------------------------- | -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ticket | str | | Ticket ID e.g. 'AIP-55' | -| region | enum | | Region of the dataset | -| source | str | s3://linz-imagery-staging/test/sample | the uri (path) to the input tiffs | -| include | regex | .tiff?$ | A regular expression to match object path(s) or name(s) from within the source path to include in standardising\*. | -| scale | enum | 500 | The scale of the TIFFs | -| validate | enum | true | Validate the TIFFs files with `tileindex-validate`. | -| retile | enum | false | Prepare the data for retiling TIFFs files to `scale` with `tileindex-validate`. | -| group | int | 50 | The number of files to group into the pods (testing has recommended using 50 for large datasets). | -| compression | enum | webp | Standardised file format | -| cutline | str | | (Optional) location of a cutline file to cut the imagery to `.fgb` or `.geojson` (leave blank if no cutline) | -| collection_id | str | | (Optional) Provide a Collection ID if re-processing an existing published survery, otherwise a ULID will be generated for the collection.json ID field. | -| category | enum | urban-aerial-photos | Dataset type for collection metadata, also used to Build Dataset title & description | -| gsd | str | 0.3m | Dataset GSD for collection metadata, also used to build dataset title | -| producer | enum | Unknown | Imagery producer :warning: Ignored if `producer_list` is used. | -| producer_list | str | | List of imagery producers, separated by semicolon (;). :warning: Has no effect unless a semicolon delimited list is entered. | -| licensor | enum | Unknown | Imagery licensor. :warning: Ignored if `licensor_list` is used. | -| licensor_list | str | | List of imagery licensors, separated by semicolon (;). :warning: Has no effect unless a semicolon delimited list is entered. | -| start_datetime | str | YYYY-MM-DD | Imagery start date (flown from), must be in default formatting | -| end_datetime | str | YYYY-MM-DD | Imagery end date (flown to), must be in default formatting | -| geographic_description | str | Hamilton | (Optional) Additional datatset description, to be used in dataset title / description in place of the Region. | -| lifeycle | enum | Completed | Lifecycle Status of Collection, from [linz STAC extension](https://github.com/linz/stac/tree/master/extensions/linz#collection-fields). Options: `completed`, `preview`, `ongoing`, `under development`, `deprecated` | -| event | str | Cyclone Gabrielle | (Optional) Event name if dataset has been captured in association with an event. | -| historic_survey_number | str | SNC8844 | (Optional) Survey Number associated with historical datasets. | -| source_epsg | str | 2193 | The EPSG code of the source imagery | -| target_epsg | str | 2193 | The target EPSG code - if different to source the imagery will be reprojected | -| publish_to_odr | str | false | Run [publish-odr](#Publish-odr) after standardising has completed successfully | -| target_bucket_name | enum | | Used only if `publish_to_odr` is true. The bucket name of the target ODR location | -| copy_option | enum | --no-clobber | Used only if `publish_to_odr` is true.
`--no-clobber`
Skip overwriting existing files.
`--force`
Overwrite all files.
`--force-no-clobber`
Overwrite only changed files, skip unchanged files.
| +| Parameter | Type | Default | Description | +| ---------------------- | ----- | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ticket | str | | Ticket ID e.g. 'AIP-55' | +| region | enum | | Region of the dataset | +| source | str | s3://linz-imagery-staging/test/sample | the uri (path) to the input tiffs | +| include | regex | .tiff?$ | A regular expression to match object path(s) or name(s) from within the source path to include in standardising\*. | +| scale | enum | 500 | The scale of the TIFFs | +| validate | enum | true | Validate the TIFFs files with `tileindex-validate`. | +| retile | enum | false | Prepare the data for retiling TIFFs files to `scale` with `tileindex-validate`. | +| group | int | 50 | The number of files to group into the pods (testing has recommended using 50 for large datasets). | +| compression | enum | webp | Standardised file format | +| create_capture_area | enum | true | Create a GeoJSON capture area for the dataset | +| cutline | str | | (Optional) location of a cutline file to cut the imagery to `.fgb` or `.geojson` (leave blank if no cutline) | +| collection_id | str | | (Optional) Provide a Collection ID if re-processing an existing published survery, otherwise a ULID will be generated for the collection.json ID field. | +| category | enum | urban-aerial-photos | Dataset type for collection metadata, also used to Build Dataset title & description | +| gsd | str | 0.3m | Dataset GSD for collection metadata, also used to build dataset title | +| producer | enum | Unknown | Imagery producer :warning: Ignored if `producer_list` is used. | +| producer_list | str | | List of imagery producers, separated by semicolon (;). :warning: Has no effect unless a semicolon delimited list is entered. | +| licensor | enum | Unknown | Imagery licensor. :warning: Ignored if `licensor_list` is used. | +| licensor_list | str | | List of imagery licensors, separated by semicolon (;). :warning: Has no effect unless a semicolon delimited list is entered. | +| start_datetime | str | YYYY-MM-DD | Imagery start date (flown from), must be in default formatting | +| end_datetime | str | YYYY-MM-DD | Imagery end date (flown to), must be in default formatting | +| geographic_description | str | Hamilton | (Optional) Additional datatset description, to be used in dataset title / description in place of the Region. | +| lifeycle | enum | Completed | Lifecycle Status of Collection, from [linz STAC extension](https://github.com/linz/stac/tree/master/extensions/linz#collection-fields). Options: `completed`, `preview`, `ongoing`, `under development`, `deprecated` | +| event | str | Cyclone Gabrielle | (Optional) Event name if dataset has been captured in association with an event. | +| historic_survey_number | str | SNC8844 | (Optional) Survey Number associated with historical datasets. | +| source_epsg | str | 2193 | The EPSG code of the source imagery | +| target_epsg | str | 2193 | The target EPSG code - if different to source the imagery will be reprojected | +| publish_to_odr | str | false | Run [publish-odr](#Publish-odr) after standardising has completed successfully | +| target_bucket_name | enum | | Used only if `publish_to_odr` is true. The bucket name of the target ODR location | +| copy_option | enum | --no-clobber | Used only if `publish_to_odr` is true.
`--no-clobber`
Skip overwriting existing files.
`--force`
Overwrite all files.
`--force-no-clobber`
Overwrite only changed files, skip unchanged files.
| \* This regex can be used to exclude paths as well, e.g. if there are RBG and RGBI directories, the following regex will only include TIFF files in the RGB directory: `RGB(?!I).*.tiff?$`. For more complicated exclusions, there is an `--exclude` parameter, which would need to be added to the Argo WorkflowTemplate. @@ -61,6 +62,7 @@ Publishing to the AWS Registry of Open Data is an optional step [publish-odr](#P | retile | false | | group | 50 | | compression | webp | +| create_capture_area | true | | cutline | s3://linz-imagery-staging/cutline/bay-of-plenty_2021-2022.fgb | | collection_id | 01FP371BHWDSREECKQAH9E8XQ | | category | rural-aerial-photos | diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 1a9612ee5..41d7d91cf 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -94,6 +94,11 @@ spec: - 'webp' - 'lzw' - 'dem_lerc' + - name: create_capture_area + value: 'true' + enum: + - 'false' + - 'true' - name: cutline description: '(Optional) location of a cutline file to cut the imagery to .fgb or .geojson' value: '' @@ -482,6 +487,8 @@ spec: - '{{=sprig.trim(workflow.parameters.end_datetime)}}' - '--collection-id' - '{{inputs.parameters.collection-id}}' + - '--create-footprints' + - '{{workflow.parameters.create_capture_area}}' - '--cutline' - '{{=sprig.trim(workflow.parameters.cutline)}}' - '--source-epsg' @@ -502,6 +509,7 @@ spec: artifacts: - name: capture-area path: '/tmp/capture-area.geojson' + optional: true archive: none: {} container: From 60973ecc651da5e0bf9bc3a7d5cbde926a4c744a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 May 2024 09:42:32 +1200 Subject: [PATCH 24/28] build(deps-dev): bump @linzjs/style from 5.1.0 to 5.2.0 (#384) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [@linzjs/style](https://github.com/linz/style-js) from 5.1.0 to 5.2.0.
Release notes

Sourced from @​linzjs/style's releases.

v5.2.0

5.2.0 (2024-01-09)

Features

  • add @​typescript-eslint/recommended-type-checked (#882) (793dd3e)
  • deps: bump eslint-config-prettier from 9.0.0 to 9.1.0 (#874) (cde1131)
  • deps: bump the typescript-eslint-prettier group with 2 updates (#895) (2c25865)
  • deps: bump the typescript-eslint-prettier group with 4 updates (#894) (ceee3cc)
  • deps: bump the typescript-eslint-prettier group with 5 updates (#880) (03ec37f)
Changelog

Sourced from @​linzjs/style's changelog.

5.2.0 (2024-01-09)

Features

  • add @​typescript-eslint/recommended-type-checked (#882) (793dd3e)
  • deps: bump eslint-config-prettier from 9.0.0 to 9.1.0 (#874) (cde1131)
  • deps: bump the typescript-eslint-prettier group with 2 updates (#895) (2c25865)
  • deps: bump the typescript-eslint-prettier group with 4 updates (#894) (ceee3cc)
  • deps: bump the typescript-eslint-prettier group with 5 updates (#880) (03ec37f)
Commits
  • bf511b2 chore(master): release 5.2.0 (#883)
  • 2c25865 feat(deps): bump the typescript-eslint-prettier group with 2 updates (#895)
  • 793dd3e feat: add @​typescript-eslint/recommended-type-checked (#882)
  • ceee3cc feat(deps): bump the typescript-eslint-prettier group with 4 updates (#894)
  • cde1131 feat(deps): bump eslint-config-prettier from 9.0.0 to 9.1.0 (#874)
  • 0907bf6 build(deps): bump google-github-actions/release-please-action from 3 to 4 (#872)
  • 03ec37f feat(deps): bump the typescript-eslint-prettier group with 5 updates (#880)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=@linzjs/style&package-manager=npm_and_yarn&previous-version=5.1.0&new-version=5.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Blayne Chard --- infra/cdk.ts | 6 +- infra/cdk8s.ts | 2 +- infra/eks/arn.ts | 8 +- infra/imports/karpenter.k8s.aws.ts | 2 + infra/imports/karpenter.sh.ts | 2 + package-lock.json | 655 ++++++----------------------- package.json | 2 +- 7 files changed, 147 insertions(+), 530 deletions(-) diff --git a/infra/cdk.ts b/infra/cdk.ts index b2aa6cb39..9fbdaa406 100644 --- a/infra/cdk.ts +++ b/infra/cdk.ts @@ -8,7 +8,7 @@ import { fetchSsmParameters } from './util/ssm.js'; const app = new App(); async function main(): Promise { - const accountId = app.node.tryGetContext('aws-account-id') ?? process.env['CDK_DEFAULT_ACCOUNT']; + const accountId = (app.node.tryGetContext('aws-account-id') as unknown) ?? process.env['CDK_DEFAULT_ACCOUNT']; const maintainerRoleArns = tryGetContextArns(app.node, 'maintainer-arns'); const slackSsmConfig = await fetchSsmParameters({ slackChannelConfigurationName: '/rds/alerts/slack/channel/name', @@ -17,7 +17,7 @@ async function main(): Promise { }); if (maintainerRoleArns == null) throw new Error('Missing context: maintainer-arns'); - if (accountId == null) { + if (typeof accountId !== 'string') { throw new Error("Missing AWS Account information, set with either '-c aws-account-id' or $CDK_DEFAULT_ACCOUNT"); } @@ -32,4 +32,4 @@ async function main(): Promise { app.synth(); } -main(); +void main(); diff --git a/infra/cdk8s.ts b/infra/cdk8s.ts index a569e2436..9702de50f 100644 --- a/infra/cdk8s.ts +++ b/infra/cdk8s.ts @@ -98,4 +98,4 @@ async function main(): Promise { app.synth(); } -main(); +void main(); diff --git a/infra/eks/arn.ts b/infra/eks/arn.ts index 7b38656b7..37f6e8a0c 100644 --- a/infra/eks/arn.ts +++ b/infra/eks/arn.ts @@ -28,10 +28,10 @@ export function validateRoleArn(arn: unknown): ArnComponents { * @throws {Error} If arn is invalid */ export function tryGetContextArn(node: Node, context: string): string | null { - const ctx = node.tryGetContext(context); + const ctx = node.tryGetContext(context) as unknown; if (ctx == null) return null; validateRoleArn(ctx); - return ctx; + return ctx as string; } /** @@ -42,12 +42,12 @@ export function tryGetContextArn(node: Node, context: string): string | null { * @returns arns if they are valid, null otherwise */ export function tryGetContextArns(node: Node, context: string): string[] | null { - let ctx = node.tryGetContext(context); + let ctx = node.tryGetContext(context) as unknown; if (ctx == null) return null; if (typeof ctx === 'string') { ctx = ctx.split(','); } if (!Array.isArray(ctx)) throw new Error('Failed to parse ARN, is not a string[]'); for (const arn of ctx) validateRoleArn(arn); - return ctx; + return ctx as string[]; } diff --git a/infra/imports/karpenter.k8s.aws.ts b/infra/imports/karpenter.k8s.aws.ts index 2225a9315..84a8fefc3 100644 --- a/infra/imports/karpenter.k8s.aws.ts +++ b/infra/imports/karpenter.k8s.aws.ts @@ -1,4 +1,6 @@ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ /* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ // generated by cdk8s import { ApiObject, ApiObjectMetadata, GroupVersionKind } from 'cdk8s'; import { Construct } from 'constructs'; diff --git a/infra/imports/karpenter.sh.ts b/infra/imports/karpenter.sh.ts index 9984e450d..046f75517 100644 --- a/infra/imports/karpenter.sh.ts +++ b/infra/imports/karpenter.sh.ts @@ -1,4 +1,6 @@ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ /* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ // generated by cdk8s import { ApiObject, ApiObjectMetadata, GroupVersionKind } from 'cdk8s'; import { Construct } from 'constructs'; diff --git a/package-lock.json b/package-lock.json index eefe83e0e..0acc2bf7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@aws-sdk/client-cloudformation": "3.451.0", "@aws-sdk/client-eks": "3.451.0", "@aws-sdk/client-ssm": "3.451.0", - "@linzjs/style": "^5.1.0", + "@linzjs/style": "^5.2.0", "aws-cdk": "2.108.x", "aws-cdk-lib": "2.108.x", "cdk8s": "^2.68.4", @@ -1180,9 +1180,9 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.3.tgz", - "integrity": "sha512-yZzuIG+jnVu6hNSzFEN07e8BxF3uAzYtQb6uDkaYZLo6oYZDCq454c5kB8zxnzfCYyP4MIuyBn10L0DqwujTmA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", "dev": true, "dependencies": { "ajv": "^6.12.4", @@ -1203,9 +1203,9 @@ } }, "node_modules/@eslint/js": { - "version": "8.53.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.53.0.tgz", - "integrity": "sha512-Kn7K8dx/5U6+cT1yEhpX1w4PCSg0M+XyRILPgvwcEBjerFWCwQj5sbr3/VmxqV0JGHCBCzyd6LxypEuehypY1w==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.56.0.tgz", + "integrity": "sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -1292,26 +1292,39 @@ "dev": true }, "node_modules/@linzjs/style": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@linzjs/style/-/style-5.1.0.tgz", - "integrity": "sha512-uMMtbqHwnvC4t4jSKRb2govImblH84Dn72NIAJUfFmaLEKplxKxf7r9dr+gRwKfZ2dU3/hsSnzt/F/2lMrjULQ==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@linzjs/style/-/style-5.2.0.tgz", + "integrity": "sha512-GrSItsMptMnyADLRE2k70jAdOxXB3mSiQE8u+aCqEDSUJyQoHl9Wh1q1spCjIODItVTbpuuFA3aemav0T6wamw==", "dev": true, "dependencies": { - "@typescript-eslint/eslint-plugin": "^6.11.0", - "@typescript-eslint/parser": "^6.11.0", - "eslint": "^8.53.0", - "eslint-config-prettier": "^9.0.0", - "eslint-plugin-prettier": "^5.0.1", + "@typescript-eslint/eslint-plugin": "^6.18.1", + "@typescript-eslint/parser": "^6.18.1", + "eslint": "^8.56.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.1.2", "eslint-plugin-react": "^7.33.2", "eslint-plugin-react-hooks": "^4.6.0", "eslint-plugin-simple-import-sort": "^10.0.0", - "prettier": "^3.1.0", - "typescript": "^5.2.2" + "prettier": "^3.1.1", + "typescript": "^5.3.3" }, "bin": { "linz-style-install": "linz-style-install.mjs" } }, + "node_modules/@linzjs/style/node_modules/typescript": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1491,19 +1504,11 @@ "@octokit/openapi-types": "^12.11.0" } }, - "node_modules/@pkgr/utils": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/@pkgr/utils/-/utils-2.4.2.tgz", - "integrity": "sha512-POgTXhjrTfbTV63DiFXav4lBHiICLKKwDeaKn9Nphwj7WH6m0hMMCaJkMyRWjgtPFyRKRVoMXXjczsTQRDEhYw==", + "node_modules/@pkgr/core": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.0.tgz", + "integrity": "sha512-Zwq5OCzuwJC2jwqmpEQt7Ds1DTi6BWSwoGkbb1n9pO3hzb35BoJELx7c0T23iDkBGkh2e7tvOtjF3tr3OaQHDQ==", "dev": true, - "dependencies": { - "cross-spawn": "^7.0.3", - "fast-glob": "^3.3.0", - "is-glob": "^4.0.3", - "open": "^9.1.0", - "picocolors": "^1.0.0", - "tslib": "^2.6.0" - }, "engines": { "node": "^12.20.0 || ^14.18.0 || >=16.0.0" }, @@ -2078,9 +2083,9 @@ "dev": true }, "node_modules/@types/semver": { - "version": "7.5.5", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.5.tgz", - "integrity": "sha512-+d+WYC1BxJ6yVOgUgzK8gWvp5qF8ssV5r4nsDcZWKRWcDQLQ619tvWAxJQYGgBrO1MnLJC7a5GtiYsAoQ47dJg==", + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz", + "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==", "dev": true }, "node_modules/@types/triple-beam": { @@ -2091,16 +2096,16 @@ "optional": true }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.11.0.tgz", - "integrity": "sha512-uXnpZDc4VRjY4iuypDBKzW1rz9T5YBBK0snMn8MaTSNd2kMlj50LnLBABELjJiOL5YHk7ZD8hbSpI9ubzqYI0w==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.18.1.tgz", + "integrity": "sha512-nISDRYnnIpk7VCFrGcu1rnZfM1Dh9LRHnfgdkjcbi/l7g16VYRri3TjXi9Ir4lOZSw5N/gnV/3H7jIPQ8Q4daA==", "dev": true, "dependencies": { "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.11.0", - "@typescript-eslint/type-utils": "6.11.0", - "@typescript-eslint/utils": "6.11.0", - "@typescript-eslint/visitor-keys": "6.11.0", + "@typescript-eslint/scope-manager": "6.18.1", + "@typescript-eslint/type-utils": "6.18.1", + "@typescript-eslint/utils": "6.18.1", + "@typescript-eslint/visitor-keys": "6.18.1", "debug": "^4.3.4", "graphemer": "^1.4.0", "ignore": "^5.2.4", @@ -2126,15 +2131,15 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.11.0.tgz", - "integrity": "sha512-+whEdjk+d5do5nxfxx73oanLL9ghKO3EwM9kBCkUtWMRwWuPaFv9ScuqlYfQ6pAD6ZiJhky7TZ2ZYhrMsfMxVQ==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.18.1.tgz", + "integrity": "sha512-zct/MdJnVaRRNy9e84XnVtRv9Vf91/qqe+hZJtKanjojud4wAVy/7lXxJmMyX6X6J+xc6c//YEWvpeif8cAhWA==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "6.11.0", - "@typescript-eslint/types": "6.11.0", - "@typescript-eslint/typescript-estree": "6.11.0", - "@typescript-eslint/visitor-keys": "6.11.0", + "@typescript-eslint/scope-manager": "6.18.1", + "@typescript-eslint/types": "6.18.1", + "@typescript-eslint/typescript-estree": "6.18.1", + "@typescript-eslint/visitor-keys": "6.18.1", "debug": "^4.3.4" }, "engines": { @@ -2154,13 +2159,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.11.0.tgz", - "integrity": "sha512-0A8KoVvIURG4uhxAdjSaxy8RdRE//HztaZdG8KiHLP8WOXSk0vlF7Pvogv+vlJA5Rnjj/wDcFENvDaHb+gKd1A==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.18.1.tgz", + "integrity": "sha512-BgdBwXPFmZzaZUuw6wKiHKIovms97a7eTImjkXCZE04TGHysG+0hDQPmygyvgtkoB/aOQwSM/nWv3LzrOIQOBw==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.11.0", - "@typescript-eslint/visitor-keys": "6.11.0" + "@typescript-eslint/types": "6.18.1", + "@typescript-eslint/visitor-keys": "6.18.1" }, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -2171,13 +2176,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.11.0.tgz", - "integrity": "sha512-nA4IOXwZtqBjIoYrJcYxLRO+F9ri+leVGoJcMW1uqr4r1Hq7vW5cyWrA43lFbpRvQ9XgNrnfLpIkO3i1emDBIA==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.18.1.tgz", + "integrity": "sha512-wyOSKhuzHeU/5pcRDP2G2Ndci+4g653V43gXTpt4nbyoIOAASkGDA9JIAgbQCdCkcr1MvpSYWzxTz0olCn8+/Q==", "dev": true, "dependencies": { - "@typescript-eslint/typescript-estree": "6.11.0", - "@typescript-eslint/utils": "6.11.0", + "@typescript-eslint/typescript-estree": "6.18.1", + "@typescript-eslint/utils": "6.18.1", "debug": "^4.3.4", "ts-api-utils": "^1.0.1" }, @@ -2198,9 +2203,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.11.0.tgz", - "integrity": "sha512-ZbEzuD4DwEJxwPqhv3QULlRj8KYTAnNsXxmfuUXFCxZmO6CF2gM/y+ugBSAQhrqaJL3M+oe4owdWunaHM6beqA==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.18.1.tgz", + "integrity": "sha512-4TuMAe+tc5oA7wwfqMtB0Y5OrREPF1GeJBAjqwgZh1lEMH5PJQgWgHGfYufVB51LtjD+peZylmeyxUXPfENLCw==", "dev": true, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -2211,16 +2216,17 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.11.0.tgz", - "integrity": "sha512-Aezzv1o2tWJwvZhedzvD5Yv7+Lpu1by/U1LZ5gLc4tCx8jUmuSCMioPFRjliN/6SJIvY6HpTtJIWubKuYYYesQ==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.18.1.tgz", + "integrity": "sha512-fv9B94UAhywPRhUeeV/v+3SBDvcPiLxRZJw/xZeeGgRLQZ6rLMG+8krrJUyIf6s1ecWTzlsbp0rlw7n9sjufHA==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.11.0", - "@typescript-eslint/visitor-keys": "6.11.0", + "@typescript-eslint/types": "6.18.1", + "@typescript-eslint/visitor-keys": "6.18.1", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", + "minimatch": "9.0.3", "semver": "^7.5.4", "ts-api-utils": "^1.0.1" }, @@ -2237,18 +2243,42 @@ } } }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@typescript-eslint/utils": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.11.0.tgz", - "integrity": "sha512-p23ibf68fxoZy605dc0dQAEoUsoiNoP3MD9WQGiHLDuTSOuqoTsa4oAy+h3KDkTcxbbfOtUjb9h3Ta0gT4ug2g==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.18.1.tgz", + "integrity": "sha512-zZmTuVZvD1wpoceHvoQpOiewmWu3uP9FuTWo8vqpy2ffsmfCE8mklRPi+vmnIYAIk9t/4kOThri2QCDgor+OpQ==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "@types/json-schema": "^7.0.12", "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.11.0", - "@typescript-eslint/types": "6.11.0", - "@typescript-eslint/typescript-estree": "6.11.0", + "@typescript-eslint/scope-manager": "6.18.1", + "@typescript-eslint/types": "6.18.1", + "@typescript-eslint/typescript-estree": "6.18.1", "semver": "^7.5.4" }, "engines": { @@ -2263,12 +2293,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.11.0.tgz", - "integrity": "sha512-+SUN/W7WjBr05uRxPggJPSzyB8zUpaYo2hByKasWbqr3PM8AXfZt8UHdNpBS1v9SA62qnSSMF3380SwDqqprgQ==", + "version": "6.18.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.18.1.tgz", + "integrity": "sha512-/kvt0C5lRqGoCfsbmm7/CwMqoSkY3zzHLIjdhHZQW3VFrnz7ATecOHR7nb7V+xn4286MBxfnQfQhAmCI0u+bJA==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.11.0", + "@typescript-eslint/types": "6.18.1", "eslint-visitor-keys": "^3.4.1" }, "engines": { @@ -2295,9 +2325,9 @@ } }, "node_modules/acorn": { - "version": "8.11.2", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.2.tgz", - "integrity": "sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==", + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", + "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -3038,15 +3068,6 @@ "dev": true, "optional": true }, - "node_modules/big-integer": { - "version": "1.6.51", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.51.tgz", - "integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==", - "dev": true, - "engines": { - "node": ">=0.6" - } - }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -3065,18 +3086,6 @@ "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", "dev": true }, - "node_modules/bplist-parser": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz", - "integrity": "sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw==", - "dev": true, - "dependencies": { - "big-integer": "^1.6.44" - }, - "engines": { - "node": ">= 5.10.0" - } - }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -3124,21 +3133,6 @@ "ieee754": "^1.1.13" } }, - "node_modules/bundle-name": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-3.0.0.tgz", - "integrity": "sha512-PKA4BeSvBpQKQ8iPOGCSiell+N8P+Tf1DlwqmYhpe2gAhKPHn8EYOxVT+ShuGmhg8lN8XiSlS80yiExKXrURlw==", - "dev": true, - "dependencies": { - "run-applescript": "^5.0.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -3878,40 +3872,6 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, - "node_modules/default-browser": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-4.0.0.tgz", - "integrity": "sha512-wX5pXO1+BrhMkSbROFsyxUm0i/cJEScyNhA4PPxc41ICuv05ZZB/MX28s8aZx6xjmatvebIapF6hLEKEcpneUA==", - "dev": true, - "dependencies": { - "bundle-name": "^3.0.0", - "default-browser-id": "^3.0.0", - "execa": "^7.1.1", - "titleize": "^3.0.0" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/default-browser-id": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-3.0.0.tgz", - "integrity": "sha512-OZ1y3y0SqSICtE8DE4S8YOE9UZOJ8wO16fKWVP5J1Qz42kV9jcnMVFrEE/noXb/ss3Q4pZIH79kxofzyNNtUNA==", - "dev": true, - "dependencies": { - "bplist-parser": "^0.2.0", - "untildify": "^4.0.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/defaults": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", @@ -3939,18 +3899,6 @@ "node": ">= 0.4" } }, - "node_modules/define-lazy-prop": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", - "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/define-properties": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", @@ -4280,15 +4228,15 @@ } }, "node_modules/eslint": { - "version": "8.53.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.53.0.tgz", - "integrity": "sha512-N4VuiPjXDUa4xVeV/GC/RV3hQW9Nw+Y463lkWaKKXKYMvmRiRDAtfpuPFLN+E1/6ZhyR8J2ig+eVREnYgUsiag==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.56.0.tgz", + "integrity": "sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.3", - "@eslint/js": "8.53.0", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.56.0", "@humanwhocodes/config-array": "^0.11.13", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", @@ -4335,9 +4283,9 @@ } }, "node_modules/eslint-config-prettier": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.0.0.tgz", - "integrity": "sha512-IcJsTkJae2S35pRsRAwoCE+925rJJStOdkKnLVgtE+tEpqU0EVVM7OqrwxqgptKdX29NUwC82I5pXsGFIgSevw==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz", + "integrity": "sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==", "dev": true, "bin": { "eslint-config-prettier": "bin/cli.js" @@ -4347,23 +4295,24 @@ } }, "node_modules/eslint-plugin-prettier": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.1.tgz", - "integrity": "sha512-m3u5RnR56asrwV/lDC4GHorlW75DsFfmUcjfCYylTUs85dBRnB7VM6xG8eCMJdeDRnppzmxZVf1GEPJvl1JmNg==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.1.3.tgz", + "integrity": "sha512-C9GCVAs4Eq7ZC/XFQHITLiHJxQngdtraXaM+LoUFoFp/lHNl2Zn8f3WQbe9HvTBBQ9YnKFB0/2Ajdqwo5D1EAw==", "dev": true, "dependencies": { "prettier-linter-helpers": "^1.0.0", - "synckit": "^0.8.5" + "synckit": "^0.8.6" }, "engines": { "node": "^14.18.0 || >=16.0.0" }, "funding": { - "url": "https://opencollective.com/prettier" + "url": "https://opencollective.com/eslint-plugin-prettier" }, "peerDependencies": { "@types/eslint": ">=8.0.0", "eslint": ">=8.0.0", + "eslint-config-prettier": "*", "prettier": ">=3.0.0" }, "peerDependenciesMeta": { @@ -4534,29 +4483,6 @@ "node": ">=0.10.0" } }, - "node_modules/execa": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-7.2.0.tgz", - "integrity": "sha512-UduyVP7TLB5IcAQl+OzLyLcS/l32W/GLg+AhHJ+ow40FOk2U3SAllPwR44v4vmdFwIWqpdwxxpQbF1n5ta9seA==", - "dev": true, - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.1", - "human-signals": "^4.3.0", - "is-stream": "^3.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^5.1.0", - "onetime": "^6.0.0", - "signal-exit": "^3.0.7", - "strip-final-newline": "^3.0.0" - }, - "engines": { - "node": "^14.18.0 || ^16.14.0 || >=18.0.0" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, "node_modules/external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -4891,18 +4817,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-symbol-description": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", @@ -4964,9 +4878,9 @@ } }, "node_modules/globals": { - "version": "13.23.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz", - "integrity": "sha512-XAmF0RjlrjY23MA51q3HltdlGxUpXPvg0GioKiD9X6HD28iMjo2dKC8Vqwm7lne4GNr78+RHTfliktR6ZH09wA==", + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -5141,15 +5055,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/human-signals": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-4.3.1.tgz", - "integrity": "sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==", - "dev": true, - "engines": { - "node": ">=14.18.0" - } - }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -5397,21 +5302,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-docker": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", - "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", - "dev": true, - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -5469,24 +5359,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-inside-container": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", - "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", - "dev": true, - "dependencies": { - "is-docker": "^3.0.0" - }, - "bin": { - "is-inside-container": "cli.js" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-interactive": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", @@ -5608,18 +5480,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", - "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", - "dev": true, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-string": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", @@ -5712,33 +5572,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-wsl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", - "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", - "dev": true, - "dependencies": { - "is-docker": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-wsl/node_modules/is-docker": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", - "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", - "dev": true, - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -6709,12 +6542,6 @@ "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==", "dev": true }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true - }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -6760,18 +6587,6 @@ "node": ">= 0.6" } }, - "node_modules/mimic-fn": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", - "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -6852,33 +6667,6 @@ } } }, - "node_modules/npm-run-path": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.1.0.tgz", - "integrity": "sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==", - "dev": true, - "dependencies": { - "path-key": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/npm-run-path/node_modules/path-key": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", - "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -7004,21 +6792,6 @@ "fn.name": "1.x.x" } }, - "node_modules/onetime": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", - "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", - "dev": true, - "dependencies": { - "mimic-fn": "^4.0.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/oo-ascii-tree": { "version": "1.91.0", "resolved": "https://registry.npmjs.org/oo-ascii-tree/-/oo-ascii-tree-1.91.0.tgz", @@ -7028,24 +6801,6 @@ "node": ">= 14.17.0" } }, - "node_modules/open": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/open/-/open-9.1.0.tgz", - "integrity": "sha512-OS+QTnw1/4vrf+9hh1jc1jnYjzSG4ttTBB8UxOwAnInG3Uo4ssetzC1ihqaIHjLJnA5GGlRl6QlZXOTQhRBUvg==", - "dev": true, - "dependencies": { - "default-browser": "^4.0.0", - "define-lazy-prop": "^3.0.0", - "is-inside-container": "^1.0.0", - "is-wsl": "^2.2.0" - }, - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/optionator": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", @@ -7206,12 +6961,6 @@ "node": ">=8" } }, - "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", - "dev": true - }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -7234,9 +6983,9 @@ } }, "node_modules/prettier": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.0.tgz", - "integrity": "sha512-TQLvXjq5IAibjh8EpBIkNKxO749UEWABoiIZehEPiY4GNpVdhaFKqSTu+QrlU6D2dPAfubRmtJTi4K4YkQ5eXw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.1.tgz", + "integrity": "sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==", "dev": true, "bin": { "prettier": "bin/prettier.cjs" @@ -7517,110 +7266,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/run-applescript": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-5.0.0.tgz", - "integrity": "sha512-XcT5rBksx1QdIhlFOCtgZkB99ZEouFZ1E2Kc2LHqNW13U3/74YGdkQRmThTwxy4QIyookibDKYZOPqX//6BlAg==", - "dev": true, - "dependencies": { - "execa": "^5.0.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/run-applescript/node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dev": true, - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/run-applescript/node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "dev": true, - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/run-applescript/node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/run-applescript/node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/run-applescript/node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true, - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/run-applescript/node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/run-applescript/node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true, - "engines": { - "node": ">=6" - } - }, "node_modules/run-async": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", @@ -7850,7 +7495,8 @@ "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true + "dev": true, + "optional": true }, "node_modules/simple-swizzle": { "version": "0.2.2", @@ -8079,18 +7725,6 @@ "node": ">=8" } }, - "node_modules/strip-final-newline": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", - "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -8148,13 +7782,13 @@ } }, "node_modules/synckit": { - "version": "0.8.5", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.8.5.tgz", - "integrity": "sha512-L1dapNV6vu2s/4Sputv8xGsCdAVlb5nRDMFU/E27D44l5U6cw1g0dGd45uLc+OXjNMmF4ntiMdCimzcjFKQI8Q==", + "version": "0.8.8", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.8.8.tgz", + "integrity": "sha512-HwOKAP7Wc5aRGYdKH+dw0PRRpbO841v2DENBtjnR5HFWoiNByAl7vrx3p0G/rCyYXQsrxqtX48TImFtPcIHSpQ==", "dev": true, "dependencies": { - "@pkgr/utils": "^2.3.1", - "tslib": "^2.5.0" + "@pkgr/core": "^0.1.0", + "tslib": "^2.6.2" }, "engines": { "node": "^14.18.0 || >=16.0.0" @@ -8238,18 +7872,6 @@ "dev": true, "optional": true }, - "node_modules/titleize": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/titleize/-/titleize-3.0.0.tgz", - "integrity": "sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -8476,15 +8098,6 @@ "node": ">= 4.0.0" } }, - "node_modules/untildify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", - "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", diff --git a/package.json b/package.json index 0c1766b95..b70f211b9 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "@aws-sdk/client-cloudformation": "3.451.0", "@aws-sdk/client-eks": "3.451.0", "@aws-sdk/client-ssm": "3.451.0", - "@linzjs/style": "^5.1.0", + "@linzjs/style": "^5.2.0", "aws-cdk": "2.108.x", "aws-cdk-lib": "2.108.x", "cdk8s": "^2.68.4", From 68990ed3bae907a1f95425c6d4819dbdc9544eb1 Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Thu, 9 May 2024 14:47:49 +1200 Subject: [PATCH 25/28] feat: stac-validate workflows use specific checksum flags TDE-1134 (#572) #### Motivation `stac-validate` has changed its flags for checksum validation in https://github.com/linz/argo-tasks/pull/982 & https://github.com/linz/argo-tasks/pull/972 #### Modification Use the `--checksum-assets` and `--checksum-links` flags #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated NA - [x] Docs updated - [x] Issue linked in Title --- templates/argo-tasks/README.md | 6 ++++-- templates/argo-tasks/stac-validate.yml | 13 +++++++++---- workflows/stac/stac-validate-parallel.yaml | 18 +++++++++++++----- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/templates/argo-tasks/README.md b/templates/argo-tasks/README.md index cc44fd635..063f16dd2 100644 --- a/templates/argo-tasks/README.md +++ b/templates/argo-tasks/README.md @@ -211,8 +211,10 @@ See (https://github.com/linz/argo-tasks#stac-validate) parameters: - name: uri value: 's3://my-bucket/path/collection.json' - - name: checksum - value: '{{workflow.parameters.checksum}}' + - name: checksum_assets + value: '{{workflow.parameters.checksum_assets}}' + - name: checksum_links + value: '{{workflow.parameters.checksum_links}}' - name: recursive value: '{{workflow.parameters.recursive}}' - name: concurrency diff --git a/templates/argo-tasks/stac-validate.yml b/templates/argo-tasks/stac-validate.yml index 77b8d6ba4..bc6a87710 100644 --- a/templates/argo-tasks/stac-validate.yml +++ b/templates/argo-tasks/stac-validate.yml @@ -28,13 +28,17 @@ spec: description: Number of requests to run concurrently default: '50' - - name: checksum - description: Validate the file:checksum if it exists + - name: checksum_assets + description: Validate the file:checksum of each asset if it exists + default: 'false' + + - name: checksum_links + description: Validate the file:checksum of each link if it exists default: 'false' - name: version description: container version to use - default: 'v3' + default: 'v4' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version)}}' @@ -51,5 +55,6 @@ spec: - 'validate' - '--concurrency={{inputs.parameters.concurrency}}' - '--recursive={{inputs.parameters.recursive}}' - - '--checksum={{inputs.parameters.checksum}}' + - '--checksum-assets={{inputs.parameters.checksum_assets}}' + - '--checksum-links={{inputs.parameters.checksum_links}}' - '{{inputs.parameters.uri}}' diff --git a/workflows/stac/stac-validate-parallel.yaml b/workflows/stac/stac-validate-parallel.yaml index 7895d390b..c791631c5 100644 --- a/workflows/stac/stac-validate-parallel.yaml +++ b/workflows/stac/stac-validate-parallel.yaml @@ -14,15 +14,21 @@ spec: arguments: parameters: - name: version_argo_tasks - value: 'v3' + value: 'v4' - name: uri description: 'Path(s) to the STAC file(s).' value: 's3://linz-imagery-staging/test/stac-validate/' - name: include value: 'collection.json$' - - name: checksum - description: 'Validate asset checksums.' + - name: checksum_assets + description: 'Validate the file:checksum of each asset if it exists' value: 'false' + enum: + - 'true' + - 'false' + - name: checksum_links + description: 'Validate the file:checksum of each link if it exists' + value: 'true' enum: - 'false' - 'true' @@ -44,8 +50,10 @@ spec: parameters: - name: uri value: '{{item}}' - - name: checksum - value: '{{workflow.parameters.checksum}}' + - name: checksum_assets + value: '{{workflow.parameters.checksum_assets}}' + - name: checksum_links + value: '{{workflow.parameters.checksum_links}}' depends: aws-list-collections withParam: '{{tasks.aws-list-collections.outputs.parameters.files}}' - name: aws-list-collections From e393dff2e1b3d26dfd7af6c80519a723dafb049b Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Thu, 9 May 2024 14:52:28 +1200 Subject: [PATCH 26/28] feat: use argo-tasks v4 (#573) #### Motivation There is a new version of argo-tasks #### Modification Use v4 #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title --- templates/argo-tasks/copy.yml | 2 +- templates/argo-tasks/create-manifest.yml | 2 +- templates/argo-tasks/generate-path.yml | 2 +- templates/argo-tasks/group.yml | 2 +- templates/argo-tasks/push-to-github.yml | 2 +- templates/argo-tasks/tile-index-validate.yml | 2 +- workflows/basemaps/imagery-import-cogify.yml | 2 +- workflows/basemaps/mapsheet-json.yaml | 2 +- workflows/basemaps/vector-etl.yaml | 2 +- workflows/raster/copy.yaml | 2 +- workflows/raster/publish-odr.yaml | 2 +- workflows/raster/standardising.yaml | 2 +- workflows/test/generate-path.yaml | 2 +- workflows/util/create-thumbnails.yaml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/templates/argo-tasks/copy.yml b/templates/argo-tasks/copy.yml index b935eb264..5f5899988 100644 --- a/templates/argo-tasks/copy.yml +++ b/templates/argo-tasks/copy.yml @@ -21,7 +21,7 @@ spec: - name: version_argo_tasks description: version of argo-tasks to use - default: 'v3' + default: 'v4' - name: copy_option description: --no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files. diff --git a/templates/argo-tasks/create-manifest.yml b/templates/argo-tasks/create-manifest.yml index cfaa953ff..dfb1b0468 100644 --- a/templates/argo-tasks/create-manifest.yml +++ b/templates/argo-tasks/create-manifest.yml @@ -24,7 +24,7 @@ spec: - name: version_argo_tasks description: version of argo-tasks to use - default: 'v3' + default: 'v4' - name: include description: A regular expression to match object path(s) or name(s) from within the source path to include in the copy diff --git a/templates/argo-tasks/generate-path.yml b/templates/argo-tasks/generate-path.yml index 81ac5e3a0..91732efd4 100644 --- a/templates/argo-tasks/generate-path.yml +++ b/templates/argo-tasks/generate-path.yml @@ -21,7 +21,7 @@ spec: description: target bucket name e.g. 'nz-imagery' - name: version description: argo-task Container version to use - default: 'v3' + default: 'v4' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{= inputs.parameters.version }}' diff --git a/templates/argo-tasks/group.yml b/templates/argo-tasks/group.yml index 1291d5eea..44ac329d4 100644 --- a/templates/argo-tasks/group.yml +++ b/templates/argo-tasks/group.yml @@ -32,7 +32,7 @@ spec: - name: version description: argo-task Container version to use - default: 'v3' + default: 'v4' outputs: parameters: diff --git a/templates/argo-tasks/push-to-github.yml b/templates/argo-tasks/push-to-github.yml index 200641ae1..ad120c149 100644 --- a/templates/argo-tasks/push-to-github.yml +++ b/templates/argo-tasks/push-to-github.yml @@ -23,7 +23,7 @@ spec: - name: version_argo_tasks description: version of argo-tasks to use - default: 'v3' + default: 'v4' - name: repository description: Repository Name diff --git a/templates/argo-tasks/tile-index-validate.yml b/templates/argo-tasks/tile-index-validate.yml index 3fe143d34..76826ad53 100644 --- a/templates/argo-tasks/tile-index-validate.yml +++ b/templates/argo-tasks/tile-index-validate.yml @@ -45,7 +45,7 @@ spec: - name: version description: container version to use - default: 'v3' + default: 'v4' outputs: artifacts: diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index e61fd1384..37cbc06d9 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -33,7 +33,7 @@ spec: - name: version_argo_tasks description: Version of the basemaps CLI docker container to use - value: v3 + value: v4 - name: ticket description: Ticket ID e.g. 'AIP-55' diff --git a/workflows/basemaps/mapsheet-json.yaml b/workflows/basemaps/mapsheet-json.yaml index 3204f314e..62cecbb1f 100644 --- a/workflows/basemaps/mapsheet-json.yaml +++ b/workflows/basemaps/mapsheet-json.yaml @@ -13,7 +13,7 @@ spec: arguments: parameters: - name: version_argo_tasks - value: 'v3' + value: 'v4' - name: layer value: '104687' - name: config diff --git a/workflows/basemaps/vector-etl.yaml b/workflows/basemaps/vector-etl.yaml index da09679c6..bccc46911 100644 --- a/workflows/basemaps/vector-etl.yaml +++ b/workflows/basemaps/vector-etl.yaml @@ -13,7 +13,7 @@ spec: parameters: - name: version_argo_tasks description: Version of the Argo Tasks CLI docker container to use - value: 'v3' + value: 'v4' - name: version_basemaps_etl description: Version of the Basemaps ETL eks container to use diff --git a/workflows/raster/copy.yaml b/workflows/raster/copy.yaml index 27b48808f..b7feabc41 100644 --- a/workflows/raster/copy.yaml +++ b/workflows/raster/copy.yaml @@ -26,7 +26,7 @@ spec: arguments: parameters: - name: version_argo_tasks - value: 'v3' + value: 'v4' - name: ticket description: Ticket ID e.g. 'AIP-55' value: '' diff --git a/workflows/raster/publish-odr.yaml b/workflows/raster/publish-odr.yaml index 98dc1615e..d72137a9d 100644 --- a/workflows/raster/publish-odr.yaml +++ b/workflows/raster/publish-odr.yaml @@ -26,7 +26,7 @@ spec: arguments: parameters: - name: version_argo_tasks - value: 'v3' + value: 'v4' - name: ticket description: Ticket ID e.g. 'AIP-55' value: '' diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 41d7d91cf..4a8b6d7f5 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -26,7 +26,7 @@ spec: arguments: parameters: - name: version_argo_tasks - value: 'v3' + value: 'v4' - name: version_basemaps_cli value: 'v7' - name: version_topo_imagery diff --git a/workflows/test/generate-path.yaml b/workflows/test/generate-path.yaml index 14030189e..89acd109b 100644 --- a/workflows/test/generate-path.yaml +++ b/workflows/test/generate-path.yaml @@ -13,7 +13,7 @@ spec: arguments: parameters: - name: version_argo_tasks - value: 'v3' + value: 'v4' - name: source value: 's3://linz-imagery-staging/test/sample/' - name: target_bucket_name diff --git a/workflows/util/create-thumbnails.yaml b/workflows/util/create-thumbnails.yaml index 2f7ecb3d4..09ab9c24e 100644 --- a/workflows/util/create-thumbnails.yaml +++ b/workflows/util/create-thumbnails.yaml @@ -16,7 +16,7 @@ spec: arguments: parameters: - name: version_argo_tasks - value: 'v3' + value: 'v4' - name: version_topo_imagery value: 'v4' - name: source From c4a6362af702b1172b95319a467ebdd4db1b1382 Mon Sep 17 00:00:00 2001 From: Rebecca Clarke Date: Mon, 13 May 2024 11:35:49 +1200 Subject: [PATCH 27/28] feat: add RSHL as a licensor (#579) #### Motivation NIWE Lidar contracted by Regional Software Holdings Limited (RSHL) #### Modification Add RSHL as a licensor #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title --- workflows/raster/standardising.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 4a8b6d7f5..8f95dfe3e 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -212,6 +212,7 @@ spec: 'Porirua City Council', 'Queenstown-Lakes District Council', 'Rangitīkei District Council', + 'Regional Software Holdings Limited', 'Rotorua District Council', 'Ruapehu District Council', 'Selwyn District Council', From 0da70b228ce4ca350ccefbbac7601973b4510bb8 Mon Sep 17 00:00:00 2001 From: paulfouquet <86932794+paulfouquet@users.noreply.github.com> Date: Thu, 16 May 2024 10:04:23 +1200 Subject: [PATCH 28/28] fix: stac-validate-collections retries should be managed by tpl-at-stac-validate only TDE-1178 (#585) #### Motivation In the case of a task in `tpl-at-stac-validate` fails more than twice and the workflow came back failing, the entire `stac-validate-collections` is retried twice which cause to run the inner stac-validate workflow. #### Modification Avoid tasks in the `stac-validate-parallel` workflow to retry by default. Force `aws-list-collections` to retry twice if it fails. This is a work around as I could not find a way to specify the retryStrategy at a task level using a `templateRef` #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [x] Issue linked in Title --- workflows/stac/stac-validate-parallel.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/workflows/stac/stac-validate-parallel.yaml b/workflows/stac/stac-validate-parallel.yaml index c791631c5..d0d75b57b 100644 --- a/workflows/stac/stac-validate-parallel.yaml +++ b/workflows/stac/stac-validate-parallel.yaml @@ -38,6 +38,8 @@ spec: image: '' templates: - name: main + retryStrategy: + limit: '0' # avoid retrying any of the following task as `tpl-at-stac-validate` already retries its own tasks. dag: tasks: - name: aws-list-collections @@ -57,6 +59,8 @@ spec: depends: aws-list-collections withParam: '{{tasks.aws-list-collections.outputs.parameters.files}}' - name: aws-list-collections + retryStrategy: + limit: '2' # force retrying this specific task container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' command: [node, /app/index.js]