Merge branch 'master' into refactor/namespace-sa-creation

README.md

@@ -1,6 +1,6 @@
 # Topo Workflows
 
-Topo workflows are run on a AWS EKS Cluster using [Argo Workflows](https://argoproj.github.io/argo-workflows/). The detailed configuration is available in [this repo](./config/).
+Topo workflows are run on a AWS EKS Cluster using [Argo Workflows](https://argoproj.github.io/argo-workflows/). The detailed configuration is available in [this repo](./infra/).
 
 To get setup you need access to the Argo user role inside the EKS cluster, you will need to contact someone from Topo Data Engineering to get access, all Imagery maintainers will already have access.
 

cdk.json

@@ -1 +1 @@
-{ "app": "npx tsx config/cdk.ts" }
+{ "app": "npx tsx infra/cdk.ts" }

cdk8s.yaml

@@ -1,4 +1,4 @@
-app: npx tsx config/cdk8s.ts
+app: npx tsx infra/cdk8s.ts
 language: typescript
 imports:
   - https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.sh_provisioners.yaml

docs/concurrency.md

@@ -3,7 +3,7 @@
 Some workflows may need limits on how many can be run concurrently. This can
 be achieved using Argo's [synchronization][1] feature.
 
-The desired limit can be added in a ConfigMap in [config/semaphores.yml][2], e.g.
+The desired limit can be added in a ConfigMap in [infra/charts/argo.semaphores.ts][2], e.g.
 
 ```yaml
 apiVersion: v1
@@ -32,4 +32,4 @@ Any further instances which are started while two are running will be queued,
 and start automatically when running workflows complete.
 
 [1]: https://argoproj.github.io/argo-workflows/synchronization/
-[2]: config/semaphores.yml
+[2]: infra/charts/argo.semaphores.ts

docs/dns.configuration.md

@@ -74,7 +74,7 @@ file: index.mjs
 ```javascript
 fetch('https://google.com').then((c) => console.log(c));
 
-import * as dns from 'dns/promises'
+import * as dns from 'dns/promises';
 
 await dns.resolve('google.com', 'A');
 await dns.resolve('google.com', 'AAAA');

config/README.md → infra/README.md

@@ -35,13 +35,25 @@ To generate the Helm Construct for a specific Chart, follow the instructions [he
 
 Specify the output for the imports:
 
-`--output config/imports/`
+`--output infra/imports/`
 
 However, some of the component Helm charts do not have a `values.schema.json`. For those we won't generate any code and use the default `Helm` construct:
 
 - aws-for-fluent-bit (<https://github.com/aws/eks-charts/issues/1011>)
 - Karpenter
 
+### Working with Helm charts
+
+#### Generate code
+
+It is possible to generate a specific Helm construct for the component if their chart includes a `value.schema.json`. This is useful to provide typing hints when specifying their configuration (<https://github.com/cdk8s-team/cdk8s/blob/master/docs/cli/import.md#values-schema>)
+
+To generate the Helm Construct for a specific Chart, follow the instructions [here](https://github.com/cdk8s-team/cdk8s/blob/master/docs/cli/import.md#values-schema)
+
+However, some of the component Helm charts do not have a `values.schema.json`. For those we won't generate any code and use the default `Helm` construct:
+
+- aws-for-fluent-bit (<https://github.com/aws/eks-charts/issues/1011>)
+
 ## Usage (for test)
 
 Ensure all dependencies are installed
@@ -58,6 +70,12 @@ Generate the kubernetes configuration yaml into `dist/`
 npx cdk8s synth
 ```
 
+To debug use the following as `cdk8s syth` swallows the errors
+
+```shell
+npx tsx infra/cdk8s.ts
+```
+
 Apply the generated yaml files
 
 ```shell

config/cdk.ts → infra/cdk.ts

@@ -1,12 +1,12 @@
 import { App } from 'aws-cdk-lib';
 
-import { CLUSTER_NAME } from './constants';
+import { ClusterName } from './constants';
 import { LinzEksCluster } from './eks/cluster';
 
 const app = new App();
 
 async function main(): Promise<void> {
-  new LinzEksCluster(app, CLUSTER_NAME, {
+  new LinzEksCluster(app, ClusterName, {
     env: { region: 'ap-southeast-2', account: process.env.CDK_DEFAULT_ACCOUNT },
   });
 

config/cdk8s.ts → infra/cdk8s.ts

@@ -1,35 +1,42 @@
 import { App } from 'cdk8s';
 
 import { ArgoSemaphore } from './charts/argo.semaphores';
+import { FluentBit } from './charts/fluentbit';
 import { Karpenter, KarpenterProvisioner } from './charts/karpenter';
 import { CoreDns } from './charts/kube-system.coredns';
-import { CfnOutputKeys } from './constants';
-import { CLUSTER_NAME } from './constants';
+import { CfnOutputKeys, ClusterName } from './constants';
 import { getCfnOutputs } from './util/cloud.formation';
 
 const app = new App();
 
 async function main(): Promise<void> {
   // Get cloudformation outputs
-  const cfnOutputs = await getCfnOutputs(CLUSTER_NAME);
-  const missingKeys = [...Object.values(CfnOutputKeys.Karpenter)].filter((f) => cfnOutputs[f] == null);
+  const cfnOutputs = await getCfnOutputs(ClusterName);
+  const missingKeys = [...Object.values(CfnOutputKeys.Karpenter), ...Object.values(CfnOutputKeys.FluentBit)].filter(
+    (f) => cfnOutputs[f] == null,
+  );
   if (missingKeys.length > 0) {
     throw new Error(`Missing CloudFormation Outputs for keys ${missingKeys.join(', ')}`);
   }
 
   new ArgoSemaphore(app, 'semaphore', {});
-  new CoreDns(app, 'Dns', {});
+  const coredns = new CoreDns(app, 'dns', {});
+  const fluentbit = new FluentBit(app, 'fluentbit', {
+    saRoleName: cfnOutputs[CfnOutputKeys.FluentBit.ServiceAccountName],
+    clusterName: ClusterName,
+  });
+  fluentbit.addDependency(coredns);
 
   const karpenter = new Karpenter(app, 'karpenter', {
-    clusterName: CLUSTER_NAME,
+    clusterName: ClusterName,
     clusterEndpoint: cfnOutputs[CfnOutputKeys.Karpenter.ClusterEndpoint],
     saRoleName: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountName],
     saRoleArn: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountRoleArn],
     instanceProfile: cfnOutputs[CfnOutputKeys.Karpenter.DefaultInstanceProfile],
   });
 
   const karpenterProvisioner = new KarpenterProvisioner(app, 'karpenter-provisioner', {
-    clusterName: CLUSTER_NAME,
+    clusterName: ClusterName,
     clusterEndpoint: cfnOutputs[CfnOutputKeys.Karpenter.ClusterEndpoint],
     saRoleName: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountName],
     saRoleArn: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountRoleArn],

infra/charts/fluentbit.ts

@@ -0,0 +1,71 @@
+import { Chart, ChartProps, Helm } from 'cdk8s';
+import { Construct } from 'constructs';
+
+import { applyDefaultLabels } from '../util/labels.js';
+
+/** version of the Helm chart (not FluentBit app) */
+const awsForFluentBitVersion = '0.1.31';
+export interface FluentBitProps {
+  saRoleName: string;
+  clusterName: string;
+}
+
+export class FluentBit extends Chart {
+  constructor(scope: Construct, id: string, props: FluentBitProps & ChartProps) {
+    super(scope, id, applyDefaultLabels(props, 'aws-for-fluent-bit', awsForFluentBitVersion, 'logs', 'workflows'));
+
+    const FluentParserName = 'containerd';
+    // This needs to be properly formatted, and it was taken directly from https://github.com/microsoft/fluentbit-containerd-cri-o-json-log
+    // The key part is the message must be parsed as "log" otherwise it wont be parsed as JSON
+    const extraParsers = `[PARSER]
+    Name ${FluentParserName}
+    Format regex
+    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+    `;
+
+    /**
+     * FIXME: We deactivated the HTTP server to avoid getting this issue:
+     * https://github.com/aws/eks-charts/issues/983
+     *
+     */
+    const extraService = `
+HTTP_Server  Off
+HTTP_Listen  [::]
+HTTP_PORT    2020
+Health_Check On 
+HC_Errors_Count 5 
+HC_Retry_Failure_Count 5 
+HC_Period 5 
+`;
+
+    new Helm(this, 'aws-for-fluent-bit', {
+      chart: 'aws-for-fluent-bit',
+      repo: 'https://aws.github.io/eks-charts',
+      namespace: 'fluentbit',
+      version: awsForFluentBitVersion,
+      values: {
+        fullnameOverride: 'fluentbit',
+        input: { parser: FluentParserName, dockerMode: 'Off' },
+        serviceAccount: { name: props.saRoleName, create: false },
+        cloudWatchLogs: {
+          enabled: true,
+          region: 'ap-southeast-2',
+          autoCreateGroup: true,
+          logRetentionDays: 30,
+          logGroupName: `/aws/eks/${props.clusterName}/logs`,
+          logGroupTemplate: `/aws/eks/${props.clusterName}/workload/$kubernetes['namespace_name']`,
+          logStreamPrefix: 'fb-',
+        },
+        firehose: { enabled: false },
+        kinesis: { enabled: false },
+        elasticsearch: { enabled: false },
+        service: { extraParsers, extraService },
+        // FIXME: `livenessProbe` and `readinessProbe` deactivated https://github.com/aws/eks-charts/issues/995
+        livenessProbe: false,
+        readinessProbe: false,
+      },
+    });
+  }
+}

config/charts/karpenter.ts → infra/charts/karpenter.ts

@@ -69,6 +69,7 @@ export class Karpenter extends Chart {
       namespace: 'karpenter',
       version: 'v0.31.1',
       values: {
+        fullnameOverride: 'karpenter', // override the karpenter-abcxywz
         serviceAccount: {
           create: false,
           name: props.saRoleName,

config/constants.ts → infra/constants.ts

@@ -1,5 +1,5 @@
 /* Cluster name */
-export const CLUSTER_NAME = 'Workflows';
+export const ClusterName = 'Workflows';
 
 /* CloudFormation Output to access from CDK8s */
 export const CfnOutputKeys = {
@@ -9,4 +9,7 @@ export const CfnOutputKeys = {
     ClusterEndpoint: 'ClusterEndpoint',
     DefaultInstanceProfile: 'DefaultInstanceProfile',
   },
+  FluentBit: {
+    ServiceAccountName: 'FluentBitServiceAccountName',
+  },
 };

config/eks/cluster.ts → infra/eks/cluster.ts

@@ -18,8 +18,7 @@ import {
   Role,
   ServicePrincipal,
 } from 'aws-cdk-lib/aws-iam';
-import { IBucket } from 'aws-cdk-lib/aws-s3';
-import { BlockPublicAccess, Bucket } from 'aws-cdk-lib/aws-s3';
+import { BlockPublicAccess, Bucket, IBucket } from 'aws-cdk-lib/aws-s3';
 import { Construct } from 'constructs';
 
 import { CfnOutputKeys } from '../constants';
@@ -132,14 +131,14 @@ export class LinzEksCluster extends Stack {
       groups: ['system:bootstrappers', 'system:nodes'],
     });
 
-    const karpenterSA = initService(this.cluster, 'karpenter');
+    const karpenterSA = initComponent(this.cluster, 'karpenter');
     // Nasty hack so this account has access to spin up EC2s inside of LINZ's network
     karpenterSA.role.addManagedPolicy(
       ManagedPolicy.fromAwsManagedPolicyName('service-role/AmazonEC2SpotFleetTaggingRole'),
     );
 
     // Allow Karpenter to start ec2 instances
-    // FIXME: some policies are missing. See https://github.com/aws/karpenter/blob/8c33a40733b90aa0bb42a6436152374f7b359f69/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml#L40
+    // @see https://github.com/aws/karpenter/blob/8c33a40733b90aa0bb42a6436152374f7b359f69/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml#L40
     // The current policies are based on https://github.com/eksctl-io/eksctl/blob/main/pkg/cfn/builder/karpenter_test.go#L111
     new Policy(this, 'ControllerPolicy', {
       roles: [karpenterSA.role],
@@ -190,8 +189,14 @@ export class LinzEksCluster extends Stack {
     new CfnOutput(this, CfnOutputKeys.Karpenter.ServiceAccountName, { value: karpenterSA.serviceAccountName });
 
     // Use fluent bit to ship logs from eks into aws
-    const fluentbitSA = initService(this.cluster, 'fluentbit');
+    const fluentbitSA = initComponent(this.cluster, 'fluentbit');
     new CfnOutput(this, 'FluentBitServiceAccountRoleArn', { value: fluentbitSA.role.roleArn });
+    // https://docs.aws.amazon.com/aws-managed-policy/latest/reference/CloudWatchAgentServerPolicy.html
+    fluentbitSA.role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('CloudWatchAgentServerPolicy'));
+    fluentbitSA.role.addToPrincipalPolicy(
+      new PolicyStatement({ actions: ['logs:PutRetentionPolicy'], resources: ['*'], effect: Effect.ALLOW }),
+    );
+    new CfnOutput(this, CfnOutputKeys.FluentBit.ServiceAccountName, { value: fluentbitSA.serviceAccountName });
 
     // Basic constructs for argo to be deployed into
     const argoNs = this.cluster.addManifest('ArgoNameSpace', {
@@ -209,12 +214,12 @@ export class LinzEksCluster extends Stack {
 }
 
 /**
- * Init a new Kubernetes service (component) by creating its namespace and service account.
+ * Init a new Kubernetes Component by creating its namespace and its initial service account.
  * @param cluster
  * @param name
  * @returns the service account created
  */
-function initService(cluster: Cluster, name: string): ServiceAccount {
+function initComponent(cluster: Cluster, name: string): ServiceAccount {
   const namespace = cluster.addManifest(`${upperCaseFirstLetter(name)}Namespace`, {
     apiVersion: 'v1',
     kind: 'Namespace',