feat: Fluentbit TDE-913

config/README.md

@@ -42,6 +42,19 @@ However, some of the component Helm charts do not have a `values.schema.json`. F
 - aws-for-fluent-bit (<https://github.com/aws/eks-charts/issues/1011>)
 - Karpenter
 
+### Working with Helm charts
+
+#### Generate code
+
+It is possible to generate a specific Helm construct for the component if their chart includes a `value.schema.json`. This is useful to provide typing hints when specifying their configuration (<https://github.com/cdk8s-team/cdk8s/blob/master/docs/cli/import.md#values-schema>)
+
+To generate the Helm Construct for a specific Chart, follow the instructions [here](https://github.com/cdk8s-team/cdk8s/blob/master/docs/cli/import.md#values-schema)
+
+However, some of the component Helm charts do not have a `values.schema.json`. For those we won't generate any code and use the default `Helm` construct:
+
+- aws-for-fluent-bit (<https://github.com/aws/eks-charts/issues/1011>)
+
+
 ## Usage (for test)
 
 Ensure all dependencies are installed

config/cdk.ts

@@ -1,12 +1,12 @@
 import { App } from 'aws-cdk-lib';
 
-import { CLUSTER_NAME } from './constants';
+import { ClusterName } from './constants';
 import { LinzEksCluster } from './eks/cluster';
 
 const app = new App();
 
 async function main(): Promise<void> {
-  new LinzEksCluster(app, CLUSTER_NAME, {
+  new LinzEksCluster(app, ClusterName, {
     env: { region: 'ap-southeast-2', account: process.env.CDK_DEFAULT_ACCOUNT },
   });
 

config/cdk8s.ts

@@ -1,35 +1,38 @@
 import { App } from 'cdk8s';
 
 import { ArgoSemaphore } from './charts/argo.semaphores';
+import { FluentBit } from './charts/fluentbit';
 import { Karpenter, KarpenterProvisioner } from './charts/karpenter';
 import { CoreDns } from './charts/kube-system.coredns';
-import { CfnOutputKeys } from './constants';
-import { CLUSTER_NAME } from './constants';
+import { CfnOutputKeys, ClusterName } from './constants';
 import { getCfnOutputs } from './util/cloud.formation';
 
 const app = new App();
 
 async function main(): Promise<void> {
   // Get cloudformation outputs
-  const cfnOutputs = await getCfnOutputs(CLUSTER_NAME);
-  const missingKeys = [...Object.values(CfnOutputKeys.Karpenter)].filter((f) => cfnOutputs[f] == null);
+  const cfnOutputs = await getCfnOutputs(ClusterName);
+  const missingKeys = [...Object.values(CfnOutputKeys.Karpenter), ...Object.values(CfnOutputKeys.FluentBit)].filter(
+    (f) => cfnOutputs[f] == null,
+  );
   if (missingKeys.length > 0) {
     throw new Error(`Missing CloudFormation Outputs for keys ${missingKeys.join(', ')}`);
   }
 
   new ArgoSemaphore(app, 'semaphore', {});
-  new CoreDns(app, 'Dns', {});
+  new FluentBit(app, 'fluentbit', { saRoleName: cfnOutputs[CfnOutputKeys.FluentBit.ServiceAccountName] });
+  new CoreDns(app, 'dns', {});
 
   const karpenter = new Karpenter(app, 'karpenter', {
-    clusterName: CLUSTER_NAME,
+    clusterName: ClusterName,
     clusterEndpoint: cfnOutputs[CfnOutputKeys.Karpenter.ClusterEndpoint],
     saRoleName: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountName],
     saRoleArn: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountRoleArn],
     instanceProfile: cfnOutputs[CfnOutputKeys.Karpenter.DefaultInstanceProfile],
   });
 
   const karpenterProvisioner = new KarpenterProvisioner(app, 'karpenter-provisioner', {
-    clusterName: CLUSTER_NAME,
+    clusterName: ClusterName,
     clusterEndpoint: cfnOutputs[CfnOutputKeys.Karpenter.ClusterEndpoint],
     saRoleName: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountName],
     saRoleArn: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountRoleArn],

config/charts/fluentbit.ts

@@ -0,0 +1,42 @@
+import { Chart, ChartProps, Helm } from 'cdk8s';
+import { Construct } from 'constructs';
+
+import { applyDefaultLabels } from '../util/labels.js';
+
+export interface FluentBitProps {
+  saRoleName: string;
+}
+
+export class FluentBit extends Chart {
+  constructor(scope: Construct, id: string, props: FluentBitProps & ChartProps) {
+    super(scope, id, applyDefaultLabels(props, 'aws-for-fluent-bit', '2.31.11', 'logs', 'workflows'));
+
+    const FluentParserName = 'containerd';
+    // This needs to be properly formatted, and it was taken directly from https://github.com/microsoft/fluentbit-containerd-cri-o-json-log
+    // The key part is the message must be parsed as "log" otherwise it wont be parsed as JSON
+    const FluentContainerParser = `[PARSER]
+        Name ${FluentParserName}
+        Format regex
+        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+    `;
+
+    new Helm(this, 'aws-for-fluent-bit', {
+      chart: 'aws-for-fluent-bit',
+      repo: 'https://aws.github.io/eks-charts',
+      namespace: 'fluent-bit',
+      version: '0.1.30',
+      values: {
+        fullnameOverride: 'fluentbit',
+        input: { parser: FluentParserName, dockerMode: 'Off' },
+        serviceAccount: { name: props.saRoleName, create: false },
+        cloudWatchLogs: { enabled: true, region: 'ap-southeast-2', autoCreateGroup: true, logRetentionDays: 30 },
+        firehose: { enabled: false },
+        kinesis: { enabled: false },
+        elasticsearch: { enabled: false },
+        service: { extraParsers: FluentContainerParser },
+      },
+    });
+  }
+}

config/charts/karpenter.ts

@@ -44,6 +44,7 @@ export class Karpenter extends Chart {
       namespace: 'karpenter',
       version: 'v0.31.0',
       values: {
+        // FIXME: fullnameOverride: 'karpenter',
         serviceAccount: {
           create: false,
           name: props.saRoleName,

config/constants.ts

@@ -1,5 +1,5 @@
 /* Cluster name */
-export const CLUSTER_NAME = 'Workflows';
+export const ClusterName = 'Workflows';
 
 /* CloudFormation Output to access from CDK8s */
 export const CfnOutputKeys = {
@@ -9,4 +9,7 @@ export const CfnOutputKeys = {
     ClusterEndpoint: 'ClusterEndpoint',
     DefaultInstanceProfile: 'DefaultInstanceProfile',
   },
+  FluentBit: {
+    ServiceAccountName: 'FluentBitServiceAccountName',
+  },
 };

config/eks/cluster.ts

@@ -11,8 +11,7 @@ import {
   Role,
   ServicePrincipal,
 } from 'aws-cdk-lib/aws-iam';
-import { IBucket } from 'aws-cdk-lib/aws-s3';
-import { BlockPublicAccess, Bucket } from 'aws-cdk-lib/aws-s3';
+import { BlockPublicAccess, Bucket, IBucket } from 'aws-cdk-lib/aws-s3';
 import { Construct } from 'constructs';
 
 import { CfnOutputKeys } from '../constants';
@@ -199,7 +198,8 @@ export class LinzEksCluster extends Stack {
       namespace: 'fluent-bit',
     });
     fluentBitSa.node.addDependency(fluentBitNs); // Ensure the namespace created first
-    new CfnOutput(this, 'FluentBitServiceAccountRoleArn', { value: fluentBitSa.role.roleArn });
+
+    new CfnOutput(this, CfnOutputKeys.FluentBit.ServiceAccountName, { value: fluentBitSa.serviceAccountName });
 
     // Basic constructs for argo to be deployed into
     const argoNs = this.cluster.addManifest('ArgoNameSpace', {