feat: config karpenter TDE-903

.gitignore

@@ -2,3 +2,4 @@ node_modules/
 dist/
 cdk.out/
 cdk.context.json
+config/imports/

cdk8s.yaml

@@ -1,2 +1,5 @@
 app: npx tsx config/cdk8s.ts
 language: typescript
+imports:
+  - https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.sh_provisioners.yaml
+  - https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml

config/cdk8s.ts

@@ -2,7 +2,7 @@ import { App } from 'cdk8s';
 
 import { CfnOutputKeys } from './cfn.output';
 import { ArgoSemaphore } from './charts/argo.semaphores';
-import { Karpenter } from './charts/karpenter';
+import { Karpenter, KarpenterProvisioner } from './charts/karpenter';
 import { getCfnOutputs } from './util/cloud.formation';
 
 const app = new App();
@@ -17,14 +17,24 @@ async function main(): Promise<void> {
 
   new ArgoSemaphore(app, 'semaphore', {});
 
-  new Karpenter(app, 'karpenter', {
+  const karpenter = new Karpenter(app, 'karpenter', {
     clusterName: 'Workflows',
     clusterEndpoint: cfnOutputs[CfnOutputKeys.Karpenter.ClusterEndpoint],
     saRoleName: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountName],
     saRoleArn: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountRoleArn],
     instanceProfile: cfnOutputs[CfnOutputKeys.Karpenter.DefaultInstanceProfile],
   });
 
+  const karpenterProvisioner = new KarpenterProvisioner(app, 'karpenter-provisioner', {
+    clusterName: 'Workflows',
+    clusterEndpoint: cfnOutputs[CfnOutputKeys.Karpenter.ClusterEndpoint],
+    saRoleName: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountName],
+    saRoleArn: cfnOutputs[CfnOutputKeys.Karpenter.ServiceAccountRoleArn],
+    instanceProfile: cfnOutputs[CfnOutputKeys.Karpenter.DefaultInstanceProfile],
+  });
+
+  karpenterProvisioner.addDependency(karpenter);
+
   app.synth();
 }
 

config/charts/karpenter.ts

@@ -1,6 +1,11 @@
-import { Chart, ChartProps, Helm } from 'cdk8s';
+import { Chart, ChartProps, Duration, Helm } from 'cdk8s';
 import { Construct } from 'constructs';
 
+import {
+  AwsNodeTemplateSpec,
+  AwsNodeTemplateSpecBlockDeviceMappingsEbsVolumeSize,
+} from '../imports/karpenter.k8s.aws.js';
+import { Provisioner, ProvisionerSpecLimitsResources } from '../imports/karpenter.sh.js';
 import { applyDefaultLabels } from '../util/labels.js';
 
 export interface KarpenterProps {
@@ -16,12 +21,32 @@ export class Karpenter extends Chart {
     // TODO: What is the component name? 'karpenter' or 'autoscaling'?
     super(scope, id, applyDefaultLabels(props, 'karpenter', '', 'karpenter', 'workflows'));
 
+    // Deploying the CRD
+    new Helm(this, 'karpenter-crd', {
+      chart: 'oci://public.ecr.aws/karpenter/karpenter-crd',
+      namespace: 'karpenter',
+      version: 'v0.31.0',
+    });
+
+    // Karpenter is using `oci` rather than regular helm repo: https://gallery.ecr.aws/karpenter/karpenter.
+    // This Helm constructor has been tricked to be able to use `oci`,
+    // the `oci` repo is passed inside `chart` instead of `repo` so the generated `helm`
+    // command is the following:
+    // [
+    //   'template',
+    //   '-f',
+    //   '/tmp/cdk8s-helm-keYZCA/overrides.yaml',
+    //   '--version',
+    //   'v0.31.0',
+    //   '--namespace',
+    //   'karpenter',
+    //   'karpenter-c870a560',
+    //   'oci://public.ecr.aws/karpenter/karpenter'
+    // ]
     new Helm(this, 'karpenter', {
-      chart: 'karpenter',
-      repo: 'oci://public.ecr.aws/karpenter/karpenter',
+      chart: 'oci://public.ecr.aws/karpenter/karpenter',
       namespace: 'karpenter',
       version: 'v0.31.0',
-      releaseName: 'karpenter',
       values: {
         serviceAccount: {
           create: false,
@@ -39,3 +64,62 @@ export class Karpenter extends Chart {
     });
   }
 }
+
+export class KarpenterProvisioner extends Chart {
+  constructor(scope: Construct, id: string, props: KarpenterProps & ChartProps) {
+    // TODO: What is the component name? 'karpenter' or 'autoscaling'?
+    super(scope, id, applyDefaultLabels(props, 'karpenter', '', 'karpenter', 'workflows'));
+
+    const provider: AwsNodeTemplateSpec = {
+      amiFamily: 'Bottlerocket',
+      subnetSelector: { 'aws-ids': '' }, // TODO How to get those?
+      securityGroupSelector: { [`kubernetes.io/cluster/${props.clusterName}`]: 'owned' },
+      instanceProfile: props.instanceProfile,
+      blockDeviceMappings: [
+        {
+          deviceName: '/dev/xvdb',
+          ebs: {
+            volumeType: 'gp3',
+            // FIXME: This does not match `'^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`
+            volumeSize: AwsNodeTemplateSpecBlockDeviceMappingsEbsVolumeSize.fromString('200Gi'),
+            deleteOnTermination: true,
+          },
+        },
+      ],
+    };
+
+    new Provisioner(this, 'ClusterAmd64WorkerNodes', {
+      metadata: { name: `eks-karpenter-${props.clusterName}-amd64`.toLowerCase(), namespace: 'karpenter' },
+      spec: {
+        requirements: [
+          { key: 'karpenter.sh/capacity-type', operator: 'In', values: ['spot'] },
+          { key: 'kubernetes.io/arch', operator: 'In', values: ['amd64'] },
+          { key: 'karpenter.k8s.aws/instance-family', operator: 'In', values: ['c5', 'c6i', 'c6a'] },
+        ],
+        limits: { resources: { cpu: ProvisionerSpecLimitsResources.fromString('20000m') } },
+        provider,
+        ttlSecondsAfterEmpty: Duration.minutes(1).toSeconds(), // optional, but never scales down if not set
+      },
+    });
+
+    new Provisioner(this, 'ClusterArmWorkerNodes', {
+      metadata: { name: `eks-karpenter-${props.clusterName}-arm64`.toLowerCase(), namespace: 'karpenter' },
+      spec: {
+        //Instances that want ARM have to tolerate the arm taint
+        // This prevenkarpenter-c870a560-76646d448b-fcq6lts some pods from accidentally trying to start on ARM
+        taints: [
+          { key: 'kubernetes.io/arch', value: 'arm64', effect: 'NoSchedule' },
+          { key: 'karpenter.sh/capacity-type', value: 'spot', effect: 'NoSchedule' },
+        ],
+        requirements: [
+          { key: 'karpenter.sh/capacity-type', operator: 'In', values: ['spot'] },
+          { key: 'kubernetes.io/arch', operator: 'In', values: ['arm64'] },
+          { key: 'karpenter.k8s.aws/instance-family', operator: 'In', values: ['c7g', 'c6g'] },
+        ],
+        limits: { resources: { cpu: ProvisionerSpecLimitsResources.fromString('20000m') } },
+        provider,
+        ttlSecondsAfterEmpty: Duration.minutes(1).toSeconds(), // optional, but never scales down if not set
+      },
+    });
+  }
+}

config/eks/cluster.ts

@@ -128,6 +128,8 @@ export class LinzEksCluster extends Stack {
     );
 
     // Allow Karpenter to start ec2 instances
+    // FIXME: some policies are missing. See https://github.com/aws/karpenter/blob/8c33a40733b90aa0bb42a6436152374f7b359f69/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml#L40
+    // The current policies are based on https://github.com/eksctl-io/eksctl/blob/main/pkg/cfn/builder/karpenter_test.go#L111
     new Policy(this, 'ControllerPolicy', {
       roles: [serviceAccount.role],
       statements: [
@@ -136,19 +138,22 @@ export class LinzEksCluster extends Stack {
             'ec2:CreateFleet',
             'ec2:CreateLaunchTemplate',
             'ec2:CreateTags',
-            'ec2:DeleteLaunchTemplate',
             'ec2:DescribeAvailabilityZones',
-            'ec2:DescribeInstances',
             'ec2:DescribeInstanceTypeOfferings',
             'ec2:DescribeInstanceTypes',
+            'ec2:DescribeInstances',
             'ec2:DescribeLaunchTemplates',
             'ec2:DescribeSecurityGroups',
             'ec2:DescribeSubnets',
+            'ec2:DeleteLaunchTemplate',
             'ec2:RunInstances',
             'ec2:TerminateInstances',
+            'ec2:DescribeImages',
+            'ec2:DescribeSpotPriceHistory',
             'iam:PassRole',
+            'iam:CreateServiceLinkedRole',
             'ssm:GetParameter',
-
+            'pricing:GetProducts',
             // LINZ requires instances to be encrypted with a KMS key
             'kms:Encrypt',
             'kms:Decrypt',