From d1b0a324a1959d36dcc0ad7e8bf97345328efe6b Mon Sep 17 00:00:00 2001
From: Ahmet AYDIN <ahmet_ayd@msn.com>
Date: Sat, 25 Nov 2023 06:57:58 -0800
Subject: [PATCH 1/6] add windows profisioner and nodetemplate

---
 aws-eks-addons/main.tf      | 108 +++++++++++++++++++++++++++++++++++-
 aws-eks-addons/variables.tf |  24 +++++++-
 2 files changed, 129 insertions(+), 3 deletions(-)

diff --git a/aws-eks-addons/main.tf b/aws-eks-addons/main.tf
index 8bf779b..72447e9 100644
--- a/aws-eks-addons/main.tf
+++ b/aws-eks-addons/main.tf
@@ -727,8 +727,6 @@ resource "kubectl_manifest" "karpenter_stateless_provisioner" {
   ]
 }
 
-
-
 resource "kubectl_manifest" "karpenter_node_template" {
   count = var.deploy_karpenter ? 1 : 0
   yaml_body = <<-YAML
@@ -758,3 +756,109 @@ resource "kubectl_manifest" "karpenter_node_template" {
     helm_release.karpenter[0]
   ]
 }
+
+resource "kubectl_manifest" "karpenter_windows_with_aws_cni" {
+  count      = var.karpenter_windows_support ? 1 : 0
+  yaml_body  = <<YAML
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: amazon-vpc-cni
+  namespace: kube-system
+data:
+  enable-windows-ipam: "true"
+YAML
+  depends_on = [helm_release.karpenter]
+}
+
+resource "kubectl_manifest" "karpenter_stateless_windows_provisioner" {
+  count = var.karpenter_windows_support ? 1 : 0
+
+  yaml_body = yamlencode({
+    apiVersion: "karpenter.sh/v1alpha5"
+    kind: "Provisioner"
+    metadata: {
+      name: "stateless-windows-provisioner"
+    }
+    spec: {
+      consolidation: {
+        enabled = true
+      }
+      ttlSecondsAfterEmpty = 30
+      providerRef = {
+        name = "windows2019"
+      }
+      limits = {
+        resources = {
+          cpu = var.stateless_total_cpu_limit
+        }
+      }
+      requirements: [
+        {
+          key = "karpenter.sh/capacity-type"
+          operator = "In"
+          values = var.stateless_windows_capacity_types
+        },
+        {
+          key = "node.kubernetes.io/instance-type"
+          operator = "In"
+          values = var.stateless_windows_instance_types
+        },
+        {
+          key = "topology.kubernetes.io/zone"
+          operator = "In"
+          values = var.stateless_instance_zones
+        },
+        {
+          key = "kubernetes.io/arch"
+          operator = "In"
+          values = var.stateless_windows_arch_types
+        },
+        {
+          key = "kubernetes.io/os"
+          operator = "In"
+          values = [
+            "windows",
+          ]
+        },
+      ]
+    }
+  })
+  depends_on = [
+    helm_release.karpenter[0]
+  ]
+}
+resource "kubectl_manifest" "karpenter_windows_node_template" {
+  count = var.karpenter_windows_support ? 1 : 0
+  yaml_body = <<-YAML
+    apiVersion: karpenter.k8s.aws/v1alpha1
+    kind: AWSNodeTemplate
+    metadata:
+      name: windows2019
+    spec:
+      blockDeviceMappings:
+        - deviceName: /dev/xvda
+          ebs:
+            volumeSize: ${var.karpenter_node_template_volume_size}
+            volumeType: ${var.karpenter_node_template_volume_type}
+            iops: ${var.karpenter_node_template_volume_iops}
+            deleteOnTermination: ${var.karpenter_node_template_delete_on_termination}
+            throughput: ${var.karpenter_node_template_throughput}
+      subnetSelector:
+        karpenter.sh/discovery: "true"
+      securityGroupSelector:
+        karpenter.sh/discovery: ${var.cluster_name}
+      tags:
+        karpenter.sh/discovery: ${var.cluster_name}
+      amiFamily: Windows2019
+      metadataOptions:
+        httpEndpoint: enabled
+        httpProtocolIPv6: disabled
+        httpPutResponseHopLimit: 2
+        httpTokens: required
+  YAML
+
+  depends_on = [
+    helm_release.karpenter[0]
+  ]
+}
\ No newline at end of file
diff --git a/aws-eks-addons/variables.tf b/aws-eks-addons/variables.tf
index 88012e1..3252048 100644
--- a/aws-eks-addons/variables.tf
+++ b/aws-eks-addons/variables.tf
@@ -231,19 +231,30 @@ variable "stateful_capacity_types" { # +
   type = list(string)
 }
 
-
 variable "stateless_capacity_types" {
   description = "instance types"
   default = ["spot"]
   type = list(string)
 }
 
+variable "stateless_windows_capacity_types" {
+  description = "instance types"
+  default = ["spot"]
+  type = list(string)
+}
+
 variable "stateful_instance_types" { 
   description = "instance types"
   default = ["r6a.xlarge", "r6a.2xlarge", "r6a.4xlarge","r6a.9xlarge"]
   type = list(string)
 }
 
+variable "stateful_windows_instance_types" { 
+  description = "instance types"
+  default = ["r6a.xlarge", "r6a.2xlarge", "r6a.4xlarge","r6a.9xlarge"]
+  type = list(string)
+}
+
 variable "stateless_instance_types" {
   description = "instance types"
   default = ["c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge","c5n.9xlarge"]
@@ -274,6 +285,12 @@ variable "stateless_arch_types" {
   type = list(string)
 }
 
+variable "stateless_windows_arch_types" {
+  description = "instance types"
+  default = ["amd64"]
+  type = list(string)
+}
+
 variable "stateful_application_toleration_value" {
   description = "stateful application tolerance value for scheduling"
   default = "stateful-application"
@@ -355,4 +372,9 @@ variable "aws_lb_controller_version" {
 variable "karpenter_version" {
   description = "Karpenter version"
   default = "v0.31.1"
+}
+
+variable "karpenter_windows_support" {
+  default = false
+  description = "Karpenter Windows Container Support"
 }
\ No newline at end of file

From 48efee2f4ee7c4bc64da8e2f7dae26e2a07759fb Mon Sep 17 00:00:00 2001
From: Ahmet AYDIN <ahmet_ayd@msn.com>
Date: Sun, 26 Nov 2023 14:23:24 -0800
Subject: [PATCH 2/6] var update

---
 aws-eks-addons/variables.tf | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/aws-eks-addons/variables.tf b/aws-eks-addons/variables.tf
index 3252048..a2019a3 100644
--- a/aws-eks-addons/variables.tf
+++ b/aws-eks-addons/variables.tf
@@ -261,6 +261,12 @@ variable "stateless_instance_types" {
   type = list(string)
 }
 
+variable "stateless_windows_instance_types" {
+  description = "instance types"
+  default = ["c5n.xlarge", "c5n.2xlarge", "c5n.4xlarge","c5n.9xlarge"]
+  type = list(string)
+}
+
 variable "stateful_instance_zones" { 
   description = "instance types"
   default = ["eu-west-1a"]

From c2490e74e265b664fe7eb7d74323f3faf753880c Mon Sep 17 00:00:00 2001
From: Ahmet AYDIN <ahmet_ayd@msn.com>
Date: Mon, 27 Nov 2023 07:59:07 -0800
Subject: [PATCH 3/6] karpenter provider format change

---
 aws-eks-addons/main.tf | 130 +++++++++++++++++++++++++----------------
 1 file changed, 80 insertions(+), 50 deletions(-)

diff --git a/aws-eks-addons/main.tf b/aws-eks-addons/main.tf
index 72447e9..ae98d8f 100644
--- a/aws-eks-addons/main.tf
+++ b/aws-eks-addons/main.tf
@@ -773,57 +773,87 @@ YAML
 
 resource "kubectl_manifest" "karpenter_stateless_windows_provisioner" {
   count = var.karpenter_windows_support ? 1 : 0
-
-  yaml_body = yamlencode({
-    apiVersion: "karpenter.sh/v1alpha5"
-    kind: "Provisioner"
-    metadata: {
-      name: "stateless-windows-provisioner"
-    }
-    spec: {
-      consolidation: {
-        enabled = true
-      }
+  yaml_body = <<-YAML
+    apiVersion: karpenter.k8s.aws/v1alpha5
+    kind: Provisioner
+    metadata:
+      name: stateless-windows-provisioner
+    spec:
       ttlSecondsAfterEmpty = 30
-      providerRef = {
-        name = "windows2019"
-      }
-      limits = {
-        resources = {
-          cpu = var.stateless_total_cpu_limit
-        }
-      }
-      requirements: [
-        {
-          key = "karpenter.sh/capacity-type"
-          operator = "In"
-          values = var.stateless_windows_capacity_types
-        },
-        {
-          key = "node.kubernetes.io/instance-type"
-          operator = "In"
-          values = var.stateless_windows_instance_types
-        },
-        {
-          key = "topology.kubernetes.io/zone"
-          operator = "In"
-          values = var.stateless_instance_zones
-        },
-        {
-          key = "kubernetes.io/arch"
-          operator = "In"
-          values = var.stateless_windows_arch_types
-        },
-        {
-          key = "kubernetes.io/os"
-          operator = "In"
-          values = [
-            "windows",
-          ]
-        },
-      ]
-    }
-  })
+      consolidateion:
+        enabled: true
+      providerRef:
+        name: "windows2019"
+      limits:
+        resources:
+          cpu: ${var.stateless_total_cpu_limit}
+      requirements:
+      - key: karpenter.sh/capacity-type
+        operator: In
+        values: ${var.stateless_windows_capacity_types}
+      - key: node.kubernetes.io/instance-type
+        operator: In
+        values: ${var.stateless_windows_instance_types}
+      - key: topology.kubernetes.io/zone
+        operator: In
+        values: ${var.stateless_instance_zones}
+      - key: kubernetes.io/arch
+        operator: In
+        values: ${var.stateless_windows_arch_types}
+      - key: kubernetes.io/os
+        operator: In
+        values: windows
+  YAML
+  # yaml_body = yamlencode({
+  #   apiVersion: "karpenter.sh/v1alpha5"
+  #   kind: "Provisioner"
+  #   metadata: {
+  #     name: "stateless-windows-provisioner"
+  #   }
+  #   spec: {
+  #     consolidation: {
+  #       enabled = true
+  #     }
+  #     ttlSecondsAfterEmpty = 30
+  #     providerRef = {
+  #       name = "windows2019"
+  #     }
+  #     limits = {
+  #       resources = {
+  #         cpu = var.stateless_total_cpu_limit
+  #       }
+  #     }
+  #     requirements: [
+  #       {
+  #         key = "karpenter.sh/capacity-type"
+  #         operator = "In"
+  #         values = var.stateless_windows_capacity_types
+  #       },
+  #       {
+  #         key = "node.kubernetes.io/instance-type"
+  #         operator = "In"
+  #         values = var.stateless_windows_instance_types
+  #       },
+  #       {
+  #         key = "topology.kubernetes.io/zone"
+  #         operator = "In"
+  #         values = var.stateless_instance_zones
+  #       },
+  #       {
+  #         key = "kubernetes.io/arch"
+  #         operator = "In"
+  #         values = var.stateless_windows_arch_types
+  #       },
+  #       {
+  #         key = "kubernetes.io/os"
+  #         operator = "In"
+  #         values = [
+  #           "windows",
+  #         ]
+  #       },
+  #     ]
+  #   }
+  # })
   depends_on = [
     helm_release.karpenter[0]
   ]

From 1064d73da1fa90139627cd4262bff3990160b051 Mon Sep 17 00:00:00 2001
From: Ahmet AYDIN <ahmet_ayd@msn.com>
Date: Mon, 27 Nov 2023 09:14:57 -0800
Subject: [PATCH 4/6] provider config

---
 aws-eks-addons/main.tf | 157 ++++++++++++++++++++---------------------
 1 file changed, 77 insertions(+), 80 deletions(-)

diff --git a/aws-eks-addons/main.tf b/aws-eks-addons/main.tf
index ae98d8f..508e455 100644
--- a/aws-eks-addons/main.tf
+++ b/aws-eks-addons/main.tf
@@ -773,87 +773,84 @@ YAML
 
 resource "kubectl_manifest" "karpenter_stateless_windows_provisioner" {
   count = var.karpenter_windows_support ? 1 : 0
-  yaml_body = <<-YAML
-    apiVersion: karpenter.k8s.aws/v1alpha5
-    kind: Provisioner
-    metadata:
-      name: stateless-windows-provisioner
-    spec:
-      ttlSecondsAfterEmpty = 30
-      consolidateion:
-        enabled: true
-      providerRef:
-        name: "windows2019"
-      limits:
-        resources:
-          cpu: ${var.stateless_total_cpu_limit}
-      requirements:
-      - key: karpenter.sh/capacity-type
-        operator: In
-        values: ${var.stateless_windows_capacity_types}
-      - key: node.kubernetes.io/instance-type
-        operator: In
-        values: ${var.stateless_windows_instance_types}
-      - key: topology.kubernetes.io/zone
-        operator: In
-        values: ${var.stateless_instance_zones}
-      - key: kubernetes.io/arch
-        operator: In
-        values: ${var.stateless_windows_arch_types}
-      - key: kubernetes.io/os
-        operator: In
-        values: windows
-  YAML
-  # yaml_body = yamlencode({
-  #   apiVersion: "karpenter.sh/v1alpha5"
-  #   kind: "Provisioner"
-  #   metadata: {
-  #     name: "stateless-windows-provisioner"
-  #   }
-  #   spec: {
-  #     consolidation: {
-  #       enabled = true
-  #     }
+  # yaml_body = <<-YAML
+  #   apiVersion: karpenter.k8s.aws/v1alpha5
+  #   kind: Provisioner
+  #   metadata:
+  #     name: stateless-windows-provisioner
+  #   spec:
   #     ttlSecondsAfterEmpty = 30
-  #     providerRef = {
-  #       name = "windows2019"
-  #     }
-  #     limits = {
-  #       resources = {
-  #         cpu = var.stateless_total_cpu_limit
-  #       }
-  #     }
-  #     requirements: [
-  #       {
-  #         key = "karpenter.sh/capacity-type"
-  #         operator = "In"
-  #         values = var.stateless_windows_capacity_types
-  #       },
-  #       {
-  #         key = "node.kubernetes.io/instance-type"
-  #         operator = "In"
-  #         values = var.stateless_windows_instance_types
-  #       },
-  #       {
-  #         key = "topology.kubernetes.io/zone"
-  #         operator = "In"
-  #         values = var.stateless_instance_zones
-  #       },
-  #       {
-  #         key = "kubernetes.io/arch"
-  #         operator = "In"
-  #         values = var.stateless_windows_arch_types
-  #       },
-  #       {
-  #         key = "kubernetes.io/os"
-  #         operator = "In"
-  #         values = [
-  #           "windows",
-  #         ]
-  #       },
-  #     ]
-  #   }
-  # })
+  #     consolidateion:
+  #       enabled: true
+  #     providerRef:
+  #       name: "windows2019"
+  #     limits:
+  #       resources:
+  #         cpu: ${var.stateless_total_cpu_limit}
+  #     requirements:
+  #     - key: karpenter.sh/capacity-type
+  #       operator: In
+  #       values: ${var.stateless_windows_capacity_types}
+  #     - key: node.kubernetes.io/instance-type
+  #       operator: In
+  #       values: ${var.stateless_windows_instance_types}
+  #     - key: topology.kubernetes.io/zone
+  #       operator: In
+  #       values: ${var.stateless_instance_zones}
+  #     - key: kubernetes.io/arch
+  #       operator: In
+  #       values: ${var.stateless_windows_arch_types}
+  #     - key: kubernetes.io/os
+  #       operator: In
+  #       values: windows
+  # YAML
+  yaml_body = yamlencode({
+    apiVersion: "karpenter.sh/v1alpha5"
+    kind: "Provisioner"
+    metadata: {
+      name: "stateless-windows-provisioner"
+    }
+    spec: {
+      ttlSecondsAfterEmpty = 30
+      providerRef = {
+        name = "windows2019"
+      }
+      limits = {
+        resources = {
+          cpu = var.stateless_total_cpu_limit
+        }
+      }
+      requirements: [
+        {
+          key = "karpenter.sh/capacity-type"
+          operator = "In"
+          values = var.stateless_windows_capacity_types
+        },
+        {
+          key = "node.kubernetes.io/instance-type"
+          operator = "In"
+          values = var.stateless_windows_instance_types
+        },
+        {
+          key = "topology.kubernetes.io/zone"
+          operator = "In"
+          values = var.stateless_instance_zones
+        },
+        {
+          key = "kubernetes.io/arch"
+          operator = "In"
+          values = var.stateless_windows_arch_types
+        },
+        {
+          key = "kubernetes.io/os"
+          operator = "In"
+          values = [
+            "windows",
+          ]
+        },
+      ]
+    }
+  })
   depends_on = [
     helm_release.karpenter[0]
   ]

From bae0c762ce59067bec1d754effd2614fe94ff3b2 Mon Sep 17 00:00:00 2001
From: Ahmet AYDIN <ahmet_ayd@msn.com>
Date: Tue, 28 Nov 2023 10:09:06 -0800
Subject: [PATCH 5/6] windows2022

---
 aws-eks-addons/main.tf | 125 ++++++++++++++++++++++++++++++-----------
 1 file changed, 92 insertions(+), 33 deletions(-)

diff --git a/aws-eks-addons/main.tf b/aws-eks-addons/main.tf
index 508e455..5986d39 100644
--- a/aws-eks-addons/main.tf
+++ b/aws-eks-addons/main.tf
@@ -771,39 +771,8 @@ YAML
   depends_on = [helm_release.karpenter]
 }
 
-resource "kubectl_manifest" "karpenter_stateless_windows_provisioner" {
+resource "kubectl_manifest" "karpenter_stateless_windows2019_provisioner" {
   count = var.karpenter_windows_support ? 1 : 0
-  # yaml_body = <<-YAML
-  #   apiVersion: karpenter.k8s.aws/v1alpha5
-  #   kind: Provisioner
-  #   metadata:
-  #     name: stateless-windows-provisioner
-  #   spec:
-  #     ttlSecondsAfterEmpty = 30
-  #     consolidateion:
-  #       enabled: true
-  #     providerRef:
-  #       name: "windows2019"
-  #     limits:
-  #       resources:
-  #         cpu: ${var.stateless_total_cpu_limit}
-  #     requirements:
-  #     - key: karpenter.sh/capacity-type
-  #       operator: In
-  #       values: ${var.stateless_windows_capacity_types}
-  #     - key: node.kubernetes.io/instance-type
-  #       operator: In
-  #       values: ${var.stateless_windows_instance_types}
-  #     - key: topology.kubernetes.io/zone
-  #       operator: In
-  #       values: ${var.stateless_instance_zones}
-  #     - key: kubernetes.io/arch
-  #       operator: In
-  #       values: ${var.stateless_windows_arch_types}
-  #     - key: kubernetes.io/os
-  #       operator: In
-  #       values: windows
-  # YAML
   yaml_body = yamlencode({
     apiVersion: "karpenter.sh/v1alpha5"
     kind: "Provisioner"
@@ -855,7 +824,62 @@ resource "kubectl_manifest" "karpenter_stateless_windows_provisioner" {
     helm_release.karpenter[0]
   ]
 }
-resource "kubectl_manifest" "karpenter_windows_node_template" {
+
+resource "kubectl_manifest" "karpenter_stateless_windows2022_provisioner" {
+  count = var.karpenter_windows_support ? 1 : 0
+  yaml_body = yamlencode({
+    apiVersion: "karpenter.sh/v1alpha5"
+    kind: "Provisioner"
+    metadata: {
+      name: "stateless-windows2019-provisioner"
+    }
+    spec: {
+      ttlSecondsAfterEmpty = 30
+      providerRef = {
+        name = "windows2022"
+      }
+      limits = {
+        resources = {
+          cpu = var.stateless_total_cpu_limit
+        }
+      }
+      requirements: [
+        {
+          key = "karpenter.sh/capacity-type"
+          operator = "In"
+          values = var.stateless_windows_capacity_types
+        },
+        {
+          key = "node.kubernetes.io/instance-type"
+          operator = "In"
+          values = var.stateless_windows_instance_types
+        },
+        {
+          key = "topology.kubernetes.io/zone"
+          operator = "In"
+          values = var.stateless_instance_zones
+        },
+        {
+          key = "kubernetes.io/arch"
+          operator = "In"
+          values = var.stateless_windows_arch_types
+        },
+        {
+          key = "kubernetes.io/os"
+          operator = "In"
+          values = [
+            "windows",
+          ]
+        },
+      ]
+    }
+  })
+  depends_on = [
+    helm_release.karpenter[0]
+  ]
+}
+
+resource "kubectl_manifest" "karpenter_windows2019_node_template" {
   count = var.karpenter_windows_support ? 1 : 0
   yaml_body = <<-YAML
     apiVersion: karpenter.k8s.aws/v1alpha1
@@ -885,6 +909,41 @@ resource "kubectl_manifest" "karpenter_windows_node_template" {
         httpTokens: required
   YAML
 
+  depends_on = [
+    helm_release.karpenter[0]
+  ]
+}
+
+resource "kubectl_manifest" "karpenter_windows2022_node_template" {
+  count = var.karpenter_windows_support ? 1 : 0
+  yaml_body = <<-YAML
+    apiVersion: karpenter.k8s.aws/v1alpha1
+    kind: AWSNodeTemplate
+    metadata:
+      name: windows2022
+    spec:
+      blockDeviceMappings:
+        - deviceName: /dev/xvda
+          ebs:
+            volumeSize: ${var.karpenter_node_template_volume_size}
+            volumeType: ${var.karpenter_node_template_volume_type}
+            iops: ${var.karpenter_node_template_volume_iops}
+            deleteOnTermination: ${var.karpenter_node_template_delete_on_termination}
+            throughput: ${var.karpenter_node_template_throughput}
+      subnetSelector:
+        karpenter.sh/discovery: "true"
+      securityGroupSelector:
+        karpenter.sh/discovery: ${var.cluster_name}
+      tags:
+        karpenter.sh/discovery: ${var.cluster_name}
+      amiFamily: Windows2022
+      metadataOptions:
+        httpEndpoint: enabled
+        httpProtocolIPv6: disabled
+        httpPutResponseHopLimit: 2
+        httpTokens: required
+  YAML
+
   depends_on = [
     helm_release.karpenter[0]
   ]

From ad850bb433ee0b42ca77893decd6fb4446db588f Mon Sep 17 00:00:00 2001
From: Ahmet AYDIN <ahmet_ayd@msn.com>
Date: Tue, 28 Nov 2023 10:33:24 -0800
Subject: [PATCH 6/6] provisioner config

---
 aws-eks-addons/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/aws-eks-addons/main.tf b/aws-eks-addons/main.tf
index 5986d39..f327d79 100644
--- a/aws-eks-addons/main.tf
+++ b/aws-eks-addons/main.tf
@@ -777,7 +777,7 @@ resource "kubectl_manifest" "karpenter_stateless_windows2019_provisioner" {
     apiVersion: "karpenter.sh/v1alpha5"
     kind: "Provisioner"
     metadata: {
-      name: "stateless-windows-provisioner"
+      name: "stateless-windows2019-provisioner"
     }
     spec: {
       ttlSecondsAfterEmpty = 30
@@ -831,7 +831,7 @@ resource "kubectl_manifest" "karpenter_stateless_windows2022_provisioner" {
     apiVersion: "karpenter.sh/v1alpha5"
     kind: "Provisioner"
     metadata: {
-      name: "stateless-windows2019-provisioner"
+      name: "stateless-windows2022-provisioner"
     }
     spec: {
       ttlSecondsAfterEmpty = 30