Added tcp keep alive config

CHANGELOG.md

@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [7.2.1] - 2024-07-01
+### Added
+- Issue where requests can hit 10min connection timeout, TCP keepalive prevents NLB closing idle connections. Similar to the issue explained here: https://paramount.tech/blog/2021/07/26/mitigation-of-connection-reset-in-aws.html
+
 ## [7.2.0] - 2024-06-26
 ### Added
 - Added `hms_ro_datanucleus_connection_pooling_type`, `hms_rw_datanucleus_connection_pooling_type`, `hms_ro_datanucleus_connection_pool_config`, `hms_rw_datanucleus_connection_pool_config`, `hms_housekeeper_db_connection_pool_size` variables to allow specifying the pooling driver and its config

k8s-readonly.tf

@@ -42,6 +42,23 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
       spec {
         service_account_name            = kubernetes_service_account_v1.hms_readonly[0].metadata.0.name
         automount_service_account_token = true
+        dynamic "security_context"  {
+          for_each = var.enable_tcp_keepalive ? ["enabled"] : []
+          content {
+            sysctl {
+              name  = "net.ipv4.tcp_keepalive_time"
+              value = var.tcp_keepalive_time
+            }
+            sysctl {
+              name  = "net.ipv4.tcp_keepalive_intvl"
+              value = var.tcp_keepalive_intvl
+            }
+            sysctl {
+              name  = "net.ipv4.tcp_keepalive_probes"
+              value = var.tcp_keepalive_probes
+            }
+          }
+        }
         dynamic "init_container" {
           for_each = var.external_database_host == "" ? ["enabled"] : []
 

k8s-readwrite.tf

@@ -42,6 +42,23 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
       spec {
         service_account_name            = kubernetes_service_account_v1.hms_readwrite[0].metadata.0.name
         automount_service_account_token = true
+        dynamic "security_context"  {
+          for_each = var.enable_tcp_keepalive ? ["enabled"] : []
+          content {
+            sysctl {
+              name  = "net.ipv4.tcp_keepalive_time"
+              value = var.tcp_keepalive_time
+            }
+            sysctl {
+              name  = "net.ipv4.tcp_keepalive_intvl"
+              value = var.tcp_keepalive_intvl
+            }
+            sysctl {
+              name  = "net.ipv4.tcp_keepalive_probes"
+              value = var.tcp_keepalive_probes
+            }
+          }
+        }
         dynamic "init_container" {
           for_each = var.external_database_host == "" ? ["enabled"] : []
           content {

templates.tf

@@ -69,6 +69,9 @@ locals{
     datadog_agent_version = var.datadog_agent_version
     datadog_agent_enabled = var.datadog_agent_enabled
     datadog_tags          = local.datadog_tags
+    tcp_keepalive_time     = var.tcp_keepalive_time
+    tcp_keepalive_intvl    = var.tcp_keepalive_intvl
+    tcp_keepalive_probes   = var.tcp_keepalive_probes
   })
 
   hms_readonly_template = templatefile("${path.module}/templates/apiary-hms-readonly.json", {
@@ -120,5 +123,8 @@ locals{
     metrics_port          = var.datadog_metrics_port
     datadog_agent_version = var.datadog_agent_version
     datadog_tags          = local.datadog_tags
+    tcp_keepalive_time     = var.tcp_keepalive_time
+    tcp_keepalive_intvl    = var.tcp_keepalive_intvl
+    tcp_keepalive_probes   = var.tcp_keepalive_probes
   })
 }

templates/apiary-hms-readonly.json

@@ -52,7 +52,21 @@
         "name": "nofile"
       }
     ],
-    "logConfiguration": {
+    "systemControls": [
+      {
+        "namespace": "net.ipv4.tcp_keepalive_time",
+        "value": "${tcp_keepalive_time}"
+      },
+      {
+        "namespace": "net.ipv4.tcp_keepalive_intvl",
+        "value": "${tcp_keepalive_intvl}"
+      },
+      {
+        "namespace": "net.ipv4.tcp_keepalive_probes",
+        "value": "${tcp_keepalive_probes}"
+     }
+    ],
+  "logConfiguration": {
         "logDriver": "awslogs",
         "options": {
             "awslogs-group": "${loggroup}",

templates/apiary-hms-readwrite.json

@@ -52,6 +52,20 @@
         "name": "nofile"
       }
     ],
+    "systemControls": [
+      {
+        "namespace": "net.ipv4.tcp_keepalive_time",
+        "value": "${tcp_keepalive_time}"
+      },
+      {
+        "namespace": "net.ipv4.tcp_keepalive_intvl",
+        "value": "${tcp_keepalive_intvl}"
+      },
+      {
+        "namespace": "net.ipv4.tcp_keepalive_probes",
+        "value": "${tcp_keepalive_probes}"
+      }
+    ],
     "logConfiguration": {
         "logDriver": "awslogs",
         "options": {

variables.tf

@@ -786,4 +786,28 @@ variable "hms_rw_datanucleus_connection_pool_config" {
   description = "A map of env vars supported by Apiary docker image that can configure the chosen Datanucleus connection pool"  
   type = map(any)
   default = {}
-}
+}
+
+variable "enable_tcp_keepalive" {
+  description = "Enable tcp keepalive settings on the hms pods"
+  type        = bool
+  default     = false
+}
+
+variable "tcp_keepalive_time" {
+  description = "Sets net.ipv4.tcp_keepalive_time (seconds)."
+  type        = number
+  default     = 200
+}
+
+variable "tcp_keepalive_intvl" {
+  description = "Sets net.ipv4.tcp_keepalive_intvl (seconds)."
+  type        = number
+  default     = 30
+}
+
+variable "tcp_keepalive_probes" {
+  description = "Sets net.ipv4.tcp_keepalive_probes (number)."
+  type        = number
+  default     = 2
+}