option to enable waggle-dance autoscaling on k8s (#100)

* variable to enable autoscaling

* fix

* configure cpu limits

* fix

* fix

* update comment

* update readme

Co-authored-by: Raj Poluri <[email protected]>

CHANGELOG.md

@@ -3,6 +3,9 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [3.4.0] - 2022-01-11
+- Variable to enable waggle-dance autoscaling on k8s.
+
 ## [3.3.15] - 2022-01-06
 - Add k8s headless service, so that clients running in k8s cluster can connect to a WD container instead of virtual ip.
 

README.md

@@ -17,14 +17,16 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
 | docker_version | Waggle Dance Docker image version. | string | - | yes |
 | domain_extension | Domain name to use for Route 53 entry and service discovery. | string | `lcl` | no |
 | enable_remote_metastore_dns | Option to enable creating DNS records for remote metastores. | string | `` | no |
+| enable_autoscaling | Enable k8s horizontal pod autoscaling. | bool | `false` | no |
 | graphite_host | Graphite server configured in Waggle Dance to send metrics to. | string | `localhost` | no |
 | graphite_port | Graphite server port. | string | `2003` | no |
 | graphite_prefix | Prefix addded to all metrics sent to Graphite from this Waggle Dance instance. | string | `waggle-dance` | no |
 | ingress_cidr | Generally allowed ingress CIDR list. | list | - | yes |
 | instance_name | Waggle Dance instance name to identify resources in multi-instance deployments. | string | `` | no |
 | k8s_namespace | K8s namespace to create waggle-dance deployment.| string | ``| no |
 | k8s_docker_registry_secret | Docker Registry authentication K8s secret name.| string | ``| no |
-| k8s_replica_count | Number of k8s pod replicas to create. | number | `3` | no |
+| k8s_replica_count | Initial Number of k8s pod replicas to create. | number | `3` | no |
+| k8s_max_replica_count | Max Number of k8s pod replicas to create. | number | `10` | no |
 | local_metastores | List of federated Metastore endpoints directly accessible on the local network. See section [`local_metastores`](#local_metastores) for more info.| list | `<list>` | no |
 | memory | The amount of memory (in MiB) used to allocate for the Waggle Dance container. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | `4096` | no |
 | primary_metastore_host | Primary Hive Metastore hostname configured in Waggle Dance. | string | `localhost` | no |
@@ -40,6 +42,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
 | vpc_id | VPC ID. | string | - | yes |
 | wd_ecs_task_count | Number of ECS tasks to create. | string | `1` | no |
 | wd_instance_type | Waggle Dance instance type, possible values: `ecs`,`k8s`. | string | `ecs` | no |
+| wd_target_cpu_percentage | Waggle Dance autoscaling threshold for CPU target usage. | number | `60` | no |
 | waggledance_version | Waggle Dance version to install on EC2 nodes | string | `3.3.2` | no |
 | root_vol_type | Waggle Dance EC2 root volume type. | string | `gp2` | no |
 | root_vol_size | Waggle Dance EC2 root volume size. | string | `10` | no |

k8s.tf

@@ -9,6 +9,8 @@ locals {
   memory_limit  = ceil((var.memory * 120) / 100)
   actuator_port = 18000
   wd_port       = 48869
+  k8s_cpu       = var.cpu / 1024
+  k8s_cpu_limit = (var.cpu / 1024) * 1.25
 }
 
 resource "kubernetes_service_account" "waggle_dance" {
@@ -84,9 +86,11 @@ resource "kubernetes_deployment" "waggle_dance" {
           }
           resources {
             limits {
+              cpu    = local.k8s_cpu_limit
               memory = "${local.memory_limit}Mi"
             }
             requests {
+              cpu    = local.k8s_cpu
               memory = "${var.memory}Mi"
             }
           }
@@ -109,6 +113,28 @@ resource "kubernetes_deployment" "waggle_dance" {
   }
 }
 
+resource "kubernetes_horizontal_pod_autoscaler" "waggle_dance" {
+  count = var.wd_instance_type == "k8s" && var.enable_autoscaling ? 1 : 0
+
+  metadata {
+    name      = local.instance_alias
+    namespace = var.k8s_namespace
+  }
+
+  spec {
+    min_replicas = var.k8s_replica_count
+    max_replicas = var.k8s_max_replica_count
+
+    target_cpu_utilization_percentage = var.wd_target_cpu_percentage
+
+    scale_target_ref {
+      api_version = "apps/v1"
+      kind        = "Deployment"
+      name        = kubernetes_deployment.waggle_dance[0].metadata[0].name
+    }
+  }
+}
+
 resource "kubernetes_service" "waggle_dance" {
   count = var.wd_instance_type == "k8s" ? 1 : 0
   metadata {

variables.tf

@@ -28,6 +28,18 @@ variable "wd_log_level" {
   default     = "info"
 }
 
+variable "enable_autoscaling" {
+  description = "Enable k8s horizontal pod autoscaling"
+  type        = bool
+  default     = false
+}
+
+variable "wd_target_cpu_percentage" {
+  description = "waggle-dance autoscaling threshold for CPU target usage."
+  type        = number
+  default     = 60
+}
+
 variable "root_vol_type" {
   description = "Waggle Dance EC2 root volume type."
   type        = string
@@ -58,11 +70,17 @@ variable "wd_ecs_task_count" {
 }
 
 variable "k8s_replica_count" {
-  description = "Number of k8s pod replicas to create."
+  description = "Initial Number of k8s pod replicas to create."
   type        = number
   default     = 3
 }
 
+variable "k8s_max_replica_count" {
+  description = "Max Number of k8s pod replicas to create."
+  type        = number
+  default     = 10
+}
+
 variable "vpc_id" {
   description = "VPC ID."
   type        = string