Use separate EBS Volumes for /var, /home folders (EKS 1.24+)

amazon-eks-al2.pkr.hcl

@@ -1,9 +1,3 @@
-locals {
- timestamp = regex_replace(timestamp(), "[- TZ:]", "")
-
- target_ami_name = "${var.ami_name_prefix}-${var.eks_version}-v${local.timestamp}"
-}
-
 data "amazon-ami" "this" {
  filters = {
  architecture = var.source_ami_arch
@@ -21,36 +15,68 @@ data "amazon-ami" "this" {
  region = var.aws_region
 }
 
-source "amazon-ebs" "this" {
- ami_block_device_mappings {
- delete_on_termination = true
- device_name = "/dev/sdb"
- volume_size = var.data_volume_size
- volume_type = "gp2"
- encrypted = true
+locals {
+ timestamp = regex_replace(timestamp(), "[- TZ:]", "")
+
+ target_ami_name = "${var.ami_name_prefix}-${var.eks_version}-v${local.timestamp}"
+
+ block_device_mappings = {
+ "/" = {
+ device_name = "/dev/xvda"
+ volume_size = var.root_volume_size
+ }
+ "/home" = {
+ device_name = "/dev/sdf"
+ volume_size = var.home_volume_size
+ }
+ "/var" = {
+ device_name = "/dev/sdg"
+ volume_size = var.var_volume_size
+ }
+ "/var/log" = {
+ device_name = "/dev/sdh"
+ volume_size = var.varlog_volume_size
+ }
+ "/var/log/audit" = {
+ device_name = "/dev/sdi"
+ volume_size = var.varlogaudit_volume_size
+ }
+ "/var/lib/containerd" = {
+ device_name = "/dev/sdj"
+ volume_size = var.varlibcontainerd_volume_size
+ }
  }
+}
 
+source "amazon-ebs" "this" {
  ami_description = "EKS Kubernetes Worker AMI with AmazonLinux2 image"
  ami_name = local.target_ami_name
  ami_virtualization_type = "hvm"
  instance_type = var.instance_type
 
- launch_block_device_mappings {
- delete_on_termination = true
- device_name = "/dev/xvda"
- volume_size = var.root_volume_size
- volume_type = "gp2"
- encrypted = true
- kms_key_id = var.kms_key_id
+ dynamic "ami_block_device_mappings" {
+ for_each = local.block_device_mappings
+
+ content {
+ device_name = ami_block_device_mappings.value.device_name
+ volume_size = ami_block_device_mappings.value.volume_size
+ delete_on_termination = true
+ volume_type = "gp3"
+ encrypted = true
+ }
  }
 
- launch_block_device_mappings {
- delete_on_termination = true
- device_name = "/dev/sdb"
- volume_size = var.data_volume_size
- volume_type = "gp2"
- encrypted = true
- kms_key_id = var.kms_key_id
+ dynamic "launch_block_device_mappings" {
+ for_each = local.block_device_mappings
+
+ content {
+ device_name = launch_block_device_mappings.value.device_name
+ volume_size = launch_block_device_mappings.value.volume_size
+ delete_on_termination = true
+ volume_type = "gp3"
+ encrypted = true
+ kms_key_id = var.kms_key_id
+ }
  }
 
  encrypt_boot = var.encrypt_boot

scripts/cis-docker.sh

@@ -23,7 +23,7 @@ echo "1.1.2 - ensure that the version of Docker is up to date"
 yum -y update docker
 
 echo "1.2.1 - ensure a separate partition for containers has been created"
-grep '/var/lib/docker\s' /proc/mounts
+#grep '/var/lib/docker\s' /proc/mounts
 
 echo "1.2.2 - ensure only trusted users are allowed to control Docker daemon"
 getent group docker

scripts/partition-disks.sh

@@ -17,21 +17,32 @@ set -o errexit
 # None
 ################################################################
 migrate_and_mount_disk() {
- local disk_name=$1
+ local device_name=$1
  local folder_path=$2
  local mount_options=$3
  local temp_path="/mnt${folder_path}"
  local old_path="${folder_path}-old"
 
- # install an ext4 filesystem to the disk
- mkfs -t ext4 ${disk_name}
+ # AWS EC2 API Block Device Mapping name to Linux NVME device name
+ disk_name="/dev/$(readlink "$device_name")"
+
+ # partition the disk (single data partition)
+ parted -a optimal -s $disk_name \
+ mklabel gpt \
+ mkpart data xfs 0% 90%
+
+ # wait for the disk to settle
+ sleep 5
+
+ # install an xfs filesystem to the disk
+ mkfs -t xfs "${disk_name}p1"
 
  # check if the folder already exists
  if [ -d "${folder_path}" ]; then
  FILE=$(ls -A ${folder_path})
  >&2 echo $FILE
  mkdir -p ${temp_path}
- mount ${disk_name} ${temp_path}
+ mount "${disk_name}p1" ${temp_path}
  # Empty folder give error on /*
  if [ ! -z "$FILE" ]; then
  cp -Rax ${folder_path}/* ${temp_path}
@@ -42,7 +53,7 @@ migrate_and_mount_disk() {
  mkdir -p ${folder_path}
 
  # add the mount point to fstab and mount the disk
- echo "UUID=$(blkid -s UUID -o value ${disk_name}) ${folder_path} ext4 ${mount_options} 0 1" >> /etc/fstab
+ echo "UUID=$(blkid -s UUID -o value "${disk_name}p1") ${folder_path} xfs ${mount_options} 0 1" >> /etc/fstab
  mount -a
 
  # if selinux is enabled restore the objects on it
@@ -51,27 +62,28 @@ migrate_and_mount_disk() {
  fi
 }
 
-disk_name='/dev/nvme1n1'
-
-# partition the disk
-parted -a optimal -s $disk_name \
- mklabel gpt \
- mkpart var ext4 0% 20% \
- mkpart varlog ext4 20% 40% \
- mkpart varlogaudit ext4 40% 60% \
- mkpart home ext4 60% 70% \
- mkpart varlibdocker ext4 70% 90%
-
-# wait for the disks to settle
-sleep 5
-
-# migrate and mount the existing
-migrate_and_mount_disk "${disk_name}p1" /var defaults,nofail,nodev
-migrate_and_mount_disk "${disk_name}p2" /var/log defaults,nofail,nodev,nosuid
-migrate_and_mount_disk "${disk_name}p3" /var/log/audit defaults,nofail,nodev,nosuid
-migrate_and_mount_disk "${disk_name}p4" /home defaults,nofail,nodev,nosuid
-
-# Create folder instead of starting/stopping docker daemon
-mkdir -p /var/lib/docker
-chown -R root:docker /var/lib/docker
-migrate_and_mount_disk "${disk_name}p5" /var/lib/docker defaults,nofail
+# migrate and mount the existing folders to dedicated EBS Volumes
+migrate_and_mount_disk "/dev/sdf" "/home" defaults,nofail,nodev,nosuid
+migrate_and_mount_disk "/dev/sdg" "/var" defaults,nofail,nodev
+migrate_and_mount_disk "/dev/sdh" "/var/log" defaults,nofail,nodev,nosuid
+migrate_and_mount_disk "/dev/sdi" "/var/log/audit" defaults,nofail,nodev,nosuid
+migrate_and_mount_disk "/dev/sdj" "/var/lib/containerd" defaults,nofail
+
+# Resize on instance launch
+cloud_init_script="/var/lib/cloud/scripts/per-boot/resize-disks.sh"
+cat > "$cloud_init_script" <<EOF
+#!/usr/bin/env bash
+
+set -x
+
+lsblk
+
+growpart "/dev/\$(readlink "/dev/sdf")" 1; xfs_growfs '/home'
+growpart "/dev/\$(readlink "/dev/sdg")" 1; xfs_growfs '/var'
+growpart "/dev/\$(readlink "/dev/sdh")" 1; xfs_growfs '/var/log'
+growpart "/dev/\$(readlink "/dev/sdi")" 1; xfs_growfs '/var/log/audit'
+growpart "/dev/\$(readlink "/dev/sdj")" 1; xfs_growfs '/var/lib/containerd'
+
+df -Th | grep -E 'Filesystem|xfs'
+EOF
+chmod +x "$cloud_init_script"

variables.pkr.hcl

@@ -4,15 +4,45 @@ variable "aws_region" {
  default = "us-west-2"
 }
 
-variable "data_volume_size" {
- description = "Size of the AMI data EBS volume"
- type = number
- default = 50
+variable "eks_version" {
+ description = "The EKS cluster version associated with the AMI created"
+ type = string
+ default = "1.22"
 }
 
 variable "root_volume_size" {
  description = "Size of the AMI root EBS volume"
  type = number
+ default = 4
+}
+
+variable "home_volume_size" {
+ description = "Size of the AMI /home EBS volume"
+ type = number
+ default = 1
+}
+
+variable "var_volume_size" {
+ description = "Size of the AMI /var EBS volume"
+ type = number
+ default = 4
+}
+
+variable "varlog_volume_size" {
+ description = "Size of the AMI /var/log EBS volume"
+ type = number
+ default = 1
+}
+
+variable "varlogaudit_volume_size" {
+ description = "Size of the AMI /var/log/audit EBS volume"
+ type = number
+ default = 1
+}
+
+variable "varlibcontainerd_volume_size" {
+ description = "Size of the AMI /var/lib/containerd EBS volume"
+ type = number
  default = 10
 }
 
@@ -34,12 +64,6 @@ variable "region_kms_key_ids" {
  default = null
 }
 
-variable "eks_version" {
- description = "The EKS cluster version associated with the AMI created"
- type = string
- default = "1.22"
-}
-
 variable "http_proxy" {
  description = "The HTTP proxy to set on the AMI created"
  type = string