diff --git a/edbterraform/data/terraform/aws/modules/machine/main.tf b/edbterraform/data/terraform/aws/modules/machine/main.tf index 7e0fe7bc..88388df1 100644 --- a/edbterraform/data/terraform/aws/modules/machine/main.tf +++ b/edbterraform/data/terraform/aws/modules/machine/main.tf @@ -161,15 +161,27 @@ locals { } locals { - script_variables = [ + volume_variables = [ for key, values in local.additional_volumes_map: { "device_names": element(local.linux_device_names, tonumber(key)) "number_of_volumes": length(lookup(var.machine.spec, "additional_volumes", [])) + 1 "mount_point": values.mount_point "mount_options": coalesce(try(join(",", values.mount_options), null), try(join(",", local.mount_options), null)) "filesystem": coalesce(values.filesystem, local.filesystem) + "volume_group": values.volume_group } ] + lvm_variables = { + for volume_group, values in var.machine.spec.volume_groups : volume_group => { + for mount_point, attibutes in values: mount_point => { + "size": coalesce(attibutes.size, "100%FREE") + "filesystem": coalesce(attibutes.filesystem, local.filesystem) + "mount_options": coalesce(try(join(",", attibutes.mount_options), null), try(join(",", local.mount_options), null)) + "type": "striped" + "stripesize": "64 KB" + } + } + } } resource "toolbox_external" "setup_volumes" { @@ -210,7 +222,7 @@ resource "toolbox_external" "setup_volumes" { fi # Execute Script - CMD="$SSH_CMD /tmp/setup_volume.sh ${base64encode(jsonencode(local.script_variables))} >> /tmp/mount.log" + CMD="$SSH_CMD /tmp/setup_volume.sh ${base64encode(jsonencode(local.volume_variables))} ${base64encode(jsonencode(local.lvm_variables))} >> /tmp/mount.log" RESULT=$($CMD) RC=$? if [[ $RC -ne 0 ]]; diff --git a/edbterraform/data/terraform/aws/modules/machine/setup_volume.sh b/edbterraform/data/terraform/aws/modules/machine/setup_volume.sh index cbf1a9a7..c48d4f62 100755 --- a/edbterraform/data/terraform/aws/modules/machine/setup_volume.sh +++ b/edbterraform/data/terraform/aws/modules/machine/setup_volume.sh @@ -14,12 +14,15 @@ fi # Expects a base64encoded json object SCRIPT_INPUTS=$(printf %s "$1" | base64 -d | jq -rc '.[]') +VOLUME_GROUPS=$(printf %s "$2" | base64 -d | jq -rc '.') +_jq_key() { + printf %s "$1" | jq -rc "$2" +} + +# Find volumes and create a filesystem or lvm volume groups for item in ${SCRIPT_INPUTS} do - _jq_key() { - printf %s "$1" | jq -rc "$2" - } # Expected device paths: ["/dev/sdX","/dev/sdY"] TARGET_DEVICES=$(_jq_key "$item" '.device_names[]') @@ -29,6 +32,7 @@ do N_NVME_DEVICE=$(_jq_key "$item" '.number_of_volumes') FSTYPE=$(_jq_key "$item" '.filesystem') FSMOUNTOPT=$(_jq_key "$item" '.mount_options') + VOLUME_GROUP=$(_jq_key "$item" '.volume_group') TARGET_NVME_DEVICE="" FSMOUNTOPT_ARG="" @@ -82,15 +86,26 @@ do exit 2 fi - if [ "${FSTYPE}" = "lvm" ]; then - if ! command -v lvm >/dev/null 2>&1 && [ -f /etc/redhat-release ]; then + if [ -n "${VOLUME_GROUP}" ] && [ "${VOLUME_GROUP}" != "null" ] + then + if ! command -v lvm >/dev/null 2>&1 && [ -f /etc/redhat-release ] + then sudo yum install lvm2 -y fi - if ! command -v lvm >/dev/null 2>&1 && [ -f /etc/debian_version ]; then + if ! command -v lvm >/dev/null 2>&1 && [ -f /etc/debian_version ] + then export DEBIAN_FRONTEND="noninteractive" sudo apt-get install lvm2 -y fi + # Check if volume exists to either create or extend volume group + if sudo vgs ${VOLUME_GROUP} >/dev/null 2>&1 + then + VG_CMD=vgextend + else + VG_CMD=vgcreate + fi sudo pvcreate "${TARGET_NVME_DEVICE}" + sudo "${VG_CMD}" "${VOLUME_GROUP}" "${TARGET_NVME_DEVICE}" else # Mount point and volume creation sudo "mkfs.${FSTYPE}" "${TARGET_NVME_DEVICE}" @@ -104,3 +119,43 @@ do fi done + +# Create logical volumes +_jq_key "${VOLUME_GROUPS}" "keys[]" | \ +while read -r VOLUME_GROUP +do + # Get the value of the key + VOLUME_DATA=$(_jq_key "${VOLUME_GROUPS}" ".\"${VOLUME_GROUP}\"") + _jq_key "${VOLUME_DATA}" "keys[]" | \ + while read -r MOUNT_POINT + do + MOUNT_DATA=$(_jq_key "${VOLUME_DATA}" ".\"${MOUNT_POINT}\"") + SIZE=$(_jq_key "${MOUNT_DATA}" '.size') + FILESYSTEM=$(_jq_key "${MOUNT_DATA}" '.filesystem') + MOUNT_OPTIONS=$(_jq_key "${MOUNT_DATA}" '.mount_options') + VOLUME_COUNT=$(sudo vgs -o pv_count --noheadings $VOLUME_GROUP | tr -d ' ') + LV_NAME=$(printf "%s" "${MOUNT_POINT}" | tr '/' '_') + LV_PATH="/dev/${VOLUME_GROUP}/${LV_NAME}" + # Create the logical volume + case "${SIZE}" in + *%*) + SIZE_CMD="--extents" + ;; + *) + SIZE_CMD="--size" + ;; + esac + sudo lvcreate "${SIZE_CMD}" "${SIZE}" --name "${LV_NAME}" --type striped --stripes "${VOLUME_COUNT}" "${VOLUME_GROUP}" + + # Create the filesystem + sudo "mkfs.${FILESYSTEM}" "${LV_PATH}" + # Create the mount point + sudo mkdir -p "${MOUNT_POINT}" + # Get device UUID with blkid as exported format: + # UUID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + printf "%s\n" "Warning: Will be mounted by UUID in /etc/fstab" + UUID=$(sudo blkid "${LV_PATH}" -o export | grep -E "^UUID=") + printf "%s\n" "${UUID} ${MOUNT_POINT} ${FILESYSTEM} ${MOUNT_OPTIONS} 0 0" | sudo tee -a /etc/fstab + sudo mount --all + done +done diff --git a/edbterraform/data/terraform/aws/modules/specification/variables.tf b/edbterraform/data/terraform/aws/modules/specification/variables.tf index d68eea8b..00bc749c 100644 --- a/edbterraform/data/terraform/aws/modules/specification/variables.tf +++ b/edbterraform/data/terraform/aws/modules/specification/variables.tf @@ -98,7 +98,7 @@ variable "spec" { # Use jbod_volumes which are meant to represent "Just a bunch of Disks(Volumes)" as an alternative # to manually manage per machine instance post-terraform additional_volumes = optional(list(object({ - mount_point = string + mount_point = optional(string) size_gb = number iops = optional(number) throughput = optional(number) @@ -106,7 +106,13 @@ variable "spec" { encrypted = optional(bool) filesystem = optional(string) mount_options = optional(string) + volume_group = optional(string) })), []) + volume_groups = optional(map(map(object({ + size = optional(string) + filesystem = optional(string) + mount_options = optional(string) + }))), {}) tags = optional(map(string), {}) })), {}) databases = optional(map(object({ diff --git a/infrastructure-examples/aws/machines-v2.yml b/infrastructure-examples/aws/machines-v2.yml index 204896c6..2474f8ff 100644 --- a/infrastructure-examples/aws/machines-v2.yml +++ b/infrastructure-examples/aws/machines-v2.yml @@ -12,14 +12,14 @@ aws: owner: 136693071363 ssh_user: admin regions: - us-east-1: + us-west-2: cidr_block: 10.2.0.0/16 zones: proxy: - zone: us-east-1b + zone: us-west-2b cidr: 10.2.20.0/24 main: - zone: us-east-1b + zone: us-west-2b cidr: 10.2.30.0/24 service_ports: - port: 22 @@ -35,10 +35,11 @@ aws: description: "ranges" machines: dbt2_driver: + spot: true image_name: debian - region: us-east-1 + region: us-west-2 zone_name: proxy - instance_type: c5.4xlarge + instance_type: t3a.medium volume: type: gp2 size_gb: 50 @@ -47,16 +48,15 @@ aws: tags: type: dbt2-driver pg1: - spot: true image_name: rocky - region: us-east-1 + region: us-west-2 zone_name: main ports: - protocol: icmp description: "ping" cidrs: - 10.2.20.0/24 - instance_type: c5.4xlarge + instance_type: t3a.medium volume: type: gp2 size_gb: 50 @@ -72,15 +72,38 @@ aws: size_gb: 100 encrypted: false additional_volumes: - - mount_point: /opt/pg_data - size_gb: 20 - type: io2 + - size_gb: 20 + type: gp3 + iops: 4000 + throughput: 1000 + encrypted: false + volume_group: data + - size_gb: 20 + type: gp2 iops: 5000 encrypted: false - - mount_point: /opt/pg_wal + volume_group: other + - mount_point: /opt/test size_gb: 20 - type: io2 + type: gp2 iops: 5000 encrypted: false + filesystem: xfs + - size_gb: 20 + type: gp3 + iops: 4000 + throughput: 1000 + encrypted: false + volume_group: data + volume_groups: + data: + /opt/data0: + size: "50%FREE" + filesystem: xfs + /opt/data1: + size: "50%FREE" + filesystem: xfs + other: + /opt/data2: {} tags: type: postgres