diff --git a/main.tf b/main.tf index d17b3aa..4c2a188 100644 --- a/main.tf +++ b/main.tf @@ -43,7 +43,7 @@ resource "aws_sqs_queue" "airflow_queue" { # --------------------------------------- module "ami_instance_profile" { - source = "git::https://github.com/traveloka/terraform-aws-iam-role//modules/instance?ref=tags/v1.0.1" + source = "git::https://github.com/traveloka/terraform-aws-iam-role//modules/instance?ref=tags/v1.0.2" service_name = "${module.airflow_labels.namespace}" cluster_role = "${module.airflow_labels.stage}" environment = "${module.airflow_labels.stage}" @@ -86,16 +86,16 @@ POLICY # ---------------------------------------------------------------------------------------- module "sg_airflow" { - source = "terraform-aws-modules/security-group/aws" - version = "3.1.0" - name = "${module.airflow_labels.id}-sg" - description = "Security group for ${module.airflow_labels.id} machines" - vpc_id = "${data.aws_vpc.default.id}" - ingress_cidr_blocks = "${var.ingress_cidr_blocks}" - ingress_rules = ["http-80-tcp", "https-443-tcp", "ssh-tcp"] + source = "terraform-aws-modules/security-group/aws" + version = "3.2.0" + name = "${module.airflow_labels.id}-sg" + description = "Security group for ${module.airflow_labels.id} machines" + vpc_id = "${data.aws_vpc.default.id}" + ingress_cidr_blocks = "${var.ingress_cidr_blocks}" + ingress_rules = ["http-80-tcp", "https-443-tcp", "ssh-tcp"] ingress_with_cidr_blocks = "${var.ingress_with_cidr_blocks}" - egress_rules = ["all-all"] - tags = "${module.airflow_labels.tags}" + egress_rules = ["all-all"] + tags = "${module.airflow_labels.tags}" } #------------------------------------------------------------------------- @@ -104,71 +104,71 @@ module "sg_airflow" { resource "aws_instance" "airflow_webserver" { count = 1 - instance_type = "${var.webserver_instance_type}" - ami = "${var.ami}" - key_name = "${aws_key_pair.auth.id}" + instance_type = "${var.webserver_instance_type}" + ami = "${var.ami}" + key_name = "${aws_key_pair.auth.id}" vpc_security_group_ids = ["${module.sg_airflow.this_security_group_id}"] - subnet_id = coalesce("${var.instance_subnet_id}", "${tolist(data.aws_subnet_ids.selected.ids)[count.index]}") - iam_instance_profile = "${module.ami_instance_profile.instance_profile_name}" + subnet_id = coalesce("${var.instance_subnet_id}", "${tolist(data.aws_subnet_ids.selected.ids)[count.index]}") + iam_instance_profile = "${module.ami_instance_profile.instance_profile_name}" associate_public_ip_address = true volume_tags = "${module.airflow_labels_webserver.tags}" root_block_device { - volume_type = "${var.root_volume_type}" - volume_size = "${var.root_volume_size}" + volume_type = "${var.root_volume_type}" + volume_size = "${var.root_volume_size}" delete_on_termination = "${var.root_volume_delete_on_termination}" } provisioner "file" { - content = "${data.template_file.custom_env.rendered}" + content = "${data.template_file.custom_env.rendered}" destination = "/tmp/custom_env" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.custom_requirements.rendered}" + content = "${data.template_file.custom_requirements.rendered}" destination = "/tmp/requirements.txt" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.airflow_environment.rendered}" + content = "${data.template_file.airflow_environment.rendered}" destination = "/tmp/airflow_environment" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.airflow_service.rendered}" + content = "${data.template_file.airflow_service.rendered}" destination = "/tmp/airflow.service" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } @@ -179,16 +179,16 @@ resource "aws_instance" "airflow_webserver" { ] connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } user_data = "${data.template_file.provisioner.rendered}" - tags = "${module.airflow_labels_webserver.tags}" + tags = "${module.airflow_labels_webserver.tags}" lifecycle { create_before_destroy = true @@ -198,71 +198,71 @@ resource "aws_instance" "airflow_webserver" { resource "aws_instance" "airflow_scheduler" { count = 1 - instance_type = "${var.scheduler_instance_type}" - ami = "${var.ami}" - key_name = "${aws_key_pair.auth.id}" + instance_type = "${var.scheduler_instance_type}" + ami = "${var.ami}" + key_name = "${aws_key_pair.auth.id}" vpc_security_group_ids = ["${module.sg_airflow.this_security_group_id}"] - subnet_id = coalesce("${var.instance_subnet_id}", "${tolist(data.aws_subnet_ids.selected.ids)[count.index]}") - iam_instance_profile = "${module.ami_instance_profile.instance_profile_name}" + subnet_id = coalesce("${var.instance_subnet_id}", "${tolist(data.aws_subnet_ids.selected.ids)[count.index]}") + iam_instance_profile = "${module.ami_instance_profile.instance_profile_name}" associate_public_ip_address = true volume_tags = "${module.airflow_labels_webserver.tags}" root_block_device { - volume_type = "${var.root_volume_type}" - volume_size = "${var.root_volume_size}" + volume_type = "${var.root_volume_type}" + volume_size = "${var.root_volume_size}" delete_on_termination = "${var.root_volume_delete_on_termination}" } provisioner "file" { - content = "${data.template_file.custom_env.rendered}" + content = "${data.template_file.custom_env.rendered}" destination = "/tmp/custom_env" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.custom_requirements.rendered}" + content = "${data.template_file.custom_requirements.rendered}" destination = "/tmp/requirements.txt" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.airflow_environment.rendered}" + content = "${data.template_file.airflow_environment.rendered}" destination = "/tmp/airflow_environment" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.airflow_service.rendered}" + content = "${data.template_file.airflow_service.rendered}" destination = "/tmp/airflow.service" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } @@ -273,16 +273,16 @@ resource "aws_instance" "airflow_scheduler" { ] connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } user_data = "${data.template_file.provisioner.rendered}" - tags = "${module.airflow_labels_scheduler.tags}" + tags = "${module.airflow_labels_scheduler.tags}" lifecycle { create_before_destroy = true @@ -292,71 +292,71 @@ resource "aws_instance" "airflow_scheduler" { resource "aws_instance" "airflow_worker" { count = "${var.worker_instance_count}" - instance_type = "${var.worker_instance_type}" - ami = "${var.ami}" - key_name = "${aws_key_pair.auth.id}" + instance_type = "${var.worker_instance_type}" + ami = "${var.ami}" + key_name = "${aws_key_pair.auth.id}" vpc_security_group_ids = ["${module.sg_airflow.this_security_group_id}"] - subnet_id = coalesce("${var.instance_subnet_id}", "${tolist(data.aws_subnet_ids.selected.ids)[count.index]}") - iam_instance_profile = "${module.ami_instance_profile.instance_profile_name}" + subnet_id = coalesce("${var.instance_subnet_id}", "${tolist(data.aws_subnet_ids.selected.ids)[count.index]}") + iam_instance_profile = "${module.ami_instance_profile.instance_profile_name}" associate_public_ip_address = true volume_tags = "${module.airflow_labels_webserver.tags}" root_block_device { - volume_type = "${var.root_volume_type}" - volume_size = "${var.root_volume_size}" + volume_type = "${var.root_volume_type}" + volume_size = "${var.root_volume_size}" delete_on_termination = "${var.root_volume_delete_on_termination}" } provisioner "file" { - content = "${data.template_file.custom_env.rendered}" + content = "${data.template_file.custom_env.rendered}" destination = "/tmp/custom_env" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.custom_requirements.rendered}" + content = "${data.template_file.custom_requirements.rendered}" destination = "/tmp/requirements.txt" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.airflow_environment.rendered}" + content = "${data.template_file.airflow_environment.rendered}" destination = "/tmp/airflow_environment" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } provisioner "file" { - content = "${data.template_file.airflow_service.rendered}" + content = "${data.template_file.airflow_service.rendered}" destination = "/tmp/airflow.service" connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } @@ -367,16 +367,16 @@ resource "aws_instance" "airflow_worker" { ] connection { - host = "${self.public_ip}" - agent = false - type = "ssh" - user = "ubuntu" + host = "${self.public_ip}" + agent = false + type = "ssh" + user = "ubuntu" private_key = "${file(var.private_key_path)}" } } user_data = "${data.template_file.provisioner.rendered}" - tags = "${module.airflow_labels_worker.tags}" + tags = "${module.airflow_labels_worker.tags}" lifecycle { create_before_destroy = true @@ -392,38 +392,39 @@ resource "aws_instance" "airflow_worker" { # ------------------------------------------------------------------------- module "sg_database" { - source = "terraform-aws-modules/security-group/aws" - name = "${module.airflow_labels.id}-database-sg" - description = "Security group for ${module.airflow_labels.id} database" - vpc_id = "${data.aws_vpc.default.id}" - ingress_cidr_blocks = "${var.ingress_cidr_blocks}" + source = "terraform-aws-modules/security-group/aws" + version = "3.2.0" + name = "${module.airflow_labels.id}-database-sg" + description = "Security group for ${module.airflow_labels.id} database" + vpc_id = "${data.aws_vpc.default.id}" + ingress_cidr_blocks = "${var.ingress_cidr_blocks}" number_of_computed_ingress_with_source_security_group_id = 1 computed_ingress_with_source_security_group_id = [ { - rule = "postgresql-tcp" + rule = "postgresql-tcp" source_security_group_id = "${module.sg_airflow.this_security_group_id}" - description = "Allow ${module.airflow_labels.id} machines" + description = "Allow ${module.airflow_labels.id} machines" } ] tags = "${module.airflow_labels.tags}" } resource "aws_db_instance" "airflow_database" { - identifier = "${module.airflow_labels.id}-db" - allocated_storage = "${var.db_allocated_storage}" - engine = "postgres" - engine_version = "11.1" - instance_class = "${var.db_instance_type}" - name = "${var.db_dbname}" - username = "${var.db_username}" - password = "${var.db_password}" - storage_type = "gp2" - backup_retention_period = 14 - multi_az = false - publicly_accessible = false - apply_immediately = true - skip_final_snapshot = true - vpc_security_group_ids = ["${module.sg_database.this_security_group_id}"] - port = "5432" - db_subnet_group_name = "${var.db_subnet_group_name}" + identifier = "${module.airflow_labels.id}-db" + allocated_storage = "${var.db_allocated_storage}" + engine = "postgres" + engine_version = "11.5" + instance_class = "${var.db_instance_type}" + name = "${var.db_dbname}" + username = "${var.db_username}" + password = "${var.db_password}" + storage_type = "gp2" + backup_retention_period = 7 + multi_az = false + publicly_accessible = false + apply_immediately = true + skip_final_snapshot = true + vpc_security_group_ids = ["${module.sg_database.this_security_group_id}"] + port = "5432" + db_subnet_group_name = "${var.db_subnet_group_name}" }