diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f876c570..a4e89459 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,6 @@ repos: - repo: https://github.com/tofuutils/pre-commit-opentofu rev: v1.0.4 hooks: + - id: tofu_validate - id: tofu_fmt - id: tofu_tflint - - id: tofu_validate diff --git a/osm/__init__.py b/osm/__init__.py index eee7f1d2..bb98193f 100644 --- a/osm/__init__.py +++ b/osm/__init__.py @@ -16,7 +16,7 @@ def get_version(): def generate_version_file(): import pkg_resources - if os.get("SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM"): + if os.getenv("SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM"): version = os.environ["SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM"] else: version = pkg_resources.get_distribution("osm").version diff --git a/osm/schemas/schema_helpers.py b/osm/schemas/schema_helpers.py index 5ea35f05..931099d5 100644 --- a/osm/schemas/schema_helpers.py +++ b/osm/schemas/schema_helpers.py @@ -176,10 +176,9 @@ def get_data_from_mongo(aggregation: list[dict] | None = None) -> Iterator[dict] aggregation = [ { "$match": { - "data_tags": "bulk_upload", - # "work.pmid": {"$regex":r"^2"}, - # "metrics.year": {"$gt": 2000}, - # "metrics.is_data_pred": {"$eq": True}, + "metrics_group": { + "$regex": "R" + } }, }, { diff --git a/web/api/Dockerfile b/web/api/Dockerfile index 99ba9fb3..fe703eb7 100644 --- a/web/api/Dockerfile +++ b/web/api/Dockerfile @@ -1,3 +1,3 @@ -FROM nimhdsst/osm_base +FROM osm_base COPY ./web/api/main.py /app/app/main.py CMD ["fastapi", "run", "--host", "0.0.0.0", "--port", "80", "--root-path", "/api"] diff --git a/web/dashboard/Dockerfile b/web/dashboard/Dockerfile index a163aa6a..ebfbf1bb 100644 --- a/web/dashboard/Dockerfile +++ b/web/dashboard/Dockerfile @@ -1,5 +1,4 @@ - -FROM nimhdsst/osm_base +FROM osm_base COPY web/dashboard/ /app ENV LOCAL_DATA_PATH=/opt/data/matches.parquet CMD ["python", "app.py"] diff --git a/web/dashboard/app.py b/web/dashboard/app.py index edbe50cc..0f6be1bb 100644 --- a/web/dashboard/app.py +++ b/web/dashboard/app.py @@ -134,6 +134,7 @@ def serve(self): allow_websocket_origin=[ "localhost:8501", "opensciencemetrics.org", + "dev.opensciencemetrics.org", ], static_dirs={ dir: str(Path(__file__).parent / dir) diff --git a/web/deploy/terraform/modules/ec2/data.tf b/web/deploy/terraform/modules/ec2/data.tf new file mode 100644 index 00000000..904a0a8f --- /dev/null +++ b/web/deploy/terraform/modules/ec2/data.tf @@ -0,0 +1,18 @@ +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-${var.ubuntu_ami_release}-amd64-server-*"] + } +} + +data "terraform_remote_state" "shared" { + backend = "s3" + config = { + bucket = "${var.state_bucket_name}-shared" + key = var.state_backend_key + dynamodb_table = "${var.state_table_name}-shared" + region = var.state_storage_region + } +} diff --git a/web/deploy/terraform/modules/ec2/main.tf b/web/deploy/terraform/modules/ec2/main.tf new file mode 100644 index 00000000..9905395e --- /dev/null +++ b/web/deploy/terraform/modules/ec2/main.tf @@ -0,0 +1,48 @@ +terraform { + required_version = ">= 1.8.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region +} + +# EC2 Instance +resource "aws_instance" "deployment" { + ami = data.aws_ami.ubuntu.id + instance_type = var.instance_type + subnet_id = data.terraform_remote_state.shared.outputs.subnet_id + key_name = var.ec2_key_name + vpc_security_group_ids = [data.terraform_remote_state.shared.outputs.security_group_id] + associate_public_ip_address = true + iam_instance_profile = data.terraform_remote_state.shared.outputs.instance_profile_name + root_block_device { + volume_size = var.ec2_root_block_device_size + volume_type = var.ec2_root_block_device_type + } + + tags = { + Name = var.environment + } + + user_data = file("${path.module}/scripts/install-docker.sh") +} + +resource "aws_eip" "deployment" { + domain = var.eip_domain + + tags = { + Name = var.environment + } +} + +resource "aws_eip_association" "deployment" { + instance_id = aws_instance.deployment.id + allocation_id = aws_eip.deployment.id +} diff --git a/web/deploy/terraform/modules/ec2/outputs.tf b/web/deploy/terraform/modules/ec2/outputs.tf new file mode 100644 index 00000000..0d21ae69 --- /dev/null +++ b/web/deploy/terraform/modules/ec2/outputs.tf @@ -0,0 +1,11 @@ +output "instance_id" { + value = aws_instance.deployment.id +} + +output "public_dns" { + value = aws_eip.deployment.public_dns +} + +output "public_ip" { + value = aws_eip.deployment.public_ip +} diff --git a/web/deploy/terraform/modules/ec2/scripts/install-docker.sh b/web/deploy/terraform/modules/ec2/scripts/install-docker.sh new file mode 100644 index 00000000..afd07760 --- /dev/null +++ b/web/deploy/terraform/modules/ec2/scripts/install-docker.sh @@ -0,0 +1,9 @@ +#!/bin/bash +apt-get update -y +apt install -y curl +apt-get install -y docker.io +curl -SL https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose +chmod a+x /usr/local/bin/docker-compose +systemctl restart sshd +systemctl start docker +systemctl enable docker diff --git a/web/deploy/terraform/modules/ec2/variables.tf b/web/deploy/terraform/modules/ec2/variables.tf new file mode 100644 index 00000000..90ad3a4b --- /dev/null +++ b/web/deploy/terraform/modules/ec2/variables.tf @@ -0,0 +1,46 @@ +variable "region" { + description = "The AWS region used by the deployment" + type = string + default = "us-east-1" +} + +variable "environment" { + description = "The name of the development environment. Usually `stage` or `prod`." + type = string +} + +variable "instance_type" { + description = "EC2 instance type" + default = "t3.large" + type = string +} + +variable "ec2_key_name" { + description = "Key name of the Key Pair to use for the instance; which can be managed using the `aws_key_pair` resource." + default = "dsst2023" + type = string +} + +variable "ec2_root_block_device_size" { + description = "Size of the volume in gibibytes (GiB)." + default = 30 + type = number +} + +variable "ec2_root_block_device_type" { + description = "Type of volume. Valid values include standard, gp2, gp3, io1, io2, sc1, or st1." + default = "gp2" + type = string +} + +variable "eip_domain" { + description = "Indicates if this EIP is for use in VPC" + default = "vpc" + type = string +} + +variable "ubuntu_ami_release" { + description = "The release of Ubuntu to use for the EC2 AMI. E.g. 20.04, 22.04, 24.04" + default = "20.04" + type = string +} diff --git a/web/deploy/terraform/modules/ec2/variables_state.tf b/web/deploy/terraform/modules/ec2/variables_state.tf new file mode 100644 index 00000000..55f44c87 --- /dev/null +++ b/web/deploy/terraform/modules/ec2/variables_state.tf @@ -0,0 +1,23 @@ +variable "state_bucket_name" { + description = "The name of the S3 bucket to store Terraform state. Must be globally unique." + type = string + default = "osm-terraform-state-storage" +} + +variable "state_table_name" { + description = "The name of the DynamoDB table. Must be unique in this AWS account." + type = string + default = "terraform-state-locks" +} + +variable "state_storage_region" { + description = "AWS region" + default = "us-east-1" + type = string +} + +variable "state_backend_key" { + description = "Path to the state file inside the S3 Bucket" + type = string + default = "terraform.tfstate" +} diff --git a/web/deploy/terraform/modules/ecr/main.tf b/web/deploy/terraform/modules/ecr/main.tf new file mode 100644 index 00000000..61f16323 --- /dev/null +++ b/web/deploy/terraform/modules/ecr/main.tf @@ -0,0 +1,27 @@ +terraform { + required_version = ">= 1.0.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region +} + +resource "aws_ecr_repository" "main" { + name = "${var.ecr_name}-${var.environment}" + image_tag_mutability = "IMMUTABLE" + + image_scanning_configuration { + scan_on_push = true + } + + encryption_configuration { + encryption_type = "AES256" + } +} diff --git a/web/deploy/terraform/modules/ecr/outputs.tf b/web/deploy/terraform/modules/ecr/outputs.tf new file mode 100644 index 00000000..df5705a5 --- /dev/null +++ b/web/deploy/terraform/modules/ecr/outputs.tf @@ -0,0 +1,3 @@ +output "arn" { + value = aws_ecr_repository.main.arn +} diff --git a/web/deploy/terraform/modules/ecr/variables.tf b/web/deploy/terraform/modules/ecr/variables.tf new file mode 100644 index 00000000..201c4474 --- /dev/null +++ b/web/deploy/terraform/modules/ecr/variables.tf @@ -0,0 +1,16 @@ +variable "environment" { + description = "The name of the environment. Usually `shared`, `stage`, or `prod`." + type = string +} + +variable "region" { + description = "AWS region" + default = "us-east-1" + type = string +} + +variable "ecr_name" { + description = "The name of the ECR repository" + default = "osm-ecr" + type = string +} diff --git a/web/deploy/terraform/modules/iam/main.tf b/web/deploy/terraform/modules/iam/main.tf new file mode 100644 index 00000000..928a0142 --- /dev/null +++ b/web/deploy/terraform/modules/iam/main.tf @@ -0,0 +1,62 @@ +terraform { + required_version = ">= 1.0.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region +} + +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "profile" { + name = "${var.instance_profile_role_name}-${var.environment}" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy_attachment" "profile" { + role = aws_iam_role.profile.name + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" +} + +resource "aws_iam_instance_profile" "profile" { + name = "${var.instance_profile_name}-${var.environment}" + role = aws_iam_role.profile.name +} + +resource "aws_iam_policy" "cd" { + name = "${var.cd_iam_policy_name}-${var.environment}" + policy = templatefile( + "${path.module}/policies/gha-policy.json.tftpl", + { + resources = jsonencode(var.cd_iam_policy_resources) + }, + ) +} + +resource "aws_iam_role" "cd" { + name = "${var.cd_iam_role_policy_name}-${var.environment}" + assume_role_policy = file("${path.module}/policies/assume-role.json") +} + +resource "aws_iam_role_policy_attachment" "cd" { + role = aws_iam_role.cd.name + policy_arn = aws_iam_policy.cd.arn +} diff --git a/web/deploy/terraform/modules/iam/outputs.tf b/web/deploy/terraform/modules/iam/outputs.tf new file mode 100644 index 00000000..ff22712d --- /dev/null +++ b/web/deploy/terraform/modules/iam/outputs.tf @@ -0,0 +1,3 @@ +output "instance_profile_name" { + value = aws_iam_instance_profile.profile.name +} diff --git a/web/deploy/terraform/modules/iam/policies/assume-role.json b/web/deploy/terraform/modules/iam/policies/assume-role.json new file mode 100644 index 00000000..e1f1d1bf --- /dev/null +++ b/web/deploy/terraform/modules/iam/policies/assume-role.json @@ -0,0 +1,22 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::123456123456:oidc-provider/token.actions.githubusercontent.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringLike": { + "token.actions.githubusercontent.com:sub": [ + "repo:nimh-dsst/osm:*" + ] + }, + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" + } + } + } + ] +} diff --git a/web/deploy/terraform/modules/iam/policies/gha-policy-nonadmin.json.tftpl b/web/deploy/terraform/modules/iam/policies/gha-policy-nonadmin.json.tftpl new file mode 100644 index 00000000..045d6912 --- /dev/null +++ b/web/deploy/terraform/modules/iam/policies/gha-policy-nonadmin.json.tftpl @@ -0,0 +1,39 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ecr:CompleteLayerUpload", + "ecr:UploadLayerPart", + "ecr:InitiateLayerUpload", + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:BatchGetImage", + "s3:GetBucketEncryption", + "s3:GetBucketTagging", + "s3:PutBucketTagging", + "s3:GetObject", + "s3:PutObject", + "s3:ListObjectsV2", + "s3:ListBuckets", + "dynamodb:CreateTable", + "dynamodb:DeleteTable", + "dynamodb:DescribeTable", + "dynamodb:ListTables", + "dynamodb:UpdateTable", + "dynamodb:PutItem", + "dynamodb:GetItem", + "dynamodb:DeleteItem", + "dynamodb:Query", + "dynamodb:Scan" + ], + "Resource": ${resources} + }, + { + "Effect": "Allow", + "Action": "ecr:GetAuthorizationToken", + "Resource": "*" + } + ] +} diff --git a/web/deploy/terraform/modules/iam/policies/gha-policy.json.tftpl b/web/deploy/terraform/modules/iam/policies/gha-policy.json.tftpl new file mode 100644 index 00000000..6756f3dd --- /dev/null +++ b/web/deploy/terraform/modules/iam/policies/gha-policy.json.tftpl @@ -0,0 +1,20 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:*", + "ecr:*", + "s3:*", + "dynamodb:*" + ], + "Resource": ${resources} + }, + { + "Effect": "Allow", + "Action": "ecr:GetAuthorizationToken", + "Resource": "*" + } + ] +} diff --git a/web/deploy/terraform/modules/iam/variables.tf b/web/deploy/terraform/modules/iam/variables.tf new file mode 100644 index 00000000..7cf2733e --- /dev/null +++ b/web/deploy/terraform/modules/iam/variables.tf @@ -0,0 +1,39 @@ +variable "environment" { + description = "The name of the environment. Usually `shared`, `stage`, or `prod`." + type = string +} + +variable "region" { + description = "AWS region" + default = "us-east-1" + type = string +} + +variable "instance_profile_name" { + description = "The name of the instance profile" + default = "osm-instance-profile" + type = string +} + +variable "instance_profile_role_name" { + description = "The name of the instance profile" + default = "osm-instance-profile-role" + type = string +} + +variable "cd_iam_policy_name" { + description = "The name of the IAM policy for continuous deployment to ECR" + default = "GitHubActions-ECR" + type = string +} + +variable "cd_iam_policy_resources" { + description = "The arn of the resource to which the IAM policy is applied" + type = list(string) +} + +variable "cd_iam_role_policy_name" { + description = "The name of the IAM role policy for continuous deployment to ECR" + default = "github-actions-role" + type = string +} diff --git a/web/deploy/terraform/modules/shared_resources/main.tf b/web/deploy/terraform/modules/networking/main.tf similarity index 60% rename from web/deploy/terraform/modules/shared_resources/main.tf rename to web/deploy/terraform/modules/networking/main.tf index 905c429b..3931dd83 100644 --- a/web/deploy/terraform/modules/shared_resources/main.tf +++ b/web/deploy/terraform/modules/networking/main.tf @@ -9,41 +9,17 @@ terraform { } } -# tflint-ignore: terraform_unused_declarations -variable "aws_region" { - description = "AWS region" - default = "us-east-1" - type = string -} - -# tflint-ignore: terraform_unused_declarations -variable "s3_bucket" { - description = "S3 bucket for Terraform state" - default = "osm-storage" - type = string -} - -# tflint-ignore: terraform_unused_declarations -variable "dynamodb_table" { - description = "DynamoDB table for Terraform state locking" - default = "terraform-locks" - type = string -} - -# tflint-ignore: terraform_unused_declarations -variable "ssh_port" { - description = "Non-standard port for SSH" - default = 22 - type = number +provider "aws" { + region = var.region } # VPC resource "aws_vpc" "main" { - cidr_block = "10.0.0.0/16" + cidr_block = var.vpc_ipv4_cidr_block enable_dns_hostnames = true enable_dns_support = true tags = { - Name = "osm-vpc" + Name = "${var.vpc_name}-${var.environment}" } } @@ -52,7 +28,7 @@ resource "aws_internet_gateway" "main" { vpc_id = aws_vpc.main.id tags = { - Name = "osm-internet-gateway" + Name = "${var.internet_gateway_name}-${var.environment}" } } @@ -61,15 +37,15 @@ resource "aws_route_table" "main" { vpc_id = aws_vpc.main.id route { - cidr_block = "0.0.0.0/0" + cidr_block = var.route_table_ipv4_cidr_block gateway_id = aws_internet_gateway.main.id } route { - ipv6_cidr_block = "::/0" + ipv6_cidr_block = var.route_table_ipv6_cidr_block gateway_id = aws_internet_gateway.main.id } tags = { - Name = "osm-route-table" + Name = "${var.route_table_name}-${var.environment}" } } @@ -103,6 +79,7 @@ resource "aws_network_acl_rule" "allow_all_outbound" { from_port = 0 to_port = 65535 } + resource "aws_security_group" "allow_all" { name = "allow_all_security_group" description = "Security group that allows all inbound and outbound traffic" @@ -141,11 +118,11 @@ resource "aws_security_group" "allow_all" { } resource "aws_vpc_dhcp_options" "main" { - domain_name = "compute-1.amazonaws.com" - domain_name_servers = ["AmazonProvidedDNS"] + domain_name = var.vpc_domain_name + domain_name_servers = var.vpc_domain_name_servers tags = { - Name = "osm-dhcp-options" + Name = "${var.vpc_dhcp_options_name}-${var.environment}" } } @@ -154,16 +131,15 @@ resource "aws_vpc_dhcp_options_association" "main" { dhcp_options_id = aws_vpc_dhcp_options.main.id } - # main Subnet resource "aws_subnet" "main" { vpc_id = aws_vpc.main.id - cidr_block = "10.0.1.0/24" - availability_zone = "us-east-1a" + cidr_block = var.subnet_ipv4_cidr_block + availability_zone = "${var.region}${var.availability_zone_letter_identifier}" map_public_ip_on_launch = true tags = { - Name = "main-subnet" + Name = "${var.subnet_name}-${var.environment}" } } @@ -178,43 +154,3 @@ resource "aws_network_acl_association" "main" { subnet_id = aws_subnet.main.id network_acl_id = aws_network_acl.allow_all.id } - - -# Security Group - - -# Data source to find the latest Ubuntu AMI -data "aws_ami" "ubuntu" { - most_recent = true - owners = ["099720109477"] - filter { - name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] - } -} - - - - -# Outputs -output "vpc_id" { - value = aws_vpc.main.id -} -output "subnet_id" { - value = aws_subnet.main.id -} -output "security_group_id" { - value = aws_security_group.allow_all.id -} -output "internet_gateway_id" { - value = aws_internet_gateway.main.id -} -output "route_table_id" { - value = aws_route_table.main.id -} -output "aws_network_acl_id" { - value = aws_network_acl.allow_all.id -} -output "ami_id" { - value = data.aws_ami.ubuntu.id -} diff --git a/web/deploy/terraform/modules/networking/outputs.tf b/web/deploy/terraform/modules/networking/outputs.tf new file mode 100644 index 00000000..0628f109 --- /dev/null +++ b/web/deploy/terraform/modules/networking/outputs.tf @@ -0,0 +1,23 @@ +output "vpc_id" { + value = aws_vpc.main.id +} + +output "subnet_id" { + value = aws_subnet.main.id +} + +output "security_group_id" { + value = aws_security_group.allow_all.id +} + +output "internet_gateway_id" { + value = aws_internet_gateway.main.id +} + +output "route_table_id" { + value = aws_route_table.main.id +} + +output "aws_network_acl_id" { + value = aws_network_acl.allow_all.id +} diff --git a/web/deploy/terraform/modules/networking/variables.tf b/web/deploy/terraform/modules/networking/variables.tf new file mode 100644 index 00000000..5b6ec11c --- /dev/null +++ b/web/deploy/terraform/modules/networking/variables.tf @@ -0,0 +1,89 @@ +variable "region" { + description = "AWS region" + default = "us-east-1" + type = string +} + +variable "availability_zone_letter_identifier" { + description = "The letter identifier for the AWS Availablity Zone. Usually `a` or `b`." + default = "a" + type = string +} + +# tflint-ignore: terraform_unused_declarations +variable "ssh_port" { + description = "Non-standard port for SSH" + default = 22 + type = number +} + +variable "environment" { + description = "The name of the environment. Usually `shared`, `stage`, or `prod`." + type = string +} + +variable "vpc_name" { + description = "The name used to tag the VPC." + default = "osm-vpc" + type = string +} + +variable "vpc_ipv4_cidr_block" { + description = "The IPv4 CIDR block for the VPC. CIDR can be explicitly set or it can be derived from IPAM using `ipv4_netmask_length`" + default = "10.0.0.0/16" + type = string +} + +variable "internet_gateway_name" { + description = "The name of the internet gateway" + default = "osm-internet-gateway" + type = string +} + +variable "route_table_name" { + description = "The name used to tag the route table." + default = "osm-route-table" + type = string +} + +variable "route_table_ipv4_cidr_block" { + description = "The IPv4 CIDR block of the route table" + default = "0.0.0.0/0" + type = string +} + +variable "route_table_ipv6_cidr_block" { + description = "The IPv6 CIDR block of the route table" + default = "::/0" + type = string +} + +variable "vpc_domain_name" { + description = "The suffix domain name to use by default when resolving non Fully Qualified Domain Names. In other words, this is what ends up being the `search` value in the `/etc/resolv.conf` file." + default = "compute-1.amazonaws.com" + type = string +} + +variable "vpc_domain_name_servers" { + description = "List of name servers to configure in `/etc/resolv.conf`. If you want to use the default AWS nameservers you should set this to `AmazonProvidedDNS`." + default = ["AmazonProvidedDNS"] + type = list(string) +} + +variable "vpc_dhcp_options_name" { + description = "The name used to tag the VPC DHCP options" + default = "osm-dhcp-options" + type = string +} + +variable "subnet_name" { + description = "The name used to tag the AWS subnet" + default = "main-subnet" + type = string +} + +variable "subnet_ipv4_cidr_block" { + description = "The IPv4 CIDR block for the subnet." + default = "10.0.1.0/24" + type = string +} diff --git a/web/deploy/terraform/modules/state/main.tf b/web/deploy/terraform/modules/state/main.tf new file mode 100644 index 00000000..e3f304b5 --- /dev/null +++ b/web/deploy/terraform/modules/state/main.tf @@ -0,0 +1,72 @@ +terraform { + required_version = ">= 1.0.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.aws_region +} + +resource "aws_s3_bucket" "tf_state" { + bucket = "${var.bucket_name}-${var.environment}" + + tags = { + Name = "${var.bucket_name}-${var.environment}" + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "tf_state" { + bucket = aws_s3_bucket.tf_state.id + rule { + id = "tf_state_${var.environment}" + status = "Enabled" + + transition { + days = 30 + storage_class = "STANDARD_IA" + } + + expiration { + days = 365 + } + } +} + +resource "aws_s3_bucket_versioning" "enabled" { + bucket = aws_s3_bucket.tf_state.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "default" { + bucket = aws_s3_bucket.tf_state.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_dynamodb_table" "tf_locks" { + name = "${var.table_name}-${var.environment}" + billing_mode = "PAY_PER_REQUEST" + hash_key = "LockID" + + attribute { + name = "LockID" + type = "S" + } + + tags = { + Name = "${var.bucket_name}-${var.environment}" + } +} diff --git a/web/deploy/terraform/modules/state/outputs.tf b/web/deploy/terraform/modules/state/outputs.tf new file mode 100644 index 00000000..660848bd --- /dev/null +++ b/web/deploy/terraform/modules/state/outputs.tf @@ -0,0 +1,9 @@ +output "s3_bucket_arn" { + value = aws_s3_bucket.tf_state.arn + description = "The ARN of the S3 bucket" +} + +output "dynamodb_table_name" { + value = aws_dynamodb_table.tf_locks.name + description = "The name of the DynamoDB table" +} diff --git a/web/deploy/terraform/modules/state/variables.tf b/web/deploy/terraform/modules/state/variables.tf new file mode 100644 index 00000000..3fef1596 --- /dev/null +++ b/web/deploy/terraform/modules/state/variables.tf @@ -0,0 +1,22 @@ +variable "bucket_name" { + description = "The name of the S3 bucket to store Terraform state. Must be globally unique." + type = string + default = "osm-terraform-state-storage" +} + +variable "table_name" { + description = "The name of the DynamoDB table. Must be unique in this AWS account." + type = string + default = "terraform-state-locks" +} + +variable "aws_region" { + description = "The AWS region used by the deployment" + type = string + default = "us-east-1" +} + +variable "environment" { + description = "The name of the development environment. Usually `stage` or `prod`." + type = string +} diff --git a/web/deploy/terraform/production/main.tf b/web/deploy/terraform/production/main.tf new file mode 100644 index 00000000..afe22b53 --- /dev/null +++ b/web/deploy/terraform/production/main.tf @@ -0,0 +1,24 @@ +terraform { + required_version = ">= 1.8.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + + backend "s3" { + bucket = "${var.state_bucket_name}-${var.environment}" + key = var.state_backend_key + region = var.state_storage_region + dynamodb_table = "${var.state_table_name}-${var.environment}" + encrypt = true + } +} + +module "ec2" { + source = "../modules/ec2/" + environment = var.environment + instance_type = var.instance_type +} diff --git a/web/deploy/terraform/production/outputs.tf b/web/deploy/terraform/production/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/web/deploy/terraform/production/variables.tf b/web/deploy/terraform/production/variables.tf new file mode 100644 index 00000000..f675f09e --- /dev/null +++ b/web/deploy/terraform/production/variables.tf @@ -0,0 +1,11 @@ +variable "environment" { + description = "The name of the environment. Usually `prod`" + default = "prod" + type = string +} + +variable "instance_type" { + description = "EC2 instance type" + default = "t3.large" + type = string +} diff --git a/web/deploy/terraform/production/variables_state.tf b/web/deploy/terraform/production/variables_state.tf new file mode 100644 index 00000000..7993c2ab --- /dev/null +++ b/web/deploy/terraform/production/variables_state.tf @@ -0,0 +1,23 @@ +variable "state_storage_region" { + description = "AWS region" + default = "us-east-1" + type = string +} + +variable "state_bucket_name" { + description = "The name of the S3 bucket to store Terraform state." + type = string + default = "osm-terraform-state-storage" +} + +variable "state_table_name" { + description = "The name of the DynamoDB table for Terraform state locks." + type = string + default = "terraform-state-locks" +} + +variable "state_backend_key" { + description = "Path to the state file inside the S3 Bucket" + type = string + default = "terraform.tfstate" +} diff --git a/web/deploy/terraform/shared/main.tf b/web/deploy/terraform/shared/main.tf new file mode 100644 index 00000000..5595b025 --- /dev/null +++ b/web/deploy/terraform/shared/main.tf @@ -0,0 +1,41 @@ +terraform { + required_version = ">= 1.8.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + + backend "s3" { + bucket = "${var.state_bucket_name}-${var.environment}" + key = var.state_backend_key + region = var.state_storage_region + dynamodb_table = "${var.state_table_name}-${var.environment}" + encrypt = true + } +} + +module "networking" { + source = "../modules/networking/" + environment = var.environment +} + +module "ecr_api" { + source = "../modules/ecr/" + environment = var.environment + ecr_name = "api" +} + +module "ecr_dashboard" { + source = "../modules/ecr/" + environment = var.environment + ecr_name = "dashboard" +} + +module "iam_role_and_policy" { + source = "../modules/iam/" + environment = var.environment + cd_iam_policy_resources = [module.ecr_api.arn, module.ecr_dashboard.arn] +} diff --git a/web/deploy/terraform/shared/outputs.tf b/web/deploy/terraform/shared/outputs.tf new file mode 100644 index 00000000..81832c89 --- /dev/null +++ b/web/deploy/terraform/shared/outputs.tf @@ -0,0 +1,11 @@ +output "subnet_id" { + value = module.networking.subnet_id +} + +output "security_group_id" { + value = module.networking.security_group_id +} + +output "instance_profile_name" { + value = module.iam_role_and_policy.instance_profile_name +} diff --git a/web/deploy/terraform/shared/variables.tf b/web/deploy/terraform/shared/variables.tf new file mode 100644 index 00000000..62bad9f4 --- /dev/null +++ b/web/deploy/terraform/shared/variables.tf @@ -0,0 +1,5 @@ +variable "environment" { + description = "The name of the environment. Usually `shared`, `stage`, or `prod`." + default = "shared" + type = string +} diff --git a/web/deploy/terraform/shared/variables_state.tf b/web/deploy/terraform/shared/variables_state.tf new file mode 100644 index 00000000..7993c2ab --- /dev/null +++ b/web/deploy/terraform/shared/variables_state.tf @@ -0,0 +1,23 @@ +variable "state_storage_region" { + description = "AWS region" + default = "us-east-1" + type = string +} + +variable "state_bucket_name" { + description = "The name of the S3 bucket to store Terraform state." + type = string + default = "osm-terraform-state-storage" +} + +variable "state_table_name" { + description = "The name of the DynamoDB table for Terraform state locks." + type = string + default = "terraform-state-locks" +} + +variable "state_backend_key" { + description = "Path to the state file inside the S3 Bucket" + type = string + default = "terraform.tfstate" +} diff --git a/web/deploy/terraform/staging/main.tf b/web/deploy/terraform/staging/main.tf index b2eae00c..afe22b53 100644 --- a/web/deploy/terraform/staging/main.tf +++ b/web/deploy/terraform/staging/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.0.0, < 2.0.0" + required_version = ">= 1.8.0, < 2.0.0" required_providers { aws = { @@ -7,96 +7,18 @@ terraform { version = "~> 5.0" } } -} - -provider "aws" { - region = "us-east-1" -} -terraform { backend "s3" { - bucket = "osm-terraform-storage" - key = "terraform/staging-state/terraform.tfstate" - region = "us-east-1" - dynamodb_table = "terraform-locks" - } -} - - -module "shared_resources" { - source = "../modules/shared_resources" -} - -# EC2 Instance -resource "aws_instance" "staging" { - ami = module.shared_resources.ami_id - instance_type = var.instance_type - subnet_id = module.shared_resources.subnet_id - key_name = "dsst2023" - vpc_security_group_ids = [module.shared_resources.security_group_id] - associate_public_ip_address = true - root_block_device { - volume_size = 30 - volume_type = "gp2" # General Purpose SSD (can be "gp2", "gp3", "io1", "io2", etc.) - } - - tags = { - Name = "staging-instance" - } - - user_data = <<-EOF - #!/bin/bash - apt-get update -y - apt install -y curl - apt-get install -y docker.io - curl -SL https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose - chmod a+x /usr/local/bin/docker-compose - systemctl restart sshd - systemctl start docker - systemctl enable docker - EOF -} - -resource "aws_eip" "staging" { - domain = "vpc" - - tags = { - Name = "staging-elastic-ip" + bucket = "${var.state_bucket_name}-${var.environment}" + key = var.state_backend_key + region = var.state_storage_region + dynamodb_table = "${var.state_table_name}-${var.environment}" + encrypt = true } } -resource "aws_eip_association" "staging" { - instance_id = aws_instance.staging.id - allocation_id = aws_eip.staging.id -} - -output "vpc_id" { - value = module.shared_resources.vpc_id -} -output "internet_gateway_id" { - value = module.shared_resources.internet_gateway_id -} -output "route_table_id" { - value = module.shared_resources.route_table_id -} -output "network_acl_id" { - value = module.shared_resources.aws_network_acl_id -} -output "security_group_id" { - value = module.shared_resources.security_group_id -} -output "subnet_id" { - value = module.shared_resources.subnet_id -} - -output "instance_id" { - value = aws_instance.staging.id -} - -output "public_dns" { - value = aws_eip.staging.public_dns -} - -output "public_ip" { - value = aws_eip.staging.public_ip +module "ec2" { + source = "../modules/ec2/" + environment = var.environment + instance_type = var.instance_type } diff --git a/web/deploy/terraform/staging/outputs.tf b/web/deploy/terraform/staging/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/web/deploy/terraform/staging/variables.tf b/web/deploy/terraform/staging/variables.tf index a6692d82..a97198da 100644 --- a/web/deploy/terraform/staging/variables.tf +++ b/web/deploy/terraform/staging/variables.tf @@ -1,3 +1,9 @@ +variable "environment" { + description = "The name of the environment. Usually `stage`." + default = "stage" + type = string +} + variable "instance_type" { description = "EC2 instance type" default = "t3.large" diff --git a/web/deploy/terraform/staging/variables_state.tf b/web/deploy/terraform/staging/variables_state.tf new file mode 100644 index 00000000..7993c2ab --- /dev/null +++ b/web/deploy/terraform/staging/variables_state.tf @@ -0,0 +1,23 @@ +variable "state_storage_region" { + description = "AWS region" + default = "us-east-1" + type = string +} + +variable "state_bucket_name" { + description = "The name of the S3 bucket to store Terraform state." + type = string + default = "osm-terraform-state-storage" +} + +variable "state_table_name" { + description = "The name of the DynamoDB table for Terraform state locks." + type = string + default = "terraform-state-locks" +} + +variable "state_backend_key" { + description = "Path to the state file inside the S3 Bucket" + type = string + default = "terraform.tfstate" +} diff --git a/web/deploy/terraform/state/deprecated/README.md b/web/deploy/terraform/state/deprecated/README.md new file mode 100644 index 00000000..90299444 --- /dev/null +++ b/web/deploy/terraform/state/deprecated/README.md @@ -0,0 +1,3 @@ +The files in this directory are deprecated and only included for reference. + +This directory might be removed in the future diff --git a/web/deploy/terraform/state_storage/dynamodb-policy.json b/web/deploy/terraform/state/deprecated/dynamodb-policy.json similarity index 100% rename from web/deploy/terraform/state_storage/dynamodb-policy.json rename to web/deploy/terraform/state/deprecated/dynamodb-policy.json diff --git a/web/deploy/terraform/state_storage/README.md b/web/deploy/terraform/state/deprecated/manual-bucket-creation.md similarity index 100% rename from web/deploy/terraform/state_storage/README.md rename to web/deploy/terraform/state/deprecated/manual-bucket-creation.md diff --git a/web/deploy/terraform/state/main.tf b/web/deploy/terraform/state/main.tf new file mode 100644 index 00000000..183795c1 --- /dev/null +++ b/web/deploy/terraform/state/main.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.0.0, < 2.0.0" +} + +module "stage_state" { + source = "../modules/state/" + environment = "stage" +} + +module "prod_state" { + source = "../modules/state/" + environment = "prod" +} + +module "shared_state" { + source = "../modules/state/" + environment = "shared" +} diff --git a/web/deploy/terraform/state_storage/state-storage.tf b/web/deploy/terraform/state_storage/state-storage.tf deleted file mode 100644 index 9970bcaa..00000000 --- a/web/deploy/terraform/state_storage/state-storage.tf +++ /dev/null @@ -1,57 +0,0 @@ -terraform { - required_version = ">= 1.0.0, < 2.0.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -provider "aws" { - region = "us-east-1" -} - -resource "aws_s3_bucket" "tf_state" { - bucket = "osm-storage" - versioning { - enabled = true - } - server_side_encryption_configuration { - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } - } - lifecycle_rule { - id = "tf_state" - enabled = true - transition { - days = 30 - storage_class = "STANDARD_IA" - } - expiration { - days = 365 - } - } - tags = { - Name = "terraform-state-storage" - } -} - -resource "aws_dynamodb_table" "tf_locks" { - name = "terraform-locks" - billing_mode = "PAY_PER_REQUEST" - hash_key = "LockID" - - attribute { - name = "LockID" - type = "S" - } - - tags = { - Name = "terraform-state-locks" - } -}