diff --git a/docs/add-ons/gitlab-runner.md b/docs/add-ons/gitlab-runner.md new file mode 100644 index 0000000000..556612227e --- /dev/null +++ b/docs/add-ons/gitlab-runner.md @@ -0,0 +1,55 @@ +# gitlab-runner + +The GitLab runner Helm Chart is the official way of deploying a GitLab runner on a Kubernetes cluster. The chart configures a GitLab runner which runs using the Kubernetes executor. For each new job, it provisions a pod within the specified namespace. + +For complete project documentation, please see [GitLab's documentation](https://docs.gitlab.com/runner/install/kubernetes.html). + +## Usage + +The GitLab Runner can be deployed by enabling the add-on via the following: + +```hcl +enable_gitlab_runner = true +``` + +At minimum a runner token must be supplied. This can be done directly, as a deploy-time value: + +```hcl + gitlab_runner_helm_config = { + set_sensitive = [ + { name = "runnerRegistrationToken", value = "someTokenValue" }, + ] + } +``` + +or using an existing secret, such as: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: gitlab-runner-secret +type: Opaque +data: + runner-registration-token: "NlZrN1pzb3NxUXlmcmVBeFhUWnIK" #base64 encoded registration token + runner-token: "" +``` +and passing it to the chart: + +```hcl + gitlab_runner_helm_config = { + set = [ + { name = "runners.secret", value = "gitlab-runner-secret" } + ] + } +``` + +By default the runner is registered on the cloud instance `gitlab.com`. An self-hosted instance can also be configured: + +```hcl + gitlab_runner_helm_config = { + set = [ + { name = "gitlabUrl", value = "https://mygitlab.com" } + ] + } +``` diff --git a/examples/ci-cd/gitlab-runner-with-managed-node-group/.README.md.swp b/examples/ci-cd/gitlab-runner-with-managed-node-group/.README.md.swp new file mode 100644 index 0000000000..7825343c9d Binary files /dev/null and b/examples/ci-cd/gitlab-runner-with-managed-node-group/.README.md.swp differ diff --git a/examples/ci-cd/gitlab-runner-with-managed-node-group/README.md b/examples/ci-cd/gitlab-runner-with-managed-node-group/README.md new file mode 100644 index 0000000000..9ab1820415 --- /dev/null +++ b/examples/ci-cd/gitlab-runner-with-managed-node-group/README.md @@ -0,0 +1,101 @@ +# GitLab Runner with Managed Node-groups + +This example deploys a new EKS cluster, with the GitLab Runner installed. The example also deploys a GitLab project under the user's namespace. This project is a GitLab provided sample: https://gitlab.com/gitlab-org/ci-sample-projects/cicd-templates/android.latest.gitlab-ci.yml-test-project. The runner spawns pods on the managed node-group when a pipeline is triggered. + +## How to Deploy + +### Prerequisites: + +Ensure that you have installed the following tools in your Mac or Windows Laptop before start working with this module and run Terraform Plan and Apply + +1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +2. [Kubectl](https://Kubernetes.io/docs/tasks/tools/) +3. [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) +4. [GitLab account](https://gitlab.com) + +### Deployment Steps + +#### Step 1: Clone the repo using the command below + +```sh +git clone https://github.com/aws-ia/terraform-aws-eks-blueprints.git +``` + +#### Step 2: Create a GitLab personal access token + +From your GitHub profile page, select "Access Tokens". Create a token with "api" privileges. Export that token as the environment variable `GITLAB_TOKEN`: + +```sh +export GITLAB_TOKEN= +``` + +#### Step 3: Run Terraform INIT + +Initialize a working directory with configuration files + +```sh +cd examples/ci-cd/gitlab-runner-with-managed-node-groups +terraform init +``` + +#### Step 4: Run Terraform PLAN + +Verify the resources created by this execution + +```sh +export AWS_REGION= # Select your own region +terraform plan +``` + +#### Step 4: Finally, Terraform APPLY + +**Deploy the pattern** + +```sh +terraform apply +``` + +Enter `yes` to apply. + +### Configure `kubectl` and test cluster + +EKS Cluster details can be extracted from terraform output or from AWS Console to get the name of cluster. +This following command used to update the `kubeconfig` in your local machine where you run kubectl commands to interact with your EKS Cluster. + +#### Step 5: Run `update-kubeconfig` command + +`~/.kube/config` file gets updated with cluster details and certificate from the below command + + $ aws eks --region update-kubeconfig --name + +#### Step 6: List all the worker nodes by running the command below + + $ kubectl get nodes + + +#### Step 7: Trigger a pipeline build + +In the GitLab UI find the project created under your profile. Trigger a pipeline with the "Run pipeline" button on the project's "CI/CD > Pipelines" page. + + +#### Step 7: List all the pods running in `gitlab-runner` namespace + + $ kubectl get pods -n gitlab-runner + +## Cleanup + +To clean up your environment, destroy the Terraform modules in reverse order. + +Destroy the Kubernetes Add-ons, EKS cluster with Node groups and VPC + +```sh +terraform destroy -target="module.eks_blueprints_kubernetes_addons" -auto-approve +terraform destroy -target="module.eks_blueprints" -auto-approve +terraform destroy -target="module.vpc" -auto-approve +``` + +Finally, destroy any additional resources that are not in the above modules + +```sh +terraform destroy -auto-approve +``` diff --git a/examples/ci-cd/gitlab-runner-with-managed-node-group/main.tf b/examples/ci-cd/gitlab-runner-with-managed-node-group/main.tf new file mode 100644 index 0000000000..70c9c2b917 --- /dev/null +++ b/examples/ci-cd/gitlab-runner-with-managed-node-group/main.tf @@ -0,0 +1,172 @@ +provider "aws" { + region = local.region +} + +provider "kubernetes" { + host = module.eks_blueprints.eks_cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token +} + +provider "helm" { + kubernetes { + host = module.eks_blueprints.eks_cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token + } +} + +data "aws_eks_cluster_auth" "this" { + name = module.eks_blueprints.eks_cluster_id +} + +data "aws_availability_zones" "available" {} + +locals { + name = basename(path.cwd) + region = "us-west-2" + + vpc_cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } +} + +module "eks_blueprints" { + source = "../../.." + + cluster_name = local.name + cluster_version = "1.24" + + vpc_id = module.vpc.vpc_id + private_subnet_ids = module.vpc.private_subnets + + #----------------------------------------------------------------------------------------------------------# + # Security groups used in this module created by the upstream modules terraform-aws-eks (https://github.com/terraform-aws-modules/terraform-aws-eks). + # Upstream module implemented Security groups based on the best practices doc https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html. + # So, by default the security groups are restrictive. Users needs to enable rules for specific ports required for App requirement or Add-ons + # See the notes below for each rule used in these examples + #----------------------------------------------------------------------------------------------------------# + node_security_group_additional_rules = { + # Extend node-to-node security group rules. Recommended and required for the Add-ons + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + # Recommended outbound traffic for Node groups + egress_all = { + description = "Node all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + # Allows Control Plane Nodes to talk to Worker nodes on all ports. Added this to simplify the example and further avoid issues with Add-ons communication with Control plane. + # This can be restricted further to specific port based on the requirement for each Add-on e.g., metrics-server 4443, spark-operator 8080, karpenter 8443 etc. + # Change this according to your security requirements if needed + ingress_cluster_to_node_all_traffic = { + description = "Cluster API to Nodegroup all traffic" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + source_cluster_security_group = true + } + } + + managed_node_groups = { + mg_5 = { + node_group_name = "managed-ondemand" + instance_types = ["m5.large"] + subnet_ids = module.vpc.private_subnets + force_update_version = true + } + } + + tags = local.tags +} + +data "gitlab_current_user" "this" {} + +resource "gitlab_project" "android_test_project" { + name = "${local.name}-android-test" + description = "${local.name} Android Test Project" + + namespace_id = data.gitlab_current_user.this.namespace_id + + # A sample GitLab project + import_url = "https://gitlab.com/gitlab-org/ci-sample-projects/cicd-templates/android.latest.gitlab-ci.yml-test-project.git" + + visibility_level = "public" + + # Only use EKS runners + shared_runners_enabled = false +} + + +module "eks_blueprints_kubernetes_addons" { + source = "../../../modules/kubernetes-addons" + + eks_cluster_id = module.eks_blueprints.eks_cluster_id + eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint + eks_oidc_provider = module.eks_blueprints.oidc_provider + eks_cluster_version = module.eks_blueprints.eks_cluster_version + eks_worker_security_group_id = module.eks_blueprints.worker_node_security_group_id + auto_scaling_group_names = module.eks_blueprints.self_managed_node_group_autoscaling_groups + + # GitLab Runner + enable_gitlab_runner = true + gitlab_runner_helm_config = { + set_sensitive = [ + { name = "runnerRegistrationToken", value = gitlab_project.android_test_project.runners_token }, + ] + } + + tags = local.tags + +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 10)] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + # Manage so we can name + manage_default_network_acl = true + default_network_acl_tags = { Name = "${local.name}-default" } + manage_default_route_table = true + default_route_table_tags = { Name = "${local.name}-default" } + manage_default_security_group = true + default_security_group_tags = { Name = "${local.name}-default" } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/examples/ci-cd/gitlab-runner-with-managed-node-group/outputs.tf b/examples/ci-cd/gitlab-runner-with-managed-node-group/outputs.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/ci-cd/gitlab-runner-with-managed-node-group/tfplan b/examples/ci-cd/gitlab-runner-with-managed-node-group/tfplan new file mode 100644 index 0000000000..cd177cefc1 Binary files /dev/null and b/examples/ci-cd/gitlab-runner-with-managed-node-group/tfplan differ diff --git a/examples/ci-cd/gitlab-runner-with-managed-node-group/variables.tf b/examples/ci-cd/gitlab-runner-with-managed-node-group/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/ci-cd/gitlab-runner-with-managed-node-group/versions.tf b/examples/ci-cd/gitlab-runner-with-managed-node-group/versions.tf new file mode 100644 index 0000000000..d0468cecad --- /dev/null +++ b/examples/ci-cd/gitlab-runner-with-managed-node-group/versions.tf @@ -0,0 +1,29 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.9" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.4.1" + } + gitlab = { + source = "gitlabhq/gitlab" + version = ">= 15.7.1" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/complete-kubernetes-addons/terraform.tfstate" + # } +} diff --git a/modules/kubernetes-addons/README.md b/modules/kubernetes-addons/README.md index 58cdcff208..853c9a52fa 100644 --- a/modules/kubernetes-addons/README.md +++ b/modules/kubernetes-addons/README.md @@ -59,6 +59,7 @@ | [external\_secrets](#module\_external\_secrets) | ./external-secrets | n/a | | [fargate\_fluentbit](#module\_fargate\_fluentbit) | ./fargate-fluentbit | n/a | | [gatekeeper](#module\_gatekeeper) | ./gatekeeper | n/a | +| [gitlab\_runner](#module\_gitlab\_runner) | ./gitlab-runner | n/a | | [grafana](#module\_grafana) | ./grafana | n/a | | [ingress\_nginx](#module\_ingress\_nginx) | ./ingress-nginx | n/a | | [karpenter](#module\_karpenter) | ./karpenter | n/a | @@ -220,6 +221,7 @@ | [enable\_external\_secrets](#input\_enable\_external\_secrets) | Enable External Secrets operator add-on | `bool` | `false` | no | | [enable\_fargate\_fluentbit](#input\_enable\_fargate\_fluentbit) | Enable Fargate FluentBit add-on | `bool` | `false` | no | | [enable\_gatekeeper](#input\_enable\_gatekeeper) | Enable Gatekeeper add-on | `bool` | `false` | no | +| [enable\_gitlab\_runner](#input\_enable\_gitlab\_runner) | Enable GitLab Runner add-on | `bool` | `false` | no | | [enable\_grafana](#input\_enable\_grafana) | Enable Grafana add-on | `bool` | `false` | no | | [enable\_ingress\_nginx](#input\_enable\_ingress\_nginx) | Enable Ingress Nginx add-on | `bool` | `false` | no | | [enable\_ipv6](#input\_enable\_ipv6) | Enable Ipv6 network. Attaches new VPC CNI policy to the IRSA role | `bool` | `false` | no | @@ -268,6 +270,7 @@ | [external\_secrets\_ssm\_parameter\_arns](#input\_external\_secrets\_ssm\_parameter\_arns) | List of Systems Manager Parameter ARNs that contain secrets to mount using External Secrets | `list(string)` |
[
"arn:aws:ssm:*:*:parameter/*"
]
| no | | [fargate\_fluentbit\_addon\_config](#input\_fargate\_fluentbit\_addon\_config) | Fargate fluentbit add-on config | `any` | `{}` | no | | [gatekeeper\_helm\_config](#input\_gatekeeper\_helm\_config) | Gatekeeper Helm Chart config | `any` | `{}` | no | +| [gitlab\_runner\_helm\_config](#input\_gitlab\_runner\_helm\_config) | GitLab Runner Helm Chart config | `any` | `{}` | no | | [grafana\_helm\_config](#input\_grafana\_helm\_config) | Kubernetes Grafana Helm Chart config | `any` | `null` | no | | [grafana\_irsa\_policies](#input\_grafana\_irsa\_policies) | IAM policy ARNs for grafana IRSA | `list(string)` | `[]` | no | | [ingress\_nginx\_helm\_config](#input\_ingress\_nginx\_helm\_config) | Ingress Nginx Helm Chart config | `any` | `{}` | no | diff --git a/modules/kubernetes-addons/gitlab-runner/README.md b/modules/kubernetes-addons/gitlab-runner/README.md new file mode 100644 index 0000000000..9ad0f1b4b3 --- /dev/null +++ b/modules/kubernetes-addons/gitlab-runner/README.md @@ -0,0 +1,44 @@ +# GitLab Runner + +This chart deploys a GitLab Runner instance into your Kubernetes cluster + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [helm\_addon](#module\_helm\_addon) | ../helm-addon | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [addon\_context](#input\_addon\_context) | Input configuration for the addon |
object({
aws_caller_identity_account_id = string
aws_caller_identity_arn = string
aws_eks_cluster_endpoint = string
aws_partition_id = string
aws_region_name = string
eks_cluster_id = string
eks_oidc_issuer_url = string
eks_oidc_provider_arn = string
tags = map(string)
})
| n/a | yes | +| [helm\_config](#input\_helm\_config) | Helm Config for calico | `any` | `{}` | no | +| [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps. | `bool` | `false` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [argocd\_gitops\_config](#output\_argocd\_gitops\_config) | Configuration used for managing the add-on with ArgoCD | +| [irsa\_arn](#output\_irsa\_arn) | IAM role ARN for the service account | +| [irsa\_name](#output\_irsa\_name) | IAM role name for the service account | +| [release\_metadata](#output\_release\_metadata) | Map of attributes of the Helm release metadata | +| [service\_account](#output\_service\_account) | Name of Kubernetes service account | + diff --git a/modules/kubernetes-addons/gitlab-runner/main.tf b/modules/kubernetes-addons/gitlab-runner/main.tf new file mode 100644 index 0000000000..2d4e02d5cf --- /dev/null +++ b/modules/kubernetes-addons/gitlab-runner/main.tf @@ -0,0 +1,25 @@ +locals { + name = "gitlab-runner" +} + +module "helm_addon" { + source = "../helm-addon" + + # https://gitlab.com/gitlab-org/charts/gitlab-runner/-/blob/main/Chart.yaml + helm_config = merge( + { + name = local.name + chart = local.name + repository = "https://charts.gitlab.io" + version = "0.48.0" + namespace = local.name + create_namespace = true + description = "GitLab runner" + values = [file("${path.module}/values.yaml")] + }, + var.helm_config + ) + manage_via_gitops = var.manage_via_gitops + + addon_context = var.addon_context +} diff --git a/modules/kubernetes-addons/gitlab-runner/outputs.tf b/modules/kubernetes-addons/gitlab-runner/outputs.tf new file mode 100644 index 0000000000..4d3d2c6515 --- /dev/null +++ b/modules/kubernetes-addons/gitlab-runner/outputs.tf @@ -0,0 +1,24 @@ +output "argocd_gitops_config" { + description = "Configuration used for managing the add-on with ArgoCD" + value = var.manage_via_gitops ? { enable = true } : null +} + +output "release_metadata" { + description = "Map of attributes of the Helm release metadata" + value = module.helm_addon.release_metadata +} + +output "irsa_arn" { + description = "IAM role ARN for the service account" + value = module.helm_addon.irsa_arn +} + +output "irsa_name" { + description = "IAM role name for the service account" + value = module.helm_addon.irsa_name +} + +output "service_account" { + description = "Name of Kubernetes service account" + value = module.helm_addon.service_account +} diff --git a/modules/kubernetes-addons/gitlab-runner/values.yaml b/modules/kubernetes-addons/gitlab-runner/values.yaml new file mode 100644 index 0000000000..8614dbed1a --- /dev/null +++ b/modules/kubernetes-addons/gitlab-runner/values.yaml @@ -0,0 +1,9 @@ +gitlabUrl: https://gitlab.com +rbac: + create: true + rules: + - resources: ["configmaps", "pods", "pods/attach", "secrets", "services"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create", "patch", "delete"] diff --git a/modules/kubernetes-addons/gitlab-runner/variables.tf b/modules/kubernetes-addons/gitlab-runner/variables.tf new file mode 100644 index 0000000000..300b0751c7 --- /dev/null +++ b/modules/kubernetes-addons/gitlab-runner/variables.tf @@ -0,0 +1,26 @@ +variable "helm_config" { + description = "Helm Config for calico" + type = any + default = {} +} + +variable "manage_via_gitops" { + description = "Determines if the add-on should be managed via GitOps." + type = bool + default = false +} + +variable "addon_context" { + description = "Input configuration for the addon" + type = object({ + aws_caller_identity_account_id = string + aws_caller_identity_arn = string + aws_eks_cluster_endpoint = string + aws_partition_id = string + aws_region_name = string + eks_cluster_id = string + eks_oidc_issuer_url = string + eks_oidc_provider_arn = string + tags = map(string) + }) +} diff --git a/modules/kubernetes-addons/gitlab-runner/versions.tf b/modules/kubernetes-addons/gitlab-runner/versions.tf new file mode 100644 index 0000000000..429c0b36d0 --- /dev/null +++ b/modules/kubernetes-addons/gitlab-runner/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.0.0" +} diff --git a/modules/kubernetes-addons/main.tf b/modules/kubernetes-addons/main.tf index 62080c9e4d..76c0af05f2 100644 --- a/modules/kubernetes-addons/main.tf +++ b/modules/kubernetes-addons/main.tf @@ -295,6 +295,16 @@ module "fargate_fluentbit" { addon_context = local.addon_context } +module "gitlab_runner" { + source = "./gitlab-runner" + + count = var.enable_gitlab_runner ? 1 : 0 + + helm_config = var.gitlab_runner_helm_config + manage_via_gitops = var.argocd_manage_add_ons + addon_context = local.addon_context +} + module "grafana" { count = var.enable_grafana ? 1 : 0 source = "./grafana" diff --git a/modules/kubernetes-addons/variables.tf b/modules/kubernetes-addons/variables.tf index 868e618501..a92d44c381 100644 --- a/modules/kubernetes-addons/variables.tf +++ b/modules/kubernetes-addons/variables.tf @@ -1275,6 +1275,19 @@ variable "datadog_operator_helm_config" { default = {} } +#-----------GitLab Operator------------- +variable "enable_gitlab_runner" { + description = "Enable GitLab Runner add-on" + type = bool + default = false +} + +variable "gitlab_runner_helm_config" { + description = "GitLab Runner Helm Chart config" + type = any + default = {} +} + #-----------Promtail ADDON------------- variable "enable_promtail" { description = "Enable Promtail add-on"