From 945110d526794689d7dc0f8c8c212943525855f3 Mon Sep 17 00:00:00 2001 From: Mrugesh Mohapatra Date: Fri, 27 Oct 2023 01:13:24 +0530 Subject: [PATCH] feat(ansible): add consul role, playbooks for datacenter --- ansible/inventory/linode.yml | 5 +- .../play-setup-datacenter--0-initialize.yml | 40 +++++++++++ .../play-setup-datacenter--1-configure.yml | 68 +++++++++++++++++++ ansible/play-setup-datacenter--2-startup.yml | 25 +++++++ .../consul/tasks/config-consul-client.yml | 16 +++++ .../consul/tasks/config-consul-server.yml | 16 +++++ ansible/roles/consul/tasks/config-systemd.yml | 5 ++ ansible/roles/consul/tasks/install-consul.yml | 59 ++++++++++++++++ ansible/roles/consul/tasks/main.yml | 24 +++++++ .../consul/templates/consul-client.env.j2 | 1 + .../consul/templates/consul-client.hcl.j2 | 29 ++++++++ .../consul/templates/consul-server.env.j2 | 3 + .../consul/templates/consul-server.hcl.j2 | 45 ++++++++++++ .../roles/consul/templates/consul.service.j2 | 20 ++++++ ansible/roles/nomad/tasks/install-nomad.yml | 2 +- .../roles/nomad/templates/nomad-client.hcl.j2 | 4 +- .../nomad/templates/nomad-client.service.j2 | 4 +- .../roles/nomad/templates/nomad-server.hcl.j2 | 6 +- .../nomad/templates/nomad-server.service.j2 | 14 ++-- 19 files changed, 369 insertions(+), 17 deletions(-) create mode 100644 ansible/play-setup-datacenter--0-initialize.yml create mode 100644 ansible/play-setup-datacenter--1-configure.yml create mode 100644 ansible/play-setup-datacenter--2-startup.yml create mode 100644 ansible/roles/consul/tasks/config-consul-client.yml create mode 100644 ansible/roles/consul/tasks/config-consul-server.yml create mode 100644 ansible/roles/consul/tasks/config-systemd.yml create mode 100644 ansible/roles/consul/tasks/install-consul.yml create mode 100644 ansible/roles/consul/tasks/main.yml create mode 100644 ansible/roles/consul/templates/consul-client.env.j2 create mode 100644 ansible/roles/consul/templates/consul-client.hcl.j2 create mode 100644 ansible/roles/consul/templates/consul-server.env.j2 create mode 100644 ansible/roles/consul/templates/consul-server.hcl.j2 create mode 100644 ansible/roles/consul/templates/consul.service.j2 diff --git a/ansible/inventory/linode.yml b/ansible/inventory/linode.yml index 5ba3c145f..6c4724235 100644 --- a/ansible/inventory/linode.yml +++ b/ansible/inventory/linode.yml @@ -24,8 +24,9 @@ groups: # mintworld Cluster mintworld_all: "'mintworld' in (tags|list)" - mintworld_leaders: "'mintworld_leader' in (tags|list)" - mintworld_workers: "'mintworld_worker' in (tags|list)" + mintworld_nomad_servers: "'nomad_svr' in (tags|list) and 'mintworld' in (tags|list)" + mintworld_consul_servers: "'consul_svr' in (tags|list) and 'mintworld' in (tags|list)" + mintworld_cluster_workers: "'cluster_wkr' in (tags|list) and 'mintworld' in (tags|list)" # Oldeworld Cluster -- ALL oldeworld_all: "'oldeworld' in (tags|list)" diff --git a/ansible/play-setup-datacenter--0-initialize.yml b/ansible/play-setup-datacenter--0-initialize.yml new file mode 100644 index 000000000..12ffc202b --- /dev/null +++ b/ansible/play-setup-datacenter--0-initialize.yml @@ -0,0 +1,40 @@ +--- +- name: Datacenter Cluster -- Pre-requisites + hosts: '{{ variable_host | default("null") }}' + become: true + vars: + nomad_svr: + '{{ variable_nomad_servers_grp | default("mintworld_nomad_servers") }}' + consul_svr: + '{{ variable_consul_servers_grp | default("mintworld_consul_servers") }}' + cluster_wkr: + '{{ variable_cluster_workers_grp | default("mintworld_cluster_workers") }}' + include_ubuntu_updates: + '{{ variable_include_ubuntu_updates | default(false) }}' + + tasks: + - name: Datacenter -- Install Pre-requisites + include_role: + name: ubuntu + when: + include_ubuntu_updates == true + + - name: Datacenter -- DNS Configuration + include_role: + name: dns + + + - name: Datacenter -- Install Nomad Binary on Nomad Servers Nodes and Cluster Workers Nodes + include_role: + name: nomad + when: + inventory_hostname in groups[nomad_svr] or inventory_hostname in + groups[cluster_wkr] + + - name: Datacenter -- Install Consul Binary on all Nodes + include_role: + name: consul + + - name: Conclusion + debug: + msg: "Datacenter -- Pre-requisites Completed, continue with configuration plays." diff --git a/ansible/play-setup-datacenter--1-configure.yml b/ansible/play-setup-datacenter--1-configure.yml new file mode 100644 index 000000000..468ca316d --- /dev/null +++ b/ansible/play-setup-datacenter--1-configure.yml @@ -0,0 +1,68 @@ +--- +- name: Datacenter Cluster -- Configure + hosts: '{{ variable_host | default("null") }}' + become: true + vars: + - nomad_svr: '{{ variable_nomad_servers_grp | default("mintworld_nomad_servers") }}' + - consul_svr: '{{ variable_consul_servers_grp | default("mintworld_consul_servers") }}' + - cluster_wkr: '{{ variable_cluster_workers_grp | default("mintworld_cluster_workers") }}' + - restart_services: '{{ variable_restart_services | default(false) }}' + + tasks: + - name: Configure Consul Servers + block: + - name: Copy the Certificates to the Consul Servers + copy: + src: '{{ variable_certificates_dir }}/consul/certs' + dest: /etc/consul.d/certs + owner: consul + group: consul + mode: 0755 + - name: Set up config for Consul Servers + include_role: + name: consul + tasks_from: config-consul-server.yml + - name: Set up systemd services for Consul on Servers + include_role: + name: consul + tasks_from: config-systemd.yml + when: inventory_hostname in groups[consul_svr] + + + - name: Configure Consul Clients + block: + - name: Set up config for Consul Clients + include_role: + name: consul + tasks_from: config-consul-client.yml + - name: Set up systemd services for Consul on Clients + include_role: + name: consul + tasks_from: config-systemd.yml + when: + inventory_hostname in groups[nomad_svr] or inventory_hostname in groups[cluster_wkr] + + - name: Configure Nomad Servers + block: + - name: Set up config for Nomad Servers + include_role: + name: nomad + tasks_from: config-nomad-server.yml + - name: Set up systemd services for Nomad on Servers + include_role: + name: nomad + tasks_from: config-systemd-server.yml + when: inventory_hostname in groups[nomad_svr] + + - name: Configure Nomad Clients + block: + - name: Set up config for Nomad Clients + include_role: + name: nomad + tasks_from: config-nomad-client.yml + - name: Set up systemd services for Nomad on Clients + include_role: + name: nomad + tasks_from: config-systemd-client.yml + when: + inventory_hostname in groups[cluster_wkr] diff --git a/ansible/play-setup-datacenter--2-startup.yml b/ansible/play-setup-datacenter--2-startup.yml new file mode 100644 index 000000000..29908e485 --- /dev/null +++ b/ansible/play-setup-datacenter--2-startup.yml @@ -0,0 +1,25 @@ +--- +- name: Datacenter Cluster -- Startup + hosts: '{{ variable_host | default("null") }}' + become: true + vars: + - nomad_svr: '{{ variable_nomad_servers_grp | default("mintworld_nomad_servers") }}' + - consul_svr: '{{ variable_consul_servers_grp | default("mintworld_consul_servers") }}' + - cluster_wkr: '{{ variable_cluster_workers_grp | default("mintworld_cluster_workers") }}' + - restart_services: '{{ variable_restart_services | default(false) }}' + + tasks: + - name: Startup Consul Servers + block: + - name: Enable and Start Consul Services on Consul Servers + service: name=consul state=started enabled=yes + when: restart_services == false + + - name: Restart Consul Services on Consul Servers + service: name=consul state=restarted + when: restart_services == true + + - name: Print Consul Status + shell: consul agent-info + when: inventory_hostname in groups[consul_svr] + diff --git a/ansible/roles/consul/tasks/config-consul-client.yml b/ansible/roles/consul/tasks/config-consul-client.yml new file mode 100644 index 000000000..23eb9e157 --- /dev/null +++ b/ansible/roles/consul/tasks/config-consul-client.yml @@ -0,0 +1,16 @@ +--- +- name: Set up Consul Client + template: + src: consul-client.hcl.j2 + dest: /etc/consul.d/consul.hcl + owner: consul + group: consul + mode: 0640 + +- name: Set up Consul Environment File + template: + src: consul-client.env.j2 + dest: /etc/consul.d/consul.env + owner: consul + group: consul + mode: 0640 diff --git a/ansible/roles/consul/tasks/config-consul-server.yml b/ansible/roles/consul/tasks/config-consul-server.yml new file mode 100644 index 000000000..011cfadfa --- /dev/null +++ b/ansible/roles/consul/tasks/config-consul-server.yml @@ -0,0 +1,16 @@ +--- +- name: Set up Consul Server + template: + src: consul-server.hcl.j2 + dest: /etc/consul.d/consul.hcl + owner: consul + group: consul + mode: 0640 + +- name: Set up Consul Environment File + template: + src: consul-server.env.j2 + dest: /etc/consul.d/consul.env + owner: consul + group: consul + mode: 0640 diff --git a/ansible/roles/consul/tasks/config-systemd.yml b/ansible/roles/consul/tasks/config-systemd.yml new file mode 100644 index 000000000..a7be8b104 --- /dev/null +++ b/ansible/roles/consul/tasks/config-systemd.yml @@ -0,0 +1,5 @@ +--- +- name: Set up Consul Systemd Service + template: + src: consul.service.j2 + dest: /etc/systemd/system/consul.service diff --git a/ansible/roles/consul/tasks/install-consul.yml b/ansible/roles/consul/tasks/install-consul.yml new file mode 100644 index 000000000..39cbd9043 --- /dev/null +++ b/ansible/roles/consul/tasks/install-consul.yml @@ -0,0 +1,59 @@ +--- +- name: Set the Consul version (pin to a specific version) + set_fact: + consul_version: 1.17.2 + +- name: Install Consul from releases + unarchive: + src: https://releases.hashicorp.com/consul/{{ consul_version }}/consul_{{ consul_version }}_linux_amd64.zip + dest: /usr/local/bin + remote_src: yes + creates: /usr/local/bin/consul + owner: root + group: root + mode: 0755 + +- name: Check consul version + command: consul version + register: consul_version_output + changed_when: false + failed_when: "'Consul v' not in consul_version_output.stdout" + +- name: Create a consul group + group: + name: consul + system: yes + +- name: Create a consul user + user: + name: consul + group: consul + comment: "Consul user" + shell: /bin/false + system: yes + create_home: yes + home: /etc/consul.d + +- name: Set the permissions on the consul home directory + file: + path: /etc/consul.d + state: directory + owner: consul + group: consul + mode: 0700 + +- name: Create a consul certificate directory + file: + path: /etc/consul.d/certs + state: directory + owner: consul + group: consul + mode: 0700 + +- name: Create a data directory + file: + path: /opt/consul + state: directory + owner: consul + group: consul + mode: 0755 diff --git a/ansible/roles/consul/tasks/main.yml b/ansible/roles/consul/tasks/main.yml new file mode 100644 index 000000000..47a6aa2a7 --- /dev/null +++ b/ansible/roles/consul/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- name: Check if Docker is installed + stat: + path: /usr/bin/docker + register: docker_installed + no_log: "{{ variable_no_log | default (true) }}" + +- name: Check if Consul is installed + stat: + path: /usr/bin/consul + register: consul_installed + no_log: "{{ variable_no_log | default (true) }}" + +- name: Install Consul if not installed using the role + include_role: + name: consul + tasks_from: install-consul.yml + when: + consul_installed.stat.exists == false and + docker_installed.stat.exists == true + +- name: Done Installing + debug: + msg: "Done Installing Consul, continue to configure with additional playbooks." diff --git a/ansible/roles/consul/templates/consul-client.env.j2 b/ansible/roles/consul/templates/consul-client.env.j2 new file mode 100644 index 000000000..d45628454 --- /dev/null +++ b/ansible/roles/consul/templates/consul-client.env.j2 @@ -0,0 +1 @@ +CONSUL_CACERT=/etc/consul.d/consul-agent-ca.pem diff --git a/ansible/roles/consul/templates/consul-client.hcl.j2 b/ansible/roles/consul/templates/consul-client.hcl.j2 new file mode 100644 index 000000000..09c1534db --- /dev/null +++ b/ansible/roles/consul/templates/consul-client.hcl.j2 @@ -0,0 +1,29 @@ +datacenter = "{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}" +data_dir = "/opt/consul" +encrypt = "{{ lookup('env', 'FCC_ANSIBLE_CONSUL_GOSSIP_ENCRYPT_SECRET') }}" + +tls { + defaults { + ca_file = "/etc/consul.d/certs/consul-agent-ca.pem" + + verify_incoming = true + verify_outgoing = true + } + internal_rpc { + verify_server_hostname = true + } +} + +auto_encrypt { + tls = true +} + +retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"] + +recursors = ["1.1.1.1"] + +acl { + enabled = true + default_policy = "deny" + enable_token_persistence = true +} diff --git a/ansible/roles/consul/templates/consul-server.env.j2 b/ansible/roles/consul/templates/consul-server.env.j2 new file mode 100644 index 000000000..b736c926b --- /dev/null +++ b/ansible/roles/consul/templates/consul-server.env.j2 @@ -0,0 +1,3 @@ +CONSUL_CACERT=/etc/consul.d/certs/consul-agent-ca.pem +CONSUL_CLIENT_CERT=/etc/consul.d/certs/"{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"-server-consul-"{{ lookup('env', 'FCC_ANSIBLE_CONSUL_CERT_NUMBER') }}".pem +CONSUL_CLIENT_KEY =/etc/consul.d/certs/"{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"-server-consul-"{{ lookup('env', 'FCC_ANSIBLE_CONSUL_CERT_NUMBER') }}"-key.pem diff --git a/ansible/roles/consul/templates/consul-server.hcl.j2 b/ansible/roles/consul/templates/consul-server.hcl.j2 new file mode 100644 index 000000000..580bad13a --- /dev/null +++ b/ansible/roles/consul/templates/consul-server.hcl.j2 @@ -0,0 +1,45 @@ +datacenter = "{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}" +data_dir = "/opt/consul" +encrypt = "{{ lookup('env', 'FCC_ANSIBLE_CONSUL_GOSSIP_ENCRYPT_SECRET') }}" + +server = true +bootstrap_expect = 3 + +bind_addr = "0.0.0.0" +client_addr = "0.0.0.0" + +connect { + enabled = true +} + +tls { + defaults { + ca_file = "/etc/consul.d/certs/consul-agent-ca.pem" + cert_file = "/etc/consul.d/certs/{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"-server-consul-"{{ lookup('env', 'FCC_ANSIBLE_CONSUL_CERT_NUMBER') }}.pem" + key_file = "/etc/consul.d/certs/{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"-server-consul-"{{ lookup('env', 'FCC_ANSIBLE_CONSUL_CERT_NUMBER') }}-key.pem" + + verify_incoming = true + verify_outgoing = true + } + internal_rpc { + verify_server_hostname = true + } +} + +auto_encrypt { + allow_tls = true +} + +retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"] + +recursors = ["1.1.1.1"] + +acl { + enabled = true + default_policy = "deny" + enable_token_persistence = true +} + +ui_config { + enabled = true +} diff --git a/ansible/roles/consul/templates/consul.service.j2 b/ansible/roles/consul/templates/consul.service.j2 new file mode 100644 index 000000000..310edf456 --- /dev/null +++ b/ansible/roles/consul/templates/consul.service.j2 @@ -0,0 +1,20 @@ +[Unit] +Description="HashiCorp Consul - A service mesh solution" +Documentation=https://www.consul.io/ +Requires=network-online.target +After=network-online.target +ConditionFileNotEmpty=/etc/consul.d/consul.hcl + +[Service] +EnvironmentFile=-/etc/consul.d/consul.env +User=consul +Group=consul +ExecStart=/usr/local/bin/consul agent -config-dir=/etc/consul.d/ +ExecReload=/bin/kill --signal HUP $MAINPID +KillMode=process +KillSignal=SIGTERM +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/nomad/tasks/install-nomad.yml b/ansible/roles/nomad/tasks/install-nomad.yml index 34d5e5951..5fabe0194 100644 --- a/ansible/roles/nomad/tasks/install-nomad.yml +++ b/ansible/roles/nomad/tasks/install-nomad.yml @@ -1,7 +1,7 @@ --- - name: Set the Nomad version (pin to a specific version) set_fact: - nomad_version: 1.6.2 + nomad_version: 1.7.3 - name: Install Nomad from releases unarchive: diff --git a/ansible/roles/nomad/templates/nomad-client.hcl.j2 b/ansible/roles/nomad/templates/nomad-client.hcl.j2 index e331c7ff4..a9aab41d3 100644 --- a/ansible/roles/nomad/templates/nomad-client.hcl.j2 +++ b/ansible/roles/nomad/templates/nomad-client.hcl.j2 @@ -1,4 +1,4 @@ -datacenter = "dc-{{ lookup('env', 'FCC_ANSIBLE_NOMAD_ENVNAME') }}" +datacenter = "{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}" data_dir = "/opt/nomad" bind_addr = "0.0.0.0" @@ -6,7 +6,7 @@ client { enabled = true server_join { - retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_NOMAD_ENVNAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"] + retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"] retry_max = 3 retry_interval = "15s" } diff --git a/ansible/roles/nomad/templates/nomad-client.service.j2 b/ansible/roles/nomad/templates/nomad-client.service.j2 index 7867c55ea..d9d71d9fc 100644 --- a/ansible/roles/nomad/templates/nomad-client.service.j2 +++ b/ansible/roles/nomad/templates/nomad-client.service.j2 @@ -7,8 +7,8 @@ After=network-online.target # When using Nomad with Consul it is not necessary to start Consul first. These # lines start Consul before Nomad as an optimization to avoid Nomad logging # that Consul is unavailable at startup. -#Wants=consul.service -#After=consul.service +Wants=consul.service +After=consul.service [Service] diff --git a/ansible/roles/nomad/templates/nomad-server.hcl.j2 b/ansible/roles/nomad/templates/nomad-server.hcl.j2 index 9c2455220..2076dc635 100644 --- a/ansible/roles/nomad/templates/nomad-server.hcl.j2 +++ b/ansible/roles/nomad/templates/nomad-server.hcl.j2 @@ -1,4 +1,4 @@ -datacenter = "dc-{{ lookup('env', 'FCC_ANSIBLE_NOMAD_ENVNAME') }}" +datacenter = "{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}" data_dir = "/opt/nomad" server { @@ -6,8 +6,8 @@ server { bootstrap_expect = 3 server_join { - retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_NOMAD_ENVNAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"] - retry_max = 3 + retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"] + retry_max = 3 retry_interval = "15s" } } diff --git a/ansible/roles/nomad/templates/nomad-server.service.j2 b/ansible/roles/nomad/templates/nomad-server.service.j2 index 45394342f..adbe05de2 100644 --- a/ansible/roles/nomad/templates/nomad-server.service.j2 +++ b/ansible/roles/nomad/templates/nomad-server.service.j2 @@ -4,16 +4,16 @@ Documentation=https://www.nomadproject.io/docs/ Wants=network-online.target After=network-online.target -# When using Nomad with Consul it is not necessary to start Consul first. These -# lines start Consul before Nomad as an optimization to avoid Nomad logging -# that Consul is unavailable at startup. -#Wants=consul.service -#After=consul.service +## When using Nomad with Consul it is not necessary to start Consul first. These +## lines start Consul before Nomad as an optimization to avoid Nomad logging +## that Consul is unavailable at startup. +# Wants=consul.service +# After=consul.service [Service] -# Nomad server should be run as the nomad user. Nomad clients -# should be run as root +## Nomad server should be run as the nomad user. Nomad clients +## should be run as root User=nomad Group=nomad