feat(ansible): add consul role, playbooks for datacenter

ansible/inventory/linode.yml

@@ -24,8 +24,9 @@ groups:
 
   # mintworld Cluster
   mintworld_all: "'mintworld' in (tags|list)"
-  mintworld_leaders: "'mintworld_leader' in (tags|list)"
-  mintworld_workers: "'mintworld_worker' in (tags|list)"
+  mintworld_nomad_servers: "'nomad_svr' in (tags|list) and 'mintworld' in (tags|list)"
+  mintworld_consul_servers: "'consul_svr' in (tags|list) and 'mintworld' in (tags|list)"
+  mintworld_cluster_workers: "'cluster_wkr' in (tags|list) and 'mintworld' in (tags|list)"
 
   # Oldeworld Cluster -- ALL
   oldeworld_all: "'oldeworld' in (tags|list)"

ansible/play-setup-datacenter--0-initialize.yml

@@ -0,0 +1,40 @@
+---
+- name: Datacenter Cluster -- Pre-requisites
+  hosts: '{{ variable_host | default("null") }}'
+  become: true
+  vars:
+    nomad_svr:
+      '{{ variable_nomad_servers_grp | default("mintworld_nomad_servers") }}'
+    consul_svr:
+      '{{ variable_consul_servers_grp | default("mintworld_consul_servers") }}'
+    cluster_wkr:
+      '{{ variable_cluster_workers_grp | default("mintworld_cluster_workers") }}'
+    include_ubuntu_updates:
+      '{{ variable_include_ubuntu_updates | default(false) }}'
+
+  tasks:
+    - name: Datacenter -- Install Pre-requisites
+      include_role:
+        name: ubuntu
+      when:
+        include_ubuntu_updates == true
+
+    - name: Datacenter -- DNS Configuration
+      include_role:
+        name: dns
+
+
+    - name: Datacenter -- Install Nomad Binary on Nomad Servers Nodes and Cluster Workers Nodes
+      include_role:
+        name: nomad
+      when:
+        inventory_hostname in groups[nomad_svr] or inventory_hostname in
+        groups[cluster_wkr]
+
+    - name: Datacenter -- Install Consul Binary on all Nodes
+      include_role:
+        name: consul
+
+    - name: Conclusion
+      debug:
+        msg: "Datacenter -- Pre-requisites Completed, continue with configuration plays."

ansible/play-setup-datacenter--1-configure.yml

@@ -0,0 +1,68 @@
+---
+- name: Datacenter Cluster -- Configure
+  hosts: '{{ variable_host | default("null") }}'
+  become: true
+  vars:
+    - nomad_svr: '{{ variable_nomad_servers_grp | default("mintworld_nomad_servers") }}'
+    - consul_svr: '{{ variable_consul_servers_grp | default("mintworld_consul_servers") }}'
+    - cluster_wkr: '{{ variable_cluster_workers_grp | default("mintworld_cluster_workers") }}'
+    - restart_services: '{{ variable_restart_services | default(false) }}'
+
+  tasks:
+    - name: Configure Consul Servers
+      block:
+        - name: Copy the Certificates to the Consul Servers
+          copy:
+            src: '{{ variable_certificates_dir }}/consul/certs'
+            dest: /etc/consul.d/certs
+            owner: consul
+            group: consul
+            mode: 0755
+        - name: Set up config for Consul Servers
+          include_role:
+            name: consul
+            tasks_from: config-consul-server.yml
+        - name: Set up systemd services for Consul on Servers
+          include_role:
+            name: consul
+            tasks_from: config-systemd.yml
+      when: inventory_hostname in groups[consul_svr]
+
+
+    - name: Configure Consul Clients
+      block:
+        - name: Set up config for Consul Clients
+          include_role:
+            name: consul
+            tasks_from: config-consul-client.yml
+        - name: Set up systemd services for Consul on Clients
+          include_role:
+            name: consul
+            tasks_from: config-systemd.yml
+      when:
+        inventory_hostname in groups[nomad_svr] or inventory_hostname in groups[cluster_wkr]
+
+    - name: Configure Nomad Servers
+      block:
+        - name: Set up config for Nomad Servers
+          include_role:
+            name: nomad
+            tasks_from: config-nomad-server.yml
+        - name: Set up systemd services for Nomad on Servers
+          include_role:
+            name: nomad
+            tasks_from: config-systemd-server.yml
+      when: inventory_hostname in groups[nomad_svr]
+
+    - name: Configure Nomad Clients
+      block:
+        - name: Set up config for Nomad Clients
+          include_role:
+            name: nomad
+            tasks_from: config-nomad-client.yml
+        - name: Set up systemd services for Nomad on Clients
+          include_role:
+            name: nomad
+            tasks_from: config-systemd-client.yml
+      when:
+        inventory_hostname in groups[cluster_wkr]

ansible/play-setup-datacenter--2-startup.yml

@@ -0,0 +1,25 @@
+---
+- name: Datacenter Cluster -- Startup
+  hosts: '{{ variable_host | default("null") }}'
+  become: true
+  vars:
+    - nomad_svr: '{{ variable_nomad_servers_grp | default("mintworld_nomad_servers") }}'
+    - consul_svr: '{{ variable_consul_servers_grp | default("mintworld_consul_servers") }}'
+    - cluster_wkr: '{{ variable_cluster_workers_grp | default("mintworld_cluster_workers") }}'
+    - restart_services: '{{ variable_restart_services | default(false) }}'
+
+  tasks:
+    - name: Startup Consul Servers
+      block:
+        - name: Enable and Start Consul Services on Consul Servers
+          service: name=consul state=started enabled=yes
+          when: restart_services == false
+
+        - name: Restart Consul Services on Consul Servers
+          service: name=consul state=restarted
+          when: restart_services == true
+
+        - name: Print Consul Status
+          shell: consul agent-info
+      when: inventory_hostname in groups[consul_svr]
+

ansible/roles/consul/tasks/config-consul-client.yml

@@ -0,0 +1,16 @@
+---
+- name: Set up Consul Client
+  template:
+    src: consul-client.hcl.j2
+    dest: /etc/consul.d/consul.hcl
+    owner: consul
+    group: consul
+    mode: 0640
+
+- name: Set up Consul Environment File
+  template:
+    src: consul-client.env.j2
+    dest: /etc/consul.d/consul.env
+    owner: consul
+    group: consul
+    mode: 0640

ansible/roles/consul/tasks/config-consul-server.yml

@@ -0,0 +1,16 @@
+---
+- name: Set up Consul Server
+  template:
+    src: consul-server.hcl.j2
+    dest: /etc/consul.d/consul.hcl
+    owner: consul
+    group: consul
+    mode: 0640
+
+- name: Set up Consul Environment File
+  template:
+    src: consul-server.env.j2
+    dest: /etc/consul.d/consul.env
+    owner: consul
+    group: consul
+    mode: 0640

ansible/roles/consul/tasks/config-systemd.yml

@@ -0,0 +1,5 @@
+---
+- name: Set up Consul Systemd Service
+  template:
+    src: consul.service.j2
+    dest: /etc/systemd/system/consul.service

ansible/roles/consul/tasks/install-consul.yml

@@ -0,0 +1,59 @@
+---
+- name: Set the Consul version (pin to a specific version)
+  set_fact:
+    consul_version: 1.17.2
+
+- name: Install Consul from releases
+  unarchive:
+    src: https://releases.hashicorp.com/consul/{{ consul_version }}/consul_{{ consul_version }}_linux_amd64.zip
+    dest: /usr/local/bin
+    remote_src: yes
+    creates: /usr/local/bin/consul
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Check consul version
+  command: consul version
+  register: consul_version_output
+  changed_when: false
+  failed_when: "'Consul v' not in consul_version_output.stdout"
+
+- name: Create a consul group
+  group:
+    name: consul
+    system: yes
+
+- name: Create a consul user
+  user:
+    name: consul
+    group: consul
+    comment: "Consul user"
+    shell: /bin/false
+    system: yes
+    create_home: yes
+    home: /etc/consul.d
+
+- name: Set the permissions on the consul home directory
+  file:
+    path: /etc/consul.d
+    state: directory
+    owner: consul
+    group: consul
+    mode: 0700
+
+- name: Create a consul certificate directory
+  file:
+    path: /etc/consul.d/certs
+    state: directory
+    owner: consul
+    group: consul
+    mode: 0700
+
+- name: Create a data directory
+  file:
+    path: /opt/consul
+    state: directory
+    owner: consul
+    group: consul
+    mode: 0755

ansible/roles/consul/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+- name: Check if Docker is installed
+  stat:
+    path: /usr/bin/docker
+  register: docker_installed
+  no_log: "{{ variable_no_log | default (true) }}"
+
+- name: Check if Consul is installed
+  stat:
+    path: /usr/bin/consul
+  register: consul_installed
+  no_log: "{{ variable_no_log | default (true) }}"
+
+- name: Install Consul if not installed using the role
+  include_role:
+    name: consul
+    tasks_from: install-consul.yml
+  when:
+    consul_installed.stat.exists == false and
+    docker_installed.stat.exists == true
+
+- name: Done Installing
+  debug:
+    msg: "Done Installing Consul, continue to configure with additional playbooks."

ansible/roles/consul/templates/consul-client.env.j2

@@ -0,0 +1 @@
+CONSUL_CACERT=/etc/consul.d/consul-agent-ca.pem

ansible/roles/consul/templates/consul-client.hcl.j2

@@ -0,0 +1,29 @@
+datacenter = "{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"
+data_dir   = "/opt/consul"
+encrypt    = "{{ lookup('env', 'FCC_ANSIBLE_CONSUL_GOSSIP_ENCRYPT_SECRET') }}"
+
+tls {
+  defaults {
+    ca_file = "/etc/consul.d/certs/consul-agent-ca.pem"
+
+    verify_incoming = true
+    verify_outgoing = true
+  }
+  internal_rpc {
+    verify_server_hostname = true
+  }
+}
+
+auto_encrypt {
+  tls = true
+}
+
+retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"]
+
+recursors = ["1.1.1.1"]
+
+acl {
+  enabled                  = true
+  default_policy           = "deny"
+  enable_token_persistence = true
+}

ansible/roles/consul/templates/consul-server.env.j2

@@ -0,0 +1,3 @@
+CONSUL_CACERT=/etc/consul.d/certs/consul-agent-ca.pem
+CONSUL_CLIENT_CERT=/etc/consul.d/certs/"{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"-server-consul-"{{ lookup('env', 'FCC_ANSIBLE_CONSUL_CERT_NUMBER') }}".pem
+CONSUL_CLIENT_KEY =/etc/consul.d/certs/"{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"-server-consul-"{{ lookup('env', 'FCC_ANSIBLE_CONSUL_CERT_NUMBER') }}"-key.pem

ansible/roles/consul/templates/consul-server.hcl.j2

@@ -0,0 +1,45 @@
+datacenter = "{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"
+data_dir   = "/opt/consul"
+encrypt    = "{{ lookup('env', 'FCC_ANSIBLE_CONSUL_GOSSIP_ENCRYPT_SECRET') }}"
+
+server           = true
+bootstrap_expect = 3
+
+bind_addr   = "0.0.0.0"
+client_addr = "0.0.0.0"
+
+connect {
+  enabled = true
+}
+
+tls {
+  defaults {
+    ca_file   = "/etc/consul.d/certs/consul-agent-ca.pem"
+    cert_file = "/etc/consul.d/certs/{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"-server-consul-"{{ lookup('env', 'FCC_ANSIBLE_CONSUL_CERT_NUMBER') }}.pem"
+    key_file  = "/etc/consul.d/certs/{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"-server-consul-"{{ lookup('env', 'FCC_ANSIBLE_CONSUL_CERT_NUMBER') }}-key.pem"
+
+    verify_incoming = true
+    verify_outgoing = true
+  }
+  internal_rpc {
+    verify_server_hostname = true
+  }
+}
+
+auto_encrypt {
+  allow_tls = true
+}
+
+retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"]
+
+recursors = ["1.1.1.1"]
+
+acl {
+  enabled                  = true
+  default_policy           = "deny"
+  enable_token_persistence = true
+}
+
+ui_config {
+  enabled = true
+}

ansible/roles/consul/templates/consul.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description="HashiCorp Consul - A service mesh solution"
+Documentation=https://www.consul.io/
+Requires=network-online.target
+After=network-online.target
+ConditionFileNotEmpty=/etc/consul.d/consul.hcl
+
+[Service]
+EnvironmentFile=-/etc/consul.d/consul.env
+User=consul
+Group=consul
+ExecStart=/usr/local/bin/consul agent -config-dir=/etc/consul.d/
+ExecReload=/bin/kill --signal HUP $MAINPID
+KillMode=process
+KillSignal=SIGTERM
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/nomad/tasks/install-nomad.yml

@@ -1,7 +1,7 @@
 ---
 - name: Set the Nomad version (pin to a specific version)
   set_fact:
-    nomad_version: 1.6.2
+    nomad_version: 1.7.3
 
 - name: Install Nomad from releases
   unarchive:

ansible/roles/nomad/templates/nomad-client.hcl.j2

@@ -1,12 +1,12 @@
-datacenter = "dc-{{ lookup('env', 'FCC_ANSIBLE_NOMAD_ENVNAME') }}"
+datacenter = "{{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }}"
 data_dir   = "/opt/nomad"
 bind_addr  = "0.0.0.0"
 
 client {
   enabled = true
 
   server_join {
-    retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_NOMAD_ENVNAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"]
+    retry_join = ["provider=linode tag_name={{ lookup('env', 'FCC_ANSIBLE_DATACENTER_NAME') }} region=us-east address_type=private_v4 api_token={{ lookup('env', 'LINODE_API_TOKEN') }}"]
     retry_max = 3
     retry_interval = "15s"
   }