Events prod deployment (#635)

* Scripts for AtlasOfLivingAustralia/extended-data-model#80

* service fixes

* set initial_master_nodes, index_buffer_size, configurable cluster.name

* es_index config

* updated docker images

* changes to allow multiple environments on the same VM

ansible/events.yml

@@ -0,0 +1,26 @@
+- hosts: events
+  roles:
+    - common
+    - webserver
+    - events
+  vars:
+    docker_dir: "/data/docker"
+
+- hosts: elasticsearch
+  vars:
+    es_version: "7.17.7"
+    es_heap_dump_path: "/data/"
+    es_api_host: "{{ elasticsearch_host | default('localhost') }}"
+    es_heap_size: "8g"
+    es_config:
+      cluster.name: "{{ elasticsearch_cluster_name | default('extended-data') }}"
+      node.name: "{{ elasticsearch_host | default('localhost') }}"
+      network.host: "{{ elasticsearch_network_host }}"
+      discovery.seed_hosts: "{{ elasticsearch_seed_hosts }}"
+      bootstrap.memory_lock: false
+      cluster.initial_master_nodes: "{{ elasticsearch_cluster_initial_master_nodes }}"
+      indices.memory.index_buffer_size: "{{ elasticsearch_index_buffer_size | default('40%') }}"
+  roles:
+    - common
+    - java
+    - { role: ansible-elasticsearch, es_templates: false, es_instance_name: 'ecodata-elasticsearch', es_data_dirs: ['/data/elasticsearch'], tags: ['elasticsearch'], when: install_elasticsearch | bool }

ansible/roles/events/handlers/main.yml

@@ -0,0 +1,2 @@
+- name: restart graphql_service
+  service: name={{ env }}-graphql state=restarted enabled="yes"

ansible/roles/events/tasks/main.yaml

@@ -0,0 +1,145 @@
+- include: ../../common/tasks/setfacts.yml
+  tags:
+    - docker
+    - update-images
+    - events
+    - service
+    - docker-config
+    - events_vhost
+
+- name: Add Docker GPG apt Key
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    state: present
+  tags:
+    - docker
+    - events
+
+- name: Add Docker Repository
+  apt_repository:
+    repo: deb https://download.docker.com/linux/ubuntu bionic stable
+    state: present
+  tags:
+    - docker
+    - events
+
+- name: Update apt and install docker-ce
+  apt: update_cache=yes name={{ item }} state=latest
+  with_items:
+    - docker-ce
+    - docker-ce-cli
+    - docker-compose
+    - containerd.io
+  tags:
+    - docker
+    - events
+
+- name: Update docker directory
+  template: src={{ item }} dest=/etc/docker/{{ item }}
+  with_items:
+    - daemon.json
+  tags:
+    - docker
+    - events
+
+- name: Restart docker
+  service:
+    name: docker
+    state: restarted
+  when: use_docker_with_pipelines is defined and use_docker_with_pipelines | bool == True
+  tags:
+    - docker
+    - events
+    - docker-restart
+
+- name: ensures {{ data_dir }}/{{ env }} dir exists
+  file:
+    path: "{{ data_dir }}/{{ env }}"
+    state: directory
+  tags:
+    - docker
+    - events
+    - update-images
+    - docker-config
+
+- name: Copy docker YAML files to {{ data_dir }}
+  template: src={{ item }} dest={{ data_dir }}/{{ env }}/{{ item }}
+  with_items:
+    - graphql-service.yml
+    - es2vt.env
+    - es-api.env
+    - graphql-api.env
+  notify:
+    - restart graphql_service
+  tags:
+    - docker
+    - events
+    - update-images
+    - docker-config
+
+- name: Copy service scripts to /usr/bin
+  template: src={{ item }} dest=/usr/bin/{{ env }}-{{ item }} mode=777
+  with_items:
+    - graphql-service.sh
+  tags:
+    - docker
+    - events
+    - service
+
+- name: Copy docker service files to /etc/systemd/system
+  template: src={{ item }} dest=/etc/systemd/system/{{ env }}-{{ item }}
+  with_items:
+    - graphql.service
+  tags:
+    - docker
+    - events
+    - service
+
+- name: enable services
+  service: name="{{ env }}-{{ item }}" enabled=yes
+  with_items:
+    - graphql.service
+  tags:
+    - docker
+    - events
+    - service
+
+- name: Start service graphql-service, if not running
+  service:
+    name: "{{ env }}-graphql"
+    state: started
+  tags:
+    - docker
+    - events
+    - service
+
+- name: add nginx vhost if configured
+  include_role:
+    name: nginx_vhost
+  vars:
+    appname: "events"
+    hostname: "{{ events_hostname }}"
+    context_path: ""
+    nginx_paths:
+      - path: "/es/"
+        sort_label: "1_es"
+        is_proxy: true
+        use_cache: true
+        force_cache: true
+        proxy_pass: "http://127.0.0.1:{{ es_port }}/"
+      - path: "/graphql"
+        sort_label: "1_graphql"
+        is_proxy: true
+        use_cache: true
+        force_cache: true
+        proxy_pass: "http://127.0.0.1:{{ graphql_port }}"
+      - path: "/tile/"
+        sort_label: "1_tile"
+        is_proxy: true
+        use_cache: true
+        force_cache: true
+        proxy_pass: "http://127.0.0.1:{{ es2vt_port }}/"
+  tags:
+    - events
+    - events_vhost
+

ansible/roles/events/templates/daemon.json

@@ -0,0 +1,4 @@
+{
+  "data-root": "{{ docker_dir }}",
+  "live-restore": true
+}

ansible/roles/events/templates/es-api.env

@@ -0,0 +1,14 @@
+event:
+  hosts: [{{ es_hosts }}]
+  requestTimeout: 30000
+  maxRetries: 3
+  maxResultWindow: 100000
+  index: {{ es_index | default('event') }}
+eventOccurrence:
+  hosts: [{{ es_hosts  }}]
+  requestTimeout: 30000
+  maxRetries: 3
+  maxResultWindow: 100000
+  index: {{ es_index | default('event') }}
+apiKey: {{ es_api_key }}
+port: {{ es_port | default('4001')  }}

ansible/roles/events/templates/es2vt.env

@@ -0,0 +1,8 @@
+event:
+  hosts: [{{ es_hosts }}]
+  index: {{ es_index | default('event') }}
+  requestTimeout: 60000
+  maxRetries: 3
+  maxResultWindow: 100000
+port: {{ es2vt_port | default('4002')  }}
+apiKey: {{ es_api_key }}

ansible/roles/events/templates/graphql-api.env

@@ -0,0 +1,15 @@
+port: {{ graphql_port }}
+debug: true
+NODE_DEBUG: http node index.js
+organization: ala
+apiEs: http:///es-api:{{ es_port | default('4001') }}
+apiEsKey: {{ es_api_key }}
+es2vt: http:///es2vt:{{ es2vt_port | default('4002') }}
+apiDownloads: "{{ downloads_url }}/dataset/{datasetKey}"
+datasetEml: "{{ collectory_url }}/ws/eml/{datasetKey}"
+alaNamematching: "{{ namesearch_url }}/api"
+ala:
+  namematching: "{{ namesearch_url }}/api"
+  bie: {{ bie_service_url }}
+  biocache: {{ biocache_ws_url }}
+  images: {{ media_store_url }}

ansible/roles/events/templates/graphql-service.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ $1 = "start" ]; then
+  docker-compose -f /data/{{ env }}/graphql-service.yml up -d
+else
+  docker-compose -f /data/{{ env }}/graphql-service.yml kill
+fi

ansible/roles/events/templates/graphql-service.yml

@@ -0,0 +1,25 @@
+version: "2.2"
+
+services:
+
+  es-api:
+    container_name: "{{ env }}-es-api"
+    image: "{{ es_docker_image }}"
+    ports:
+      - "{{ es_port | default('4001') }}:{{ es_port | default('4001') }}"
+    volumes:
+      - /data/{{ env }}/es-api.env:/usr/src/.env
+  graphql-api:
+    container_name: "{{ env }}-graphql-api"
+    image: "{{ graphql_docker_image }}"
+    ports:
+      - "{{ graphql_port | default('4000') }}:{{ graphql_port | default('4000') }}"
+    volumes:
+      - /data/{{ env }}/graphql-api.env:/usr/src/.env
+  es2vt:
+    container_name: "{{ env }}-es2vt"
+    image: "{{ es2vt_docker_image }}"
+    ports:
+      - "{{ es2vt_port | default('4002') }}:{{ es2vt_port | default('4002') }}"
+    volumes:
+      - /data/{{ env }}/es2vt.env:/usr/src/.env

ansible/roles/events/templates/graphql.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=ALA {{ env }} graphql service
+
+[Service]
+Restart=always
+ExecStart=/usr/bin/{{ env }}-graphql-service.sh start
+ExecStop=/usr/bin/{{ env }}-graphql-service.sh stop
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/events/vars/main.yml

@@ -0,0 +1,3 @@
+graphql_image: "{{ graphql_image | default('atlasoflivingaustralia/graphql-api:5cf0cb50')}}"
+es_image: "{{ es_image | default('atlasoflivingaustralia/es-api:79e116ca')}}"
+es2vt_image: "{{ es2vt_image | default('atlasoflivingaustralia/es2vt:8af41c39')}}"