#15 import from cyverse

ansible/playbooks/support-filebeat.yaml

@@ -0,0 +1,113 @@
+# Installs filebeat to all managed systems in inventory
+#
+---
+- hosts: services:ui:&systems
+  become: true
+  vars:
+      ssl_docker:
+          image: "{{docker.registry.base}}/{{data_container.image_name}}"
+          container: "{{data_container.container_name}}"
+          cert_path: "{{logstash.ssl.cert}}"
+  roles:
+      - role: infra-import-trusted-ca-cert
+
+- hosts: services:ui:&systems
+  become: true
+  vars:
+      filebeat_logstash_host: "{{groups['elasticsearch'][0]}}:{{logstash.port}}"
+      filebeat_index_pattern: "de-filebeat"
+      filebeat_config:
+         filebeat:
+           prospectors:
+             - paths:
+                  - "/var/log/de/anon-files-docker.log"
+               document_type: "anon-files-log"
+             - paths:
+                  - "/var/log/de/clockwork-docker.log"
+               document_type: "clockwork-log"
+             - paths:
+                  - "/var/log/de/condor-log-monitor-docker.log"
+               document_type: "condor-log-monitor-log"
+             - paths:
+                  - "/var/log/de/data-info-docker.log"
+               document_type: "data-info-log"
+             - paths:
+                  - "/var/log/de/nginx-de-ui.log"
+               document_type: "de-ui-nginx-log"
+             - paths:
+                  - "/var/log/de/de-ui.log"
+               document_type: "ui-log"
+             - paths:
+                  - "/var/log/de/dewey-docker.log"
+               document_type: "dewey-log"
+             - paths:
+                  - "/var/log/de/terrain-docker.log"
+               document_type: "terrain-log"
+             - paths:
+                  - "/var/log/de/exim-docker.log"
+               document_type: "exim-sender-log"
+             - paths:
+                  - "/var/log/de/infosquito-docker.log"
+               document_type: "infosquito-log"
+             - paths:
+                  - "/var/log/de/info-typer-docker.log"
+               document_type: "info-typer-log"
+             - paths:
+                  - "/var/log/de/iplant-email-docker.log"
+               document_type: "iplant-email-log"
+             - paths:
+                  - "/var/log/de/iplant-groups-docker.log"
+               document_type: "iplant-groups-log"
+             - paths:
+                  - "/var/log/de/jex/jex.log"
+               document_type: "jex-log"
+             - paths:
+                  - "/var/log/de/jex-events-docker.log"
+               document_type: "jexevents-log"
+             - paths:
+                  - "/var/log/de/kifshare-docker.log"
+               document_type: "kifshare-log"
+             - paths:
+                  - "/var/log/de/apps-docker.log"
+               document_type: "apps-log"
+             - paths:
+                  - "/var/log/de/metadata-docker.log"
+               document_type: "metadata-log"
+             - paths:
+                  - "/var/log/de/monkey-docker.log"
+               document_type: "monkey-log"
+             - paths:
+                  - "/var/log/de/notificationagent-docker.log"
+               document_type: "notificationagent-log"
+             - paths:
+                  - "/var/log/de/saved-searches-docker.log"
+               document_type: "saved-searches-log"
+             - paths:
+                  - "/var/log/de/tree-urls-docker.log"
+               document_type: "tree-urls-log"
+             - paths:
+                  - "/var/log/de/user-preferences-docker.log"
+               document_type: "user-preferences-log"
+             - paths:
+                  - "/var/log/de/user-sessions-docker.log"
+               document_type: "user-sessions-log"
+         output:
+          logstash:
+            hosts:
+              - "{{filebeat_logstash_host}}"
+            tls:
+              certificate_authorities: []
+            index: "{{filebeat_index_pattern}}"
+         shipper:
+           name: "{{ansible_hostname}}"
+           tags:
+               - "{{ inventory_file.split('/')[-1].split('.')[0] }}"
+         logging:
+           to_files: true
+           level: info
+           files:
+             path: /var/log/filebeat
+             name: filebeat
+             rotateeverybytes: 10485760 
+  roles:
+     - role: support-filebeat

ansible/playbooks/support-logstash-cfg.yaml

@@ -0,0 +1,27 @@
+# Installs logstash to the elasticsearch client node
+#
+---
+- hosts: elasticsearch[0]:&systems
+  become: true
+  vars:
+      ssl_docker:
+          image: "{{docker.registry.base}}/{{data_container.image_name}}"
+          container: "{{data_container.container_name}}"
+          cert_path: "{{logstash.ssl.cert}}"
+          key_path: "{{logstash.ssl.key}}"
+      ssl_dest:
+          cert_path: "{{logstash.ssl.cert}}"
+          key_path: "{{logstash.ssl.key}}"
+          owner: "logstash"
+          group: "logstash"
+  roles:
+      - role: infra-place-ssl-keys
+
+- hosts: elasticsearch[0]:&systems
+  become: true
+  vars:
+      logstash_elasticsearch_host: "{{groups['elasticsearch'][0]}}:{{elasticsearch.network_http_port}}"
+      logstash_plugins:
+          - plugin: logstash-input-beats
+  roles:
+      - role: support-logstash-cfg

ansible/roles/support-filebeat/.travis.yml

@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/

ansible/roles/support-filebeat/README.md

@@ -0,0 +1,45 @@
+support-filebeat
+================
+
+Installs and configures a Filebeat instance.
+
+https://www.elastic.co/guide/en/beats/filebeat/current/index.html
+
+Requirements
+------------
+
+Ansible 2.x
+Requires sudo.
+
+Role Variables
+--------------
+
+|   Variable                         | required | default                 | choices | comments                                               |
+|------------------------------------|----------|-------------------------|---------|--------------------------------------------------------|
+| filebeat_version                   |  no      |                         |         | When defined, will install the specified version of Filebeat, otherwise, the most current will be installed. |
+| filebeat_install                   |  no      | true                    |         | A flag used to control whether the role should perform installation steps. |
+| filebeat_config                    |  no      |                         |         | If defined, is used to populate the filebeat config file. If undefined, the config file is unchanged. |
+
+
+Dependencies
+------------
+
+N/A
+
+Example Playbook
+----------------
+
+    - hosts: systems
+      vars:
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+Jonathan Strootman "[email protected]"

ansible/roles/support-filebeat/defaults/main.yaml

@@ -0,0 +1,6 @@
+---
+# defaults file for support-filebeat
+filebeat_install: true
+filebeat_apt_repo:
+    gpgkey: "http://packages.elastic.co/GPG-KEY-elasticsearch"
+    repo: "deb https://packages.elastic.co/beats/apt stable main"

ansible/roles/support-filebeat/files/beats.repo

@@ -0,0 +1,7 @@
+[beats]
+name=Elastic Beats Repository
+baseurl=https://packages.elastic.co/beats/yum/el/$basearch
+enabled=1
+gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
+gpgcheck=1
+

ansible/roles/support-filebeat/handlers/main.yaml

@@ -0,0 +1,6 @@
+---
+# handlers file for support-filebeat
+- name: restart filebeat
+  service:
+      name: filebeat
+      state: restarted

ansible/roles/support-filebeat/meta/main.yaml

@@ -0,0 +1,37 @@
+galaxy_info:
+  author: Jonathan Strootman
+  description:  Installs filebeat instance
+  company: CyVerse
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+  min_ansible_version: 1.9
+  platforms:
+  - name: EL
+    versions:
+    - 6
+    - 7
+  - name: Ubuntu
+    versions:
+    - trusty
+    - utopic
+    - vivid
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is
+    # a keyword that describes and categorizes the role.
+    # Users find roles by searching for tags. Be sure to
+    # remove the '[]' above if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of
+    # alphanumeric characters. Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line.
+  # Be sure to remove the '[]' above if you add dependencies
+  # to this list.

ansible/roles/support-filebeat/tasks/CentOS.yaml

@@ -0,0 +1,19 @@
+---
+- name: Add Elastic repo public signing key
+  shell: rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
+  tags: 
+     - install
+
+- name: Add Beats repo
+  copy: src=beats.repo dest=/etc/yum.repos.d/ mode=0644
+  tags: 
+     - install
+
+- name: Install Filebeat
+  yum:
+    name: "filebeat{% if filebeat_version is defined %}-{{filebeat_version}}{% endif %}"
+    update_cache: yes
+    state: present
+  tags: 
+     - install
+

ansible/roles/support-filebeat/tasks/Ubuntu.yaml

@@ -0,0 +1,22 @@
+---
+- name: Add Logstash repo key
+  when:
+  apt_key: "url={{filebeat_apt_repo.gpgkey}} state=present"
+  tags: 
+     - install
+
+- name: Add Beats repo
+  apt_repository: 
+      repo: "{{filebeat_apt_repo.repo}}"
+      state: present
+  tags: 
+     - install
+
+- name: Install filebeat
+  apt: 
+    name: "filebeat{% if filebeat_version is defined %}-{{filebeat_version}}{% endif %}"
+    update_cache: yes
+    state: present
+  tags: 
+     - install
+

ansible/roles/support-filebeat/tasks/check.yaml

@@ -0,0 +1,17 @@
+---
+- debug: var=ansible_distribution
+  when: (ansible_distribution == "CentOS" and
+         ansible_distribution_major_version | version_compare('6', '<')) or
+        (ansible_distribution == "Ubuntu" and 
+         ansible_distribution_major_version | version_compare('14', '<'))
+- debug: var=ansible_distribution_major_version 
+  when: (ansible_distribution == "CentOS" and
+         ansible_distribution_major_version | version_compare('6', '<')) or
+        (ansible_distribution == "Ubuntu" and 
+         ansible_distribution_major_version | version_compare('14', '<'))
+- name: Verify OS
+  fail: msg="This role only supported on Ubuntu vivid or CentOS 7"
+  failed_when: (ansible_distribution == "CentOS" and
+                ansible_distribution_major_version | version_compare('6', '<')) or
+               (ansible_distribution == "Ubuntu" and 
+                ansible_distribution_major_version | version_compare('14', '<'))

ansible/roles/support-filebeat/tasks/main.yaml

@@ -0,0 +1,29 @@
+---
+# tasks file for support-filebeat
+- include: check.yaml
+
+- when: filebeat_install
+  block:
+       - include: Ubuntu.yaml
+         when: ansible_distribution == 'Ubuntu'
+       - include: CentOS.yaml
+         when: ansible_distribution == "CentOS"
+       - name: Enable Filebeat at boot
+         service:
+             name: filebeat
+             enabled: yes
+         notify: restart filebeat
+         tags:
+             - install
+
+- debug: var=filebeat_config             
+- name: Configure Filebeat
+  when: filebeat_config is defined
+  template:
+      src: "filebeat.yml.j2"
+      dest: "/etc/filebeat/filebeat.yml"
+  notify: restart filebeat
+  tags: 
+     - config
+
+

ansible/roles/support-filebeat/templates/filebeat.yml.j2

@@ -0,0 +1,3 @@
+## {{ ansible_managed }}
+---
+{{filebeat_config | to_nice_yaml }}

ansible/roles/support-filebeat/tests/inventory

@@ -0,0 +1 @@
+localhost

ansible/roles/support-filebeat/tests/test.yaml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - support-filebeat