Merge pull request #549 from deNBI/staging

Staging
deNBI · Dec 8, 2020 · bb912eb · bb912eb
2 parents 055fdb9 + df82f14
commit bb912eb
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 49 deletions.
diff --git a/.env.in b/.env.in
@@ -5,6 +5,8 @@ FORC_API_KEY=
 ELASTIC_USER=
 ELASTIC_PASSWORD=
 FILEBEAT_TAG=7.1.0
+HOST=portal-dev.denbi.de
+REGION=Giessen-Staging
 
 OS_AUTH_URL=
 OS_PROJECT_ID=

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -6,16 +6,14 @@ updates:
         interval: "weekly"
     target-branch: "dev"
     reviewers:
-      - "dweinholz"
-      - "eKatchko"
-      - "vktrrdk"
+      - "deNBI/portal-dev"
+
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
         interval: "weekly"
     target-branch: "dev"
     reviewers:
-      - "dweinholz"
-      - "eKatchko"
-      - "vktrrdk"
+      - "deNBI/portal-dev"
+
 
diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml
@@ -26,7 +26,6 @@ services:
     networks:
      - portal
 
-
   redis:
     image: redis:5.0.5
     expose:
@@ -36,10 +35,8 @@ services:
 
   bibigrid:
     image: bibiserv/bibigrid:${BIBIGRID_TAG}
-
     env_file:
       - .env
-
     environment:
       - OS_AUTH_URL
       - OS_PROJECT_ID
@@ -56,14 +53,15 @@ services:
     networks:
       - portal
 
-
-
   # filebeat
   filebeat:
     image: docker.elastic.co/beats/filebeat:${FILEBEAT_TAG}
+    env_file:
+      - .env
     environment:
       - ELASTIC_USER
       - ELASTIC_PASSWORD
+      - HOST
     volumes:
       - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
       - ./logs:/usr/share/filebeat/log:ro

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -38,9 +38,12 @@ services:
   # filebeat
   filebeat:
     image: docker.elastic.co/beats/filebeat:${FILEBEAT_TAG}
+    env_file:
+      - .env
     environment:
       - ELASTIC_USER
       - ELASTIC_PASSWORD
+      - HOST
     volumes:
       - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
       - ./logs:/usr/share/filebeat/log:ro

diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
@@ -1,16 +1,23 @@
+fields_under_root: true
+fields:
+  region: '${REGION}'
+
 output.elasticsearch:
     hosts:
-      - https://cloud.denbi.de:443
+      - 'https://${HOST}:443'
     path: /elasticsearch/
-    username: "${ELASTIC_USER}"
-    password: "${ELASTIC_PASSWORD}"
+    username: '${ELASTIC_USER}'
+    password: '${ELASTIC_PASSWORD}'
     ssl:
       verification_mode: none
     timeout: 15
+    headers:
+      HOST: '${HOST}'
 
 filebeat.inputs:
   - type: log
     enabled: true
+    exclude_files: ['\.swp$']
     paths:
       - /usr/share/filebeat/log/client/*.log
     multiline:

diff --git a/plays/general_variables.yml b/plays/general_variables.yml
@@ -3,4 +3,5 @@ client:
   CONFIG_DIR_PATH: /home/ubuntu/client/VirtualMachineService/config
   CERT_FILE_PATH: /home/ubuntu/client/VirtualMachineService/keys
   REPO_PATH: /home/ubuntu/client
+  FILEBEAT_DATA_APPEND: filebeat/data
 
diff --git a/plays/setup_basics.yml b/plays/setup_basics.yml
@@ -27,7 +27,6 @@
         cache_valid_time: 86400 #One day
       tags: always
 
-
     - name: Checkout cloud-portal-client repository
       git:
         repo: "[email protected]:deNBI/cloud-portal-client.git"
@@ -38,26 +37,20 @@
       become: no
       tags: always
 
-
     - name: Install python3-pip
       apt:
         state: latest
         name: python3-pip
       tags: always
 
-
     - name: Install list of packages
       apt:
         name:  ['libffi-dev', 'libssl-dev', 'jq', 'python3-setuptools']
         state: present
         update_cache: yes
       tags: always
 
-
     - name: Install docker-compose with pip
       pip:
         name: docker-compose
       tags: always
-
-
-
diff --git a/plays/setup_client.yml b/plays/setup_client.yml
@@ -4,18 +4,12 @@
   vars_files:
     - general_variables.yml
   tasks:
-
+    # Checks
     - name: Setup persistent client directory
       file:
         path: "{{ client.PERSISTENT_PATH }}"
         state: directory
       tags: always
-    - name: Setup persistent client cert directory
-      file:
-        path: "{{ client.PERSISTENT_PATH }}"
-        state: directory
-      tags: always
-
 
     - name: Check if client config file exists already
       stat:
@@ -28,7 +22,16 @@
       register: stat_client_pem
       tags: always
 
+    - name: Setup filebeat data directory
+      file:
+        path: "{{ client.REPO_PATH }}/{{ client.FILEBEAT_DATA_APPEND }}"
+        state: directory
+        owner: ubuntu
+        group: ubuntu
+        recurse: yes
+      tags: always
 
+    # Copy configs
     - name: Copy client config file from config folder
       copy: 
         remote_src: True 
@@ -37,33 +40,28 @@
       when: client_config is not defined and stat_client_config.stat.exists == False
       tags: always
 
-
     - name: Copy specified client config file to remote machine 
       copy: src={{ item.src }} dest={{ item.dest }}
       with_items:
         - { src: "{{client_config}}", dest:  "{{ client.PERSISTENT_PATH }}/config.yml" }
       when: client_config is defined
       tags: always
 
-
-
     - name: Copy specified server pem file to remote machine persistent
       copy: src={{ item.src }} dest={{ item.dest }}
       with_items:
         - { src: "{{client_server_pem}}", dest: "{{ client.PERSISTENT_PATH }}/server.pem" }
       when: client_server_pem is defined
       tags: always
 
-
     - name: Copy specified server pem file to remote machine persistent from default path
       copy: src={{ item.src }} dest={{ item.dest }}
       with_items:
         - { src:  "{{client.CERT_FILE_PATH}}", dest: "{{ client.PERSISTENT_PATH }}/server.pem" }
       when: client_server_pem is not defined and stat_client_pem.exists == False
       tags: always
 
-
-
+    # Copy envs
     - name: Copy env file default
       copy:
         src: "../.env"
@@ -72,7 +70,6 @@
       when: env_file is not defined
       tags: always
 
-
     - name: Copy env file
       copy:
         src: "{{env_file}}"
@@ -81,8 +78,6 @@
       when: env_file is defined
       tags: always
 
-
-
     - name: Copy secrets file default
       copy:
         src: "../.secrets"
@@ -91,7 +86,6 @@
       when: secrets_file is not defined
       tags: always
 
-
     - name: Copy secrets file
       copy:
         src: "{{secrets_file}}"
@@ -100,9 +94,6 @@
       when: secrets_file is defined
       tags: always
 
-
-
-
     - name: Create variable files
       copy: 
         content: "{{ item.content | dict2items | map('to_json') | map('regex_replace', '\"key\":\\s\"(.*)\"', lookup('vars', 'regex_env')) | map('from_json') | list}}"
@@ -114,7 +105,6 @@
         - { content: "{{ client }}", name: 'client' }
       tags: always
 
-
     - name: Transform json to properties file
       shell: rm -f "{{ client.REPO_PATH }}"/.ansible_environment && jq -r '.[]|"\(.key)=\(.value)"' "{{ client.REPO_PATH }}"/.ansible_environment_*.json >> "{{ client.REPO_PATH }}"/.ansible_environment
       tags: always
@@ -123,7 +113,7 @@
       shell: cat "{{ client.REPO_PATH }}"/.secrets "{{ client.REPO_PATH }}"/.ansible_environment >> "{{ client.REPO_PATH }}"/.env
       tags: always
 
-
+    # Start
     - name: Start client docker container without bibigrid
       docker_compose:
         recreate: always
@@ -134,7 +124,6 @@
       become: yes
       when: "'bibigrid' not in ansible_run_tags"
 
-
     - name: Start docker container with bibigrid
       docker_compose:
           recreate: always
@@ -144,7 +133,3 @@
             - docker-compose.bibigrid.yml
       become: yes
       tags: ['never', 'bibigrid']
-
-
-
-
diff --git a/requirements.txt b/requirements.txt
@@ -5,7 +5,7 @@ openstacksdk ==0.51.0
 deprecated == 1.2.10
 Click==7.1.2
 flake8==3.8.4
-ansible==2.10.3
+ansible==2.10.4
 ruamel.yaml<0.17.0
 paramiko==2.7.2
 pyvim==3.0.2