diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml index 888dd6c9c..e9da7ad77 100644 --- a/.github/workflows/provision.yml +++ b/.github/workflows/provision.yml @@ -74,7 +74,6 @@ jobs: mongodb_admin_username: ${{ secrets.MONGODB_ADMIN_USER }} mongodb_admin_password: ${{ secrets.MONGODB_ADMIN_PASSWORD }} backup_encryption_passphrase: ${{ secrets.BACKUP_ENCRYPTION_PASSPHRASE }} - restore_backup_encryption_passphrase: ${{ secrets.RESTORE_BACKUP_ENCRYPTION_PASSPHRASE }} elasticsearch_superuser_password: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }} external_backup_server_ssh_port: 22 external_backup_server_ip: ${{ secrets.BACKUP_HOST }} diff --git a/infrastructure/deployment/deploy.sh b/infrastructure/deployment/deploy.sh index 426cb14d3..b21c137c9 100755 --- a/infrastructure/deployment/deploy.sh +++ b/infrastructure/deployment/deploy.sh @@ -352,6 +352,12 @@ EOF # Setup configuration files and compose file for the deployment domain configured_ssh " + HOST=$HOST + SMTP_HOST=$SMTP_HOST + SMTP_PORT=$SMTP_PORT + ALERT_EMAIL=$ALERT_EMAIL + SENDER_EMAIL_ADDRESS=$SENDER_EMAIL_ADDRESS + DOMAIN=$DOMAIN MINIO_ROOT_USER=$MINIO_ROOT_USER MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD /opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST | tee -a $LOG_LOCATION/setup-deploy-config.log" @@ -366,6 +372,7 @@ echo echo "Waiting 2 mins for mongo to deploy before working with data. Please note it can take up to 10 minutes for the entire stack to deploy in some scenarios." echo +sleep 120 # Required as Kibana cannot be immediately contacted echo "Setting up Kibana config & alerts" while true; do diff --git a/infrastructure/docker-compose.deploy.yml b/infrastructure/docker-compose.deploy.yml index c53c0fd78..7e7b0e834 100644 --- a/infrastructure/docker-compose.deploy.yml +++ b/infrastructure/docker-compose.deploy.yml @@ -405,6 +405,8 @@ services: - '/opt/opencrvs/infrastructure/monitoring/elastalert/rules:/opt/elastalert/rules' networks: - overlay_net + depends_on: + - elasticsearch deploy: labels: - 'traefik.enable=false' diff --git a/infrastructure/docker-compose.development-deploy.yml b/infrastructure/docker-compose.development-deploy.yml index 53e03684a..276f4f8c5 100644 --- a/infrastructure/docker-compose.development-deploy.yml +++ b/infrastructure/docker-compose.development-deploy.yml @@ -15,6 +15,7 @@ services: - jwt-public-key.{{ts}} environment: - NODE_ENV=production + - QA_ENV=true - FHIR_URL=http://hearth:3447/fhir - AUTH_URL=http://auth:4040 - APPLICATION_CONFIG_URL=http://config:2021 diff --git a/infrastructure/monitoring/elastalert/elastalert.yaml b/infrastructure/monitoring/elastalert/elastalert.yaml index 2c81b51bc..b74db9981 100644 --- a/infrastructure/monitoring/elastalert/elastalert.yaml +++ b/infrastructure/monitoring/elastalert/elastalert.yaml @@ -16,8 +16,6 @@ buffer_time: es_host: elasticsearch es_port: 9200 -es_username: '{{ES_USERNAME}}' -es_password: '{{ES_PASSWORD}}' writeback_index: elastalert_status diff --git a/infrastructure/monitoring/kibana/setup-config.sh b/infrastructure/monitoring/kibana/setup-config.sh index 3fe3cdfff..92e456d1d 100755 --- a/infrastructure/monitoring/kibana/setup-config.sh +++ b/infrastructure/monitoring/kibana/setup-config.sh @@ -14,14 +14,14 @@ set -e docker_command="docker run --rm -v /opt/opencrvs/infrastructure/monitoring/kibana/config.ndjson:/config.ndjson --network=opencrvs_overlay_net curlimages/curl" # First delete all alerts. This is because the import doesn't remove alerts that are no longer in the config -$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do - $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id" +$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do + $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id" done -$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null +$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null # Re-enable all alerts. This is because after importing a config, all alerts are disabled by default -$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do - $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_disable" - $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_enable" +$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do + $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_disable" + $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_enable" done \ No newline at end of file diff --git a/infrastructure/server-setup/backups.yml b/infrastructure/server-setup/backups.yml index fa8c4ff0a..ca6b9d31f 100644 --- a/infrastructure/server-setup/backups.yml +++ b/infrastructure/server-setup/backups.yml @@ -87,16 +87,6 @@ vars: manager_hostname: "{{ groups['docker-manager-first'][0] }}" tasks: - - name: Ensure backup user is present - user: - name: '{{ external_backup_server_user }}' - state: present - create_home: true - home: '/home/{{ external_backup_server_user }}' - shell: /bin/bash - tags: - - backups - - set_fact: external_backup_server_user_home: '/home/{{ external_backup_server_user }}' tags: diff --git a/infrastructure/server-setup/playbook.yml b/infrastructure/server-setup/playbook.yml index 74376cc44..b12b2c4c3 100644 --- a/infrastructure/server-setup/playbook.yml +++ b/infrastructure/server-setup/playbook.yml @@ -34,7 +34,9 @@ - hosts: docker-manager-first, docker-workers become: yes - become_method: sudo + become_method: sudo + vars: + crontab_user: root tasks: - include_tasks: file: tasks/application.yml diff --git a/infrastructure/server-setup/staging.yml b/infrastructure/server-setup/staging.yml index ab7590dd5..1d34f18ec 100644 --- a/infrastructure/server-setup/staging.yml +++ b/infrastructure/server-setup/staging.yml @@ -15,7 +15,6 @@ all: - 165.22.110.53 enable_backups: false periodic_restore_from_backup: true - # restore_backup_encryption_passphrase: Defined in --extra-vars by the provisioning pipeline # external_backup_server_ssh_port: Defined in --extra-vars by the provisioning pipeline # external_backup_server_ip: Defined in --extra-vars by the provisioning pipeline users: diff --git a/infrastructure/server-setup/tasks/backups/crontab.yml b/infrastructure/server-setup/tasks/backups/crontab.yml index e1424628e..7527efea2 100644 --- a/infrastructure/server-setup/tasks/backups/crontab.yml +++ b/infrastructure/server-setup/tasks/backups/crontab.yml @@ -16,10 +16,10 @@ periodic_restore_from_backup: false when: periodic_restore_from_backup is not defined -- name: Throw an error if periodic_restore_from_backup is true but restore_backup_encryption_passphrase is not defined +- name: Throw an error if periodic_restore_from_backup is true but backup_encryption_passphrase is not defined fail: - msg: 'Error: restore_backup_encryption_passphrase is not defined. It usually means you haven't set RESTORE_BACKUP_ENCRYPTION_PASSPHRASE in your staging environments secrets' - when: periodic_restore_from_backup and restore_backup_encryption_passphrase is not defined + msg: 'Error: backup_encryption_passphrase is not defined. It usually means you have not set backup_encryption_passphrase in your staging environments secrets' + when: periodic_restore_from_backup and backup_encryption_passphrase is not defined - name: 'Setup crontab to download a backup periodically the opencrvs data' cron: @@ -27,8 +27,8 @@ name: 'download opencrvs backup' minute: '30' hour: '0' - job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ restore_backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1' - state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}" + job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1' + state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}" - name: 'Setup crontab to restore the opencrvs data' cron: @@ -37,4 +37,4 @@ minute: '0' hour: '1' job: 'cd / && bash /opt/opencrvs/infrastructure/backups/restore.sh --replicas=1 >> /var/log/opencrvs-restore.log 2>&1' - state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}" + state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}" diff --git a/infrastructure/server-setup/tasks/updates.yml b/infrastructure/server-setup/tasks/updates.yml index 098149e63..2317bbde1 100644 --- a/infrastructure/server-setup/tasks/updates.yml +++ b/infrastructure/server-setup/tasks/updates.yml @@ -25,6 +25,9 @@ content: | Unattended-Upgrade::Package-Blacklist {}; Unattended-Upgrade::DevRelease "auto"; + Unattended-Upgrade::Remove-Unused-Kernel-Packages "true"; + Unattended-Upgrade::Remove-New-Unused-Dependencies "true"; + Unattended-Upgrade::Remove-Unused-Dependencies "true"; Unattended-Upgrade::Allowed-Origins { "${distro_id}:${distro_codename}-security"; }; diff --git a/infrastructure/setup-deploy-config.sh b/infrastructure/setup-deploy-config.sh index f9d8bacfd..6ce09263d 100755 --- a/infrastructure/setup-deploy-config.sh +++ b/infrastructure/setup-deploy-config.sh @@ -26,6 +26,19 @@ done KIBANA_ENCRYPTION_KEY=`uuidgen` sed -i "s/{{KIBANA_ENCRYPTION_KEY}}/$KIBANA_ENCRYPTION_KEY/g" /opt/opencrvs/infrastructure/monitoring/kibana/kibana.yml +# Move metabase file +mv /opt/opencrvs/infrastructure/metabase.init.db.sql /data/metabase/metabase.init.db.sql + +# Replace environment variables from all alert definition files +for file in /opt/opencrvs/infrastructure/monitoring/elastalert/rules/*.yaml; do + sed -i -e "s%{{HOST}}%$1%" $file + sed -i -e "s%{{SMTP_HOST}}%$SMTP_HOST%" $file + sed -i -e "s%{{SMTP_PORT}}%$SMTP_PORT%" $file + sed -i -e "s%{{ALERT_EMAIL}}%$ALERT_EMAIL%" $file + sed -i -e "s%{{SENDER_EMAIL_ADDRESS}}%$SENDER_EMAIL_ADDRESS%" $file + sed -i -e "s%{{DOMAIN}}%$DOMAIN%" $file +done + sed -i -e "s%{{MINIO_ROOT_USER}}%$MINIO_ROOT_USER%" /opt/opencrvs/infrastructure/mc-config/config.json sed -i -e "s%{{MINIO_ROOT_PASSWORD}}%$MINIO_ROOT_PASSWORD%" /opt/opencrvs/infrastructure/mc-config/config.json