diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..64273b9 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,90 @@ +name: Build & Test + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + branches: [main, develop] + push: + branches: [main, develop] + +jobs: + deploy: + if: github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main') + runs-on: ubuntu-latest + env: + SSH_ADDRESS: ${{ github.ref == 'refs/heads/main' && secrets.SSH_ADDRESS_PROD || secrets.SSH_ADDRESS }} + SSH_USER: ${{ github.ref == 'refs/heads/main' && secrets.SSH_USER_PROD || secrets.SSH_USER }} + SSH_KEY: ${{ github.ref == 'refs/heads/main' && secrets.SSH_KEY_PROD || secrets.SSH_KEY }} + DEPLOY_ENVIRONMENT: ${{ github.ref == 'refs/heads/main' && 'PRODUCTION' || 'STAGING' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Add SSH key + run: | + mkdir -p ~/.ssh + echo "${{ env.SSH_KEY }}" > ~/.ssh/id_rsa + chmod 700 ~/.ssh/id_rsa + - name: Deploy to server + run: | + echo "Attempting to deploy to $DEPLOY_ENVIRONMENT environment" + echo "Copying files to server located at $SSH_ADDRESS" + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "mkdir -p /home/${{ env.SSH_USER }}/t-rex-monitor" + scp -v -o StrictHostKeyChecking=no -r ./* ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }}:/home/${{ env.SSH_USER }}/t-rex-monitor + echo "Finished deploying to $DEPLOY_ENVIRONMENT environment" + - name: Move service files + run: | + echo "Moving Grafana service and config files to correct path" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/alertmanager.yml /etc/alertmanager/alertmanager.yml" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/ben-error-rule.yml /etc/prometheus/ben-error-rule.yml" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/loki-local-config.yml /etc/loki-local-config.yml" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/loki.service /etc/systemd/system/loki.service" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/node_exporter.service /etc/systemd/system/node_exporter.service" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/overview.json /var/lib/grafana/dashboards/overview.json" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/prometheus.service /etc/systemd/system/prometheus.service" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/prometheus.yml /etc/prometheus" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/promtail-local-config.yml /etc/promtail-local-config.yml" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/promtail.service /etc/systemd/system/promtail.service" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/grafana/main/pushgateway.service /etc/systemd/system/pushgateway.service" + + echo "Moving Ben scheduler service and config files to correct path" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/scheduler/jobs/ben-jobs.service /etc/systemd/system/ben-jobs.service" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/scheduler/jobs/ben-jobs.timer /etc/systemd/system/ben-jobs.timer" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/scheduler/supabase/ben-supabase.service /etc/systemd/system/ben-supabase.service" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/scheduler/supabase/ben-supabase.timer /etc/systemd/system/ben-supabase.timer" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/scheduler/logs/ben-logs.service /etc/systemd/system/ben-logs.service" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo mv /home/${{ env.SSH_USER }}/t-rex-monitor/scheduler/logs/ben-logs.timer /etc/systemd/system/ben-logs.timer" + + echo "Finished moving service and config files." + - name: Restart service and workers + run : | + echo "Restarting Grafana and scheduler services" + + echo "Restarting Grafana services" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo systemctl daemon-reload; sudo systemctl restart prometheus.service; sudo systemctl restart node_exporter.service; sudo systemctl restart pushgateway.service; sudo systemctl restart alertmanager.service; sudo systemctl restart loki.service; sudo systemctl restart promtail.service; sudo systemctl restart grafana-server.service;" + + echo "Restarting Ben scheduler services" + + ssh -o StrictHostKeyChecking=no ${{ env.SSH_USER }}@${{ env.SSH_ADDRESS }} "sudo systemctl daemon-reload; sudo systemctl restart ben-logs.service; sudo systemctl restart ben-logs.timer; sudo systemctl restart ben-supabase.service; sudo systemctl restart ben-supabase.timer; sudo systemctl restart ben-jobs.service; sudo systemctl restart ben-jobs.timer;" + + echo "Finished restarting services" + - name: Clean up SSH key + run: | + rm -f ~/.ssh/id_rsa \ No newline at end of file diff --git a/grafana/main/main_grafana_setup.sh b/grafana/main/main_grafana_setup.sh index 1f0e3ce..663f11c 100755 --- a/grafana/main/main_grafana_setup.sh +++ b/grafana/main/main_grafana_setup.sh @@ -100,7 +100,7 @@ sudo mv loki-linux-amd64 /usr/local/bin/loki loki --version sudo mkdir -p /data/loki -sudo cp loki-local-config.yaml /etc/loki-local-config.yaml +sudo cp loki-local-config.yml /etc/loki-local-config.yml sudo cp loki.service /etc/systemd/system/loki.service curl -O -L "https://github.com/grafana/loki/releases/download/v$PROMTAIL_VERSION/promtail-linux-amd64.zip" @@ -110,7 +110,7 @@ rm promtail-linux-amd64.zip sudo mv promtail-linux-amd64 /usr/local/bin/promtail promtail --version -sudo cp promtail-local-config.yaml /etc/promtail-local-config.yaml +sudo cp promtail-local-config.yml /etc/promtail-local-config.yml sudo cp promtail.service /etc/systemd/system/promtail.service sudo systemctl daemon-reload diff --git a/scheduler/jobs/ben-jobs.service b/scheduler/jobs/ben-jobs.service index c84399c..45adb97 100644 --- a/scheduler/jobs/ben-jobs.service +++ b/scheduler/jobs/ben-jobs.service @@ -5,7 +5,7 @@ Requires=grafana-server.service [Service] User=ubuntu -WorkingDirectory=/etc/ben/jobs +WorkingDirectory=/home/ubuntu/t-rex-monitor/scheduler/jobs Environment="PATH=/usr/local/bin:/usr/bin:/bin:/home/ubuntu/.conda/envs/webhook/bin" ExecStart=/usr/local/miniconda/bin/python3 node_util.py Restart=always diff --git a/scheduler/ben-logs.service b/scheduler/logs/ben-logs.service similarity index 68% rename from scheduler/ben-logs.service rename to scheduler/logs/ben-logs.service index 8088a0b..950f855 100644 --- a/scheduler/ben-logs.service +++ b/scheduler/logs/ben-logs.service @@ -4,7 +4,7 @@ After=network-online.target [Service] User=ubuntu -ExecStart=/bin/bash /etc/ben/error_counter.sh +ExecStart=/bin/bash /home/ubuntu/t-rex-monitor/scheduler/logs/error_counter.sh Type=simple Restart=on-failure diff --git a/scheduler/ben-logs.timer b/scheduler/logs/ben-logs.timer similarity index 100% rename from scheduler/ben-logs.timer rename to scheduler/logs/ben-logs.timer diff --git a/scheduler/error_counter.sh b/scheduler/logs/error_counter.sh similarity index 100% rename from scheduler/error_counter.sh rename to scheduler/logs/error_counter.sh diff --git a/scheduler/main_ben_setup.sh b/scheduler/main_ben_setup.sh index f486e68..8b50f51 100755 --- a/scheduler/main_ben_setup.sh +++ b/scheduler/main_ben_setup.sh @@ -1,6 +1,6 @@ #! /bin/bash # main_grafana_setup.sh needs to run before this -BEN_VERSION='2.14' +BEN_VERSION='2.16' # ben setup wget https://www.poirrier.ca/ben/ben-$BEN_VERSION.tar.gz @@ -14,8 +14,7 @@ sudo mkdir -p /etc/ben/jobs sudo mkdir -p /etc/ben/queue sudo mv ben/ben /etc/ben/ben -sudo cp node_util.py error_counter.sh /etc/ben/ -sudo cp ben-jobs.service ben-jobs.timer ben-logs.service ben-logs.timer /etc/systemd/system +sudo cp jobs/ben-jobs.service jobs/ben-jobs.timer supabase/ben-supabase.service supabase/ben-supabase.timer logs/ben-logs.service logs/ben-logs.timer /etc/systemd/system # start ben /etc/ben/ben server --snapshot /etc/ben/queue/ecopcr.ini -s /tmp/ben-ecopcr -d