Collect Daily Stats in SSP #383
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Collect Daily Stats in SSP | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| record_stats: | |
| name: Collect Daily SSP Stats | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Set up env | |
| run: | | |
| python -m pip install requests | |
| # Set up AWS CLI | |
| export AWSCLI_SSP_ACCESS=${{ secrets.AWSCLI_SSP_ACCESS }} | |
| export AWSCLI_SSP_SECRET=${{ secrets.AWSCLI_SSP_SECRET }} | |
| python -m pip install awscli | |
| mkdir ~/.aws | |
| echo "[default]" > ~/.aws/config | |
| echo "region = us-east-2" >> ~/.aws/config | |
| echo "[default]" > ~/.aws/credentials | |
| echo "aws_access_key_id = $AWSCLI_SSP_ACCESS" >> ~/.aws/credentials | |
| echo "aws_secret_access_key = $AWSCLI_SSP_SECRET" >> ~/.aws/credentials | |
| - name: Clone zap-mgmt-scripts | |
| run: | | |
| # Setup git details | |
| export GITHUB_USER=zapbot | |
| export GITHUB_TOKEN=${{ secrets.ZAPBOT_TOKEN }} | |
| git config --global user.email "[email protected]" | |
| git config --global user.name $GITHUB_USER | |
| git clone https://github.com/$GITHUB_USER/zap-mgmt-scripts.git | |
| cp -R zap-mgmt-scripts/ master | |
| cp -R zap-mgmt-scripts/ gh-pages | |
| cd gh-pages | |
| git checkout gh-pages | |
| git remote set-url origin https://$GITHUB_USER:[email protected]/$GITHUB_USER/zap-mgmt-scripts.git | |
| cd .. | |
| - name: Set envs | |
| run: | | |
| # c/o https://stackoverflow.com/questions/13168463/using-date-command-to-get-previous-current-and-next-month | |
| echo "START_MONTH=$(date +%Y-%m-1)" >> $GITHUB_ENV | |
| echo "THIS_MONTH=$(date -d '${{ env.START_MONTH }} -0 month' +%Y-%m)" >> $GITHUB_ENV | |
| echo "LAST_MONTH=$(date -d '${{ env.START_MONTH }} -1 month' +%Y-%m)" >> $GITHUB_ENV | |
| - name: Sync S3 files | |
| run: | | |
| mkdir project-zap | |
| mkdir project-zap/stats | |
| # Just sync the last 2 months worth of files | |
| echo "This month: ${{ env.THIS_MONTH }} last month: ${{ env.LAST_MONTH }}" | |
| aws s3 sync s3://ssp-project-zap/stats/ project-zap/stats/ --exclude "*" --include "*${{ env.THIS_MONTH }}*" --include "*${{ env.LAST_MONTH }}*" | |
| - name: Collect todays stats | |
| run: | | |
| export BITLY_TOKEN=${{ secrets.BITLY_TOKEN }} | |
| cd master/stats | |
| python3 stats.py collect | |
| # TODO raise an issue if not empty | |
| echo "Any errors?" | |
| cat errors.txt | |
| cd ../.. | |
| - name: Daily post process | |
| run: | | |
| cd master/stats | |
| python3 stats.py daily | |
| cd ../.. | |
| - name: Update S3 files | |
| run: | | |
| aws s3 sync project-zap/stats/ s3://ssp-project-zap/stats/ |