Post process the ZAP Telemetry data in SSP #8351
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Post process the ZAP Telemetry data in SSP | |
| on: | |
| schedule: | |
| - cron: '30 * * * *' | |
| workflow_dispatch: | |
| jobs: | |
| record_stats: | |
| name: Post process the ZAP Telemetry data | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Clone zap-mgmt-scripts | |
| run: | | |
| # Setup git details | |
| export GITHUB_USER=zapbot | |
| export GITHUB_TOKEN=${{ secrets.ZAPBOT_TOKEN }} | |
| git config --global user.email "[email protected]" | |
| git config --global user.name $GITHUB_USER | |
| git clone https://github.com/$GITHUB_USER/zap-mgmt-scripts.git | |
| - name: Sync the Stats | |
| run: | | |
| python -m pip install requests | |
| # Set up AWS CLI | |
| export AWSCLI_SSP_ACCESS=${{ secrets.AWSCLI_SSP_ACCESS }} | |
| export AWSCLI_SSP_SECRET=${{ secrets.AWSCLI_SSP_SECRET }} | |
| python -m pip install awscli | |
| mkdir ~/.aws | |
| echo "[default]" > ~/.aws/config | |
| echo "region = us-east-2" >> ~/.aws/config | |
| echo "[default]" > ~/.aws/credentials | |
| echo "aws_access_key_id = $AWSCLI_SSP_ACCESS" >> ~/.aws/credentials | |
| echo "aws_secret_access_key = $AWSCLI_SSP_SECRET" >> ~/.aws/credentials | |
| cd zap-mgmt-scripts/stats | |
| # The telemetry/raw directory should just contain unprocessed files so remain small | |
| # Move the current files so we don't delete any that come in while we are processing the current ones | |
| aws s3 mv s3://ssp-project-zap/telemetry/raw/ s3://ssp-project-zap/telemetry/process/ --recursive | |
| aws s3 sync s3://ssp-project-zap/telemetry/process/ telemetry/raw/ | |
| python3 tel_postprocess.py | |
| aws s3 mv telemetry/ s3://ssp-project-zap/telemetry/ --recursive | |
| aws s3 sync telemetry/raw/ s3://ssp-project-zap/telemetry/process/ --delete | |