archive: use data downloader queue #1078
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: scan-vulns-deps | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - '**' | |
| jobs: | |
| build-and-deploy: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| BRANCH_NAME: ${{ steps.myvars.outputs.BRANCH_NAME }} | |
| GIT_HASH_SHORT: ${{ steps.myvars.outputs.GIT_HASH_SHORT }} | |
| DATE_IN_SECS: ${{ steps.myvars.outputs.DATE_IN_SECS }} | |
| CONTAINER_TAG: ${{ steps.myvars.outputs.CONTAINER_TAG }} | |
| SHORT_ENV_OUT: ${{ steps.myvars.outputs.SHORT_ENV_OUT }} | |
| CONTAINER_NAME: ${{ steps.myvars.outputs.CONTAINER_NAME }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set myvars | |
| id: myvars | |
| run: | | |
| branchname=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '-' ) | |
| dateinsecs=$(date +%s) | |
| githashshort=$(git rev-parse --short HEAD) | |
| echo "BRANCH_NAME=$branchname" >> $GITHUB_OUTPUT | |
| echo "GIT_HASH_SHORT=$githashshort" >> $GITHUB_OUTPUT | |
| echo "DATE_IN_SECS=$dateinsecs" >> $GITHUB_OUTPUT | |
| if [ "$branchname" = "develop" ]; then | |
| echo "CURRENT_ENVIRONMENT=development" >> $GITHUB_OUTPUT | |
| echo "SHORT_ENV_OUT=DEV" >> $GITHUB_OUTPUT | |
| containertag="commit-race-$githashshort" | |
| elif [ "$branchname" = "main" ]; then | |
| echo "CURRENT_ENVIRONMENT=production" >> $GITHUB_OUTPUT | |
| echo "SHORT_ENV_OUT=PROD" >> $GITHUB_OUTPUT | |
| containertag="commit-race-$githashshort" | |
| else | |
| echo "BRANCH_NAME=test" >> $GITHUB_OUTPUT | |
| echo "CURRENT_ENVIRONMENT=testing" >> $GITHUB_OUTPUT | |
| echo "SHORT_ENV_OUT=TEST" >> $GITHUB_OUTPUT | |
| containertag="commit-$githashshort" | |
| fi | |
| echo "CONTAINER_NAME=vocdoni-node" >> $GITHUB_OUTPUT | |
| echo "CONTAINER_TAG=$containertag" >> $GITHUB_OUTPUT | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: '1.21' | |
| - name: Setup Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Push Docker image to ghcr.io | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| #push: true | |
| push: false | |
| tags: | | |
| ghcr.io/vocdoni/go-dvote:latest, | |
| ghcr.io/vocdoni/go-dvote:${{ steps.myvars.outputs.BRANCH_NAME }}, | |
| ghcr.io/vocdoni/go-dvote:commit-${{ steps.myvars.outputs.GIT_HASH_SHORT }}, | |
| ghcr.io/vocdoni/go-dvote:${{ steps.myvars.outputs.BRANCH_NAME }}-${{ steps.myvars.outputs.DATE_IN_SECS }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| #outputs: type=tar,dest=${{ steps.myvars.outputs.CONTAINER_NAME }}-${{ steps.myvars.outputs.CONTAINER_TAG }}-tar | |
| outputs: type=docker,dest=${{ steps.myvars.outputs.CONTAINER_NAME }}-${{ steps.myvars.outputs.CONTAINER_TAG }}-oci-tar | |
| - name: Push Docker image to ghcr.io (race enabled) | |
| uses: docker/build-push-action@v4 | |
| if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' | |
| with: | |
| context: . | |
| #push: true | |
| push: false | |
| build-args: | | |
| BUILDARGS=-race | |
| tags: | | |
| ghcr.io/vocdoni/go-dvote:latest-race, | |
| ghcr.io/vocdoni/go-dvote:${{ steps.myvars.outputs.BRANCH_NAME }}-race, | |
| ghcr.io/vocdoni/go-dvote:commit-${{ steps.myvars.outputs.GIT_HASH_SHORT }}, | |
| ghcr.io/vocdoni/go-dvote:${{ steps.myvars.outputs.BRANCH_NAME }}-race-${{ steps.myvars.outputs.DATE_IN_SECS }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| #outputs: type=tar,dest=${{ steps.myvars.outputs.CONTAINER_NAME }}-${{ steps.myvars.outputs.CONTAINER_TAG }}-tar | |
| outputs: type=docker,dest=${{ steps.myvars.outputs.CONTAINER_NAME }}-${{ steps.myvars.outputs.CONTAINER_TAG }}-oci-tar | |
| - name: Upload Container Img Tarball as Artifact | |
| uses: actions/upload-artifact@v3 | |
| if: success() || failure() | |
| with: | |
| name: ${{ steps.myvars.outputs.CONTAINER_NAME }}-${{ steps.myvars.outputs.CONTAINER_TAG }}-docker-img | |
| path: ${{ steps.myvars.outputs.CONTAINER_NAME }}-${{ steps.myvars.outputs.CONTAINER_TAG }}-oci-tar | |
| scan-vulns-repo: | |
| name: Scan Vulns in Repo | |
| runs-on: ubuntu-latest | |
| needs: [build-and-deploy] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: '1.21' | |
| - name: Scan in Repo (html) | |
| uses: aquasecurity/trivy-action@master | |
| if: success() || failure() | |
| with: | |
| scan-type: fs | |
| scanners: vuln,secret,config | |
| scan-ref: . | |
| format: template | |
| template: '@/contrib/html.tpl' | |
| output: trivy-results-repo-${{ needs.build-and-deploy.outputs.GIT_HASH_SHORT }}.html | |
| env: | |
| TRIVY_USERNAME: ${{ github.repository_owner }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Scan in Repo (sarif) | |
| uses: aquasecurity/trivy-action@master | |
| if: success() || failure() | |
| with: | |
| scan-type: fs | |
| scanners: vuln,secret,config | |
| scan-ref: . | |
| format: sarif | |
| output: trivy-results-repo-${{ needs.build-and-deploy.outputs.GIT_HASH_SHORT }}.sarif | |
| env: | |
| TRIVY_USERNAME: ${{ github.repository_owner }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Publish Repo Scan Results as Artifact | |
| uses: actions/upload-artifact@v3 | |
| if: success() || failure() | |
| with: | |
| name: trivy-results-repo-${{ needs.build-and-deploy.outputs.DATE_IN_SECS }} | |
| path: trivy-results-repo-${{ needs.build-and-deploy.outputs.GIT_HASH_SHORT }}.* | |
| - name: Load Repo Scan Results (sarif) to Github | |
| uses: github/codeql-action/upload-sarif@v2 | |
| if: always() | |
| #if: false ## false = bypass | |
| with: | |
| sarif_file: trivy-results-repo-${{ needs.build-and-deploy.outputs.GIT_HASH_SHORT }}.sarif | |
| category: vulns-in-repo | |
| scan-vulns-docker: | |
| name: Scan Vulns in Docker | |
| runs-on: ubuntu-latest | |
| needs: [build-and-deploy] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Download Container Img Tarball as Artifact | |
| uses: actions/download-artifact@v3 | |
| id: container_img_tar | |
| with: | |
| name: ${{ needs.build-and-deploy.outputs.CONTAINER_NAME }}-${{ needs.build-and-deploy.outputs.CONTAINER_TAG }}-docker-img | |
| path: _tmp/ | |
| - name: Check Container Image Tarball | |
| run: | | |
| cd _tmp/ | |
| mkdir _tar/ | |
| ls -la | |
| file ${{ needs.build-and-deploy.outputs.CONTAINER_NAME }}-${{ needs.build-and-deploy.outputs.CONTAINER_TAG }}-oci-tar | |
| ## we remove 'z' flag because file is not compressed (gz), only archived (tar) | |
| tar -xvf ${{ needs.build-and-deploy.outputs.CONTAINER_NAME }}-${{ needs.build-and-deploy.outputs.CONTAINER_TAG }}-oci-tar -C _tar/ | |
| ls -la _tar/ | |
| - name: Vuln scan in Docker (table) | |
| uses: aquasecurity/trivy-action@master | |
| if: always() | |
| with: | |
| scan-type: image | |
| scanners: vuln,secret,config | |
| ## it can be the dir with content of untar file or it can be the tar file | |
| input: _tmp/_tar/ | |
| ##input: _tmp/${{ needs.build-and-deploy.outputs.CONTAINER_NAME }}-${{ needs.build-and-deploy.outputs.CONTAINER_TAG }}-oci-tar | |
| format: table | |
| env: | |
| TRIVY_USERNAME: ${{ github.repository_owner }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Vuln scan in Docker (html) | |
| uses: aquasecurity/trivy-action@master | |
| if: always() | |
| with: | |
| scan-type: image | |
| scanners: vuln,secret,config | |
| ## it can be the dir with content of untar file or it can be the tar file | |
| ## input: _tmp/_tar/ | |
| input: _tmp/${{ needs.build-and-deploy.outputs.CONTAINER_NAME }}-${{ needs.build-and-deploy.outputs.CONTAINER_TAG }}-oci-tar | |
| format: template | |
| template: '@/contrib/html.tpl' | |
| output: trivy-results-docker-${{ needs.build-and-deploy.outputs.GIT_HASH_SHORT }}.html | |
| env: | |
| TRIVY_USERNAME: ${{ github.repository_owner }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Vuln scan in Docker (sarif) | |
| uses: aquasecurity/trivy-action@master | |
| if: always() | |
| with: | |
| scan-type: image | |
| scanners: vuln,secret,config | |
| ## it can be the dir with content of untar file or it can be the tar file | |
| ## input: _tmp/_tar/ | |
| input: _tmp/${{ needs.build-and-deploy.outputs.CONTAINER_NAME }}-${{ needs.build-and-deploy.outputs.CONTAINER_TAG }}-oci-tar | |
| format: sarif | |
| output: trivy-results-docker-${{ needs.build-and-deploy.outputs.GIT_HASH_SHORT }}.sarif | |
| env: | |
| TRIVY_USERNAME: ${{ github.repository_owner }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Publish Docker Scan Results as Artifact | |
| uses: actions/upload-artifact@v3 | |
| if: success() || failure() | |
| with: | |
| name: trivy-results-docker-${{ needs.build-and-deploy.outputs.DATE_IN_SECS }} | |
| path: trivy-results-docker-${{ needs.build-and-deploy.outputs.GIT_HASH_SHORT }}.* | |
| - name: Load Docker Scan Results (sarif) to Github | |
| uses: github/codeql-action/upload-sarif@v2 | |
| if: always() | |
| #if: false ## false = bypass | |
| with: | |
| sarif_file: trivy-results-docker-${{ needs.build-and-deploy.outputs.GIT_HASH_SHORT }}.sarif | |
| category: vulns-in-docker |