Reverse sort file list and handle FileNotFound exceptions #225
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "VxIngest CI" | |
| on: | |
| push: | |
| paths-ignore: | |
| - "meta_update_middleware/**" | |
| tags: | |
| - "[0-9]+.[0-9]+.[0-9]+" | |
| - "[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" | |
| branches: [main] | |
| pull_request: | |
| paths-ignore: | |
| - "meta_update_middleware/**" | |
| workflow_dispatch: # Manually | |
| env: | |
| REGISTRY: ghcr.io/noaa-gsl/vxingest | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install poetry | |
| run: pipx install poetry | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| poetry env use 3.11 | |
| poetry install | |
| - name: Format with Ruff | |
| run: poetry run ruff format --check src tests | |
| - name: Lint with Ruff | |
| run: poetry run ruff check --output-format=github src tests | |
| test: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install poetry | |
| run: pipx install poetry | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| poetry env use 3.11 | |
| poetry install | |
| - name: Unit test with pytest | |
| run: | | |
| poetry run coverage run -m pytest -m "not integration" tests | |
| poetry run coverage report | |
| poetry run coverage xml | |
| - name: Code Coverage Report | |
| uses: irongut/[email protected] | |
| with: | |
| filename: coverage.xml | |
| badge: true | |
| fail_below_min: false | |
| hide_complexity: true | |
| format: markdown | |
| output: both | |
| thresholds: "60 80" | |
| - name: Add Coverage PR Comment | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| if: github.event_name == 'pull_request' | |
| with: | |
| recreate: true | |
| path: code-coverage-results.md | |
| - name: Add Coverage to Job Summary | |
| run: cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY | |
| build-ingest: | |
| name: Build Ingest image | |
| runs-on: ubuntu-latest | |
| needs: [ lint, test ] | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Generate image metadata | |
| uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: | | |
| ghcr.io/noaa-gsl/vxingest/ingest | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=sha | |
| # set latest tag for default branch | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| labels: | | |
| org.opencontainers.image.vendor=NOAA's Global Systems Laboratory | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - name: Login to GHCR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build image and push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| file: docker/ingest/Dockerfile | |
| context: . | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| build-import: | |
| name: Build Import image | |
| runs-on: ubuntu-latest | |
| needs: [ lint, test ] | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Generate image metadata | |
| uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: | | |
| ghcr.io/noaa-gsl/vxingest/import | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=sha | |
| # set latest tag for default branch | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| labels: | | |
| org.opencontainers.image.vendor=NOAA's Global Systems Laboratory | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - name: Login to GHCR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build image and push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| file: docker/import/Dockerfile | |
| context: . | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| scan-ingest: | |
| runs-on: ubuntu-latest | |
| needs: build-ingest | |
| steps: | |
| - uses: actions/checkout@v4 # Supply git info to upload-serif | |
| - name: Extract SHORT_SHA | |
| run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> "$GITHUB_ENV" | |
| - name: Scan image with Trivy | |
| uses: aquasecurity/[email protected] | |
| with: | |
| image-ref: "ghcr.io/noaa-gsl/vxingest/ingest:sha-${{ env.SHORT_SHA }}" | |
| format: "table" # Generate actionable output | |
| ignore-unfixed: true | |
| severity: "CRITICAL,HIGH" | |
| exit-code: "1" | |
| env: | |
| TRIVY_USERNAME: ${{ github.actor }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Generate Trivy output formatted for the GitHub Security tab | |
| if: always() | |
| uses: aquasecurity/[email protected] | |
| with: | |
| image-ref: "ghcr.io/noaa-gsl/vxingest/ingest:sha-${{ env.SHORT_SHA }}" | |
| format: "sarif" | |
| output: "trivy-ingest-results.sarif" | |
| ignore-unfixed: true | |
| severity: "CRITICAL,HIGH" | |
| limit-severities-for-sarif: true | |
| env: | |
| TRIVY_USERNAME: ${{ github.actor }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| if: always() | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: "trivy-ingest-results.sarif" | |
| scan-import: | |
| runs-on: ubuntu-latest | |
| needs: build-import | |
| steps: | |
| - uses: actions/checkout@v4 # Supply git info to upload-serif | |
| - name: Extract SHORT_SHA | |
| run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> "$GITHUB_ENV" | |
| - name: Scan image with Trivy | |
| uses: aquasecurity/[email protected] | |
| with: | |
| image-ref: "ghcr.io/noaa-gsl/vxingest/import:sha-${{ env.SHORT_SHA }}" | |
| format: "table" # Generate actionable output | |
| ignore-unfixed: true | |
| severity: "CRITICAL,HIGH" | |
| # exit-code: "1" # FIXME: allow failures for now. Couchbase needs to update cbtools | |
| env: | |
| TRIVY_USERNAME: ${{ github.actor }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Generate Trivy output formatted for the GitHub Security tab | |
| if: always() | |
| uses: aquasecurity/[email protected] | |
| with: | |
| image-ref: "ghcr.io/noaa-gsl/vxingest/import:sha-${{ env.SHORT_SHA }}" | |
| format: "sarif" | |
| output: "trivy-import-results.sarif" | |
| ignore-unfixed: true | |
| severity: "CRITICAL,HIGH" | |
| limit-severities-for-sarif: true | |
| env: | |
| TRIVY_USERNAME: ${{ github.actor }} | |
| TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| if: always() | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: "trivy-import-results.sarif" |