Add a CI job to build our container images

.github/workflows/ci.yml

@@ -0,0 +1,146 @@
+name: "CI"
+on:
+  push:
+    tags:
+      - "[0-9]+.[0-9]+.[0-9]+"
+      - "[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"
+    branches: [main]
+  pull_request:
+  workflow_dispatch: # Manually
+env:
+  REGISTRY: ghcr.io/noaa-gsl/vxingest
+
+jobs:
+  build-ingest:
+    name: Build Ingest image
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Generate image metadata
+        uses: docker/metadata-action@v4
+        id: meta
+        with:
+          images: |
+            ghcr.io/noaa-gsl/vxingest/ingest
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+            # set latest tag for default branch
+            type=raw,value=latest,enable={{is_default_branch}}
+          labels: |
+            org.opencontainers.image.vendor=NOAA's Global Systems Laboratory
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build image and push
+        uses: docker/build-push-action@v5
+        with:
+          file: docker/ingest/Dockerfile
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
+          push: true
+          # Use a cache to speed up builds
+          # Note: this may cause security issues if the apt updates are cached
+          # It may make more sense to build eccodes as its own image instead.
+          cache-from: type=registry,ref=ghcr.io/noaa-gsl/vxingest/cache/ingest:buildcache
+          cache-to: type=registry,ref=ghcr.io/noaa-gsl/vxingest/cache/ingest:buildcache,mode=max
+  build-import:
+    name: Build Import image
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Generate image metadata
+        uses: docker/metadata-action@v4
+        id: meta
+        with:
+          images: |
+            ghcr.io/noaa-gsl/vxingest/import
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+            # set latest tag for default branch
+            type=raw,value=latest,enable={{is_default_branch}}
+          labels: |
+            org.opencontainers.image.vendor=NOAA's Global Systems Laboratory
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build image and push
+        uses: docker/build-push-action@v5
+        with:
+          file: docker/import/Dockerfile
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
+          push: true
+  scan-ingest:
+    runs-on: ubuntu-latest
+    needs: build-ingest
+    steps:
+      - name: Extract SHORT_SHA
+        run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> "$GITHUB_ENV"
+      - name: Scan image with Trivy
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: "ghcr.io/noaa-gsl/vxingest/ingest:sha-${{ env.SHORT_SHA }}"
+          format: "sarif"
+          output: "trivy-ingest-results.sarif"
+          ignore-unfixed: true
+          severity: "CRITICAL,HIGH"
+          limit-severities-for-sarif: true
+          exit-code: "1"
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload Trivy scan results to GitHub Security tab
+        if: always()
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-ingest-results.sarif"
+  scan-import:
+    runs-on: ubuntu-latest
+    needs: build-import
+    steps:
+      - name: Extract SHORT_SHA
+        run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> "$GITHUB_ENV"
+      - name: Scan image with Trivy
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: "ghcr.io/noaa-gsl/vxingest/import:sha-${{ env.SHORT_SHA }}"
+          format: "sarif"
+          output: "trivy-import-results.sarif"
+          ignore-unfixed: true
+          severity: "CRITICAL,HIGH"
+          limit-severities-for-sarif: true
+          # exit-code: "1" # FIXME: allow failures for now. Couchbase needs to update cbtools
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload Trivy scan results to GitHub Security tab
+        if: always()
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-import-results.sarif"

docker/import/Dockerfile

@@ -30,9 +30,13 @@ RUN groupadd --gid ${ID} amb-verif && \
 
 WORKDIR /app
 
+# Install the couchbase tools
+ENV CB_VERSION=7.2.2
+RUN curl -L https://packages.couchbase.com/releases/${CB_VERSION}/couchbase-server-tools_${CB_VERSION}-linux_$(uname -m).tar.gz | tar xz --directory /usr/local
+
 # Copy the scripts and metadata dirs so the import script can run
-COPY ./scripts/ /app/
-COPY ./mats_metadata_and_indexes /app/
+COPY ./scripts /app/scripts
+COPY ./mats_metadata_and_indexes /app/mats_metadata_and_indexes
 
 # TODO - install the cbtools directly and remove from the git repo
 # See: https://docs.couchbase.com/cloud/reference/command-line-tools.html#download-and-install-the-couchbase-command-line-tools

scripts/VXingest_utilities/import_docs.sh

@@ -98,7 +98,7 @@ do_import() {
   sleep 10
   cat ${file_list} | while read f; do
     echo "cbimport json --cluster couchbase://${host} --bucket ${bucket}  --scope-collection-exp ${scope}.${collection} --username ${user} --password ${pwd} --format list --generate-key %id% --dataset file:///${f}"
-    ${curdir}/scripts/cbtools/bin/cbimport json --cluster couchbase://${host} --bucket ${bucket}  --scope-collection-exp ${scope}.${collection} --username ${user} --password ${pwd} --format list --generate-key %id% --dataset file:///${f}
+    cbimport json --cluster couchbase://${host} --bucket ${bucket}  --scope-collection-exp ${scope}.${collection} --username ${user} --password ${pwd} --format list --generate-key %id% --dataset file:///${f}
   done
 }