Merge pull request #6518 from pulibrary/6501-cog

Update GDAL command to use built COG output format

.circleci/Dockerfile

@@ -1,4 +1,4 @@
-FROM cimg/ruby:3.1.0-browsers
+FROM cimg/ruby:3.1.6-browsers
 
 RUN sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
  && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
  build:
  working_directory: ~/figgy
  docker:
- - image: pulibrary/ci-figgy:1.6
+ - image: ghcr.io/pulibrary/figgy:ci
  environment:
  RAILS_ENV: test
  FIGGY_DB_HOST: localhost
@@ -60,7 +60,7 @@ jobs:
  test:
  working_directory: ~/figgy
  docker:
- - image: pulibrary/ci-figgy:1.6
+ - image: ghcr.io/pulibrary/figgy:ci
  environment:
  RAILS_ENV: test
  FIGGY_DB_HOST: localhost
@@ -137,7 +137,7 @@ jobs:
  coverage_report:
  working_directory: ~/figgy
  docker:
- - image: pulibrary/ci-figgy:1.6
+ - image: ghcr.io/pulibrary/figgy:ci
  resource_class: large
  steps:
  - attach_workspace:
@@ -154,7 +154,7 @@ jobs:
  rubocop:
  working_directory: ~/figgy
  docker:
- - image: pulibrary/ci-figgy:1.6
+ - image: ghcr.io/pulibrary/figgy:ci
  environment:
  RAILS_ENV: test
  FIGGY_DB_HOST: localhost

.github/workflows/build-ci-docker.yml

@@ -0,0 +1,69 @@
+name: Create and publish a Docker CI image
+
+on:
+ push:
+ paths:
+ - .circleci/Dockerfile
+ - bin/**
+ branches:
+ - main
+ pull_request:
+ paths:
+ - .circleci/Dockerfile
+ - bin/**
+ branches:
+ - main
+ workflow_dispatch:
+
+env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+ build-and-push-image:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ tags: |
+ type=ref,event=branch
+ type=ref,event=pr
+ type=sha
+ env:
+ DOCKER_METADATA_PR_HEAD_SHA: true
+
+ - name: Build and push Docker image
+ if: (github.event_name == 'workflow_dispatch' || github.event_name == 'push') && github.ref == 'refs/heads/main'
+ uses: docker/build-push-action@v6
+ with:
+ context: "${{ github.workspace }}"
+ file: "${{ github.workspace }}/.circleci/Dockerfile"
+ push: true
+ tags: ghcr.io/pulibrary/figgy:ci
+ labels: ${{ steps.meta.outputs.labels }}
+
+ - name: Build and push Docker image
+ if: (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && github.ref != 'refs/heads/main'
+ uses: docker/build-push-action@v6
+ with:
+ context: "${{ github.workspace }}"
+ file: "${{ github.workspace }}/.circleci/Dockerfile"
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}

.github/workflows/prune-containers.yml

@@ -0,0 +1,19 @@
+name: Prune containers
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: "0 0 * * SUN" # every sunday at midnight
+
+jobs:
+ clean:
+ runs-on: ubuntu-latest
+ name: Delete old images
+ steps:
+ - uses: snok/[email protected]
+ with:
+ account: pulibrary
+ image-names: figgy
+ token: ${{ secrets.GITHUB_TOKEN }}
+ image-tags: "!ci !main"
+ cut-off: 2w

app/derivative_services/geo_derivatives/processors/gdal.rb

@@ -26,7 +26,7 @@ def self.translate(in_path, out_path, _options)
  # @param options [Hash] creation options
  def self.warp(in_path, out_path, options)
  execute "gdalwarp -q -t_srs #{options[:output_srid]} "\
- "\"#{in_path}\" #{out_path} -co TILED=YES -co COMPRESS=NONE"
+ "\"#{in_path}\" #{out_path} -co COMPRESS=NONE"
  end
 
  # Executes a gdal_translate command. Used to compress
@@ -46,13 +46,12 @@ def self.compress(in_path, out_path, options)
  # @param out_path [String] processor output file path
  # @param options [Hash] creation options
  def self.cloud_optimized_geotiff(in_path, out_path, _options)
- system("gdaladdo -q -r average \"#{in_path}\" 2 4 8 16 32")
- execute("gdal_translate -q -expand rgb \"#{in_path}\" #{out_path} -ot Byte -co TILED=YES "\
- "-a_nodata 256 -co COMPRESS=LZW -co COPY_SRC_OVERVIEWS=YES")
+ execute("gdal_translate -q -expand rgb \"#{in_path}\" #{out_path} -ot Byte -of COG "\
+ "-a_nodata 256 -co COMPRESS=LZW -co TILING_SCHEME=GoogleMapsCompatible")
  rescue StandardError
  # Try without expanding rgb
- execute("gdal_translate -q \"#{in_path}\" #{out_path} -ot Byte -co TILED=YES "\
- "-a_nodata 256 -co COMPRESS=LZW -co COPY_SRC_OVERVIEWS=YES")
+ execute("gdal_translate -q \"#{in_path}\" #{out_path} -ot Byte -of COG "\
+ "-a_nodata 256 -co COMPRESS=LZW -co TILING_SCHEME=GoogleMapsCompatible")
  end
 
  # Executes a gdal_rasterize command. Used to rasterize vector

spec/derivative_services/geo_derivatives/processors/base_geo_processor_spec.rb

@@ -73,7 +73,7 @@ def source_path; end
 
  describe "#warp" do
  it "executes a reproject command" do
- command = "gdalwarp -q -t_srs EPSG:4326 \"files/geo.tif\" output/geo.png -co TILED=YES -co COMPRESS=NONE"
+ command = "gdalwarp -q -t_srs EPSG:4326 \"files/geo.tif\" output/geo.png -co COMPRESS=NONE"
  processor.class.warp(file_name, output_file, options)
  expect(processor.class).to have_received(:execute).with command
  end