Merge/Sync Stable back into cwbi-dev

api/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine AS builder
+FROM golang:1.21-alpine AS builder
 # Install Git
 RUN apk update && apk add --no-cache ca-certificates
 # Copy In Source Code

api/go.mod

@@ -1,50 +1,51 @@
 module github.com/USACE/cumulus-api/api
 
-go 1.20
+go 1.21
+
+toolchain go1.21.0
 
 require (
 	github.com/USACE/go-simple-asyncer v0.0.0-20201015223104-446ae10887a8
-	github.com/aws/aws-sdk-go v1.47.1
+	github.com/aws/aws-sdk-go v1.54.13
 	github.com/btcsuite/btcutil v1.0.2
 	github.com/georgysavva/scany v1.2.1
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/uuid v1.4.0
 	github.com/gosimple/slug v1.13.1
-	github.com/jackc/pgx/v4 v4.18.1
+	github.com/jackc/pgx/v4 v4.18.3
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/labstack/echo-contrib v0.15.0
-	github.com/labstack/echo/v4 v4.11.2
-	golang.org/x/crypto v0.14.0
-	golang.org/x/net v0.17.0
+	github.com/labstack/echo-contrib v0.17.1
+	github.com/labstack/echo/v4 v4.12.0
+	golang.org/x/crypto v0.24.0
+	golang.org/x/net v0.26.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/gosimple/unidecode v1.0.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.14.1 // indirect
+	github.com/jackc/pgconn v1.14.3 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.3.2 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgtype v1.14.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.3 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/pgtype v1.14.3 // indirect
 	github.com/jackc/puddle v1.3.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/labstack/gommon v0.4.0 // indirect
+	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
-	github.com/prometheus/client_golang v1.17.0 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/prometheus/client_golang v1.19.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.53.0 // indirect
+	github.com/prometheus/procfs v0.13.0 // indirect
 	github.com/streadway/amqp v1.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )

api/models/download.go

@@ -73,6 +73,7 @@ type PackagerRequest struct {
 type Extent struct {
 	Name string    `json:"name"`
 	Bbox []float64 `json:"bbox"`
+	Srid float64   `json:"srid"`
 }
 
 // PackagerContentItem is a single item for Packager to include in output file
@@ -181,9 +182,12 @@ func GetDownloadPackagerRequest(db *pgxpool.Pool, downloadID *uuid.UUID) (*Packa
 		       json_build_object(
 				   'name', w.name,
 				   'bbox', ARRAY[
-					   ST_XMin(w.geometry),ST_Ymin(w.geometry),
-					   ST_XMax(w.geometry),ST_YMax(w.geometry)
-					]
+					   ST_XMin(ST_Transform(w.geometry,w.output_srid)),
+					   ST_YMin(ST_Transform(w.geometry,w.output_srid)),
+					   ST_XMax(ST_Transform(w.geometry,w.output_srid)),
+					   ST_YMax(ST_Transform(w.geometry,w.output_srid))
+					],
+					'srid', w.output_srid
 			   ) AS extent,
 			   CONCAT(
 				   'cumulus/download/', f.abbreviation,

async_geoprocess/Dockerfile

@@ -1,20 +1,15 @@
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.7.0
-
-# Note: 
-# 1) The apt-get upgrade below is due to the latest gdal:ubuntu-full-3.7.0
-#    base image being slightly out of date with patching.
-# 2) The individual lines below to install/upgrade was to get
-#    the packages to properly update and the docker vulnerabilities
-#    to be resolved.
-
-RUN apt-get update -y && apt-get upgrade -y \
-  && apt-get install -y python3-pip git \
-  && rm -rf /var/lib/apt/lists/* \
-  && python3 -m pip install --no-cache-dir --upgrade pip \
-  && python3 -m pip install --no-cache-dir --upgrade setuptools \
-  && python3 -m pip install --no-cache-dir --upgrade wheel \
-  && python3 -m pip install --no-cache-dir --upgrade pillow \
-  && python3 -m pip install --no-cache-dir --upgrade numpy
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.9.1
+
+RUN apt-get update -y && apt-get install -y \
+  git \
+  python3-pip \
+  python3-venv \
+  python3-dev \
+  postgresql-server-dev-all \
+  libhdf5-dev \
+  zlib1g-dev \
+  libnetcdf-dev \
+  && rm -rf /var/lib/apt/lists/*
 
 RUN useradd appuser
 
@@ -29,7 +24,7 @@ ENV PYTHONUNBUFFERED=1
 ENV GDAL_DATA=/usr/share/gdal
 
 # pip install the cumulus package
-RUN python3 -m pip install $CUMULUS_GEOPROC_PIP_URL
+RUN python3 -m pip install --break-system-packages $CUMULUS_GEOPROC_PIP_URL
 
 WORKDIR /app/async_geoprocess
 

async_listener/Dockerfile

@@ -1,5 +1,5 @@
 # BUILD GO BINARIES
-FROM golang:1.19-alpine AS builder
+FROM golang:1.21-alpine AS builder
 # Install Git
 RUN apk update && apk add --no-cache git ca-certificates
 # Copy In Source Code

async_listener/go.mod

@@ -9,7 +9,7 @@ require (
 )
 
 require (
-	github.com/aws/aws-sdk-go v1.44.290 // indirect
+	github.com/aws/aws-sdk-go v1.54.13 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/streadway/amqp v1.1.0 // indirect
 )

async_listener/go.sum

@@ -1,8 +1,8 @@
 github.com/USACE/go-simple-asyncer v0.0.0-20201015223104-446ae10887a8 h1:ZiPYYp2OgNoJ489M5jiyqt23LV3yQ9w+ZiqiswQLWP8=
 github.com/USACE/go-simple-asyncer v0.0.0-20201015223104-446ae10887a8/go.mod h1:2Zftz61ghmwOivA7LUAWt8rUV1CXmHuKYYvK2bFSxMY=
 github.com/aws/aws-sdk-go v1.35.7/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
-github.com/aws/aws-sdk-go v1.44.290 h1:Md4+os9DQtJjow0lWLMzeJljsimD+XS2xwwHDr5Z+Lk=
-github.com/aws/aws-sdk-go v1.44.290/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.54.13 h1:zpCuiG+/mFdDY/klKJvmSioAZWk45F4rLGq0JWVAAzk=
+github.com/aws/aws-sdk-go v1.54.13/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@@ -20,36 +20,10 @@ github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1Sd
 github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM=
 github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

async_packager/src/cumulus_packager/heclib/__init__.py

@@ -60,6 +60,7 @@
 AXIS["Easting",EAST],AXIS["Northing",NORTH]]'
 """str: UTM WKT"""
 
+
 # ProjectionDatum
 class ProjectionDatum(Enum):
     UNDEFINED_PROJECTION_DATUM = 0
@@ -79,6 +80,7 @@ class CompressionMethod(Enum):
 
 compression_method = {i.name: i.value for i in CompressionMethod}
 
+
 # StorageDataType
 class StorageDataType(Enum):
     GRID_FLOAT = 0
@@ -89,6 +91,7 @@ class StorageDataType(Enum):
 
 storage_data_type = {i.name: i.value for i in StorageDataType}
 
+
 # DataType
 class DataType(Enum):
     PER_AVER = 0
@@ -101,42 +104,49 @@ class DataType(Enum):
 
 data_type = {i.name.replace("_", "-"): i.value for i in DataType}
 
+
 # GridStructVersion
 class GridStructVersion(Enum):
     VERSION_100 = -100
 
 
 grid_struct_version = {i.name: i.value for i in GridStructVersion}
 
+
 # DssGridType
 class DssGridType(Enum):
     UNDEFINED_GRID_TYPE = 400
     HRAP = 410
     SHG = ALBERS = 420
     SPECIFIED_GRID_TYPE = 430
+    UTM6N = 430
 
 
 dss_grid_type = {i.name: i.value for i in DssGridType}
 
+
 # DssGridTypeName
 class DssGridTypeName(Enum):
     HRAP = "HRAP"
     SHG = "ALBERS"
-    UTM = "UMT%s%s"
+    UTM = "UTM%s%s"
 
 
 dss_grid_type_name = {i.name: i.value for i in DssGridTypeName}
 
+
 # SpatialRefereceDefinition
 class SpatialReferenceDefinition(Enum):
     UNDEFINED_GRID_TYPE = None
     HRAP = HRAP_SRC_DEFINITION
     SHG = ALBERS = SHG_SRC_DEFINITION
     SPECIFIED_GRID_TYPE = None
+    UTM6N = UTM_SRC_DEFINITION % ("6", "N", "-147", "0")
 
 
 spatial_reference_definition = {i.name: i.value for i in SpatialReferenceDefinition}
 
+
 # TimeZones
 class TimeZone(Enum):
     GMT = UTC = 0
@@ -194,6 +204,7 @@ def __init__(self, *args, **kw):
         self._nullValue = UNDEFINED
         super().__init__(*args, **kw)
 
+
 def zwrite_record(
     dssfilename: str,
     gridStructStore: zStructSpatialGrid,

async_packager/src/cumulus_packager/packager/handler.py

@@ -85,14 +85,18 @@ def handle_message(payload_resp: namedtuple, dst: str):
 
     """
     logger.info(f'Handle message with plugin "{payload_resp.format}"')
+    extent = payload_resp.extent
+    logger.info(f"extent {extent}")
+    dst_srs = f'EPSG:{extent["srid"]}'
+    logger.info(f"output projection of grids will be {dst_srs}")
     result = pkg_writer(
         plugin=payload_resp.format,
         id=payload_resp.download_id,
         src=json.dumps(payload_resp.contents),
-        extent=json.dumps(payload_resp.extent),
+        extent=json.dumps(extent),
         dst=dst,
         cellsize=2000,
-        dst_srs="EPSG:5070",
+        dst_srs=dst_srs,
     )
 
     return result

async_packager/src/cumulus_packager/writers/dss7.py

@@ -1,6 +1,7 @@
 """DSS7 package writer
 
 """
+
 import json
 import os
 import sys
@@ -69,8 +70,15 @@ def writer(
     destination_srs.ImportFromEPSG(int(epsg_code))
 
     ###### this can go away when the payload has the resolution ######
-    grid_type_name = "SHG"
-    grid_type = heclib.dss_grid_type[grid_type_name]
+    if epsg_code == "26906":
+        grid_type_name = "UTM6N"
+        grid_type = 430
+    else:
+        grid_type_name = "SHG"
+        grid_type = heclib.dss_grid_type[grid_type_name]
+    logger.info(
+        f"grid type name {grid_type_name}",
+    )
     zcompression = heclib.compression_method["ZLIB_COMPRESSION"]
     srs_definition = heclib.spatial_reference_definition[grid_type_name]
     tz_name = "GMT"

docker-compose.minio.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 networks:
   default:
     name: cumulus-api_default

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3"
-
 services:
   cumulusdb:
     image: postgis/postgis:13-3.1