From d29072c19dadf1782db8ad666a5b471a581e3763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Tue, 7 May 2024 10:04:42 +0200 Subject: [PATCH 01/22] Remove not supporting raster extension as a non valid reason --- .../java/org/hortonmachine/gears/io/stac/HMStacAsset.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java index bcb7ea8ed..f6b35634c 100644 --- a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java +++ b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java @@ -49,9 +49,8 @@ public HMStacAsset( String id, JsonNode assetNode ) { } if (type.toLowerCase().contains("profile=cloud-optimized")) { JsonNode rasterBandNode = assetNode.get("raster:bands"); + assetUrl = assetNode.get("href").textValue(); if (rasterBandNode != null && !rasterBandNode.isEmpty()) { - assetUrl = assetNode.get("href").textValue(); - Iterator rbIterator = rasterBandNode.elements(); while( rbIterator.hasNext() ) { JsonNode rbNode = rbIterator.next(); @@ -64,9 +63,6 @@ public HMStacAsset( String id, JsonNode assetNode ) { resolution = resolNode.asDouble(); } } - } else { - isValid = false; - nonValidReason = "raster bands metadata missing"; } } else { isValid = false; From c7a1230cd476644249042ccf81c27ca1f71357a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Thu, 9 May 2024 10:29:24 +0200 Subject: [PATCH 02/22] Update test for raster:bands --- .../java/org/hortonmachine/gears/modules/TestStacAsset.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java index 4ad6a3952..1d8e3ca56 100644 --- a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java +++ b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java @@ -52,15 +52,14 @@ public void testCreateInvalidStacAssetNotACOG() throws JsonProcessingException { assertEquals("not a COG", asset.getNonValidReason()); } - public void testCreateInvalidStacAssetRasterBandsMetadataMissing() throws JsonProcessingException { + public void testCreateStacAssetRasterBandsMetadataMissingIsValid() throws JsonProcessingException { String assetJSON = "{\"title\":\"Band 1 (coastal) BOA reflectance\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"roles\":[\"data\"],\"gsd\":60,\"eo:bands\":[{\"name\":\"B01\",\"common_name\":\"coastal\",\"center_wavelength\":0.4439,\"full_width_half_max\":0.027}],\"href\":\"https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/33/S/VB/2021/2/S2B_33SVB_20210221_0_L2A/B01.tif\",\"proj:shape\":[1830,1830],\"proj:transform\":[60,0,399960,0,-60,4200000,0,0,1],\"raster:bands\":[]}"; ObjectMapper mapper = new ObjectMapper(); JsonNode node = mapper.readTree(assetJSON); HMStacAsset asset = new HMStacAsset("B01", node); - assertFalse(asset.isValid()); - assertEquals("raster bands metadata missing", asset.getNonValidReason()); + assertTrue(asset.isValid()); } } From 6136550786e90ff1888af1835ec4c795e16db5a2 Mon Sep 17 00:00:00 2001 From: Andrea Antonello Date: Fri, 10 May 2024 07:57:50 +0200 Subject: [PATCH 03/22] set version to snapshot --- apps/pom.xml | 4 ++-- dbs/pom.xml | 2 +- gears/pom.xml | 4 ++-- gui/pom.xml | 4 ++-- hmachine/pom.xml | 4 ++-- lesto/pom.xml | 4 ++-- modules/pom.xml | 4 ++-- pom.xml | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/pom.xml b/apps/pom.xml index 400b08272..15aa58895 100644 --- a/apps/pom.xml +++ b/apps/pom.xml @@ -7,7 +7,7 @@ org.hortonmachine hortonmachine - 0.10.8 + 0.10.9-SNAPSHOT hm-apps @@ -27,7 +27,7 @@ org.hortonmachine hm-gui - 0.10.8 + 0.10.9-SNAPSHOT org.mapsforge diff --git a/dbs/pom.xml b/dbs/pom.xml index d57d3ce8b..f812c7adb 100644 --- a/dbs/pom.xml +++ b/dbs/pom.xml @@ -7,7 +7,7 @@ org.hortonmachine hortonmachine - 0.10.8 + 0.10.9-SNAPSHOT ../pom.xml diff --git a/gears/pom.xml b/gears/pom.xml index f4e8844eb..810ac1e01 100644 --- a/gears/pom.xml +++ b/gears/pom.xml @@ -7,7 +7,7 @@ org.hortonmachine hortonmachine ../pom.xml - 0.10.8 + 0.10.9-SNAPSHOT hm-gears @@ -18,7 +18,7 @@ org.hortonmachine hm-dbs - 0.10.8 + 0.10.9-SNAPSHOT diff --git a/gui/pom.xml b/gui/pom.xml index 7f13aa4bd..b96413703 100644 --- a/gui/pom.xml +++ b/gui/pom.xml @@ -5,7 +5,7 @@ org.hortonmachine hortonmachine - 0.10.8 + 0.10.9-SNAPSHOT hm-gui @@ -16,7 +16,7 @@ org.hortonmachine hm-modules - 0.10.8 + 0.10.9-SNAPSHOT diff --git a/hmachine/pom.xml b/hmachine/pom.xml index 4821f8345..fc92dbfd2 100644 --- a/hmachine/pom.xml +++ b/hmachine/pom.xml @@ -6,7 +6,7 @@ org.hortonmachine hortonmachine - 0.10.8 + 0.10.9-SNAPSHOT ../pom.xml @@ -18,7 +18,7 @@ org.hortonmachine hm-gears - 0.10.8 + 0.10.9-SNAPSHOT diff --git a/lesto/pom.xml b/lesto/pom.xml index 08e595203..af39333ac 100644 --- a/lesto/pom.xml +++ b/lesto/pom.xml @@ -6,7 +6,7 @@ org.hortonmachine hortonmachine - 0.10.8 + 0.10.9-SNAPSHOT hm-lesto @@ -18,7 +18,7 @@ org.hortonmachine hm-hmachine - 0.10.8 + 0.10.9-SNAPSHOT diff --git a/modules/pom.xml b/modules/pom.xml index f86655a4e..958e69e84 100644 --- a/modules/pom.xml +++ b/modules/pom.xml @@ -7,7 +7,7 @@ org.hortonmachine hortonmachine - 0.10.8 + 0.10.9-SNAPSHOT hm-modules @@ -18,7 +18,7 @@ org.hortonmachine hm-lesto - 0.10.8 + 0.10.9-SNAPSHOT diff --git a/pom.xml b/pom.xml index 94cdf5776..27c3f018e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.hortonmachine hortonmachine - 0.10.8 + 0.10.9-SNAPSHOT pom The Horton Machine From 2dad14403684338ece75ad1a37e97627acdc1cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Thu, 16 May 2024 13:09:42 +0200 Subject: [PATCH 04/22] set bbox filter for STAC search --- .../gears/io/stac/HMStacCollection.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java index a6d7a919c..7c2898a03 100644 --- a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java +++ b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java @@ -9,6 +9,7 @@ import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.data.simple.SimpleFeatureCollection; import org.geotools.data.simple.SimpleFeatureIterator; +import org.geotools.filter.IllegalFilterException; import org.geotools.filter.text.cql2.CQL; import org.geotools.filter.text.cql2.CQLException; import org.geotools.geometry.jts.JTS; @@ -108,10 +109,26 @@ public HMStacCollection setTimestampFilter( Date startTimestamp, Date endTimesta public HMStacCollection setGeometryFilter( Geometry intersectionGeometry ) { if (search == null) search = new SearchQuery(); + else if (search.getBbox() != null) + throw new IllegalStateException("Cannot add intersects filter. Only one of either intersects or bbox may be specified"); search.setIntersects(intersectionGeometry); return this; } + /** + * Set the geometry bbox filter for search query + * @param bbox + * @return the current collection. + */ + public HMStacCollection setBboxFilter( double[] bbox ) { + if (search == null) + search = new SearchQuery(); + else if (search.getIntersects() != null) + throw new IllegalStateException("Cannot add intersects filter. Only one of either intersects or bbox may be specified"); + search.setBbox(bbox); + return this; + } + /** * Set cql filter for search query; * From 6219ebf1cc12925e1ec3cd8d7f6a98079609bb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Tue, 14 May 2024 16:44:06 +0200 Subject: [PATCH 05/22] use POST as default query method for STAC search --- .../gears/io/stac/HMStacCollection.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java index 7c2898a03..67cff709b 100644 --- a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java +++ b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java @@ -1,5 +1,6 @@ package org.hortonmachine.gears.io.stac; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -143,12 +144,21 @@ public HMStacCollection setCqlFilter( String cqlFilter ) throws CQLException { return this; } + private SimpleFeatureCollection queryFeatureCollection() throws IOException { + try { + return stacClient.search(search, STACClient.SearchMode.POST); + } catch (IOException e) { + pm.message("POST search query not supported by the endpoint. GET will be tried."); + } + return stacClient.search(search, STACClient.SearchMode.GET); + } + public List searchItems() throws Exception { if (search == null) search = new SearchQuery(); search.setCollections(Arrays.asList(getId())); - SimpleFeatureCollection fc = stacClient.search(search, STACClient.SearchMode.GET); + SimpleFeatureCollection fc = queryFeatureCollection(); SimpleFeatureIterator iterator = fc.features(); pm.beginTask("Extracting STAC items...", -1); List stacItems = new ArrayList<>(); From 7a32369354ab6efe97052523afb370e7d854677a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Tue, 14 May 2024 17:03:20 +0200 Subject: [PATCH 06/22] add tests for stac collection search query --- .../gears/modules/TestHMStacCollection.java | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 gears/src/test/java/org/hortonmachine/gears/modules/TestHMStacCollection.java diff --git a/gears/src/test/java/org/hortonmachine/gears/modules/TestHMStacCollection.java b/gears/src/test/java/org/hortonmachine/gears/modules/TestHMStacCollection.java new file mode 100644 index 000000000..a46019e95 --- /dev/null +++ b/gears/src/test/java/org/hortonmachine/gears/modules/TestHMStacCollection.java @@ -0,0 +1,78 @@ +package org.hortonmachine.gears.modules; + +import org.hortonmachine.gears.io.stac.HMStacCollection; +import org.hortonmachine.gears.io.stac.HMStacItem; +import org.hortonmachine.gears.io.stac.HMStacManager; +import org.hortonmachine.gears.libs.monitor.DummyProgressMonitor; +import org.hortonmachine.gears.utils.HMTestCase; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Polygon; + +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.List; + +public class TestHMStacCollection extends HMTestCase { + private GeometryFactory gf = new GeometryFactory(); + private DummyProgressMonitor pm = new DummyProgressMonitor(); + private HMStacManager manager; + + private Coordinate[] createBbox(double xmin, double ymin, double xmax, double ymax) { + Coordinate[] bboxCoordinates = new Coordinate[5]; + bboxCoordinates[0] = new Coordinate(xmin, ymin); + bboxCoordinates[1] = new Coordinate(xmin, ymax); + bboxCoordinates[2] = new Coordinate(xmax, ymax); + bboxCoordinates[3] = new Coordinate(xmax, ymin); + bboxCoordinates[4] = new Coordinate(xmin, ymin); + return bboxCoordinates; + } + + protected void setUp() throws Exception { + super.setUp(); + + String catalogUrl = "https://planetarycomputer.microsoft.com/api/stac/v1"; + manager = new HMStacManager(catalogUrl, pm); + manager.open(); + } + + public void testSearchCatalogWithValidSpatioTemporalFilters() throws Exception { + HMStacCollection collection = manager.getCollectionById("io-lulc-annual-v02"); + Instant start = Instant.ofEpochMilli(1514764800000L); // 2018-01-01 + Instant end = start.plus(Duration.ofDays(10)); + Polygon bboxPolygon = gf.createPolygon(createBbox(0.0, 0.0, 10.0, 10.0)); + + List items = collection.setTimestampFilter(Date.from(start), Date.from(end)) + .setGeometryFilter(bboxPolygon) + .searchItems(); + + assertTrue(!items.isEmpty()); + } + + public void testSearchCatalogWithInvalidTemporalFilters() throws Exception { + HMStacCollection collection = manager.getCollectionById("io-lulc-annual-v02"); + Instant start = Instant.ofEpochMilli(946688400000L); // 2000-01-01, a year with no data + Instant end = start.plus(Duration.ofDays(10)); + Polygon bboxPolygon = gf.createPolygon(createBbox(0.0, 0.0, 10.0, 10.0)); + + List items = collection.setTimestampFilter(Date.from(start), Date.from(end)) + .setGeometryFilter(bboxPolygon) + .searchItems(); + + assertTrue(items.isEmpty()); + } + + public void testSearchCatalogWithInvalidSpatialFilters() throws Exception { + HMStacCollection collection = manager.getCollectionById("io-lulc-annual-v02"); + Instant start = Instant.ofEpochMilli(1514764800000L); // 2018-01-01 + Instant end = start.plus(Duration.ofDays(10)); + Polygon bboxPolygon = gf.createPolygon(createBbox(190.0, 90.0, 200.0, 100.0)); // out of bounds + + List items = collection.setTimestampFilter(Date.from(start), Date.from(end)) + .setGeometryFilter(bboxPolygon) + .searchItems(); + + assertTrue(items.isEmpty()); + } +} From f50cfaa00e080428f06054b5e9d5deda60806956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Mon, 20 May 2024 16:16:25 +0200 Subject: [PATCH 07/22] test bbox tests for STAC search --- .../gears/modules/TestHMStacCollection.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/gears/src/test/java/org/hortonmachine/gears/modules/TestHMStacCollection.java b/gears/src/test/java/org/hortonmachine/gears/modules/TestHMStacCollection.java index a46019e95..5f15feba4 100644 --- a/gears/src/test/java/org/hortonmachine/gears/modules/TestHMStacCollection.java +++ b/gears/src/test/java/org/hortonmachine/gears/modules/TestHMStacCollection.java @@ -14,6 +14,8 @@ import java.util.Date; import java.util.List; +import static org.junit.Assert.assertThrows; + public class TestHMStacCollection extends HMTestCase { private GeometryFactory gf = new GeometryFactory(); private DummyProgressMonitor pm = new DummyProgressMonitor(); @@ -75,4 +77,31 @@ public void testSearchCatalogWithInvalidSpatialFilters() throws Exception { assertTrue(items.isEmpty()); } + + public void testSearchCatalogUsingBbox() throws Exception { + HMStacCollection collection = manager.getCollectionById("io-lulc-annual-v02"); + Instant start = Instant.ofEpochMilli(1514764800000L); // 2018-01-01 + Instant end = start.plus(Duration.ofDays(10)); + double[] bbox = {0.0, 0.0, 10.0, 10.0}; + + List items = collection.setTimestampFilter(Date.from(start), Date.from(end)) + .setBboxFilter(bbox) + .searchItems(); + + assertTrue(!items.isEmpty()); + } + + public void testSearchThrowsExceptionUsingBothBboxAndIntersect() throws Exception { + HMStacCollection collection = manager.getCollectionById("io-lulc-annual-v02"); + Instant start = Instant.ofEpochMilli(1514764800000L); // 2018-01-01 + Instant end = start.plus(Duration.ofDays(10)); + Polygon bboxPolygon = gf.createPolygon(createBbox(0.0, 0.0, 10.0, 10.0)); + double[] bbox = {0.0, 0.0, 10.0, 10.0}; + + assertThrows(IllegalStateException.class, () -> collection.setTimestampFilter(Date.from(start), Date.from(end)) + .setGeometryFilter(bboxPolygon) + .setBboxFilter(bbox) + .searchItems() + ); + } } From cc4f54ab89613830deb2478598127a47c5ec2cad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Thu, 16 May 2024 17:36:53 +0200 Subject: [PATCH 08/22] add rangereader-s3 dependency --- gears/pom.xml | 6 +++++- pom.xml | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/gears/pom.xml b/gears/pom.xml index 810ac1e01..cbf1bba75 100644 --- a/gears/pom.xml +++ b/gears/pom.xml @@ -145,7 +145,11 @@ it.geosolutions.imageio-ext imageio-ext-cog-rangereader-http - + + it.geosolutions.imageio-ext + imageio-ext-cog-rangereader-s3 + + org.geotools gt-imagemosaic diff --git a/pom.xml b/pom.xml index 27c3f018e..78dd1a67e 100644 --- a/pom.xml +++ b/pom.xml @@ -397,6 +397,11 @@ imageio-ext-cog-rangereader-http ${imageio.ext.version} + + it.geosolutions.imageio-ext + imageio-ext-cog-rangereader-s3 + ${imageio.ext.version} + joda-time From 287fb8a3c98c02de083023eae7c20d4530303429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Thu, 16 May 2024 17:38:42 +0200 Subject: [PATCH 09/22] read asset rasters from S3 if required --- .../gears/io/stac/HMStacAsset.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java index f6b35634c..f5c74a111 100644 --- a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java +++ b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java @@ -1,5 +1,6 @@ package org.hortonmachine.gears.io.stac; +import java.net.URI; import java.util.Iterator; import org.geotools.coverage.grid.GridCoverage2D; @@ -18,6 +19,8 @@ import it.geosolutions.imageioimpl.plugins.cog.CogImageReaderSpi; import it.geosolutions.imageioimpl.plugins.cog.CogSourceSPIProvider; import it.geosolutions.imageioimpl.plugins.cog.HttpRangeReader; +import it.geosolutions.imageioimpl.plugins.cog.S3RangeReader; +import it.geosolutions.imageioimpl.plugins.cog.RangeReader; /** * An asset from a stac item. @@ -87,22 +90,35 @@ public String toString() { return sb.toString(); } + private RangeReader createRangeReader(BasicAuthURI cogUri) { + if (assetUrl.startsWith("s3://")) { + return new S3RangeReader(cogUri.getUri(), CogImageReadParam.DEFAULT_HEADER_LENGTH); + } + return new HttpRangeReader(cogUri.getUri(), CogImageReadParam.DEFAULT_HEADER_LENGTH); + } + /** * Read the asset's coverage into a local raster. * * @param region and optional region to read from. * @param user an optional user in case of authentication. * @param password an optional password in case of authentication. - * @return the read raster from the asset's url.. + * @param awsRegion an optional value for the AWS region in case of S3 authentication. + * @return the read raster from the asset's url. * @throws Exception */ - public GridCoverage2D readRaster( RegionMap region, String user, String password ) throws Exception { + public GridCoverage2D readRaster( RegionMap region, String user, String password, String awsRegion ) throws Exception { BasicAuthURI cogUri = new BasicAuthURI(assetUrl, false); if (user != null && password != null) { cogUri.setUser(user); cogUri.setPassword(password); } - HttpRangeReader rangeReader = new HttpRangeReader(cogUri.getUri(), CogImageReadParam.DEFAULT_HEADER_LENGTH); + if (awsRegion != null) { + URI uri = cogUri.getUri(); + URI newUri = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(), "region=" + awsRegion, uri.getFragment()); + cogUri.setUri(newUri); + } + RangeReader rangeReader = createRangeReader(cogUri); CogSourceSPIProvider inputProvider = new CogSourceSPIProvider(cogUri, new CogImageReaderSpi(), new CogImageInputStreamSpi(), rangeReader.getClass().getName()); GeoTiffReader reader = new GeoTiffReader(inputProvider); @@ -117,7 +133,11 @@ public GridCoverage2D readRaster( RegionMap region, String user, String password } public GridCoverage2D readRaster( RegionMap region ) throws Exception { - return readRaster(region, null, null); + return readRaster(region, null, null, null); + } + + public GridCoverage2D readRaster( RegionMap region, String awsRegion ) throws Exception { + return readRaster(region, null, null, awsRegion); } public String getId() { From b17577d054c0aa048bdc59a34547cf25a8d8c0c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Wed, 22 May 2024 08:48:19 +0200 Subject: [PATCH 10/22] remove parameter for aws region --- .../gears/io/stac/HMStacAsset.java | 38 +++++-------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java index f5c74a111..96a389d0b 100644 --- a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java +++ b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java @@ -1,8 +1,9 @@ package org.hortonmachine.gears.io.stac; -import java.net.URI; -import java.util.Iterator; - +import com.fasterxml.jackson.databind.JsonNode; +import it.geosolutions.imageio.core.BasicAuthURI; +import it.geosolutions.imageio.plugins.cog.CogImageReadParam; +import it.geosolutions.imageioimpl.plugins.cog.*; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.gce.geotiff.GeoTiffReader; import org.hortonmachine.gears.libs.modules.HMConstants; @@ -11,20 +12,11 @@ import org.opengis.parameter.GeneralParameterValue; import org.opengis.referencing.crs.CoordinateReferenceSystem; -import com.fasterxml.jackson.databind.JsonNode; - -import it.geosolutions.imageio.core.BasicAuthURI; -import it.geosolutions.imageio.plugins.cog.CogImageReadParam; -import it.geosolutions.imageioimpl.plugins.cog.CogImageInputStreamSpi; -import it.geosolutions.imageioimpl.plugins.cog.CogImageReaderSpi; -import it.geosolutions.imageioimpl.plugins.cog.CogSourceSPIProvider; -import it.geosolutions.imageioimpl.plugins.cog.HttpRangeReader; -import it.geosolutions.imageioimpl.plugins.cog.S3RangeReader; -import it.geosolutions.imageioimpl.plugins.cog.RangeReader; +import java.util.Iterator; /** * An asset from a stac item. - * + * * @author Andrea Antonello (www.hydrologis.com) * */ @@ -99,25 +91,19 @@ private RangeReader createRangeReader(BasicAuthURI cogUri) { /** * Read the asset's coverage into a local raster. - * + * * @param region and optional region to read from. * @param user an optional user in case of authentication. * @param password an optional password in case of authentication. - * @param awsRegion an optional value for the AWS region in case of S3 authentication. * @return the read raster from the asset's url. - * @throws Exception + * @throws Exception */ - public GridCoverage2D readRaster( RegionMap region, String user, String password, String awsRegion ) throws Exception { + public GridCoverage2D readRaster( RegionMap region, String user, String password ) throws Exception { BasicAuthURI cogUri = new BasicAuthURI(assetUrl, false); if (user != null && password != null) { cogUri.setUser(user); cogUri.setPassword(password); } - if (awsRegion != null) { - URI uri = cogUri.getUri(); - URI newUri = new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), uri.getPath(), "region=" + awsRegion, uri.getFragment()); - cogUri.setUri(newUri); - } RangeReader rangeReader = createRangeReader(cogUri); CogSourceSPIProvider inputProvider = new CogSourceSPIProvider(cogUri, new CogImageReaderSpi(), new CogImageInputStreamSpi(), rangeReader.getClass().getName()); @@ -133,11 +119,7 @@ public GridCoverage2D readRaster( RegionMap region, String user, String password } public GridCoverage2D readRaster( RegionMap region ) throws Exception { - return readRaster(region, null, null, null); - } - - public GridCoverage2D readRaster( RegionMap region, String awsRegion ) throws Exception { - return readRaster(region, null, null, awsRegion); + return readRaster(region, null, null ); } public String getId() { From 05fdfc7a9af1c650f950a7e3b3f3e3b233a94f81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Wed, 22 May 2024 09:13:24 +0200 Subject: [PATCH 11/22] test readRaster from HTTP plus a disabled test for S3 --- .../gears/modules/TestStacAsset.java | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java index 1d8e3ca56..3e75c6404 100644 --- a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java +++ b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java @@ -3,10 +3,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.geotools.coverage.grid.GridCoverage2D; import org.hortonmachine.gears.io.stac.HMStacAsset; +import org.hortonmachine.gears.io.stac.HMStacUtils; import org.hortonmachine.gears.utils.HMTestCase; +import org.hortonmachine.gears.utils.RegionMap; public class TestStacAsset extends HMTestCase { + private ObjectMapper mapper = new ObjectMapper(); protected void setUp() throws Exception { @@ -17,7 +21,6 @@ protected void setUp() throws Exception { public void testCreateValidStacAsset() throws JsonProcessingException { String assetJSON = "{\"title\":\"Band 1 (coastal) BOA reflectance\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"roles\":[\"data\"],\"gsd\":60,\"eo:bands\":[{\"name\":\"B01\",\"common_name\":\"coastal\",\"center_wavelength\":0.4439,\"full_width_half_max\":0.027}],\"href\":\"https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/33/S/VB/2021/2/S2B_33SVB_20210221_0_L2A/B01.tif\",\"proj:shape\":[1830,1830],\"proj:transform\":[60,0,399960,0,-60,4200000,0,0,1],\"raster:bands\":[{\"data_type\":\"uint16\",\"spatial_resolution\":60,\"bits_per_sample\":15,\"nodata\":0,\"statistics\":{\"minimum\":1,\"maximum\":20567,\"mean\":2339.4759595597,\"stddev\":3026.6973619954,\"valid_percent\":99.83}}]}"; - ObjectMapper mapper = new ObjectMapper(); JsonNode node = mapper.readTree(assetJSON); HMStacAsset asset = new HMStacAsset("B01", node); @@ -32,7 +35,6 @@ public void testCreateValidStacAsset() throws JsonProcessingException { public void testCreateInvalidStacAssetTypeInformationNotAvailable() throws JsonProcessingException { String assetJSON = "{\"title\":\"Band 1 (coastal) BOA reflectance\",\"roles\":[\"data\"],\"gsd\":60,\"eo:bands\":[{\"name\":\"B01\",\"common_name\":\"coastal\",\"center_wavelength\":0.4439,\"full_width_half_max\":0.027}],\"href\":\"https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/33/S/VB/2021/2/S2B_33SVB_20210221_0_L2A/B01.tif\",\"proj:shape\":[1830,1830],\"proj:transform\":[60,0,399960,0,-60,4200000,0,0,1],\"raster:bands\":[{\"data_type\":\"uint16\",\"spatial_resolution\":60,\"bits_per_sample\":15,\"nodata\":0,\"statistics\":{\"minimum\":1,\"maximum\":20567,\"mean\":2339.4759595597,\"stddev\":3026.6973619954,\"valid_percent\":99.83}}]}"; - ObjectMapper mapper = new ObjectMapper(); JsonNode node = mapper.readTree(assetJSON); HMStacAsset asset = new HMStacAsset("B01", node); @@ -43,7 +45,6 @@ public void testCreateInvalidStacAssetTypeInformationNotAvailable() throws JsonP public void testCreateInvalidStacAssetNotACOG() throws JsonProcessingException { String assetJSON = "{\"title\":\"Band 1 (coastal) BOA reflectance\",\"type\":\"image/tiff;\",\"roles\":[\"data\"],\"gsd\":60,\"eo:bands\":[{\"name\":\"B01\",\"common_name\":\"coastal\",\"center_wavelength\":0.4439,\"full_width_half_max\":0.027}],\"href\":\"https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/33/S/VB/2021/2/S2B_33SVB_20210221_0_L2A/B01.tif\",\"proj:shape\":[1830,1830],\"proj:transform\":[60,0,399960,0,-60,4200000,0,0,1],\"raster:bands\":[{\"data_type\":\"uint16\",\"spatial_resolution\":60,\"bits_per_sample\":15,\"nodata\":0,\"statistics\":{\"minimum\":1,\"maximum\":20567,\"mean\":2339.4759595597,\"stddev\":3026.6973619954,\"valid_percent\":99.83}}]}"; - ObjectMapper mapper = new ObjectMapper(); JsonNode node = mapper.readTree(assetJSON); HMStacAsset asset = new HMStacAsset("B01", node); @@ -54,7 +55,6 @@ public void testCreateInvalidStacAssetNotACOG() throws JsonProcessingException { public void testCreateStacAssetRasterBandsMetadataMissingIsValid() throws JsonProcessingException { String assetJSON = "{\"title\":\"Band 1 (coastal) BOA reflectance\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"roles\":[\"data\"],\"gsd\":60,\"eo:bands\":[{\"name\":\"B01\",\"common_name\":\"coastal\",\"center_wavelength\":0.4439,\"full_width_half_max\":0.027}],\"href\":\"https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/33/S/VB/2021/2/S2B_33SVB_20210221_0_L2A/B01.tif\",\"proj:shape\":[1830,1830],\"proj:transform\":[60,0,399960,0,-60,4200000,0,0,1],\"raster:bands\":[]}"; - ObjectMapper mapper = new ObjectMapper(); JsonNode node = mapper.readTree(assetJSON); HMStacAsset asset = new HMStacAsset("B01", node); @@ -62,4 +62,28 @@ public void testCreateStacAssetRasterBandsMetadataMissingIsValid() throws JsonPr assertTrue(asset.isValid()); } + public void testReadRasterFromHTTP() throws Exception { + String assetJSON = "{\"title\":\"Band 1 (coastal) BOA reflectance\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"roles\":[\"data\"],\"gsd\":60,\"eo:bands\":[{\"name\":\"B01\",\"common_name\":\"coastal\",\"center_wavelength\":0.4439,\"full_width_half_max\":0.027}],\"href\":\"https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/33/S/VB/2021/2/S2B_33SVB_20210221_0_L2A/B01.tif\",\"proj:shape\":[1830,1830],\"proj:transform\":[60,0,399960,0,-60,4200000,0,0,1],\"raster:bands\":[{\"data_type\":\"uint16\",\"spatial_resolution\":60,\"bits_per_sample\":15,\"nodata\":0,\"statistics\":{\"minimum\":1,\"maximum\":20567,\"mean\":2339.4759595597,\"stddev\":3026.6973619954,\"valid_percent\":99.83}}]}"; + JsonNode node = mapper.readTree(assetJSON); + String assetId = "rainfall"; + HMStacAsset hmAsset = new HMStacAsset(assetId, node); + RegionMap regionMap = RegionMap.fromBoundsAndGrid(1640000.0, 1640200.0, 5140000.0, 5140160.0, 20, 16); + + GridCoverage2D grid = hmAsset.readRaster(regionMap); + + assertNotNull(grid); + } + + // need setup for AWS credentials in System + public void disabledTestReadRasterFromS3() throws Exception { + String assetJSON = "{\"href\":\"s3://sentinel-cogs/sentinel-s2-l2a-cogs/5/C/MK/2018/10/S2B_5CMK_20181020_0_L2A/B01.tif\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"title\":\"rainfall\",\"eo:bands\":[{\"name\":\"rainfall\"}],\"proj:epsg\":4326,\"proj:shape\":[1600,1500],\"proj:transform\":[0.05000000074505806,0,-20,0,-0.05000000074505806,40,0,0,1],\"roles\":[\"data\"]}},\"bbox\":[-20,-40.000001192092896,55.00000111758709,40],\"stac_extensions\":[\"https://stac-extensions.github.io/eo/v1.1.0/schema.json\",\"https://stac-extensions.github.io/projection/v1.1.0/schema.json\"],\"collection\":\"rainfall_chirps_monthly\"}"; + JsonNode node = mapper.readTree(assetJSON); + String assetId = "rainfall"; + HMStacAsset hmAsset = new HMStacAsset(assetId, node); + RegionMap regionMap = RegionMap.fromBoundsAndGrid(1640000.0, 1640200.0, 5140000.0, 5140160.0, 20, 16); + + GridCoverage2D grid = hmAsset.readRaster(regionMap); + + assertNotNull(grid); + } } From 3f1bee90843a7318dbc77a4d47358a7c4ebba427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Wed, 22 May 2024 14:28:11 +0200 Subject: [PATCH 12/22] test ReadRaster from S3 --- .../gears/modules/TestStacAsset.java | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java index 3e75c6404..e71625916 100644 --- a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java +++ b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java @@ -5,9 +5,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.geotools.coverage.grid.GridCoverage2D; import org.hortonmachine.gears.io.stac.HMStacAsset; -import org.hortonmachine.gears.io.stac.HMStacUtils; import org.hortonmachine.gears.utils.HMTestCase; import org.hortonmachine.gears.utils.RegionMap; +import org.junit.After; + +import static org.junit.Assert.assertThrows; public class TestStacAsset extends HMTestCase { private ObjectMapper mapper = new ObjectMapper(); @@ -74,8 +76,10 @@ public void testReadRasterFromHTTP() throws Exception { assertNotNull(grid); } - // need setup for AWS credentials in System - public void disabledTestReadRasterFromS3() throws Exception { + public void testReadRasterFromS3() throws Exception { + System.setProperty("aws.accessKeyId", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + System.setProperty("aws.secretKey", "AKIAIOSFODNN7EXAMPLE"); + System.setProperty("iio.s3.aws.region", "us-west-2"); String assetJSON = "{\"href\":\"s3://sentinel-cogs/sentinel-s2-l2a-cogs/5/C/MK/2018/10/S2B_5CMK_20181020_0_L2A/B01.tif\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"title\":\"rainfall\",\"eo:bands\":[{\"name\":\"rainfall\"}],\"proj:epsg\":4326,\"proj:shape\":[1600,1500],\"proj:transform\":[0.05000000074505806,0,-20,0,-0.05000000074505806,40,0,0,1],\"roles\":[\"data\"]}},\"bbox\":[-20,-40.000001192092896,55.00000111758709,40],\"stac_extensions\":[\"https://stac-extensions.github.io/eo/v1.1.0/schema.json\",\"https://stac-extensions.github.io/projection/v1.1.0/schema.json\"],\"collection\":\"rainfall_chirps_monthly\"}"; JsonNode node = mapper.readTree(assetJSON); String assetId = "rainfall"; @@ -86,4 +90,26 @@ public void disabledTestReadRasterFromS3() throws Exception { assertNotNull(grid); } + + public void testReadRasterFromS3FailBucketNotInRegion() throws Exception { + System.setProperty("aws.accessKeyId", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + System.setProperty("aws.secretKey", "AKIAIOSFODNN7EXAMPLE"); + System.setProperty("iio.s3.aws.region", "us-west-1"); // The bucket is not in this region + + String assetJSON = "{\"href\":\"s3://sentinel-cogs/sentinel-s2-l2a-cogs/5/C/MK/2018/10/S2B_5CMK_20181020_0_L2A/B01.tif\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"title\":\"rainfall\",\"eo:bands\":[{\"name\":\"rainfall\"}],\"proj:epsg\":4326,\"proj:shape\":[1600,1500],\"proj:transform\":[0.05000000074505806,0,-20,0,-0.05000000074505806,40,0,0,1],\"roles\":[\"data\"]}},\"bbox\":[-20,-40.000001192092896,55.00000111758709,40],\"stac_extensions\":[\"https://stac-extensions.github.io/eo/v1.1.0/schema.json\",\"https://stac-extensions.github.io/projection/v1.1.0/schema.json\"],\"collection\":\"rainfall_chirps_monthly\"}"; + JsonNode node = mapper.readTree(assetJSON); + String assetId = "rainfall"; + HMStacAsset hmAsset = new HMStacAsset(assetId, node); + RegionMap regionMap = RegionMap.fromBoundsAndGrid(1640000.0, 1640200.0, 5140000.0, 5140160.0, 20, 16); + + assertThrows(RuntimeException.class, () -> hmAsset.readRaster(regionMap) + ); + } + + @After + public void after(){ + System.clearProperty("aws.accessKeyId"); + System.clearProperty("aws.secretKey"); + System.clearProperty("iio.s3.aws.region"); + } } From 5a647d5e7dc0938f65fb572d1198c10cb8a312c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Mon, 27 May 2024 16:54:28 +0200 Subject: [PATCH 13/22] Get S3 credentials as parameters --- .../gears/io/stac/HMStacAsset.java | 39 ++++++++++++------- .../gears/io/stac/HMStacCollection.java | 19 ++++----- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java index 96a389d0b..68ec20a8a 100644 --- a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java +++ b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java @@ -1,5 +1,7 @@ package org.hortonmachine.gears.io.stac; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.S3Object; import com.fasterxml.jackson.databind.JsonNode; import it.geosolutions.imageio.core.BasicAuthURI; import it.geosolutions.imageio.plugins.cog.CogImageReadParam; @@ -12,6 +14,8 @@ import org.opengis.parameter.GeneralParameterValue; import org.opengis.referencing.crs.CoordinateReferenceSystem; +import java.io.IOException; +import java.io.InputStream; import java.util.Iterator; /** @@ -82,13 +86,6 @@ public String toString() { return sb.toString(); } - private RangeReader createRangeReader(BasicAuthURI cogUri) { - if (assetUrl.startsWith("s3://")) { - return new S3RangeReader(cogUri.getUri(), CogImageReadParam.DEFAULT_HEADER_LENGTH); - } - return new HttpRangeReader(cogUri.getUri(), CogImageReadParam.DEFAULT_HEADER_LENGTH); - } - /** * Read the asset's coverage into a local raster. * @@ -98,16 +95,22 @@ private RangeReader createRangeReader(BasicAuthURI cogUri) { * @return the read raster from the asset's url. * @throws Exception */ - public GridCoverage2D readRaster( RegionMap region, String user, String password ) throws Exception { + public GridCoverage2D readRaster( RegionMap region, String user, String password, AmazonS3 s3Client ) throws Exception { BasicAuthURI cogUri = new BasicAuthURI(assetUrl, false); if (user != null && password != null) { cogUri.setUser(user); cogUri.setPassword(password); } - RangeReader rangeReader = createRangeReader(cogUri); - CogSourceSPIProvider inputProvider = new CogSourceSPIProvider(cogUri, new CogImageReaderSpi(), - new CogImageInputStreamSpi(), rangeReader.getClass().getName()); - GeoTiffReader reader = new GeoTiffReader(inputProvider); + GeoTiffReader reader; + if (assetUrl.startsWith("s3://")) { + InputStream inputProvider = readS3Raster(cogUri, s3Client); + reader = new GeoTiffReader(inputProvider); + } else { + RangeReader rangeReader = new HttpRangeReader(cogUri.getUri(), CogImageReadParam.DEFAULT_HEADER_LENGTH); + CogSourceSPIProvider inputProvider = new CogSourceSPIProvider(cogUri, new CogImageReaderSpi(), + new CogImageInputStreamSpi(), rangeReader.getClass().getName()); + reader = new GeoTiffReader(inputProvider); + } CoordinateReferenceSystem crs = reader.getCoordinateReferenceSystem(); GeneralParameterValue[] generalParameter = null; @@ -118,8 +121,18 @@ public GridCoverage2D readRaster( RegionMap region, String user, String password return coverage; } + public InputStream readS3Raster(BasicAuthURI cogUri, AmazonS3 s3Client ) throws IOException { + String[] bucketAndObject = assetUrl.split("://")[1].split("/", 2); + S3Object object = s3Client.getObject(bucketAndObject[0], bucketAndObject[1]); + return object.getObjectContent(); + } + public GridCoverage2D readRaster( RegionMap region ) throws Exception { - return readRaster(region, null, null ); + return readRaster(region, null, null, null ); + } + + public GridCoverage2D readRaster( RegionMap region, AmazonS3 s3Client ) throws Exception { + return readRaster(region, null, null, s3Client ); } public String getId() { diff --git a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java index 67cff709b..2a35afc9f 100644 --- a/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java +++ b/gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java @@ -1,16 +1,9 @@ package org.hortonmachine.gears.io.stac; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.stream.Collectors; - +import com.amazonaws.services.s3.AmazonS3; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.data.simple.SimpleFeatureCollection; import org.geotools.data.simple.SimpleFeatureIterator; -import org.geotools.filter.IllegalFilterException; import org.geotools.filter.text.cql2.CQL; import org.geotools.filter.text.cql2.CQLException; import org.geotools.geometry.jts.JTS; @@ -39,6 +32,13 @@ import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.opengis.referencing.operation.MathTransform; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + @SuppressWarnings({"rawtypes"}) /** * A stac collection. @@ -51,6 +51,7 @@ public class HMStacCollection { private Collection collection; private SearchQuery search; private IHMProgressMonitor pm; + private AmazonS3 s3Client; HMStacCollection( STACClient stacClient, Collection collection, IHMProgressMonitor pm ) { this.stacClient = stacClient; @@ -191,7 +192,7 @@ public List searchItems() throws Exception { * @return the final raster. * @throws Exception */ - public static HMRaster readRasterBandOnRegion( RegionMap latLongRegionMap, String bandName, List items, + public HMRaster readRasterBandOnRegion( RegionMap latLongRegionMap, String bandName, List items, boolean allowTransform, MergeMode mergeMode, IHMProgressMonitor pm ) throws Exception { if (!allowTransform) { From 59698c7671945b6d2ccb8d97b3657e020ad5e301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Mon, 27 May 2024 16:55:02 +0200 Subject: [PATCH 14/22] Test S3 credentials as parameters --- .../gears/modules/TestStacAsset.java | 49 +++++++++---------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java index e71625916..6f0f6fe42 100644 --- a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java +++ b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java @@ -1,5 +1,9 @@ package org.hortonmachine.gears.modules; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -7,7 +11,6 @@ import org.hortonmachine.gears.io.stac.HMStacAsset; import org.hortonmachine.gears.utils.HMTestCase; import org.hortonmachine.gears.utils.RegionMap; -import org.junit.After; import static org.junit.Assert.assertThrows; @@ -76,40 +79,32 @@ public void testReadRasterFromHTTP() throws Exception { assertNotNull(grid); } - public void testReadRasterFromS3() throws Exception { - System.setProperty("aws.accessKeyId", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); - System.setProperty("aws.secretKey", "AKIAIOSFODNN7EXAMPLE"); - System.setProperty("iio.s3.aws.region", "us-west-2"); - String assetJSON = "{\"href\":\"s3://sentinel-cogs/sentinel-s2-l2a-cogs/5/C/MK/2018/10/S2B_5CMK_20181020_0_L2A/B01.tif\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"title\":\"rainfall\",\"eo:bands\":[{\"name\":\"rainfall\"}],\"proj:epsg\":4326,\"proj:shape\":[1600,1500],\"proj:transform\":[0.05000000074505806,0,-20,0,-0.05000000074505806,40,0,0,1],\"roles\":[\"data\"]}},\"bbox\":[-20,-40.000001192092896,55.00000111758709,40],\"stac_extensions\":[\"https://stac-extensions.github.io/eo/v1.1.0/schema.json\",\"https://stac-extensions.github.io/projection/v1.1.0/schema.json\"],\"collection\":\"rainfall_chirps_monthly\"}"; - JsonNode node = mapper.readTree(assetJSON); - String assetId = "rainfall"; - HMStacAsset hmAsset = new HMStacAsset(assetId, node); - RegionMap regionMap = RegionMap.fromBoundsAndGrid(1640000.0, 1640200.0, 5140000.0, 5140160.0, 20, 16); - - GridCoverage2D grid = hmAsset.readRaster(regionMap); - - assertNotNull(grid); + private AmazonS3 createDefaultS3Client(String region) { + AWSCredentials credentials = new BasicAWSCredentials( + "accessKey", + "secretKey" + ); + AmazonS3 client = AmazonS3ClientBuilder + .standard() + // We use anonymized credentials for this example. + // For testing purposes, add your own credentials and uncomment the following line. + //.withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withRegion("us-west-2") + .build(); + client.listBuckets(); + return client; } - public void testReadRasterFromS3FailBucketNotInRegion() throws Exception { - System.setProperty("aws.accessKeyId", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); - System.setProperty("aws.secretKey", "AKIAIOSFODNN7EXAMPLE"); - System.setProperty("iio.s3.aws.region", "us-west-1"); // The bucket is not in this region - + public void testReadRasterFromS3() throws Exception { String assetJSON = "{\"href\":\"s3://sentinel-cogs/sentinel-s2-l2a-cogs/5/C/MK/2018/10/S2B_5CMK_20181020_0_L2A/B01.tif\",\"type\":\"image/tiff; application=geotiff; profile=cloud-optimized\",\"title\":\"rainfall\",\"eo:bands\":[{\"name\":\"rainfall\"}],\"proj:epsg\":4326,\"proj:shape\":[1600,1500],\"proj:transform\":[0.05000000074505806,0,-20,0,-0.05000000074505806,40,0,0,1],\"roles\":[\"data\"]}},\"bbox\":[-20,-40.000001192092896,55.00000111758709,40],\"stac_extensions\":[\"https://stac-extensions.github.io/eo/v1.1.0/schema.json\",\"https://stac-extensions.github.io/projection/v1.1.0/schema.json\"],\"collection\":\"rainfall_chirps_monthly\"}"; JsonNode node = mapper.readTree(assetJSON); String assetId = "rainfall"; HMStacAsset hmAsset = new HMStacAsset(assetId, node); RegionMap regionMap = RegionMap.fromBoundsAndGrid(1640000.0, 1640200.0, 5140000.0, 5140160.0, 20, 16); + AmazonS3 s3Client = createDefaultS3Client("us-west-2"); - assertThrows(RuntimeException.class, () -> hmAsset.readRaster(regionMap) - ); - } + GridCoverage2D grid = hmAsset.readRaster(regionMap, s3Client); - @After - public void after(){ - System.clearProperty("aws.accessKeyId"); - System.clearProperty("aws.secretKey"); - System.clearProperty("iio.s3.aws.region"); + assertNotNull(grid); } } From 22816ebde1be8fa14c4a7ea726ec515dfbf48cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Tue, 28 May 2024 12:17:51 +0200 Subject: [PATCH 15/22] replace rangereader-s3 dependencies with aws-sdk --- gears/pom.xml | 8 ++++---- pom.xml | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/gears/pom.xml b/gears/pom.xml index cbf1bba75..cf5860552 100644 --- a/gears/pom.xml +++ b/gears/pom.xml @@ -145,10 +145,6 @@ it.geosolutions.imageio-ext imageio-ext-cog-rangereader-http - - it.geosolutions.imageio-ext - imageio-ext-cog-rangereader-s3 - org.geotools @@ -207,6 +203,10 @@ netcdf4 + + com.amazonaws + aws-java-sdk + diff --git a/pom.xml b/pom.xml index 78dd1a67e..aca3bd38c 100644 --- a/pom.xml +++ b/pom.xml @@ -55,6 +55,7 @@ false + 1.12.730 @@ -397,12 +398,6 @@ imageio-ext-cog-rangereader-http ${imageio.ext.version} - - it.geosolutions.imageio-ext - imageio-ext-cog-rangereader-s3 - ${imageio.ext.version} - - joda-time joda-time @@ -899,6 +894,12 @@ test + + com.amazonaws + aws-java-sdk + ${aws.version} + + From 821b6fcd216b4830dc65017f3dae696dee125d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Mon, 24 Jun 2024 11:46:55 +0200 Subject: [PATCH 16/22] Replace AWS dependency --- gears/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gears/pom.xml b/gears/pom.xml index cf5860552..52607bee8 100644 --- a/gears/pom.xml +++ b/gears/pom.xml @@ -205,7 +205,7 @@ com.amazonaws - aws-java-sdk + aws-java-sdk-core diff --git a/pom.xml b/pom.xml index aca3bd38c..ac9180ee2 100644 --- a/pom.xml +++ b/pom.xml @@ -896,7 +896,7 @@ com.amazonaws - aws-java-sdk + aws-java-sdk-core ${aws.version} From 2aef7aa64950ebbc4367e84421d8a04905343f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Tue, 13 Aug 2024 18:17:13 +0200 Subject: [PATCH 17/22] Remove unused listBuckets() call --- .../test/java/org/hortonmachine/gears/modules/TestStacAsset.java | 1 - 1 file changed, 1 deletion(-) diff --git a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java index 6f0f6fe42..a57cb7804 100644 --- a/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java +++ b/gears/src/test/java/org/hortonmachine/gears/modules/TestStacAsset.java @@ -91,7 +91,6 @@ private AmazonS3 createDefaultS3Client(String region) { //.withCredentials(new AWSStaticCredentialsProvider(credentials)) .withRegion("us-west-2") .build(); - client.listBuckets(); return client; } From 8f09dbb04d85caf18d0593b4d48c3190647df99c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Wed, 14 Aug 2024 11:40:20 +0200 Subject: [PATCH 18/22] Create new action to test branches on push --- .github/workflows/dev.build.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/dev.build.yml diff --git a/.github/workflows/dev.build.yml b/.github/workflows/dev.build.yml new file mode 100644 index 000000000..2e328eb28 --- /dev/null +++ b/.github/workflows/dev.build.yml @@ -0,0 +1,26 @@ +name: Build and Test Branch + +on: + push: + +jobs: + build: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + + - name: Build and Test Branch + run: | + mvn --no-transfer-progress \ + --batch-mode \ + clean install \ + -Dmaven.javadoc.skip=true From c2e4c807cefad9d44c31aa734d9fb19891cdd4fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Cobi=C3=A1n?= Date: Tue, 20 Aug 2024 09:25:00 +0200 Subject: [PATCH 19/22] Unify workflows into a single file --- .github/workflows/build.yml | 24 +++++++++++++++++++++++- .github/workflows/dev.build.yml | 26 -------------------------- 2 files changed, 23 insertions(+), 27 deletions(-) delete mode 100644 .github/workflows/dev.build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a7c7417fa..0fda56f0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,12 +3,13 @@ name: Build and Release Snapshot on: push: branches: - - master jobs: build-and-release: runs-on: ubuntu-22.04 + if: github.ref == 'refs/heads/master' + steps: - name: Checkout code uses: actions/checkout@v3 @@ -44,3 +45,24 @@ jobs: -P release \ -Dmaven.javadoc.skip=true \ -Dgpg.skip=false + + build-and-test: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + + - name: Build and Test Branch + run: | + mvn --no-transfer-progress \ + --batch-mode \ + clean install \ + -Dmaven.javadoc.skip=true diff --git a/.github/workflows/dev.build.yml b/.github/workflows/dev.build.yml deleted file mode 100644 index 2e328eb28..000000000 --- a/.github/workflows/dev.build.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Build and Test Branch - -on: - push: - -jobs: - build: - runs-on: ubuntu-22.04 - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Setup Java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - - name: Build and Test Branch - run: | - mvn --no-transfer-progress \ - --batch-mode \ - clean install \ - -Dmaven.javadoc.skip=true From d4cbb73ee57df84188820d898b7b55fd10e5c35f Mon Sep 17 00:00:00 2001 From: Andrea Antonello Date: Tue, 20 Aug 2024 09:55:40 +0200 Subject: [PATCH 20/22] try to fix release workflow --- .github/workflows/build.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0fda56f0f..0f5e473d4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,8 +21,14 @@ jobs: java-version: '11' cache: 'maven' server-id: ossrh - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD + server-username: ${{ secrets.OSSRH_USERNAME }} + server-password: ${{ secrets.OSSRH_TOKEN }} + + - name: Run Tests + run: | + mvn --no-transfer-progress \ + --batch-mode \ + clean verify - id: install-secret-key name: Install gpg secret key @@ -33,9 +39,6 @@ jobs: gpg --list-secret-keys --keyid-format LONG - name: Build and Deploy Snapshot - env: - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} run: | mvn --no-transfer-progress \ --batch-mode \ From cb18f9c3f408fda1b3e3ddc2553e5f1b594c3e85 Mon Sep 17 00:00:00 2001 From: Andrea Antonello Date: Tue, 20 Aug 2024 10:13:48 +0200 Subject: [PATCH 21/22] add user/token to env --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f5e473d4..bfc164f81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,8 @@ jobs: mvn --no-transfer-progress \ --batch-mode \ -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} \ + -Dossrh.username=${{ secrets.OSSRH_USERNAME }} \ + -Dossrh.password=${{ secrets.OSSRH_TOKEN }} \ clean deploy \ -DskipTests=true \ -P release \ From 70a9625668eaaad7fa7a2d963c02ced522b7abca Mon Sep 17 00:00:00 2001 From: Andrea Antonello Date: Tue, 20 Aug 2024 10:17:25 +0200 Subject: [PATCH 22/22] add auth to pom --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index ac9180ee2..baa0afc4a 100644 --- a/pom.xml +++ b/pom.xml @@ -1128,6 +1128,8 @@ ossrh https://oss.sonatype.org/ true + ${env.OSSRH_USERNAME} + ${env.OSSRH_TOKEN}