Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/TheHortonMachine/hortonma…
Browse files Browse the repository at this point in the history 
…chine
  • Loading branch information
@moovida
moovida committed Oct 28, 2024
2 parents a8374a6 + 70a9625 commit 2c8cb15
Show file tree
Hide file tree
Showing 13 changed files with 290 additions and 62 deletions.
39 changes: 33 additions & 6 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ name: Build and Release Snapshot
on:
push:
branches:
- master

jobs:
build-and-release:
runs-on: ubuntu-22.04

if: github.ref == 'refs/heads/master'

steps:
- name: Checkout code
uses: actions/checkout@v3
Expand All @@ -20,8 +21,14 @@ jobs:
java-version: '11'
cache: 'maven'
server-id: ossrh
server-username: MAVEN_USERNAME
server-password: MAVEN_PASSWORD
server-username: ${{ secrets.OSSRH_USERNAME }}
server-password: ${{ secrets.OSSRH_TOKEN }}

- name: Run Tests
run: |
mvn --no-transfer-progress \
--batch-mode \
clean verify
- id: install-secret-key
name: Install gpg secret key
Expand All @@ -32,15 +39,35 @@ jobs:
gpg --list-secret-keys --keyid-format LONG
- name: Build and Deploy Snapshot
env:
MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}
run: |
mvn --no-transfer-progress \
--batch-mode \
-Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} \
-Dossrh.username=${{ secrets.OSSRH_USERNAME }} \
-Dossrh.password=${{ secrets.OSSRH_TOKEN }} \
clean deploy \
-DskipTests=true \
-P release \
-Dmaven.javadoc.skip=true \
-Dgpg.skip=false
build-and-test:
runs-on: ubuntu-22.04

steps:
- name: Checkout code
uses: actions/checkout@v3

- name: Setup Java
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '11'
cache: 'maven'

- name: Build and Test Branch
run: |
mvn --no-transfer-progress \
--batch-mode \
clean install \
-Dmaven.javadoc.skip=true
4 changes: 2 additions & 2 deletions apps/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<parent>
<groupId>org.hortonmachine</groupId>
<artifactId>hortonmachine</artifactId>
<version>0.10.8</version>
<version>0.10.9-SNAPSHOT</version>
</parent>

<artifactId>hm-apps</artifactId>
Expand All @@ -27,7 +27,7 @@
<dependency>
<groupId>org.hortonmachine</groupId>
<artifactId>hm-gui</artifactId>
<version>0.10.8</version>
<version>0.10.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.mapsforge</groupId>
Expand Down
2 changes: 1 addition & 1 deletion dbs/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<parent>
<groupId>org.hortonmachine</groupId>
<artifactId>hortonmachine</artifactId>
<version>0.10.8</version>
<version>0.10.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
10 changes: 7 additions & 3 deletions gears/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<groupId>org.hortonmachine</groupId>
<artifactId>hortonmachine</artifactId>
<relativePath>../pom.xml</relativePath>
<version>0.10.8</version>
<version>0.10.9-SNAPSHOT</version>
</parent>

<artifactId>hm-gears</artifactId>
Expand All @@ -18,7 +18,7 @@
<dependency>
<groupId>org.hortonmachine</groupId>
<artifactId>hm-dbs</artifactId>
<version>0.10.8</version>
<version>0.10.9-SNAPSHOT</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -145,7 +145,7 @@
<groupId>it.geosolutions.imageio-ext</groupId>
<artifactId>imageio-ext-cog-rangereader-http</artifactId>
</dependency>

<dependency>
<groupId>org.geotools</groupId>
<artifactId>gt-imagemosaic</artifactId>
Expand Down Expand Up @@ -203,6 +203,10 @@
<artifactId>netcdf4</artifactId>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
</dependency>
</dependencies>

</project>
61 changes: 36 additions & 25 deletions gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacAsset.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package org.hortonmachine.gears.io.stac;

import java.util.Iterator;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.S3Object;
import com.fasterxml.jackson.databind.JsonNode;
import it.geosolutions.imageio.core.BasicAuthURI;
import it.geosolutions.imageio.plugins.cog.CogImageReadParam;
import it.geosolutions.imageioimpl.plugins.cog.*;
import org.geotools.coverage.grid.GridCoverage2D;
import org.geotools.gce.geotiff.GeoTiffReader;
import org.hortonmachine.gears.libs.modules.HMConstants;
Expand All @@ -10,18 +14,13 @@
import org.opengis.parameter.GeneralParameterValue;
import org.opengis.referencing.crs.CoordinateReferenceSystem;

import com.fasterxml.jackson.databind.JsonNode;

import it.geosolutions.imageio.core.BasicAuthURI;
import it.geosolutions.imageio.plugins.cog.CogImageReadParam;
import it.geosolutions.imageioimpl.plugins.cog.CogImageInputStreamSpi;
import it.geosolutions.imageioimpl.plugins.cog.CogImageReaderSpi;
import it.geosolutions.imageioimpl.plugins.cog.CogSourceSPIProvider;
import it.geosolutions.imageioimpl.plugins.cog.HttpRangeReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;

/**
* An asset from a stac item.
*
*
* @author Andrea Antonello (www.hydrologis.com)
*
*/
Expand Down Expand Up @@ -49,9 +48,8 @@ public HMStacAsset( String id, JsonNode assetNode ) {
}
if (type.toLowerCase().contains("profile=cloud-optimized")) {
JsonNode rasterBandNode = assetNode.get("raster:bands");
assetUrl = assetNode.get("href").textValue();
if (rasterBandNode != null && !rasterBandNode.isEmpty()) {
assetUrl = assetNode.get("href").textValue();

Iterator<JsonNode> rbIterator = rasterBandNode.elements();
while( rbIterator.hasNext() ) {
JsonNode rbNode = rbIterator.next();
Expand All @@ -64,9 +62,6 @@ public HMStacAsset( String id, JsonNode assetNode ) {
resolution = resolNode.asDouble();
}
}
} else {
isValid = false;
nonValidReason = "raster bands metadata missing";
}
} else {
isValid = false;
Expand All @@ -93,23 +88,29 @@ public String toString() {

/**
* Read the asset's coverage into a local raster.
*
*
* @param region and optional region to read from.
* @param user an optional user in case of authentication.
* @param password an optional password in case of authentication.
* @return the read raster from the asset's url..
* @throws Exception
* @return the read raster from the asset's url.
* @throws Exception
*/
public GridCoverage2D readRaster( RegionMap region, String user, String password ) throws Exception {
public GridCoverage2D readRaster( RegionMap region, String user, String password, AmazonS3 s3Client ) throws Exception {
BasicAuthURI cogUri = new BasicAuthURI(assetUrl, false);
if (user != null && password != null) {
cogUri.setUser(user);
cogUri.setPassword(password);
}
HttpRangeReader rangeReader = new HttpRangeReader(cogUri.getUri(), CogImageReadParam.DEFAULT_HEADER_LENGTH);
CogSourceSPIProvider inputProvider = new CogSourceSPIProvider(cogUri, new CogImageReaderSpi(),
new CogImageInputStreamSpi(), rangeReader.getClass().getName());
GeoTiffReader reader = new GeoTiffReader(inputProvider);
GeoTiffReader reader;
if (assetUrl.startsWith("s3://")) {
InputStream inputProvider = readS3Raster(cogUri, s3Client);
reader = new GeoTiffReader(inputProvider);
} else {
RangeReader rangeReader = new HttpRangeReader(cogUri.getUri(), CogImageReadParam.DEFAULT_HEADER_LENGTH);
CogSourceSPIProvider inputProvider = new CogSourceSPIProvider(cogUri, new CogImageReaderSpi(),
new CogImageInputStreamSpi(), rangeReader.getClass().getName());
reader = new GeoTiffReader(inputProvider);
}
CoordinateReferenceSystem crs = reader.getCoordinateReferenceSystem();

GeneralParameterValue[] generalParameter = null;
Expand All @@ -120,8 +121,18 @@ public GridCoverage2D readRaster( RegionMap region, String user, String password
return coverage;
}

public InputStream readS3Raster(BasicAuthURI cogUri, AmazonS3 s3Client ) throws IOException {
String[] bucketAndObject = assetUrl.split("://")[1].split("/", 2);
S3Object object = s3Client.getObject(bucketAndObject[0], bucketAndObject[1]);
return object.getObjectContent();
}

public GridCoverage2D readRaster( RegionMap region ) throws Exception {
return readRaster(region, null, null);
return readRaster(region, null, null, null );
}

public GridCoverage2D readRaster( RegionMap region, AmazonS3 s3Client ) throws Exception {
return readRaster(region, null, null, s3Client );
}

public String getId() {
Expand Down
44 changes: 36 additions & 8 deletions gears/src/main/java/org/hortonmachine/gears/io/stac/HMStacCollection.java
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,6 @@
package org.hortonmachine.gears.io.stac;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;

import com.amazonaws.services.s3.AmazonS3;
import org.geotools.coverage.grid.GridCoverage2D;
import org.geotools.data.simple.SimpleFeatureCollection;
import org.geotools.data.simple.SimpleFeatureIterator;
Expand Down Expand Up @@ -37,6 +32,13 @@
import org.opengis.referencing.crs.CoordinateReferenceSystem;
import org.opengis.referencing.operation.MathTransform;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;

@SuppressWarnings({"rawtypes"})
/**
* A stac collection.
Expand All @@ -49,6 +51,7 @@ public class HMStacCollection {
private Collection collection;
private SearchQuery search;
private IHMProgressMonitor pm;
private AmazonS3 s3Client;

HMStacCollection( STACClient stacClient, Collection collection, IHMProgressMonitor pm ) {
this.stacClient = stacClient;
Expand Down Expand Up @@ -108,10 +111,26 @@ public HMStacCollection setTimestampFilter( Date startTimestamp, Date endTimesta
public HMStacCollection setGeometryFilter( Geometry intersectionGeometry ) {
if (search == null)
search = new SearchQuery();
else if (search.getBbox() != null)
throw new IllegalStateException("Cannot add intersects filter. Only one of either intersects or bbox may be specified");
search.setIntersects(intersectionGeometry);
return this;
}

/**
* Set the geometry bbox filter for search query
* @param bbox
* @return the current collection.
*/
public HMStacCollection setBboxFilter( double[] bbox ) {
if (search == null)
search = new SearchQuery();
else if (search.getIntersects() != null)
throw new IllegalStateException("Cannot add intersects filter. Only one of either intersects or bbox may be specified");
search.setBbox(bbox);
return this;
}

/**
* Set cql filter for search query;
*
Expand All @@ -126,12 +145,21 @@ public HMStacCollection setCqlFilter( String cqlFilter ) throws CQLException {
return this;
}

private SimpleFeatureCollection queryFeatureCollection() throws IOException {
try {
return stacClient.search(search, STACClient.SearchMode.POST);
} catch (IOException e) {
pm.message("POST search query not supported by the endpoint. GET will be tried.");
}
return stacClient.search(search, STACClient.SearchMode.GET);
}

public List<HMStacItem> searchItems() throws Exception {
if (search == null)
search = new SearchQuery();
search.setCollections(Arrays.asList(getId()));

SimpleFeatureCollection fc = stacClient.search(search, STACClient.SearchMode.GET);
SimpleFeatureCollection fc = queryFeatureCollection();
SimpleFeatureIterator iterator = fc.features();
pm.beginTask("Extracting STAC items...", -1);
List<HMStacItem> stacItems = new ArrayList<>();
Expand Down Expand Up @@ -164,7 +192,7 @@ public List<HMStacItem> searchItems() throws Exception {
* @return the final raster.
* @throws Exception
*/
public static HMRaster readRasterBandOnRegion( RegionMap latLongRegionMap, String bandName, List<HMStacItem> items,
public HMRaster readRasterBandOnRegion( RegionMap latLongRegionMap, String bandName, List<HMStacItem> items,
boolean allowTransform, MergeMode mergeMode, IHMProgressMonitor pm ) throws Exception {

if (!allowTransform) {
Expand Down
Loading

0 comments on commit 2c8cb15

Please sign in to comment.