diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexGrid.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexGrid.java index 749564c2..813cb462 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexGrid.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexGrid.java @@ -214,6 +214,22 @@ public static VortexGridBuilder builder(){ return new VortexGridBuilder(); } + public static VortexGrid empty() { + return VortexGrid.builder() + .shortName("") + .fullName("") + .description("") + .fileName("") + .nx(0).ny(0) + .dx(0).dy(0) + .wkt("") + .data(new float[0]) + .noDataValue(Double.NaN) + .units("") + .dataType(VortexDataType.UNDEFINED) + .build(); + } + public double dx() { return dx; } @@ -298,24 +314,10 @@ public Duration interval() { return interval; } - VortexDataType dataType() { + public VortexDataType dataType() { return dataType != null ? dataType : inferDataType(); } - public double[] xCoordinates() { - double[] xCoordinates = new double[nx]; - // xCoordinates[i] = midpoint of xEdges[i] and xEdges[i + 1] - for (int i = 0; i < nx; i++) xCoordinates[i] = originX + (i + 1) * dx - (dx / 2); - return xCoordinates; - } - - public double[] yCoordinates() { - double[] yCoordinates = new double[ny]; - // yCoordinates[i] = midpoint of yEdges[i] and yEdges[i + 1] - for (int i = 0; i < ny; i++) yCoordinates[i] = originY + (i + 1) * dy - (dy / 2); - return yCoordinates; - } - public float[][][] data3D() { float[][][] data3D = new float[1][ny][nx]; for (int y = 0; y < ny; y++) System.arraycopy(data, y * nx, data3D[0][y], 0, nx); @@ -332,22 +334,49 @@ private VortexDataType inferDataType() { }; } + public double getValue(int index) { + return data[index]; + } + + public boolean isTemporal() { + return startTime != null && endTime != null; + } + + private boolean hasSameData(VortexGrid that) { + float[] thisData = this.data(); + float[] thatData = that.data(); + + if (thisData.length != thatData.length) { + return false; + } + + for (int i = 0; i < thisData.length; i++) { + float thisValue = thisData[i]; + float thatValue = thatData[i]; + + boolean sameFloatValue = thisValue == thatValue; + boolean bothAreNoData = thisValue == this.noDataValue && thatValue == that.noDataValue; + + if (!sameFloatValue && !bothAreNoData) { + return false; + } + } + + return true; + } + @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof VortexGrid)) return false; - - VortexGrid that = (VortexGrid) o; - + if (!(o instanceof VortexGrid that)) return false; if (Double.compare(that.dx, dx) != 0) return false; if (Double.compare(that.dy, dy) != 0) return false; if (nx != that.nx) return false; if (ny != that.ny) return false; if (Double.compare(that.originX, originX) != 0) return false; if (Double.compare(that.originY, originY) != 0) return false; - if (Double.compare(that.noDataValue, noDataValue) != 0) return false; if (!ReferenceUtils.equals(wkt, that.wkt)) return false; - if (!Arrays.equals(data, that.data)) return false; + if (!this.hasSameData(that)) return false; if (!UnitUtil.equals(units, that.units)) return false; if (!startTime.isEqual(that.startTime)) return false; if (!endTime.isEqual(that.endTime)) return false; @@ -358,21 +387,13 @@ public boolean equals(Object o) { @Override public int hashCode() { int result; - long temp; - temp = Double.doubleToLongBits(dx); - result = (int) (temp ^ (temp >>> 32)); - temp = Double.doubleToLongBits(dy); - result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = Double.hashCode(dx); + result = 31 * result + Double.hashCode(dy); result = 31 * result + nx; result = 31 * result + ny; - temp = Double.doubleToLongBits(originX); - result = 31 * result + (int) (temp ^ (temp >>> 32)); - temp = Double.doubleToLongBits(originY); - result = 31 * result + (int) (temp ^ (temp >>> 32)); + result = 31 * result + Double.hashCode(originX); + result = 31 * result + Double.hashCode(originY); result = 31 * result + (wkt != null ? wkt.hashCode() : 0); - result = 31 * result + Arrays.hashCode(data); - temp = Double.doubleToLongBits(noDataValue); - result = 31 * result + (int) (temp ^ (temp >>> 32)); result = 31 * result + (units != null ? units.hashCode() : 0); result = 31 * result + (startTime != null ? startTime.hashCode() : 0); result = 31 * result + (endTime != null ? endTime.hashCode() : 0); @@ -380,5 +401,32 @@ public int hashCode() { result = 31 * result + (dataType != null ? dataType.hashCode() : 0); return result; } + + // Add to help with debugging (when testing VortexGrid::equals) + @Override + public String toString() { + return "VortexGrid{" + + "dx=" + dx + + ", dy=" + dy + + ", nx=" + nx + + ", ny=" + ny + + ", originX=" + originX + + ", originY=" + originY + + ", wkt='" + wkt + '\'' + + ", data=" + Arrays.toString(data) + + ", noDataValue=" + noDataValue + + ", units='" + units + '\'' + + ", fileName='" + fileName + '\'' + + ", shortName='" + shortName + '\'' + + ", fullName='" + fullName + '\'' + + ", description='" + description + '\'' + + ", startTime=" + startTime + + ", endTime=" + endTime + + ", interval=" + interval + + ", dataType=" + dataType + + ", terminusX=" + terminusX + + ", terminusY=" + terminusY + + '}'; + } } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexGridCollection.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexGridCollection.java deleted file mode 100644 index 3e9ebc6e..00000000 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexGridCollection.java +++ /dev/null @@ -1,224 +0,0 @@ -package mil.army.usace.hec.vortex; - -import mil.army.usace.hec.vortex.io.NetcdfGridWriter; -import mil.army.usace.hec.vortex.geo.WktParser; -import ucar.nc2.constants.CF; -import ucar.unidata.geoloc.LatLonPoint; -import ucar.unidata.geoloc.Projection; -import ucar.unidata.geoloc.ProjectionPoint; -import ucar.unidata.geoloc.projection.LatLonProjection; -import ucar.unidata.util.Parameter; - -import java.time.Duration; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.logging.Logger; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -public class VortexGridCollection { - private static final Logger logger = Logger.getLogger(VortexGridCollection.class.getName()); - - private final List vortexGridList; - private final VortexGrid defaultGrid; - private final ZonedDateTime baseTime; - - public VortexGridCollection(List vortexGrids) { - vortexGridList = vortexGrids; - defaultGrid = !vortexGridList.isEmpty() ? vortexGrids.get(0) : null; - baseTime = initBaseTime(); - cleanCollection(); - } - - /* Init */ - private ZonedDateTime initBaseTime() { - return ZonedDateTime.of(1900,1,1,0,0,0,0, ZoneId.of("UTC")); - } - - private void cleanCollection() { - String shortName = defaultGrid.shortName(); - String wkt = defaultGrid.wkt(); - Duration interval = defaultGrid.interval(); - - if (shortName.isEmpty()) logger.warning("Short name not found"); - if (wkt.isEmpty()) logger.warning("Wkt not found"); - if (interval.equals(Duration.ZERO)) logger.warning("Interval not found"); - - vortexGridList.removeIf(g -> !Objects.equals(shortName, g.shortName())); - vortexGridList.removeIf(g -> !Objects.equals(wkt, g.wkt())); - vortexGridList.removeIf(g -> !Objects.equals(interval, g.interval())); - } - - /* Conditionals */ - public boolean isGeographic() { - return getProjection() instanceof LatLonProjection; - } - - public boolean hasTimeBounds() { - return !getInterval().isZero(); - } - - /* Data */ - public Stream> getCollectionDataStream() { - return IntStream.range(0, vortexGridList.size()).parallel().mapToObj(i -> Map.entry(i, vortexGridList.get(i))); - } - - public float getNoDataValue() { - return (float) defaultGrid.noDataValue(); - } - - public String getDataUnit() { - return defaultGrid.units(); - } - - public VortexDataType getDataType() { - return defaultGrid.dataType(); - } - - /* Name & Description */ - public String getShortName() { - return defaultGrid.shortName(); - } - - public String getDescription() { - return defaultGrid.description(); - } - - /* Projection */ - public Projection getProjection() { - return WktParser.getProjection(getWkt()); - } - - public String getProjectionName() { - Projection projection = getProjection(); - for (Parameter parameter : projection.getProjectionParameters()) { - if (parameter.getName().equals(CF.GRID_MAPPING_NAME)) { - return parameter.getStringValue(); - } - } - return null; - } - - public String getProjectionUnit() { - return WktParser.getProjectionUnit(getWkt()); - } - - public String getWkt() { - return defaultGrid.wkt(); - } - - /* Y and X */ - public double[] getYCoordinates() { - return defaultGrid.yCoordinates(); - } - - public double[] getXCoordinates() { - return defaultGrid.xCoordinates(); - } - - public int getNy() { - return defaultGrid.ny(); - } - - public int getNx() { - return defaultGrid.nx(); - } - - /* Lat & Lon */ - public Map getLatLonCoordinates() { - Map latLonMap = new HashMap<>(); - - int ny = getNy(); - int nx = getNx(); - double[] yCoordinates = getYCoordinates(); - double[] xCoordinates = getXCoordinates(); - Projection projection = getProjection(); - - double[][] latData = new double[ny][nx]; - double[][] lonData = new double[ny][nx]; - - for (int y = 0; y < ny; y++) { - for (int x = 0; x < nx; x++) { - ProjectionPoint point = ProjectionPoint.create(xCoordinates[x], yCoordinates[y]); - LatLonPoint latlonPoint = projection.projToLatLon(point); - latData[y][x] = latlonPoint.getLatitude(); - lonData[y][x] = latlonPoint.getLongitude(); - } - } - - latLonMap.put("lat", latData); - latLonMap.put("lon", lonData); - - return latLonMap; - } - - /* Time */ - public int getTimeLength() { - return vortexGridList.size(); - } - - public String getTimeUnits() { - DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss XXX"); - String durationUnit = getDurationUnit(getBaseDuration()).toString(); - return durationUnit + " since " + baseTime.format(dateTimeFormatter); - } - - public float[] getTimeData() { - if (vortexGridList.size() == 1) { - return new float[] {getNumDurationsFromBaseTime(vortexGridList.get(0).endTime())}; - } - - int numData = vortexGridList.size(); - float[] timeData = new float[numData]; - for (int i = 0; i < numData; i++) { - VortexGrid grid = vortexGridList.get(i); - float startTime = getNumDurationsFromBaseTime(grid.startTime()); - float endTime = getNumDurationsFromBaseTime(grid.endTime()); - float midTime = (startTime + endTime) / 2; - timeData[i] = midTime; - } - - return timeData; - } - - public float[][] getTimeBoundsArray() { - float[][] timeBoundArray = new float[getTimeLength()][NetcdfGridWriter.BOUNDS_LEN]; - - for (int i = 0; i < vortexGridList.size(); i++) { - VortexGrid grid = vortexGridList.get(i); - float startTime = getNumDurationsFromBaseTime(grid.startTime()); - float endTime = getNumDurationsFromBaseTime(grid.endTime()); - timeBoundArray[i][0] = startTime; - timeBoundArray[i][1] = endTime; - } - - return timeBoundArray; - } - - public Duration getInterval() { - return defaultGrid.interval(); - } - - /* Helpers */ - private float getNumDurationsFromBaseTime(ZonedDateTime dateTime) { - ZonedDateTime zDateTime = dateTime.withZoneSameInstant(ZoneId.of("Z")); - Duration durationBetween = Duration.between(baseTime, zDateTime); - Duration divisor = getBaseDuration(); - return durationBetween.dividedBy(divisor); - } - - private ChronoUnit getDurationUnit(Duration duration) { - if (duration.toDays() > 0) return ChronoUnit.DAYS; - if (duration.toHours() > 0) return ChronoUnit.HOURS; - if (duration.toMinutes() > 0) return ChronoUnit.MINUTES; - return ChronoUnit.SECONDS; - } - - private Duration getBaseDuration() { - Duration interval = getInterval(); - return interval.isZero() ? Duration.ofHours(1) : interval; - } -} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexVariable.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexVariable.java index bc6296e4..48094510 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexVariable.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/VortexVariable.java @@ -1,6 +1,9 @@ package mil.army.usace.hec.vortex; import java.util.Arrays; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; // long names taken from https://cfconventions.org/Data/cf-standard-names/current/build/cf-standard-name-table.html // DSS names taken from https://www.hec.usace.army.mil/confluence/hmsdocs/hmsum/latest/shared-component-data/grid-data @@ -39,21 +42,16 @@ public enum VortexVariable { private final String longName; private final String dssName; + private static final Logger logger = Logger.getLogger(VortexVariable.class.getName()); + VortexVariable(String shortName, String longName, String dssName) { this.shortName = shortName; this.longName = longName; this.dssName = dssName; } - public static VortexVariable fromDssName(String dssName) { - return Arrays.stream(VortexVariable.values()) - .filter(n -> n.dssName.equals(dssName)) - .findFirst() - .orElse(UNDEFINED); - } - public String getShortName() { - return shortName; + return normalizeString(shortName); } public String getLongName() { @@ -64,6 +62,31 @@ public String getDssCPart() { return dssName; } + private static boolean isUndefined(VortexVariable variable) { + return variable.equals(UNDEFINED); + } + + public static VortexVariable fromGrid(VortexGrid vortexGrid) { + return VortexVariable.fromNames( + vortexGrid.shortName(), + vortexGrid.fullName(), + vortexGrid.description() + ); + } + + public static VortexVariable fromNames(String... names) { + Set matchedSet = Arrays.stream(names) + .map(VortexVariable::fromName) + .filter(v -> !isUndefined(v)) + .collect(Collectors.toSet()); + + if (matchedSet.size() > 1) { + logger.warning("More than 1 variable matched with names"); + } + + return matchedSet.stream().findFirst().orElse(UNDEFINED); + } + public static VortexVariable fromName(String name) { if (name == null || name.isEmpty()) return UNDEFINED; name = name.toLowerCase().replaceAll("\\s", ""); @@ -95,11 +118,13 @@ public static VortexVariable fromName(String name) { if (isValidSnowDepthName(name)) return SNOW_DEPTH; if (isValidSnowSublimationName(name)) return SNOW_SUBLIMATION; if (isValidSnowMeltName(name)) return SNOW_MELT; + if (isValidHumidityName(name)) return HUMIDITY; return UNDEFINED; } private static boolean isValidPrecipitationName(String shortName) { return shortName.equals("precipitation") + || matchesDssName(shortName, PRECIPITATION) || shortName.equals("precip") || shortName.equals("precip-inc") || shortName.equals("precipitationcal") @@ -118,119 +143,141 @@ private static boolean isValidPrecipitationName(String shortName) { } private static boolean isValidTemperatureName(String shortName) { - return shortName.contains("temperature") + return matchesDssName(shortName, TEMPERATURE) + || shortName.contains("temperature") || shortName.equals("airtemp") || shortName.equals("tasmin") || shortName.equals("tasmax") + || shortName.equals("temp") || shortName.equals("temp-air"); } private static boolean isValidShortwaveRadiationName(String shortName) { - return (shortName.contains("short") && shortName.contains("wave") - || shortName.contains("solar")) && shortName.contains("radiation"); + return matchesDssName(shortName, SHORTWAVE_RADIATION) + || (shortName.contains("shortwave") && shortName.contains("radiation")) + || ((shortName.contains("solar") && shortName.contains("radiation"))); } private static boolean isValidLongwaveRadiationName(String shortName) { - return shortName.matches("long.*radiation"); + return matchesDssName(shortName, LONGWAVE_RADIATION) || shortName.matches("long.*radiation"); } private static boolean isValidCropCoefficientName(String shortName) { - return shortName.contains("crop") && shortName.contains("coefficient"); + return matchesDssName(shortName, CROP_COEFFICIENT) || (shortName.contains("crop") && shortName.contains("coefficient")); } private static boolean isValidStorageCapacityName(String shortName) { - return shortName.contains("storage") && shortName.contains("capacity"); + return matchesDssName(shortName, STORAGE_CAPACITY) || (shortName.contains("storage") && shortName.contains("capacity")); } private static boolean isValidPercolationRateName(String shortName) { - return shortName.equals("percolation") - || shortName.matches("percolation\\s?rate"); + return matchesDssName(shortName, PERCOLATION_RATE) || shortName.equals("percolation") || shortName.matches("percolation\\s?rate"); } private static boolean isValidStorageCoefficientName(String shortName) { - return shortName.contains("storage") && shortName.contains("coefficient"); + return matchesDssName(shortName, STORAGE_COEFFICIENT) || (shortName.contains("storage") && shortName.contains("coefficient")); } private static boolean isValidMoistureDeficitName(String shortName) { - return shortName.matches("moisture\\s?deficit"); + return matchesDssName(shortName, MOISTURE_DEFICIT) || shortName.matches("moisture\\s?deficit"); } private static boolean isValidImperviousAreaName(String shortName) { - return shortName.matches("impervious\\s?area"); + return matchesDssName(shortName, IMPERVIOUS_AREA) || shortName.matches("impervious\\s?area"); } private static boolean isValidCurveNumberName(String shortName) { - return shortName.matches("curve\\s?number"); + return matchesDssName(shortName, CURVE_NUMBER) || shortName.matches("curve\\s?number"); } private static boolean isValidColdContentName(String shortName) { - return shortName.equals("cold content"); + return matchesDssName(shortName, COLD_CONTENT) || equalsIgnoreCaseAndSpace(shortName, "cold content"); } private static boolean isValidColdContentATIName(String shortName) { - return shortName.equals("cold content ati"); + return matchesDssName(shortName, COLD_CONTENT_ATI) || equalsIgnoreCaseAndSpace(shortName, "cold content ati"); } private static boolean isValidMeltrateATIName(String shortName) { - return shortName.equals("meltrate ati"); + return matchesDssName(shortName, MELTRATE_ATI) || equalsIgnoreCaseAndSpace(shortName, "meltrate ati"); } private static boolean isValidLiquidWaterName(String shortName) { - return shortName.contains("snow") && shortName.contains("melt") && shortName.contains("runoff") - || shortName.equals("liquid water"); + return matchesDssName(shortName, LIQUID_WATER) + || (shortName.contains("snow") && shortName.contains("melt") && shortName.contains("runoff")) + || equalsIgnoreCaseAndSpace(shortName, "liquid water"); } private static boolean isValidSnowWaterEquivalentName(String shortName) { - return shortName.contains("snow") && shortName.contains("water") && shortName.contains("equivalent") + return matchesDssName(shortName, SNOW_WATER_EQUIVALENT) + || shortName.contains("snow") && shortName.contains("water") && shortName.contains("equivalent") || shortName.equals("swe") || shortName.equals("weasd"); } private static boolean isValidWaterContentName(String shortName) { - return shortName.contains("water") && shortName.contains("content"); + return matchesDssName(shortName, WATER_CONTENT) || shortName.contains("water") && shortName.contains("content"); } private static boolean isValidWaterPotentialName(String shortName) { - return shortName.contains("water") && shortName.contains("potential"); + return matchesDssName(shortName, WATER_POTENTIAL) || shortName.contains("water") && shortName.contains("potential"); } private static boolean isValidHumidityName(String shortName) { - return shortName.contains("humidity"); + return matchesDssName(shortName, HUMIDITY) || shortName.contains("humidity"); } private static boolean isValidWindSpeedName(String shortName) { - return shortName.contains("wind") && shortName.contains("speed"); + return matchesDssName(shortName, WINDSPEED) || (shortName.contains("wind") && shortName.contains("speed")); } private static boolean isValidPressureName(String shortName) { - return shortName.contains("pressure"); + return matchesDssName(shortName, PRESSURE) || shortName.contains("pressure"); } private static boolean isValidPrecipitationFrequencyName(String shortName) { - return shortName.contains("precipitation") && shortName.contains("frequency"); + return matchesDssName(shortName, PRECIPITATION_FREQUENCY) || shortName.contains("precipitation") && shortName.contains("frequency"); } private static boolean isValidAlbedoName(String shortName) { - return shortName.contains("albedo"); + return matchesDssName(shortName, ALBEDO) || shortName.contains("albedo"); } private static boolean isValidEnergyName(String shortName) { - return shortName.equals("albedo"); + return matchesDssName(shortName, ENERGY) || shortName.equals("energy"); } private static boolean isValidSnowfallAccumulationName(String shortName) { - return shortName.contains("snowfall") && shortName.contains("accumulation"); + return matchesDssName(shortName, SNOWFALL_ACCUMULATION) || (shortName.contains("snowfall") && shortName.contains("accumulation")); } private static boolean isValidSnowDepthName(String shortName) { - return shortName.contains("snow") && shortName.contains("depth"); + return matchesDssName(shortName, SNOW_DEPTH) || (shortName.contains("snow") && shortName.contains("depth")); } private static boolean isValidSnowSublimationName(String shortName) { - return shortName.contains("snow") && shortName.contains("sublimation"); + return matchesDssName(shortName, SNOW_SUBLIMATION) || (shortName.contains("snow") && shortName.contains("sublimation")); } private static boolean isValidSnowMeltName(String shortName) { - return shortName.equals("snow melt"); + return matchesDssName(shortName, SNOW_MELT) || equalsIgnoreCaseAndSpace(shortName, "snow melt"); + } + + private static boolean matchesDssName(String name, VortexVariable variable) { + String normalizedName = normalizeString(name); + String normalizedDSS = normalizeString(variable.getDssCPart()); + return normalizedName.equals(normalizedDSS); + } + + private static boolean equalsIgnoreCaseAndSpace(String one, String two) { + String normalizedLeft = normalizeString(one); + String normalizedRight = normalizeString(two); + return normalizedLeft.equals(normalizedRight); + } + + private static String normalizeString(String name) { + return name.toLowerCase() + .replaceAll("\\s+", "") + .replace("_", ""); } } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/Grid.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/Grid.java index 40a5ff7d..c1c3bd91 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/Grid.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/Grid.java @@ -37,6 +37,20 @@ public static class Builder { private int ny; private String crs; + Builder() { + // Empty constructor + } + + Builder(Grid copyGrid) { + this.originX = copyGrid.getOriginX(); + this.originY = copyGrid.getOriginY(); + this.dx = copyGrid.getDx(); + this.dy = copyGrid.getDy(); + this.nx = copyGrid.getNx(); + this.ny = copyGrid.getNy(); + this.crs = copyGrid.getCrs(); + } + public Builder originX(double originX) { this.originX = originX; return this; @@ -81,6 +95,10 @@ public static Builder builder() { return new Builder(); } + public static Builder toBuilder(Grid grid) { + return new Builder(grid); + } + public List getGridCells() { if (gridCells == null) { gridCells = new ArrayList<>(); diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/RasterUtils.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/RasterUtils.java index 61838bb0..d99f06af 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/RasterUtils.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/RasterUtils.java @@ -69,6 +69,10 @@ public static VortexGrid resetNoDataValue(VortexGrid grid, double noDataValue) { } public static float[] flipVertically (float[] data, int nx) { + if (nx <= 0) { + return data; + } + int length = data.length; float[] flipped = new float[length]; int ny = length / nx; diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/WktFactory.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/WktFactory.java index 9c43dbcd..df1da7a3 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/WktFactory.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/WktFactory.java @@ -7,12 +7,16 @@ import org.gdal.osr.SpatialReference; import ucar.unidata.geoloc.Projection; import ucar.unidata.geoloc.projection.*; +import ucar.unidata.geoloc.projection.proj4.AlbersEqualAreaEllipse; import ucar.unidata.geoloc.projection.proj4.LambertConformalConicEllipse; import ucar.unidata.geoloc.projection.proj4.StereographicAzimuthalProjection; import ucar.unidata.geoloc.projection.proj4.TransverseMercatorProjection; import ucar.unidata.util.Parameter; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -42,6 +46,26 @@ public static String createWkt(Projection projection) { return wkt; } else if (projection instanceof AlbersEqualArea in) { + SpatialReference srs = new SpatialReference(); + srs.SetProjCS("Albers Equal Area"); + setGcsParameters(in, srs); + srs.SetACEA( + in.getParallelOne(), + in.getParallelTwo(), + in.getOriginLat(), + in.getOriginLon(), + in.getFalseEasting(), + in.getFalseNorthing() + ); + srs.SetLinearUnits(SRS_UL_METER, 1.0); + + String wkt = srs.ExportToPrettyWkt(); + + srs.delete(); + + return wkt; + + } else if (projection instanceof AlbersEqualAreaEllipse in) { SpatialReference srs = new SpatialReference(); srs.SetProjCS("Albers Equal Area Conic"); setGcsParameters(in, srs); diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/WktParser.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/WktParser.java index 9f271a24..0f6f331c 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/WktParser.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/geo/WktParser.java @@ -4,6 +4,7 @@ import ucar.unidata.geoloc.Earth; import ucar.unidata.geoloc.Projection; import ucar.unidata.geoloc.projection.*; +import ucar.unidata.geoloc.projection.proj4.AlbersEqualAreaEllipse; import ucar.unidata.geoloc.projection.proj4.LambertConformalConicEllipse; public class WktParser { @@ -20,7 +21,7 @@ public static Projection getProjection(String wkt) { if (srs.IsGeographic() == 1) return parseLatLong(srs); String projectionType = srs.GetAttrValue("PROJECTION"); if (projectionType.matches("(?i).*albers.*conic.*equal.*area.*")) - return parseAlbersEqualArea(srs); + return parseAlbersConicEqualArea(srs); if (projectionType.matches("(?i).*lambert.*conformal.*conic.*")) return parseLambertConformalConic(srs); if (projectionType.matches("(?i).*lambert.*conformal.*")) @@ -47,13 +48,10 @@ public static Projection getProjection(String wkt) { public static String getProjectionUnit(String wkt) { SpatialReference srs = new SpatialReference(wkt); String unitName = srs.GetLinearUnitsName().toLowerCase(); - switch (unitName) { - case "meter": - case "metre": - return "m"; - default: - return unitName; - } + return switch (unitName) { + case "meter", "metre" -> "m"; + default -> unitName; + }; } private static Projection parseLatLong(SpatialReference srs) { @@ -77,6 +75,18 @@ private static Projection parseAlbersEqualArea(SpatialReference srs) { ); } + private static Projection parseAlbersConicEqualArea(SpatialReference srs) { + return new AlbersEqualAreaEllipse( + getCenterLatitude(srs), + getCenterLongitude(srs), + getStandardParallel1(srs), + getStandardParallel2(srs), + getFalseEasting(srs), + getFalseNorthing(srs), + getEarth(srs) + ); + } + private static Projection parseLambertConformalConic(SpatialReference srs) { return new LambertConformalConicEllipse( getCenterLatitude(srs), diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscDataReader.java index 5c590bf9..5ce6b255 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscDataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscDataReader.java @@ -302,4 +302,12 @@ public VortexData getDto(int idx) { return null; } } + + @Override + public List getDataIntervals() { + return getDtos() + .stream() + .map(VortexDataInterval::of) + .toList(); + } } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscDataWriter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscDataWriter.java index a22b6508..a3dd7d0c 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscDataWriter.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscDataWriter.java @@ -12,7 +12,7 @@ import java.util.Vector; import java.util.stream.Collectors; -public class AscDataWriter extends DataWriter { +class AscDataWriter extends DataWriter { private static final double NO_DATA_VALUE = -9999; AscDataWriter(Builder builder) { diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscZipDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscZipDataReader.java index 39346075..e2f0be88 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscZipDataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/AscZipDataReader.java @@ -4,13 +4,12 @@ import mil.army.usace.hec.vortex.VortexData; import org.gdal.gdal.gdal; -import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; -public class AscZipDataReader extends DataReader implements VirtualFileSystem{ +class AscZipDataReader extends DataReader implements VirtualFileSystem { static { GdalRegister.getInstance(); } @@ -75,4 +74,11 @@ public VortexData getDto(int idx) { return null; } // Extended getDto(): Asc Zip + @Override + public List getDataIntervals() { + return getDtos().stream() + .map(VortexDataInterval::of) + .toList(); + } + } // AscZipDataReader class \ No newline at end of file diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BatchImporter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BatchImporter.java index b4e81a17..ea009cb8 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BatchImporter.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BatchImporter.java @@ -93,10 +93,15 @@ private boolean isConcurrentWritable() { } public BatchImporter build() { - if (destination == null) throw new IllegalStateException("Invalid destination."); + if (destination == null) { + throw new IllegalStateException("Invalid destination."); + } - if (isConcurrentWritable()) return new ConcurrentBatchImporter(this); - else return new SerialBatchImporter(this); + if (isConcurrentWritable()) { + return new ConcurrentBatchImporter(this); + } else { + return new NetcdfBatchImporter(this); + } } } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BilDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BilDataReader.java index 32d351bc..3f065d1a 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BilDataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BilDataReader.java @@ -272,4 +272,11 @@ public VortexData getDto(int idx) { return null; } } + + @Override + public List getDataIntervals() { + return getDtos().stream() + .map(VortexDataInterval::of) + .toList(); + } } // BilDataReader class diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BilZipDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BilZipDataReader.java index 9fdd458b..adb3486d 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BilZipDataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/BilZipDataReader.java @@ -4,13 +4,12 @@ import mil.army.usace.hec.vortex.VortexData; import org.gdal.gdal.gdal; -import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Vector; import java.util.concurrent.atomic.AtomicInteger; -public class BilZipDataReader extends DataReader implements VirtualFileSystem{ +class BilZipDataReader extends DataReader implements VirtualFileSystem { static { GdalRegister.getInstance(); } @@ -78,4 +77,11 @@ public VortexData getDto(int idx) { return null; } // Extended getDto(): Bil Zip + @Override + public List getDataIntervals() { + return getDtos().stream() + .map(VortexDataInterval::of) + .toList(); + } + } // BilZipDataReader class diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DataReader.java index be24ff16..04edc2b0 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DataReader.java @@ -1,6 +1,7 @@ package mil.army.usace.hec.vortex.io; import mil.army.usace.hec.vortex.VortexData; +import mil.army.usace.hec.vortex.util.FilenameUtil; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; @@ -72,7 +73,7 @@ public DataReader build(){ return new DssDataReader(this); } - return new NetcdfDataReader(this); + return NetcdfDataReader.createInstance(path, variableName); } // build() } // DataReaderBuilder class @@ -83,25 +84,25 @@ public DataReader build(){ public static Set getVariables(String path){ String fileName = new File(path).getName().toLowerCase(); - if (fileName.matches(".*\\.(asc|tif|tiff)$") || fileName.endsWith("asc.zip")){ + if (FilenameUtil.endsWithExtensions(fileName, ".asc", ".tif", ".tiff", "asc.zip")) { return AscDataReader.getVariables(path); - } - if (fileName.endsWith(".bil") || fileName.endsWith("bil.zip")){ + } else if (FilenameUtil.endsWithExtensions(fileName, ".bil", "bil.zip")) { return BilDataReader.getVariables(path); - } - if (fileName.matches(".*snodas.*\\.(dat|tar|tar.gz)")){ + } else if (fileName.contains("snodas") && FilenameUtil.endsWithExtensions(fileName, ".dat", ".tar", ".tar.gz")) { return SnodasDataReader.getVariables(path); - } - if (fileName.endsWith(".dss")){ + } else if (FilenameUtil.endsWithExtensions(fileName, ".dss")) { return DssDataReader.getVariables(path); + } else { + return NetcdfDataReader.getVariables(path); } - return NetcdfDataReader.getVariables(path); } // builder() public abstract int getDtoCount(); public abstract VortexData getDto(int idx); + public abstract List getDataIntervals(); + public static boolean isVariableRequired(String pathToFile) { String fileName = new File(pathToFile).getName().toLowerCase(); diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataReader.java index 2efe658b..4c4c9916 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataReader.java @@ -1,6 +1,9 @@ package mil.army.usace.hec.vortex.io; -import hec.heclib.dss.*; +import hec.heclib.dss.DSSPathname; +import hec.heclib.dss.DssDataType; +import hec.heclib.dss.HecDSSDataAttributes; +import hec.heclib.dss.HecDssCatalog; import hec.heclib.grid.GridData; import hec.heclib.grid.GridInfo; import hec.heclib.grid.GridUtilities; @@ -25,38 +28,56 @@ import static hec.heclib.dss.HecDSSDataAttributes.*; class DssDataReader extends DataReader { + private final List catalogPathnameList; DssDataReader(DataReaderBuilder builder) { super(builder); + catalogPathnameList = getCatalogPathnames(path, variableName); + } + + private static List getCatalogPathnames(String path, String variableName) { + if (!variableName.contains("*")) { + return List.of(new DSSPathname(variableName)); + } + + HecDssCatalog hecDssCatalog = new HecDssCatalog(); + hecDssCatalog.setDSSFileName(path); + String[] dssPathnames = hecDssCatalog.getCatalog(true, variableName); + + return Arrays.stream(dssPathnames).map(DSSPathname::new).toList(); } @Override public List getDtos() { - HecDSSFileAccess.setDefaultDSSFileName(path); - String[] paths; - if (variableName.contains("*")) { - HecDssCatalog catalog = new HecDssCatalog(); - paths = catalog.getCatalog(true, variableName); - } else { - paths = new String[1]; - paths[0] = variableName; - } List dtos = new ArrayList<>(); - Arrays.stream(paths).forEach(path -> { - int[] status = new int[1]; - GriddedData griddedData = new GriddedData(); - griddedData.setDSSFileName(this.path); - griddedData.setPathname(path); - GridData gridData = new GridData(); - griddedData.retrieveGriddedData(true, gridData, status); - if (status[0] == 0) { - dtos.add(dssToDto(gridData, path)); + catalogPathnameList.forEach(path -> { + GridData gridData = retrieveGriddedData(this.path, path.getPathname()); + if (gridData != null) { + dtos.add(dssToDto(gridData, path.getPathname())); } - }); return dtos; } + private static GridData retrieveGriddedData(String dssFileName, String dssPathname) { + int[] status = new int[1]; + GriddedData griddedData = new GriddedData(); + griddedData.setDSSFileName(dssFileName); + griddedData.setPathname(dssPathname); + GridData gridData = new GridData(); + + try { + griddedData.retrieveGriddedData(true, gridData, status); + if (status[0] != 0) { + return null; + } + } catch (Exception e) { + return null; + } + + return gridData; + } + private VortexGrid dssToDto(GridData gridData, String pathname){ GridInfo gridInfo = gridData.getGridInfo(); String wkt = WktFactory.fromGridInfo(gridInfo); @@ -182,30 +203,16 @@ private static Set getGridRecordTypes() { @Override public int getDtoCount() { - HecDSSFileAccess.setDefaultDSSFileName(path); - String[] paths; - if (variableName.contains("*")) { - HecDssCatalog catalog = new HecDssCatalog(); - paths = catalog.getCatalog(true, variableName); - } else { - paths = new String[1]; - paths[0] = variableName; - } - return paths.length; + return catalogPathnameList.size(); } @Override public VortexData getDto(int idx) { - HecDSSFileAccess.setDefaultDSSFileName(path); - String[] paths; - if (variableName.contains("*")) { - HecDssCatalog catalog = new HecDssCatalog(); - paths = catalog.getCatalog(true, variableName); - } else { - paths = new String[1]; - paths[0] = variableName; + if (idx < 0 || idx >= catalogPathnameList.size()) { + return null; } - String dssPath = paths[idx]; + + String dssPath = catalogPathnameList.get(idx).pathname(); int[] status = new int[1]; GridData gridData = GridUtilities.retrieveGridFromDss(this.path, dssPath, status); if (gridData != null) { @@ -213,4 +220,12 @@ public VortexData getDto(int idx) { } return null; } + + @Override + public List getDataIntervals() { + return catalogPathnameList.stream() + .map(DSSPathname::toString) + .map(VortexDataInterval::of) + .toList(); + } } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataWriter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataWriter.java index a188bff4..c5693e96 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataWriter.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataWriter.java @@ -3,7 +3,9 @@ import hec.heclib.dss.DSSPathname; import hec.heclib.dss.DssDataType; import hec.heclib.dss.HecTimeSeries; -import hec.heclib.grid.*; +import hec.heclib.grid.GridData; +import hec.heclib.grid.GridInfo; +import hec.heclib.grid.GriddedData; import hec.heclib.util.HecTime; import hec.heclib.util.HecTimeArray; import hec.heclib.util.Heclib; @@ -15,8 +17,8 @@ import mil.army.usace.hec.vortex.VortexVariable; import mil.army.usace.hec.vortex.geo.RasterUtils; import mil.army.usace.hec.vortex.geo.ZonalStatistics; +import mil.army.usace.hec.vortex.util.DssUtil; import mil.army.usace.hec.vortex.util.UnitUtil; -import org.gdal.osr.SpatialReference; import javax.measure.Unit; import java.time.Duration; @@ -29,16 +31,14 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import static javax.measure.MetricPrefix.KILO; import static javax.measure.MetricPrefix.MILLI; import static mil.army.usace.hec.vortex.VortexVariable.*; -import static systems.uom.common.USCustomary.*; +import static systems.uom.common.USCustomary.FAHRENHEIT; +import static systems.uom.common.USCustomary.INCH; import static tech.units.indriya.AbstractUnit.ONE; -import static tech.units.indriya.unit.Units.HOUR; -import static tech.units.indriya.unit.Units.MINUTE; import static tech.units.indriya.unit.Units.*; -public class DssDataWriter extends DataWriter { +class DssDataWriter extends DataWriter { private static final Logger logger = Logger.getLogger(DssDataWriter.class.getName()); @@ -58,7 +58,7 @@ public void write() { .toList(); for (VortexGrid grid : grids) { - GridInfo gridInfo = getGridInfo(grid); + GridInfo gridInfo = DssUtil.getGridInfo(grid); float[] data; if (grid.dy() < 0) { @@ -372,91 +372,6 @@ private static DSSPathname updatePathname(DSSPathname pathnameIn, Map unit) { - if (unit.equals(MILLI(METRE))) { - return "MM"; - } - if (unit.equals(INCH)) { - return "IN"; - } - if (unit.equals(INCH.divide(HOUR))) { - return "IN/HR"; - } - if (unit.equals(MILLI(METRE).divide(SECOND))) { - return "MM/S"; - } - if (unit.equals(MILLI(METRE).divide(HOUR))) { - return "MM/HR"; - } - if (unit.equals(MILLI(METRE).divide(DAY))) { - return "MM/DAY"; - } - if (unit.equals(CUBIC_METRE.divide(SECOND))) { - return "M3/S"; - } - if (unit.equals(CUBIC_FOOT.divide(SECOND))) { - return "CFS"; - } - if (unit.equals(METRE)) { - return "M"; - } - if (unit.equals(FOOT)) { - return "FT"; - } - if (unit.equals(CELSIUS)) { - return "DEG C"; - } - if (unit.equals(FAHRENHEIT)) { - return "DEG F"; - } - if (unit.equals(WATT.divide(SQUARE_METRE))) { - return "WATT/M2"; - } - if (unit.equals(KILOMETRE_PER_HOUR)) { - return "KPH"; - } - if (unit.equals(METRE_PER_SECOND)) { - return "M/S"; - } - if (unit.equals(MILE_PER_HOUR)) { - return "MPH"; - } - if (unit.equals(FOOT_PER_SECOND)) { - return "FT/S"; - } - if (unit.equals(KILO(PASCAL))) { - return "KPA"; - } - if (unit.equals(PASCAL)) { - return "PA"; - } - if (unit.equals(PERCENT)) { - return "%"; - } - if (unit.equals(KILO(METRE))) { - return "KM"; - } - if (unit.equals(MILE)) { - return "MILE"; - } - if (unit.equals(ONE)) { - return "UNSPECIF"; - } - if (unit.equals(TON)) { - return "TONS"; - } - if (unit.equals(MILLI(GRAM).divide(LITRE))) { - return "MG/L"; - } - if (unit.equals(CELSIUS.multiply(DAY))) { - return "DEGC-D"; - } - if (unit.equals(MINUTE)) { - return "MINUTES"; - } - return unit.toString(); - } - private static String getEPart(int seconds) { int minutes = seconds / SECONDS_PER_MINUTE; return switch (minutes) { @@ -491,96 +406,6 @@ private static String getEPart(int seconds) { }; } - private static GridInfo getGridInfo(VortexGrid grid) { - GridInfo gridInfo; - - SpatialReference srs = new SpatialReference(grid.wkt()); - String crsName; - if (srs.IsProjected() == 1) { - crsName = srs.GetAttrValue("projcs"); - } else if (srs.IsGeographic() == 1) { - crsName = srs.GetAttrValue("geogcs"); - } else { - crsName = ""; - } - - if (crsName.toLowerCase().contains("albers")) { - - AlbersInfo albersInfo = new AlbersInfo(); - albersInfo.setCoordOfGridCellZero(0, 0); - - String datum = srs.GetAttrValue("geogcs"); - if (datum.contains("83")) { - albersInfo.setProjectionDatum(GridInfo.getNad83()); - } - - String units = srs.GetLinearUnitsName(); - albersInfo.setProjectionUnits(units); - - double centralMeridian = srs.GetProjParm("central_meridian"); - albersInfo.setCentralMeridian((float) centralMeridian); - - double falseEasting = srs.GetProjParm("false_easting"); - double falseNorthing = srs.GetProjParm("false_northing"); - albersInfo.setFalseEastingAndNorthing((float) falseEasting, (float) falseNorthing); - - double latitudeOfOrigin = srs.GetProjParm("latitude_of_origin"); - albersInfo.setLatitudeOfProjectionOrigin((float) latitudeOfOrigin); - - double standardParallel1 = srs.GetProjParm("standard_parallel_1"); - double standardParallel2 = srs.GetProjParm("standard_parallel_2"); - albersInfo.setStandardParallels((float) standardParallel1, (float) standardParallel2); - - gridInfo = albersInfo; - } else { - SpecifiedGridInfo specifiedGridInfo = new SpecifiedGridInfo(); - specifiedGridInfo.setSpatialReference(crsName, grid.wkt(), 0, 0); - gridInfo = specifiedGridInfo; - } - - double llx = grid.originX(); - double lly; - if (grid.dy() < 0) { - lly = grid.originY() + grid.dy() * grid.ny(); - } else { - lly = grid.originY(); - } - double dx = grid.dx(); - double dy = grid.dy(); - float cellSize = (float) ((Math.abs(dx) + Math.abs(dy)) / 2.0); - int minX = (int) Math.round(llx / cellSize); - int minY = (int) Math.round(lly / cellSize); - - gridInfo.setCellInfo(minX, minY, grid.nx(), grid.ny(), cellSize); - - Unit units = UnitUtil.getUnits(grid.units()); - String unitsString = getUnitsString(units); - gridInfo.setDataUnits(unitsString); - - ZonedDateTime startTime = grid.startTime(); - - if (startTime == null) - return gridInfo; - - ZonedDateTime endTime = grid.endTime(); - if (!startTime.equals(endTime) && units.isCompatible(CELSIUS)) { - gridInfo.setDataType(DssDataType.PER_AVER.value()); - } else if (startTime.equals(endTime)) { - gridInfo.setDataType(DssDataType.INST_VAL.value()); - } else { - gridInfo.setDataType(DssDataType.PER_CUM.value()); - } - - HecTime hecTimeStart = getHecTime(startTime); - hecTimeStart.showTimeAsBeginningOfDay(true); - HecTime hecTimeEnd = getHecTime(endTime); - hecTimeEnd.showTimeAsBeginningOfDay(false); - - gridInfo.setGridTimes(hecTimeStart, hecTimeEnd); - - return gridInfo; - } - private static HecTime getHecTime(ZonedDateTime zonedDateTime) { return new HecTime(zonedDateTime.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/GridDatasetReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/GridDatasetReader.java new file mode 100644 index 00000000..43f33c99 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/GridDatasetReader.java @@ -0,0 +1,425 @@ +package mil.army.usace.hec.vortex.io; + +import mil.army.usace.hec.vortex.VortexData; +import mil.army.usace.hec.vortex.VortexGrid; +import mil.army.usace.hec.vortex.geo.*; +import mil.army.usace.hec.vortex.util.UnitUtil; +import org.locationtech.jts.geom.Coordinate; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.dataset.*; +import ucar.nc2.dt.GridCoordSystem; +import ucar.nc2.dt.GridDatatype; +import ucar.nc2.dt.grid.GridDataset; +import ucar.nc2.time.CalendarDate; + +import javax.measure.Unit; +import java.io.IOException; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import static mil.army.usace.hec.vortex.io.GridDatasetReader.SpecialFileType.UNDEFINED; +import static mil.army.usace.hec.vortex.io.GridDatasetReader.SpecialFileType.values; +import static tech.units.indriya.unit.Units.METRE; + +class GridDatasetReader extends NetcdfDataReader { + private static final Logger logger = Logger.getLogger(GridDatasetReader.class.getName()); + + private final GridDataset gridDataset; + private final SpecialFileType specialFileType; + private final GridDatatype gridDatatype; + private final GridCoordSystem gridCoordSystem; + private final VariableDS variableDS; + private final Grid gridDefinition; + private final List timeBounds; + + private static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z"); + + enum SpecialFileType { + ABRSC_GAUGE(".*gaugecorr.*qpe.*01h.*grib2", ".*"), + ABRSC_RADAR(".*radaronly.*qpe.*01h.*grib2", ".*"), + MRMS_IDP(".*multisensor.*qpe.*01h.*grib2", ".*"), + MRMS_PRECIP("mrms_preciprate.*", ".*"), + MRMS_PRECIP_2_MIN("preciprate_.*\\.grib2", ".*"), + GPM(".*hhr\\.ms\\.mrg.*hdf.*", ".*"), + AORC_APCP(".*aorc.*apcp.*nc4.*", ".*"), + AORC_TMP(".*aorc.*tmp.*nc4.*", ".*"), + UA_SWE("[0-9]{2}.nc", ".*"), + NLDAS_APCP("nldas_fora0125_h.a.*", "APCP"), + CMORPH(".*cmorph.*h.*ly.*", ".*"), + GEFS("ge.*\\.pgrb2.*\\.0p.*\\.f.*\\..*", ".*"), + LIVNEH_PRECIP("prec.\\d{4}.nc", ".*"), + HRRR_WRFSFCF("hrrr.*wrfsfcf.*", ".*"), + GFS("gfs.nc", "Precipitation_rate_surface"), + UNDEFINED("", ".*"); + + private final Pattern filenamePattern; + private final Pattern variableNamePattern; + + SpecialFileType(String filenameRegex, String variableRegex) { + this.filenamePattern = Pattern.compile(filenameRegex); + this.variableNamePattern = Pattern.compile(variableRegex); + } + + boolean matches(String fileName, String variableName) { + return filenamePattern.matcher(fileName).matches() && variableNamePattern.matcher(variableName).matches(); + } + } + + /* Constructor */ + GridDatasetReader(GridDataset gridDataset, String variableName) { + super(new DataReaderBuilder().path(gridDataset.getLocation()).variable(variableName)); + this.gridDataset = gridDataset; + specialFileType = determineSpecialFileType(gridDataset.getLocation()); + + gridDatatype = gridDataset.findGridDatatype(variableName); + gridCoordSystem = gridDatatype.getCoordinateSystem(); + variableDS = gridDatatype.getVariable(); + + gridDefinition = getGridDefinition(gridCoordSystem); + timeBounds = getTimeBounds(gridCoordSystem); + } + + private SpecialFileType determineSpecialFileType(String pathToFile) { + String filename = Path.of(pathToFile).getFileName().toString().toLowerCase(); + return Arrays.stream(values()) + .filter(t -> t.matches(filename, variableName)) + .findFirst() + .orElse(UNDEFINED); + } + + /* Public */ + @Override + public List getDtos() { + Dimension timeDim = gridDatatype.getTimeDimension(); + Dimension endDim = gridDatatype.getEnsembleDimension(); + Dimension rtDim = gridDatatype.getRunTimeDimension(); + + if (timeDim != null && endDim != null && rtDim != null) { + return getMultiDimensionGrids(timeDim, endDim, rtDim); + } else if (timeDim != null) { + return getTemporalGrids(); + } else { + return getStaticGrid(); + } + } + + @Override + public VortexData getDto(int timeIndex) { + return createGridFromDataSlice(0, 0, timeIndex); + } + + @Override + public int getDtoCount() { + Dimension timeDim = gridDatatype.getTimeDimension(); + return timeDim != null ? timeDim.getLength() : 1; + } + + @Override + public List getDataIntervals() { + return timeBounds; + } + + /* Grid Data Helpers */ + private List getMultiDimensionGrids(Dimension timeDim, Dimension endDim, Dimension rtDim) { + List gridList = new ArrayList<>(); + + for (int rtIndex = 0; rtIndex < rtDim.getLength(); rtIndex++) { + for (int endIndex = 0; endIndex < endDim.getLength(); endIndex++) { + for (int timeIndex = 0; timeIndex < timeDim.getLength(); timeIndex++) { + VortexGrid vortexGrid = createGridFromDataSlice(rtIndex, endIndex, timeIndex); + // Note: still add to list even if vortexGrid is null + gridList.add(vortexGrid); + } + } + } + + return gridList; + } + + private List getTemporalGrids() { + List gridList = new ArrayList<>(); + for (int i = 0; i < timeBounds.size(); i++) { + VortexData vortexGrid = getDto(i); + gridList.add(vortexGrid); + } + return gridList; + } + + private List getStaticGrid() { + VortexData staticGrid = createGridFromDataSlice(0, 0, 0); + return staticGrid != null ? List.of(staticGrid) : Collections.emptyList(); + } + + /** + * Creates a VortexGrid object by reading a specific data slice and assembling it into a grid. + * This method retrieves a slice of data based on specified indices for runtime, ensemble, and time dimensions, + * converts the raw data into a float array, and then builds a grid using the processed data and associated time record. + * + * @param rtIndex the runtime dimension index. If < 0, all runtimes are included. + * @param endIndex the ensemble dimension index. If < 0, all ensembles are included. + * @param timeIndex the time dimension index. Specifies a particular time slice to include. + * @return a VortexGrid object representing the specified data slice, or null if an IOException occurs. + */ + private VortexGrid createGridFromDataSlice(int rtIndex, int endIndex, int timeIndex) { + try { + Array array = gridDatatype.readDataSlice(rtIndex, endIndex, timeIndex, -1, -1, -1); + float[] slice = getFloatArray(array); + float[] data = getGridData(slice); + VortexDataInterval timeRecord = (timeIndex < timeBounds.size()) ? timeBounds.get(timeIndex) : VortexDataInterval.UNDEFINED; + return buildGrid(data, timeRecord); + } catch (IOException e) { + logger.severe(e.getMessage()); + return null; + } + } + + private float[] getFloatArray(Array array) { + // Check commit [939d691a] for details + return (float[]) array.get1DJavaArray(DataType.FLOAT); + } + + private float[] getGridData(float[] slice) { + if (gridCoordSystem.isRegularSpatial()) { + return slice; + } + + IndexSearcher indexSearcher = IndexSearcherFactory.INSTANCE.getOrCreate(gridCoordSystem); + indexSearcher.addPropertyChangeListener(support::firePropertyChange); + + Coordinate[] coordinates = gridDefinition.getGridCellCentroidCoords(); + indexSearcher.cacheCoordinates(coordinates); + + float[] data = new float[coordinates.length]; + for (int i = 0; i < data.length; i++) { + Coordinate coordinate = coordinates[i]; + int index = indexSearcher.getIndex(coordinate.x, coordinate.y); + data[i] = index >= 0 ? slice[index] : (float) variableDS.getFillValue(); + } + + return data; + } + + private VortexGrid buildGrid(float[] data, VortexDataInterval timeRecord) { + // Grid must be shifted after getData call since getData uses the original locations to map values. + Grid grid = Grid.toBuilder(gridDefinition).build(); + shiftGrid(grid); + + return VortexGrid.builder() + .dx(grid.getDx()).dy(grid.getDy()) + .nx(grid.getNx()).ny(grid.getNy()) + .originX(grid.getOriginX()).originY(grid.getOriginY()) + .wkt(grid.getCrs()) + .data(data) + .noDataValue(variableDS.getFillValue()) + .units(getUnits(variableDS)) + .fileName(path) + .shortName(gridDatatype.getShortName()) + .fullName(gridDatatype.getFullName()) + .description(gridDatatype.getDescription()) + .startTime(timeRecord.startTime()) + .endTime(timeRecord.endTime()) + .interval(timeRecord.getRecordDuration()) + .dataType(getVortexDataType(variableDS)) + .build(); + } + + /* Grid Definition Helpers */ + private static Grid getGridDefinition(GridCoordSystem coordinateSystem) { + AtomicReference grid = new AtomicReference<>(); + + CoordinateAxis xAxis = coordinateSystem.getXHorizAxis(); + CoordinateAxis yAxis = coordinateSystem.getYHorizAxis(); + + int nx; + int ny; + + double[] edgesX; + double[] edgesY; + + if (xAxis instanceof CoordinateAxis1D xCoord && yAxis instanceof CoordinateAxis1D yCoord) { + nx = (int) xAxis.getSize(); + ny = (int) yAxis.getSize(); + edgesX = xCoord.getCoordEdges(); + edgesY = yCoord.getCoordEdges(); + } else if (xAxis instanceof CoordinateAxis2D xAxis2D && yAxis instanceof CoordinateAxis2D yAxis2D) { + int shapeX = xAxis2D.getEdges().getShape()[1] - 1; + double minX = xAxis2D.getMinValue(); + double maxX = xAxis2D.getMaxValue(); + double dx = (maxX - minX) / (shapeX - 1); + + int shapeY = xAxis2D.getEdges().getShape()[0] - 1; + double minY = yAxis2D.getMinValue(); + double maxY = yAxis2D.getMaxValue(); + double dy = (maxY - minY) / (shapeY - 1); + + double cellSize = (dx + dy) / 2; + + nx = (int) Math.round((maxX - minX) / cellSize); + ny = (int) Math.round((maxY - minY) / cellSize); + + edgesX = new double[nx]; + for (int i = 0; i < nx; i++) { + edgesX[i] = minX + i * cellSize; + } + + edgesY = new double[ny]; + for (int i = 0; i < ny; i++) { + edgesY[i] = minY + i * cellSize; + } + } else { + throw new IllegalStateException(); + } + + double ulx = edgesX[0]; + double urx = edgesX[edgesX.length - 1]; + double dx = (urx - ulx) / nx; + + double uly = edgesY[0]; + double lly = edgesY[edgesY.length - 1]; + double dy = (lly - uly) / ny; + + String wkt = WktFactory.createWkt(coordinateSystem.getProjection()); + + grid.set(Grid.builder() + .nx(nx).ny(ny) + .dx(dx).dy(dy) + .originX(ulx).originY(uly) + .crs(wkt) + .build()); + + String xAxisUnits = Objects.requireNonNull(xAxis).getUnitsString(); + Unit cellUnits = UnitUtil.getUnits(xAxisUnits.toLowerCase()); + Unit csUnits = ReferenceUtils.getLinearUnits(wkt); + + // This will scale the grid if cellUnits and csUnits do not align + // e.g. cellUnits are in meters but csUnits are in kilometers + // isCompatible is simply checking if the units are of type length so that no scaling is attempted + // between DEGREE_ANGLE and ONE + if (cellUnits.isCompatible(METRE) && csUnits.isCompatible(METRE) && !cellUnits.equals(csUnits)) { + grid.set(scaleGrid(grid.get(), cellUnits, csUnits)); + } + + return grid.get(); + } + + private String getUnits(VariableDS variableDS) { + // Add handling logic for MRMS IDP version 12.0.0 + if (variableName.toLowerCase().contains("var209-6")) { + return "mm"; + } + + return variableDS.getUnitsString(); + } + + /* Time Record Helpers */ + private List getTimeBounds(GridCoordSystem gcs) { + if (gcs.hasTimeAxis1D()) { + return getTimeRecordsWithTimeAxis1D(gcs.getTimeAxis1D()); + } else if (gcs.hasTimeAxis()) { + return this.getDataIntervals(); + } else { + VortexDataInterval timeRecord = getTimeRecordFromAttributes(); + return timeRecord != null ? List.of(timeRecord) : Collections.emptyList(); + } + } + + private VortexDataInterval getTimeRecordFromAttributes() { + // No time axes found. Try reading times from attributes. + VortexDataInterval startStopDateRecord = getTimeRecordFromStartStopDate(); + return startStopDateRecord != null ? startStopDateRecord : getTimeRecordFromNominalProductTime(); + } + + private VortexDataInterval getTimeRecordFromStartStopDate() { + Attribute startDateAttribute = gridDatatype.findAttributeIgnoreCase("start_date"); + Attribute endDateAttribute = gridDatatype.findAttributeIgnoreCase("stop_date"); + + String startDateString = getAttributeStringValue(startDateAttribute); + String endDateString = getAttributeStringValue(endDateAttribute); + if (startDateString == null || endDateString == null) return null; + + ZonedDateTime startDate = ZonedDateTime.parse(startDateString, dateTimeFormatter); + ZonedDateTime endDate = ZonedDateTime.parse(endDateString, dateTimeFormatter); + return VortexDataInterval.of(startDate, endDate); + } + + private VortexDataInterval getTimeRecordFromNominalProductTime() { + Attribute nominalProductTimeAttribute = gridDataset.findGlobalAttributeIgnoreCase("nominal_product_time"); + String timeString = getAttributeStringValue(nominalProductTimeAttribute); + if (timeString == null) return null; + + CalendarDate calendarDate = CalendarDate.parseISOformat(null, timeString); + LocalDateTime ldt = LocalDateTime.parse(calendarDate.toString(), DateTimeFormatter.ISO_DATE_TIME); + ZonedDateTime time = ZonedDateTime.of(ldt, ZoneId.of("UTC")); + return VortexDataInterval.of(time, time); + } + + private String getAttributeStringValue(Attribute attribute) { + return Optional.ofNullable(attribute).map(Attribute::getStringValue).orElse(null); + } + + private List getTimeRecordsWithTimeAxis1D(CoordinateAxis1DTime tAxis) { + List list = new ArrayList<>(); + if (!tAxis.isInterval() && !isSpecialTimeBounds()) { + return getTimeInstants(tAxis); + } + + for (int i = 0; i < (int) tAxis.getSize(); i++) { + VortexDataInterval timeRecord = adjustTimeForSpecialFile(tAxis, i); + list.add(timeRecord); + } + + return list; + } + + private VortexDataInterval adjustTimeForSpecialFile(CoordinateAxis1DTime tAxis, int time) { + CalendarDate[] dates = tAxis.getCoordBoundsDate(time); + ZonedDateTime startTime = TimeConverter.toZonedDateTime(dates[0]); + ZonedDateTime endTime = TimeConverter.toZonedDateTime(dates[1]); + ZonedDateTime instantTime = TimeConverter.toZonedDateTime(tAxis.getCalendarDate(time)); + ZonedDateTime initialTime = TimeConverter.toZonedDateTime(tAxis.getCalendarDate(0)); + + ZonedDateTime adjustedStart = switch (specialFileType) { + case ABRSC_GAUGE, ABRSC_RADAR, MRMS_IDP, NLDAS_APCP -> startTime.minusHours(1); + case MRMS_PRECIP -> startTime.minusMinutes(5); + case MRMS_PRECIP_2_MIN -> startTime.minusMinutes(2); + case GPM, AORC_TMP, LIVNEH_PRECIP -> instantTime; + case AORC_APCP -> instantTime.minusHours(1); + case UA_SWE -> initialTime; + case GFS -> startTime.plusMinutes(90); + default -> startTime; + }; + + ZonedDateTime adjustedEnd = switch (specialFileType) { + case HRRR_WRFSFCF, CMORPH -> adjustedStart.plusHours(1); + case GPM -> adjustedStart.plusMinutes(30); + case AORC_APCP, AORC_TMP -> instantTime; + case UA_SWE, LIVNEH_PRECIP -> adjustedStart.plusDays(1); + case GEFS -> adjustedStart.plusHours(3); + case GFS -> endTime.plusMinutes(90); + default -> endTime; + }; + + return VortexDataInterval.of(adjustedStart, adjustedEnd); + } + + private boolean isSpecialTimeBounds() { + return specialFileType != null && specialFileType != UNDEFINED; + } + + private List getTimeInstants(CoordinateAxis1DTime timeAxis) { + return timeAxis.getCalendarDates().stream() + .map(TimeConverter::toZonedDateTime) + .map(t -> VortexDataInterval.of(t, t)) + .toList(); + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/Hdf5RasPrecipDataWriter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/Hdf5RasPrecipDataWriter.java index f3a42363..f66d7bc8 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/Hdf5RasPrecipDataWriter.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/Hdf5RasPrecipDataWriter.java @@ -1,10 +1,10 @@ package mil.army.usace.hec.vortex.io; +import hdf.hdf5lib.H5; +import hdf.hdf5lib.HDF5Constants; import hdf.hdf5lib.exceptions.HDF5Exception; import hdf.hdf5lib.exceptions.HDF5LibraryException; import mil.army.usace.hec.vortex.VortexGrid; -import hdf.hdf5lib.H5; -import hdf.hdf5lib.HDF5Constants; import java.io.File; import java.nio.file.Path; @@ -15,7 +15,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; -public class Hdf5RasPrecipDataWriter extends DataWriter { +class Hdf5RasPrecipDataWriter extends DataWriter { private static final Logger logger = Logger.getLogger(Hdf5RasPrecipDataWriter.class.getName()); diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/IndexMap.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/IndexMap.java new file mode 100644 index 00000000..b15b1ab9 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/IndexMap.java @@ -0,0 +1,24 @@ +package mil.army.usace.hec.vortex.io; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +final class IndexMap { + private final Map internalMap; + + private IndexMap(Map internalMap) { + this.internalMap = internalMap; + } + + static IndexMap of(List sortedList) { + Map map = IntStream.range(0, sortedList.size()).boxed() + .collect(Collectors.toMap(sortedList::get, index -> index)); + return new IndexMap<>(map); + } + + int indexOf(K object) { + return internalMap.getOrDefault(object, -1); + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/InstantaneousRecordIndexQuery.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/InstantaneousRecordIndexQuery.java new file mode 100644 index 00000000..bddce9d7 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/InstantaneousRecordIndexQuery.java @@ -0,0 +1,96 @@ +package mil.army.usace.hec.vortex.io; + +import java.time.ZonedDateTime; +import java.util.*; +import java.util.logging.Logger; + +final class InstantaneousRecordIndexQuery implements RecordIndexQuery { + private static final Logger logger = Logger.getLogger(InstantaneousRecordIndexQuery.class.getName()); + private final NavigableMap instantaneousDataTree; + + private InstantaneousRecordIndexQuery(List recordList) { + this.instantaneousDataTree = initInstantaneousDataTree(recordList); + } + + static InstantaneousRecordIndexQuery from(List recordList) { + return new InstantaneousRecordIndexQuery(recordList); + } + + /* Query */ + @Override + public List query(ZonedDateTime startTime, ZonedDateTime endTime) { + if (startTime.isEqual(endTime)) { + return queryPoint(instantaneousDataTree, startTime); + } else { + return queryPeriod(instantaneousDataTree, startTime, endTime); + } + } + + @Override + public ZonedDateTime getEarliestStartTime() { + return !instantaneousDataTree.isEmpty() ? instantaneousDataTree.firstKey() : null; + } + + @Override + public ZonedDateTime getLatestEndTime() { + return !instantaneousDataTree.isEmpty() ? instantaneousDataTree.lastKey() : null; + } + + private static List queryPoint(NavigableMap instantaneousDataTree, ZonedDateTime time) { + Map.Entry floorEntry = instantaneousDataTree.floorEntry(time); + Map.Entry ceilingEntry = instantaneousDataTree.ceilingEntry(time); + + if (floorEntry == null || ceilingEntry == null) { + logger.info("Unable to find overlapped instant grid(s)"); + return Collections.emptyList(); + } + + int floor = floorEntry.getValue(); + int ceiling = ceilingEntry.getValue(); + + if (floor == ceiling) { + return Collections.singletonList(floor); + } else { + return List.of(floor, ceiling); + } + } + + private static List queryPeriod(NavigableMap instantaneousDataTree, ZonedDateTime start, ZonedDateTime end) { + ZonedDateTime adjustedStart = Optional.ofNullable(instantaneousDataTree.floorEntry(start)) + .map(Map.Entry::getKey) + .orElse(start); + + ZonedDateTime adjustedEnd = Optional.ofNullable(instantaneousDataTree.ceilingEntry(end)) + .map(Map.Entry::getKey) + .orElse(end); + + boolean inclusiveStart = true; + boolean inclusiveEnd = true; + SortedMap subMap = instantaneousDataTree.subMap(adjustedStart, inclusiveStart, adjustedEnd, inclusiveEnd); + + if (subMap == null || subMap.isEmpty()) { + logger.info("Unable to find overlapped grid(s) for period instant data"); + return Collections.emptyList(); + } + + return subMap.values().stream().toList(); + } + + /* Helpers */ + private static NavigableMap initInstantaneousDataTree(List recordList) { + TreeMap treeMap = new TreeMap<>(); + + for (int i = 0; i < recordList.size(); i++) { + VortexDataInterval timeRecord = recordList.get(i); + boolean isUndefined = !VortexDataInterval.isDefined(timeRecord); + + if (isUndefined || !timeRecord.isInstantaneous()) { + continue; + } + + treeMap.put(timeRecord.startTime(), i); + } + + return Collections.unmodifiableNavigableMap(treeMap); + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/Interval.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/Interval.java new file mode 100644 index 00000000..13e5d2d6 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/Interval.java @@ -0,0 +1,33 @@ +package mil.army.usace.hec.vortex.io; + +/** + * Closed-open, [), interval on the integer number line. + */ +interface Interval extends Comparable { + + /** + * Returns the starting point of this. + */ + long startEpochSecond(); + + /** + * Returns the ending point of this. + *

+ * The interval does not include this point. + */ + long endEpochSecond(); + + default boolean overlaps(Interval o) { + return endEpochSecond() > o.startEpochSecond() && o.endEpochSecond() > startEpochSecond(); + } + + default int compareTo(Interval o) { + if (startEpochSecond() > o.startEpochSecond()) { + return 1; + } else if (startEpochSecond() < o.startEpochSecond()) { + return -1; + } else { + return Long.compare(endEpochSecond(), o.endEpochSecond()); + } + } +} \ No newline at end of file diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/IntervalTree.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/IntervalTree.java new file mode 100644 index 00000000..9a4e22f3 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/IntervalTree.java @@ -0,0 +1,117 @@ +package mil.army.usace.hec.vortex.io; + +import java.util.*; + +final class IntervalTree { + private final Node root; + + /* Constructor */ + private IntervalTree(List intervals) { + List sortedByStart = intervals.stream() + .filter(Objects::nonNull) + .sorted(Comparator.comparing(T::startEpochSecond)) + .toList(); + root = buildTree(sortedByStart, 0, intervals.size() - 1); + } + + static IntervalTree from(List intervals) { + return new IntervalTree<>(intervals); + } + + /* Tree's Node (includes maxEnd for IntervalTree algorithm) */ + private class Node { + private final T interval; + private long maxEnd; + private Node left; + private Node right; + + Node(T interval) { + this.interval = interval; + } + } + + /* Tree Initialization */ + private Node buildTree(List intervals, int start, int end) { + if (intervals.isEmpty() || start > end) { + return null; + } + + int mid = (start + end) / 2; + + Node node = new Node(intervals.get(mid)); + node.left = buildTree(intervals, start, mid - 1); + node.right = buildTree(intervals, mid + 1, end); + node.maxEnd = intervals.get(mid).endEpochSecond(); + + if (node.left != null) { + node.maxEnd = Math.max(node.maxEnd, node.left.maxEnd); + } + + if (node.right != null) { + node.maxEnd = Math.max(node.maxEnd, node.right.maxEnd); + } + + return node; + } + + /* Queries */ + public List findOverlaps(T target) { + if (root == null) { + return Collections.emptyList(); + } + + List overlaps = new ArrayList<>(); + collectOverlaps(root, target, overlaps); + return overlaps; + } + + public T findMinimum() { + if (root == null) { + return null; + } + + Node current = root; + while (current.left != null) { + current = current.left; + } + return current.interval; + } + + public T findMaximum() { + if (root == null) { + return null; + } + + Node current = root; + long maxEnd = current.maxEnd; // Start with the maxEnd of the root + while (current != null) { + if (current.maxEnd == maxEnd) { + if (current.right != null && current.right.maxEnd == maxEnd) { + current = current.right; + } else { + return current.interval; // This node contains the max end + } + } else { + current = current.right; // Continue to node with potentially higher maxEnd + } + } + return null; + } + + /* Helpers */ + private void collectOverlaps(Node node, T target, List overlaps) { + if (node == null) return; + // Check for overlap with the current node's interval + if (node.interval.overlaps(target)) { + overlaps.add(node.interval); + } + // Continue searching in the left subtree if there's a potential overlap + if (node.left != null && node.left.maxEnd > target.startEpochSecond()) { + collectOverlaps(node.left, target, overlaps); + } + // Continue searching in the right subtree + if (node.right != null && node.interval.startEpochSecond() < target.endEpochSecond()) { + collectOverlaps(node.right, target, overlaps); + } + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/MemoryDynamicWriteDataBuffer.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/MemoryDynamicWriteDataBuffer.java new file mode 100644 index 00000000..64212b9a --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/MemoryDynamicWriteDataBuffer.java @@ -0,0 +1,47 @@ +package mil.army.usace.hec.vortex.io; + +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.function.Consumer; +import java.util.stream.Stream; + +final class MemoryDynamicWriteDataBuffer implements WriteDataBuffer { + private final Queue buffer = new ConcurrentLinkedQueue<>(); + + @Override + public void add(T data) { + buffer.add(data); + } + + @Override + public void clear() { + buffer.clear(); + } + + @Override + public boolean isFull() { + return !MemoryManager.isMemoryAvailable(); + } + + @Override + public Stream getBufferAsStream() { + return buffer.stream(); + } + + @Override + public synchronized void processBufferAndClear(Consumer> bufferProcessFunction, boolean isForced) { + if (isFull() || isForced) { + bufferProcessFunction.accept(getBufferAsStream()); + clear(); + } + } + + @Override + public synchronized void addAndProcessWhenFull(T data, Consumer> bufferProcessFunction, boolean isForced) { + if (isFull() || isForced) { + processBufferAndClear(bufferProcessFunction, isForced); + } + + add(data); + } +} \ No newline at end of file diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/MemoryManager.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/MemoryManager.java new file mode 100644 index 00000000..7105e0ac --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/MemoryManager.java @@ -0,0 +1,14 @@ +package mil.army.usace.hec.vortex.io; + +final class MemoryManager { + private static final Runtime runtime = Runtime.getRuntime(); + private static final double DEFAULT_MEMORY_THRESHOLD = 0.8; + + private MemoryManager() { + // Utility Class + } + + static boolean isMemoryAvailable() { + return (runtime.totalMemory() - runtime.freeMemory()) / (double) runtime.maxMemory() < DEFAULT_MEMORY_THRESHOLD; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfBatchImporter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfBatchImporter.java new file mode 100644 index 00000000..446a4238 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfBatchImporter.java @@ -0,0 +1,128 @@ +package mil.army.usace.hec.vortex.io; + +import hec.heclib.dss.DSSPathname; +import mil.army.usace.hec.vortex.VortexData; +import mil.army.usace.hec.vortex.VortexGrid; +import mil.army.usace.hec.vortex.VortexProperty; +import mil.army.usace.hec.vortex.geo.GeographicProcessor; +import mil.army.usace.hec.vortex.util.Stopwatch; + +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.logging.Logger; +import java.util.stream.Stream; + +class NetcdfBatchImporter extends BatchImporter { + private static final Logger logger = Logger.getLogger(NetcdfBatchImporter.class.getName()); + + private final AtomicInteger totalDataCount = new AtomicInteger(0); + private final AtomicInteger doneDataCount = new AtomicInteger(0); + + // Netcdf Batch Importer + NetcdfBatchImporter(Builder builder) { + super(builder); + } + + @Override + public void process() { + Stopwatch stopwatch = new Stopwatch(); + stopwatch.start(); + + // Geoprocessor + GeographicProcessor geoProcessor = new GeographicProcessor(geoOptions); + + // Prepare NetCDF file to append + List dataReaders = getDataReaders(); + totalDataCount.set(dataReaders.stream().mapToInt(DataReader::getDtoCount).sum()); + + Stream sortedRecordStream = dataReaders.parallelStream() + .map(DataReader::getDataIntervals) + .flatMap(Collection::stream) + .distinct() + .sorted(Comparator.comparing(VortexDataInterval::startTime)); + NetcdfWriterPrep.initializeForAppend(destination.toString(), representativeCollection(dataReaders, geoOptions), sortedRecordStream); + + // Buffered Data Writer + WriteDataBuffer bufferedDataWriter = WriteDataBuffer.of(WriteDataBuffer.Type.MEMORY_DYNAMIC); + + // Buffered Read and Process + Consumer> bufferProcessFunction = stream -> writeBufferedData(destination, writeOptions, stream); + dataReaders.parallelStream() + .map(DataReader::getDtos) + .flatMap(Collection::stream) + .filter(VortexGrid.class::isInstance) + .map(VortexGrid.class::cast) + .map(geoProcessor::process) + .forEach(grid -> bufferedDataWriter.addAndProcessWhenFull(grid, bufferProcessFunction, false)); + bufferedDataWriter.processBufferAndClear(bufferProcessFunction, true); + + stopwatch.end(); + String timeMessage = "Batch import time: " + stopwatch; + logger.info(timeMessage); + + support.firePropertyChange(VortexProperty.STATUS, null, null); + } + + private void writeBufferedData(Path destination, Map writeOptions, Stream bufferedData) { + Comparator comparator = Comparator.comparing(VortexData::startTime).thenComparing(VortexData::endTime); + List bufferedDataList = bufferedData.sorted(comparator).toList(); + + DataWriter dataWriter = DataWriter.builder() + .destination(destination) + .options(writeOptions) + .data(bufferedDataList) + .build(); + + if (!(dataWriter instanceof NetcdfDataWriter netcdfDataWriter)) { + return; + } + + netcdfDataWriter.appendData(); + + // Update Progress + double doneCount = doneDataCount.addAndGet(bufferedDataList.size()); + double totalCount = totalDataCount.addAndGet(bufferedDataList.size()); + int progress = (int) ((doneCount / totalCount) * 100); + support.firePropertyChange(VortexProperty.PROGRESS, null, progress); + logger.info(() -> "Completed Buffered Write Count: " + bufferedDataList.size()); + } + + private static VortexGridCollection representativeCollection(List readers, Map geoOptions) { + GeographicProcessor geoProcessor = new GeographicProcessor(geoOptions); + Set seenVariables = new HashSet<>(); + + List processedFirstGrids = readers.stream() + .filter(reader -> isUniqueVariableReader(seenVariables, reader)) + .map(r -> r.getDto(0)) + .filter(VortexGrid.class::isInstance) + .map(VortexGrid.class::cast) + .map(geoProcessor::process) + .toList(); + VortexGridCollection vortexGridCollection = VortexGridCollection.of(processedFirstGrids); + List uniqueGrids = List.copyOf(vortexGridCollection.getRepresentativeGridNameMap().values()); + return VortexGridCollection.of(uniqueGrids); + } + + private static boolean isUniqueVariableReader(Set seenVariables, DataReader dataReader) { + String pathToFile = dataReader.path; + String variableName = dataReader.variableName; + + if (pathToFile.toLowerCase().endsWith(".dss")) { + variableName = retrieveDssVariableName(variableName); + } + + if (seenVariables.contains(variableName)) { + return false; + } else { + seenVariables.add(variableName); + return true; + } + } + + private static String retrieveDssVariableName(String variableName) { + DSSPathname dssPathname = new DSSPathname(variableName); + return dssPathname.cPart(); + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfDataReader.java index 0487b3ca..35f64d2e 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfDataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfDataReader.java @@ -1,166 +1,89 @@ package mil.army.usace.hec.vortex.io; -import mil.army.usace.hec.vortex.*; -import mil.army.usace.hec.vortex.geo.*; -import mil.army.usace.hec.vortex.util.TimeConverter; -import org.locationtech.jts.geom.Coordinate; -import ucar.ma2.Array; -import ucar.ma2.DataType; -import ucar.nc2.Attribute; -import ucar.nc2.Dimension; +import mil.army.usace.hec.vortex.VortexData; +import mil.army.usace.hec.vortex.VortexDataType; +import mil.army.usace.hec.vortex.geo.Grid; +import mil.army.usace.hec.vortex.geo.ReferenceUtils; import ucar.nc2.Variable; import ucar.nc2.constants.AxisType; import ucar.nc2.constants.CF; import ucar.nc2.constants.FeatureType; import ucar.nc2.dataset.*; -import ucar.nc2.dt.GridCoordSystem; -import ucar.nc2.dt.GridDatatype; import ucar.nc2.dt.grid.GridDataset; import ucar.nc2.ft.FeatureDataset; import ucar.nc2.ft.FeatureDatasetFactoryManager; -import ucar.nc2.time.CalendarDate; -import ucar.unidata.geoloc.Projection; -import ucar.unidata.geoloc.ProjectionImpl; import javax.measure.IncommensurableException; import javax.measure.Unit; import javax.measure.UnitConverter; -import java.io.File; import java.io.IOException; -import java.time.*; -import java.time.format.DateTimeFormatter; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.util.*; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import static javax.measure.MetricPrefix.KILO; -import static systems.uom.common.USCustomary.DEGREE_ANGLE; -import static tech.units.indriya.AbstractUnit.ONE; -import static tech.units.indriya.unit.Units.METRE; +abstract class NetcdfDataReader extends DataReader { + private static final Logger logger = Logger.getLogger(NetcdfDataReader.class.getName()); + final NetcdfDataset ncd; -public class NetcdfDataReader extends DataReader { + /* Factory Method */ + public static NetcdfDataReader createInstance(String pathToFile, String pathToData) { + NetcdfDataset dataset = getNetcdfDataset(pathToFile); + if (dataset == null) { + return null; + } - private static final Logger logger = Logger.getLogger(NetcdfDataReader.class.getName()); + GridDataset gridDataset = getGridDataset(dataset); + if (gridDataset != null) { + return new GridDatasetReader(gridDataset, pathToData); + } + + VariableDS variableDS = getVariableDataset(dataset, pathToData); + if (variableDS != null) { + return new VariableDsReader(variableDS, pathToData); + } + + return null; + } NetcdfDataReader(DataReaderBuilder builder) { super(builder); + ncd = getNetcdfDataset(path); } @Override - public List getDtos() { - try (NetcdfDataset ncd = NetcdfDatasets.openDataset(path); Formatter errlog = new Formatter()) { - FeatureDataset dataset = FeatureDatasetFactoryManager.wrap(FeatureType.GRID, ncd, null, errlog); - if (dataset != null) { - FeatureType ftype = dataset.getFeatureType(); - if (ftype == FeatureType.GRID - && dataset instanceof GridDataset gridDataset - && gridDataset.findGridDatatype(variableName) != null) { - return getData(gridDataset, variableName); - } - } + public abstract List getDtos(); - List variables = ncd.getVariables(); - for (Variable variable : variables) { - if (variable.getShortName().equals(variableName) - && variable instanceof VariableDS variableDS) { - int count = getDtoCount(variableDS); - - VariableDsReader reader = VariableDsReader.builder() - .setNetcdfFile(ncd) - .setVariableName(variableName) - .build(); - - List dataList = new ArrayList<>(); - for (int i = 0; i < count; i++) { - VortexData data = reader.read(i); - dataList.add(data); - } - return dataList; - } - } + @Override + public abstract VortexData getDto(int idx); - } catch (IOException e) { - logger.log(Level.SEVERE, e, e::getMessage); + @Override + public List getDataIntervals() { + if (!(ncd.findCoordinateAxis(AxisType.Time) instanceof CoordinateAxis1D timeAxis)) { return Collections.emptyList(); } - return Collections.emptyList(); - } - - @Override - public VortexData getDto(int idx) { - try (NetcdfDataset ncd = NetcdfDatasets.openDataset(path); Formatter errlog = new Formatter()) { - FeatureDataset dataset = FeatureDatasetFactoryManager.wrap(FeatureType.GRID, ncd, null, errlog); - if (dataset != null) { - FeatureType ftype = dataset.getFeatureType(); - if (ftype == FeatureType.GRID - && dataset instanceof GridDataset gridDataset - && gridDataset.findGridDatatype(variableName) != null) { - return getData(gridDataset, variableName, idx); - } - } - List variables = ncd.getVariables(); - for (Variable variable : variables) { - if (variable.getShortName().equals(variableName) - && variable instanceof VariableDS variableDS) { - List coordinateSystems = variableDS.getCoordinateSystems(); + String timeAxisUnits = timeAxis.getUnitsString(); - boolean isLatLon = ncd.findCoordinateAxis(AxisType.Lon) != null - && ncd.findCoordinateAxis(AxisType.Lat) != null; - - if (!coordinateSystems.isEmpty() || isLatLon) { - VariableDsReader reader = VariableDsReader.builder() - .setNetcdfFile(ncd) - .setVariableName(variableName) - .build(); + int count = (int) timeAxis.getSize(); + double[] startTimes = timeAxis.getBound1(); + double[] endTimes = timeAxis.getBound2(); - return reader.read(idx); - } - } - } - } catch (IOException e) { - logger.log(Level.SEVERE, e, e::getMessage); + List timeRecords = new ArrayList<>(); + for (int i = 0; i < count; i++) { + ZonedDateTime startTime = parseTime(timeAxisUnits, startTimes[i]); + ZonedDateTime endTime = parseTime(timeAxisUnits, endTimes[i]); + VortexDataInterval timeRecord = VortexDataInterval.of(startTime, endTime); + timeRecords.add(timeRecord); } - return null; + return timeRecords; } @Override - public int getDtoCount() { - try (NetcdfDataset ncd = NetcdfDatasets.openDataset(path); Formatter errlog = new Formatter()) { - FeatureDataset dataset = FeatureDatasetFactoryManager.wrap(FeatureType.GRID, ncd, null, errlog); - if (dataset != null) { - FeatureType ftype = dataset.getFeatureType(); - if (ftype == FeatureType.GRID - && dataset instanceof GridDataset gridDataset - && gridDataset.findGridDatatype(variableName) != null) { - return getDtoCount(gridDataset, variableName); - } - } - - List variables = ncd.getVariables(); - for (Variable variable : variables) { - if (variable.getShortName().equals(variableName) && variable instanceof VariableDS variableDS) { - List coordinateSystems = variableDS.getCoordinateSystems(); - - boolean isLatLon = ncd.findCoordinateAxis(AxisType.Lon) != null - && ncd.findCoordinateAxis(AxisType.Lat) != null; - - if (!coordinateSystems.isEmpty() || isLatLon) { - return getDtoCount(variableDS); - } - } - } - - } catch (IOException e) { - logger.log(Level.SEVERE, e, e::getMessage); - } - - return 0; - } + public abstract int getDtoCount(); public static Set getVariables(String path) { try (NetcdfDataset ncd = NetcdfDatasets.openDataset(path)) { @@ -186,483 +109,66 @@ public static Set getVariables(String path) { return Collections.emptySet(); } - private static boolean isSelectableVariable(Variable variable) { - boolean isVariableDS = variable instanceof VariableDS; - boolean isNotAxis = !(variable instanceof CoordinateAxis); - return isVariableDS && isNotAxis; - } - - private float[] getFloatArray(Array array) { - return (float[]) array.get1DJavaArray(DataType.FLOAT); - } - - private List getData(GridDataset gridDataset, String variable) { - GridDatatype gridDatatype = gridDataset.findGridDatatype(variable); - GridCoordSystem gcs = gridDatatype.getCoordinateSystem(); - - VariableDS variableDs = gridDatatype.getVariable(); - double noDataValue = variableDs.getFillValue(); - - Grid grid = getGrid(gcs); - String wkt = getWkt(gcs.getProjection()); - - VortexDataType vortexDataType = getVortexDataType(variableDs); - - List times = getTimeBounds(gridDataset, variable); - - Dimension timeDim = gridDatatype.getTimeDimension(); - Dimension endDim = gridDatatype.getEnsembleDimension(); - Dimension rtDim = gridDatatype.getRunTimeDimension(); - - List grids = new ArrayList<>(); - - if (timeDim != null && endDim != null && rtDim != null) { - IntStream.range(0, rtDim.getLength()).forEach(rtIndex -> - IntStream.range(0, endDim.getLength()).forEach(ensIndex -> - IntStream.range(0, timeDim.getLength()).forEach(timeIndex -> { - Array array; - try { - array = gridDatatype.readDataSlice(rtIndex, ensIndex, timeIndex, -1, -1, -1); - float[] slice = getFloatArray(array); - float[] data = getData(slice, gcs, grid, noDataValue); - - ZonedDateTime startTime = times.get(timeIndex)[0]; - ZonedDateTime endTime = times.get(timeIndex)[1]; - Duration interval = Duration.between(startTime, endTime); - - // Grid must be shifted after getData call since getData uses the original locations - // to map values. - shiftGrid(grid); - - grids.add(VortexGrid.builder() - .dx(grid.getDx()).dy(grid.getDy()) - .nx(grid.getNx()).ny(grid.getNy()) - .originX(grid.getOriginX()).originY(grid.getOriginY()) - .wkt(wkt) - .data(data) - .noDataValue(noDataValue) - .units(gridDatatype.getUnitsString()) - .fileName(gridDataset.getLocation()) - .shortName(gridDatatype.getShortName()) - .fullName(gridDatatype.getFullName()) - .description(gridDatatype.getDescription()) - .startTime(startTime) - .endTime(endTime) - .interval(interval) - .dataType(vortexDataType) - .build()); - } catch (IOException e) { - logger.log(Level.SEVERE, e, e::getMessage); - } - }))); - } else if (timeDim != null) { - IntStream.range(0, times.size()).forEach(timeIndex -> { - try { - Array array = gridDatatype.readDataSlice(timeIndex, -1, -1, -1); - float[] slice = getFloatArray(array); - float[] data = getData(slice, gcs, grid, noDataValue); - - ZonedDateTime startTime = times.get(timeIndex)[0]; - ZonedDateTime endTime = times.get(timeIndex)[1]; - Duration interval = Duration.between(startTime, endTime); - - // Grid must be shifted after getData call since getData uses the original locations - // to map values. - shiftGrid(grid); - - grids.add(VortexGrid.builder() - .dx(grid.getDx()).dy(grid.getDy()) - .nx(grid.getNx()).ny(grid.getNy()) - .originX(grid.getOriginX()).originY(grid.getOriginY()) - .wkt(wkt) - .data(data) - .noDataValue(noDataValue) - .units(gridDatatype.getUnitsString()) - .fileName(gridDataset.getLocation()) - .shortName(gridDatatype.getShortName()) - .fullName(gridDatatype.getFullName()) - .description(gridDatatype.getDescription()) - .startTime(startTime) - .endTime(endTime) - .interval(interval) - .dataType(vortexDataType) - .build()); - } catch (IOException e) { - logger.log(Level.SEVERE, e, e::getMessage); - } - }); - } else { - try { - Array array = gridDatatype.readDataSlice(1, -1, -1, -1); - float[] slice = getFloatArray(array); - float[] data = getData(slice, gcs, grid, noDataValue); - - ZonedDateTime startTime; - ZonedDateTime endTime; - Duration interval; - if (!times.isEmpty()) { - startTime = times.get(0)[0]; - endTime = times.get(0)[1]; - interval = Duration.between(startTime, endTime); - } else { - startTime = null; - endTime = null; - interval = null; - } - - // Grid must be shifted after getData call since getData uses the original locations - // to map values. - shiftGrid(grid); - - grids.add(VortexGrid.builder() - .dx(grid.getDx()).dy(grid.getDy()) - .nx(grid.getNx()).ny(grid.getNy()) - .originX(grid.getOriginX()).originY(grid.getOriginY()) - .wkt(wkt) - .data(data) - .noDataValue(noDataValue) - .units(gridDatatype.getUnitsString()) - .fileName(gridDataset.getLocation()) - .shortName(gridDatatype.getShortName()) - .fullName(gridDatatype.getFullName()) - .description(gridDatatype.getDescription()) - .startTime(startTime) - .endTime(endTime) - .interval(interval) - .dataType(vortexDataType) - .build()); - } catch (IOException e) { - logger.log(Level.SEVERE, e, e::getMessage); - } - } - return grids; - } - - private static String getWkt(Projection projection) { - return WktFactory.createWkt((ProjectionImpl) projection); - } - - private List getTimeBounds(GridDataset gridDataset, String variable) { - GridDatatype gridDatatype = gridDataset.findGridDatatype(variable); - GridCoordSystem gcs = gridDatatype.getCoordinateSystem(); - - List list = new ArrayList<>(); - if (gcs.hasTimeAxis1D()) { - CoordinateAxis1DTime tAxis = gcs.getTimeAxis1D(); - - boolean isSpecialTimeBounds = isSpecialTimeBounds(); - - if (!tAxis.isInterval() && !isSpecialTimeBounds) - return getTimeInstants(tAxis); - - if (isSpecialTimeBounds) { - for (int i = 0; i < tAxis.getSize(); i++) { - ZonedDateTime[] zonedDateTimes = getSpecialTimeBounds(tAxis, i); - list.add(zonedDateTimes); - } - } else { - for (int i = 0; i < tAxis.getSize(); i++) { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - ZonedDateTime zdt0 = convert(dates[0]); - ZonedDateTime zdt1 = convert(dates[1]); - ZonedDateTime[] zonedDateTimes = new ZonedDateTime[]{zdt0, zdt1}; - list.add(zonedDateTimes); - } - } - - return list; - } - - if (gcs.hasTimeAxis()) { - CoordinateAxis1D tAxis = (CoordinateAxis1D) gcs.getTimeAxis(); - String units = tAxis.getUnitsString(); - - IntStream.range(0, (int) tAxis.getSize()).forEach(time -> { - String dateTimeString = (units.split(" ", 3)[2]).replaceFirst(" ", "T").split(" ")[0].replace(".", ""); - - ZonedDateTime origin; - - if (dateTimeString.contains("T")) { - origin = ZonedDateTime.of(LocalDateTime.parse(dateTimeString, DateTimeFormatter.ISO_DATE_TIME), ZoneId.of("UTC")); - } else { - origin = ZonedDateTime.of(LocalDate.parse(dateTimeString, DateTimeFormatter.ofPattern("uuuu-M-d")), LocalTime.of(0, 0), ZoneId.of("UTC")); - } - ZonedDateTime startTime; - ZonedDateTime endTime; - if (units.toLowerCase().matches("^month[s]? since.*$")) { - startTime = origin.plusMonths((long) tAxis.getBound1()[time]); - endTime = origin.plusMonths((long) tAxis.getBound2()[time]); - } else if (units.toLowerCase().matches("^day[s]? since.*$")) { - startTime = origin.plusSeconds((long) tAxis.getBound1()[time] * 86400); - endTime = origin.plusSeconds((long) tAxis.getBound2()[time] * 86400); - } else if (units.toLowerCase().matches("^hour[s]? since.*$")) { - startTime = origin.plusSeconds((long) tAxis.getBound1()[time] * 3600); - endTime = origin.plusSeconds((long) tAxis.getBound2()[time] * 3600); - } else if (units.toLowerCase().matches("^minute[s]? since.*$")) { - endTime = origin.plusSeconds((long) tAxis.getBound2()[time] * 60); - startTime = origin.plusSeconds((long) tAxis.getBound1()[time] * 60); - } else if (units.toLowerCase().matches("^second[s]? since.*$")) { - startTime = origin.plusSeconds((long) tAxis.getBound1()[time]); - endTime = origin.plusSeconds((long) tAxis.getBound2()[time]); - } else { - startTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); - endTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); - } - ZonedDateTime[] zonedDateTimes = new ZonedDateTime[2]; - zonedDateTimes[0] = startTime; - zonedDateTimes[1] = endTime; - list.add(zonedDateTimes); - }); - return list; - } - - // No time axes found. Try reading times from attributes. - Attribute startDateAttribute = gridDatatype.findAttributeIgnoreCase("start_date"); - Attribute endDateAttribute = gridDatatype.findAttributeIgnoreCase("stop_date"); - if (startDateAttribute != null && endDateAttribute != null) { - String startTimeString = startDateAttribute.getStringValue(); - if (startTimeString == null) return Collections.emptyList(); - ZonedDateTime startTime = ZonedDateTime.parse(startTimeString, - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")); - - String endTimeString = endDateAttribute.getStringValue(); - if (endTimeString == null) return Collections.emptyList(); - ZonedDateTime endTime = ZonedDateTime.parse(endTimeString, - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")); - - ZonedDateTime[] zonedDateTimes = new ZonedDateTime[2]; - zonedDateTimes[0] = startTime; - zonedDateTimes[1] = endTime; - - list.add(zonedDateTimes); - return list; - } - - Attribute nominalProductTimeAttribute = gridDataset.findGlobalAttributeIgnoreCase("nominal_product_time"); - if (nominalProductTimeAttribute != null) { - String timeString = nominalProductTimeAttribute.getStringValue(); - if (timeString == null) return Collections.emptyList(); - - CalendarDate calendarDate = CalendarDate.parseISOformat(null, timeString); - LocalDateTime ldt = LocalDateTime.parse(calendarDate.toString(), DateTimeFormatter.ISO_DATE_TIME); - ZonedDateTime time = ZonedDateTime.of(ldt, ZoneId.of("UTC")); - - ZonedDateTime[] zonedDateTimes = new ZonedDateTime[2]; - zonedDateTimes[0] = time; - zonedDateTimes[1] = time; - - list.add(zonedDateTimes); - return list; + /* Helpers */ + private static NetcdfDataset getNetcdfDataset(String pathToFile) { + try { + return NetcdfDatasets.openDataset(pathToFile); + } catch (IOException e) { + logger.severe(e.getMessage()); + return null; } - - return Collections.emptyList(); } - private boolean isSpecialTimeBounds() { - String fileName = new File(path).getName().toLowerCase(); - - String[] regexPatterns = { - ".*gaugecorr.*qpe.*01h.*grib2", - ".*radaronly.*qpe.*01h.*grib2", - ".*multisensor.*qpe.*01h.*grib2", - "mrms_preciprate.*", - "preciprate_.*\\.grib2", - ".*hhr\\.ms\\.mrg.*hdf.*", - ".*aorc.*apcp.*nc4.*", - ".*aorc.*tmp.*nc4.*", - "[0-9]{2}.nc", - "nldas_fora0125_h.a.*", - "hrrr.*wrfsfcf.*", - ".*cmorph.*h.*ly.*", - "ge.*\\.pgrb2.*\\.0p.*\\.f.*\\..*", - "prec.[0-9]{4}.nc", - "livneh_unsplit_precip.\\d{4}-\\d{2}-\\d{2}.\\d{4}.nc" - }; - - if (fileName.matches("nldas_fora0125_h.a.*") && variableName.equals("APCP") - || fileName.matches("gfs.nc") && variableName.equals("Precipitation_rate_surface")) { - return true; + private static GridDataset getGridDataset(NetcdfDataset netcdfDataset) { + try { + Formatter errorLog = new Formatter(); + FeatureDataset dataset = FeatureDatasetFactoryManager.wrap(FeatureType.GRID, netcdfDataset, null, errorLog); + if (dataset == null) return null; + boolean isGrid = dataset.getFeatureType() == FeatureType.GRID; + return isGrid && dataset instanceof GridDataset gridDataset ? gridDataset : null; + } catch (IOException e) { + logger.severe(e.getMessage()); + return null; } - - return Arrays.stream(regexPatterns).anyMatch(fileName::matches); } - private ZonedDateTime[] getSpecialTimeBounds(CoordinateAxis1DTime tAxis, int i) { - ZonedDateTime[] zonedDateTimes = new ZonedDateTime[2]; - - String fileName = new File(path).getName().toLowerCase(); - - if (fileName.matches(".*gaugecorr.*qpe.*01h.*grib2") - || fileName.matches(".*radaronly.*qpe.*01h.*grib2") - || fileName.matches(".*multisensor.*qpe.*01h.*grib2") - || (fileName.matches("nldas_fora0125_h.a.*") && variableName.equals("APCP"))) { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - zonedDateTimes[0] = convert(dates[0]).minusHours(1); - zonedDateTimes[1] = convert(dates[1]); - } else if (fileName.matches("mrms_preciprate.*")) { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - zonedDateTimes[0] = convert(dates[0]).minusMinutes(5); - zonedDateTimes[1] = convert(dates[1]); - } else if (fileName.matches("preciprate_.*\\.grib2")) { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - zonedDateTimes[0] = convert(dates[0]).minusMinutes(2); - zonedDateTimes[1] = convert(dates[1]); - } else if (fileName.matches(".*hhr\\.ms\\.mrg.*hdf.*")) { - zonedDateTimes[0] = convert(tAxis.getCalendarDate(i)); - zonedDateTimes[1] = zonedDateTimes[0].plusMinutes(30); - } else if (fileName.matches(".*aorc.*apcp.*nc4.*")) { - ZonedDateTime zdt = convert(tAxis.getCalendarDate(i)); - zonedDateTimes[0] = zdt.minusHours(1); - zonedDateTimes[1] = zdt; - } else if (fileName.matches(".*aorc.*tmp.*nc4.*")) { - ZonedDateTime zdt = convert(tAxis.getCalendarDate(i)); - zonedDateTimes[0] = zdt; - zonedDateTimes[1] = zdt; - } else if (fileName.matches("[0-9]{2}.nc")) { - zonedDateTimes[0] = convert(tAxis.getCalendarDate(0)); - zonedDateTimes[1] = zonedDateTimes[0].plusDays(1); - } else if (fileName.matches("hrrr.*wrfsfcf.*")) { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - zonedDateTimes[0] = convert(dates[0]); - zonedDateTimes[1] = zonedDateTimes[0].plusHours(1); - } else if (fileName.matches(".*cmorph.*h.*ly.*")) { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - ZonedDateTime zdt0 = convert(dates[0]); - zonedDateTimes[0] = zdt0; - zonedDateTimes[1] = zdt0.plusHours(1); - } else if (fileName.matches("ge.*\\.pgrb2.*\\.0p.*\\.f.*\\..*")) { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - zonedDateTimes[0] = convert(dates[0]); - zonedDateTimes[1] = zonedDateTimes[0].plusHours(3); - } else if (fileName.matches("^prec.[0-9]{4}.nc$") - || fileName.matches("^livneh_unsplit_precip.\\d{4}-\\d{2}-\\d{2}.\\d{4}.nc$")) { - // Livneh and Livneh unsplit precipitation data - zonedDateTimes[0] = convert(tAxis.getCalendarDate(i)); - zonedDateTimes[1] = zonedDateTimes[0].plusDays(1); - } else if (fileName.matches("gfs.nc")) { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - zonedDateTimes[0] = convert(dates[0]).plusMinutes(90); - zonedDateTimes[1] = convert(dates[1]).plusMinutes(90); - } else { - CalendarDate[] dates = tAxis.getCoordBoundsDate(i); - zonedDateTimes[0] = convert(dates[0]); - zonedDateTimes[1] = convert(dates[1]); - } - - return zonedDateTimes; + private static VariableDS getVariableDataset(NetcdfDataset netcdfDataset, String variableName) { + return netcdfDataset.getVariables().stream() + .filter(v -> v.getShortName().equals(variableName) && v instanceof VariableDS) + .map(VariableDS.class::cast) + .filter(v -> !v.getCoordinateSystems().isEmpty() || isLatLon(netcdfDataset)) + .findFirst() + .orElse(null); } - private List getTimeInstants(CoordinateAxis1DTime timeAxis) { - return timeAxis.getCalendarDates().stream() - .map(TimeConverter::toZonedDateTime) - .map(t -> new ZonedDateTime[] {t, t}) - .collect(Collectors.toList()); + private static boolean isLatLon(NetcdfDataset ncd) { + return ncd.findCoordinateAxis(AxisType.Lon) != null && ncd.findCoordinateAxis(AxisType.Lat) != null; } - private static ZonedDateTime convert(CalendarDate date) { - return ZonedDateTime.ofInstant(date.toDate().toInstant(), ZoneId.of("Z")); + private ZonedDateTime parseTime(String timeAxisUnits, double timeValue) { + String[] split = timeAxisUnits.split("since"); + String chronoUnitStr = split[0].trim().toUpperCase(Locale.ROOT); + String dateTimeStr = split[1].trim(); + ChronoUnit chronoUnit = ChronoUnit.valueOf(chronoUnitStr); + ZonedDateTime origin = TimeConverter.toZonedDateTime(dateTimeStr); + if (origin == null) return undefinedTime(); + return origin.plus((long) timeValue, chronoUnit); } - private static Grid getGrid(GridCoordSystem coordinateSystem) { - AtomicReference grid = new AtomicReference<>(); - - CoordinateAxis xAxis = coordinateSystem.getXHorizAxis(); - CoordinateAxis yAxis = coordinateSystem.getYHorizAxis(); - - int nx; - int ny; - - double[] edgesX; - double[] edgesY; - - if (xAxis instanceof CoordinateAxis1D - && yAxis instanceof CoordinateAxis1D) { - nx = (int) xAxis.getSize(); - ny = (int) yAxis.getSize(); - edgesX = ((CoordinateAxis1D) xAxis).getCoordEdges(); - edgesY = ((CoordinateAxis1D) yAxis).getCoordEdges(); - } else if (xAxis instanceof CoordinateAxis2D xAxis2D - && yAxis instanceof CoordinateAxis2D yAxis2D) { - int shapeX = xAxis2D.getEdges().getShape()[1] - 1; - double minX = xAxis2D.getMinValue(); - double maxX = xAxis2D.getMaxValue(); - double dx = (maxX - minX) / (shapeX - 1); - - int shapeY = xAxis2D.getEdges().getShape()[0] - 1; - double minY = yAxis2D.getMinValue(); - double maxY = yAxis2D.getMaxValue(); - double dy = (maxY - minY) / (shapeY - 1); - - double cellSize = (dx + dy) / 2; - - nx = (int) Math.round((maxX - minX) / cellSize); - ny = (int) Math.round((maxY - minY) / cellSize); - - edgesX = new double[nx]; - for (int i = 0; i < nx; i++) { - edgesX[i] = minX + i * cellSize; - } - - edgesY = new double[ny]; - for (int i = 0; i < ny; i++) { - edgesY[i] = minY + i * cellSize; - } - } else { - throw new IllegalStateException(); - } - - double ulx = edgesX[0]; - double urx = edgesX[edgesX.length - 1]; - double dx = (urx - ulx) / nx; - - double uly = edgesY[0]; - double lly = edgesY[edgesY.length - 1]; - double dy = (lly - uly) / ny; - - String wkt = getWkt(coordinateSystem.getProjection()); - - grid.set(Grid.builder() - .nx(nx) - .ny(ny) - .dx(dx) - .dy(dy) - .originX(ulx) - .originY(uly) - .crs(wkt) - .build()); - - String xAxisUnits = Objects.requireNonNull(xAxis).getUnitsString(); - - Unit cellUnits = switch (xAxisUnits.toLowerCase()) { - case "m", "meter", "metre" -> METRE; - case "km" -> KILO(METRE); - case "degrees", "degrees_east", "degrees_north" -> DEGREE_ANGLE; - default -> ONE; - }; - - Unit csUnits = ReferenceUtils.getLinearUnits(wkt); - - // This will scale the grid if cellUnits and csUnits do not align - // e.g. cellUnits are in meters but csUnits are in kilometers - // isCompatible is simply checking if the units are of type length so that no scaling is attempted - // between DEGREE_ANGLE and ONE - if (cellUnits.isCompatible(METRE) && csUnits.isCompatible(METRE) && !cellUnits.equals(csUnits)) { - grid.set(scaleGrid(grid.get(), cellUnits, csUnits)); - } - - return grid.get(); + private ZonedDateTime undefinedTime() { + return ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); } - private static void shiftGrid(Grid grid) { - String crs = grid.getCrs(); - boolean isGeographic = ReferenceUtils.isGeographic(crs); - if (isGeographic && (grid.getOriginX() > 180 || grid.getTerminusX() > 180)) - grid.shift(-360, 0); + private static boolean isSelectableVariable(Variable variable) { + boolean isVariableDS = variable instanceof VariableDS; + boolean isNotAxis = !(variable instanceof CoordinateAxis); + return isVariableDS && isNotAxis; } - private static Grid scaleGrid(Grid grid, Unit cellUnits, Unit csUnits){ + static Grid scaleGrid(Grid grid, Unit cellUnits, Unit csUnits) { Grid scaled; try { + // Converting xAxis and yAxis to be consistent with the wkt units UnitConverter converter = cellUnits.getConverterToAny(csUnits); scaled = Grid.builder() .originX(converter.convert(grid.getOriginX())) @@ -671,6 +177,7 @@ private static Grid scaleGrid(Grid grid, Unit cellUnits, Unit csUnits){ .dy(converter.convert(grid.getDy())) .nx(grid.getNx()) .ny(grid.getNy()) + .crs(grid.getCrs()) .build(); } catch (IncommensurableException e) { return null; @@ -678,120 +185,15 @@ private static Grid scaleGrid(Grid grid, Unit cellUnits, Unit csUnits){ return scaled; } - private int getDtoCount(GridDataset dataset, String variable) { - GridDatatype gridDatatype = dataset.findGridDatatype(variable); - Dimension timeDim = gridDatatype.getTimeDimension(); - if (timeDim != null) - return timeDim.getLength(); - - return 1; - - } - - private int getDtoCount(VariableDS variableDS) { - List dimensions = variableDS.getDimensions(); - for (Dimension dimension : dimensions) { - if (dimension.getShortName().equals("time")) { - return dimension.getLength(); - } - } - - return 1; - } - - private VortexData getData(GridDataset gridDataset, String variable, int idx) { - GridDatatype gridDatatype = gridDataset.findGridDatatype(variable); - GridCoordSystem gcs = gridDatatype.getCoordinateSystem(); - - VariableDS variableDS = gridDatatype.getVariable(); - double noDataValue = variableDS.getFillValue(); - - Grid grid = getGrid(gcs); - String wkt = getWkt(gcs.getProjection()); - - List times = getTimeBounds(gridDataset, variable); - - ZonedDateTime startTime; - ZonedDateTime endTime; - Duration interval; - if (!times.isEmpty()) { - startTime = times.get(idx)[0]; - endTime = times.get(idx)[1]; - interval = Duration.between(startTime, endTime); - } else { - startTime = null; - endTime = null; - interval = null; - } - - float[] slice; - try { - Array array = gridDatatype.readDataSlice(idx, -1, -1, -1); - slice = getFloatArray(array); - } catch (IOException e) { - logger.log(Level.SEVERE, e, e::getMessage); - return null; - } - - float[] data = getData(slice, gcs, grid, noDataValue); - - String units; - if (variable.toLowerCase().contains("var209-6")) { - units = "mm"; - } else { - units = gridDatatype.getUnitsString(); - } - - // Grid must be shifted after getData call since getData uses the original locations - // to map values. - shiftGrid(grid); - - VortexDataType vortexDataType = getVortexDataType(variableDS); - - return VortexGrid.builder() - .dx(grid.getDx()) - .dy(grid.getDy()) - .nx(grid.getNx()) - .ny(grid.getNy()) - .originX(grid.getOriginX()) - .originY(grid.getOriginY()) - .wkt(wkt) - .data(data) - .noDataValue(noDataValue) - .units(units) - .fileName(gridDataset.getLocation()) - .shortName(gridDatatype.getShortName()) - .fullName(gridDatatype.getFullName()) - .description(gridDatatype.getDescription()) - .startTime(startTime) - .endTime(endTime) - .interval(interval) - .dataType(vortexDataType) - .build(); - } - - private float[] getData(float[] slice, GridCoordSystem gcs, Grid gridDefinition, double noDataValue) { - if (gcs.isRegularSpatial()) - return slice; - - IndexSearcher indexSearcher = IndexSearcherFactory.INSTANCE.getOrCreate(gcs); - indexSearcher.addPropertyChangeListener(support::firePropertyChange); - - Coordinate[] coordinates = gridDefinition.getGridCellCentroidCoords(); - indexSearcher.cacheCoordinates(coordinates); - - float[] data = new float[coordinates.length]; - int count = data.length; - for (int i = 0; i < count; i++) { - Coordinate coordinate = coordinates[i]; - int index = indexSearcher.getIndex(coordinate.x, coordinate.y); - data[i] = index >= 0 ? slice[index] : (float) noDataValue; + static void shiftGrid(Grid grid) { + String crs = grid.getCrs(); + boolean isGeographic = ReferenceUtils.isGeographic(crs); + if (isGeographic && (grid.getOriginX() > 180 || grid.getTerminusX() > 180)) { + grid.shift(-360, 0); } - - return data; } - private VortexDataType getVortexDataType(VariableDS variableDS) { + static VortexDataType getVortexDataType(VariableDS variableDS) { String cellMethods = variableDS.findAttributeString(CF.CELL_METHODS, ""); return VortexDataType.fromString(cellMethods); } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfDataWriter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfDataWriter.java index 561c4562..840a3b87 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfDataWriter.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfDataWriter.java @@ -2,25 +2,19 @@ import mil.army.usace.hec.vortex.VortexGrid; import mil.army.usace.hec.vortex.VortexProperty; -import ucar.nc2.Attribute; import ucar.nc2.write.Nc4Chunking; import ucar.nc2.write.Nc4ChunkingStrategy; import ucar.nc2.write.NetcdfFileFormat; import ucar.nc2.write.NetcdfFormatWriter; import java.beans.PropertyChangeListener; -import java.nio.file.Files; -import java.util.Collections; -import java.util.HashMap; +import java.util.Comparator; import java.util.List; -import java.util.Map; +import java.util.stream.Stream; -public class NetcdfDataWriter extends DataWriter { +class NetcdfDataWriter extends DataWriter { private final List vortexGridList; - private static final Map fileFirstWriteCompleted = Collections.synchronizedMap(new HashMap<>()); - private final boolean overwriteExistingFile; - // NetCDF4 Settings public static final Nc4Chunking.Strategy CHUNKING_STRATEGY = Nc4Chunking.Strategy.standard; public static final int DEFLATE_LEVEL = 9; @@ -30,66 +24,37 @@ public class NetcdfDataWriter extends DataWriter { /* Constructor */ NetcdfDataWriter(Builder builder) { super(builder); - // This synchronized lock makes sure that each NetcdfDataWriter instance gets initialized one at a time - // Avoid race conditions of multiple instances trying to be the first one to overwrite the destination file. - synchronized (NetcdfDataWriter.class) { - // Check if file exists and get user preference for overwriting - boolean fileExists = Files.exists(destination); - boolean userPrefersOverwrite = getOverwritePreference(); - - String pathToFile = destination.toString(); - boolean firstWriteCompleted = fileFirstWriteCompleted.getOrDefault(pathToFile, false); - - if (!fileExists || userPrefersOverwrite && !firstWriteCompleted) { - overwriteExistingFile = true; - fileFirstWriteCompleted.put(pathToFile, true); - } else { - overwriteExistingFile = false; - } - } - - vortexGridList = data.stream() .filter(VortexGrid.class::isInstance) .map(VortexGrid.class::cast) .toList(); } - private boolean getOverwritePreference() { - String overwriteOption = options.get("isOverwrite"); - return overwriteOption == null || Boolean.parseBoolean(overwriteOption); - } - /* Write */ @Override public void write() { - if (overwriteExistingFile) overwriteData(); - else appendData(); - } - - private void overwriteData() { - NetcdfFormatWriter.Builder writerBuilder = initWriterBuilder(); - addGlobalAttributes(writerBuilder); - - NetcdfGridWriter gridWriter = new NetcdfGridWriter(vortexGridList); - gridWriter.addListener(writerPropertyListener()); - gridWriter.write(writerBuilder); + VortexGridCollection collection = VortexGridCollection.of(vortexGridList); + Stream timeRecordStream = vortexGridList.stream() + .map(VortexDataInterval::of) + .sorted(Comparator.comparing(VortexDataInterval::startTime)); + NetcdfWriterPrep.initializeForAppend(destination.toString(), collection, timeRecordStream); + appendData(); } public void appendData() { - NetcdfFormatWriter.Builder writerBuilder = initWriterBuilder(); + NetcdfFormatWriter.Builder writerBuilder = initAppendWriterBuilder(destination.toString()); - NetcdfGridWriter gridWriter = new NetcdfGridWriter(vortexGridList); + NetcdfGridWriter gridWriter = NetcdfGridWriter.create(vortexGridList); gridWriter.addListener(writerPropertyListener()); gridWriter.appendData(writerBuilder); } - private NetcdfFormatWriter.Builder initWriterBuilder() { + private static NetcdfFormatWriter.Builder initAppendWriterBuilder(String ncDestination) { Nc4Chunking chunker = Nc4ChunkingStrategy.factory(CHUNKING_STRATEGY, DEFLATE_LEVEL, SHUFFLE); return NetcdfFormatWriter.builder() - .setNewFile(overwriteExistingFile) + .setNewFile(false) .setFormat(NETCDF_FORMAT) - .setLocation(destination.toString()) + .setLocation(ncDestination) .setChunker(chunker); } @@ -104,8 +69,4 @@ private PropertyChangeListener writerPropertyListener() { }; } - /* Add Global Attributes */ - private void addGlobalAttributes(NetcdfFormatWriter.Builder writerBuilder) { - writerBuilder.addAttribute(new Attribute("Conventions", "CF-1.10")); - } } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfGridWriter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfGridWriter.java index 57127e45..2ec242a1 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfGridWriter.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfGridWriter.java @@ -1,22 +1,14 @@ package mil.army.usace.hec.vortex.io; +import mil.army.usace.hec.vortex.VortexGrid; import mil.army.usace.hec.vortex.VortexProperty; import mil.army.usace.hec.vortex.VortexVariable; -import mil.army.usace.hec.vortex.VortexGrid; -import mil.army.usace.hec.vortex.VortexGridCollection; -import mil.army.usace.hec.vortex.util.UnitUtil; import ucar.ma2.Array; -import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; -import ucar.nc2.Attribute; -import ucar.nc2.Dimension; import ucar.nc2.Variable; -import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; import ucar.nc2.write.NetcdfFormatWriter; -import ucar.unidata.util.Parameter; -import javax.measure.Unit; import java.beans.PropertyChangeListener; import java.beans.PropertyChangeSupport; import java.io.IOException; @@ -29,44 +21,25 @@ import java.util.logging.Logger; import java.util.stream.Collectors; -import static javax.measure.MetricPrefix.*; -import static systems.uom.common.USCustomary.*; -import static tech.units.indriya.unit.Units.HOUR; -import static tech.units.indriya.unit.Units.*; - -public class NetcdfGridWriter { +class NetcdfGridWriter { private static final Logger logger = Logger.getLogger(NetcdfGridWriter.class.getName()); private final PropertyChangeSupport support = new PropertyChangeSupport(this); - public static final int BOUNDS_LEN = 2; - public static final String CRS_WKT = "crs_wkt"; private final Map gridCollectionMap; - private final VortexGridCollection defaultCollection; - // Dimensions - private final Dimension timeDim; - private final Dimension latDim; - private final Dimension lonDim; - private final Dimension yDim; - private final Dimension xDim; - private final Dimension boundsDim; // For all axis that has bounds (interval data) - public NetcdfGridWriter(List vortexGridList) { + private NetcdfGridWriter(List vortexGridList) { gridCollectionMap = initGridCollectionMap(vortexGridList); - defaultCollection = getAnyCollection(); - // Dimensions - timeDim = Dimension.builder().setName(CF.TIME).setIsUnlimited(true).build(); - latDim = Dimension.builder().setName(CF.LATITUDE).setLength(defaultCollection.getNy()).build(); - lonDim = Dimension.builder().setName(CF.LONGITUDE).setLength(defaultCollection.getNx()).build(); - yDim = Dimension.builder().setName("y").setLength(defaultCollection.getNy()).build(); - xDim = Dimension.builder().setName("x").setLength(defaultCollection.getNx()).build(); - boundsDim = Dimension.builder().setName("nv").setIsUnlimited(true).build(); + } + + static NetcdfGridWriter create(List vortexGridList) { + return new NetcdfGridWriter(vortexGridList); } private Map initGridCollectionMap(List vortexGridList) { Map map = vortexGridList.stream() .collect(Collectors.groupingBy( VortexGrid::shortName, - Collectors.collectingAndThen(Collectors.toList(), VortexGridCollection::new)) + Collectors.collectingAndThen(Collectors.toList(), VortexGridCollection::of)) ); boolean isValidMap = verifyGridCollectionMap(map); return isValidMap ? map : Collections.emptyMap(); @@ -77,8 +50,7 @@ private boolean verifyGridCollectionMap(Map map) { // No need to check lat & lon since they are generated from (x & y & projection) boolean yMatched = isUnique(VortexGridCollection::getYCoordinates, map); boolean xMatched = isUnique(VortexGridCollection::getXCoordinates, map); - boolean timeMatched = isUnique(VortexGridCollection::getTimeData, map); - return projectionMatched && yMatched && xMatched && timeMatched; + return projectionMatched && yMatched && xMatched; } private boolean isUnique(Function propertyGetter, Map map) { @@ -87,51 +59,13 @@ private boolean isUnique(Function propertyGetter, Map (o instanceof double[] || o instanceof float[]) ? Arrays.deepToString(new Object[] {o}) : o) .distinct() .count() == 1; - if (!isUnique) logger.severe("Data is not the same for all grids"); - return isUnique; - } - - private VortexGridCollection getAnyCollection() { - return gridCollectionMap.values().stream().findAny().orElse(null); - } - - /* Write Methods */ - public void write(NetcdfFormatWriter.Builder writerBuilder) { - addDimensions(writerBuilder); - addVariables(writerBuilder); - - try (NetcdfFormatWriter writer = writerBuilder.build()) { - writeDimensions(writer); - writeVariableGrids(writer, 0); - } catch (IOException | InvalidRangeException e) { - logger.severe(e.getMessage()); + if (!isUnique) { + logger.severe("Data is not the same for all grids"); } + return isUnique; } - private void writeDimensions(NetcdfFormatWriter writer) throws InvalidRangeException, IOException { - writer.write(timeDim.getShortName(), Array.makeFromJavaArray(defaultCollection.getTimeData())); - - if (defaultCollection.hasTimeBounds()) - writer.write(getBoundsName(timeDim), Array.makeFromJavaArray(defaultCollection.getTimeBoundsArray())); - - if (defaultCollection.isGeographic()) writeDimensionsGeographic(writer); - else writeDimensionsProjected(writer); - } - - private void writeDimensionsGeographic(NetcdfFormatWriter writer) throws InvalidRangeException, IOException { - writer.write(latDim.getShortName(), Array.makeFromJavaArray(defaultCollection.getYCoordinates())); - writer.write(lonDim.getShortName(), Array.makeFromJavaArray(defaultCollection.getXCoordinates())); - } - - private void writeDimensionsProjected(NetcdfFormatWriter writer) throws InvalidRangeException, IOException { - Map latLonMap = defaultCollection.getLatLonCoordinates(); - writer.write(yDim.getShortName(), Array.makeFromJavaArray(defaultCollection.getYCoordinates())); - writer.write(xDim.getShortName(), Array.makeFromJavaArray(defaultCollection.getXCoordinates())); - writer.write(latDim.getShortName(), Array.makeFromJavaArray(latLonMap.get("lat"))); - writer.write(lonDim.getShortName(), Array.makeFromJavaArray(latLonMap.get("lon"))); - } - - private void writeVariableGrids(NetcdfFormatWriter writer, int startIndex) { + private void writeVariableGrids(NetcdfFormatWriter writer) { AtomicBoolean hasErrors = new AtomicBoolean(false); for (VortexGridCollection collection : gridCollectionMap.values()) { @@ -145,8 +79,8 @@ private void writeVariableGrids(NetcdfFormatWriter writer, int startIndex) { collection.getCollectionDataStream().forEach(entry -> { try { - int index = entry.getKey() + startIndex; VortexGrid grid = entry.getValue(); + int index = NetcdfWriterPrep.getTimeRecordIndex(writer.getOutputFile().getLocation(), VortexDataInterval.of(grid)); int[] origin = {index, 0, 0}; writer.write(variable, origin, Array.makeFromJavaArray(grid.data3D())); } catch (IOException | InvalidRangeException e) { @@ -157,168 +91,20 @@ private void writeVariableGrids(NetcdfFormatWriter writer, int startIndex) { } if (hasErrors.get()) { - boolean isAppend = startIndex > 0; +// boolean isAppend = startIndex > 0; String overwriteErrorMessage = "Failed to overwrite file."; String appendErrorMessage = "Some reasons may be:\n* Attempted to append to non-existing variable\n* Attempted to append data with different projection\n* Attempted to append data with different location"; - String message = isAppend ? appendErrorMessage : overwriteErrorMessage; - support.firePropertyChange(VortexProperty.ERROR, null, message); - } - } - - /* Add Dimensions */ - private void addDimensions(NetcdfFormatWriter.Builder writerBuilder) { - writerBuilder.addDimension(timeDim); - if (defaultCollection.hasTimeBounds()) writerBuilder.addDimension(boundsDim); - - if (defaultCollection.isGeographic()) { - writerBuilder.addDimension(latDim); - writerBuilder.addDimension(lonDim); - } else { - writerBuilder.addDimension(yDim); - writerBuilder.addDimension(xDim); - } - } - - /* Add Variables */ - private void addVariables(NetcdfFormatWriter.Builder writerBuilder) { - addVariableTime(writerBuilder); - if (defaultCollection.hasTimeBounds()) addVariableTimeBounds(writerBuilder); - addVariableLat(writerBuilder); - addVariableLon(writerBuilder); - addVariableGridCollection(writerBuilder); - - if (!defaultCollection.isGeographic()) { - addVariableProjection(writerBuilder); - addVariableX(writerBuilder); - addVariableY(writerBuilder); - } - } - - private void addVariableTime(NetcdfFormatWriter.Builder writerBuilder) { - Variable.Builder v = writerBuilder.addVariable(timeDim.getShortName(), DataType.FLOAT, List.of(timeDim)); - v.addAttribute(new Attribute(CF.STANDARD_NAME, CF.TIME)); - v.addAttribute(new Attribute(CF.CALENDAR, "standard")); - v.addAttribute(new Attribute(CF.UNITS, defaultCollection.getTimeUnits())); - if (defaultCollection.hasTimeBounds()) v.addAttribute(new Attribute(CF.BOUNDS, getBoundsName(timeDim))); - } - - private void addVariableTimeBounds(NetcdfFormatWriter.Builder writerBuilder) { - writerBuilder.addVariable(getBoundsName(timeDim), DataType.FLOAT, List.of(timeDim, boundsDim)); - } - - private void addVariableLat(NetcdfFormatWriter.Builder writerBuilder) { - boolean isGeographic = defaultCollection.isGeographic(); - List dimensions = isGeographic ? List.of(latDim) : List.of(yDim, xDim); - writerBuilder.addVariable(latDim.getShortName(), DataType.DOUBLE, dimensions) - .addAttribute(new Attribute(CF.UNITS, CDM.LAT_UNITS)) - .addAttribute(new Attribute(CF.LONG_NAME, "latitude coordinate")) - .addAttribute(new Attribute(CF.STANDARD_NAME, CF.LATITUDE)); - } - - private void addVariableLon(NetcdfFormatWriter.Builder writerBuilder) { - boolean isGeographic = defaultCollection.isGeographic(); - List dimensions = isGeographic ? List.of(lonDim) : List.of(yDim, xDim); - writerBuilder.addVariable(lonDim.getShortName(), DataType.DOUBLE, dimensions) - .addAttribute(new Attribute(CF.UNITS, CDM.LON_UNITS)) - .addAttribute(new Attribute(CF.LONG_NAME, "longitude coordinate")) - .addAttribute(new Attribute(CF.STANDARD_NAME, CF.LONGITUDE)); - } - - private void addVariableY(NetcdfFormatWriter.Builder writerBuilder) { - writerBuilder.addVariable(yDim.getShortName(), DataType.DOUBLE, List.of(yDim)) - .addAttribute(new Attribute(CF.UNITS, defaultCollection.getProjectionUnit())) - .addAttribute(new Attribute(CF.STANDARD_NAME, CF.PROJECTION_Y_COORDINATE)); - } - - private void addVariableX(NetcdfFormatWriter.Builder writerBuilder) { - writerBuilder.addVariable(xDim.getShortName(), DataType.DOUBLE, List.of(xDim)) - .addAttribute(new Attribute(CF.UNITS, defaultCollection.getProjectionUnit())) - .addAttribute(new Attribute(CF.STANDARD_NAME, CF.PROJECTION_X_COORDINATE)); - } - - private void addVariableProjection(NetcdfFormatWriter.Builder writerBuilder) { - Variable.Builder variableBuilder = writerBuilder.addVariable(defaultCollection.getProjectionName(), DataType.SHORT, Collections.emptyList()); - for (Parameter parameter : defaultCollection.getProjection().getProjectionParameters()) { - String name = parameter.getName(); - String stringValue = parameter.getStringValue(); - double[] numericValues = parameter.getNumericValues(); - - if (stringValue == null && numericValues == null) { - String logMessage = String.format("Parameter '%s' has no value", parameter.getName()); - logger.severe(logMessage); - continue; - } - - Attribute attribute = (stringValue != null) ? - Attribute.builder().setName(name).setStringValue(stringValue).build() : - Attribute.builder().setName(name).setValues(Array.makeFromJavaArray(numericValues)).build(); - variableBuilder.addAttribute(attribute); - } - - // Adding CRS WKT for Grid's Coordinate System information - // CF Conventions: https://cfconventions.org/Data/cf-conventions/cf-conventions-1.11/cf-conventions.html#use-of-the-crs-well-known-text-format - variableBuilder.addAttribute(new Attribute(CRS_WKT, defaultCollection.getWkt())); - } - - private void addVariableGridCollection(NetcdfFormatWriter.Builder writerBuilder) { - boolean isGeographic = defaultCollection.isGeographic(); - List dimensions = isGeographic ? List.of(timeDim, latDim, lonDim) : List.of(timeDim, yDim, xDim); - for (VortexGridCollection collection : gridCollectionMap.values()) { - VortexVariable variable = getVortexVariable(collection); - Unit dataUnit = UnitUtil.getUnits(collection.getDataUnit()); - writerBuilder.addVariable(variable.getShortName(), DataType.FLOAT, dimensions) - .addAttribute(new Attribute(CF.LONG_NAME, variable.getLongName())) - .addAttribute(new Attribute(CF.UNITS, getUnitsString(dataUnit))) - .addAttribute(new Attribute(CF.GRID_MAPPING, defaultCollection.getProjectionName())) - .addAttribute(new Attribute(CF.COORDINATES, "latitude longitude")) - .addAttribute(new Attribute(CF.MISSING_VALUE, collection.getNoDataValue())) - .addAttribute(new Attribute(CF._FILLVALUE, collection.getNoDataValue())) - .addAttribute(new Attribute(CF.CELL_METHODS, collection.getDataType().getNcString())); +// String message = isAppend ? appendErrorMessage : overwriteErrorMessage; + support.firePropertyChange(VortexProperty.ERROR, null, appendErrorMessage); } } /* Helpers */ - private String getBoundsName(Dimension dimension) { - return dimension.getShortName() + "_bnds"; - } - private static VortexVariable getVortexVariable(VortexGridCollection collection) { VortexVariable name = VortexVariable.fromName(collection.getShortName()); return name.equals(VortexVariable.UNDEFINED) ? VortexVariable.fromName(collection.getDescription()) : name; } - private static String getUnitsString(Unit unit){ - if (unit.equals(MILLI(METRE).divide(SECOND))) return "mm/s"; - if (unit.equals(MILLI(METRE).divide(HOUR))) return "mm/hr"; - if (unit.equals(MILLI(METRE).divide(DAY))) return "mm/day"; - if (unit.equals(MILLI(METRE))) return "mm"; - if (unit.equals(INCH)) return "in"; - if (unit.equals(CELSIUS)) return "degC"; - if (unit.equals(CELSIUS.multiply(DAY))) return "degC-d"; - if (unit.equals(FAHRENHEIT)) return "degF"; - if (unit.equals(KELVIN)) return "K"; - if (unit.equals(WATT.divide(SQUARE_METRE))) return "W m-2"; - if (unit.equals(JOULE.divide(SQUARE_METRE))) return "J m-2"; - if (unit.equals(KILO(METRE).divide(HOUR))) return "kph"; - if (unit.equals(KILOMETRE_PER_HOUR)) return "km h-1"; - if (unit.equals(PERCENT)) return "%"; - if (unit.equals(HECTO(PASCAL))) return "hPa"; - if (unit.equals(PASCAL)) return "Pa"; - if (unit.equals(METRE)) return "m"; - if (unit.equals(CUBIC_METRE.divide(SECOND))) return "m3 s-1"; - if (unit.equals(CUBIC_FOOT.divide(SECOND))) return "cfs"; - if (unit.equals(FOOT)) return "ft"; - if (unit.equals(METRE_PER_SECOND)) return "m/s"; - if (unit.equals(MILE_PER_HOUR)) return "mph"; - if (unit.equals(FOOT_PER_SECOND)) return "ft/s"; - if (unit.equals(KILO(PASCAL))) return "kPa"; - if (unit.equals(KILO(METRE))) return "km"; - if (unit.equals(MILE)) return "mi"; - if (unit.equals(TON)) return "ton"; - if (unit.equals(MILLI(GRAM).divide(LITRE))) return "mg L-1"; - return "Unspecified"; - } - /* Property Change */ public void addListener(PropertyChangeListener pcl) { this.support.addPropertyChangeListener(pcl); @@ -337,13 +123,7 @@ public void appendData(NetcdfFormatWriter.Builder writerBuilder) { return; } - int startIndex = timeVar.getShape(0); - writeVariableGrids(writer, startIndex); - writer.write(timeVar, new int[] {startIndex}, Array.makeFromJavaArray(defaultCollection.getTimeData())); - - if (defaultCollection.hasTimeBounds()) { - writer.write(getBoundsName(timeDim), new int[] {startIndex, 0}, Array.makeFromJavaArray(defaultCollection.getTimeBoundsArray())); - } + writeVariableGrids(writer); } catch (Exception e) { logger.severe(e.getMessage()); } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfTimeUtils.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfTimeUtils.java new file mode 100644 index 00000000..d991ca08 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfTimeUtils.java @@ -0,0 +1,38 @@ +package mil.army.usace.hec.vortex.io; + +import java.time.Duration; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; + +class NetcdfTimeUtils { + private static final ZoneId ORIGIN_ZONE_ID = ZoneId.of("UTC"); + static final ZonedDateTime ORIGIN_TIME = ZonedDateTime.of(1900, 1, 1, 0, 0, 0, 0, ORIGIN_ZONE_ID); + + static long getNumIntervalsFromBaseTime(ZonedDateTime dateTime, ChronoUnit intervalUnit) { + return intervalUnit.between(ORIGIN_TIME, dateTime); + } + + static ChronoUnit getMinimumDeltaTimeUnit(List timeRecords) { + ChronoUnit minimum = ChronoUnit.DAYS; + for (VortexDataInterval timeRecord : timeRecords) { + ChronoUnit unit = getDeltaTimeUnit(timeRecord.getRecordDuration()); + if (unit.getDuration().toSeconds() < minimum.getDuration().toSeconds()) { + minimum = unit; + } + } + return minimum; + } + + static ChronoUnit getDeltaTimeUnit(Duration duration) { + // http://cfconventions.org/cf-conventions/cf-conventions.html#time-coordinate + if (duration.toDays() > 0) { + return ChronoUnit.DAYS; + } else if (duration.toHours() > 0) { + return ChronoUnit.HOURS; + } else { + return ChronoUnit.MINUTES; + } + } +} \ No newline at end of file diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfWriterPrep.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfWriterPrep.java new file mode 100644 index 00000000..eb0275ee --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfWriterPrep.java @@ -0,0 +1,314 @@ +package mil.army.usace.hec.vortex.io; + +import mil.army.usace.hec.vortex.VortexGrid; +import mil.army.usace.hec.vortex.VortexVariable; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.Variable; +import ucar.nc2.constants.CDM; +import ucar.nc2.constants.CF; +import ucar.nc2.write.Nc4Chunking; +import ucar.nc2.write.Nc4ChunkingStrategy; +import ucar.nc2.write.NetcdfFileFormat; +import ucar.nc2.write.NetcdfFormatWriter; +import ucar.unidata.util.Parameter; + +import java.io.IOException; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.logging.Logger; +import java.util.stream.Stream; + +final class NetcdfWriterPrep { + private static final Logger logger = Logger.getLogger(NetcdfWriterPrep.class.getName()); + + // NetCDF4 Settings + private static final Nc4Chunking.Strategy CHUNKING_STRATEGY = Nc4Chunking.Strategy.standard; + private static final int DEFLATE_LEVEL = 9; + private static final boolean SHUFFLE = false; + private static final NetcdfFileFormat NETCDF_FORMAT = NetcdfFileFormat.NETCDF4; + + // Map + private static final Map> recordIndexMap = new HashMap<>(); + + private NetcdfWriterPrep() { + // Utility Class + } + + public static void initializeForAppend(String destination, VortexGridCollection gridCollection, Stream sortedRecordStream) { + NetcdfFormatWriter.Builder builder = overwriteWriterBuilder(destination, gridCollection); + if (builder == null) { + logger.warning("Failed to create writer builder to prep file"); + return; + } + + try (NetcdfFormatWriter writer = builder.build()) { + writeProjectionDimensions(writer, gridCollection); + List sortedRecordList = sortedRecordStream.toList(); + writeTimeDimension(writer, sortedRecordList); + recordIndexMap.put(destination, IndexMap.of(sortedRecordList)); + logger.info("Generated NetCDF File. Ready for Append."); + } catch (Exception e) { + logger.warning("Failed to prep file"); + logger.warning(e.getMessage()); + } + } + + public static int getTimeRecordIndex(String destination, VortexDataInterval timeRecord) { + return recordIndexMap.get(destination).indexOf(timeRecord); + } + + /* Helpers */ + private static NetcdfFormatWriter.Builder overwriteWriterBuilder(String destination, VortexGridCollection gridCollection) { + try { + Nc4Chunking chunker = Nc4ChunkingStrategy.factory(CHUNKING_STRATEGY, DEFLATE_LEVEL, SHUFFLE); + NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.builder() + .setNewFile(true) + .setFormat(NETCDF_FORMAT) + .setLocation(destination) + .setChunker(chunker); + addDimensions(builder, gridCollection); + addVariables(builder, gridCollection); + addGlobalAttributes(builder); + return builder; + } catch (Exception e) { + logger.warning("Failed to create builder"); + logger.warning(e.getMessage()); + return null; + } + } + + /* Dimensions */ + private static void addDimensions(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + if (gridCollection.hasTimeDimension()) { + writerBuilder.addDimension(getTimeDimension()); + } + + if (gridCollection.hasTimeBounds()) { + writerBuilder.addDimension(getTimeBoundsDimension()); + } + + if (gridCollection.isGeographic()) { + writerBuilder.addDimension(getLatitudeDimension(gridCollection.getNy())); + writerBuilder.addDimension(getLongitudeDimension(gridCollection.getNx())); + } else { + writerBuilder.addDimension(getYDimension(gridCollection.getNy())); + writerBuilder.addDimension(getXDimension(gridCollection.getNx())); + } + + } + + private static Dimension getTimeDimension() { + return Dimension.builder().setName(CF.TIME).setIsUnlimited(true).build(); + } + + private static Dimension getTimeBoundsDimension() { + return Dimension.builder().setName("nv").setLength(2).build(); + } + + private static Dimension getLatitudeDimension(int ny) { + return Dimension.builder().setName(CF.LATITUDE).setLength(ny).build(); + } + + private static Dimension getLongitudeDimension(int nx) { + return Dimension.builder().setName(CF.LONGITUDE).setLength(nx).build(); + } + + private static Dimension getYDimension(int ny) { + return Dimension.builder().setName("y").setLength(ny).build(); + } + + private static Dimension getXDimension(int nx) { + return Dimension.builder().setName("x").setLength(nx).build(); + } + + /* Variables */ + private static void addVariables(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + if (gridCollection.hasTimeDimension()) { + addTimeVariable(writerBuilder, gridCollection); + } + + if (gridCollection.hasTimeBounds()) { + addVariableTimeBounds(writerBuilder); + } + + if (gridCollection.isGeographic()) { + addVariableLat(writerBuilder, gridCollection); + addVariableLon(writerBuilder, gridCollection); + } else { + addVariableY(writerBuilder, gridCollection); + addVariableX(writerBuilder, gridCollection); + } + + addVariableProjection(writerBuilder, gridCollection); + addVariableGridCollection(writerBuilder, gridCollection); + + } + + private static void addTimeVariable(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + if (!gridCollection.hasTimeDimension()) { + return; + } + + Variable.Builder v = writerBuilder.addVariable(CF.TIME, DataType.ULONG, List.of(getTimeDimension())); + v.addAttribute(new Attribute(CF.STANDARD_NAME, CF.TIME)); + v.addAttribute(new Attribute(CF.CALENDAR, "standard")); + v.addAttribute(new Attribute(CF.UNITS, gridCollection.getTimeUnits())); + + if (gridCollection.hasTimeBounds()) { + v.addAttribute(new Attribute(CF.BOUNDS, "time_bnds")); + } + + } + + private static void addVariableTimeBounds(NetcdfFormatWriter.Builder writerBuilder) { + Dimension timeDim = getTimeDimension(); + Dimension timeBoundsDim = getTimeBoundsDimension(); + writerBuilder.addVariable("time_bnds", DataType.ULONG, List.of(timeDim, timeBoundsDim)); + } + + private static void addVariableLat(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + Dimension latDim = getLatitudeDimension(gridCollection.getNy()); + Dimension yDim = getYDimension(gridCollection.getNy()); + Dimension xDim = getXDimension(gridCollection.getNx()); + + boolean isGeographic = gridCollection.isGeographic(); + List dimensions = isGeographic ? List.of(latDim) : List.of(yDim, xDim); + writerBuilder.addVariable(latDim.getShortName(), DataType.DOUBLE, dimensions) + .addAttribute(new Attribute(CF.UNITS, CDM.LAT_UNITS)) + .addAttribute(new Attribute(CF.LONG_NAME, "latitude coordinate")) + .addAttribute(new Attribute(CF.STANDARD_NAME, CF.LATITUDE)); + + } + + private static void addVariableLon(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + Dimension lonDim = getLongitudeDimension(gridCollection.getNx()); + Dimension yDim = getYDimension(gridCollection.getNy()); + Dimension xDim = getXDimension(gridCollection.getNx()); + + boolean isGeographic = gridCollection.isGeographic(); + List dimensions = isGeographic ? List.of(lonDim) : List.of(yDim, xDim); + writerBuilder.addVariable(lonDim.getShortName(), DataType.DOUBLE, dimensions) + .addAttribute(new Attribute(CF.UNITS, CDM.LON_UNITS)) + .addAttribute(new Attribute(CF.LONG_NAME, "longitude coordinate")) + .addAttribute(new Attribute(CF.STANDARD_NAME, CF.LONGITUDE)); + + } + + private static void addVariableY(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + Dimension yDim = getYDimension(gridCollection.getNy()); + writerBuilder.addVariable(yDim.getShortName(), DataType.DOUBLE, List.of(yDim)) + .addAttribute(new Attribute(CF.UNITS, gridCollection.getProjectionUnit())) + .addAttribute(new Attribute(CF.STANDARD_NAME, CF.PROJECTION_Y_COORDINATE)); + } + + private static void addVariableX(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + Dimension xDim = getXDimension(gridCollection.getNx()); + writerBuilder.addVariable(xDim.getShortName(), DataType.DOUBLE, List.of(xDim)) + .addAttribute(new Attribute(CF.UNITS, gridCollection.getProjectionUnit())) + .addAttribute(new Attribute(CF.STANDARD_NAME, CF.PROJECTION_X_COORDINATE)); + } + + private static void addVariableProjection(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + Variable.Builder variableBuilder = writerBuilder.addVariable(gridCollection.getProjectionName(), DataType.SHORT, Collections.emptyList()); + for (Parameter parameter : gridCollection.getProjection().getProjectionParameters()) { + String name = parameter.getName(); + String stringValue = parameter.getStringValue(); + double[] numericValues = parameter.getNumericValues(); + + if (stringValue == null && numericValues == null) { + String logMessage = String.format("Parameter '%s' has no value", parameter.getName()); + logger.severe(logMessage); + continue; + } + + Attribute attribute = (stringValue != null) ? + Attribute.builder().setName(name).setStringValue(stringValue).build() : + Attribute.builder().setName(name).setValues(Array.makeFromJavaArray(numericValues)).build(); + variableBuilder.addAttribute(attribute); + } + + // Adding CRS WKT for Grid's Coordinate System information + // CF Conventions: https://cfconventions.org/Data/cf-conventions/cf-conventions-1.11/cf-conventions.html#use-of-the-crs-well-known-text-format + variableBuilder.addAttribute(new Attribute("crs_wkt", gridCollection.getWkt())); + + } + + private static void addVariableGridCollection(NetcdfFormatWriter.Builder writerBuilder, VortexGridCollection gridCollection) { + Dimension timeDim = getTimeDimension(); + + List dimensions = new ArrayList<>(); + if (gridCollection.hasTimeDimension()) { + dimensions.add(timeDim); + } + + if (gridCollection.isGeographic()) { + dimensions.add(getLatitudeDimension(gridCollection.getNy())); + dimensions.add(getLongitudeDimension(gridCollection.getNx())); + } else { + dimensions.add(getYDimension(gridCollection.getNy())); + dimensions.add(getXDimension(gridCollection.getNx())); + } + + Map availableVortexVariables = gridCollection.getRepresentativeGridNameMap(); + for (VortexGrid vortexGrid : availableVortexVariables.values()) { + VortexVariable variable = VortexVariable.fromGrid(vortexGrid); + writerBuilder.addVariable(variable.getShortName(), DataType.FLOAT, dimensions) + .addAttribute(new Attribute(CF.LONG_NAME, variable.getLongName())) + .addAttribute(new Attribute(CF.UNITS, vortexGrid.units())) + .addAttribute(new Attribute(CF.GRID_MAPPING, gridCollection.getProjectionName())) + .addAttribute(new Attribute(CF.COORDINATES, "latitude longitude")) + .addAttribute(new Attribute(CF.MISSING_VALUE, (float) vortexGrid.noDataValue())) + .addAttribute(new Attribute(CF._FILLVALUE, (float) vortexGrid.noDataValue())) + .addAttribute(new Attribute(CF.CELL_METHODS, vortexGrid.dataType().getNcString())); + } + + } + + private static void addGlobalAttributes(NetcdfFormatWriter.Builder writerBuilder) { + writerBuilder.addAttribute(new Attribute("Conventions", "CF-1.10")); + } + + /* Write Dimensions */ + private static void writeProjectionDimensions(NetcdfFormatWriter writer, VortexGridCollection gridCollection) throws InvalidRangeException, IOException { + if (gridCollection.isGeographic()) { + writer.write(CF.LATITUDE, Array.makeFromJavaArray(gridCollection.getYCoordinates())); + writer.write(CF.LONGITUDE, Array.makeFromJavaArray(gridCollection.getXCoordinates())); + } else { + writer.write("y", Array.makeFromJavaArray(gridCollection.getYCoordinates())); + writer.write("x", Array.makeFromJavaArray(gridCollection.getXCoordinates())); + } + } + + private static void writeTimeDimension(NetcdfFormatWriter writer, List timeRecords) throws InvalidRangeException, IOException { + int numData = timeRecords.size(); + + long[][] timeBoundsData = new long[numData][2]; + long[] midTimeData = new long[numData]; + + ChronoUnit minimumUnit = NetcdfTimeUtils.getMinimumDeltaTimeUnit(timeRecords); + for (int i = 0; i < numData; i++) { + VortexDataInterval timeRecord = timeRecords.get(i); + long startTime = NetcdfTimeUtils.getNumIntervalsFromBaseTime(timeRecord.startTime(), minimumUnit); + long endTime = NetcdfTimeUtils.getNumIntervalsFromBaseTime(timeRecord.endTime(), minimumUnit); + long midTime = (startTime + endTime) / 2; + + timeBoundsData[i][0] = startTime; + timeBoundsData[i][1] = endTime; + midTimeData[i] = midTime; + } + + Variable timeVar = writer.findVariable(CF.TIME); + if (timeVar != null) { + writer.write(timeVar, new int[]{0}, Array.makeFromJavaArray(midTimeData)); + } + + Variable timeBoundsVar = writer.findVariable("time_bnds"); + if (timeBoundsVar != null) { + writer.write(timeBoundsVar, Array.makeFromJavaArray(timeBoundsData)); + } + } +} \ No newline at end of file diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NoRecordIndexQuery.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NoRecordIndexQuery.java new file mode 100644 index 00000000..d9907821 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NoRecordIndexQuery.java @@ -0,0 +1,22 @@ +package mil.army.usace.hec.vortex.io; + +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.List; + +class NoRecordIndexQuery implements RecordIndexQuery { + @Override + public List query(ZonedDateTime startTime, ZonedDateTime endTime) { + return Collections.emptyList(); + } + + @Override + public ZonedDateTime getEarliestStartTime() { + return null; + } + + @Override + public ZonedDateTime getLatestEndTime() { + return null; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/PeriodRecordIndexQuery.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/PeriodRecordIndexQuery.java new file mode 100644 index 00000000..d8307e85 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/PeriodRecordIndexQuery.java @@ -0,0 +1,53 @@ +package mil.army.usace.hec.vortex.io; + +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +final class PeriodRecordIndexQuery implements RecordIndexQuery { + private final Map originalIndexMap; + private final IntervalTree intervalTree; + + private PeriodRecordIndexQuery(List recordList) { + this.originalIndexMap = initOriginalIndexMap(recordList); + this.intervalTree = IntervalTree.from(recordList); + } + + static PeriodRecordIndexQuery from(List recordList) { + return new PeriodRecordIndexQuery(recordList); + } + + /* Query */ + @Override + public List query(ZonedDateTime startTime, ZonedDateTime endTime) { + List overlappers = intervalTree.findOverlaps(VortexDataInterval.of(startTime, endTime)); + return overlappers.stream().map(originalIndexMap::get).toList(); + } + + @Override + public ZonedDateTime getEarliestStartTime() { + return Optional.ofNullable(intervalTree.findMinimum()) + .map(VortexDataInterval::startTime) + .orElse(null); + } + + @Override + public ZonedDateTime getLatestEndTime() { + return Optional.ofNullable(intervalTree.findMaximum()) + .map(VortexDataInterval::endTime) + .orElse(null); + } + + private static Map initOriginalIndexMap(List recordList) { + Map indexMap = new HashMap<>(); + + for (int i = 0; i < recordList.size(); i++) { + VortexDataInterval timeRecord = recordList.get(i); + indexMap.put(timeRecord, i); + } + + return indexMap; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/RecordIndexQuery.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/RecordIndexQuery.java new file mode 100644 index 00000000..48cb9f84 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/RecordIndexQuery.java @@ -0,0 +1,22 @@ +package mil.army.usace.hec.vortex.io; + +import mil.army.usace.hec.vortex.VortexDataType; + +import java.time.ZonedDateTime; +import java.util.List; + +interface RecordIndexQuery { + List query(ZonedDateTime startTime, ZonedDateTime endTime); + + ZonedDateTime getEarliestStartTime(); + + ZonedDateTime getLatestEndTime(); + + static RecordIndexQuery of(VortexDataType dataType, List indexedRecords) { + return switch (dataType) { + case AVERAGE, ACCUMULATION -> PeriodRecordIndexQuery.from(indexedRecords); + case INSTANTANEOUS -> InstantaneousRecordIndexQuery.from(indexedRecords); + default -> new NoRecordIndexQuery(); + }; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SerialBatchImporter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SerialBatchImporter.java deleted file mode 100644 index 4c131b0a..00000000 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SerialBatchImporter.java +++ /dev/null @@ -1,38 +0,0 @@ -package mil.army.usace.hec.vortex.io; - -import mil.army.usace.hec.vortex.VortexProperty; -import mil.army.usace.hec.vortex.util.Stopwatch; - -import java.util.List; -import java.util.logging.Logger; - -class SerialBatchImporter extends BatchImporter { - private static final Logger logger = Logger.getLogger(SerialBatchImporter.class.getName()); - - SerialBatchImporter(Builder builder) { - super(builder); - } - - @Override - public void process() { - Stopwatch stopwatch = new Stopwatch(); - stopwatch.start(); - - List importableUnits = getImportableUnits(); - - for (ImportableUnit importableUnit : importableUnits) { - totalCount += importableUnit.getDtoCount(); - } - - importableUnits.forEach(importableUnit -> { - importableUnit.addPropertyChangeListener(propertyChangeListener()); - importableUnit.process(); - }); - - stopwatch.end(); - String timeMessage = "Batch import time: " + stopwatch; - logger.info(timeMessage); - - support.firePropertyChange(VortexProperty.STATUS, null, null); - } -} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SnodasDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SnodasDataReader.java index 63cf2d6e..c8e42904 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SnodasDataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SnodasDataReader.java @@ -58,6 +58,13 @@ public VortexData getDto(int idx) { return null; } // getDto() + @Override + public List getDataIntervals() { + return getDtos().stream() + .map(VortexDataInterval::of) + .toList(); + } + private static Map parseFile(String fileName) { Map info = new HashMap<>(); diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SnodasTarDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SnodasTarDataReader.java index 84ddc6a5..6a2ac9a8 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SnodasTarDataReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/SnodasTarDataReader.java @@ -4,12 +4,14 @@ import mil.army.usace.hec.vortex.VortexData; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.utils.IOUtils; import org.gdal.gdal.gdal; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -82,6 +84,13 @@ public VortexData getDto(int idx) { return null; } // getDto() + @Override + public List getDataIntervals() { + return getDtos().stream() + .map(VortexDataInterval::of) + .toList(); + } + private void updateTar(String pathToFile) throws IOException { // Get DirectoryPath, TarFilePath, and TarFileName String directoryPath = Paths.get(pathToFile).getParent().toString(); @@ -139,28 +148,6 @@ else if(productCode.equals("1038") && variableName.equals("Snow Pack Average Tem return false; } // matchedVariable() returns true if fileName matches with the variableName - - private File tarFolder(String directoryPath, File inputFolder) throws IOException { - // File tempTarFile = tarFolder(directoryPath, gzFolder); - // ^ Use that line in updateTar to compress the zip folder into a tar - // Create a temp Tar File - File tempTarFile = new File(directoryPath, "tempTar.tar"); - tempTarFile.createNewFile(); - TarArchiveOutputStream newStream = new TarArchiveOutputStream(new FileOutputStream(tempTarFile)); - - for(File currentFile : inputFolder.listFiles()) { - ArchiveEntry entry = newStream.createArchiveEntry(currentFile, currentFile.getName()); - newStream.putArchiveEntry(entry); - InputStream folderStream = Files.newInputStream(currentFile.toPath()); - IOUtils.copy(folderStream, newStream); - folderStream.close(); - newStream.closeArchiveEntry(); - } - newStream.close(); - - return tempTarFile; - } // tarFolder - private File unTarFile(String directoryPath, TarArchiveInputStream iStream, String tarName) throws IOException { String fileSeparator = File.separator; ArchiveEntry nextEntry = null; diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TemporalDataCalculator.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TemporalDataCalculator.java new file mode 100644 index 00000000..cf7d1df8 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TemporalDataCalculator.java @@ -0,0 +1,246 @@ +package mil.army.usace.hec.vortex.io; + +import hec.heclib.util.Heclib; +import mil.army.usace.hec.vortex.VortexGrid; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; + +class TemporalDataCalculator { + private static final Logger logger = Logger.getLogger(TemporalDataCalculator.class.getName()); + + private TemporalDataCalculator() { + // Utility Class + } + + static float[] calculateWeightedAccumulation(List grids, ZonedDateTime startTime, ZonedDateTime endTime) { + // Accumulation Data = [Sum of grids' (data * overlap ratio)] + if (hasNoTargetData(grids, startTime, endTime)) return undefinedGridData(); + double[] weights = calculateAccumulationWeights(grids, startTime, endTime); + return calculateWeightedData(grids, weights); + } + + static float[] calculateWeightedAverage(List grids, ZonedDateTime startTime, ZonedDateTime endTime) { + // Average Data = [Sum of grids' (data * overlap ratio)] / [total overlap ratio] + if (hasNoTargetData(grids, startTime, endTime)) return undefinedGridData(); + double[] weights = calculateAverageWeights(grids, startTime, endTime); + return calculateWeightedData(grids, weights); + } + + static float[] calculatePointInstant(List vortexGrids, ZonedDateTime time) { + // Instant Data = floor grid + (rate of change between floor and ceiling) * (overlap time) + if (hasNoTargetData(vortexGrids, time, time)) return undefinedGridData(); + double[] weights = calculatePointInstantWeights(vortexGrids, time); + return calculateWeightedData(vortexGrids, weights); + } + + static float[] calculatePeriodInstant(List grids, ZonedDateTime startTime, ZonedDateTime endTime) { + if (hasNoTargetData(grids, startTime, endTime)) return undefinedGridData(); + float[] result = new float[grids.get(0).data().length]; + + ZonedDateTime remainingStart = startTime; + + for (int i = 0; i < grids.size() - 1; i++) { + VortexGrid grid1 = grids.get(i); + VortexGrid grid2 = grids.get(i + 1); + + ZonedDateTime avgEnd = grid2.startTime(); + double[] avgData = calculateAverageOfTwoGrids(grid1, grid2, remainingStart, endTime); + + for (int j = 0; j < avgData.length; j++) { + result[j] = (float) (result[j] + avgData[j]); + } + + remainingStart = avgEnd; + } + + int totalInterval = (int) Duration.ofSeconds(endTime.toEpochSecond() - startTime.toEpochSecond()).toMinutes(); + for (int i = 0; i < result.length; i++) { + result[i] /= totalInterval; + } + + return result; + } + + static float[][] getMinMaxForGrids(List grids) { + int dataLength = grids.get(0).data().length; + + float[][] minMaxData = new float[2][dataLength]; + Arrays.fill(minMaxData[0], Float.MAX_VALUE); + Arrays.fill(minMaxData[1], -Float.MAX_VALUE); + + for (VortexGrid grid : grids) { + float[] currentData = grid.data(); + double noDataValue = grid.noDataValue(); + + for (int i = 0; i < dataLength; i++) { + float datum = currentData[i]; + + if (datum == noDataValue) + continue; + + if (datum < minMaxData[0][i]) + minMaxData[0][i] = datum; + + if (datum > minMaxData[1][i]) + minMaxData[1][i] = datum; + } + } + + return minMaxData; + } + + /* Helpers */ + static VortexGrid buildGrid(VortexGrid baseGrid, ZonedDateTime startTime, ZonedDateTime endTime, float[] data) { + if (data == null || data.length == 0) { + data = new float[baseGrid.data().length]; + Arrays.fill(data, (float) baseGrid.noDataValue()); + } + + return VortexGrid.builder() + .dx(baseGrid.dx()).dy(baseGrid.dy()) + .nx(baseGrid.nx()).ny(baseGrid.ny()) + .originX(baseGrid.originX()).originY(baseGrid.originY()) + .wkt(baseGrid.wkt()) + .data(data) + .noDataValue(baseGrid.noDataValue()) + .units(baseGrid.units()) + .fileName(baseGrid.fileName()) + .shortName(baseGrid.shortName()) + .fullName(baseGrid.fullName()) + .description(baseGrid.description()) + .startTime(startTime).endTime(endTime) + .interval(Duration.between(startTime, endTime)) + .dataType(baseGrid.dataType()) + .build(); + } + + private static float[] calculateWeightedData(List grids, double[] weights) { + // Result = data * weight. If any grid's data[i] is NaN, then result[i] is NaN + if (grids.isEmpty() || grids.size() != weights.length) { + logger.severe("Invalid grids and/or weights"); + return undefinedGridData(); + } + + int numGrids = grids.size(); + int numData = grids.get(0).data().length; + + float[] result = new float[numData]; + + for (int gridIndex = 0; gridIndex < numGrids; gridIndex++) { + VortexGrid grid = grids.get(gridIndex); + float[] gridData = grid.data(); + double noDataValue = grid.noDataValue(); + + for (int dataIndex = 0; dataIndex < numData; dataIndex++) { + double data = gridData[dataIndex]; + boolean isNaN = result[dataIndex] == noDataValue || data == noDataValue; + + if (!isNaN) { + double weightedData = gridData[dataIndex] * weights[gridIndex]; + result[dataIndex] = (float) (result[dataIndex] + weightedData); + } else { + result[dataIndex] = (float) grid.noDataValue(); + } + } + } + + return result; + } + + private static double[] calculatePointInstantWeights(List vortexGrids, ZonedDateTime targetTime) { + return switch (vortexGrids.size()) { + case 1 -> new double[]{1f}; + case 2 -> { + VortexGrid floorGrid = vortexGrids.get(0); + VortexGrid ceilingGrid = vortexGrids.get(1); + long grid1Time = floorGrid.startTime().toEpochSecond(); + long grid2Time = ceilingGrid.startTime().toEpochSecond(); + double fraction = (double) (targetTime.toEpochSecond() - grid1Time) / (grid2Time - grid1Time); + yield new double[]{1 - fraction, fraction}; + } + default -> { + logger.severe("Should not have more than two grids to interpolate"); + yield new double[0]; + } + }; + } + + private static double[] calculateAccumulationWeights(List grids, ZonedDateTime start, ZonedDateTime end) { + VortexDataInterval queryRecord = VortexDataInterval.of(start, end); + return grids.stream() + .map(VortexDataInterval::of) + .mapToDouble(timeRecord -> timeRecord.getPercentOverlapped(queryRecord)) + .toArray(); + } + + private static double[] calculateAverageWeights(List grids, ZonedDateTime start, ZonedDateTime end) { + double[] ratios = new double[grids.size()]; + + List timeRecords = grids.stream().map(VortexDataInterval::of).toList(); + long totalDuration = end.toEpochSecond() - start.toEpochSecond(); + + VortexDataInterval queryRecord = VortexDataInterval.of(start, end); + for (int i = 0; i < timeRecords.size(); i++) { + VortexDataInterval timeRecord = timeRecords.get(i); + double overlapDuration = timeRecord.getOverlapDuration(queryRecord).toSeconds(); + double ratio = overlapDuration / totalDuration; + ratios[i] = ratio; + } + + return ratios; + } + + private static double[] calculateAverageOfTwoGrids(VortexGrid firstGrid, VortexGrid secondGrid, ZonedDateTime start, ZonedDateTime end) { + VortexDataInterval timeRecord = VortexDataInterval.of(firstGrid.startTime(), secondGrid.startTime()); + VortexDataInterval queryRecord = VortexDataInterval.of(start, end); + int overlapInterval = (int) timeRecord.getOverlapDuration(queryRecord).toMinutes(); + int wholeInterval = (int) timeRecord.getRecordDuration().toMinutes(); + + float[] data1 = firstGrid.data(); + float[] data2 = secondGrid.data(); + double[] result = new double[data1.length]; + + for (int i = 0; i < data1.length; i++) { + double value1 = data1[i]; + double value2 = data2[i]; + + if (value1 == Heclib.UNDEFINED_DOUBLE || value2 == Heclib.UNDEFINED_DOUBLE) { + result[i] = Heclib.UNDEFINED_DOUBLE; + continue; + } + + boolean adjustStart = timeRecord.startTime().toEpochSecond() != start.toEpochSecond(); + boolean adjustEnd = timeRecord.endTime().toEpochSecond() != end.toEpochSecond(); + double difference = value2 - value1; + + value1 = adjustStart ? value2 - overlapInterval * difference / wholeInterval : value1; + value2 = adjustEnd ? value1 + overlapInterval * difference / wholeInterval : value2; + + double avg = 0.5 * (value1 + value2) * overlapInterval; + result[i] = avg; + } + + return result; + } + + private static boolean hasNoTargetData(List vortexGrids, ZonedDateTime targetStart, ZonedDateTime targetEnd) { + // No grids found for time period [targetStart, targetEnd] + if (vortexGrids == null || vortexGrids.isEmpty()) return true; + + VortexGrid firstGrid = vortexGrids.get(0); + VortexGrid lastGrid = vortexGrids.get(vortexGrids.size() - 1); + + boolean afterFirstGrid = targetStart.toEpochSecond() >= firstGrid.startTime().toEpochSecond(); + boolean beforeLastGrid = targetEnd.toEpochSecond() <= lastGrid.endTime().toEpochSecond(); + + return !afterFirstGrid || !beforeLastGrid; + } + + private static float[] undefinedGridData() { + return new float[0]; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TemporalDataReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TemporalDataReader.java new file mode 100644 index 00000000..34ced29a --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TemporalDataReader.java @@ -0,0 +1,306 @@ +package mil.army.usace.hec.vortex.io; + +import hec.heclib.util.Heclib; +import mil.army.usace.hec.vortex.VortexDataType; +import mil.army.usace.hec.vortex.VortexGrid; +import mil.army.usace.hec.vortex.geo.Grid; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.*; + +/** + * The TemporalDataReader class provides functionality to read and process temporal + * data from a given data source. It supports various operations like reading data + * for a specific time range and getting grid definitions. + */ +public class TemporalDataReader { + private final DataReader dataReader; + private final RecordIndexQuery recordIndexQuery; + private final VortexGrid baseGrid; + + /* Constructor */ + private TemporalDataReader(DataReader dataReader) { + this.dataReader = dataReader; + this.baseGrid = Optional.ofNullable(dataReader.getDto(0)) + .filter(VortexGrid.class::isInstance) + .map(VortexGrid.class::cast) + .orElse(null); + + if (baseGrid != null) { + VortexDataType dataType = baseGrid.dataType(); + List timeIntervals = this.dataReader.getDataIntervals(); + recordIndexQuery = RecordIndexQuery.of(dataType, timeIntervals); + } else { + recordIndexQuery = RecordIndexQuery.of(VortexDataType.UNDEFINED, Collections.emptyList()); + } + } + + /* Factory */ + public static TemporalDataReader create(DataReader dataReader) { + return new TemporalDataReader(dataReader); + } + + /** + * Reads and returns an Optional containing a VortexGrid for the specified time range. + * If no data is found for the specified time range, an empty Optional is returned. + * + * @param startTime The start time for the data reading. + * @param endTime The end time for the data reading. + * @return An Optional containing a VortexGrid with data for the specified time range, + * or an empty Optional if no data is found. + */ + public Optional read(ZonedDateTime startTime, ZonedDateTime endTime) { + if (baseGrid == null) { + return Optional.empty(); + } + + // Special case + if (!baseGrid.isTemporal() && dataReader.getDtoCount() == 1) { + return Optional.of(baseGrid); + } + + VortexDataType dataType = baseGrid.dataType(); + return Optional.ofNullable(read(dataType, startTime, endTime)); + } + + /** + * Retrieves the minimum and maximum grid data within a specified time range. + * + * @param startTime The start time for the data range. + * @param endTime The end time for the data range. + * @return An array of VortexGrid objects, where the first element is the minimum grid + * and the second element is the maximum grid for the specified time range. + * Returns an empty array if no data is found. + */ + public VortexGrid[] getMinMaxGridData(ZonedDateTime startTime, ZonedDateTime endTime) { + if (baseGrid == null) return new VortexGrid[0]; + VortexDataType dataType = baseGrid.dataType(); + return getMinMaxGridData(dataType, startTime, endTime); + } + + /** + * Retrieves the grid definition of the underlying data. + * If no base grid is available, an empty Optional is returned. + * + * @return An Optional containing a Grid object representing the grid definition, + * or an empty Optional if no base grid is available. + */ + public Optional getGridDefinition() { + if (baseGrid == null) { + return Optional.empty(); + } + + Grid gridDefinition = Grid.builder() + .dx(baseGrid.dx()) + .dy(baseGrid.dy()) + .nx(baseGrid.nx()) + .ny(baseGrid.ny()) + .originX(baseGrid.originX()) + .originY(baseGrid.originY()) + .crs(baseGrid.wkt()) + .build(); + + return Optional.of(gridDefinition); + } + + /** + * Retrieves the start time of the data records. + * If no record is found, an empty Optional is returned. + * + * @return An Optional containing the ZonedDateTime representing the earliest start time + * in the record list, or an empty Optional if no record is found. + */ + public Optional getStartTime() { + return Optional.ofNullable(recordIndexQuery.getEarliestStartTime()); + } + + /** + * Retrieves the end time of the data records. + * If no record is found, an empty Optional is returned. + * + * @return An Optional containing the ZonedDateTime representing the latest end time + * in the record list, or an empty Optional if no record is found. + */ + public Optional getEndTime() { + return Optional.ofNullable(recordIndexQuery.getLatestEndTime()); + } + + /** + * Retrieves the units string of the data. + * If no base grid is available, an empty Optional is returned. + * + * @return An Optional containing a string representing the units of the data, + * or an empty Optional if no base grid is available. + */ + public Optional getDataUnits() { + return Optional.ofNullable(baseGrid).map(VortexGrid::units); + } + + /* Read Methods */ + private VortexGrid read(VortexDataType dataType, ZonedDateTime startTime, ZonedDateTime endTime) { + if (startTime == null || endTime == null) { + return baseGrid; + } + + return switch (dataType) { + case ACCUMULATION -> readAccumulationData(startTime, endTime); + case AVERAGE -> readAverageData(startTime, endTime); + case INSTANTANEOUS -> readInstantaneousData(startTime, endTime); + default -> null; + }; + } + + private VortexGrid readAccumulationData(ZonedDateTime startTime, ZonedDateTime endTime) { + List relevantGrids = getGridsForPeriod(startTime, endTime); + float[] data = TemporalDataCalculator.calculateWeightedAccumulation(relevantGrids, startTime, endTime); + return TemporalDataCalculator.buildGrid(baseGrid, startTime, endTime, data); + } + + private VortexGrid readAverageData(ZonedDateTime startTime, ZonedDateTime endTime) { + List relevantGrids = getGridsForPeriod(startTime, endTime); + float[] data = TemporalDataCalculator.calculateWeightedAverage(relevantGrids, startTime, endTime); + return TemporalDataCalculator.buildGrid(baseGrid, startTime, endTime, data); + } + + private VortexGrid readInstantaneousData(ZonedDateTime startTime, ZonedDateTime endTime) { + List relevantGrids = recordIndexQuery.query(startTime, endTime).stream() + .map(dataReader::getDto) + .filter(VortexGrid.class::isInstance) + .map(VortexGrid.class::cast) + .filter(grid -> grid.startTime() != null) + .sorted(Comparator.comparing(VortexGrid::startTime)) + .toList(); + + if (startTime.isEqual(endTime)) { + float[] data = TemporalDataCalculator.calculatePointInstant(relevantGrids, startTime); + return TemporalDataCalculator.buildGrid(baseGrid, startTime, startTime, data); + } else { + float[] data = TemporalDataCalculator.calculatePeriodInstant(relevantGrids, startTime, endTime); + return TemporalDataCalculator.buildGrid(baseGrid, startTime, endTime, data); + } + } + + /* Get Min & Max Methods */ + private VortexGrid[] getMinMaxGridData(VortexDataType dataType, ZonedDateTime startTime, ZonedDateTime endTime) { + return switch (dataType) { + case ACCUMULATION, AVERAGE -> getMinMaxForPeriodGrids(startTime, endTime); + case INSTANTANEOUS -> getMinMaxForInstantaneousGrids(startTime, endTime); + default -> null; + }; + } + + private VortexGrid[] getMinMaxForPeriodGrids(ZonedDateTime startTime, ZonedDateTime endTime) { + List indices = new ArrayList<>(recordIndexQuery.query(startTime, endTime)); + List relevantGrids = indices.stream() + .map(dataReader::getDto) + .filter(VortexGrid.class::isInstance) + .map(VortexGrid.class::cast) + .toList(); + return buildMinMaxGrids(relevantGrids, startTime, endTime); + } + + private VortexGrid[] getMinMaxForInstantaneousGrids(ZonedDateTime startTime, ZonedDateTime endTime) { + List indices = new ArrayList<>(recordIndexQuery.query(startTime, endTime)); + indices.remove(indices.size() - 1); // Drop last index + + List relevantGrids = indices.stream() + .map(dataReader::getDto) + .filter(VortexGrid.class::isInstance) + .map(VortexGrid.class::cast) + .filter(grid -> grid.startTime() != null) + .filter(g -> g.startTime().toEpochSecond() >= startTime.toEpochSecond()) + .toList(); + return buildMinMaxGrids(relevantGrids, startTime, endTime); + } + + private VortexGrid[] buildMinMaxGrids(List grids, ZonedDateTime startTime, ZonedDateTime endTime) { + float[][] minMaxData; + + if (grids.isEmpty()) { + minMaxData = new float[][]{new float[0], new float[0]}; + } else { + minMaxData = TemporalDataCalculator.getMinMaxForGrids(grids); + } + + float[] minData = minMaxData[0]; + float[] maxData = minMaxData[1]; + + VortexGrid minGrid = TemporalDataCalculator.buildGrid(baseGrid, startTime, endTime, minData); + VortexGrid maxGrid = TemporalDataCalculator.buildGrid(baseGrid, startTime, endTime, maxData); + + return new VortexGrid[]{minGrid, maxGrid}; + } + + private List getGridsForPeriod(ZonedDateTime startTime, ZonedDateTime endTime) { + List relevantGrids = new ArrayList<>(); + long coveredUntil = startTime.toEpochSecond(); + + List indices = recordIndexQuery.query(startTime, endTime); + List overlappingGrids = indices.stream() + .map(dataReader::getDto) + .filter(VortexGrid.class::isInstance) + .map(VortexGrid.class::cast) + .sorted(gridPrioritizationComparator()) + .toList(); + + for (VortexGrid grid : overlappingGrids) { + VortexDataInterval timeRecord = VortexDataInterval.of(grid); + long recordStart = timeRecord.startTime().toEpochSecond(); + long recordEnd = timeRecord.endTime().toEpochSecond(); + + boolean isRelevant = recordEnd > coveredUntil && recordStart <= coveredUntil; + + if (isRelevant) { + relevantGrids.add(grid); + coveredUntil = recordEnd; + } + + if (coveredUntil >= endTime.toEpochSecond()) { + break; + } + } + + return relevantGrids; + } + + /* Grid Selection/Prioritization Logic */ + private static Comparator gridPrioritizationComparator() { + return Comparator.comparing(VortexGrid::endTime) + .thenComparing(prioritizeHigherResolutionDataComparator()) + .thenComparing(prioritizeLessNoDataComparator()); + } + + private static Comparator prioritizeHigherResolutionDataComparator() { + return (o1, o2) -> { + Duration duration1 = VortexDataInterval.of(o1).getRecordDuration(); + Duration duration2 = VortexDataInterval.of(o2).getRecordDuration(); + return duration1.compareTo(duration2); + }; + } + + private static Comparator prioritizeLessNoDataComparator() { + return (o1, o2) -> { + float[] o1DataArray = o1.data(); + float[] o2DataArray = o2.data(); + int o1Count = 0; + int o2Count = 0; + double noDataValue = o1.noDataValue(); + + for (int i = 0; i < o1DataArray.length; i++) { + float o1Data = o1DataArray[i]; + float o2Data = o2DataArray[i]; + + if (Float.isNaN(o1Data) || o1Data == noDataValue || o1Data == Heclib.UNDEFINED_FLOAT) { + o1Count++; + } + + if (Float.isNaN(o2Data) || o2Data == noDataValue || o2Data == Heclib.UNDEFINED_FLOAT) { + o2Count++; + } + } + + return Integer.compare(o1Count, o2Count); + }; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TiffDataWriter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TiffDataWriter.java index dbad9d2a..5ef844d0 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TiffDataWriter.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TiffDataWriter.java @@ -12,7 +12,7 @@ import java.util.Vector; import java.util.stream.Collectors; -public class TiffDataWriter extends DataWriter { +class TiffDataWriter extends DataWriter { private static final double NO_DATA_VALUE = -9999; TiffDataWriter(Builder builder) { diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TimeConverter.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TimeConverter.java new file mode 100644 index 00000000..9ac0ee2e --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/TimeConverter.java @@ -0,0 +1,105 @@ +package mil.army.usace.hec.vortex.io; + +import hec.heclib.util.HecTime; +import ucar.nc2.time.CalendarDate; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.util.Locale; +import java.util.logging.Logger; + +class TimeConverter { + private static final Logger logger = Logger.getLogger(TimeConverter.class.getName()); + private static final ZoneId UTC = ZoneId.of("UTC"); + + private static final DateTimeFormatter ymdFormatter = new DateTimeFormatterBuilder() + .appendPattern("yyyy[-][/][MM][M][-][/][dd]['T'][ ][HH][:][mm][:][ss][ ][VV][XXX]") + .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) + .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) + .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) + .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) + .toFormatter(); + + private static final DateTimeFormatter mdyFormatter = new DateTimeFormatterBuilder() + .appendPattern("[MM][M][-][/][dd][-][/]yyyy['T'][ ][HH][:][mm][:][ss][ ][VV][XXX]") + .parseDefaulting(ChronoField.DAY_OF_MONTH, 1) + .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) + .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) + .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) + .toFormatter(); + + private static final DateTimeFormatter hecTimeFormatter = new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .appendPattern("ddMMMuuuu:HHmm") + .toFormatter(Locale.ENGLISH); + + private TimeConverter() { + // Utility class - Private constructor + } + + static ZonedDateTime toZonedDateTime(HecTime hecTime) { + return ZonedDateTime.parse(hecTime.getXMLDateTime(0)); + } + + static ZonedDateTime toZonedDateTime(CalendarDate calendarDate) { + return ZonedDateTime.ofInstant(calendarDate.toDate().toInstant(), ZoneId.of("UTC")); + } + + static ZonedDateTime toZonedDateTime(String dateTimeString) { + TemporalAccessor parsedDate = parseDate(dateTimeString); + + if (parsedDate instanceof ZonedDateTime zonedDateTime) { + return zonedDateTime.withZoneSameInstant(UTC); + } else if (parsedDate instanceof LocalDateTime localDateTime) { + return localDateTime.atZone(UTC); + } else if (parsedDate instanceof LocalDate localDate) { + return localDate.atStartOfDay(UTC); + } else { + String message = String.format("Unable to parse: %s", dateTimeString); + logger.warning(message); + return null; + } + } + + private static TemporalAccessor parseDate(String dateTimeString) { + TemporalAccessor parsedDate = parseYearMonthDayFormat(dateTimeString); + if (parsedDate == null) { + parsedDate = parseMonthDayYear(dateTimeString); + } + if (parsedDate == null) { + parsedDate = parseHecTime(dateTimeString); + } + + return parsedDate; + } + + private static TemporalAccessor parseYearMonthDayFormat(String dateTimeString) { + try { + return ymdFormatter.parseBest(dateTimeString, ZonedDateTime::from, LocalDateTime::from, LocalDate::from); + } catch (Exception e) { + return null; + } + } + + private static TemporalAccessor parseMonthDayYear(String dateTimeString) { + try { + return mdyFormatter.parseBest(dateTimeString, ZonedDateTime::from, LocalDateTime::from, LocalDate::from); + } catch (Exception e) { + return null; + } + } + + private static TemporalAccessor parseHecTime(String dateTimeString) { + try { + return hecTimeFormatter.parseBest(dateTimeString, ZonedDateTime::from, LocalDateTime::from, LocalDate::from); + } catch (Exception e) { + return null; + } + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VariableDsReader.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VariableDsReader.java index 401c643c..e7d63f50 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VariableDsReader.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VariableDsReader.java @@ -1,11 +1,11 @@ package mil.army.usace.hec.vortex.io; +import mil.army.usace.hec.vortex.VortexData; import mil.army.usace.hec.vortex.VortexGrid; +import mil.army.usace.hec.vortex.geo.Grid; import mil.army.usace.hec.vortex.geo.ReferenceUtils; import mil.army.usace.hec.vortex.geo.WktFactory; -import si.uom.NonSI; -import tech.units.indriya.AbstractUnit; -import tech.units.indriya.unit.Units; +import mil.army.usace.hec.vortex.util.UnitUtil; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; @@ -13,105 +13,135 @@ import ucar.nc2.Variable; import ucar.nc2.constants.AxisType; import ucar.nc2.dataset.*; -import ucar.nc2.time.CalendarDate; -import javax.measure.IncommensurableException; import javax.measure.Unit; -import javax.measure.UnitConverter; import java.io.IOException; -import java.time.*; +import java.time.YearMonth; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.logging.Level; import java.util.logging.Logger; -import static javax.measure.MetricPrefix.KILO; - -public class VariableDsReader { +class VariableDsReader extends NetcdfDataReader { private static final Logger logger = Logger.getLogger(VariableDsReader.class.getName()); - private final NetcdfDataset ncd; - private final String variableName; - - private VariableDS variableDS; - private CoordinateSystem coordinateSystem; - private int nx; - private int ny; - private double dx; - private double dy; - private double ulx; - private double uly; - private String wkt; - - private VariableDsReader(Builder builder) { - ncd = builder.ncd; - variableName = builder.variableName; + private final VariableDS variableDS; + private final Grid gridDefinition; + private final List timeBounds; + + /* Constructor */ + public VariableDsReader(VariableDS variableDS, String variableName) { + super(new DataReaderBuilder().path(variableDS.getDatasetLocation()).variable(variableName)); + this.variableDS = getVariableDS(ncd, variableName); + CoordinateSystem coordinateSystem = getCoordinateSystem(variableDS); + this.gridDefinition = getGridDefinition(ncd, coordinateSystem); + this.timeBounds = getTimeBounds(); } - public static class Builder { - private NetcdfDataset ncd; - private String variableName; + private static VariableDS getVariableDS(NetcdfDataset ncd, String variableName) { + Variable variable = ncd.findVariable(variableName); + return variable instanceof VariableDS variableDS ? variableDS : null; + } - public Builder setNetcdfFile(NetcdfDataset ncd) { - this.ncd = ncd; - return this; - } + private static CoordinateSystem getCoordinateSystem(VariableDS variableDS) { + List coordinateSystems = variableDS.getCoordinateSystems(); + return !coordinateSystems.isEmpty() ? coordinateSystems.get(0) : null; + } - public Builder setVariableName(String variableName) { - this.variableName = variableName; - return this; + /* Public Methods */ + @Override + public List getDtos() { + List dataList = new ArrayList<>(); + for (int i = 0; i < getDtoCount(); i++) { + VortexData data = getDto(i); + dataList.add(data); } - public VariableDsReader build() { - return new VariableDsReader(this); - } + return dataList; } - public static Builder builder() { - return new Builder(); + @Override + public VortexGrid getDto(int index) { + return getTimeAxis() != null ? buildGridWithTimeAxis(index) : buildGridWithoutTimeAxis(); } - public VortexGrid read(int index) { - - try { - Variable variable = ncd.findVariable(variableName); - if (variable instanceof VariableDS) { - this.variableDS = (VariableDS) variable; - - List coordinateSystems = variableDS.getCoordinateSystems(); - if (!coordinateSystems.isEmpty()) { - coordinateSystem = coordinateSystems.get(0); - } + @Override + public int getDtoCount() { + List dimensions = variableDS.getDimensions(); + for (Dimension dimension : dimensions) { + if (dimension.getShortName().equals("time")) { + return dimension.getLength(); } - } catch (Exception e) { - logger.log(Level.SEVERE, e, e::getMessage); } - processCellInfo(); + return 1; + } - CoordinateAxis1D timeAxis; - if (ncd.findCoordinateAxis(AxisType.Time) != null) { - timeAxis = (CoordinateAxis1D) ncd.findCoordinateAxis(AxisType.Time); - } else { - timeAxis = (CoordinateAxis1D) ncd.findCoordinateAxis("time"); - } + /* Helpers */ + private List getTimeBounds() { + CoordinateAxis1D timeAxis = getTimeAxis(); + if (timeAxis == null) return Collections.emptyList(); - if (timeAxis == null) { - Array array; - try { - array = variableDS.read(); - } catch (IOException e) { - logger.log(Level.SEVERE, e, e::getMessage); - array = ucar.ma2.Array.factory(DataType.FLOAT, new int[]{}); - } + String timeAxisUnits = timeAxis.getUnitsString(); + boolean isYearMonth = timeAxisUnits.equalsIgnoreCase("yyyymm"); + return isYearMonth ? getYearMonthTimeRecords(timeAxis) : getDataIntervals(); + } + + private CoordinateAxis1D getTimeAxis() { + CoordinateAxis timeAxis = ncd.findCoordinateAxis(AxisType.Time); + timeAxis = timeAxis == null ? ncd.findCoordinateAxis("time") : timeAxis; + return timeAxis instanceof CoordinateAxis1D axis ? axis : null; + } - float[] data = getFloatArray(array); - return createDTO(data); + private List getYearMonthTimeRecords(CoordinateAxis1D timeAxis) { + List timeRecords = new ArrayList<>(); + for (int i = 0; i < getDtoCount(); i++) { + VortexDataInterval timeRecord = getYearMonthTimeRecord(timeAxis, i); + timeRecords.add(timeRecord); } + return timeRecords; + } + private VortexDataInterval getYearMonthTimeRecord(CoordinateAxis1D timeAxis, int timeIndex) { + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("uuuuMM"); + YearMonth startYearMonth = YearMonth.parse(String.valueOf(Math.round(timeAxis.getBound1()[timeIndex])), formatter); + YearMonth endYearMonth = YearMonth.parse(String.valueOf(Math.round(timeAxis.getBound2()[timeIndex])), formatter); + ZonedDateTime startTime = startYearMonth.atDay(1).atStartOfDay(ZoneId.of("UTC")); + ZonedDateTime endTime = endYearMonth.atDay(1).atStartOfDay(ZoneId.of("UTC")); + return VortexDataInterval.of(startTime, endTime); + } catch (DateTimeParseException e) { + logger.info(e::getMessage); + return undefinedTimeRecord(); + } + } + + private VortexDataInterval undefinedTimeRecord() { + ZonedDateTime undefinedStartTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); + ZonedDateTime undefinedEndTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); + return VortexDataInterval.of(undefinedStartTime, undefinedEndTime); + } + + private VortexGrid buildGridWithTimeAxis(int timeIndex) { + int dimensionIndex = getTimeDimensionIndex(); + float[] data = readSlicedData(dimensionIndex, timeIndex); + VortexDataInterval timeRecord = timeBounds.get(timeIndex); + return buildGrid(data, timeRecord); + } + + private VortexGrid buildGridWithoutTimeAxis() { + float[] data = readAllData(); + return buildGrid(data, undefinedTimeRecord()); + } + + private int getTimeDimensionIndex() { List dimensions = variableDS.getDimensions(); + int timeDimension = -1; for (int i = 0; i < dimensions.size(); i++) { if (dimensions.get(i).getShortName().contains("time")) { @@ -119,102 +149,43 @@ public VortexGrid read(int index) { } } - Array array; + return timeDimension; + } + + private float[] readAllData() { try { - if (timeDimension >= 0) { - array = variableDS.slice(timeDimension, index).read(); - } else { - array = variableDS.read(); - } - } catch (IOException | InvalidRangeException e) { - logger.log(Level.SEVERE, e, e::getMessage); - array = ucar.ma2.Array.factory(DataType.FLOAT, new int[]{}); + Array array = variableDS.read(); + return getFloatArray(array); + } catch (IOException e) { + logger.severe(e.getMessage()); + return new float[0]; } + } - float[] data = getFloatArray(array); - - String timeAxisUnits = timeAxis.getUnitsString(); - - if (timeAxisUnits.equalsIgnoreCase("yyyymm")) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("uuuuMM"); - try { - YearMonth startYearMonth = YearMonth.parse(String.valueOf(Math.round(timeAxis.getBound1()[index])), formatter); - YearMonth endYearMonth = YearMonth.parse(String.valueOf(Math.round(timeAxis.getBound2()[index])), formatter); - ZonedDateTime startTime = ZonedDateTime.of(startYearMonth.getYear(), startYearMonth.getMonth().getValue(), - 1, 0, 0, 0, 0, ZoneId.of("UTC")); - ZonedDateTime endTime = ZonedDateTime.of(endYearMonth.getYear(), endYearMonth.getMonth().getValue(), - 1, 0, 0, 0, 0, ZoneId.of("UTC")); - Duration interval = Duration.between(startTime, endTime); - return createDTO(data, startTime, endTime, interval); - } catch (DateTimeParseException e) { - logger.info(e::getMessage); - ZonedDateTime startTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); - ZonedDateTime endTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); - Duration interval = Duration.ofMinutes(0); - return createDTO(data, startTime, endTime, interval); - } - } else { - String dateTimeString = (timeAxisUnits.split(" ", 3)[2]).replaceFirst(" ", "T").split(" ")[0].replace(".", ""); - - ZonedDateTime origin; - if (dateTimeString.contains("T")) { - CalendarDate calendarDate = CalendarDate.parseISOformat(null, dateTimeString); - origin = ZonedDateTime.of(LocalDateTime.parse(calendarDate.toString(), DateTimeFormatter.ISO_DATE_TIME), ZoneId.of("UTC")); - } else { - origin = ZonedDateTime.of(LocalDate.parse(dateTimeString, DateTimeFormatter.ofPattern("uuuu-M-d")), LocalTime.of(0, 0), ZoneId.of("UTC")); - } + private float[] readSlicedData(int timeDimension, int timeIndex) { + if (timeDimension < 0) { + return readAllData(); + } - try { - ZonedDateTime startTime; - ZonedDateTime endTime; - if (timeAxisUnits.toLowerCase().matches("^month[s]? since.*$")) { - startTime = origin.plusMonths((long) timeAxis.getBound1()[index]); - endTime = origin.plusMonths((long) timeAxis.getBound2()[index]); - } else if (timeAxisUnits.toLowerCase().matches("^day[s]? since.*$")) { - startTime = origin.plusSeconds((long) timeAxis.getBound1()[index] * 86400); - endTime = origin.plusSeconds((long) timeAxis.getBound2()[index] * 86400); - } else if (timeAxisUnits.toLowerCase().matches("^hour[s]? since.*$")) { - startTime = origin.plusSeconds((long) timeAxis.getBound1()[index] * 3600); - if (ncd.getLocation().toLowerCase().matches("hrrr.*wrfsfcf.*")) { - endTime = startTime.plusHours(1); - } else { - endTime = origin.plusSeconds((long) timeAxis.getBound2()[index] * 3600); - } - } else if (timeAxisUnits.toLowerCase().matches("^minute[s]? since.*$")) { - endTime = origin.plusSeconds((long) timeAxis.getBound2()[index] * 60); - if (variableDS.getDescription().toLowerCase().contains("qpe01h")) { - startTime = endTime.minusHours(1); - } else { - startTime = origin.plusSeconds((long) timeAxis.getBound1()[index] * 60); - } - } else if (timeAxisUnits.toLowerCase().matches("^second[s]? since.*$")) { - startTime = origin.plusSeconds((long) timeAxis.getBound1()[index]); - endTime = origin.plusSeconds((long) timeAxis.getBound2()[index]); - } else { - startTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); - endTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); - } - Duration interval = Duration.between(startTime, endTime); - return createDTO(data, startTime, endTime, interval); - } catch (Exception e) { - logger.info(e::getMessage); - ZonedDateTime startTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); - ZonedDateTime endTime = ZonedDateTime.of(0, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")); - Duration interval = Duration.ofMinutes(0); - return createDTO(data, startTime, endTime, interval); - } + try { + Array array = variableDS.slice(timeDimension, timeIndex).read(); + return getFloatArray(array); + } catch (IOException | InvalidRangeException e) { + logger.severe("Error reading sliced data: " + e.getMessage()); + return new float[0]; } } - private VortexGrid createDTO(float[] data, ZonedDateTime startTime, ZonedDateTime endTime, Duration interval) { + private VortexGrid buildGrid(float[] data, VortexDataInterval timeRecord) { + // Grid must be shifted after getData call since getData uses the original locations to map values. + Grid grid = Grid.toBuilder(gridDefinition).build(); + shiftGrid(grid); + return VortexGrid.builder() - .dx(dx) - .dy(dy) - .nx(nx) - .ny(ny) - .originX(ulx) - .originY(uly) - .wkt(wkt) + .dx(grid.getDx()).dy(grid.getDy()) + .nx(grid.getNx()).ny(grid.getNy()) + .originX(grid.getOriginX()).originY(grid.getOriginY()) + .wkt(grid.getCrs()) .data(data) .noDataValue(variableDS.getFillValue()) .units(variableDS.getUnitsString()) @@ -222,42 +193,18 @@ private VortexGrid createDTO(float[] data, ZonedDateTime startTime, ZonedDateTim .shortName(variableDS.getShortName()) .fullName(variableDS.getFullName()) .description(variableDS.getDescription()) - .startTime(startTime) - .endTime(endTime) - .interval(interval) + .startTime(timeRecord.startTime()) + .endTime(timeRecord.endTime()) + .interval(timeRecord.getRecordDuration()) + .dataType(getVortexDataType(variableDS)) .build(); } - private VortexGrid createDTO(float[] data) { - return createDTO(data, null, null, Duration.ZERO); - } - - private float[] getFloatArray(ucar.ma2.Array array) { - float[] data; - Object myArr; - try { - myArr = array.copyTo1DJavaArray(); - if (myArr instanceof float[]) { - data = (float[]) myArr; - return data; - } else if (myArr instanceof double[] doubleArray) { - data = new float[(int) array.getSize()]; - float[] datalocal = data; - for (int i = 0; i < data.length; i++) { - datalocal[i] = (float) (doubleArray[i]); - } - } else { - // Could not parse - data = new float[]{}; - } - } catch (ClassCastException e) { - data = new float[]{}; - } - return data; + private float[] getFloatArray(Array array) { + return (float[]) array.get1DJavaArray(DataType.FLOAT); } - private void processCellInfo() { - + private static Grid getGridDefinition(NetcdfDataset ncd, CoordinateSystem coordinateSystem) { CoordinateAxis lonAxis = ncd.findCoordinateAxis(AxisType.Lon); CoordinateAxis latAxis = ncd.findCoordinateAxis(AxisType.Lat); @@ -277,59 +224,43 @@ private void processCellInfo() { throw new IllegalStateException(); } - nx = (int) xAxis.getSize(); - ny = (int) yAxis.getSize(); + int nx = (int) xAxis.getSize(); + int ny = (int) yAxis.getSize(); double[] edgesX = ((CoordinateAxis1D) xAxis).getCoordEdges(); - ulx = edgesX[0]; + double ulx = edgesX[0]; double urx = edgesX[edgesX.length - 1]; - dx = (urx - ulx) / nx; + double dx = (urx - ulx) / nx; double[] edgesY = ((CoordinateAxis1D) yAxis).getCoordEdges(); - uly = edgesY[0]; + double uly = edgesY[0]; double lly = edgesY[edgesY.length - 1]; - dy = (lly - uly) / ny; + double dy = (lly - uly) / ny; + String wkt = null; if (coordinateSystem != null) { wkt = WktFactory.createWkt(coordinateSystem.getProjection()); - } - - // If there is no wkt at this point, assume WGS84 - if (wkt == null || wkt.isBlank()) { + } else if (lonAxis != null && latAxis != null) { wkt = WktFactory.fromEpsg(4326); } - String xAxisUnits = Objects.requireNonNull(xAxis).getUnitsString(); - - Unit cellUnits = switch (xAxisUnits.toLowerCase()) { - case "m", "meter", "metre" -> Units.METRE; - case "km" -> KILO(Units.METRE); - case "degrees_east", "degrees_north" -> NonSI.DEGREE_ANGLE; - default -> AbstractUnit.ONE; - }; - - if (cellUnits == NonSI.DEGREE_ANGLE && ulx == 0) { - ulx = -180; - } - - if (cellUnits == NonSI.DEGREE_ANGLE && ulx > 180) { - ulx = ulx - 360; - } + Grid gridDefinition = Grid.builder() + .nx(nx).ny(ny) + .dx(dx).dy(dy) + .originX(ulx).originY(uly) + .crs(wkt) + .build(); + String xAxisUnits = Objects.requireNonNull(xAxis).getUnitsString(); + Unit cellUnits = UnitUtil.getUnits(xAxisUnits.toLowerCase()); Unit csUnits = ReferenceUtils.getLinearUnits(wkt); - if (cellUnits.isCompatible(csUnits)) { - try { - UnitConverter converter = cellUnits.getConverterToAny(csUnits); - ulx = converter.convert(ulx); - uly = converter.convert(uly); - dx = converter.convert(dx); - dy = converter.convert(dy); - } catch (IncommensurableException e) { - logger.log(Level.SEVERE, e, e::getMessage); - } + if (cellUnits.isCompatible(csUnits) && !cellUnits.equals(csUnits)) { + gridDefinition = scaleGrid(gridDefinition, cellUnits, csUnits); } + + return gridDefinition; } } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VortexDataInterval.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VortexDataInterval.java new file mode 100644 index 00000000..340c51fc --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VortexDataInterval.java @@ -0,0 +1,125 @@ +package mil.army.usace.hec.vortex.io; + +import hec.heclib.util.HecTime; +import mil.army.usace.hec.vortex.VortexData; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Optional; + +record VortexDataInterval(ZonedDateTime startTime, ZonedDateTime endTime) implements Interval { + static final VortexDataInterval UNDEFINED = VortexDataInterval.of(null, null); + + static VortexDataInterval of(ZonedDateTime startTime, ZonedDateTime endTime) { + return new VortexDataInterval(startTime, endTime); + } + + static VortexDataInterval of(String dssPathname) { + String[] parts = getDssParts(dssPathname); + + HecTime hecStart = new HecTime(parts[3]); + HecTime hecEnd = new HecTime(parts[4]); + + if (!hecStart.isDefined()) { + return null; + } + + if (!hecEnd.isDefined()) { + hecEnd = hecStart; + } + + ZonedDateTime start = TimeConverter.toZonedDateTime(hecStart); + ZonedDateTime end = TimeConverter.toZonedDateTime(hecEnd); + + return VortexDataInterval.of(start, end); + } + + static VortexDataInterval of(VortexData vortexData) { + ZonedDateTime start = vortexData.startTime(); + ZonedDateTime end = vortexData.endTime(); + return VortexDataInterval.of(start, end); + } + + static boolean isDefined(VortexDataInterval timeRecord) { + return timeRecord != null && timeRecord.startTime != null && timeRecord.endTime != null; + } + + Duration getRecordDuration() { + return isDefined(this) ? Duration.between(this.startTime, this.endTime) : Duration.ZERO; + } + + double getPercentOverlapped(VortexDataInterval other) { + long overlapDuration = getOverlapDuration(other).toSeconds(); + long recordDuration = getRecordDuration().toSeconds(); + return (double) overlapDuration / recordDuration; + } + + Duration getOverlapDuration(VortexDataInterval other) { + long recordStart = startTime.toEpochSecond(); + long recordEnd = endTime.toEpochSecond(); + + long overlapStart = Math.max(other.startTime().toEpochSecond(), recordStart); + long overlapEnd = Math.min(other.endTime().toEpochSecond(), recordEnd); + long overlapDuration = overlapEnd > overlapStart ? overlapEnd - overlapStart : 0; + + return Duration.ofSeconds(overlapDuration); + } + + boolean isInstantaneous() { + return startTime.isEqual(endTime); + } + + /* Interval */ + @Override + public long startEpochSecond() { + return startTime.toEpochSecond(); + } + + @Override + public long endEpochSecond() { + return endTime.toEpochSecond(); + } + + /* Override equals and hashCode to be the same for time with the same offset but different IDs */ + @Override + public boolean equals(Object o) { + if (!(o instanceof VortexDataInterval that)) { + return false; + } + + return startTime.toEpochSecond() == that.startTime.toEpochSecond() && + endTime.toEpochSecond() == that.endTime.toEpochSecond() && + startTime.getOffset().equals(that.startTime.getOffset()) && + endTime.getOffset().equals(that.endTime.getOffset()); + } + + @Override + public int hashCode() { + int result = Long.hashCode(startTime.toEpochSecond()); + result = 31 * result + startTime.getOffset().hashCode(); + result = 31 * result + Long.hashCode(endTime.toEpochSecond()); + result = 31 * result + endTime.getOffset().hashCode(); + return result; + } + + /* Helpers */ + private static String[] getDssParts(String pathname) { + String trimmedPathname = Optional.ofNullable(pathname) + .map(String::trim) + .filter(p -> p.startsWith("/") && p.endsWith("/")) + .map(p -> p.substring(1, p.length() - 1)) + .orElse(""); + + if (trimmedPathname.isEmpty()) { + return new String[0]; + } + + String[] parts = Arrays.stream(trimmedPathname.split("/", -1)) + .map(String::trim) + .toArray(String[]::new); + + // There should be exactly 6 parts + return parts.length == 6 ? parts : new String[0]; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VortexGridCollection.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VortexGridCollection.java new file mode 100644 index 00000000..acb56732 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/VortexGridCollection.java @@ -0,0 +1,173 @@ +package mil.army.usace.hec.vortex.io; + +import mil.army.usace.hec.vortex.VortexGrid; +import mil.army.usace.hec.vortex.geo.WktParser; +import ucar.nc2.constants.CF; +import ucar.unidata.geoloc.Projection; +import ucar.unidata.geoloc.projection.LatLonProjection; +import ucar.unidata.util.Parameter; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +class VortexGridCollection { + private static final Logger logger = Logger.getLogger(VortexGridCollection.class.getName()); + + private final List vortexGridList; + private final VortexGrid defaultGrid; + private final double[] xCoordinates; + private final double[] yCoordinates; + private final ZonedDateTime baseTime; + + private VortexGridCollection(List vortexGrids) { + baseTime = NetcdfTimeUtils.ORIGIN_TIME; + + vortexGridList = sanitizeCollection(vortexGrids); + if (!vortexGridList.isEmpty()) { + defaultGrid = vortexGrids.get(0); + xCoordinates = generateCoordinates(defaultGrid.originX(), defaultGrid.dx(), defaultGrid.nx()); + yCoordinates = generateCoordinates(defaultGrid.originY(), defaultGrid.dy(), defaultGrid.ny()); + } else { + defaultGrid = VortexGrid.empty(); + xCoordinates = new double[0]; + yCoordinates = new double[0]; + } + } + + /* Factory */ + static VortexGridCollection of(List vortexGrids) { + return new VortexGridCollection(vortexGrids); + } + + /* Init */ + private static List sanitizeCollection(List vortexGrids) { + if (vortexGrids == null || vortexGrids.isEmpty()) { + return Collections.emptyList(); + } + + VortexGrid baseGrid = vortexGrids.get(0); + String baseWkt = baseGrid.wkt(); + + Predicate predicate = vortexGrid -> { + boolean sameWkt = Objects.equals(baseWkt, vortexGrid.wkt()); + + if (sameWkt) { + return true; + } else { + logger.info(() -> "Filtered from collection: " + vortexGrid); + return false; + } + }; + + return vortexGrids.stream() + .filter(predicate) + .toList(); + } + + private static double[] generateCoordinates(double origin, double stepSize, int count) { + double[] coordinates = new double[count]; + + for (int i = 0; i < count; i++) { + coordinates[i] = origin + (i + 1) * stepSize - (stepSize / 2); + } + + return coordinates; + } + + /* Conditionals */ + boolean isGeographic() { + return getProjection() instanceof LatLonProjection; + } + + boolean hasTimeDimension() { + return vortexGridList.stream().allMatch(VortexGrid::isTemporal); + } + + boolean hasTimeBounds() { + Duration interval = getInterval(); + return interval != null && !interval.isZero(); + } + + /* Data */ + Stream> getCollectionDataStream() { + return IntStream.range(0, vortexGridList.size()).parallel().mapToObj(i -> Map.entry(i, vortexGridList.get(i))); + } + + /* Name & Description */ + Map getRepresentativeGridNameMap() { + return vortexGridList.stream().collect(Collectors.toMap(VortexGrid::shortName, g -> g, (existing, replacement) -> existing)); + } + + String getShortName() { + return defaultGrid.shortName(); + } + + String getDescription() { + return defaultGrid.description(); + } + + /* Projection */ + Projection getProjection() { + return WktParser.getProjection(getWkt()); + } + + String getProjectionName() { + Projection projection = getProjection(); + for (Parameter parameter : projection.getProjectionParameters()) { + if (parameter.getName().equals(CF.GRID_MAPPING_NAME)) { + return parameter.getStringValue(); + } + } + return "Unknown Projection Name"; + } + + String getProjectionUnit() { + return WktParser.getProjectionUnit(getWkt()); + } + + String getWkt() { + return defaultGrid.wkt(); + } + + /* Y and X */ + double[] getYCoordinates() { + return yCoordinates.clone(); + } + + double[] getXCoordinates() { + return xCoordinates.clone(); + } + + int getNy() { + return defaultGrid.ny(); + } + + int getNx() { + return defaultGrid.nx(); + } + + String getTimeUnits() { + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss XXX"); + String durationUnit = NetcdfTimeUtils.getDeltaTimeUnit(getBaseDuration()).toString(); + return durationUnit + " since " + baseTime.format(dateTimeFormatter); + } + + private Duration getInterval() { + return defaultGrid.interval(); + } + + private Duration getBaseDuration() { + Duration interval = getInterval(); + return interval.isZero() ? Duration.ofMinutes(1) : interval; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/WriteDataBuffer.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/WriteDataBuffer.java new file mode 100644 index 00000000..9b853277 --- /dev/null +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/io/WriteDataBuffer.java @@ -0,0 +1,35 @@ +package mil.army.usace.hec.vortex.io; + +import java.util.function.Consumer; +import java.util.stream.Stream; + +interface WriteDataBuffer { + enum Type {MEMORY_DYNAMIC} + + void add(T data); + + void clear(); + + boolean isFull(); + + Stream getBufferAsStream(); + + default void processBufferAndClear(Consumer> bufferProcessFunction, boolean isForced) { + bufferProcessFunction.accept(getBufferAsStream()); + clear(); + } + + default void addAndProcessWhenFull(T data, Consumer> bufferProcessFunction, boolean isForced) { + if (isFull() || isForced) { + processBufferAndClear(bufferProcessFunction, isForced); + } + + add(data); + } + + static WriteDataBuffer of(Type type) { + return switch (type) { + case MEMORY_DYNAMIC -> new MemoryDynamicWriteDataBuffer<>(); + }; + } +} diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/util/DssUtil.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/util/DssUtil.java index 19531952..e376b193 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/util/DssUtil.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/util/DssUtil.java @@ -1,9 +1,27 @@ package mil.army.usace.hec.vortex.util; import hec.heclib.dss.DSSPathname; +import hec.heclib.dss.DssDataType; +import hec.heclib.grid.AlbersInfo; +import hec.heclib.grid.GridInfo; +import hec.heclib.grid.SpecifiedGridInfo; +import hec.heclib.util.HecTime; +import mil.army.usace.hec.vortex.VortexGrid; +import org.gdal.osr.SpatialReference; +import javax.measure.Unit; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; +import static javax.measure.MetricPrefix.KILO; +import static javax.measure.MetricPrefix.MILLI; +import static systems.uom.common.USCustomary.*; +import static tech.units.indriya.AbstractUnit.ONE; +import static tech.units.indriya.unit.Units.HOUR; +import static tech.units.indriya.unit.Units.MINUTE; +import static tech.units.indriya.unit.Units.*; + public class DssUtil { private DssUtil(){} @@ -26,4 +44,158 @@ public static Map> getPathnameParts(List pathnames){ parts.put("fParts", fParts); return parts; } + + public static GridInfo getGridInfo(VortexGrid grid) { + GridInfo gridInfo; + + SpatialReference srs = new SpatialReference(grid.wkt()); + String crsName; + if (srs.IsProjected() == 1) { + crsName = srs.GetAttrValue("projcs"); + } else if (srs.IsGeographic() == 1) { + crsName = srs.GetAttrValue("geogcs"); + } else { + crsName = ""; + } + + if (crsName.toLowerCase().contains("albers")) { + + AlbersInfo albersInfo = new AlbersInfo(); + albersInfo.setCoordOfGridCellZero(0, 0); + + String datum = srs.GetAttrValue("geogcs"); + if (datum.contains("83")) { + albersInfo.setProjectionDatum(GridInfo.getNad83()); + } + + String units = srs.GetLinearUnitsName(); + albersInfo.setProjectionUnits(units); + + double centralMeridian = srs.GetProjParm("central_meridian"); + albersInfo.setCentralMeridian((float) centralMeridian); + + double falseEasting = srs.GetProjParm("false_easting"); + double falseNorthing = srs.GetProjParm("false_northing"); + albersInfo.setFalseEastingAndNorthing((float) falseEasting, (float) falseNorthing); + + double latitudeOfOrigin = srs.GetProjParm("latitude_of_origin"); + albersInfo.setLatitudeOfProjectionOrigin((float) latitudeOfOrigin); + + double standardParallel1 = srs.GetProjParm("standard_parallel_1"); + double standardParallel2 = srs.GetProjParm("standard_parallel_2"); + albersInfo.setStandardParallels((float) standardParallel1, (float) standardParallel2); + + gridInfo = albersInfo; + } else { + SpecifiedGridInfo specifiedGridInfo = new SpecifiedGridInfo(); + specifiedGridInfo.setSpatialReference(crsName, grid.wkt(), 0, 0); + gridInfo = specifiedGridInfo; + } + + double llx = grid.originX(); + double lly; + if (grid.dy() < 0) { + lly = grid.originY() + grid.dy() * grid.ny(); + } else { + lly = grid.originY(); + } + double dx = grid.dx(); + double dy = grid.dy(); + float cellSize = (float) ((Math.abs(dx) + Math.abs(dy)) / 2.0); + int minX = (int) Math.round(llx / cellSize); + int minY = (int) Math.round(lly / cellSize); + + gridInfo.setCellInfo(minX, minY, grid.nx(), grid.ny(), cellSize); + + Unit units = UnitUtil.getUnits(grid.units()); + String unitsString = getUnitsString(units); + gridInfo.setDataUnits(unitsString); + + ZonedDateTime startTime = grid.startTime(); + + if (startTime == null) + return gridInfo; + + ZonedDateTime endTime = grid.endTime(); + if (!startTime.equals(endTime) && units.isCompatible(CELSIUS)) { + gridInfo.setDataType(DssDataType.PER_AVER.value()); + } else if (startTime.equals(endTime)) { + gridInfo.setDataType(DssDataType.INST_VAL.value()); + } else { + gridInfo.setDataType(DssDataType.PER_CUM.value()); + } + + HecTime hecTimeStart = getHecTime(startTime); + hecTimeStart.showTimeAsBeginningOfDay(true); + HecTime hecTimeEnd = getHecTime(endTime); + hecTimeEnd.showTimeAsBeginningOfDay(false); + + gridInfo.setGridTimes(hecTimeStart, hecTimeEnd); + + return gridInfo; + } + + private static String getUnitsString(Unit unit) { + if (unit.equals(MILLI(METRE))) { + return "MM"; + } else if (unit.equals(INCH)) { + return "IN"; + } else if (unit.equals(INCH.divide(HOUR))) { + return "IN/HR"; + } else if (unit.equals(MILLI(METRE).divide(SECOND))) { + return "MM/S"; + } else if (unit.equals(MILLI(METRE).divide(HOUR))) { + return "MM/HR"; + } else if (unit.equals(MILLI(METRE).divide(DAY))) { + return "MM/DAY"; + } else if (unit.equals(CUBIC_METRE.divide(SECOND))) { + return "M3/S"; + } else if (unit.equals(CUBIC_FOOT.divide(SECOND))) { + return "CFS"; + } else if (unit.equals(METRE)) { + return "M"; + } else if (unit.equals(FOOT)) { + return "FT"; + } else if (unit.equals(CELSIUS)) { + return "DEG C"; + } else if (unit.equals(FAHRENHEIT)) { + return "DEG F"; + } else if (unit.equals(WATT.divide(SQUARE_METRE))) { + return "WATT/M2"; + } else if (unit.equals(KILOMETRE_PER_HOUR)) { + return "KPH"; + } else if (unit.equals(METRE_PER_SECOND)) { + return "M/S"; + } else if (unit.equals(MILE_PER_HOUR)) { + return "MPH"; + } else if (unit.equals(FOOT_PER_SECOND)) { + return "FT/S"; + } else if (unit.equals(KILO(PASCAL))) { + return "KPA"; + } else if (unit.equals(PASCAL)) { + return "PA"; + } else if (unit.equals(PERCENT)) { + return "%"; + } else if (unit.equals(KILO(METRE))) { + return "KM"; + } else if (unit.equals(MILE)) { + return "MILE"; + } else if (unit.equals(ONE)) { + return "UNSPECIF"; + } else if (unit.equals(TON)) { + return "TONS"; + } else if (unit.equals(MILLI(GRAM).divide(LITRE))) { + return "MG/L"; + } else if (unit.equals(CELSIUS.multiply(DAY))) { + return "DEGC-D"; + } else if (unit.equals(MINUTE)) { + return "MINUTES"; + } else { + return unit.toString(); + } + } + + private static HecTime getHecTime(ZonedDateTime zonedDateTime) { + return new HecTime(zonedDateTime.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); + } } diff --git a/vortex-api/src/main/java/mil/army/usace/hec/vortex/util/FilenameUtil.java b/vortex-api/src/main/java/mil/army/usace/hec/vortex/util/FilenameUtil.java index 71008593..95e632e7 100644 --- a/vortex-api/src/main/java/mil/army/usace/hec/vortex/util/FilenameUtil.java +++ b/vortex-api/src/main/java/mil/army/usace/hec/vortex/util/FilenameUtil.java @@ -12,4 +12,13 @@ public static String removeExtension(String filename, boolean removeAllExtension String extPattern = "(? getUnits(String units) { case "%" -> PERCENT; case "hpa" -> HECTO(PASCAL); case "pa" -> PASCAL; - case "m" -> METRE; + case "m", "meter", "metre" -> METRE; case "min" -> MINUTE; + case "km" -> KILO(METRE); + case "degrees", "degrees_east", "degrees_north" -> USCustomary.DEGREE_ANGLE; default -> ONE; }; } diff --git a/vortex-api/src/test/java/mil/army/usace/hec/vortex/VortexGridTest.java b/vortex-api/src/test/java/mil/army/usace/hec/vortex/VortexGridTest.java new file mode 100644 index 00000000..e52577d4 --- /dev/null +++ b/vortex-api/src/test/java/mil/army/usace/hec/vortex/VortexGridTest.java @@ -0,0 +1,27 @@ +package mil.army.usace.hec.vortex; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class VortexGridTest { + @Test + void getValue() { + float[] data = new float[10]; + for (int i = 0; i < data.length; i++) { + data[i] = (float) i * 5; + } + + VortexGrid grid = VortexGrid.builder().data(data).build(); + assertEquals(0, grid.getValue(0)); + assertEquals(5, grid.getValue(1)); + assertEquals(10, grid.getValue(2)); + assertEquals(15, grid.getValue(3)); + assertEquals(20, grid.getValue(4)); + assertEquals(25, grid.getValue(5)); + assertEquals(30, grid.getValue(6)); + assertEquals(35, grid.getValue(7)); + assertEquals(40, grid.getValue(8)); + assertEquals(45, grid.getValue(9)); + } +} \ No newline at end of file diff --git a/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/NetcdfDataReaderTest.java b/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/NetcdfDataReaderTest.java index 9bd80173..4942b51c 100644 --- a/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/NetcdfDataReaderTest.java +++ b/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/NetcdfDataReaderTest.java @@ -171,8 +171,8 @@ void ArizonaSweDaily(){ VortexData vortexData = reader.getDto(0); VortexGrid vortexGrid = (VortexGrid) vortexData; - assertEquals(ZonedDateTime.of(2020, 3, 1, 0, 0, 0, 0, ZoneId.of("Z")), vortexGrid.startTime()); - assertEquals(ZonedDateTime.of(2020, 3, 2, 0, 0, 0, 0, ZoneId.of("Z")), vortexGrid.endTime()); + assertEquals(ZonedDateTime.of(2020, 3, 1, 0, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.startTime()); + assertEquals(ZonedDateTime.of(2020, 3, 2, 0, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.endTime()); } @Test @@ -197,8 +197,8 @@ void NLDAS_Forcing_APCP(){ VortexData vortexData = reader.getDto(0); VortexGrid vortexGrid = (VortexGrid) vortexData; - assertEquals(ZonedDateTime.of(1981, 12, 31, 23, 0, 0, 0, ZoneId.of("Z")), vortexGrid.startTime()); - assertEquals(ZonedDateTime.of(1982, 1, 1, 0, 0, 0, 0, ZoneId.of("Z")), vortexGrid.endTime()); + assertEquals(ZonedDateTime.of(1981, 12, 31, 23, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.startTime()); + assertEquals(ZonedDateTime.of(1982, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.endTime()); } @Test @@ -212,8 +212,8 @@ void NLDAS_Forcing_TMP(){ VortexData vortexData = reader.getDto(0); VortexGrid vortexGrid = (VortexGrid) vortexData; - assertEquals(ZonedDateTime.of(1982, 1, 1, 0, 0, 0, 0, ZoneId.of("Z")), vortexGrid.startTime()); - assertEquals(ZonedDateTime.of(1982, 1, 1, 0, 0, 0, 0, ZoneId.of("Z")), vortexGrid.endTime()); + assertEquals(ZonedDateTime.of(1982, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.startTime()); + assertEquals(ZonedDateTime.of(1982, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.endTime()); } @Test @@ -227,8 +227,8 @@ void MRMS_RadarOnly_QPE(){ VortexData vortexData = reader.getDto(0); VortexGrid vortexGrid = (VortexGrid) vortexData; - assertEquals(ZonedDateTime.of(2021, 7, 5, 23, 0, 0, 0, ZoneId.of("Z")), vortexGrid.startTime()); - assertEquals(ZonedDateTime.of(2021, 7, 6, 0, 0, 0, 0, ZoneId.of("Z")), vortexGrid.endTime()); + assertEquals(ZonedDateTime.of(2021, 7, 5, 23, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.startTime()); + assertEquals(ZonedDateTime.of(2021, 7, 6, 0, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.endTime()); } @Test @@ -242,8 +242,8 @@ void MRMS_MultiSensor_QPE(){ VortexData vortexData = reader.getDto(0); VortexGrid vortexGrid = (VortexGrid) vortexData; - assertEquals(ZonedDateTime.of(2021, 7, 5, 23, 0, 0, 0, ZoneId.of("Z")), vortexGrid.startTime()); - assertEquals(ZonedDateTime.of(2021, 7, 6, 0, 0, 0, 0, ZoneId.of("Z")), vortexGrid.endTime()); + assertEquals(ZonedDateTime.of(2021, 7, 5, 23, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.startTime()); + assertEquals(ZonedDateTime.of(2021, 7, 6, 0, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.endTime()); } @Test @@ -259,8 +259,8 @@ void MRMS_PrecipRate(){ VortexData vortexData = reader.getDto(0); VortexGrid vortexGrid = (VortexGrid) vortexData; - assertEquals(ZonedDateTime.of(2021, 7, 5, 23, 58, 0, 0, ZoneId.of("Z")), vortexGrid.startTime()); - assertEquals(ZonedDateTime.of(2021, 7, 6, 0, 0, 0, 0, ZoneId.of("Z")), vortexGrid.endTime()); + assertEquals(ZonedDateTime.of(2021, 7, 5, 23, 58, 0, 0, ZoneId.of("UTC")), vortexGrid.startTime()); + assertEquals(ZonedDateTime.of(2021, 7, 6, 0, 0, 0, 0, ZoneId.of("UTC")), vortexGrid.endTime()); } @Test diff --git a/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/NetcdfDataWriterTest.java b/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/NetcdfDataWriterTest.java index 8c215a5c..d0bdebdb 100644 --- a/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/NetcdfDataWriterTest.java +++ b/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/NetcdfDataWriterTest.java @@ -22,7 +22,10 @@ class NetcdfDataWriterTest { @Test void InstantTimeCircleTest() throws IOException { - ZonedDateTime time = ZonedDateTime.of(1900,2,2,0,0,0,0, ZoneId.of("Z")); + String outputPath = TestUtil.createTempFile("InstantTimeCircleTest.nc"); + Assertions.assertNotNull(outputPath); + + ZonedDateTime time = ZonedDateTime.of(1900, 2, 2, 0, 0, 0, 0, ZoneId.of("UTC")); List originalGrids = new ArrayList<>(); for (int i = 0; i < 5; i++) { @@ -37,6 +40,8 @@ void InstantTimeCircleTest() throws IOException { .data(data) .noDataValue(-9999) .units("degC") + .fileName(outputPath) + .fullName("temperature") .shortName("temperature") .description("air_temperature") .startTime(time.plusHours(i)) @@ -48,9 +53,6 @@ void InstantTimeCircleTest() throws IOException { originalGrids.add(grid); } - String outputPath = TestUtil.createTempFile("InstantTimeCircleTest.nc"); - Assertions.assertNotNull(outputPath); - DataWriter writer = DataWriter.builder() .data(originalGrids) .destination(outputPath) @@ -74,7 +76,10 @@ void InstantTimeCircleTest() throws IOException { @Test void IntervalTimeCircleTest() throws IOException { - ZonedDateTime startTime = ZonedDateTime.of(1900,2,2,0,0,0,0, ZoneId.of("Z")); + String outputPath = TestUtil.createTempFile("IntervalTimeCircleTest.nc"); + Assertions.assertNotNull(outputPath); + + ZonedDateTime startTime = ZonedDateTime.of(1900, 2, 2, 0, 0, 0, 0, ZoneId.of("UTC")); ZonedDateTime endTime = startTime.plusHours(1); List originalGrids = new ArrayList<>(); @@ -90,6 +95,7 @@ void IntervalTimeCircleTest() throws IOException { .data(data) .noDataValue(-9999) .units("m") + .fileName(outputPath) .shortName("precipitation") .description("lwe_thickness_of_precipitation_amount") .startTime(startTime.plusHours(i)) @@ -101,9 +107,6 @@ void IntervalTimeCircleTest() throws IOException { originalGrids.add(grid); } - String outputPath = TestUtil.createTempFile("IntervalTimeCircleTest.nc"); - Assertions.assertNotNull(outputPath); - DataWriter writer = DataWriter.builder() .data(originalGrids) .destination(outputPath) diff --git a/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/TemporalDataReaderTest.java b/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/TemporalDataReaderTest.java new file mode 100644 index 00000000..23d76051 --- /dev/null +++ b/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/TemporalDataReaderTest.java @@ -0,0 +1,491 @@ +package mil.army.usace.hec.vortex.io; + +import hec.heclib.util.Heclib; +import mil.army.usace.hec.vortex.TestUtil; +import mil.army.usace.hec.vortex.VortexData; +import mil.army.usace.hec.vortex.VortexDataType; +import mil.army.usace.hec.vortex.VortexGrid; +import mil.army.usace.hec.vortex.geo.Grid; +import mil.army.usace.hec.vortex.geo.WktFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static mil.army.usace.hec.vortex.VortexDataType.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class TemporalDataReaderTest { + private static String tempDssFile; + private static TemporalDataReader accumulationReader; + private static TemporalDataReader averageReader; + private static TemporalDataReader instantReader; + + /* Set Up */ + @BeforeAll + static void setUp() { + tempDssFile = TestUtil.createTempFile("TemporalDataReaderTest.dss"); + + writeAccumulationData(); + writeAverageData(); + writeInstantData(); + + accumulationReader = createTemporalDataReader(ACCUMULATION); + averageReader = createTemporalDataReader(AVERAGE); + instantReader = createTemporalDataReader(INSTANTANEOUS); + } + + /* Getter Tests */ + @Test + void getAccumulationMinMaxGridData() { + ZonedDateTime startTime = buildTestTime(2, 0); + ZonedDateTime endTime = buildTestTime(4, 15); + float[] expectedMin = buildTestData(0f); + float[] expectedMax = buildTestData(3f); + VortexGrid[] actual = accumulationReader.getMinMaxGridData(startTime, endTime); + assertFloatArrayEquals(expectedMin, actual[0].data()); + assertFloatArrayEquals(expectedMax, actual[1].data()); + } + + @Test + void getAverageMinMaxGridData() { + ZonedDateTime startTime = buildTestTime(1, 30); + ZonedDateTime endTime = buildTestTime(3, 20); + float[] expectedMin = buildTestData(7.5f); + float[] expectedMax = buildTestData(7.7f); + VortexGrid[] actual = averageReader.getMinMaxGridData(startTime, endTime); + assertFloatArrayEquals(expectedMin, actual[0].data()); + assertFloatArrayEquals(expectedMax, actual[1].data()); + } + + @Test + void getInstantMinMaxGridData() { + ZonedDateTime startTime = buildTestTime(3, 0); + ZonedDateTime endTime = buildTestTime(5, 30); + float[] expectedMin = buildTestData(18f); + float[] expectedMax = buildTestData(22f); + VortexGrid[] actual = instantReader.getMinMaxGridData(startTime, endTime); + assertFloatArrayEquals(expectedMin, actual[0].data()); + assertFloatArrayEquals(expectedMax, actual[1].data()); + } + + @Test + void getGridDefinition() { + String expectedWkt = WktFactory.getShg(); + + Optional actualAccumulationGridDefinition = accumulationReader.getGridDefinition(); + assertEquals(expectedWkt, actualAccumulationGridDefinition.map(Grid::getCrs).orElse("")); + + Optional actualAverageGridDefinition = averageReader.getGridDefinition(); + assertEquals(expectedWkt, actualAverageGridDefinition.map(Grid::getCrs).orElse("")); + + Optional actualInstantGridDefinition = instantReader.getGridDefinition(); + assertEquals(expectedWkt, actualInstantGridDefinition.map(Grid::getCrs).orElse("")); + } + + @Test + void getStartAndEndTime() { + ZonedDateTime expectedAccumulationStartTime = TimeConverter.toZonedDateTime("01JAN2020:0100"); + ZonedDateTime expectedAccumulationEndTime = TimeConverter.toZonedDateTime("01JAN2020:0600"); + assertEquals(expectedAccumulationStartTime, accumulationReader.getStartTime().orElse(null)); + assertEquals(expectedAccumulationEndTime, accumulationReader.getEndTime().orElse(null)); + + ZonedDateTime expectedAverageStartTime = TimeConverter.toZonedDateTime("01JAN2020:0100"); + ZonedDateTime expectedAverageEndTime = TimeConverter.toZonedDateTime("01JAN2020:0600"); + assertEquals(expectedAverageStartTime, averageReader.getStartTime().orElse(null)); + assertEquals(expectedAverageEndTime, averageReader.getEndTime().orElse(null)); + + ZonedDateTime expectedInstantStartTime = TimeConverter.toZonedDateTime("01JAN2020:0100"); + ZonedDateTime expectedInstantEndTime = TimeConverter.toZonedDateTime("01JAN2020:0500"); + assertEquals(expectedInstantStartTime, instantReader.getStartTime().orElse(null)); + assertEquals(expectedInstantEndTime, instantReader.getEndTime().orElse(null)); + } + + @Test + void getDataUnits() { + String expectedAccumulationUnits = "MM"; + assertEquals(expectedAccumulationUnits, accumulationReader.getDataUnits().orElse(null)); + + String expectedAverageUnits = "DEG C"; + assertEquals(expectedAverageUnits, averageReader.getDataUnits().orElse(null)); + + String expectedInstantUnits = "UNSPECIF"; + assertEquals(expectedInstantUnits, instantReader.getDataUnits().orElse(null)); + } + + /* Accumulation Tests */ + @Test + void readAccumulationSingleGridOverlap() { + TemporalDataReader reader = accumulationReader; + ZonedDateTime start = buildTestTime(2, 0); + ZonedDateTime end = buildTestTime(3, 0); + + float[] expectedData = buildTestData(1.5f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readAccumulationFullDataOverlap() { + TemporalDataReader reader = accumulationReader; + ZonedDateTime start = buildTestTime(1, 0); + ZonedDateTime end = buildTestTime(6, 0); + + float[] expectedData = buildTestData(9f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readAccumulationPartialFirstGridOverlap() { + TemporalDataReader reader = accumulationReader; + ZonedDateTime start = buildTestTime(1, 30); + ZonedDateTime end = buildTestTime(3, 0); + + float[] expectedData = buildTestData(2.5f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readAccumulationPartialMultipleGridsOverlap() { + TemporalDataReader reader = accumulationReader; + ZonedDateTime start = buildTestTime(1, 15); + ZonedDateTime end = buildTestTime(3, 15); + + float[] expectedData = buildTestData(3f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readAccumulationBeforeStart() { + TemporalDataReader reader = accumulationReader; + ZonedDateTime start = buildTestTime(0, 15); + ZonedDateTime end = buildTestTime(3, 0); + + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertMissingData(actualGrid); + } + + @Test + void readAccumulationTimeAfterEnd() { + TemporalDataReader reader = accumulationReader; + ZonedDateTime start = buildTestTime(4, 30); + ZonedDateTime end = buildTestTime(6, 15); + + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertMissingData(actualGrid); + } + + /* Average Tests */ + @Test + void readAverageNoOverlap() { + TemporalDataReader reader = averageReader; + ZonedDateTime start = buildTestTime(7, 0); + ZonedDateTime end = buildTestTime(8, 0); + + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertMissingData(actualGrid); + } + + @Test + void readAverageSingleGridOverlap() { + TemporalDataReader reader = averageReader; + ZonedDateTime start = buildTestTime(1, 0); + ZonedDateTime end = buildTestTime(2, 0); + + float[] expectedData = buildTestData(7.5f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readAverageFullDataOverlap() { + TemporalDataReader reader = averageReader; + ZonedDateTime start = buildTestTime(1, 0); + ZonedDateTime end = buildTestTime(6, 0); + + float[] expectedData = buildTestData(7.56f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readAveragePartialFirstGridOverlap() { + TemporalDataReader reader = averageReader; + ZonedDateTime start = buildTestTime(1, 30); + ZonedDateTime end = buildTestTime(3, 0); + + float[] expectedData = buildTestData(7.566667f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readAveragePartialMultipleGridsOverlap() { + TemporalDataReader reader = averageReader; + ZonedDateTime start = buildTestTime(1, 15); + ZonedDateTime end = buildTestTime(3, 15); + + float[] expectedData = buildTestData(7.575f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + /* Point Instant Tests */ + @Test + void readPointInstantSingleGridOverlap() { + TemporalDataReader reader = instantReader; + ZonedDateTime time = buildTestTime(2, 0); + + float[] expectedData = buildTestData(20f); + VortexGrid actualGrid = reader.read(time, time).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readPointInstantInBetweenGrids() { + TemporalDataReader reader = instantReader; + ZonedDateTime time = buildTestTime(1, 15); + + float[] expectedData = buildTestData(16.25f); + VortexGrid actualGrid = reader.read(time, time).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readPointInstantBeforeStart() { + TemporalDataReader reader = instantReader; + ZonedDateTime time = buildTestTime(0, 15); + + VortexGrid actualGrid = reader.read(time, time).orElse(null); + + assertNotNull(actualGrid); + assertMissingData(actualGrid); + } + + @Test + void readPointInstantAfterEnd() { + TemporalDataReader reader = instantReader; + ZonedDateTime time = buildTestTime(5, 30); + + VortexGrid actualGrid = reader.read(time, time).orElse(null); + + assertNotNull(actualGrid); + assertMissingData(actualGrid); + } + + /* Period Instant Tests */ + @Test + void readPeriodInstantPartialMultipleGrids() { + TemporalDataReader reader = instantReader; + ZonedDateTime start = buildTestTime(2, 15); + ZonedDateTime end = buildTestTime(4, 50); + + float[] expectedData = buildTestData(19.341398f); + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertFloatArrayEquals(expectedData, actualGrid.data()); + } + + @Test + void readPeriodInstantMissingSomeDataBeforeStart() { + TemporalDataReader reader = instantReader; + ZonedDateTime start = buildTestTime(0, 30); + ZonedDateTime end = buildTestTime(1, 0); + + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertMissingData(actualGrid); + } + + @Test + void readPeriodInstantMissingSomeDataAfterEnd() { + TemporalDataReader reader = instantReader; + ZonedDateTime start = buildTestTime(5, 0); + ZonedDateTime end = buildTestTime(6, 15); + + VortexGrid actualGrid = reader.read(start, end).orElse(null); + + assertNotNull(actualGrid); + assertMissingData(actualGrid); + } + + /* Helpers */ + private static void writeAccumulationData() { + List accumulationGrids = List.of( + buildTestGridNew(ACCUMULATION, 1, 2, 2), + buildTestGridNew(ACCUMULATION, 2, 3, 1.5f), + buildTestGridNew(ACCUMULATION, 3, 4, 0), + buildTestGridNew(ACCUMULATION, 4, 5, 3), + buildTestGridNew(ACCUMULATION, 5, 6, 2.5f) + ); + + + DataWriter dataWriter = DataWriter.builder() + .data(accumulationGrids) + .destination(tempDssFile) + .build(); + + dataWriter.write(); + } + + private static void writeAverageData() { + List averageGrids = List.of( + buildTestGridNew(AVERAGE, 1, 2, 7.5f), + buildTestGridNew(AVERAGE, 2, 3, 7.6f), + buildTestGridNew(AVERAGE, 3, 4, 7.7f), + buildTestGridNew(AVERAGE, 4, 5, 7.4f), + buildTestGridNew(AVERAGE, 5, 6, 7.6f) + ); + + DataWriter dataWriter = DataWriter.builder() + .data(averageGrids) + .destination(tempDssFile) + .build(); + + dataWriter.write(); + } + + private static void writeInstantData() { + List instantGrids = List.of( + buildTestGridNew(INSTANTANEOUS, 1, 1, 15), + buildTestGridNew(INSTANTANEOUS, 2, 2, 20), + buildTestGridNew(INSTANTANEOUS, 3, 3, 18), + buildTestGridNew(INSTANTANEOUS, 4, 4, 22), + buildTestGridNew(INSTANTANEOUS, 5, 5, 15) + ); + + DataWriter dataWriter = DataWriter.builder() + .data(instantGrids) + .destination(tempDssFile) + .build(); + + dataWriter.write(); + } + + private static TemporalDataReader createTemporalDataReader(VortexDataType type) { + String variableName = getTestGridDataName(type); + String pathToData = String.format("///%s/*/*//", variableName); + DataReader dataReader = DataReader.builder().path(tempDssFile).variable(pathToData).build(); + return TemporalDataReader.create(dataReader); + } + + private static VortexGrid buildTestGridNew(VortexDataType type, int hourStart, int hourEnd, float dataValue) { + float[] data = buildTestData(dataValue); + + ZonedDateTime startTime = buildTestTime(hourStart, 0); + ZonedDateTime endTime = buildTestTime(hourEnd, 0); + + return VortexGrid.builder() + .dx(1).dy(1) + .nx(3).ny(3) + .originX(0).originY(0) + .fileName(tempDssFile) + .shortName(getTestGridDataName(type)) + .fullName(getTestGridDataName(type)) + .units(getTestGridDataUnits(type)) + .description("") + .wkt(WktFactory.getShg()) + .data(data) + .noDataValue(Heclib.UNDEFINED_FLOAT) + .startTime(startTime) + .endTime(endTime) + .interval(Duration.between(startTime, endTime)) + .dataType(type) + .build(); + } + + private static String getTestGridDataName(VortexDataType type) { + return switch (type) { + case ACCUMULATION -> "PRECIPITATION"; + case AVERAGE -> "TEMPERATURE"; + case INSTANTANEOUS -> "ALBEDO"; + default -> ""; + }; + } + + private static String getTestGridDataUnits(VortexDataType type) { + return switch (type) { + case ACCUMULATION -> "MM"; + case AVERAGE -> "DEG C"; + case INSTANTANEOUS -> "UNSPECIF"; + default -> ""; + }; + } + + private static ZonedDateTime buildTestTime(int hour, int minute) { + ZoneId zoneId = ZoneId.of("UTC"); + return ZonedDateTime.of(2020, 1, 1, hour, minute, 0, 0, zoneId); + } + + private static float[] buildTestData(float value) { + float[] data = new float[9]; + Arrays.fill(data, value); + return data; + } + + private void assertMissingData(VortexGrid grid) { + float[] expectedData = new float[grid.data().length]; + Arrays.fill(expectedData, (float) grid.noDataValue()); + assertFloatArrayEquals(expectedData, grid.data()); + } + + private void assertFloatArrayEquals(float[] expected, float[] actual) { + assertEquals(expected.length, actual.length, "Array lengths are different"); + + float epsilon = (float) Math.pow(10, -4); + + for (int i = 0; i < expected.length; i++) { + assertEquals(expected[i], actual[i], epsilon, + "Arrays differ at index " + i + ". Expected: " + expected[i] + ", but was: " + actual[i]); + } + } + + /* Tear Down */ + @AfterAll + static void tearDown() throws IOException { + if (tempDssFile != null) { + Files.deleteIfExists(Path.of(tempDssFile)); + } + } +} \ No newline at end of file diff --git a/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/TimeConverterTest.java b/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/TimeConverterTest.java new file mode 100644 index 00000000..de4a68f1 --- /dev/null +++ b/vortex-api/src/test/java/mil/army/usace/hec/vortex/io/TimeConverterTest.java @@ -0,0 +1,123 @@ +package mil.army.usace.hec.vortex.io; + +import org.junit.jupiter.api.Test; + +import java.time.*; +import java.time.format.DateTimeFormatter; + +import static org.junit.jupiter.api.Assertions.*; + +class TimeConverterTest { + @Test + void testToZonedDateTimeWithZ() { + String input = "2021-07-20 15:50:55 Z"; + ZonedDateTime expected = ZonedDateTime.of(2021, 7, 20, 15, 50, 55, 0, ZoneId.of("Z")); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + @Test + void testToZonedDateTimeWithoutZ() { + String input = "2021-07-20 15:50:55"; + ZonedDateTime expected = ZonedDateTime.of(2021, 7, 20, 15, 50, 55, 0, ZoneId.of("UTC")); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + @Test + void testToZonedDateTimeIsoDateTime() { + String input = "2021-07-20T15:50:55Z"; + ZonedDateTime expected = ZonedDateTime.parse(input); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + @Test + void testToZonedDateTimeIsoLocalDateTime() { + String input = "2021-07-20T15:50:55"; + ZonedDateTime expected = LocalDateTime.parse(input).atZone(ZoneId.of("UTC")); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + @Test + void testToZonedDateTimeSimpleDate() { + String input = "2021-7-20"; + ZonedDateTime expected = LocalDate.parse(input, DateTimeFormatter.ofPattern("uuuu-M-d")).atStartOfDay(ZoneId.of("UTC")); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + @Test + void testToZonedDateTimeFullDateTimeWithZoneOffset() { + String input = "2021-07-20 15:50:55 -07:00"; + ZonedDateTime expected = ZonedDateTime.of(LocalDateTime.of(2021, 7, 20, 15, 50, 55), ZoneOffset.ofHours(-7)); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected.withZoneSameInstant(ZoneId.of("UTC")), result); + } + + @Test + void testToZonedDateTimeDateOnly() { + String input = "2021-07-20"; + ZonedDateTime expected = LocalDate.parse(input).atStartOfDay(ZoneId.of("UTC")); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + @Test + void testToZonedDateTimeMonthYearOnly() { + String input = "2021-07"; + ZonedDateTime expected = YearMonth.parse(input, DateTimeFormatter.ofPattern("uuuu-MM")).atDay(1).atStartOfDay(ZoneId.of("UTC")); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + @Test + void testToZonedDateTimeWithDifferentZoneId() { + String input = "2021-07-20 15:50:55 America/New_York"; + ZonedDateTime expected = ZonedDateTime.of(LocalDateTime.of(2021, 7, 20, 15, 50, 55), ZoneId.of("America/New_York")); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected.withZoneSameInstant(ZoneId.of("UTC")), result); + } + + @Test + void testToZonedDateTimeWithHyphenatedAmericanFormat() { + String input = "07-20-2021 15:50"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM-dd-yyyy HH:mm").withZone(ZoneId.of("UTC")); + ZonedDateTime expected = ZonedDateTime.parse(input, formatter); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + @Test + void testToZonedDateTimeWithSlashAmericanFormat() { + String input = "07/20/2021 15:50"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm").withZone(ZoneId.of("UTC")); + ZonedDateTime expected = ZonedDateTime.parse(input, formatter); + ZonedDateTime result = TimeConverter.toZonedDateTime(input); + assertNotNull(result); + assertTimeEquals(expected, result); + } + + // Test for an invalid format + @Test + void testToZonedDateTimeInvalidFormat() { + String input = "invalid-date-time"; + assertNull(TimeConverter.toZonedDateTime(input)); + } + + private void assertTimeEquals(ZonedDateTime expected, ZonedDateTime result) { + boolean isEqual = expected.isEqual(result); + assertTrue(isEqual); + } +} \ No newline at end of file