From 58eb5802c93cd4153e9446b1c4d7363d0b58b26c Mon Sep 17 00:00:00 2001 From: Ben Hall Date: Thu, 7 Oct 2021 14:16:42 -0400 Subject: [PATCH] Use 'nccopy' to pull/convert/compress OPeNDAP -> netCDF4/HDF5; remove DodsToNetcdf4Converter --- .../com/rpsgroup/DodsToNetcdf4Converter.java | 37 ------------------- src/main/java/com/rpsgroup/FileManager.java | 12 +----- src/main/java/com/rpsgroup/HarvesterMain.java | 33 ++++++++++------- src/main/java/com/rpsgroup/HttpHandler.java | 32 +--------------- .../java/com/rpsgroup/PropertiesHolder.java | 14 ++++--- src/main/resources/application.properties | 4 +- src/main/resources/logback.xml | 1 - 7 files changed, 31 insertions(+), 102 deletions(-) delete mode 100644 src/main/java/com/rpsgroup/DodsToNetcdf4Converter.java diff --git a/src/main/java/com/rpsgroup/DodsToNetcdf4Converter.java b/src/main/java/com/rpsgroup/DodsToNetcdf4Converter.java deleted file mode 100644 index 7e7ee2d..0000000 --- a/src/main/java/com/rpsgroup/DodsToNetcdf4Converter.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.rpsgroup; - -import java.io.File; -import java.io.IOException; - -import ucar.nc2.NetcdfFile; -import ucar.nc2.dataset.NetcdfDatasets; -import ucar.nc2.write.NetcdfCopier; -import ucar.nc2.write.NetcdfFileFormat; -import ucar.nc2.write.NetcdfFormatWriter; - - -public class DodsToNetcdf4Converter { - - private PropertiesHolder props = new PropertiesHolder(); - - public void convert(File dodsInFile, File nc4OutFile) throws IOException { - - NetcdfFile ncfile = NetcdfDatasets.openFile("file:" + dodsInFile.toString(), null); - NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf4( - NetcdfFileFormat.NETCDF4, - nc4OutFile.toString(), - null); - - System.setProperty("jna.library.path", props.jnaLibDirectory); - NetcdfCopier copier = NetcdfCopier.create(ncfile, builder); - - NetcdfFile out = copier.write(null); - - ncfile.close(); - out.close(); - } - - public void setProps(PropertiesHolder props) { - this.props = props; - } -} diff --git a/src/main/java/com/rpsgroup/FileManager.java b/src/main/java/com/rpsgroup/FileManager.java index b9942ea..682b083 100644 --- a/src/main/java/com/rpsgroup/FileManager.java +++ b/src/main/java/com/rpsgroup/FileManager.java @@ -64,24 +64,14 @@ public int getFileCount() { return dataDir.listFiles(filter).length; } - public void setProps(PropertiesHolder props) { this.props = props; } - public File getDodsOutfile() { - - return new File( - new File(props.downloadDirectory), - Long.toString(Instant.now().getEpochSecond()) + "_" + props.dodsFilename); - } - public File getNC4Outfile() { - String ncname = props.dodsFilename.replace(".dods", ""); - return new File( new File(props.storageDirectory), - Long.toString(Instant.now().getEpochSecond()) + "_" + ncname); + Long.toString(Instant.now().getEpochSecond()) + "_" + props.ncFilename); } } diff --git a/src/main/java/com/rpsgroup/HarvesterMain.java b/src/main/java/com/rpsgroup/HarvesterMain.java index 4e7a983..8d9c34e 100644 --- a/src/main/java/com/rpsgroup/HarvesterMain.java +++ b/src/main/java/com/rpsgroup/HarvesterMain.java @@ -1,6 +1,5 @@ package com.rpsgroup; -import java.io.File; import java.io.PrintWriter; import java.io.StringWriter; import java.net.MalformedURLException; @@ -18,25 +17,31 @@ public class HarvesterMain { private static final SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); private static final PropertiesHolder props = new PropertiesHolder(); - private DodsToNetcdf4Converter converter = new DodsToNetcdf4Converter(); private HttpHandler http = new HttpHandler(); private FileManager file = new FileManager(); private Logger log = LoggerFactory.getLogger(HarvesterMain.class); public static void main(String[] args) throws MalformedURLException, Exception { + run(props.ncUrl); + } + + private String getNcCopyCommand(String url) { + return props.ncCopyCommand + " " + url + " " + file.getNC4Outfile().toString(); + } + + private static void run(String url) throws MalformedURLException, Exception { HarvesterMain runner = new HarvesterMain(); - File dodsFile = null; try { - runner.converter.setProps(props); - runner.http.setProps(props); runner.file.setProps(props); + runner.http.setProps(props); runner.http.setFileManager(runner.file); runner.http.setDateFormat(sdf); + // Why dods url? It accepts HEAD requests and returns last-mod time, others don't. Date remoteModTime = runner.http.getLastModified(new URL(props.dodsURL)); Date mostRecentLocal = runner.file.getMostRecentDataTime(); @@ -48,13 +53,17 @@ public static void main(String[] args) throws MalformedURLException, Exception { if (remoteModTime.after(mostRecentLocal) || firstRun) { runner.log.info("Downloading latest data..."); - dodsFile = runner.http.downloadDods(); - runner.log.info("Downloaded .dods file: " + dodsFile.toString()); + String command = runner.getNcCopyCommand(url); + Process copyProc = Runtime.getRuntime().exec(command); + + int exitCode = copyProc.waitFor(); - File netcdf = runner.file.getNC4Outfile(); - runner.converter.convert(dodsFile, netcdf); + if (exitCode != 0) { + runner.log.error("Fatal Error execing '" + command + "'. Exit code: " + Integer.toString(exitCode)); + } else { + runner.log.info("Successfully exec'd '" + command + "'."); + } - runner.log.info("Converted .dods to netCDF4: " + netcdf.toString()); while (runner.file.getFileCount() > props.storageMaxCount) { Path oldest = runner.file.getOldestFile().toPath(); @@ -77,10 +86,6 @@ public static void main(String[] args) throws MalformedURLException, Exception { } finally { - if (dodsFile != null) { - Files.delete(dodsFile.toPath()); - runner.log.info("Deleted .dods file"); - } runner.log.info("Done."); } diff --git a/src/main/java/com/rpsgroup/HttpHandler.java b/src/main/java/com/rpsgroup/HttpHandler.java index 33f446f..849a1d5 100644 --- a/src/main/java/com/rpsgroup/HttpHandler.java +++ b/src/main/java/com/rpsgroup/HttpHandler.java @@ -1,13 +1,6 @@ package com.rpsgroup; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; import java.net.URL; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.ReadableByteChannel; import java.text.SimpleDateFormat; import java.util.Date; @@ -20,8 +13,6 @@ public class HttpHandler { private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); - private PropertiesHolder props; - private FileManager fmgr; private SimpleDateFormat sdf; public Date getLastModified(URL url) throws Exception { @@ -33,33 +24,12 @@ public Date getLastModified(URL url) throws Exception { return sdf.parse(lastMod); - } - - public File downloadDods() throws IOException { - - URL getUrl = new URL(props.dodsURL); - InputStream webIs = getUrl.openStream(); - - ReadableByteChannel readableByteChannel = Channels.newChannel(webIs); - - File outFile = fmgr.getDodsOutfile(); - - try (FileOutputStream fileOutputStream = new FileOutputStream(outFile); - FileChannel fileChannel = fileOutputStream.getChannel()) { - - fileChannel.transferFrom(readableByteChannel, 0, Long.MAX_VALUE); - } - - return outFile; } public void setProps(PropertiesHolder props) { - this.props = props; } - public void setFileManager(FileManager fmgr) { - this.fmgr = fmgr; - } + public void setFileManager(FileManager fmgr) { } public void setDateFormat(SimpleDateFormat sdf) { this.sdf = sdf; diff --git a/src/main/java/com/rpsgroup/PropertiesHolder.java b/src/main/java/com/rpsgroup/PropertiesHolder.java index 83be184..4054e5b 100644 --- a/src/main/java/com/rpsgroup/PropertiesHolder.java +++ b/src/main/java/com/rpsgroup/PropertiesHolder.java @@ -10,11 +10,11 @@ public class PropertiesHolder { private InputStream propIs; public String dodsURL; - public String dodsFilename; public String ncFilename; - public String downloadDirectory; + public String ncUrl; + public String ncCopyCommand; public String storageDirectory; - public String jnaLibDirectory; +// public String jnaLibDirectory; public int storageMaxCount; public PropertiesHolder() { @@ -34,12 +34,14 @@ public PropertiesHolder() { } dodsURL = getProp("dods.url"); - dodsFilename = getProp("dods.filename"); + ncFilename = getProp("nc.filename"); + ncUrl = getProp("nc.url"); - downloadDirectory = getProp("directory.download"); + ncCopyCommand = getProp("nc.copy.cmd"); storageDirectory = getProp("directory.storage"); - jnaLibDirectory = getProp("directory.jna.lib"); + +// jnaLibDirectory = getProp("directory.jna.lib"); storageMaxCount = Integer.parseInt(getProp("directory.storage.maxcount")); } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index a177d3d..ed13e1d 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -5,8 +5,8 @@ dods.url=http://nopp.dms.uconn.edu:8080/thredds/dodsC/FVCOM/Forecast/fvcom_lis_f ## Used to filter filenames in directory listings nc.filename=fvcom_lis_forecast.nc -## Directory where temporary .dods files are written -directory.download=/tmp +nc.url=http://nopp.dms.uconn.edu:8080/thredds/dodsC/FVCOM/Forecast/fvcom_lis_forecast.nc +nc.copy.cmd=nccopy -k4 -s -d5 ## Final resting place for harvested data #directory.storage=/tmp diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 483b1ce..159c4b5 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -14,7 +14,6 @@ -