Use 'nccopy' to pull/convert/compress OPeNDAP -> netCDF4/HDF5; remove…

… DodsToNetcdf4Converter

src/main/java/com/rpsgroup/FileManager.java

@@ -64,24 +64,14 @@ public int getFileCount() {
 		return dataDir.listFiles(filter).length;
 	}
 
-
 	public void setProps(PropertiesHolder props) {
 		this.props = props;
 	}
 
-	public File getDodsOutfile() {
-
-		return new File(
-				new File(props.downloadDirectory),
-				Long.toString(Instant.now().getEpochSecond()) +  "_" + props.dodsFilename);
-	}
-
 	public File getNC4Outfile() {
-		String ncname = props.dodsFilename.replace(".dods", "");
-
 		return new File(
 				new File(props.storageDirectory),
-				Long.toString(Instant.now().getEpochSecond()) +  "_" + ncname);
+				Long.toString(Instant.now().getEpochSecond()) +  "_" + props.ncFilename);
 	}
 
 }

src/main/java/com/rpsgroup/HarvesterMain.java

@@ -1,6 +1,5 @@
 package com.rpsgroup;
 
-import java.io.File;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.net.MalformedURLException;
@@ -18,25 +17,31 @@ public class HarvesterMain {
 	private static final SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
 	private static final PropertiesHolder props = new PropertiesHolder();
 
-	private DodsToNetcdf4Converter converter = new DodsToNetcdf4Converter();
 	private HttpHandler http = new HttpHandler();
 
 	private FileManager file = new FileManager();
 	private Logger log = LoggerFactory.getLogger(HarvesterMain.class);
 
 	public static void main(String[] args) throws MalformedURLException, Exception {
+		run(props.ncUrl);
+	}
+
+	private String getNcCopyCommand(String url) {
+		return props.ncCopyCommand + " " + url + " " + file.getNC4Outfile().toString();
+	}
+
+	private static void run(String url) throws MalformedURLException, Exception {
 
 		HarvesterMain runner = new HarvesterMain();	
-		File dodsFile = null;
 
 		try {
-			runner.converter.setProps(props);
-			runner.http.setProps(props);
 			runner.file.setProps(props);
+			runner.http.setProps(props);
 
 			runner.http.setFileManager(runner.file);
 			runner.http.setDateFormat(sdf);
 
+			// Why dods url? It accepts HEAD requests and returns last-mod time, others don't.
 			Date remoteModTime = runner.http.getLastModified(new URL(props.dodsURL));
 			Date mostRecentLocal = runner.file.getMostRecentDataTime();
 
@@ -48,13 +53,17 @@ public static void main(String[] args) throws MalformedURLException, Exception {
 			if (remoteModTime.after(mostRecentLocal) || firstRun) {
 				runner.log.info("Downloading latest data...");
 
-				dodsFile = runner.http.downloadDods();
-				runner.log.info("Downloaded .dods file: " + dodsFile.toString());
+				String command = runner.getNcCopyCommand(url);
+				Process copyProc = Runtime.getRuntime().exec(command);
+
+				int exitCode = copyProc.waitFor();
 
-				File netcdf = runner.file.getNC4Outfile();
-				runner.converter.convert(dodsFile, netcdf);
+				if (exitCode != 0) {
+					runner.log.error("Fatal Error execing '" + command + "'. Exit code: " + Integer.toString(exitCode));
+				} else {
+					runner.log.info("Successfully exec'd '" + command + "'.");
+				}
 
-				runner.log.info("Converted .dods to netCDF4: " + netcdf.toString());				
 				while (runner.file.getFileCount() > props.storageMaxCount) {	
 
 					Path oldest = runner.file.getOldestFile().toPath();
@@ -77,10 +86,6 @@ public static void main(String[] args) throws MalformedURLException, Exception {
 
 		} finally {
 
-			if (dodsFile != null) {
-				Files.delete(dodsFile.toPath());
-				runner.log.info("Deleted .dods file");
-			}
 			runner.log.info("Done.");
 		}
 

src/main/java/com/rpsgroup/HttpHandler.java

@@ -1,13 +1,6 @@
 package com.rpsgroup;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
 import java.net.URL;
-import java.nio.channels.Channels;
-import java.nio.channels.FileChannel;
-import java.nio.channels.ReadableByteChannel;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
@@ -20,8 +13,6 @@
 public class HttpHandler {
 
     private static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport();
-	private PropertiesHolder props;
-	private FileManager fmgr; 
 	private SimpleDateFormat sdf;
 
     public Date getLastModified(URL url) throws Exception {
@@ -33,33 +24,12 @@ public Date getLastModified(URL url) throws Exception {
 
         return sdf.parse(lastMod);
 
-    }
-
-    public File downloadDods() throws IOException {
-
-    	URL getUrl = new URL(props.dodsURL);
-    	InputStream webIs = getUrl.openStream();
-
-    	ReadableByteChannel readableByteChannel = Channels.newChannel(webIs);
-
-    	File outFile = fmgr.getDodsOutfile();
-
-    	try (FileOutputStream fileOutputStream = new FileOutputStream(outFile);
-    		FileChannel fileChannel = fileOutputStream.getChannel()) {
-
-    		fileChannel.transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
-    	}
-
-    	return outFile;
     }
 
 	public void setProps(PropertiesHolder props) {
-		this.props = props;
 	}
 
-	public void setFileManager(FileManager fmgr) {
-		this.fmgr = fmgr;
-	}
+	public void setFileManager(FileManager fmgr) {	}
 
 	public void setDateFormat(SimpleDateFormat sdf) {
 		this.sdf = sdf;

src/main/java/com/rpsgroup/PropertiesHolder.java

@@ -10,11 +10,11 @@ public class PropertiesHolder {
 	private InputStream propIs;
 
 	public String dodsURL;
-	public String dodsFilename;
 	public String ncFilename;
-	public String downloadDirectory;
+	public String ncUrl;
+	public String ncCopyCommand;
 	public String storageDirectory;
-	public String jnaLibDirectory;
+//	public String jnaLibDirectory;
 	public int storageMaxCount;
 
 	public PropertiesHolder() {
@@ -34,12 +34,14 @@ public PropertiesHolder() {
 		}
 
 		dodsURL = getProp("dods.url");
-		dodsFilename = getProp("dods.filename");
+
 		ncFilename = getProp("nc.filename");
+		ncUrl = getProp("nc.url");
 
-		downloadDirectory = getProp("directory.download");
+		ncCopyCommand = getProp("nc.copy.cmd");
 		storageDirectory = getProp("directory.storage");
-		jnaLibDirectory = getProp("directory.jna.lib");
+
+//		jnaLibDirectory = getProp("directory.jna.lib");
 		storageMaxCount = Integer.parseInt(getProp("directory.storage.maxcount"));
 
 	}

src/main/resources/application.properties

@@ -5,8 +5,8 @@ dods.url=http://nopp.dms.uconn.edu:8080/thredds/dodsC/FVCOM/Forecast/fvcom_lis_f
 ## Used to filter filenames in directory listings
 nc.filename=fvcom_lis_forecast.nc
 
-## Directory where temporary .dods files are written
-directory.download=/tmp
+nc.url=http://nopp.dms.uconn.edu:8080/thredds/dodsC/FVCOM/Forecast/fvcom_lis_forecast.nc
+nc.copy.cmd=nccopy -k4 -s -d5
 
 ## Final resting place for harvested data
 #directory.storage=/tmp

src/main/resources/logback.xml

@@ -14,7 +14,6 @@
 			</pattern>
 		</encoder>
 	</appender>
-
 	<root level="trace">
 		<appender-ref ref="FILE" />
 		<appender-ref ref="STDOUT" />