Use local docker for file copier (#3)

Make file copier binary-safe

README.md

@@ -40,23 +40,23 @@ Executes jobs on remote Docker containers managed by the Rancher host.
 
 Features:
 
- - Can select first container in a service so only one needs to run.
+ - Can select first container in a service so only one needs to run (filter by "seen:1").
  - Reconstructs the STDERR channel that is missing in output from Rancher API.
 
 ### Rancher File Copier
 
-Should be considered beta. Probably limited to text files.
+Copies files to a node. Can be configured to use Rancher CLI if it is installed and
+available. Otherwise assembles file from Base64-encoded parts transmitted via Rancher
+API.
+
+To distribute to all containers in a stack's service, omit the filter for "seen".
 
 ## Road Map
 
- - 0.6.x Make File Copier binary-safe.
+ - 0.6.6 Make File Copier binary-safe.
  - 0.7 Provide container upgrade node step plugin.
  - 0.9.x Provide reasonable if not complete test coverage prior to 1.x
 
-## Known Bugs
-
- - File Copier handles only test files.
-
 ## Compatibility
 
 This has been tested with Rundeck 3.1.3 and Rancher 1.6.26.

src/main/java/com/bioraft/rundeck/rancher/RancherFileCopier.java

@@ -24,23 +24,33 @@
 
 package com.bioraft.rundeck.rancher;
 
-import java.io.ByteArrayInputStream;
+import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.function.Consumer;
 
+import com.dtolabs.rundeck.core.Constants;
 import com.dtolabs.rundeck.core.common.INodeEntry;
 import com.dtolabs.rundeck.core.execution.ExecutionContext;
+import com.dtolabs.rundeck.core.execution.impl.common.BaseFileCopier;
+import com.dtolabs.rundeck.core.execution.script.ScriptfileUtils;
 import com.dtolabs.rundeck.core.execution.service.FileCopier;
 import com.dtolabs.rundeck.core.execution.service.FileCopierException;
+import com.dtolabs.rundeck.core.execution.utils.ResolverUtil;
+import com.dtolabs.rundeck.core.execution.workflow.steps.FailureReason;
 import com.dtolabs.rundeck.core.plugins.Plugin;
+import com.dtolabs.rundeck.core.plugins.configuration.Describable;
+import com.dtolabs.rundeck.core.plugins.configuration.Description;
+import com.dtolabs.rundeck.core.plugins.configuration.PropertyUtil;
 import com.dtolabs.rundeck.core.storage.ResourceMeta;
 import com.dtolabs.rundeck.plugins.ServiceNameConstants;
 import com.dtolabs.rundeck.plugins.descriptions.PluginDescription;
+import com.dtolabs.rundeck.plugins.util.DescriptionBuilder;
 
 /**
  * RancherStubFileCopier provider for the FileCopier service
@@ -50,64 +60,201 @@
  */
 @Plugin(name = RancherShared.SERVICE_PROVIDER_NAME, service = ServiceNameConstants.FileCopier)
 @PluginDescription(title = "Rancher File Copier", description = "Copies a file to a Rancher-mananged Docker container.")
-public class RancherFileCopier implements FileCopier {
+public class RancherFileCopier implements FileCopier, Describable {
+
+	static final Description DESC;
+
+	static {
+		DescriptionBuilder builder = DescriptionBuilder.builder();
+		builder.name(RancherShared.SERVICE_PROVIDER_NAME);
+		builder.title("Rancher File Copier");
+		builder.description("Copies a file to a Rancher-mananged Docker container");
+
+		builder.property(PropertyUtil.string(RancherShared.CONFIG_RANCHER_CLI_PATH, "Search path ",
+				"A search path on the Rundeck host that finds rancher, docker, sh, and base64 (e.g., /usr/local/bin:/usr/bin:/bin)",
+				false, ""));
+
+		builder.mapping(RancherShared.CONFIG_RANCHER_CLI_PATH, "project." + RancherShared.CONFIG_RANCHER_CLI_PATH);
+		builder.frameworkMapping(RancherShared.CONFIG_RANCHER_CLI_PATH,
+				"framework." + RancherShared.CONFIG_RANCHER_CLI_PATH);
+
+		DESC = builder.build();
+	}
+
+	@Override
+	public Description getDescription() {
+		return DESC;
+	}
+
+	@Override
+	public String copyFileStream(ExecutionContext context, InputStream input, INodeEntry node, String destination)
+			throws FileCopierException {
+		return copyFile(context, null, input, null, node, destination);
+	}
 
 	@Override
-	public String copyFileStream(final ExecutionContext context, final InputStream input, final INodeEntry node,
-			final String destination) throws FileCopierException {
+	public String copyFile(final ExecutionContext context, File file, INodeEntry node, final String destination)
+			throws FileCopierException {
+		return copyFile(context, file, null, null, node, destination);
+	}
+
+	@Override
+	public String copyScriptContent(ExecutionContext context, String script, INodeEntry node, final String destination)
+			throws FileCopierException {
+		return copyFile(context, null, null, script, node, destination);
+	}
+
+	private String copyFile(final ExecutionContext context, final File scriptfile, final InputStream input,
+			final String script, final INodeEntry node, final String destinationPath) throws FileCopierException {
+
+		String remotefile;
+
 		Map<String, String> nodeAttributes = node.getAttributes();
 		String accessKey;
 		String secretKey;
 		try {
 			accessKey = this.loadStoragePathData(context, nodeAttributes.get(RancherShared.CONFIG_ACCESSKEY_PATH));
 			secretKey = this.loadStoragePathData(context, nodeAttributes.get(RancherShared.CONFIG_SECRETKEY_PATH));
 		} catch (IOException e) {
-			throw new FileCopierException(e.getMessage(), RancherFileCopierFailureReason.AuthenticationFailure);
+			throw new FileCopierException(e.getMessage(), FileCopyFailureReason.AuthenticationFailure);
 		}
-		String url = nodeAttributes.get("execute");
 
-		try {
-			RancherWebSocketListener.putFile(url, accessKey, secretKey, input, destination);
-		} catch (IOException | InterruptedException e) {
-			throw new FileCopierException(e.getMessage(), RancherFileCopierFailureReason.ConnectionFailure);
+		if (null == destinationPath) {
+			String identity = null != context.getDataContext() && null != context.getDataContext().get("job")
+					? context.getDataContext().get("job").get("execid")
+					: null;
+			remotefile = BaseFileCopier.generateRemoteFilepathForNode(node,
+					context.getFramework().getFrameworkProjectMgr().getFrameworkProject(context.getFrameworkProject()),
+					context.getFramework(), (null != scriptfile ? scriptfile.getName() : "dispatch-script"), null,
+					identity);
+		} else {
+			remotefile = destinationPath;
 		}
-		return destination;
-	}
+		// write to a local temp file or use the input file
+		final File localTempfile = null != scriptfile ? scriptfile
+				: BaseFileCopier.writeTempFile(context, scriptfile, input, script);
 
-	@Override
-	public String copyFile(final ExecutionContext context, final File file, final INodeEntry node,
-			final String destination) throws FileCopierException {
-		FileInputStream fileStream = null;
+		/**
+		 * 
+		 * Copy the file over
+		 */
+		System.out.println("copying file: '" + localTempfile.getAbsolutePath() + "' to: '" + node.getNodename() + ":"
+				+ remotefile + "'");
+
+		String searchPath = ResolverUtil.resolveProperty(RancherShared.CONFIG_RANCHER_CLI_PATH, "", node,
+				context.getFramework().getFrameworkProjectMgr().getFrameworkProject(context.getFrameworkProject()),
+				context.getFramework());
 		try {
-			fileStream = new FileInputStream(file);
-			return copyFileStream(context, fileStream, node, destination);
-		} catch (FileNotFoundException fnf) {
-			throw new FileCopierException(fnf.getMessage(), RancherFileCopierFailureReason.ConnectionFailure);
+			if (searchPath.equals("")) {
+				return copyViaApi(nodeAttributes, accessKey, secretKey, localTempfile, remotefile);
+			} else {
+				return copyViaCli(nodeAttributes, accessKey, secretKey, localTempfile, remotefile, searchPath);
+			}
 		} finally {
-			if (fileStream != null) {
-				try {
-					fileStream.close();
-				} catch (IOException e) {
-					// TODO Auto-generated catch block
-					e.printStackTrace();
+			if (null == scriptfile) {
+				if (!ScriptfileUtils.releaseTempFile(localTempfile)) {
+					context.getExecutionListener().log(Constants.WARN_LEVEL,
+							"Unable to remove local temp file: " + localTempfile.getAbsolutePath());
 				}
 			}
 		}
 	}
 
-	@Override
-	public String copyScriptContent(final ExecutionContext context, final String script, final INodeEntry node,
-			final String destination) throws FileCopierException {
-		ByteArrayInputStream scriptStream = new ByteArrayInputStream(script.getBytes());
+	private String copyViaCli(Map<String, String> nodeAttributes, String accessKey, String secretKey,
+			File localTempFile, String remotefile, String searchPath) throws FileCopierException {
+		String path = localTempFile.getAbsolutePath();
+		String instance = nodeAttributes.get("externalId");
+		String[] command = { "rancher", "docker", "cp", path, instance + ":" + remotefile };
+
+		boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows");
 		try {
-			return copyFileStream(context, scriptStream, node, destination);
-		} finally {
-			try {
-				scriptStream.close();
-			} catch (IOException e) {
-				// TODO Auto-generated catch block
-				e.printStackTrace();
+			ProcessBuilder builder = new ProcessBuilder();
+			Map<String, String> environment = builder.environment();
+			environment.put("PATH", searchPath);
+			environment.put("RANCHER_ENVIRONMENT", nodeAttributes.get("environment"));
+			environment.put("RANCHER_DOCKER_HOST", nodeAttributes.get("hostId"));
+			environment.put("RANCHER_URL", nodeAttributes.get("execute").replaceFirst("/projects/.*$", ""));
+			environment.put("RANCHER_ACCESS_KEY", accessKey);
+			environment.put("RANCHER_SECRET_KEY", secretKey);
+			if (isWindows) {
+				throw new FileCopierException("Windows is not currently supported.",
+						FileCopyFailureReason.UnsupportedOperatingSystem);
+			} else {
+				builder.command(command);
 			}
+			builder.directory(new File(System.getProperty("user.home")));
+			Process process = builder.start();
+			StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println);
+			Executors.newSingleThreadExecutor().submit(streamGobbler);
+			int exitCode = process.waitFor();
+			assert exitCode == 0;
+		} catch (IOException e) {
+			throw new FileCopierException("Child process IO Exception", FileCopyFailureReason.IOException, e);
+		} catch (InterruptedException e) {
+			throw new FileCopierException("Child process interrupted", FileCopyFailureReason.InterruptedException, e);
+		}
+
+		return remotefile;
+	}
+
+	private String copyViaApi(Map<String, String> nodeAttributes, String accessKey, String secretKey, File file,
+			String destination) throws FileCopierException {
+		try {
+			String url = nodeAttributes.get("execute");
+			RancherWebSocketListener.putFile(url, accessKey, secretKey, file, destination);
+		} catch (IOException | InterruptedException e) {
+			throw new FileCopierException(e.getMessage(), FileCopyFailureReason.ConnectionFailure);
+		}
+		return destination;
+	}
+
+	public enum FileCopyFailureReason implements FailureReason {
+		/**
+		 * Requested file could not be found
+		 */
+		FileNotFound,
+		/**
+		 * Process could not be started
+		 */
+		IOException,
+		/**
+		 * Process was interrupted
+		 */
+		InterruptedException,
+		/**
+		 * Timeout on connection
+		 */
+		ConnectionTimeout,
+		/**
+		 * Connection unsuccessful
+		 */
+		ConnectionFailure,
+		/**
+		 * Authentication unsuccessful
+		 */
+		AuthenticationFailure,
+		/**
+		 * Command or script execution result code was not zero
+		 */
+		NonZeroResultCode,
+		/**
+		 * Operating system not currently supported
+		 */
+		UnsupportedOperatingSystem
+	}
+
+	private static class StreamGobbler implements Runnable {
+		private InputStream inputStream;
+		private Consumer<String> consumer;
+
+		public StreamGobbler(InputStream inputStream, Consumer<String> consumer) {
+			this.inputStream = inputStream;
+			this.consumer = consumer;
+		}
+
+		@Override
+		public void run() {
+			new BufferedReader(new InputStreamReader(inputStream)).lines().forEach(consumer);
 		}
 	}
 

src/main/java/com/bioraft/rundeck/rancher/RancherNodeExecutorPlugin.java

@@ -24,7 +24,6 @@
 
 import javax.xml.bind.DatatypeConverter;
 
-import com.dtolabs.rundeck.core.common.Framework;
 import com.dtolabs.rundeck.core.common.INodeEntry;
 import com.dtolabs.rundeck.core.execution.ExecutionContext;
 import com.dtolabs.rundeck.core.execution.ExecutionListener;
@@ -56,8 +55,6 @@ public class RancherNodeExecutorPlugin implements NodeExecutor, Describable {
 	private String accessKey;
 	private String secretKey;
 
-	private Framework framework;
-
 	static {
 		DescriptionBuilder builder = DescriptionBuilder.builder();
 		builder.name(RancherShared.SERVICE_PROVIDER_NAME);
@@ -74,10 +71,6 @@ public class RancherNodeExecutorPlugin implements NodeExecutor, Describable {
 		DESC = builder.build();
 	}
 
-	public RancherNodeExecutorPlugin(Framework framework) {
-		this.framework = framework;
-	}
-
 	@Override
 	public Description getDescription() {
 		return DESC;