Skip to content

Commit

Permalink
Improve logging and handling of service-based nodes
Browse files Browse the repository at this point in the history 
* Replace static calls to RancherWebSocketListener in RancherNodeExecutorPlugin
* Run node executor on services with multiple containers
* Increase support for handling services in file copier
* Better logging of errors in http client class

Co-authored-by: Karl DeBisschop <[email protected]>
  • Loading branch information
@kdebisschop
kdebisschop and Karl DeBisschop authored Apr 20, 2020
1 parent 354d4ef commit 19e0de0
Show file tree
Hide file tree
Showing 11 changed files with 395 additions and 128 deletions.
5 changes: 3 additions & 2 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,11 @@ configurations {
dependencies {
implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1'

implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.13.1'
implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.12.0'

implementation (
'org.rundeck:rundeck-core:3.2.+',
'org.rundeck:rundeck-core:3.2.4-20200318',
'org.rundeck:rundeck-storage-api:3.2.4-20200318',
)

testImplementation group: 'junit', name: 'junit', version:'4.12'
Expand Down
38 changes: 36 additions & 2 deletions src/main/java/com/bioraft/rundeck/rancher/HttpClient.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.bioraft.rundeck.rancher;

import com.dtolabs.rundeck.core.execution.ExecutionLogger;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.*;
Expand All @@ -8,10 +9,14 @@
import java.util.Map;
import java.util.Objects;

import static com.dtolabs.rundeck.core.Constants.DEBUG_LEVEL;

public class HttpClient {

private String accessKey;
private String secretKey;

private ExecutionLogger logger;
protected final OkHttpClient client;

public HttpClient() {
Expand All @@ -30,6 +35,10 @@ public void setSecretKey(String secretKey) {
this.secretKey = secretKey;
}

public void setLogger(ExecutionLogger logger) {
this.logger = logger;
}

protected JsonNode get(String url) throws IOException {
return this.get(url, null);
}
Expand All @@ -44,7 +53,8 @@ protected JsonNode get(String url, Map<String, String> query) throws IOException
Response response = client.newCall(builder.build()).execute();
// Since URL comes from the Rancher server itself, assume there are no redirects.
if (response.code() >= 300) {
throw new IOException("API get failed" + response.message());
logError(response);
throw new IOException("API get failed: " + response.message());
}
ObjectMapper mapper = new ObjectMapper();
if (response.body() == null) {
Expand All @@ -66,12 +76,36 @@ protected JsonNode post(String url, String data) throws IOException {
Response response = client.newCall(builder.build()).execute();
// Since URL comes from the Rancher server itself, assume there are no redirects.
if (response.code() >= 300) {
throw new IOException("API post failed" + response.message());
logError(response);
throw new IOException("API post failed: " + response.message());
}
ObjectMapper mapper = new ObjectMapper();
if (response.body() == null) {
return mapper.readTree("");
}
return mapper.readTree(response.body().string());
}

private void logError(Response response) {
if (logger == null) {
return;
}
ResponseBody body = response.body();
if (body != null) {
String text;
try {
text = body.string();
} catch (IOException e) {
return;
}
try {
ObjectMapper mapper = new ObjectMapper();
JsonNode node = mapper.readTree(text);
logger.log(DEBUG_LEVEL, node.toPrettyString());
} catch (IOException e) {
logger.log(DEBUG_LEVEL, text);
}
body.close();
}
}
}
17 changes: 11 additions & 6 deletions src/main/java/com/bioraft/rundeck/rancher/RancherAddService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@

import static com.bioraft.rundeck.rancher.Constants.*;
import static com.bioraft.rundeck.rancher.Errors.ErrorCause.*;
import static com.dtolabs.rundeck.core.Constants.ERR_LEVEL;
import static com.dtolabs.rundeck.core.Constants.INFO_LEVEL;
import static com.dtolabs.rundeck.core.Constants.*;
import static com.dtolabs.rundeck.core.plugins.configuration.PropertyResolverFactory.FRAMEWORK_PREFIX;
import static com.dtolabs.rundeck.core.plugins.configuration.PropertyResolverFactory.PROJECT_PREFIX;
import static com.dtolabs.rundeck.core.plugins.configuration.StringRenderingConstants.CODE_SYNTAX_MODE;
Expand Down Expand Up @@ -78,7 +77,7 @@ public class RancherAddService implements StepPlugin {
@PluginProperty(title = "Secret IDs", description = "List of secrets IDs, space or comma separated")
private String secrets;

final HttpClient client;
private final HttpClient client;

public RancherAddService () {
this.client = new HttpClient();
Expand Down Expand Up @@ -123,6 +122,7 @@ public void executeStep(final PluginStepContext context, final Map<String, Objec
Framework framework = context.getFramework();
String project = context.getFrameworkProject();
PluginLogger logger = context.getLogger();
client.setLogger(logger);

String endpoint = cfgFromProjectOrFramework(framework, project, RANCHER_CONFIG_ENDPOINT);
String spec = endpoint + (new Strings()).apiPath(environmentId, "/services");
Expand Down Expand Up @@ -168,16 +168,21 @@ public void executeStep(final PluginStepContext context, final Map<String, Objec
throw new StepException("Stack does not exist: " + stackName, INVALID_CONFIGURATION);
}

ObjectMapper mapper = new ObjectMapper();
try {
Map<String, Object> map = ImmutableMap.<String, Object>builder()
Map<String, Object> map = ImmutableMap.<String, Object>builder().put("type", "service")
.put("assignServiceIpAddress", false).put("startOnCreate", true).put("name", serviceName)
.put("stackId", stackId).put("rancherCompose", "").put("launchConfig", mapBuilder.build()).build();
.put("scale", 1).put("serviceIndexStrategy", "deploymentUnitBased")
.put("launchConfig", mapBuilder.build())
.put("stackId", stackId).build();
String payload = mapper.writeValueAsString(map);
logger.log(DEBUG_LEVEL, mapper.readTree(payload).toPrettyString());
JsonNode serviceResult = client.post(spec, map);
logger.log(INFO_LEVEL, "Success!");
logger.log(INFO_LEVEL, "New service ID:" + serviceResult.path("id").asText());
logger.log(INFO_LEVEL, "New service name:" + serviceResult.path("name").asText());
} catch (IOException e) {
throw new StepException("Failed posting to " + spec, e, INVALID_CONFIGURATION);
throw new StepException("Failed at " + spec + "\n" + e.getMessage(), e, INVALID_CONFIGURATION);
}
}

Expand Down
28 changes: 28 additions & 0 deletions src/main/java/com/bioraft/rundeck/rancher/RancherCredentials.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package com.bioraft.rundeck.rancher;

import com.dtolabs.rundeck.core.execution.ExecutionContext;

import java.io.IOException;
import java.util.Map;

import static com.bioraft.rundeck.rancher.Constants.CONFIG_ACCESSKEY_PATH;
import static com.bioraft.rundeck.rancher.Constants.CONFIG_SECRETKEY_PATH;

public class RancherCredentials {
private String accessKey;
private String secretKey;

public RancherCredentials(ExecutionContext context, Map<String, String> nodeAttributes) throws IOException {
Storage storage = new Storage(context);
accessKey = storage.loadStoragePathData(nodeAttributes.get(CONFIG_ACCESSKEY_PATH));
secretKey = storage.loadStoragePathData(nodeAttributes.get(CONFIG_SECRETKEY_PATH));
}

public String getAccessKey() {
return accessKey;
}

public String getSecretKey() {
return secretKey;
}
}
161 changes: 96 additions & 65 deletions src/main/java/com/bioraft/rundeck/rancher/RancherFileCopier.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,36 +111,22 @@ public String copyScriptContent(ExecutionContext context, String script, INodeEn
return copyFile(context, null, null, script, node, destination);
}

private String copyFile(final ExecutionContext context, final File scriptfile, final InputStream input,
private String copyFile(final ExecutionContext context, final File scriptFile, final InputStream input,
final String script, final INodeEntry node, final String destinationPath) throws FileCopierException {

Map<String, String> nodeAttributes = node.getAttributes();

if (nodeAttributes.get("type").equals("service")) {
String message = "File copier is not currently supported for services";
throw new FileCopierException(message, UNSUPPORTED_NODE_TYPE);
}

String accessKey;
String secretKey;
try {
Storage storage = new Storage(context);
accessKey = storage.loadStoragePathData(nodeAttributes.get(CONFIG_ACCESSKEY_PATH));
secretKey = storage.loadStoragePathData(nodeAttributes.get(CONFIG_SECRETKEY_PATH));
} catch (IOException e) {
throw new FileCopierException(e.getMessage(), AUTHENTICATION_FAILURE);
}

String remotefile = getRemoteFile(destinationPath, context, node, scriptfile);
String remoteFile = getRemoteFile(destinationPath, context, node, scriptFile);

// write to a local temp file or use the input file
final File localTempfile = (null != scriptfile) ? scriptfile
final File localTempFile = (null != scriptFile) ? scriptFile
: BaseFileCopier.writeTempFile(context, null, input, script);

// Copy the file over
ExecutionLogger logger = context.getExecutionLogger();
String absPath = localTempfile.getAbsolutePath();
String message = "copying file: '" + absPath + "' to: '" + node.getNodename() + ":" + remotefile + "'";
String absPath = localTempFile.getAbsolutePath();
String message = "copying file: '" + absPath + "' to: '" + node.getNodename() + ":" + remoteFile + "'";
logger.log(DEBUG_LEVEL, message);

Framework framework = context.getFramework();
Expand All @@ -152,90 +138,135 @@ private String copyFile(final ExecutionContext context, final File scriptfile, f

try {
String result;
if (searchPath == null || searchPath.equals("")) {
result = copyViaApi(context, nodeAttributes, accessKey, secretKey, localTempfile, remotefile);
// Use API for multiple containers for now, because we do not have externalId list in service-type of node.
if (searchPath == null || searchPath.equals("") || nodeAttributes.get("type").equals("service")) {
result = copyViaApi(context, nodeAttributes, localTempFile, remoteFile);
} else {
result = copyViaCli(context, nodeAttributes, accessKey, secretKey, localTempfile, remotefile, searchPath);
CliCopier cliCopier = new CliCopier(localTempFile, searchPath, context, nodeAttributes);
result = cliCopier.copyViaCli(nodeAttributes, remoteFile, searchPath);
}
context.getExecutionLogger().log(DEBUG_LEVEL, "Copied '" + localTempfile + "' to '" + result );
context.getExecutionLogger().log(DEBUG_LEVEL, "Copied '" + localTempFile + "' to '" + result);
return result;
} catch (IOException e) {
throw new FileCopierException(e.getMessage(), IO_EXCEPTION);
} finally {
if (null == scriptfile && !ScriptfileUtils.releaseTempFile(localTempfile)) {
if (null == scriptFile && !ScriptfileUtils.releaseTempFile(localTempFile)) {
context.getExecutionListener().log(Constants.WARN_LEVEL,
"Unable to remove local temp file: " + localTempfile.getAbsolutePath());
"Unable to remove local temp file: " + localTempFile.getAbsolutePath());
}
}
}

private String getRemoteFile(final String destinationPath, final ExecutionContext context, final INodeEntry node, final File scriptfile) {
private String getRemoteFile(final String destinationPath, final ExecutionContext context, final INodeEntry node, final File scriptFile) {
if (null == destinationPath) {
String identity = null != context.getDataContext() && null != context.getDataContext().get("job")
? context.getDataContext().get("job").get("execid")
: null;
return BaseFileCopier.generateRemoteFilepathForNode(node,
context.getFramework().getFrameworkProjectMgr().getFrameworkProject(context.getFrameworkProject()),
context.getFramework(), (null != scriptfile ? scriptfile.getName() : "dispatch-script"), null,
context.getFramework(), (null != scriptFile ? scriptFile.getName() : "dispatch-script"), null,
identity);
} else {
return destinationPath;
}
}

private String copyViaCli(final ExecutionContext context, Map<String, String> nodeAttributes, String accessKey, String secretKey,
File localTempFile, String remotefile, String searchPath) throws FileCopierException {
ExecutionLogger logger = context.getExecutionLogger();
logger.log(DEBUG_LEVEL, "PATH: '" + searchPath + "'");
private String copyViaApi(final ExecutionContext context, Map<String, String> nodeAttributes, File file, String destination)
throws FileCopierException {
try {
RancherCredentials rancherCredentials = new RancherCredentials(context, nodeAttributes);
String[] instanceIds;
if (nodeAttributes.get("type").equals("service")) {
// "self": "https://rancher.example.com/v2-beta/projects/1a10/services/1s56"
// "execute": "https://rancher.example.com/v2-beta/projects/1a10/containers/1i234/?action=execute",
String self = nodeAttributes.get(NODE_ATT_SELF);
instanceIds = nodeAttributes.get("instanceIds").split(",");
for (String instance : instanceIds) {
String url = self.replaceFirst("/services/[0-9]+s[0-9]+", "/containers/" + instance + "/?action=execute");
webSocketListener.putFile(url, rancherCredentials.getAccessKey(), rancherCredentials.getSecretKey(), file, destination);
context.getExecutionLogger().log(DEBUG_LEVEL, "PUT: '" + file + "' to " + url);
}
} else {
String url = nodeAttributes.get("execute");
webSocketListener.putFile(url, rancherCredentials.getAccessKey(), rancherCredentials.getSecretKey(), file, destination);
context.getExecutionLogger().log(DEBUG_LEVEL, "PUT: '" + file + "'");
}
} catch (IOException e) {
throw new FileCopierException(e.getMessage(), CONNECTION_FAILURE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FileCopierException(e.getMessage(), CONNECTION_FAILURE);
}
return destination;
}

String path = localTempFile.getAbsolutePath();
String instance = nodeAttributes.get("externalId");
String[] command = {"rancher", "docker", "cp", path, instance + ":" + remotefile};

boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows");
try {
private static class CliCopier {
private final String searchPath;
private final String accessKey;
private final String secretKey;

private final String path;
private final ExecutionLogger logger;

public CliCopier(File localTempFile, String searchPath, final ExecutionContext context, Map<String, String> nodeAttributes)
throws IOException {
this.path = localTempFile.getAbsolutePath();
this.searchPath = searchPath;
RancherCredentials rancherCredentials = new RancherCredentials(context, nodeAttributes);
this.accessKey = rancherCredentials.getAccessKey();
this.secretKey = rancherCredentials.getSecretKey();

this.logger = context.getExecutionLogger();
}

public String copyViaCli(Map<String, String> nodeAttributes, String remoteFile, String searchPath)
throws FileCopierException {
boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows");
if (isWindows) {
throw new FileCopierException("Windows is not currently supported.", UNSUPPORTED_OPERATING_SYSTEM);
}

logger.log(DEBUG_LEVEL, "PATH: '" + searchPath + "'");

try {
String instance = nodeAttributes.get("externalId");
String[] command = {"rancher", "docker", "cp", path, instance + ":" + remoteFile};
logger.log(DEBUG_LEVEL, "OS Copy: '" + String.join(" ", command) + "'");
this.toOneHostByCli(instance, remoteFile, nodeAttributes);
} catch (IOException e) {
throw new FileCopierException("Child process IO Exception", IO_EXCEPTION, e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FileCopierException("Child process interrupted", INTERRUPTED, e);
}

return remoteFile;
}

private void toOneHostByCli(String instance, String remoteFile, Map<String, String> nodeAttributes)
throws IOException, InterruptedException {
String[] command = {"rancher", "docker", "cp", path, instance + ":" + remoteFile};
ProcessBuilder builder = new ProcessBuilder();
Map<String, String> environment = builder.environment();
logger.log(DEBUG_LEVEL, "CMD: '" + String.join(" ", command) + "'");
logger.log(DEBUG_LEVEL, "CLI Copy: '" + String.join(" ", command) + "'");
environment.put("PATH", searchPath);
environment.put("RANCHER_ENVIRONMENT", nodeAttributes.get("environment"));
environment.put("RANCHER_DOCKER_HOST", nodeAttributes.get("hostname"));
environment.put("RANCHER_URL", nodeAttributes.get("execute").replaceFirst("/projects/.*$", ""));
environment.put("RANCHER_ACCESS_KEY", accessKey);
environment.put("RANCHER_SECRET_KEY", secretKey);
if (isWindows) {
throw new FileCopierException("Windows is not currently supported.", UNSUPPORTED_OPERATING_SYSTEM);
} else {
builder.command(command);
}
logger.log(DEBUG_LEVEL, "CMD: '" + String.join(" ", command) + "'");
builder.command(command);
builder.directory(new File(System.getProperty("java.io.tmpdir")));
Process process = builder.start();
StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println);
Executors.newSingleThreadExecutor().submit(streamGobbler);
int exitCode = process.waitFor();
assert exitCode == 0;
} catch (IOException e) {
throw new FileCopierException("Child process IO Exception", IO_EXCEPTION, e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FileCopierException("Child process interrupted", INTERRUPTED, e);
if (exitCode != 0) {
throw new IOException("CLI process failed");
}
}

return remotefile;
}

private String copyViaApi(final ExecutionContext context, Map<String, String> nodeAttributes, String accessKey, String secretKey, File file,
String destination) throws FileCopierException {
try {
String url = nodeAttributes.get("execute");
webSocketListener.putFile(url, accessKey, secretKey, file, destination);
context.getExecutionLogger().log(DEBUG_LEVEL, "PUT: '" + file + "'");
} catch (IOException e) {
throw new FileCopierException(e.getMessage(), CONNECTION_FAILURE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FileCopierException(e.getMessage(), CONNECTION_FAILURE);
}
return destination;
}

private static class StreamGobbler implements Runnable {
Expand Down
Loading

0 comments on commit 19e0de0

Please sign in to comment.