From 1a5fe234e6e008716fb018426f742f261b4f5285 Mon Sep 17 00:00:00 2001 From: Fede Galland <99492720+f-galland@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:39:29 -0300 Subject: [PATCH] Implement Http Client Logic (#93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add plugin-security.policy * Add dependencies and async http requests service class. * Make the plugin perform periodical http queries using placeholder scheduler and config classes * Adapt the plugin for POST requests to a local server * Small fixes * Switch to using HttpClient * Fix slf4j nop warnings * Fix test failing on forbidden api usage * Add json body * Use doPrivileged * Fix socket permission denied error * Make close() work again * Add lifecycle component back to jobscheduler placeholder class * Make AsyncRequestRepository a singleton * Remove unneeded dependencies * Add license and notice files * Fix forbiddenapis error * Skip checks dependency license checks and dependencies forbidden apis check * Refactor HttpClient and add unit tests (#102) * Refactor HttpClient and add unit tests * Add more JavaDocs * Fix Javadocs --------- Signed-off-by: Álex Ruiz Co-authored-by: Álex Ruiz --- plugins/command-manager/build.gradle | 26 +++- .../commandmanager/CommandManagerPlugin.java | 19 +++ .../commandmanager/index/CommandIndex.java | 16 ++- .../httpclient/HttpResponseCallback.java | 53 +++++++ .../utils/httpclient/HttpRestClient.java | 131 ++++++++++++++++++ .../utils/httpclient/HttpRestClientDemo.java | 50 +++++++ .../plugin-metadata/plugin-security.policy | 3 + .../commandmanager/CommandManagerTests.java | 46 ++++++ 8 files changed, 338 insertions(+), 6 deletions(-) create mode 100644 plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpResponseCallback.java create mode 100644 plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpRestClient.java create mode 100644 plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpRestClientDemo.java create mode 100644 plugins/command-manager/src/main/plugin-metadata/plugin-security.policy diff --git a/plugins/command-manager/build.gradle b/plugins/command-manager/build.gradle index 7defcd1..3c6ab2c 100644 --- a/plugins/command-manager/build.gradle +++ b/plugins/command-manager/build.gradle @@ -51,12 +51,35 @@ opensearchplugin { noticeFile rootProject.file('NOTICE.txt') } +def versions = [ + httpclient5: "5.4", + httpcore5: "5.3", + slf4j: "1.7.36", + log4j: "2.23.1", + conscrypt: "2.5.2" +] + +dependencies { + api "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" + api "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" + api "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" + api "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" + api "org.slf4j:slf4j-api:${versions.slf4j}" + api "org.conscrypt:conscrypt-openjdk-uber:${versions.conscrypt}" +} + // This requires an additional Jar not published as part of build-tools loggerUsageCheck.enabled = false // No need to validate pom, as we do not upload to maven/sonatype validateNebulaPom.enabled = false +// Skip forbiddenAPIs check on dependencies +thirdPartyAudit.enabled = false + +//Skip checking for third party licenses +dependencyLicenses.enabled = false + buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "2.16.0") @@ -121,5 +144,4 @@ task updateVersion { // String tokenization to support -SNAPSHOT ant.replaceregexp(file: 'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags: 'g', byline: true) } -} - +} \ No newline at end of file diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java index 267e0f4..073216c 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/CommandManagerPlugin.java @@ -9,6 +9,8 @@ import com.wazuh.commandmanager.index.CommandIndex; import com.wazuh.commandmanager.rest.action.RestPostCommandAction; +import com.wazuh.commandmanager.utils.httpclient.HttpRestClient; +import com.wazuh.commandmanager.utils.httpclient.HttpRestClientDemo; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; @@ -30,6 +32,7 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -63,6 +66,11 @@ public Collection createComponents( Supplier repositoriesServiceSupplier ) { this.commandIndex = new CommandIndex(client, clusterService, threadPool); + + // HttpRestClient stuff + String uri = "https://httpbin.org/post"; + String payload = "{\"message\": \"Hello world!\"}"; + HttpRestClientDemo.run(uri, payload); return Collections.emptyList(); } @@ -77,4 +85,15 @@ public List getRestHandlers( ) { return Collections.singletonList(new RestPostCommandAction(this.commandIndex)); } + + /** + * Close the resources opened by this plugin. + * + * @throws IOException if the plugin failed to close its resources + */ + @Override + public void close() throws IOException { + super.close(); + HttpRestClient.getInstance().stopHttpAsyncClient(); + } } diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java index 27495dd..d0138da 100644 --- a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/index/CommandIndex.java @@ -31,6 +31,9 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; +/** + * Class to manage the Command Manager index and index template. + */ public class CommandIndex implements IndexingOperationListener { private static final Logger logger = LogManager.getLogger(CommandIndex.class); @@ -53,7 +56,7 @@ public CommandIndex(Client client, ClusterService clusterService, ThreadPool thr } /** - * @param document: A Command model object + * @param document instance of the document model to persist in the index. * @return A CompletableFuture with the RestStatus response from the operation */ public CompletableFuture asyncCreate(Document document) { @@ -95,9 +98,10 @@ public CompletableFuture asyncCreate(Document document) { } /** + * Checks for the existence of the given index template in the cluster. * - * @param template_name - * @return + * @param template_name index template name within the resources folder + * @return whether the index template exists. */ public boolean indexTemplateExists(String template_name) { Map templates = this.clusterService @@ -127,7 +131,11 @@ public void putIndexTemplate(String templateName) { .patterns((List) template.get("index_patterns")); executor.submit(() -> { - AcknowledgedResponse acknowledgedResponse = this.client.admin().indices().putTemplate(putIndexTemplateRequest).actionGet(); + AcknowledgedResponse acknowledgedResponse = this.client + .admin() + .indices() + .putTemplate(putIndexTemplateRequest) + .actionGet(); if (acknowledgedResponse.isAcknowledged()) { logger.info( "Index template created successfully: {}", diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpResponseCallback.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpResponseCallback.java new file mode 100644 index 0000000..eaf357d --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpResponseCallback.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.utils.httpclient; + +import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.core5.concurrent.FutureCallback; +import org.apache.hc.core5.http.message.StatusLine; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class HttpResponseCallback implements FutureCallback { + + private static final Logger log = LogManager.getLogger(HttpResponseCallback.class); + + /** + * The Http get request. + */ + SimpleHttpRequest httpRequest; + + /** + * The Error message. + */ + String errorMessage; + + public HttpResponseCallback(SimpleHttpRequest httpRequest, + String errorMessage) { + this.httpRequest = httpRequest; + this.errorMessage = errorMessage; + } + + @Override + public void completed(SimpleHttpResponse response) { + log.debug("{}->{}", httpRequest, new StatusLine(response)); + log.debug("Got response: {}", response.getBody()); + } + + @Override + public void failed(Exception ex) { + log.error("{}->{}", httpRequest, ex); +// throw new HttpException(errorMessage, ex); + } + + @Override + public void cancelled() { + log.debug(httpRequest + " cancelled"); + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpRestClient.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpRestClient.java new file mode 100644 index 0000000..f21cd58 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpRestClient.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.utils.httpclient; + +import org.apache.hc.client5.http.async.methods.*; +import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient; +import org.apache.hc.client5.http.impl.async.HttpAsyncClients; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.io.CloseMode; +import org.apache.hc.core5.reactor.IOReactorConfig; +import org.apache.hc.core5.util.Timeout; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.Randomness; + +import java.net.URI; +import java.util.concurrent.Future; + +/** + * HTTP Rest client. Currently used to perform + * POST requests against the Wazuh Server. + */ +public class HttpRestClient { + + private static final Logger log = LogManager.getLogger(HttpRestClient.class); + private static HttpRestClient instance; + private CloseableHttpAsyncClient httpClient; + + /** + * Private default constructor + */ + private HttpRestClient() { + startHttpAsyncClient(); + } + + /** + * Singleton instance accessor + * + * @return {@link HttpRestClient#instance} + */ + public static HttpRestClient getInstance() { + if (HttpRestClient.instance == null) { + instance = new HttpRestClient(); + } + return HttpRestClient.instance; + } + + /** + * Starts http async client. + */ + private void startHttpAsyncClient() { + if (this.httpClient == null) { + try { + PoolingAsyncClientConnectionManager cm = + PoolingAsyncClientConnectionManagerBuilder.create().build(); + + IOReactorConfig ioReactorConfig = IOReactorConfig.custom() + .setSoTimeout(Timeout.ofSeconds(5)) + .build(); + + httpClient = HttpAsyncClients.custom() + .setIOReactorConfig(ioReactorConfig) + .setConnectionManager(cm) + .build(); + + httpClient.start(); + } catch (Exception e) { + // handle exception + log.error("Error starting async Http client {}", e.getMessage()); + } + } + } + + /** + * Stop http async client. + */ + public void stopHttpAsyncClient() { + if (this.httpClient != null) { + log.info("Shutting down."); + httpClient.close(CloseMode.GRACEFUL); + httpClient = null; + } + } + + /** + * Sends a POST request. + * + * @param uri Well-formed URI + * @param payload data to send + * @return HTTP response + */ + public SimpleHttpResponse post(URI uri, String payload) { + Long id = Randomness.get().nextLong(); + + try { + // Create request + HttpHost httpHost = HttpHost.create(uri.getHost()); + + SimpleHttpRequest httpPostRequest = SimpleRequestBuilder + .post() + .setHttpHost(httpHost) + .setPath(uri.getPath()) + .setBody(payload, ContentType.APPLICATION_JSON) + .build(); + + // log request + Future future = + this.httpClient.execute( + SimpleRequestProducer.create(httpPostRequest), + SimpleResponseConsumer.create(), + new HttpResponseCallback( + httpPostRequest, + "Failed to send data for ID: " + id + ) + ); + + return future.get(); + } catch (Exception e) { + log.error("Failed to send data for ID: {}", id); + } + return null; + } +} diff --git a/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpRestClientDemo.java b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpRestClientDemo.java new file mode 100644 index 0000000..448ece4 --- /dev/null +++ b/plugins/command-manager/src/main/java/com/wazuh/commandmanager/utils/httpclient/HttpRestClientDemo.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package com.wazuh.commandmanager.utils.httpclient; + +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.net.URI; +import java.net.URISyntaxException; +import java.security.AccessController; +import java.security.PrivilegedAction; + +/** + * Demo class to test the {@link HttpRestClient} class. + */ +public class HttpRestClientDemo { + + private static final Logger log = LogManager.getLogger(HttpRestClientDemo.class); + + /** + * Demo method to test the {@link HttpRestClient} class. + * + * @param endpoint POST's requests endpoint as a well-formed URI + * @param body POST's request body as a JSON string. + */ + public static void run(String endpoint, String body) { + log.info("Executing POST request"); + AccessController.doPrivileged( + (PrivilegedAction) () -> { + HttpRestClient httpClient = HttpRestClient.getInstance(); + URI host; + try { + host = new URIBuilder(endpoint).build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + SimpleHttpResponse postResponse = httpClient.post(host, body); + log.info(postResponse.getBodyText()); + return postResponse; + } + ); + } +} diff --git a/plugins/command-manager/src/main/plugin-metadata/plugin-security.policy b/plugins/command-manager/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000..6e4d716 --- /dev/null +++ b/plugins/command-manager/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,3 @@ +grant { + permission java.net.SocketPermission "*", "connect,resolve"; +}; \ No newline at end of file diff --git a/plugins/command-manager/src/test/java/com/wazuh/commandmanager/CommandManagerTests.java b/plugins/command-manager/src/test/java/com/wazuh/commandmanager/CommandManagerTests.java index cb4da7e..eacd0b1 100644 --- a/plugins/command-manager/src/test/java/com/wazuh/commandmanager/CommandManagerTests.java +++ b/plugins/command-manager/src/test/java/com/wazuh/commandmanager/CommandManagerTests.java @@ -7,8 +7,54 @@ */ package com.wazuh.commandmanager; +import com.wazuh.commandmanager.utils.httpclient.HttpRestClient; +import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; +import org.junit.Assert; import org.opensearch.test.OpenSearchTestCase; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.AccessController; +import java.security.PrivilegedAction; + public class CommandManagerTests extends OpenSearchTestCase { // Add unit tests for your plugin + + private HttpRestClient httpClient; + + public void testPost_success() { + try { + AccessController.doPrivileged( + (PrivilegedAction) () -> { + this.httpClient = HttpRestClient.getInstance(); + URI uri = null; + try { + uri = new URI("https://httpbin.org/post"); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + String payload = "{\"message\": \"Hello world!\"}"; + SimpleHttpResponse postResponse = this.httpClient.post(uri, payload); + + String responseText = postResponse.getBodyText(); + assertNotEquals(null, postResponse); + assertNotEquals(null, responseText); + assertEquals(200, postResponse.getCode()); + assertNotEquals(0, responseText.length()); + assertTrue(responseText.contains("Hello world!")); + return postResponse; + } + ); + } catch (Exception e) { + Assert.fail("Failed to execute HTTP request: " + e); + } finally { + this.httpClient.stopHttpAsyncClient(); + } + } + + public void testPost_badUri() { + } + + public void testPost_badPayload() { + } }