From 35157880a1bb6aa91460d09886f89e4ec40d6aa5 Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Tue, 14 Dec 2021 12:37:42 +0100 Subject: [PATCH 1/4] [maven-release-plugin] prepare release 1.0.3 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a7bb7f6..f96111e 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.plugin graylog-plugin-http-output - 1.0.3-SNAPSHOT + 1.0.3 jar ${project.artifactId} @@ -36,7 +36,7 @@ scm:git:git@github.com:SekoiaLab/graylog-http-plugin.git scm:git:git@github.com:SekoiaLab/graylog-http-plugin.git https://github.com/SekoiaLab/graylog-http-plugin - HEAD + 1.0.3 From ef8f8dc6bd2ed85342318dcb90599b1aede86929 Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Tue, 14 Dec 2021 12:37:46 +0100 Subject: [PATCH 2/4] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f96111e..046c561 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.plugin graylog-plugin-http-output - 1.0.3 + 1.0.4-SNAPSHOT jar ${project.artifactId} @@ -36,7 +36,7 @@ scm:git:git@github.com:SekoiaLab/graylog-http-plugin.git scm:git:git@github.com:SekoiaLab/graylog-http-plugin.git https://github.com/SekoiaLab/graylog-http-plugin - 1.0.3 + HEAD From 3759c6b468ddbea68e7b49e7ed27d57f00fad829 Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Tue, 14 Dec 2021 14:41:56 +0100 Subject: [PATCH 3/4] fix(GraylogHTTPPlugin): change the payload send to SEKOIA.IO HTTP intake --- src/main/java/com/plugin/HttpOutput.java | 29 +++++++++++++----------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/plugin/HttpOutput.java b/src/main/java/com/plugin/HttpOutput.java index 8b8149d..d687791 100644 --- a/src/main/java/com/plugin/HttpOutput.java +++ b/src/main/java/com/plugin/HttpOutput.java @@ -5,6 +5,7 @@ import java.net.URL; import java.util.List; import java.util.Map; +import java.util.HashMap; import java.util.concurrent.TimeUnit; import okhttp3.*; @@ -33,14 +34,14 @@ * interfaces. (i.e. AlarmCallback, MessageInput, MessageOutput) */ public class HttpOutput implements MessageOutput { - - public static final int HTTP_BATCH_SIZE = 500; private final OkHttpClient httpClient; private final Gson gson = new Gson(); private boolean shutdown; private boolean raise_exception_on_http_error = false; private String url; + private String intake_key; private static final String CK_OUTPUT_API = "output_api"; + private static final String CK_INTAKE_KEY = "intake_key"; private static final String CK_GZIP_REQUEST = "gzip_request"; private static final String CK_RAISE_EXCEPTION = "raise_exception"; private static final Logger LOG = LoggerFactory.getLogger(HttpOutput.class); @@ -92,11 +93,13 @@ public void writeTo(BufferedSink sink) throws IOException { public HttpOutput(@Assisted Stream stream, @Assisted Configuration conf) throws HttpOutputException { this.url = conf.getString(CK_OUTPUT_API); + this.intake_key = conf.getString(CK_INTAKE_KEY); this.raise_exception_on_http_error = conf.getBoolean(CK_RAISE_EXCEPTION); this.shutdown = false; LOG.info(" Http Output Plugin has been configured with the following parameters:"); LOG.info(CK_OUTPUT_API + " : " + this.url); + LOG.info(CK_INTAKE_KEY + " : " + this.intake_key); LOG.info(CK_GZIP_REQUEST + " : " + conf.getBoolean(CK_GZIP_REQUEST)); LOG.info(CK_RAISE_EXCEPTION + " : " + this.raise_exception_on_http_error); @@ -132,26 +135,22 @@ public void stop() { @Override public void write(List msgs) throws Exception { - for (List partition : ListUtils.partition(msgs, HTTP_BATCH_SIZE)) { - List> payload = new FastArrayList(); - for (Message msg : partition) { - payload.add(msg.getFields()); - } - this.executeRequest(RequestBody.create( - JSON, - this.gson.toJson(payload) - )); + for(Message message: msgs) { + this.write(message); } - } @Override public void write(Message msg) throws Exception { + Map payload = new HashMap<>(); + payload.put("intake_key", this.intake_key); + payload.put("json", this.gson.toJson(msg.getFields())); + this.executeRequest( RequestBody.create( JSON, - this.gson.toJson(msg.getFields()) + this.gson.toJson(payload) ) ); } @@ -204,6 +203,10 @@ public ConfigurationRequest getRequestedConfiguration() { new TextField(CK_OUTPUT_API, "API to forward the stream data.", "/", "HTTP address where the stream data to be sent.", ConfigurationField.Optional.NOT_OPTIONAL)); + configurationRequest.addField( + new TextField(CK_INTAKE_KEY, "Intake key", "", + "The intake key to identify the events", ConfigurationField.Optional.NOT_OPTIONAL)); + configurationRequest.addField(new BooleanField(CK_GZIP_REQUEST, "GZIP request", false, "Enable GZIP compression for HTTP requests.")); From 3d5d2e39f47e25128178c2a4afebbd3274ff8bb2 Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Tue, 14 Dec 2021 14:47:05 +0100 Subject: [PATCH 4/4] refactor(GraylogHttpOutput): remove the ability to gzip sent payloads --- src/main/java/com/plugin/HttpOutput.java | 50 ------------------------ 1 file changed, 50 deletions(-) diff --git a/src/main/java/com/plugin/HttpOutput.java b/src/main/java/com/plugin/HttpOutput.java index d687791..01ca25a 100644 --- a/src/main/java/com/plugin/HttpOutput.java +++ b/src/main/java/com/plugin/HttpOutput.java @@ -42,53 +42,10 @@ public class HttpOutput implements MessageOutput { private String intake_key; private static final String CK_OUTPUT_API = "output_api"; private static final String CK_INTAKE_KEY = "intake_key"; - private static final String CK_GZIP_REQUEST = "gzip_request"; private static final String CK_RAISE_EXCEPTION = "raise_exception"; private static final Logger LOG = LoggerFactory.getLogger(HttpOutput.class); private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - /** - * This interceptor compresses the HTTP request body. Many webservers can't handle this! - *

- * taken from https://square.github.io/okhttp/interceptors/ - */ - final class GzipRequestInterceptor implements Interceptor { - @Override - public Response intercept(Interceptor.Chain chain) throws IOException { - Request originalRequest = chain.request(); - if (originalRequest.body() == null || originalRequest.header("Content-Encoding") != null) { - return chain.proceed(originalRequest); - } - - Request compressedRequest = originalRequest.newBuilder() - .header("Content-Encoding", "gzip") - .method(originalRequest.method(), gzip(originalRequest.body())) - .build(); - return chain.proceed(compressedRequest); - } - - private RequestBody gzip(final RequestBody body) { - return new RequestBody() { - @Override - public MediaType contentType() { - return body.contentType(); - } - - @Override - public long contentLength() { - return -1; // We don't know the compressed length in advance! - } - - @Override - public void writeTo(BufferedSink sink) throws IOException { - BufferedSink gzipSink = Okio.buffer(new GzipSink(sink)); - body.writeTo(gzipSink); - gzipSink.close(); - } - }; - } - } - @Inject public HttpOutput(@Assisted Stream stream, @Assisted Configuration conf) throws HttpOutputException { @@ -100,7 +57,6 @@ public HttpOutput(@Assisted Stream stream, @Assisted Configuration conf) throws LOG.info(" Http Output Plugin has been configured with the following parameters:"); LOG.info(CK_OUTPUT_API + " : " + this.url); LOG.info(CK_INTAKE_KEY + " : " + this.intake_key); - LOG.info(CK_GZIP_REQUEST + " : " + conf.getBoolean(CK_GZIP_REQUEST)); LOG.info(CK_RAISE_EXCEPTION + " : " + this.raise_exception_on_http_error); try { @@ -115,9 +71,6 @@ public HttpOutput(@Assisted Stream stream, @Assisted Configuration conf) throws .writeTimeout(10, TimeUnit.SECONDS) .readTimeout(5, TimeUnit.SECONDS); - if (conf.getBoolean(CK_GZIP_REQUEST)) { - clientBuilder.addInterceptor(new GzipRequestInterceptor()); - } this.httpClient = clientBuilder.build(); } @@ -207,9 +160,6 @@ public ConfigurationRequest getRequestedConfiguration() { new TextField(CK_INTAKE_KEY, "Intake key", "", "The intake key to identify the events", ConfigurationField.Optional.NOT_OPTIONAL)); - configurationRequest.addField(new BooleanField(CK_GZIP_REQUEST, "GZIP request", false, - "Enable GZIP compression for HTTP requests.")); - configurationRequest.addField(new BooleanField(CK_RAISE_EXCEPTION, "Raise exception", false, "Raise an exception on HTTP error"));