From 5a0d3cbef1139642477d02d7b5cd28d498f84172 Mon Sep 17 00:00:00 2001 From: rfoltyns Date: Mon, 20 Jun 2022 00:39:44 +0100 Subject: [PATCH] Elasticsearch 8 compatibility - set default mapping type to null (#81) * HCBatchOperations constructor deprecated * HCHttp.mappingType deprecated and moved to ElasticsearchBulkApi * ApiRequestFactory added to allow custom client API builders * ElasticsearchBulkApi added to handle Elasticsearch builders * ElasticsearchBulkApiPlugin added to handle Log4j2 config --- README.md | 3 +- .../elasticsearch/IndexTemplateTest.java | 4 +- log4j2-elasticsearch-hc/README.md | 65 +++++--- log4j2-elasticsearch-hc/pom.xml | 2 +- .../log4j2/elasticsearch/hc/BatchRequest.java | 118 ++++++++++---- .../elasticsearch/hc/ClientAPIFactory.java | 31 ++++ .../hc/ElasticsearchBulkAPI.java | 108 +++++++++++++ .../hc/ElasticsearchBulkPlugin.java | 130 +++++++++++++++ .../elasticsearch/hc/HCBatchOperations.java | 69 ++++---- .../log4j2/elasticsearch/hc/HCHttp.java | 34 +--- .../log4j2/elasticsearch/hc/HCHttpPlugin.java | 28 +++- .../elasticsearch/hc/HCRequestFactory.java | 20 ++- .../elasticsearch/hc/HttpClientProvider.java | 1 - .../log4j2/elasticsearch/hc/IndexRequest.java | 4 - .../log4j2/elasticsearch/hc/Request.java | 4 +- ...ava => AbstractHCBatchOperationsTest.java} | 29 ++-- .../elasticsearch/hc/BatchRequestTest.java | 149 ++++++++++++------ .../hc/CheckBootstrapIndexTest.java | 4 +- ...tAPIFactoryBasedHCBatchOperationsTest.java | 55 +++++++ .../hc/ElasticsearchBulkAPITest.java | 126 +++++++++++++++ .../hc/ElasticsearchBulkPluginTest.java | 135 ++++++++++++++++ .../elasticsearch/hc/HCHttpPluginTest.java | 27 ++++ .../log4j2/elasticsearch/hc/HCHttpTest.java | 31 ++-- .../hc/HCRequestFactoryTest.java | 34 +++- .../elasticsearch/hc/HttpClientTest.java | 16 +- .../elasticsearch/hc/IndexRequestTest.java | 7 +- .../hc/LegacyHCBatchOperationsTest.java | 54 +++++++ .../hc/PutIndexTemplateTest.java | 12 +- ...ncBatchEmitterWithBatchOperationsTest.java | 5 +- .../elasticsearch/hc/smoke/SmokeTest.java | 14 +- .../src/test/resources/log4j2.properties | 46 +++++- .../src/test/resources/log4j2.xml | 39 +++-- 32 files changed, 1147 insertions(+), 257 deletions(-) create mode 100644 log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactory.java create mode 100644 log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkAPI.java create mode 100644 log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPlugin.java rename log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/{HCBatchOperationsTest.java => AbstractHCBatchOperationsTest.java} (88%) create mode 100644 log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactoryBasedHCBatchOperationsTest.java create mode 100644 log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkAPITest.java create mode 100644 log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPluginTest.java create mode 100644 log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/LegacyHCBatchOperationsTest.java diff --git a/README.md b/README.md index 18d04d29..43a46e8f 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Latest released code (1.5.x) is available [here](https://github.com/rfoltyns/log Project consists of: * `log4j2-elasticsearch-core` - skeleton provider for conrete implementations -* `log4j2-elasticsearch-hc` - optimized Apache Async HTTP client compatible with Elasticsearch 2.x, 5.x, 6.x and 7.x clusters +* `log4j2-elasticsearch-hc` - optimized Apache Async HTTP client compatible with Elasticsearch 2.x, 5.x, 6.x, 7.x and 8.x clusters * `log4j2-elasticsearch-jest` - [Jest HTTP Client](https://github.com/searchbox-io/Jest) compatible with Elasticsearch 2.x, 5.x, 6.x and 7.x clusters * `log4j2-elasticsearch2-bulkprocessor` - [TCP client](https://www.elastic.co/guide/en/elasticsearch/client/java-api/2.4/java-docs-bulk-processor.html) compatible with 2.x clusters * `log4j2-elasticsearch5-bulkprocessor` - [TCP client](https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.6/java-docs-bulk-processor.html) compatible with 5.x and 6.x clusters @@ -39,6 +39,7 @@ Project consists of: * [Component templates](log4j2-elasticsearch-core#component-templates) configuration * [Composable index templates](log4j2-elasticsearch-core#composable-index-template) configuration * [Service Discovery](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-hc#service-discovery) for HC module +* (1.6) Elasticsearch 8.x support (`null` mapping type) ### Roadmap [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/rfoltyns/log4j2-elasticsearch) diff --git a/log4j2-elasticsearch-core/src/test/java/org/appenders/log4j2/elasticsearch/IndexTemplateTest.java b/log4j2-elasticsearch-core/src/test/java/org/appenders/log4j2/elasticsearch/IndexTemplateTest.java index 0e080643..7f5145fa 100644 --- a/log4j2-elasticsearch-core/src/test/java/org/appenders/log4j2/elasticsearch/IndexTemplateTest.java +++ b/log4j2-elasticsearch-core/src/test/java/org/appenders/log4j2/elasticsearch/IndexTemplateTest.java @@ -68,7 +68,7 @@ public void startsWhenSetupCorrectlyWithNonDefaultApiVersion() { final IndexTemplate.Builder builder = createTestIndexTemplateBuilder(); builder.withName(TEST_INDEX_TEMPLATE) .withPath(TEST_PATH) - .withApiVersion(8); + .withApiVersion(7); // when final IndexTemplate indexTemplate = builder.build(); @@ -76,7 +76,7 @@ public void startsWhenSetupCorrectlyWithNonDefaultApiVersion() { // then assertNotNull(indexTemplate); assertNotEquals(IndexTemplate.DEFAULT_API_VERSION, indexTemplate.getApiVersion()); - assertEquals(8, indexTemplate.getApiVersion()); + assertEquals(7, indexTemplate.getApiVersion()); assertEquals(TEST_INDEX_TEMPLATE, indexTemplate.getName()); assertNotNull(indexTemplate.getSource()); assertEquals(IndexTemplate.TYPE_NAME, indexTemplate.getType()); diff --git a/log4j2-elasticsearch-hc/README.md b/log4j2-elasticsearch-hc/README.md index b4cfa56b..15165329 100644 --- a/log4j2-elasticsearch-hc/README.md +++ b/log4j2-elasticsearch-hc/README.md @@ -42,19 +42,20 @@ It's highly recommended to put this plugin behind `AsyncLogger`. See [log4j2.xml ``` ### HCHttp Properties -| Name | Type | Required | Default | Description | -|----------------------------------|-----------|-----------------------------------------------------------------|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| serverUris | Attribute | no (MUST be specified by either `HCHttp` or `ServiceDiscovery`) | None | List of semicolon-separated `http[s]://host:[port]` addresses of Elasticsearch nodes to connect with. | -| connTimeout | Attribute | no | 1000 | Number of milliseconds before ConnectException is thrown while attempting to connect. | -| readTimeout | Attribute | no | 0 | Number of milliseconds before SocketTimeoutException is thrown while waiting for response bytes. | -| maxTotalConnections | Attribute | no | 8 | Number of connections available. | -| ioThreadCount | Attribute | no | No. of available processors | Number of `I/O Dispatcher` threads started by Apache HC `IOReactor` | -| itemSourceFactory | Element | yes | None | `ItemSourceFactory` used to create wrappers for batch requests. `PooledItemSourceFactory` and it's extensions can be used. | -| mappingType | Attribute | no | `_doc` | Name of index mapping type to use in ES cluster. `_doc` is used by default for compatibility with Elasticsearch 7.x. | -| pooledResponseBuffers | Attribute | no | yes | If `true`, pooled `SimpleInputBuffer`s will be used to handle responses. Otherwise, new `SimpleInputBuffer` wil be created for every response. | -| pooledResponseBuffersSizeInBytes | Attribute | no | 1MB (1048756 bytes) | Single response buffer size. | -| auth | Element | no | None | Security config. [Security](#pem-cert-config) | -| serviceDiscovery | Element | no | None | Service discovery config. [ServiceDiscovery](#service-discovery) | +| Name | Type | Required | Default | Description | +|----------------------------------|-----------|-----------------------------------------------------------------|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| serverUris | Attribute | no (MUST be specified by either `HCHttp` or `ServiceDiscovery`) | None | List of semicolon-separated `http[s]://host:[port]` addresses of Elasticsearch nodes to connect with. | +| connTimeout | Attribute | no | 1000 | Number of milliseconds before ConnectException is thrown while attempting to connect. | +| readTimeout | Attribute | no | 0 | Number of milliseconds before SocketTimeoutException is thrown while waiting for response bytes. | +| maxTotalConnections | Attribute | no | 8 | Number of connections available. | +| ioThreadCount | Attribute | no | No. of available processors | Number of `I/O Dispatcher` threads started by Apache HC `IOReactor` | +| itemSourceFactory | Element | yes | None | `ItemSourceFactory` used to create wrappers for batch requests. `PooledItemSourceFactory` and it's extensions can be used. | +| mappingType | Attribute | no | `null` since 1.6 | Name of index mapping type to use. Applicable to Elasticsearch <8.x. See [removal of types](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html).
DEPRECATED: As of 1.7, this attribute will be removed. Use [ElasticsearchBulk](#elasticsearchbulk) instead. | +| pooledResponseBuffers | Attribute | no | yes | If `true`, pooled `SimpleInputBuffer`s will be used to handle responses. Otherwise, new `SimpleInputBuffer` wil be created for every response. | +| pooledResponseBuffersSizeInBytes | Attribute | no | 1MB (1048756 bytes) | Single response buffer size. | +| auth | Element | no | None | Security config. [Security](#pem-cert-config) | +| serviceDiscovery | Element | no | None | Service discovery config. [ServiceDiscovery](#service-discovery) | +| clientAPIFactory | Element | no | `ElasticsearchBulk` | Batch API factory. [ElasticsearchBulk](#elasticsearchbulk) | ### Service Discovery @@ -123,6 +124,29 @@ Example: NOTE: Config policies were added for convenience. Recommended configuration should contain `configPolices=none`, `serverUris` configured ONLY for `ServiceDiscovery` and separate `Security` configs if needed. +### Client API Factory + +Since 1.6, [ClientAPIFactory](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactory.java) can be configured to further customize the output and runtime capabilities of batches and batch items. [ElasticsearchBulk](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPlugin.java) is used by default. + +#### ElasticsearchBulk + +Default. + +Configures builders and serializers for: +* [BatchRequest](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/BatchRequest.java) - `/_bulk` request (batch) +* [IndexRequest](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/IndexRequest.java) - document (batch item) + +```xml + + + +``` + +#### ElasticsearchBulk Properties +| Name | Type | Required | Default | Description | +|-------------|-----------|----------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mappingType | Attribute | no | `null` since 1.6 | Name of index mapping type to use. Applicable to Elasticsearch <8.x. See [removal of types](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html). | + ### Programmatic config See [programmatc config example](https://github.com/rfoltyns/log4j2-elasticsearch/blob/master/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/smoke/SmokeTest.java). @@ -167,6 +191,9 @@ Example: See [index name](../log4j2-elasticsearch-core#index-name) or [index rollover](../log4j2-elasticsearch-core#index-rollover) ### Index template + +Since 1.6, this module is compatible with Elasticsearch 8.x by default. Use `apiVersion` for older clusters. + See [index template docs](../log4j2-elasticsearch-core#index-template) ### SSL/TLS @@ -200,12 +227,12 @@ Can be configured using `Security` tag: ### Compatibility matrix -| Feature/Version | 2.x | 5.x | 6.x | 7.x | -|------------------|------------|------------|------------|------------| -| IndexTemplate | Yes | Yes | Yes | Yes | -| BasicCredentials | Yes | Yes | Yes | Yes | -| JKS | Yes | Not tested | Not tested | Not tested | -| PEM | Not tested | Yes | Yes | Yes | +| Feature/Version | 2.x | 5.x | 6.x | 7.x | 8.x | +|------------------|------------|------------|------------|------------|------------| +| IndexTemplate | Yes | Yes | Yes | Yes | Yes | +| BasicCredentials | Yes | Yes | Yes | Yes | Yes | +| JKS | Yes | Not tested | Not tested | Not tested | Not tested | +| PEM | Not tested | Yes | Yes | Yes | Yes | ## Pluggable JCTools diff --git a/log4j2-elasticsearch-hc/pom.xml b/log4j2-elasticsearch-hc/pom.xml index a7153626..454a1d96 100644 --- a/log4j2-elasticsearch-hc/pom.xml +++ b/log4j2-elasticsearch-hc/pom.xml @@ -8,7 +8,7 @@ 4.0.0 log4j2-elasticsearch-hc Log4j2 Elasticsearch Apache HC HTTP client - Log4j2 Appender plugin pushing logs in batches to Elasticsearch (2.x/5.x/6.x/7.x) clusters + Log4j2 Appender plugin pushing logs in batches to Elasticsearch (2.x/5.x/6.x/7.x/8.x) clusters diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/BatchRequest.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/BatchRequest.java index 852fafc6..af15f407 100644 --- a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/BatchRequest.java +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/BatchRequest.java @@ -23,10 +23,13 @@ import com.fasterxml.jackson.databind.ObjectWriter; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufOutputStream; +import org.appenders.log4j2.elasticsearch.Deserializer; import org.appenders.log4j2.elasticsearch.ItemSource; +import org.appenders.log4j2.elasticsearch.JacksonSerializer; +import org.appenders.log4j2.elasticsearch.Serializer; import java.io.IOException; -import java.io.OutputStream; +import java.io.InputStream; import java.util.Collection; import static org.appenders.log4j2.elasticsearch.QueueFactory.getQueueFactoryInstance; @@ -41,46 +44,54 @@ public class BatchRequest implements Batch { public static final String HTTP_METHOD_NAME = "POST"; public static final char LINE_SEPARATOR = '\n'; - private final ObjectWriter objectWriter; - private ItemSource itemSource; + private final Serializer itemSerializer; + private final Deserializer resultDeserializer; + private ItemSource buffer; protected final Collection indexRequests; - protected BatchRequest(Builder builder) { + protected BatchRequest(final Builder builder) { this.indexRequests = getQueueFactoryInstance(BatchRequest.class.getSimpleName()).toIterable(builder.items); - this.objectWriter = builder.objectWriter; - this.itemSource = builder.itemSource; + this.itemSerializer = builder.itemSerializer; + this.resultDeserializer = builder.resultDeserializer; + this.buffer = builder.itemSource; } /** - * Serializes and writes {@link #indexRequests} into {@link #itemSource} + * Serializes and writes {@link #indexRequests} into {@link #buffer} * * @return underlying buffer filled with serialized indexRequests * @throws IOException if serialization failed */ - public ItemSource serialize() throws IOException { + public ItemSource serialize() throws Exception { - ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(itemSource.getSource()); + ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(buffer.getSource()); // in current impl with no IDs, it's possible to reduce serialization by reusing first action IndexRequest identicalAction = uniformAction(indexRequests); - byte[] actionTemplate = identicalAction != null ? objectWriter.writeValueAsBytes(identicalAction) : null; + byte[] actionTemplate = identicalAction != null ? itemSerializer.writeAsBytes(identicalAction) : null; for (IndexRequest action : indexRequests) { if (actionTemplate == null) { - objectWriter.writeValue((OutputStream) byteBufOutputStream, action); + itemSerializer.write(byteBufOutputStream, action); } else { byteBufOutputStream.write(actionTemplate); } byteBufOutputStream.writeByte(LINE_SEPARATOR); ByteBuf source = action.getSource().getSource(); - itemSource.getSource().writeBytes(source); + buffer.getSource().writeBytes(source); byteBufOutputStream.writeByte(LINE_SEPARATOR); } - return itemSource; + + return buffer; + + } + + public BatchResult deserialize(final InputStream responseBody) throws IOException { + return resultDeserializer.read(responseBody); } /** @@ -90,20 +101,28 @@ public ItemSource serialize() throws IOException { * @param indexRequests collection of items to be checked * @return {@link IndexRequest} first action in given collection if all items are equal, null otherwise */ - IndexRequest uniformAction(Collection indexRequests) { + IndexRequest uniformAction(final Collection indexRequests) { IndexRequest current = null; for (IndexRequest indexRequest : indexRequests) { + if (current == null) { current = indexRequest; continue; } - if (!current.index.equals(indexRequest.index) || !current.type.equals(indexRequest.type)) { + + final boolean sameIndex = current.sameIndex(indexRequest); + final boolean sameType = current.sameType(indexRequest); + + if (!sameIndex || !sameType) { // fail fast and serialize each item return null; } + } + return current; + } /** @@ -111,13 +130,14 @@ IndexRequest uniformAction(Collection indexRequests) { *

MUST be called when request is completed. Otherwise it may lead to excessive resource usage and memory leaks */ public void completed() { + for (IndexRequest indexRequest : indexRequests) { indexRequest.completed(); } indexRequests.clear(); - itemSource.release(); - itemSource = null; + buffer.release(); + buffer = null; } @@ -141,44 +161,86 @@ public String getHttpMethodName() { public static class Builder { - private static final int INITIAL_SIZE = Integer.parseInt(System.getProperty("appenders." + BatchRequest.class.getSimpleName() + ".initialSize", "10000")); + private static final int INITIAL_SIZE = Integer.parseInt(System.getProperty("appenders." + BatchRequest.class.getSimpleName() + ".initialSize", "8192")); - protected final Collection items = getQueueFactoryInstance(BatchRequest.class.getSimpleName()).tryCreateMpscQueue(INITIAL_SIZE); + protected final Collection items; private ItemSource itemSource; - private ObjectWriter objectWriter; + private Serializer itemSerializer; + private Deserializer resultDeserializer; - public Builder add(IndexRequest item) { + public Builder() { + this.items = getQueueFactoryInstance(BatchRequest.class.getSimpleName()).tryCreateMpscQueue(INITIAL_SIZE); + } + + public Builder add(final Object item) { + add((IndexRequest)item); + return this; + } + + public Builder add(final IndexRequest item) { this.items.add(item); return this; } - public Builder add(Collection items) { + public Builder add(final Collection items) { this.items.addAll(items); return this; } public BatchRequest build() { + + validate(); + + return new BatchRequest(this); + + } + + protected void validate() { + if (itemSource == null) { throw new IllegalArgumentException("buffer cannot be null"); } - if (objectWriter == null) { - throw new IllegalArgumentException("objectWriter cannot be null"); + if (itemSerializer == null) { + throw new IllegalArgumentException("itemSerializer cannot be null"); + } + + if (resultDeserializer == null) { + throw new IllegalArgumentException("resultDeserializer cannot be null"); } - return new BatchRequest(this); } - public Builder withBuffer(ItemSource buffer) { + public Builder withBuffer(final ItemSource buffer) { this.itemSource = buffer; return this; } - public Builder withObjectWriter(ObjectWriter objectWriter) { - this.objectWriter = objectWriter; + /** + * Will be replaced with Serializer. Use {@link #withItemSerializer(Serializer)} + * + * @param objectWriter item serializer + * @return this + * @deprecated As of 1.7, this method will be removed. Use {@link #withItemSerializer(Serializer)} instead + */ + @Deprecated + public Builder withObjectWriter(final ObjectWriter objectWriter) { + if (objectWriter == null) { + throw new IllegalArgumentException("objectWriter cannot be null"); + } + this.itemSerializer = new JacksonSerializer<>(objectWriter); return this; } + public Builder withItemSerializer(final Serializer serializer) { + this.itemSerializer = serializer; + return this; + } + + public Builder withResultDeserializer(final Deserializer deserializer) { + this.resultDeserializer = deserializer; + return this; + } } } diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactory.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactory.java new file mode 100644 index 00000000..df8f0fcd --- /dev/null +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactory.java @@ -0,0 +1,31 @@ +package org.appenders.log4j2.elasticsearch.hc; + +/*- + * #%L + * log4j2-elasticsearch + * %% + * Copyright (C) 2022 Rafal Foltynski + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.appenders.log4j2.elasticsearch.ItemSource; + +public interface ClientAPIFactory { + + ITEM_BUILDER_TYPE itemBuilder(String target, ItemSource payload); + + BATCH_BUILDER_TYPE batchBuilder(); + +} diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkAPI.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkAPI.java new file mode 100644 index 00000000..f9670b2a --- /dev/null +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkAPI.java @@ -0,0 +1,108 @@ +package org.appenders.log4j2.elasticsearch.hc; + +/*- + * #%L + * log4j2-elasticsearch + * %% + * Copyright (C) 2022 Rafal Foltynski + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MappingJsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.introspect.VisibilityChecker; +import org.appenders.log4j2.elasticsearch.Deserializer; +import org.appenders.log4j2.elasticsearch.ExtendedObjectMapper; +import org.appenders.log4j2.elasticsearch.ItemSource; +import org.appenders.log4j2.elasticsearch.JacksonDeserializer; +import org.appenders.log4j2.elasticsearch.JacksonSerializer; +import org.appenders.log4j2.elasticsearch.Serializer; + +public class ElasticsearchBulkAPI implements ClientAPIFactory { + + private final String mappingType; + private final Serializer itemSerializer; + private final Deserializer resultDeserializer; + + public ElasticsearchBulkAPI(final String mappingType) { + this.mappingType = mappingType; + this.itemSerializer = createItemSerializer(); + this.resultDeserializer = createResultDeserializer(); + } + + /** + * @param itemSerializer index request metadata serializer + * @param resultDeserializer batch response deserializer + * @param mappingType Elasticsearch mapping type + */ + public ElasticsearchBulkAPI( + final String mappingType, + final Serializer itemSerializer, + final Deserializer resultDeserializer + ) { + this.mappingType = mappingType; + this.itemSerializer = itemSerializer; + this.resultDeserializer = resultDeserializer; + } + + @Override + public IndexRequest.Builder itemBuilder(final String target, final ItemSource payload) { + return new IndexRequest.Builder(payload) + .index(target) + .type(mappingType); + } + + @Override + public BatchRequest.Builder batchBuilder() { + return new BatchRequest.Builder() + .withItemSerializer(itemSerializer) + .withResultDeserializer(resultDeserializer); + } + + protected Serializer createItemSerializer() { + + final ObjectWriter objectWriter = new ExtendedObjectMapper(new MappingJsonFactory()) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) + .addMixIn(IndexRequest.class, IndexRequestMixIn.class) + .writerFor(IndexRequest.class); + + return new JacksonSerializer<>(objectWriter); + + } + + @SuppressWarnings("DuplicatedCode") + protected Deserializer createResultDeserializer() { + + final ObjectReader objectReader = new ObjectMapper() + .setVisibility(VisibilityChecker.Std.defaultInstance().with(JsonAutoDetect.Visibility.ANY)) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) + .configure(SerializationFeature.CLOSE_CLOSEABLE, false) + .configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) + .addMixIn(BatchResult.class, BatchResultMixIn.class) + .addMixIn(Error.class, ErrorMixIn.class) + .addMixIn(BatchItemResult.class, BatchItemResultMixIn.class) + .readerFor(BatchResult.class); + + return new JacksonDeserializer<>(objectReader); + + } + +} diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPlugin.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPlugin.java new file mode 100644 index 00000000..937c77c9 --- /dev/null +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPlugin.java @@ -0,0 +1,130 @@ +package org.appenders.log4j2.elasticsearch.hc; + +/*- + * #%L + * log4j2-elasticsearch + * %% + * Copyright (C) 2022 Rafal Foltynski + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MappingJsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.introspect.VisibilityChecker; +import org.apache.logging.log4j.core.config.ConfigurationException; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; +import org.appenders.log4j2.elasticsearch.Deserializer; +import org.appenders.log4j2.elasticsearch.ExtendedObjectMapper; +import org.appenders.log4j2.elasticsearch.JacksonDeserializer; +import org.appenders.log4j2.elasticsearch.JacksonSerializer; +import org.appenders.log4j2.elasticsearch.Serializer; + +@Plugin(name = "ElasticsearchBulk", category = Node.CATEGORY, elementType = "clientAPIFactory", printObject = true) +public class ElasticsearchBulkPlugin extends ElasticsearchBulkAPI { + + private ElasticsearchBulkPlugin( + final Serializer itemSerializer, + final Deserializer resultDeserializer, + final String mappingType + ) { + super(mappingType, itemSerializer, resultDeserializer); + } + + @PluginBuilderFactory + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder implements org.apache.logging.log4j.core.util.Builder { + + @PluginAttribute(value = "mappingType") + protected String mappingType; + + private Serializer itemSerializer = createItemSerializer(); + private Deserializer resultDeserializer = createResultDeserializer(); + + @Override + public ElasticsearchBulkPlugin build() + { + if (itemSerializer == null) { + throw new ConfigurationException("itemSerializer cannot be null"); + } + + if (resultDeserializer == null) { + throw new ConfigurationException("resultDeserializer cannot be null"); + } + + return new ElasticsearchBulkPlugin(itemSerializer, resultDeserializer, mappingType); + } + + public Builder withMappingType(final String mappingType) { + this.mappingType = mappingType; + return this; + } + + public Builder withItemSerializer(final Serializer itemSerializer) { + this.itemSerializer = itemSerializer; + return this; + } + + public Builder withResultDeserializer(final Deserializer resultDeserializer) { + this.resultDeserializer = resultDeserializer; + return this; + } + + /** + * @return index request metadata serializer + */ + protected Serializer createItemSerializer() { + final ObjectWriter objectWriter = new ExtendedObjectMapper(new MappingJsonFactory()) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) + .addMixIn(IndexRequest.class, IndexRequestMixIn.class) + .writerFor(IndexRequest.class); + return new JacksonSerializer<>(objectWriter); + } + + /** + * @return batch response deserializer + */ + @SuppressWarnings("DuplicatedCode") + protected Deserializer createResultDeserializer() { + + final ObjectReader objectReader = new ObjectMapper() + .setVisibility(VisibilityChecker.Std.defaultInstance().with(JsonAutoDetect.Visibility.ANY)) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) + .configure(SerializationFeature.CLOSE_CLOSEABLE, false) + .configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) + .addMixIn(BatchResult.class, BatchResultMixIn.class) + .addMixIn(Error.class, ErrorMixIn.class) + .addMixIn(BatchItemResult.class, BatchItemResultMixIn.class) + .readerFor(BatchResult.class); + + return new JacksonDeserializer<>(objectReader); + + } + + } + +} + diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCBatchOperations.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCBatchOperations.java index b69c8346..d0580754 100644 --- a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCBatchOperations.java +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCBatchOperations.java @@ -20,12 +20,9 @@ */ -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.MappingJsonFactory; import com.fasterxml.jackson.databind.ObjectWriter; import org.appenders.log4j2.elasticsearch.BatchBuilder; import org.appenders.log4j2.elasticsearch.BatchOperations; -import org.appenders.log4j2.elasticsearch.ExtendedObjectMapper; import org.appenders.log4j2.elasticsearch.ItemSource; import org.appenders.log4j2.elasticsearch.LifeCycle; import org.appenders.log4j2.elasticsearch.PooledItemSourceFactory; @@ -34,48 +31,68 @@ public class HCBatchOperations implements BatchOperations, LifeCyc private volatile State state = State.STOPPED; - private final PooledItemSourceFactory pooledItemSourceFactory; + protected final PooledItemSourceFactory batchBufferFactory; + private final ClientAPIFactory builderFactory; private final String mappingType; - private final ObjectWriter objectWriter; - public HCBatchOperations(PooledItemSourceFactory pooledItemSourceFactory, String mappingType) { - this.pooledItemSourceFactory = pooledItemSourceFactory; + /** + * @param batchBufferFactory batch buffer factory + * @param mappingType Elasticsearch mapping type + * @deprecated This constructor will be removed in future releases. Use {@link #HCBatchOperations(PooledItemSourceFactory, ClientAPIFactory)} instead + */ + @Deprecated + public HCBatchOperations(final PooledItemSourceFactory batchBufferFactory, final String mappingType) { + this.batchBufferFactory = batchBufferFactory; + this.builderFactory = new ElasticsearchBulkAPI(mappingType); + + // bad decisions pit.. this.mappingType = mappingType; - this.objectWriter = configuredWriter(); } - public HCBatchOperations(PooledItemSourceFactory pooledItemSourceFactory) { - this(pooledItemSourceFactory, "_doc"); + /** + * @param batchBufferFactory batch buffer factory + * @deprecated This constructor will be removed in future releases. Use {@link #HCBatchOperations(PooledItemSourceFactory, ClientAPIFactory)} instead + */ + @Deprecated + public HCBatchOperations(final PooledItemSourceFactory batchBufferFactory) { + this(batchBufferFactory, (String) null); } + public HCBatchOperations(final PooledItemSourceFactory batchBufferFactory, final ClientAPIFactory builderFactory) { + this.batchBufferFactory = batchBufferFactory; + this.builderFactory = builderFactory; + this.mappingType = null; // irrelevant in this context + } + + /** + * @return Elasticsearch mapping type + * @deprecated This method will be removed in future releases. Use {@link ClientAPIFactory} instead. + */ + @Deprecated public String getMappingType() { return mappingType; } @Override - public Object createBatchItem(String indexName, Object source) { + public Object createBatchItem(final String target, final Object source) { throw new UnsupportedOperationException("Use ItemSource based API instead"); } @Override - public Object createBatchItem(String indexName, ItemSource source) { - return new IndexRequest.Builder(source) - .index(indexName) - .type(mappingType) - .build(); + public Object createBatchItem(final String target, final ItemSource payload) { + return builderFactory.itemBuilder(target, payload).build(); } @Override public BatchBuilder createBatchBuilder() { return new BatchBuilder() { - private final BatchRequest.Builder builder = new BatchRequest.Builder() - .withBuffer(pooledItemSourceFactory.createEmptySource()) - .withObjectWriter(objectWriter); + private final BatchRequest.Builder builder = builderFactory.batchBuilder() + .withBuffer(batchBufferFactory.createEmptySource()); @Override public void add(Object item) { - builder.add((IndexRequest)item); + builder.add(item); } @Override @@ -88,13 +105,11 @@ public BatchRequest build() { /** * @return {@code com.fasterxml.jackson.databind.ObjectWriter} to serialize {@link IndexRequest} instances + * @deprecated This method will be removed along with {@link #HCBatchOperations(PooledItemSourceFactory, String)} constructor. Use {@link #HCBatchOperations(PooledItemSourceFactory, ClientAPIFactory)} instead. */ - // FIXME: design - wrap with Serializer(?) to allow other implementations + @Deprecated protected ObjectWriter configuredWriter() { - return new ExtendedObjectMapper(new MappingJsonFactory()) - .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) - .addMixIn(IndexRequest.class, IndexRequestMixIn.class) - .writerFor(IndexRequest.class); + throw new UnsupportedOperationException("Moved to ElasticsearchBulk or peer"); } @Override @@ -104,7 +119,7 @@ public void start() { return; } - pooledItemSourceFactory.start(); + batchBufferFactory.start(); state = State.STARTED; @@ -117,7 +132,7 @@ public void stop() { return; } - pooledItemSourceFactory.stop(); + batchBufferFactory.stop(); state = State.STOPPED; diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCHttp.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCHttp.java index b065201d..94a67b36 100644 --- a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCHttp.java +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCHttp.java @@ -20,13 +20,6 @@ */ -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.introspect.VisibilityChecker; import org.appenders.log4j2.elasticsearch.BatchOperations; import org.appenders.log4j2.elasticsearch.LifeCycle; import org.appenders.log4j2.elasticsearch.OperationFactory; @@ -48,13 +41,10 @@ public class HCHttp extends BatchingClientObjectFactory batchOperations; protected final OperationFactory operationFactory; - private final ObjectReader objectReader; - public HCHttp(Builder builder) { super(builder); this.batchOperations = builder.batchOperations; this.operationFactory = builder.operationFactory; - this.objectReader = configuredReader(); } @Override @@ -67,24 +57,6 @@ public OperationFactory setupOperationFactory() { return operationFactory; } - /** - * @return {@code com.fasterxml.jackson.databind.ObjectReader} to deserialize {@link BatchResult} - * @deprecated This method will be removed in future releases (not earlier than 1.6) - */ - @Deprecated - protected ObjectReader configuredReader() { - // TODO: Inject..? - return new ObjectMapper() - .setVisibility(VisibilityChecker.Std.defaultInstance().with(JsonAutoDetect.Visibility.ANY)) - .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) - .configure(SerializationFeature.CLOSE_CLOSEABLE, false) - .configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) - .addMixIn(BatchResult.class, BatchResultMixIn.class) - .addMixIn(Error.class, ErrorMixIn.class) - .addMixIn(BatchItemResult.class, BatchItemResultMixIn.class) - .readerFor(BatchResult.class); - } - protected ResponseHandler createResultHandler(BatchRequest request, Function failureHandler) { return new HCResponseHandler(request, failureHandler); } @@ -166,7 +138,7 @@ private class HCResponseHandler implements ResponseHandler { private final BatchRequest request; private final Function failureHandler; - public HCResponseHandler(BatchRequest request, Function failureHandler) { + public HCResponseHandler(final BatchRequest request, final Function failureHandler) { this.request = request; this.failureHandler = failureHandler; @@ -201,8 +173,8 @@ public void failed(Exception ex) { } @Override - public BatchResult deserializeResponse(InputStream responseBody) throws IOException { - return objectReader.readValue(responseBody); + public BatchResult deserializeResponse(final InputStream responseBody) throws IOException { + return request.deserialize(responseBody); } } diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCHttpPlugin.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCHttpPlugin.java index dbf3baed..155ca9da 100644 --- a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCHttpPlugin.java +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCHttpPlugin.java @@ -98,8 +98,12 @@ public static class Builder implements org.apache.logging.log4j.core.util.Builde @PluginElement(ItemSourceFactory.ELEMENT_TYPE) protected PooledItemSourceFactory pooledItemSourceFactory; + /** + * @deprecated This field will be removed in future releases. Use {@link ClientAPIFactory} + */ @PluginBuilderAttribute - protected String mappingType = "_doc"; + @Deprecated + protected String mappingType; @PluginElement(BackoffPolicy.NAME) protected BackoffPolicy backoffPolicy; @@ -109,6 +113,9 @@ public static class Builder implements org.apache.logging.log4j.core.util.Builde protected ValueResolver valueResolver; + @PluginElement("clientAPIFactory") + private ClientAPIFactory clientAPIFactory; + @Override public HCHttpPlugin build() { @@ -186,7 +193,13 @@ private HCBatchOperations createBatchOperations() { if (pooledItemSourceFactory == null) { throw new IllegalArgumentException(String.format("No %s provided for %s", PooledItemSourceFactory.class.getSimpleName(), HCHttp.class.getSimpleName())); } - return new HCBatchOperations(pooledItemSourceFactory, mappingType); + + if (clientAPIFactory == null) { + return new HCBatchOperations(pooledItemSourceFactory, mappingType); + } else { + return new HCBatchOperations(pooledItemSourceFactory, clientAPIFactory); + } + } public Builder withServerUris(String serverUris) { @@ -229,11 +242,22 @@ public Builder withAuth(Auth auth) { return this; } + /** + * @param mappingType Elasticsearch mapping type + * @return this + * @deprecated This method will be removed in future released. Use {@link #withClientAPIFactory(ClientAPIFactory)} instead. + */ + @Deprecated public Builder withMappingType(String mappingType) { this.mappingType = mappingType; return this; } + public Builder withClientAPIFactory(ClientAPIFactory clientAPIFactory) { + this.clientAPIFactory = clientAPIFactory; + return this; + } + public Builder withPooledResponseBuffers(boolean pooledResponseBuffersEnabled) { this.pooledResponseBuffers = pooledResponseBuffersEnabled; return this; diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCRequestFactory.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCRequestFactory.java index e09f6a8c..e045c3da 100644 --- a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCRequestFactory.java +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HCRequestFactory.java @@ -29,6 +29,7 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.entity.ContentType; +import org.appenders.log4j2.elasticsearch.ItemSource; import java.io.IOException; @@ -64,13 +65,20 @@ public HttpUriRequest create(String url, Request request) throws IOException { protected HttpEntity createHttpEntity(Request request) throws IOException { - ByteBuf byteBuf = (ByteBuf) request.serialize().getSource(); + try { + final ItemSource serialize = request.serialize(); + final ByteBuf byteBuf = (ByteBuf) serialize.getSource(); - return new ByteBufEntityBuilder() - .setByteBuf(byteBuf) - .setContentLength(byteBuf.writerIndex()) - .setContentType(requestContentType) - .build(); + return new ByteBufEntityBuilder() + .setByteBuf(byteBuf) + .setContentLength(byteBuf.writerIndex()) + .setContentType(requestContentType) + .build(); + + } catch (Exception e) { + // repackage for HTTP stack for now + throw new IOException("Unable to create HTTP entity", e); + } } diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HttpClientProvider.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HttpClientProvider.java index 2a55f5cf..e78124fd 100644 --- a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HttpClientProvider.java +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/HttpClientProvider.java @@ -22,7 +22,6 @@ import org.appenders.log4j2.elasticsearch.ClientProvider; import org.appenders.log4j2.elasticsearch.LifeCycle; -import org.appenders.log4j2.elasticsearch.hc.discovery.HCServiceDiscovery; import static org.appenders.core.logging.InternalLogging.getLogger; diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/IndexRequest.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/IndexRequest.java index 12b30467..b191f535 100644 --- a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/IndexRequest.java +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/IndexRequest.java @@ -119,10 +119,6 @@ protected void validate() { throw new IllegalArgumentException("index cannot be null"); } - if (type == null) { - throw new IllegalArgumentException("type cannot be null"); - } - } } diff --git a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/Request.java b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/Request.java index 4dd5a2f1..21f1114c 100644 --- a/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/Request.java +++ b/log4j2-elasticsearch-hc/src/main/java/org/appenders/log4j2/elasticsearch/hc/Request.java @@ -22,14 +22,12 @@ import org.appenders.log4j2.elasticsearch.ItemSource; -import java.io.IOException; - public interface Request { String getURI(); String getHttpMethodName(); - ItemSource serialize() throws IOException; + ItemSource serialize() throws Exception; } diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCBatchOperationsTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/AbstractHCBatchOperationsTest.java similarity index 88% rename from log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCBatchOperationsTest.java rename to log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/AbstractHCBatchOperationsTest.java index 58b8e194..4954ac22 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCBatchOperationsTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/AbstractHCBatchOperationsTest.java @@ -21,7 +21,6 @@ */ import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufInputStream; import org.apache.logging.log4j.core.LoggerContext; @@ -35,10 +34,10 @@ import org.appenders.log4j2.elasticsearch.PooledItemSourceFactoryTest; import org.junit.jupiter.api.Test; -import java.io.IOException; import java.util.Scanner; import java.util.UUID; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -50,7 +49,11 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; -public class HCBatchOperationsTest { +public abstract class AbstractHCBatchOperationsTest { + + abstract public HCBatchOperations createDefaultBatchOperations(PooledItemSourceFactory itemSourceFactory); + + abstract public HCBatchOperations createDefaultBatchOperations(PooledItemSourceFactory itemSourceFactory, String mappingType); @Test public void throwsOnStringSource() { @@ -70,14 +73,6 @@ public void throwsOnStringSource() { } - private HCBatchOperations createDefaultBatchOperations(PooledItemSourceFactory itemSourceFactory) { - return createDefaultBatchOperations(itemSourceFactory, "_doc"); - } - - private HCBatchOperations createDefaultBatchOperations(PooledItemSourceFactory itemSourceFactory, String mappingType) { - return new HCBatchOperations(itemSourceFactory, mappingType); - } - @Test public void createsBatchBuilder() { @@ -93,22 +88,22 @@ public void createsBatchBuilder() { } @Test - public void createsConfiguredWriter() { + public void throwsOnDeprecatedConfiguredWriter() { // given - PooledItemSourceFactory itemSourceFactory = PooledItemSourceFactoryTest.createDefaultTestSourceFactoryConfig().build(); - HCBatchOperations batchOperations = createDefaultBatchOperations(itemSourceFactory, null); + final PooledItemSourceFactory itemSourceFactory = PooledItemSourceFactoryTest.createDefaultTestSourceFactoryConfig().build(); + final HCBatchOperations batchOperations = createDefaultBatchOperations(itemSourceFactory, null); // when - ObjectWriter writer = batchOperations.configuredWriter(); + final UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, batchOperations::configuredWriter); // then - assertNotNull(writer); + assertThat(exception.getMessage(), containsString("Moved to ElasticsearchBulk")); } @Test - public void defaultWriterCanSerializeBatchRequest() throws IOException { + public void defaultWriterCanSerializeBatchRequest() throws Exception { // given PooledItemSourceFactory itemSourceFactory = PooledItemSourceFactoryTest.createDefaultTestSourceFactoryConfig().build(); diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/BatchRequestTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/BatchRequestTest.java index 1eb399d2..30c0135b 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/BatchRequestTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/BatchRequestTest.java @@ -24,12 +24,13 @@ import com.fasterxml.jackson.databind.ObjectWriter; import io.netty.buffer.ByteBuf; import org.appenders.log4j2.elasticsearch.ByteBufItemSource; +import org.appenders.log4j2.elasticsearch.Deserializer; import org.appenders.log4j2.elasticsearch.ItemSource; +import org.appenders.log4j2.elasticsearch.JacksonSerializer; +import org.appenders.log4j2.elasticsearch.Serializer; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -import java.io.IOException; -import java.io.OutputStream; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -52,6 +53,21 @@ public abstract class BatchRequestTest { + @SuppressWarnings("unchecked") + public static BatchRequest createTestBatch(final BatchRequest.Builder builder, final ItemSource... payloads) { + + //noinspection unchecked + builder.withBuffer(createTestItemSource()) + .withItemSerializer(mock(Serializer.class)) + .withResultDeserializer(mock(Deserializer.class)); + + for (ItemSource payload : payloads) { + builder.add(createIndexRequestBuilder(payload) + .build()); + } + return spy(builder.build()); + } + @Test public void builderBuildsSuccessfully() { @@ -63,6 +79,8 @@ public void builderBuildsSuccessfully() { // then assertNotNull(batchRequest); + assertEquals("/_bulk", batchRequest.getURI()); + assertEquals("POST", batchRequest.getHttpMethodName()); } @@ -83,28 +101,74 @@ public void builderFailsWhenBufferIsNull() { } @Test - public void builderFailsWhenObjectWriterIsNull() { + public void builderFailsWhenSerializerIsNull() { // given - BatchRequest.Builder builder = createDefaultTestObjectBuilder(); + final BatchRequest.Builder builder = createDefaultTestObjectBuilder(); + builder.withItemSerializer(null); - builder.withObjectWriter(null); + // when + final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, builder::build); + + // then + assertThat(exception.getMessage(), containsString("itemSerializer cannot be null")); + + } + + @Test + public void builderFailsWhenResultDeserializerIsNull() { + + // given + final BatchRequest.Builder builder = createDefaultTestObjectBuilder(); + builder.withResultDeserializer(null); // when final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, builder::build); + // then + assertThat(exception.getMessage(), containsString("resultDeserializer cannot be null")); + + } + + @Test + public void builderFailsWhenObjectWriterIsNull() { + + // given + final BatchRequest.Builder builder = createDefaultTestObjectBuilder(); + + // when + final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> builder.withObjectWriter(null)); + // then assertThat(exception.getMessage(), containsString("objectWriter cannot be null")); } + @Test + public void builderSupportsLegacyObjectWriterSetter() { + + // given + final BatchRequest.Builder builder = createDefaultTestObjectBuilder() + .withItemSerializer(null); + + final ObjectWriter objectWriter = mock(ObjectWriter.class); + + // when + builder.withObjectWriter(objectWriter); + final BatchRequest batchRequest = builder.build(); + + // then + assertNotNull(batchRequest); + + } + @Test public void builderCanStoreAction() { // given BatchRequest.Builder builder = createDefaultTestObjectBuilder(); - builder.withObjectWriter(null); + builder.withItemSerializer(null); // when builder.add(mock(IndexRequest.class)); @@ -135,29 +199,31 @@ public void builderCanStoreMultipleActionsAtOnce() { } @Test - public void canSerializeUniqueItemsSeparately() throws IOException { + public void canSerializeUniqueItemsSeparately() throws Exception { // given - ObjectWriter writer = spy(new ObjectMapper().writerFor(IndexRequest.class)); + final Serializer serializer = spy(new JacksonSerializer<>(new ObjectMapper().writerFor(IndexRequest.class))); - ItemSource source1 = createTestItemSource(); - String index1 = UUID.randomUUID().toString(); - String mappingType = UUID.randomUUID().toString(); - IndexRequest action1 = createIndexRequestBuilder(source1) + final ItemSource source1 = createTestItemSource(); + final String index1 = UUID.randomUUID().toString(); + final String mappingType = UUID.randomUUID().toString(); + final IndexRequest action1 = createIndexRequestBuilder(source1) .index(index1) .type(mappingType) .build(); - ItemSource source2 = createTestItemSource(); - String index2 = UUID.randomUUID().toString(); + final ItemSource source2 = createTestItemSource(); + final String index2 = UUID.randomUUID().toString(); - IndexRequest action2 = createIndexRequestBuilder(source2) + final IndexRequest action2 = createIndexRequestBuilder(source2) .index(index2) .type(mappingType) .build(); - BatchRequest request = new BatchRequest.Builder() - .withObjectWriter(writer) + @SuppressWarnings("unchecked") + final BatchRequest request = new BatchRequest.Builder() + .withItemSerializer(serializer) + .withResultDeserializer(mock(Deserializer.class)) .withBuffer(createTestItemSource()) .add(action1) .add(action2) @@ -167,9 +233,9 @@ public void canSerializeUniqueItemsSeparately() throws IOException { request.serialize(); // then - ArgumentCaptor captor = ArgumentCaptor.forClass(IndexRequest.class); - verify(writer, times(2)).writeValue((OutputStream)any(), captor.capture()); - List allValues = captor.getAllValues(); + final ArgumentCaptor captor = ArgumentCaptor.forClass(IndexRequest.class); + verify(serializer, times(2)).write(any(), captor.capture()); + final List allValues = captor.getAllValues(); assertEquals(2, allValues.size()); assertEquals(index1, allValues.get(0).getIndex()); assertEquals(index2, allValues.get(1).getIndex()); @@ -297,39 +363,30 @@ public void itemsAreNotIdenticalIfBothIndicesAndTypesAreDifferent() { } - public static BatchRequest createTestBatch(BatchRequest.Builder builder, ItemSource... payloads) { - builder.withBuffer(createTestItemSource()); - builder.withObjectWriter(mock(ObjectWriter.class)); - - for (ItemSource payload : payloads) { - builder.add(createIndexRequestBuilder(payload) - .build()); - } - return spy(builder.build()); - } - @Test - public void canSerializeOnceIfAllItemsAreTheSame() throws IOException { + public void canSerializeOnceIfAllItemsAreTheSame() throws Exception { // given - ObjectWriter writer = spy(new ObjectMapper().writerFor(IndexRequest.class)); + final Serializer serializer = spy(new JacksonSerializer<>(new ObjectMapper().writerFor(IndexRequest.class))); - ItemSource source1 = createTestItemSource(); - String index = UUID.randomUUID().toString(); - String mappingType = UUID.randomUUID().toString(); - IndexRequest action1 = createIndexRequestBuilder(source1) + final ItemSource source1 = createTestItemSource(); + final String index = UUID.randomUUID().toString(); + final String mappingType = UUID.randomUUID().toString(); + final IndexRequest action1 = createIndexRequestBuilder(source1) .index(index) .type(mappingType) .build(); - ItemSource source2 = createTestItemSource(); - IndexRequest action2 = createIndexRequestBuilder(source2) + final ItemSource source2 = createTestItemSource(); + final IndexRequest action2 = createIndexRequestBuilder(source2) .index(index) .type(mappingType) .build(); - BatchRequest batchRequest = new BatchRequest.Builder() - .withObjectWriter(writer) + @SuppressWarnings("unchecked") + final BatchRequest batchRequest = new BatchRequest.Builder() + .withItemSerializer(serializer) + .withResultDeserializer(mock(Deserializer.class)) .withBuffer(createTestItemSource()) .add(action1) .add(action2) @@ -339,10 +396,10 @@ public void canSerializeOnceIfAllItemsAreTheSame() throws IOException { batchRequest.serialize(); // then - ArgumentCaptor captor = ArgumentCaptor.forClass(IndexRequest.class); - verify(writer, times(1)).writeValueAsBytes(captor.capture()); + final ArgumentCaptor captor = ArgumentCaptor.forClass(IndexRequest.class); + verify(serializer, times(1)).writeAsBytes(captor.capture()); - List allValues = captor.getAllValues(); + final List allValues = captor.getAllValues(); assertEquals(1, allValues.size()); assertEquals(index, allValues.get(0).getIndex()); @@ -393,8 +450,10 @@ public void callingCompletedReleasesActions() { } public static BatchRequest.Builder createDefaultTestObjectBuilder() { + //noinspection unchecked return new BatchRequest.Builder() - .withObjectWriter(mock(ObjectWriter.class)) + .withItemSerializer(mock(Serializer.class)) + .withResultDeserializer(mock(Deserializer.class)) .withBuffer(mock(ByteBufItemSource.class)); } diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/CheckBootstrapIndexTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/CheckBootstrapIndexTest.java index 3090b528..2eca0a9f 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/CheckBootstrapIndexTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/CheckBootstrapIndexTest.java @@ -27,8 +27,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import java.io.IOException; - import static org.appenders.core.logging.InternalLogging.setLogger; import static org.appenders.core.logging.InternalLoggingTest.mockTestLogger; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -163,7 +161,7 @@ public void onResponseCodeZeroLogsAndFails() { } @Test - public void createsGenericRequest() throws IOException { + public void createsGenericRequest() throws Exception { // given CheckBootstrapIndex setupStep = new CheckBootstrapIndex(TEST_ROLLOVER_ALIAS); diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactoryBasedHCBatchOperationsTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactoryBasedHCBatchOperationsTest.java new file mode 100644 index 00000000..d7b90648 --- /dev/null +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ClientAPIFactoryBasedHCBatchOperationsTest.java @@ -0,0 +1,55 @@ +package org.appenders.log4j2.elasticsearch.hc; + +/*- + * #%L + * log4j2-elasticsearch + * %% + * Copyright (C) 2022 Rafal Foltynski + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.appenders.log4j2.elasticsearch.PooledItemSourceFactory; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +public class ClientAPIFactoryBasedHCBatchOperationsTest extends AbstractHCBatchOperationsTest { + + public HCBatchOperations createDefaultBatchOperations(PooledItemSourceFactory itemSourceFactory) { + return createDefaultBatchOperations(itemSourceFactory, null); + } + + public HCBatchOperations createDefaultBatchOperations(PooledItemSourceFactory itemSourceFactory, String mappingType) { + final ElasticsearchBulkPlugin clientAPIFactory = ElasticsearchBulkPlugin.newBuilder().withMappingType(mappingType).build(); + return new HCBatchOperations(itemSourceFactory, clientAPIFactory); + } + + @Test + public void mappingTypeIsNullWithClientAPIFactoryBasedConstructor() { + + // given + final PooledItemSourceFactory itemSourceFactory = mock(PooledItemSourceFactory.class); + final HCBatchOperations batchOperations = createDefaultBatchOperations(itemSourceFactory); + + // when + final String result = batchOperations.getMappingType(); + + // then + assertNull(result); + + } + +} diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkAPITest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkAPITest.java new file mode 100644 index 00000000..a6ebfd5f --- /dev/null +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkAPITest.java @@ -0,0 +1,126 @@ +package org.appenders.log4j2.elasticsearch.hc; + +/*- + * #%L + * log4j2-elasticsearch + * %% + * Copyright (C) 2022 Rafal Foltynski + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.CompositeByteBuf; +import org.appenders.log4j2.elasticsearch.ByteBufItemSourceTest; +import org.appenders.log4j2.elasticsearch.Deserializer; +import org.appenders.log4j2.elasticsearch.IndexNamePluginTest; +import org.appenders.log4j2.elasticsearch.ItemSource; +import org.appenders.log4j2.elasticsearch.Serializer; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +public class ElasticsearchBulkAPITest { + + @Test + public void createsIndexRequestBuilder() { + + // given + final String payloadString = UUID.randomUUID().toString(); + final ItemSource payload = createTestItemSource(payloadString); + + @SuppressWarnings("unchecked") final Serializer serializer = mock(Serializer.class); + final Deserializer deserializer = mock(Deserializer.class); + + final String mappingType = UUID.randomUUID().toString(); + final ElasticsearchBulkAPI builder = new ElasticsearchBulkAPI(mappingType, serializer, deserializer); + + final String target = IndexNamePluginTest.TEST_INDEX_NAME; + + // when + final IndexRequest.Builder requestBuilder = builder.itemBuilder(target, payload); + final IndexRequest request = requestBuilder.build(); + + // then + assertNotNull(request); + assertEquals(target, request.index); + assertEquals(payload, request.source); + + } + + @Test + public void createsBatchRequestBuilder() throws Exception { + + // given + final Serializer serializer = spy(ElasticsearchBulkPlugin.newBuilder().createItemSerializer()); + final Deserializer deserializer = mock(Deserializer.class); + + final String mappingType = UUID.randomUUID().toString(); + final ElasticsearchBulkAPI builder = new ElasticsearchBulkAPI(mappingType, serializer, deserializer); + + final ItemSource batchBuffer = createDefaultTestBatchBuffer(); + + final String target = IndexNamePluginTest.TEST_INDEX_NAME; + + final String payloadString = UUID.randomUUID().toString(); + final ItemSource payload = createTestItemSource(payloadString); + final IndexRequest.Builder indexRequestBuilder = builder.itemBuilder(target, payload); + final IndexRequest indexRequest = spy(indexRequestBuilder.build()); + + // when + final BatchRequest.Builder requestBuilder = builder.batchBuilder(); + requestBuilder.withBuffer(batchBuffer); + requestBuilder.add(indexRequest); + + final BatchRequest request = requestBuilder.build(); + final ItemSource serialized = request.serialize(); + + // then + verify(serializer).writeAsBytes(eq(indexRequest)); + + final ByteBuf source = (ByteBuf) serialized.getSource(); + final String batchString = source.toString(StandardCharsets.UTF_8); + + assertThat(batchString, containsString(target)); + assertThat(batchString, containsString(mappingType)); + assertThat(batchString, containsString(payloadString)); + + } + + ItemSource createTestItemSource(final String payloadString) { + + final CompositeByteBuf buffer = ByteBufItemSourceTest.createDefaultTestByteBuf(); + buffer.writeBytes(payloadString.getBytes(StandardCharsets.UTF_8)); + + return ByteBufItemSourceTest.createTestItemSource(buffer, itemSource -> {}); + + } + + private ItemSource createDefaultTestBatchBuffer() { + final CompositeByteBuf batchByteBuf = ByteBufItemSourceTest.createDefaultTestByteBuf(); + return ByteBufItemSourceTest.createTestItemSource(batchByteBuf, itemSource -> {}); + } + +} diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPluginTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPluginTest.java new file mode 100644 index 00000000..25fc25e0 --- /dev/null +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/ElasticsearchBulkPluginTest.java @@ -0,0 +1,135 @@ +package org.appenders.log4j2.elasticsearch.hc; + +/*- + * #%L + * log4j2-elasticsearch + * %% + * Copyright (C) 2022 Rafal Foltynski + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.apache.logging.log4j.core.config.ConfigurationException; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class ElasticsearchBulkPluginTest extends ElasticsearchBulkAPITest { + + public static final String TEST_PAYLOAD_STRING = "{}"; + public static final String TEST_INDEX_NAME = UUID.randomUUID().toString(); + + @Test + public void defaultBuilderBuildsSuccessfully() { + + // given + final ElasticsearchBulkPlugin.Builder builder = ElasticsearchBulkPlugin.newBuilder(); + + // when + final ElasticsearchBulkPlugin plugin = builder.build(); + + // then + assertNotNull(plugin); + + } + + @Test + public void defaultBuilderDoesNotSetMappingType() { + + // given + final ElasticsearchBulkPlugin.Builder builder = ElasticsearchBulkPlugin.newBuilder(); + + final ElasticsearchBulkPlugin plugin = builder.build(); + + // when + final IndexRequest indexRequest = plugin.itemBuilder(TEST_INDEX_NAME, createTestItemSource(TEST_PAYLOAD_STRING)).build(); + + // then + assertNull(indexRequest.type); + + } + + @Test + public void builderBuildsWhenMappingTypeIsNull() { + + // given + final ElasticsearchBulkPlugin.Builder builder = ElasticsearchBulkPlugin.newBuilder() + .withMappingType(null); + + final ElasticsearchBulkPlugin plugin = builder.build(); + + // when + final IndexRequest indexRequest = plugin.itemBuilder(TEST_INDEX_NAME, createTestItemSource(TEST_PAYLOAD_STRING)).build(); + + // then + assertNull(indexRequest.type); + + } + + @Test + public void builderSetsConfiguredMappingType() { + + // given + final String expectedMappingType = UUID.randomUUID().toString(); + final ElasticsearchBulkPlugin.Builder builder = ElasticsearchBulkPlugin.newBuilder() + .withMappingType(expectedMappingType); + + final ElasticsearchBulkPlugin plugin = builder.build(); + + // when + final IndexRequest indexRequest = plugin.itemBuilder(TEST_INDEX_NAME, createTestItemSource(TEST_PAYLOAD_STRING)).build(); + + // then + assertEquals(expectedMappingType, indexRequest.type); + + } + + @Test + public void builderThrowsWhenItemSerializerIsNull() { + + // given + final ElasticsearchBulkPlugin.Builder builder = ElasticsearchBulkPlugin.newBuilder() + .withItemSerializer(null); + + // when + final ConfigurationException exception = assertThrows(ConfigurationException.class, builder::build); + + // then + assertThat(exception.getMessage(), containsString("itemSerializer cannot be null")); + + } + + @Test + public void builderThrowsWhenResultDeserializerIsNull() { + + // given + final ElasticsearchBulkPlugin.Builder builder = ElasticsearchBulkPlugin.newBuilder() + .withResultDeserializer(null); + + // when + final ConfigurationException exception = assertThrows(ConfigurationException.class, builder::build); + + // then + assertThat(exception.getMessage(), containsString("resultDeserializer cannot be null")); + + } + +} diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCHttpPluginTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCHttpPluginTest.java index 98e38faa..841ab347 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCHttpPluginTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCHttpPluginTest.java @@ -23,7 +23,9 @@ import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.lookup.StrSubstitutor; import org.appenders.log4j2.elasticsearch.Auth; +import org.appenders.log4j2.elasticsearch.ByteBufItemSourceTest; import org.appenders.log4j2.elasticsearch.IndexTemplate; +import org.appenders.log4j2.elasticsearch.ItemSourceFactory; import org.appenders.log4j2.elasticsearch.PooledItemSourceFactory; import org.appenders.log4j2.elasticsearch.PooledItemSourceFactoryTest; import org.appenders.log4j2.elasticsearch.ValueResolver; @@ -51,6 +53,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -226,4 +229,28 @@ public void serviceDiscoveryIsUsedIfConfigured() { } + @Test + public void clientAPIFactoryIsUsedIfConfigured() { + + // given + final ClientAPIFactory clientAPIFactory = mock(ClientAPIFactory.class); + when(clientAPIFactory.batchBuilder()).thenReturn(new BatchRequest.Builder()); + + final ItemSourceFactory itemSourceFactory = mock(ItemSourceFactory.class); + when(itemSourceFactory.createEmptySource()).thenReturn(ByteBufItemSourceTest.createTestItemSource()); + final HCHttpPlugin.Builder builder = spy(createDefaultHttpObjectFactoryBuilder()) + .withClientAPIFactory(clientAPIFactory); + + verify(clientAPIFactory, never()).batchBuilder(); + + // when + final HCHttpPlugin plugin = builder.build(); + plugin.start(); + plugin.batchOperations.createBatchBuilder(); + + // then + verify(clientAPIFactory).batchBuilder(); + + } + } diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCHttpTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCHttpTest.java index 8b9b7bb2..f33c216a 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCHttpTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCHttpTest.java @@ -20,7 +20,6 @@ * #L% */ -import com.fasterxml.jackson.databind.ObjectReader; import io.netty.buffer.ByteBuf; import io.netty.buffer.CompositeByteBuf; import org.appenders.log4j2.elasticsearch.Auth; @@ -28,6 +27,7 @@ import org.appenders.log4j2.elasticsearch.ByteBufItemSourceTest; import org.appenders.log4j2.elasticsearch.ClientObjectFactory; import org.appenders.log4j2.elasticsearch.ClientProvider; +import org.appenders.log4j2.elasticsearch.Deserializer; import org.appenders.log4j2.elasticsearch.FailoverPolicy; import org.appenders.log4j2.elasticsearch.ItemSource; import org.appenders.log4j2.elasticsearch.LifeCycle; @@ -96,9 +96,10 @@ public static HCHttp.Builder createDefaultHttpObjectFactoryBuilder() { .createDefaultTestSourceFactoryConfig() .build(); + return new HCHttp.Builder() .withOperationFactory(new ElasticsearchOperationFactory(step -> Result.SUCCESS, ValueResolver.NO_OP)) - .withBatchOperations(new HCBatchOperations(itemSourceFactory)) + .withBatchOperations(new HCBatchOperations(itemSourceFactory, new ElasticsearchBulkAPI(null))) .withClientProvider(HttpClientProviderTest.createDefaultTestClientProvider()); } @@ -293,29 +294,35 @@ public void authIsNotAppliedIfNull() { } @Test - public void resultHandlerUsesGivenObjectReader() throws IOException { + public void resultHandlerUsesConfiguredResponseDeserializer() throws IOException { // given - ObjectReader mockedObjectReader = mock(ObjectReader.class); - HCHttp factory = new HCHttp(createDefaultHttpObjectFactoryBuilder()) { + final PooledItemSourceFactory itemSourceFactory = PooledItemSourceFactoryTest + .createDefaultTestSourceFactoryConfig() + .build(); + + final Deserializer deserializer = mock(Deserializer.class); + final HCBatchOperations batchOperations = new HCBatchOperations(itemSourceFactory, new ElasticsearchBulkAPI(null) { @Override - protected ObjectReader configuredReader() { - return mockedObjectReader; + protected Deserializer createResultDeserializer() { + return deserializer; } - }; + }); + final HCHttp factory = new HCHttp(createDefaultHttpObjectFactoryBuilder() + .withBatchOperations(batchOperations)); - ResponseHandler resultHandler = factory.createResultHandler( - BatchRequestTest.createDefaultTestObjectBuilder().build(), + final ResponseHandler resultHandler = factory.createResultHandler( + batchOperations.createBatchBuilder().build(), batchRequest -> true ); - InputStream inputStream = mock(InputStream.class); + final InputStream inputStream = mock(InputStream.class); // when resultHandler.deserializeResponse(inputStream); // then - verify(mockedObjectReader).readValue(eq(inputStream)); + verify(deserializer).read(eq(inputStream)); } @Test diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCRequestFactoryTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCRequestFactoryTest.java index c801b597..ab66009d 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCRequestFactoryTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HCRequestFactoryTest.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.net.URISyntaxException; import java.util.UUID; import static org.appenders.log4j2.elasticsearch.GenericItemSourcePoolTest.byteBufAllocator; @@ -49,7 +48,7 @@ public class HCRequestFactoryTest { @Test - public void throwsOnUnknownHttpMethodName() throws IOException { + public void throwsOnUnknownHttpMethodName() throws Exception { // given HCRequestFactory factory = createDefaultTestObject(); @@ -66,7 +65,26 @@ public void throwsOnUnknownHttpMethodName() throws IOException { } @Test - public void createsPostRequest() throws IOException, URISyntaxException { + public void throwsOnSerializationExceptions() throws Exception { + + // given + final HCRequestFactory factory = createDefaultTestObject(); + final String expectedUrl = UUID.randomUUID().toString(); + final String httpMethodName = UUID.randomUUID().toString(); + final Request request = createDefaultMockRequest(expectedUrl, httpMethodName); + when(request.serialize()).thenThrow(new Exception("Cannot serialize")); + + // when + final IOException exception = assertThrows(IOException.class, () -> factory.createHttpEntity(request)); + + // then + assertThat(exception.getMessage(), containsString("Unable to create HTTP entity")); + assertThat(exception.getCause().getMessage(), containsString("Cannot serialize")); + + } + + @Test + public void createsPostRequest() throws Exception { // given HCRequestFactory factory = createDefaultTestObject(); @@ -83,7 +101,7 @@ public void createsPostRequest() throws IOException, URISyntaxException { } @Test - public void createsPutRequest() throws IOException, URISyntaxException { + public void createsPutRequest() throws Exception { // given HCRequestFactory factory = createDefaultTestObject(); @@ -100,7 +118,7 @@ public void createsPutRequest() throws IOException, URISyntaxException { } @Test - public void createsHeadRequest() throws IOException, URISyntaxException { + public void createsHeadRequest() throws Exception { // given HCRequestFactory factory = createDefaultTestObject(); @@ -117,7 +135,7 @@ public void createsHeadRequest() throws IOException, URISyntaxException { } @Test - public void createsGetRequest() throws IOException, URISyntaxException { + public void createsGetRequest() throws Exception { // given HCRequestFactory factory = createDefaultTestObject(); @@ -134,7 +152,7 @@ public void createsGetRequest() throws IOException, URISyntaxException { } @Test - public void createsEntityUsingGivenSource() throws IOException { + public void createsEntityUsingGivenSource() throws Exception { // given HCRequestFactory factory = createDefaultTestObject(); @@ -163,7 +181,7 @@ public void createsEntityUsingGivenSource() throws IOException { } - public static Request createDefaultMockRequest(String expectedUrl, String httpMethodName) throws IOException { + public static Request createDefaultMockRequest(String expectedUrl, String httpMethodName) throws Exception { Request request = mock(Request.class); when(request.getURI()).thenReturn(expectedUrl); when(request.getHttpMethodName()).thenReturn(httpMethodName); diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HttpClientTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HttpClientTest.java index 09001d2e..3177580e 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HttpClientTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/HttpClientTest.java @@ -252,7 +252,7 @@ public void executeAsyncDelegatesToConfiguredAsyncClient() { } @Test - public void executeAsyncCallbackCallsResultHandlerCompleted() throws IOException { + public void executeAsyncCallbackCallsResultHandlerCompleted() throws Exception { // given ResponseHandler responseHandler = createMockTestResultHandler(); @@ -282,7 +282,7 @@ public void executeAsyncCallbackCallsResultHandlerCompleted() throws IOException } @Test - public void executeAsyncCallbackCallsResultHandlerFailedOnIOException() throws IOException { + public void executeAsyncCallbackCallsResultHandlerFailedOnIOException() throws Exception { // given ResponseHandler responseHandler = createMockTestResultHandler(); @@ -310,7 +310,7 @@ public void executeAsyncCallbackCallsResultHandlerFailedOnIOException() throws I } @Test - public void executeAsyncCallbackCallsResultHandlerFailedOnThrowable() throws IOException { + public void executeAsyncCallbackCallsResultHandlerFailedOnThrowable() throws Exception { // given ResponseHandler responseHandler = mock(ResponseHandler.class); @@ -341,7 +341,7 @@ public void executeAsyncCallbackCallsResultHandlerFailedOnThrowable() throws IOE } @Test - public void executeAsyncCallbackCallsResultHandlerCompletedOnInputStreamCloseException() throws IOException { + public void executeAsyncCallbackCallsResultHandlerCompletedOnInputStreamCloseException() throws Exception { // given ResponseHandler responseHandler = createMockTestResultHandler(); @@ -378,7 +378,7 @@ public void close() throws IOException { } @Test - public void executeAsyncCallbackCallsResultHandlerWhenCancelled() throws IOException { + public void executeAsyncCallbackCallsResultHandlerWhenCancelled() throws Exception { // given ResponseHandler responseHandler = createMockTestResultHandler(); @@ -421,7 +421,7 @@ public void executeAsyncCallbackDoesNotRethrowOnResponseHandlerExceptions() { } @Test - public void executeAsyncCallbackHandlesHttpResponse() throws IOException { + public void executeAsyncCallbackHandlesHttpResponse() throws Exception { // given ResponseHandler responseHandler = createMockTestResultHandler(); @@ -449,7 +449,7 @@ public void executeAsyncCallbackHandlesHttpResponse() throws IOException { } @Test - public void executeAsyncCallbackHandlesNonSuccessfulResponse() throws IOException { + public void executeAsyncCallbackHandlesNonSuccessfulResponse() throws Exception { // given ResponseHandler responseHandler = createMockTestResultHandler(); @@ -658,7 +658,7 @@ private HttpResponse createDefaultTestHttpResponse() { return httpResponse; } - private HCResultCallback mockHttpResponseCallback(ResponseHandler responseHandler) throws IOException { + private HCResultCallback mockHttpResponseCallback(ResponseHandler responseHandler) throws Exception { HttpClient client = Mockito.spy(createDefaultTestObject()); CloseableHttpAsyncClient asyncClient = mockAsyncClient(client); diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/IndexRequestTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/IndexRequestTest.java index 6c15b424..08035391 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/IndexRequestTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/IndexRequestTest.java @@ -30,6 +30,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -66,17 +67,17 @@ public void builderFailsWhenIndexIsNull() { } @Test - public void builderFailsWhenMappingTypeIsNull() { + public void builderBuildsWhenMappingTypeIsNull() { // given IndexRequest.Builder builder = createIndexRequestBuilder() .type(null); // when - final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, builder::build); + final IndexRequest request = builder.build(); // then - assertThat(exception.getMessage(), containsString("type cannot be null")); + assertNotNull(request); } diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/LegacyHCBatchOperationsTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/LegacyHCBatchOperationsTest.java new file mode 100644 index 00000000..c201bd50 --- /dev/null +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/LegacyHCBatchOperationsTest.java @@ -0,0 +1,54 @@ +package org.appenders.log4j2.elasticsearch.hc; + +/*- + * #%L + * log4j2-elasticsearch + * %% + * Copyright (C) 2018 Rafal Foltynski + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.appenders.log4j2.elasticsearch.PooledItemSourceFactory; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +public class LegacyHCBatchOperationsTest extends AbstractHCBatchOperationsTest { + + public HCBatchOperations createDefaultBatchOperations(PooledItemSourceFactory itemSourceFactory) { + return new HCBatchOperations(itemSourceFactory); + } + + public HCBatchOperations createDefaultBatchOperations(PooledItemSourceFactory itemSourceFactory, String mappingType) { + return new HCBatchOperations(itemSourceFactory, mappingType); + } + + @Test + public void defaultMappingTypeIsNullWithDeprecatedConstructor() { + + // given + final PooledItemSourceFactory itemSourceFactory = mock(PooledItemSourceFactory.class); + final HCBatchOperations batchOperations = createDefaultBatchOperations(itemSourceFactory); + + // when + final String result = batchOperations.getMappingType(); + + // then + assertEquals(null, result); + + } + +} diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/PutIndexTemplateTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/PutIndexTemplateTest.java index 88883b6c..98de4a44 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/PutIndexTemplateTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/PutIndexTemplateTest.java @@ -29,8 +29,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import java.io.IOException; - import static org.appenders.core.logging.InternalLogging.setLogger; import static org.appenders.core.logging.InternalLoggingTest.mockTestLogger; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -144,7 +142,7 @@ public void onResponseLogsOnNonSuccess() { } @Test - public void createsActualRequestIfComposable() throws IOException { + public void createsActualRequestIfComposable() throws Exception { // given PutIndexTemplate setupStep = new PutIndexTemplate(IndexTemplate.DEFAULT_API_VERSION + 1, TEST_TEMPLATE_NAME, TEST_SOURCE); @@ -160,10 +158,10 @@ public void createsActualRequestIfComposable() throws IOException { } @Test - public void createsActualRequestIfNotComposable() throws IOException { + public void createsActualRequestIfNotComposable() throws Exception { // given - PutIndexTemplate setupStep = new PutIndexTemplate(IndexTemplate.DEFAULT_API_VERSION, TEST_TEMPLATE_NAME, TEST_SOURCE); + PutIndexTemplate setupStep = new PutIndexTemplate(6, TEST_TEMPLATE_NAME, TEST_SOURCE); // when Request request = setupStep.createRequest(); @@ -176,7 +174,7 @@ public void createsActualRequestIfNotComposable() throws IOException { } @Test - public void defaultRequestNotComposable() throws IOException { + public void defaultRequestIsComposable() throws Exception { // given PutIndexTemplate setupStep = new PutIndexTemplate(TEST_TEMPLATE_NAME, TEST_SOURCE); @@ -186,7 +184,7 @@ public void defaultRequestNotComposable() throws IOException { // then assertEquals("PUT", request.getHttpMethodName()); - assertEquals("_template/" + TEST_TEMPLATE_NAME, request.getURI()); + assertEquals("_index_template/" + TEST_TEMPLATE_NAME, request.getURI()); assertTrue(request.serialize() == TEST_SOURCE); } diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/jmh/AsyncBatchEmitterWithBatchOperationsTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/jmh/AsyncBatchEmitterWithBatchOperationsTest.java index e5071c4c..d49527d1 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/jmh/AsyncBatchEmitterWithBatchOperationsTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/jmh/AsyncBatchEmitterWithBatchOperationsTest.java @@ -58,7 +58,6 @@ import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.infra.Blackhole; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Queue; @@ -92,12 +91,12 @@ public class AsyncBatchEmitterWithBatchOperationsTest { "10", "1000", "10000", - "100000", }) public int itemPoolSize; @Param({ "512", + "1024", "2048", "4096", "8192", @@ -168,7 +167,7 @@ public void release() { bytesSerialized.addAndGet(serialize.getSource().writerIndex()); batchRequest.completed(); return true; - } catch (IOException e) { + } catch (Exception e) { e.printStackTrace(); return false; } diff --git a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/smoke/SmokeTest.java b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/smoke/SmokeTest.java index 0d5c51b0..39af6d5e 100644 --- a/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/smoke/SmokeTest.java +++ b/log4j2-elasticsearch-hc/src/test/java/org/appenders/log4j2/elasticsearch/hc/smoke/SmokeTest.java @@ -114,7 +114,7 @@ protected void configure() { .add("servicediscovery.enabled", Boolean.parseBoolean(System.getProperty("smokeTest.servicediscovery.enabled", "true"))) .add("servicediscovery.nodesFilter", System.getProperty("smokeTest.servicediscovery.nodesFilter", ElasticsearchNodesQuery.DEFAULT_NODES_FILTER)) .add("chroniclemap.sequenceId", 1) - .add("api.version", System.getProperty("smokeTest.api.version", "7.10.2")); + .add("api.version", System.getProperty("smokeTest.api.version", "8.3.2")); } private TestConfig addSecurityConfig(TestConfig target) { @@ -357,7 +357,17 @@ private OpSource[] setupOpSources(final Version version, final String indexName, } private String mappingType(final Version version) { - return !version.lowerThan("7.0.0") ? "_doc" : "index"; + + if (version.lowerThan("7.0.0")) { + return "index"; + } + + if (version.lowerThan("8.0.0")) { + return "_doc"; + } + + return null; + } } diff --git a/log4j2-elasticsearch-hc/src/test/resources/log4j2.properties b/log4j2-elasticsearch-hc/src/test/resources/log4j2.properties index 4fd89a98..702636fe 100644 --- a/log4j2-elasticsearch-hc/src/test/resources/log4j2.properties +++ b/log4j2-elasticsearch-hc/src/test/resources/log4j2.properties @@ -10,7 +10,7 @@ appender.es.type = Elasticsearch # with index 'log4j2-elasticsearch-hc' rolling hourly appender.es.indexNameFormatter.type = RollingIndexName -appender.es.indexNameFormatter.indexName = log4j2-elasticsearch-hc +appender.es.indexNameFormatter.indexName = ${sys:smokeTest.indexName:-log4j2-elasticsearch-hc} appender.es.indexNameFormatter.pattern = yyyy-MM-dd-HH # with AsyncBatchDelivery every 3 seconds or each 5000 logs @@ -18,18 +18,37 @@ appender.es.batchDelivery.type = AsyncBatchDelivery appender.es.batchDelivery.batchSize = 5000 appender.es.batchDelivery.deliveryInterval = 3000 -# with index template -appender.es.batchDelivery.indexTemplate.type = IndexTemplate -appender.es.batchDelivery.indexTemplate.name = log4j2-elasticsearch-hc -appender.es.batchDelivery.indexTemplate.path = classpath:indexTemplate-7.json +# with index template (broken example) +# TL;DR: Use complete index template without components +# Broken as order cannot be guaranteed due to log4j2-core loading this file to java.util.Properties +# As a result, mapping might not be fully configured. +# This bug does not manifest itself with other config methods since property loading methods are not based on java.util.Properties +appender.es.batchDelivery.mappings.type = ComponentTemplate +appender.es.batchDelivery.mappings.name = ${sys:smokeTest.indexName:-log4j2-elasticsearch-hc}-mappings +appender.es.batchDelivery.mappings.path = classpath:componentTemplate-7-mappings.json + +appender.es.batchDelivery.settings.type = ComponentTemplate +appender.es.batchDelivery.settings.name = ${sys:smokeTest.indexName:-log4j2-elasticsearch-hc}-settings +appender.es.batchDelivery.settings.path = classpath:componentTemplate-7-settings.json + +appender.es.batchDelivery.settings-ilm.type = ComponentTemplate +appender.es.batchDelivery.settings-ilm.name = ${sys:smokeTest.indexName:-log4j2-elasticsearch-hc}-settings-ilm +appender.es.batchDelivery.settings-ilm.path = classpath:componentTemplate-7-settings-ilm.json + +appender.es.batchDelivery.index-template.type = IndexTemplate +appender.es.batchDelivery.index-template.name = ${sys:smokeTest.indexName:-log4j2-elasticsearch-hc} +appender.es.batchDelivery.index-template.path = classpath:composableIndexTemplate-7.json + +appender.es.batchDelivery.ilm-policy.type = ILMPolicy +appender.es.batchDelivery.ilm-policy.name = ${sys:smokeTest.indexName:-log4j2-elasticsearch-hc} +appender.es.batchDelivery.ilm-policy.path = classpath:ilmPolicy-7.json # with HC HTTP client appender.es.batchDelivery.objectFactory.type = HCHttp -appender.es.batchDelivery.objectFactory.serverUris = http://localhost:9200 +appender.es.batchDelivery.objectFactory.serverUris = https://localhost:9200 appender.es.batchDelivery.objectFactory.connTimeout = 500 appender.es.batchDelivery.objectFactory.readTimeout = 10000 appender.es.batchDelivery.objectFactory.maxTotalConnections = 8 -appender.es.batchDelivery.objectFactory.mappingType = _doc # with buffers for serialized batchRequest objects appender.es.batchDelivery.objectFactory.itemSourceFactory.type = PooledItemSourceFactory @@ -40,6 +59,19 @@ appender.es.batchDelivery.objectFactory.itemSourceFactory.monitored = true appender.es.batchDelivery.objectFactory.itemSourceFactory.monitorTaskInterval = 5000 appender.es.batchDelivery.objectFactory.itemSourceFactory.resizeTimeout = 100 +# with Security +appender.es.batchDelivery.objectFactory.auth.type = Security + +appender.es.batchDelivery.objectFactory.auth.credentials.type = BasicCredentials +appender.es.batchDelivery.objectFactory.auth.credentials.username = admin +appender.es.batchDelivery.objectFactory.auth.credentials.password = changeme + +appender.es.batchDelivery.objectFactory.auth.certInfo.type = PEM +appender.es.batchDelivery.objectFactory.auth.certInfo.keyPath=${sys:pemCertInfo.keyPathWithPassphrase} +appender.es.batchDelivery.objectFactory.auth.certInfo.keyPassphrase=${sys:pemCertInfo.keyPassphrase} +appender.es.batchDelivery.objectFactory.auth.certInfo.clientCertPath=${sys:pemCertInfo.clientCertPath} +appender.es.batchDelivery.objectFactory.auth.certInfo.caPath=${sys:pemCertInfo.caPath} + # with JacksonJsonLayout appender.es.layout.type = JacksonJsonLayout diff --git a/log4j2-elasticsearch-hc/src/test/resources/log4j2.xml b/log4j2-elasticsearch-hc/src/test/resources/log4j2.xml index bcf73f07..adf549a4 100644 --- a/log4j2-elasticsearch-hc/src/test/resources/log4j2.xml +++ b/log4j2-elasticsearch-hc/src/test/resources/log4j2.xml @@ -3,7 +3,7 @@ - + @@ -22,8 +22,11 @@ - - + + + + + @@ -36,22 +39,23 @@ - - - - - - - - - - - - + configPolicies="security" /> + + + + + + + + - +