Skip to content

Commit

Permalink
Add DataStream support (#62)
Browse files Browse the repository at this point in the history
* Add DataStream to ElasticsearchOperationFactory
* Add DataStreamItem - batch item
* Add DataStreamBatchRequest - batch
* Add ElasticsearchDataStreamAPI - builders and serializers
* Add ElasticsearchDataStreamAPIPlugin - Log4j2 config
  • Loading branch information
rfoltyns committed Oct 5, 2022
1 parent 5a0d3cb commit 98d5bf3
Show file tree
Hide file tree
Showing 39 changed files with 2,734 additions and 44 deletions.
17 changes: 16 additions & 1 deletion log4j2-elasticsearch-core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,15 @@ NOTE: Be aware that template parsing errors on cluster side MAY NOT prevent plug
### Index lifecycle management
Since 1.5, [ILM Policy](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/index-lifecycle-management.html) can be created during appender startup. Policy can be loaded from specified file or defined directly in the XML config:

| Config property | Type | Required | Default | Description |
|----------------------|-----------|-----------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------|
| name | Attribute | yes | None | ILM Policy resource name |
| createBootstrapIndex | Attribute | no | `true` | If `true`, bootstrap index will be created and set as write index unless an index with the same `name` already exists |
| rolloverAlias | Attribute | yes, if `createBootstrapIndex` is `true` | None | Rollover alias |
| path | Attribute | yes, if document not defined with `sourceString` (see examples below) | None | Path to policy document. E.g. `classpath:ilm-policy.json` or `<DIR>/ilm-policy.json` |

If used with [DataStream](#data-stream), `createBootstrapIndex` MUST be false.

```xml
<Appenders>
<Elasticsearch name="elasticsearchAsyncBatch">
Expand All @@ -228,7 +237,7 @@ Since 1.5, [ILM Policy](https://www.elastic.co/guide/en/elasticsearch/reference/
</Elasticsearch>
</Appenders>
```
or
or with `sourceString`
```xml
<Appenders>
<Elasticsearch name="elasticsearchAsyncBatch">
Expand All @@ -252,6 +261,12 @@ NOTE: This feature is supported by [log4j2-elasticsearch-jest](https://github.co

NOTE: Be aware that policy parsing errors on cluster side MAY NOT prevent plugin from loading - error is logged on client side and startup continues.

### Data streams support

Since 1.6, [Data streams](https://www.elastic.co/guide/en/elasticsearch/reference/current/data-streams.html) are supported with `DataStream` setup operation in several modules.

See submodules documentation to check support.

### Message output

There are numerous ways to generate JSON output:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package org.appenders.log4j2.elasticsearch;

/*-
* #%L
* log4j2-elasticsearch
* %%
* Copyright (C) 2022 Rafal Foltynski
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/

public class DataStream implements OpSource {

public static final String TYPE_NAME = "DataStream";
private static final String EMPTY_SOURCE = "";

private final String name;

/**
* @param name Data stream name
*/
public DataStream(final String name) {
this.name = name;
}

/**
* @return Data stream name
*/
public String getName() {
return name;
}

/**
* @return {@link #TYPE_NAME}
*/
@Override
public String getType() {
return TYPE_NAME;
}

/**
* @return Empty String {@code ""}
*/
@Override
public String getSource() {
return EMPTY_SOURCE;
}

public static class Builder {

protected String name;

public DataStream build() {
validate();
return new DataStream(name);
}

void validate() {

if (name == null) {
throw new IllegalArgumentException("No name provided for " + DataStream.class.getSimpleName());
}

}

/**
* @param name Data stream name
* @return this
*/
public Builder withName(String name) {
this.name = name;
return this;
}

}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package org.appenders.log4j2.elasticsearch;

/*-
* #%L
* log4j2-elasticsearch
* %%
* Copyright (C) 2022 Rafal Foltynski
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/

import org.apache.logging.log4j.core.config.ConfigurationException;
import org.apache.logging.log4j.core.config.Node;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute;
import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory;


@Plugin(name = DataStreamPlugin.PLUGIN_NAME, category = Node.CATEGORY, elementType = "setupOperation", printObject = true)
public class DataStreamPlugin extends DataStream {

public static final String PLUGIN_NAME = "DataStream";

/**
* @param name Data stream name
*/
protected DataStreamPlugin(final String name) {
super(name);
}

@PluginBuilderFactory
public static DataStreamPlugin.Builder newBuilder() {
return new Builder();
}

public static class Builder implements org.apache.logging.log4j.core.util.Builder<DataStreamPlugin> {

@PluginBuilderAttribute
private String name;

@Override
public DataStreamPlugin build() {

if (name == null) {
throw new ConfigurationException("No name provided for " + PLUGIN_NAME);
}

return new DataStreamPlugin(name);
}

public Builder withName(String name) {
this.name = name;
return this;
}

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,30 @@ public class ILMPolicy implements OpSource {

private final String name;
private final String rolloverAlias;
private final boolean createBootstrapIndex;
private final String source;

/**
* @param name ILM policy name
* @param rolloverAlias index rollover alias
* @param source ILM policy document
* @deprecated As of 1.7, this method will be removed. Use {@link #ILMPolicy(String, String, boolean, String)} instead.
*/
public ILMPolicy(String name, String rolloverAlias, String source) {
@Deprecated
public ILMPolicy(final String name, final String rolloverAlias, final String source) {
this(name, rolloverAlias, true, source);
}

/**
* @param name ILM policy name
* @param rolloverAlias index rollover alias
* @param createBootstrapIndex should bootstrap index be created or not
* @param source ILM policy document
*/
public ILMPolicy(final String name, final String rolloverAlias, final boolean createBootstrapIndex, final String source) {
this.name = name;
this.rolloverAlias = rolloverAlias;
this.createBootstrapIndex = createBootstrapIndex;
this.source = source;
}

Expand All @@ -64,6 +78,10 @@ public String getRolloverAlias() {
return this.rolloverAlias;
}

public boolean isCreateBootstrapIndex() {
return createBootstrapIndex;
}

/**
* @return ILM policy document
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ public class ILMPolicyPlugin extends ILMPolicy {
/**
* {@inheritDoc}
*/
protected ILMPolicyPlugin(String policyName, String rolloverAlias, String source) {
super(policyName, rolloverAlias, source);
protected ILMPolicyPlugin(String policyName, String rolloverAlias, boolean createBootstrapIndex, String source) {
super(policyName, rolloverAlias, createBootstrapIndex, source);
}

@PluginBuilderFactory
Expand All @@ -57,9 +57,11 @@ public static class Builder implements org.apache.logging.log4j.core.util.Builde
private String name;

@PluginAttribute("rolloverAlias")
@Required
private String rolloverAlias;

@PluginAttribute("createBootstrapIndex")
private boolean createBootstrapIndex = true;

@PluginAttribute("path")
private String path;

Expand All @@ -72,7 +74,7 @@ public ILMPolicyPlugin build() {
throw new ConfigurationException("No name provided for " + PLUGIN_NAME);
}

if (rolloverAlias == null) {
if (createBootstrapIndex && rolloverAlias == null) {
throw new ConfigurationException("No rolloverAlias provided for " + PLUGIN_NAME);
}

Expand All @@ -82,7 +84,7 @@ public ILMPolicyPlugin build() {
throw new ConfigurationException("Either path or source must to be provided for " + PLUGIN_NAME);
}

return new ILMPolicyPlugin(name, rolloverAlias, loadSource());
return new ILMPolicyPlugin(name, rolloverAlias, createBootstrapIndex, loadSource());
}

private String loadSource() {
Expand Down Expand Up @@ -124,6 +126,11 @@ public Builder withRolloverAlias(String rolloverAlias) {
return this;
}

public Builder withCreateBootstrapIndex(boolean createBootstrapIndex) {
this.createBootstrapIndex = createBootstrapIndex;
return this;
}

/**
* @param source ILM policy document. MAY contain placeholders resolvable by {@link ValueResolver}
* @return this
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,15 @@
/**
* Index template definition. Supports both composable index templates and templates deprecated in 7.8.
* Set {@link #apiVersion} to 8 to indicate that this template is composable
* Set {@link #apiVersion} to 7 to use legacy Index Template API
*
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html">Composable index templates</a>
* and <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html">Deprecated index templates</a>
*/
public class IndexTemplate implements OpSource {

public static final String TYPE_NAME = "IndexTemplate";
public static final int DEFAULT_API_VERSION = 7;
public static final int DEFAULT_API_VERSION = 8;

private final int apiVersion;
private final String name;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package org.appenders.log4j2.elasticsearch.json.jackson;

/*-
* #%L
* log4j2-elasticsearch
* %%
* Copyright (C) 2022 Rafal Foltynski
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/

public abstract class LogEventDataStreamMixIn extends ExtendedLogEventJacksonJsonMixIn implements LogEventTimestampMixIn {

}

Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package org.appenders.log4j2.elasticsearch.json.jackson;

/*-
* #%L
* log4j2-elasticsearch
* %%
* Copyright (C) 2022 Rafal Foltynski
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/

import com.fasterxml.jackson.annotation.JsonProperty;

public interface LogEventTimestampMixIn {

@JsonProperty("@timestamp")
long getTimeMillis();

}
Loading

0 comments on commit 98d5bf3

Please sign in to comment.