diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/LuceneChangesSnapshotBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/LuceneChangesSnapshotBenchmark.java deleted file mode 100644 index dde9d056853d7..0000000000000 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/LuceneChangesSnapshotBenchmark.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.benchmark.index; - -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SegmentCommitInfo; -import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryCachingPolicy; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.similarities.BM25Similarity; -import org.apache.lucene.store.FSDirectory; -import org.elasticsearch.benchmark.index.mapper.MapperServiceFactory; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.core.IOUtils; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.codec.PerFieldMapperCodec; -import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.LuceneChangesSnapshot; -import org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshot; -import org.elasticsearch.index.engine.SearchBasedChangesSnapshot; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OperationsPerInvocation; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Random; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; - -import static org.elasticsearch.index.engine.Engine.ROOT_DOC_FIELD_NAME; - -@Fork(value = 1) -@Warmup(iterations = 3, time = 3) -@Measurement(iterations = 5, time = 3) -@BenchmarkMode(Mode.Throughput) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@State(Scope.Thread) -public class LuceneChangesSnapshotBenchmark { - @Param({ "default", "logsdb@1kb", "logsdb@4MB" }) - String mode; - - @Param({ "false", "true" }) - boolean sequential; - - @Param({ "logs-endpoint-events-process", "logs-endpoint-events-security", "logs-kafka-log" }) - String dataset; - - static final int NUM_OPS = 10000; - - Path path; - MapperService mapperService; - FSDirectory dir; - IndexReader reader; - Engine.Searcher searcher; - - static { - LogConfigurator.configureESLogging(); // native access requires logging to be initialized - } - - @Setup - public void setup() throws IOException { - this.path = Files.createTempDirectory("snapshot_changes"); - Settings settings = mode.startsWith("logsdb") - ? Settings.builder().put("index.mode", "logsdb").put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true).build() - : Settings.EMPTY; - this.mapperService = MapperServiceFactory.create(settings, readMappings(dataset)); - IndexWriterConfig config = new IndexWriterConfig(); - config.setCodec( - new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, BigArrays.NON_RECYCLING_INSTANCE) - ); - if (sequential == false) { - config.setIndexSort(new Sort(new SortField[] { new SortField("rand", SortField.Type.LONG) })); - config.setParentField(ROOT_DOC_FIELD_NAME); - } - try (FSDirectory dir = FSDirectory.open(path); IndexWriter writer = new IndexWriter(dir, config);) { - try ( - var inputStream = readSampleDocs(dataset); - XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, inputStream) - ) { - int id = 0; - // find the hits array - while (parser.nextToken() != XContentParser.Token.START_ARRAY) { - parser.nextToken(); - } - Random rand = new Random(); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - // skip start object - parser.nextToken(); - // skip _source field name - parser.nextToken(); - XContentBuilder source = XContentBuilder.builder(XContentType.JSON.xContent()); - source.copyCurrentStructure(parser); - var sourceBytes = BytesReference.bytes(source); - SourceToParse sourceToParse = new SourceToParse(Integer.toString(id), sourceBytes, XContentType.JSON); - ParsedDocument doc = mapperService.documentMapper().parse(sourceToParse); - doc.rootDoc().add(new NumericDocValuesField("rand", rand.nextInt())); - doc.updateSeqID(id, 0); - doc.version().setLongValue(0); - writer.addDocuments(doc.docs()); - id++; - parser.nextToken(); - } - } - } - this.dir = FSDirectory.open(path); - this.reader = ElasticsearchDirectoryReader.wrap( - DirectoryReader.open(dir), - new ShardId(mapperService.getIndexSettings().getIndex(), 0) - ); - long sizeInBytes = 0; - for (LeafReaderContext readerContext : reader.leaves()) { - // we go on the segment level here to get accurate numbers - final SegmentReader segmentReader = Lucene.segmentReader(readerContext.reader()); - SegmentCommitInfo info = segmentReader.getSegmentInfo(); - try { - sizeInBytes += info.sizeInBytes(); - } catch (IOException e) {} - } - System.out.println("Size: " + ByteSizeValue.ofBytes(sizeInBytes)); - - this.searcher = new Engine.Searcher("snapshot", reader, new BM25Similarity(), null, new QueryCachingPolicy() { - @Override - public void onUse(Query query) {} - - @Override - public boolean shouldCache(Query query) throws IOException { - return false; - } - }, () -> {}); - } - - @TearDown - public void tearDown() { - try { - for (var file : dir.listAll()) { - dir.deleteFile(file); - } - Files.delete(path); - } catch (IOException e) { - throw new RuntimeException(e); - } - IOUtils.closeWhileHandlingException(searcher, reader, dir); - } - - @Benchmark - @OperationsPerInvocation(NUM_OPS) - public long recover() throws IOException { - String indexMode = mode.split("@")[0]; - Translog.Snapshot snapshot = switch (mapperService.getIndexSettings().getMode()) { - case LOGSDB: - assert indexMode.equals("logsdb"); - long maxMemorySize = ByteSizeValue.parseBytesSizeValue(mode.split("@")[1], "").getBytes(); - yield new LuceneSyntheticSourceChangesSnapshot( - mapperService.mappingLookup(), - searcher, - SearchBasedChangesSnapshot.DEFAULT_BATCH_SIZE, - maxMemorySize, - 0, - NUM_OPS - 1, - true, - true, - IndexVersion.current() - ); - - default: - assert indexMode.equals("default"); - yield new LuceneChangesSnapshot( - searcher, - SearchBasedChangesSnapshot.DEFAULT_BATCH_SIZE, - 0, - NUM_OPS - 1, - true, - true, - true, - IndexVersion.current() - ); - }; - - long totalSize = 0; - try (snapshot) { - Translog.Operation op; - while ((op = snapshot.next()) != null) { - totalSize += op.estimateSize(); - } - } - return totalSize; - } - - private String readMappings(String dataset) throws IOException { - return Streams.readFully(LuceneChangesSnapshotBenchmark.class.getResourceAsStream(dataset + "-mappings.json")).utf8ToString(); - } - - private InputStream readSampleDocs(String dataset) throws IOException { - return new GZIPInputStream(LuceneChangesSnapshotBenchmark.class.getResourceAsStream(dataset + "-sample-docs.json.gz")); - } -} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java index 524576018f376..d3f210f774782 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java @@ -43,13 +43,9 @@ import java.util.Map; public class MapperServiceFactory { - public static MapperService create(String mappings) { - return create(Settings.EMPTY, mappings); - } - public static MapperService create(Settings userSettings, String mappings) { + public static MapperService create(String mappings) { Settings settings = Settings.builder() - .put(userSettings) .put("index.number_of_replicas", 0) .put("index.number_of_shards", 1) .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) diff --git a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-process-mappings.json b/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-process-mappings.json deleted file mode 100644 index 2d3cc77c28b58..0000000000000 --- a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-process-mappings.json +++ /dev/null @@ -1,1950 +0,0 @@ -{ - "dynamic": "false", - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "endpoint" - } - }, - "_data_stream_timestamp": { - "enabled": true - }, - "date_detection": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "agent": { - "properties": { - "ephemeral_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "data_stream": { - "properties": { - "dataset": { - "type": "keyword" - }, - "namespace": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "destination": { - "properties": { - "geo": { - "properties": { - "city_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "continent_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "continent_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "country_iso_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "country_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "location": { - "type": "geo_point" - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "postal_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "region_iso_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "region_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "timezone": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "ecs": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "event": { - "properties": { - "action": { - "type": "keyword", - "ignore_above": 1024 - }, - "agent_id_status": { - "type": "keyword", - "ignore_above": 1024 - }, - "category": { - "type": "keyword", - "ignore_above": 1024 - }, - "code": { - "type": "keyword", - "ignore_above": 1024 - }, - "created": { - "type": "date" - }, - "dataset": { - "type": "keyword", - "ignore_above": 1024 - }, - "hash": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "ingested": { - "type": "date", - "format": "strict_date_time_no_millis||strict_date_optional_time||epoch_millis" - }, - "kind": { - "type": "keyword", - "ignore_above": 1024 - }, - "module": { - "type": "keyword", - "ignore_above": 1024 - }, - "outcome": { - "type": "keyword", - "ignore_above": 1024 - }, - "provider": { - "type": "keyword", - "ignore_above": 1024 - }, - "sequence": { - "type": "long" - }, - "severity": { - "type": "long" - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "group": { - "properties": { - "Ext": { - "properties": { - "real": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "host": { - "properties": { - "architecture": { - "type": "keyword", - "ignore_above": 1024 - }, - "boot": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "hostname": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "ip": { - "type": "ip" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "os": { - "properties": { - "Ext": { - "properties": { - "variant": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "family": { - "type": "keyword", - "ignore_above": 1024 - }, - "full": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "kernel": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "platform": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "pid_ns_ino": { - "type": "keyword", - "ignore_above": 1024 - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - }, - "uptime": { - "type": "long" - } - } - }, - "message": { - "type": "text" - }, - "package": { - "properties": { - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "process": { - "properties": { - "Ext": { - "properties": { - "ancestry": { - "type": "keyword", - "ignore_above": 1024 - }, - "architecture": { - "type": "keyword", - "ignore_above": 1024 - }, - "authentication_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "code_signature": { - "type": "nested", - "properties": { - "exists": { - "type": "boolean" - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "subject_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "trusted": { - "type": "boolean" - }, - "valid": { - "type": "boolean" - } - } - }, - "defense_evasions": { - "type": "keyword", - "ignore_above": 1024 - }, - "dll": { - "properties": { - "Ext": { - "properties": { - "code_signature": { - "type": "nested", - "properties": { - "exists": { - "type": "boolean" - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "subject_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "trusted": { - "type": "boolean" - }, - "valid": { - "type": "boolean" - } - } - }, - "mapped_address": { - "type": "keyword" - }, - "mapped_size": { - "type": "keyword" - } - } - }, - "code_signature": { - "properties": { - "exists": { - "type": "boolean" - }, - "signing_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "subject_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "team_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "trusted": { - "type": "boolean" - }, - "valid": { - "type": "boolean" - } - } - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "path": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "protection": { - "type": "keyword", - "ignore_above": 1024 - }, - "session": { - "type": "keyword", - "ignore_above": 1024 - }, - "token": { - "properties": { - "elevation": { - "type": "boolean" - }, - "elevation_level": { - "type": "keyword", - "ignore_above": 1024 - }, - "elevation_type": { - "type": "keyword", - "ignore_above": 1024 - }, - "integrity_level_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "security_attributes": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "args": { - "type": "keyword", - "ignore_above": 1024 - }, - "args_count": { - "type": "long" - }, - "code_signature": { - "properties": { - "exists": { - "type": "boolean" - }, - "signing_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "subject_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "team_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "trusted": { - "type": "boolean" - }, - "valid": { - "type": "boolean" - } - } - }, - "command_line": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "entry_leader": { - "properties": { - "args": { - "type": "keyword", - "ignore_above": 1024 - }, - "args_count": { - "type": "long" - }, - "command_line": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "entry_meta": { - "properties": { - "source": { - "properties": { - "ip": { - "type": "ip" - } - } - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "executable": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "interactive": { - "type": "boolean" - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "parent": { - "properties": { - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "pid": { - "type": "long" - }, - "session_leader": { - "properties": { - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "pid": { - "type": "long" - }, - "start": { - "type": "date" - } - } - }, - "start": { - "type": "date" - } - } - }, - "pid": { - "type": "long" - }, - "real_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "real_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "same_as_process": { - "type": "boolean" - }, - "saved_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "saved_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "start": { - "type": "date" - }, - "supplemental_groups": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "tty": { - "properties": { - "char_device": { - "properties": { - "major": { - "type": "long" - }, - "minor": { - "type": "long" - } - } - } - } - }, - "user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "working_directory": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - } - } - }, - "env_vars": { - "type": "object" - }, - "executable": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "exit_code": { - "type": "long" - }, - "group_leader": { - "properties": { - "args": { - "type": "keyword", - "ignore_above": 1024 - }, - "args_count": { - "type": "long" - }, - "command_line": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "executable": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "interactive": { - "type": "boolean" - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "pid": { - "type": "long" - }, - "real_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "real_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "same_as_process": { - "type": "boolean" - }, - "saved_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "saved_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "start": { - "type": "date" - }, - "supplemental_groups": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "tty": { - "properties": { - "char_device": { - "properties": { - "major": { - "type": "long" - }, - "minor": { - "type": "long" - } - } - } - } - }, - "user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "working_directory": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - } - } - }, - "hash": { - "properties": { - "md5": { - "type": "keyword", - "ignore_above": 1024 - }, - "sha1": { - "type": "keyword", - "ignore_above": 1024 - }, - "sha256": { - "type": "keyword", - "ignore_above": 1024 - }, - "sha512": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "interactive": { - "type": "boolean" - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "parent": { - "properties": { - "Ext": { - "properties": { - "architecture": { - "type": "keyword", - "ignore_above": 1024 - }, - "code_signature": { - "type": "nested", - "properties": { - "exists": { - "type": "boolean" - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "subject_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "trusted": { - "type": "boolean" - }, - "valid": { - "type": "boolean" - } - } - }, - "protection": { - "type": "keyword", - "ignore_above": 1024 - }, - "real": { - "properties": { - "pid": { - "type": "long" - } - } - }, - "user": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "args": { - "type": "keyword", - "ignore_above": 1024 - }, - "args_count": { - "type": "long" - }, - "code_signature": { - "properties": { - "exists": { - "type": "boolean" - }, - "signing_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "status": { - "type": "keyword", - "ignore_above": 1024 - }, - "subject_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "team_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "trusted": { - "type": "boolean" - }, - "valid": { - "type": "boolean" - } - } - }, - "command_line": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "executable": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "exit_code": { - "type": "long" - }, - "group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "group_leader": { - "properties": { - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "pid": { - "type": "long" - }, - "start": { - "type": "date" - } - } - }, - "hash": { - "properties": { - "md5": { - "type": "keyword", - "ignore_above": 1024 - }, - "sha1": { - "type": "keyword", - "ignore_above": 1024 - }, - "sha256": { - "type": "keyword", - "ignore_above": 1024 - }, - "sha512": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "interactive": { - "type": "boolean" - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "pe": { - "properties": { - "company": { - "type": "keyword", - "ignore_above": 1024 - }, - "description": { - "type": "keyword", - "ignore_above": 1024 - }, - "file_version": { - "type": "keyword", - "ignore_above": 1024 - }, - "imphash": { - "type": "keyword", - "ignore_above": 1024 - }, - "original_file_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "product": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "pgid": { - "type": "long" - }, - "pid": { - "type": "long" - }, - "ppid": { - "type": "long" - }, - "real_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "real_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "saved_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "saved_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "start": { - "type": "date" - }, - "supplemental_groups": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "thread": { - "properties": { - "id": { - "type": "long" - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "title": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - }, - "tty": { - "properties": { - "char_device": { - "properties": { - "major": { - "type": "long" - }, - "minor": { - "type": "long" - } - } - } - } - }, - "uptime": { - "type": "long" - }, - "user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "working_directory": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - } - } - }, - "pe": { - "properties": { - "company": { - "type": "keyword", - "ignore_above": 1024 - }, - "description": { - "type": "keyword", - "ignore_above": 1024 - }, - "file_version": { - "type": "keyword", - "ignore_above": 1024 - }, - "imphash": { - "type": "keyword", - "ignore_above": 1024 - }, - "original_file_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "product": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "pgid": { - "type": "long" - }, - "pid": { - "type": "long" - }, - "ppid": { - "type": "long" - }, - "previous": { - "properties": { - "args": { - "type": "keyword", - "ignore_above": 1024 - }, - "args_count": { - "type": "long" - }, - "executable": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - } - } - }, - "real_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "real_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "saved_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "saved_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "session_leader": { - "properties": { - "args": { - "type": "keyword", - "ignore_above": 1024 - }, - "args_count": { - "type": "long" - }, - "command_line": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "executable": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "interactive": { - "type": "boolean" - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "parent": { - "properties": { - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "pid": { - "type": "long" - }, - "session_leader": { - "properties": { - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "pid": { - "type": "long" - }, - "start": { - "type": "date" - } - } - }, - "start": { - "type": "date" - } - } - }, - "pid": { - "type": "long" - }, - "real_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "real_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "same_as_process": { - "type": "boolean" - }, - "saved_group": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "saved_user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "start": { - "type": "date" - }, - "supplemental_groups": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "tty": { - "properties": { - "char_device": { - "properties": { - "major": { - "type": "long" - }, - "minor": { - "type": "long" - } - } - } - } - }, - "user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "working_directory": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - } - } - }, - "start": { - "type": "date" - }, - "supplemental_groups": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "thread": { - "properties": { - "id": { - "type": "long" - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "title": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - }, - "tty": { - "properties": { - "char_device": { - "properties": { - "major": { - "type": "long" - }, - "minor": { - "type": "long" - } - } - } - } - }, - "uptime": { - "type": "long" - }, - "user": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "working_directory": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - } - } - }, - "source": { - "properties": { - "geo": { - "properties": { - "city_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "continent_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "continent_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "country_iso_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "country_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "location": { - "type": "geo_point" - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "postal_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "region_iso_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "region_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "timezone": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "user": { - "properties": { - "Ext": { - "properties": { - "real": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "email": { - "type": "keyword", - "ignore_above": 1024 - }, - "full_name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - }, - "group": { - "properties": { - "Ext": { - "properties": { - "real": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "hash": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-process-sample-docs.json.gz b/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-process-sample-docs.json.gz deleted file mode 100644 index ec0b8542a4677..0000000000000 Binary files a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-process-sample-docs.json.gz and /dev/null differ diff --git a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-security-mappings.json b/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-security-mappings.json deleted file mode 100644 index 98fd747f98b71..0000000000000 --- a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-security-mappings.json +++ /dev/null @@ -1,492 +0,0 @@ -{ - "dynamic": "false", - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "endpoint" - } - }, - "_data_stream_timestamp": { - "enabled": true - }, - "date_detection": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "agent": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "data_stream": { - "properties": { - "dataset": { - "type": "keyword" - }, - "namespace": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "destination": { - "properties": { - "geo": { - "properties": { - "city_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "continent_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "continent_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "country_iso_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "country_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "location": { - "type": "geo_point" - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "postal_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "region_iso_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "region_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "timezone": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "ecs": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "event": { - "properties": { - "action": { - "type": "keyword", - "ignore_above": 1024 - }, - "agent_id_status": { - "type": "keyword", - "ignore_above": 1024 - }, - "category": { - "type": "keyword", - "ignore_above": 1024 - }, - "code": { - "type": "keyword", - "ignore_above": 1024 - }, - "created": { - "type": "date" - }, - "dataset": { - "type": "keyword", - "ignore_above": 1024 - }, - "hash": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "ingested": { - "type": "date", - "format": "strict_date_time_no_millis||strict_date_optional_time||epoch_millis" - }, - "kind": { - "type": "keyword", - "ignore_above": 1024 - }, - "module": { - "type": "keyword", - "ignore_above": 1024 - }, - "outcome": { - "type": "keyword", - "ignore_above": 1024 - }, - "provider": { - "type": "keyword", - "ignore_above": 1024 - }, - "sequence": { - "type": "long" - }, - "severity": { - "type": "long" - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "group": { - "properties": { - "Ext": { - "properties": { - "real": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "host": { - "properties": { - "architecture": { - "type": "keyword", - "ignore_above": 1024 - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "hostname": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "ip": { - "type": "ip" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "os": { - "properties": { - "Ext": { - "properties": { - "variant": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "family": { - "type": "keyword", - "ignore_above": 1024 - }, - "full": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "kernel": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "platform": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - }, - "uptime": { - "type": "long" - } - } - }, - "message": { - "type": "text" - }, - "process": { - "properties": { - "Ext": { - "properties": { - "ancestry": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "executable": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "caseless": { - "type": "keyword", - "ignore_above": 1024, - "normalizer": "lowercase" - }, - "text": { - "type": "text" - } - } - }, - "pid": { - "type": "long" - }, - "thread": { - "properties": { - "id": { - "type": "long" - } - } - } - } - }, - "source": { - "properties": { - "geo": { - "properties": { - "city_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "continent_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "continent_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "country_iso_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "country_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "location": { - "type": "geo_point" - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "postal_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "region_iso_code": { - "type": "keyword", - "ignore_above": 1024 - }, - "region_name": { - "type": "keyword", - "ignore_above": 1024 - }, - "timezone": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "user": { - "properties": { - "Ext": { - "properties": { - "real": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "email": { - "type": "keyword", - "ignore_above": 1024 - }, - "full_name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - }, - "group": { - "properties": { - "Ext": { - "properties": { - "real": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - } - } - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "hash": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - } - } - } - } -} \ No newline at end of file diff --git a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-security-sample-docs.json.gz b/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-security-sample-docs.json.gz deleted file mode 100644 index 7fb91c8d0bee6..0000000000000 Binary files a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-endpoint-events-security-sample-docs.json.gz and /dev/null differ diff --git a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-kafka-log-mappings.json b/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-kafka-log-mappings.json deleted file mode 100644 index 217bc64761bd5..0000000000000 --- a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-kafka-log-mappings.json +++ /dev/null @@ -1,479 +0,0 @@ -{ - "_meta": { - "managed_by": "fleet", - "managed": true, - "package": { - "name": "kafka" - } - }, - "_data_stream_timestamp": { - "enabled": true - }, - "date_detection": false, - "runtime": { - "rally.doc_size": { - "type": "long" - }, - "rally.message_size": { - "type": "long" - } - }, - "properties": { - "@timestamp": { - "type": "date", - "ignore_malformed": false - }, - "agent": { - "properties": { - "ephemeral_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "hostname": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "cloud": { - "properties": { - "account": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "availability_zone": { - "type": "keyword", - "ignore_above": 1024 - }, - "image": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "instance": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "machine": { - "properties": { - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "project": { - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "provider": { - "type": "keyword", - "ignore_above": 1024 - }, - "region": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "container": { - "dynamic": "true", - "properties": { - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "image": { - "properties": { - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "labels": { - "type": "object", - "dynamic": "true" - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "data_stream": { - "properties": { - "dataset": { - "type": "keyword" - }, - "namespace": { - "type": "keyword" - }, - "type": { - "type": "keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "error": { - "properties": { - "log": { - "type": "keyword", - "ignore_above": 1024 - }, - "message": { - "type": "text" - } - } - }, - "event": { - "properties": { - "agent_id_status": { - "type": "keyword", - "ignore_above": 1024 - }, - "created": { - "type": "date", - "format": "strict_date_optional_time" - }, - "dataset": { - "type": "keyword" - }, - "end": { - "type": "date" - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "ingested": { - "type": "date", - "format": "strict_date_optional_time" - }, - "kind": { - "type": "keyword", - "ignore_above": 1024 - }, - "module": { - "type": "keyword" - }, - "start": { - "type": "date" - }, - "timezone": { - "type": "keyword", - "ignore_above": 1024 - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "fileset": { - "properties": { - "name": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "host": { - "properties": { - "architecture": { - "type": "keyword", - "ignore_above": 1024 - }, - "containerized": { - "type": "boolean" - }, - "domain": { - "type": "keyword", - "ignore_above": 1024 - }, - "hostname": { - "type": "keyword", - "ignore_above": 1024 - }, - "id": { - "type": "keyword", - "ignore_above": 1024 - }, - "ip": { - "type": "ip" - }, - "mac": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024 - }, - "os": { - "properties": { - "build": { - "type": "keyword", - "ignore_above": 1024 - }, - "codename": { - "type": "keyword", - "ignore_above": 1024 - }, - "family": { - "type": "keyword", - "ignore_above": 1024 - }, - "kernel": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "ignore_above": 1024, - "fields": { - "text": { - "type": "text" - } - } - }, - "platform": { - "type": "keyword", - "ignore_above": 1024 - }, - "version": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "input": { - "properties": { - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "kafka": { - "properties": { - "log": { - "properties": { - "class": { - "type": "keyword", - "ignore_above": 1024 - }, - "component": { - "type": "keyword", - "ignore_above": 1024 - }, - "thread": { - "type": "keyword", - "ignore_above": 1024 - }, - "trace": { - "properties": { - "class": { - "type": "keyword", - "ignore_above": 1024 - }, - "message": { - "type": "text" - } - } - } - } - } - } - }, - "kubernetes": { - "properties": { - "container": { - "properties": { - "image": { - "type": "keyword", - "ignore_above": 1024 - }, - "name": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "labels": { - "properties": { - "app": { - "type": "keyword", - "ignore_above": 1024 - }, - "controller-revision-hash": { - "type": "keyword", - "ignore_above": 1024 - }, - "release": { - "type": "keyword", - "ignore_above": 1024 - }, - "statefulset_kubernetes_io/pod-name": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "namespace": { - "type": "keyword", - "ignore_above": 1024 - }, - "node": { - "properties": { - "name": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "pod": { - "properties": { - "name": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "uid": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "statefulset": { - "properties": { - "name": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - } - } - } - } - }, - "log": { - "properties": { - "file": { - "properties": { - "path": { - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - } - } - }, - "flags": { - "type": "keyword", - "ignore_above": 1024 - }, - "level": { - "type": "keyword", - "ignore_above": 1024 - }, - "offset": { - "type": "long" - } - } - }, - "message": { - "type": "text" - }, - "rally": { - "type": "object" - }, - "service": { - "properties": { - "type": { - "type": "keyword", - "ignore_above": 1024 - } - } - }, - "stream": { - "type": "keyword", - "ignore_above": 1024 - }, - "tags": { - "type": "keyword", - "ignore_above": 1024 - } - } -} \ No newline at end of file diff --git a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-kafka-log-sample-docs.json.gz b/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-kafka-log-sample-docs.json.gz deleted file mode 100644 index ca8f3539fcbd7..0000000000000 Binary files a/benchmarks/src/main/resources/org/elasticsearch/benchmark/index/logs-kafka-log-sample-docs.json.gz and /dev/null differ