diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index e35cec326b..ce1bfb08de 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3803,6 +3803,62 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-endpoint_x_diagnostic_x_collection: + index_sorting: false + index_template: + composed_of: + - event-mappings + - logs-endpoint.diagnostic.collection@custom + - logs-endpoint.diagnostic.collection@package + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-endpoint.diagnostic.collection-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-endpoint.diagnostic.collection-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + sort: + field: '@timestamp' + order: desc + policy: + _meta: + managed: true + managed_by: security_onion + package: + name: elastic_agent + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-endpoint_x_events_x_api: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json index 2390705f37..7bcc34de93 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json @@ -1,382 +1,383 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent-1.13.1", - "mapping": { - "total_fields": { - "limit": "10000" +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs" + }, + "codec": "best_compression", + "default_pipeline": "logs-elastic_agent-1.13.1", + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "query": { + "default_field": [ + "cloud.account.id", + "cloud.availability_zone", + "cloud.instance.id", + "cloud.instance.name", + "cloud.machine.type", + "cloud.provider", + "cloud.region", + "cloud.project.id", + "cloud.image.id", + "container.id", + "container.image.name", + "container.name", + "host.architecture", + "host.hostname", + "host.id", + "host.mac", + "host.name", + "host.os.family", + "host.os.kernel", + "host.os.name", + "host.os.platform", + "host.os.version", + "host.os.build", + "host.os.codename", + "host.type", + "ecs.version", + "agent.build.original", + "agent.ephemeral_id", + "agent.id", + "agent.name", + "agent.type", + "agent.version", + "log.level", + "message", + "elastic_agent.id", + "elastic_agent.process", + "elastic_agent.version", + "component.id", + "component.type", + "component.binary", + "component.state", + "component.old_state", + "unit.id", + "unit.type", + "unit.state", + "unit.old_state" + ] + } + } + }, + "mappings": { + "dynamic": false, + "dynamic_templates": [ + { + "container.labels": { + "path_match": "container.labels.*", + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "container": { + "properties": { + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version", - "component.id", - "component.type", - "component.binary", - "component.state", - "component.old_state", - "unit.id", - "unit.type", - "unit.state", - "unit.old_state" - ] } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" + } + }, + "agent": { + "properties": { + "build": { + "properties": { + "original": { + "ignore_above": 1024, + "type": "keyword" } } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "message": { - "type": "text" - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "elastic_agent": { + "properties": { + "process": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "snapshot": { + "type": "boolean" + } + } + }, + "message": { + "type": "text" + }, + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "component": { - "properties": { - "binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "old_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "wildcard" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } + } + }, + "instance": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "unit": { - "properties": { - "old_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "wildcard" - }, - "state": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } + } + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + }, + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } + } + } + } + }, + "component": { + "properties": { + "binary": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "wildcard" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "unit": { + "properties": { + "old_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "wildcard" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "type": "date" + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "text": { + "type": "text" } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" } + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" } } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" } } }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true + "event": { + "properties": { + "dataset": { + "type": "constant_keyword" + } + } } } + } + }, + "_meta": { + "package": { + "name": "elastic_agent" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@custom.json new file mode 100644 index 0000000000..1bf9313a9f --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@custom.json @@ -0,0 +1,12 @@ +{ + "template": { + "settings": {} + }, + "_meta": { + "package": { + "name": "endpoint" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@package.json new file mode 100644 index 0000000000..bf60f2543b --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@package.json @@ -0,0 +1,132 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs-endpoint.collection-diagnostic" + }, + "codec": "best_compression", + "default_pipeline": "logs-endpoint.diagnostic.collection-8.10.2", + "mapping": { + "total_fields": { + "limit": "10000" + }, + "ignore_malformed": "true" + }, + "query": { + "default_field": [ + "ecs.version", + "event.action", + "event.category", + "event.code", + "event.dataset", + "event.hash", + "event.id", + "event.kind", + "event.module", + "event.outcome", + "event.provider", + "event.type" + ] + } + } + }, + "mappings": { + "dynamic": false, + "properties": { + "@timestamp": { + "ignore_malformed": false, + "type": "date" + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "event": { + "properties": { + "severity": { + "type": "long" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence": { + "type": "long" + }, + "ingested": { + "type": "date" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "endpoint" + }, + "managed_by": "fleet", + "managed": true + } +}