Skip to content

Commit

Permalink
Add community_id, fingerprint, network_direction, registered_domain p…
Browse files Browse the repository at this point in the history 
…rocessors (#3011)

Add ingest processor definitions for `community_id`, `fingerprint`, `network_direction`, and `registered_domain` processors. All parameters and descriptions were scraped from the 8.16 docs.

Add `ip` to the list of types supported by the `convert` processor.

Add `ecs_compatibility` to grok processor.

Add `output_format` to date processor.

Related issues

- Fixes #2309
- Fixes #2553
- Fixes #2593
- Fixes #2617
  • Loading branch information
@andrewkroh
andrewkroh authored Oct 14, 2024
1 parent 7610041 commit d1857ea
Show file tree
Hide file tree
Showing 2 changed files with 201 additions and 2 deletions.
4 changes: 4 additions & 0 deletions specification/_doc_ids/table.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ cluster-stats,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/c
cluster-update-settings,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cluster-update-settings.html
cluster,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cluster.html
common-options,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/common-options.html
community-id-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/community-id-processor.html
connector-sync-job-cancel,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cancel-connector-sync-job-api.html
connector-sync-job-delete,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/delete-connector-sync-job-api.html
connector-sync-job-get,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/get-connector-sync-job-api.html
Expand Down Expand Up @@ -157,6 +158,7 @@ fail-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/
field-and-document-access-control,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/field-and-document-access-control.html
field-usage-stats,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/field-usage-stats.html
find-structure,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/find-structure.html
fingerprint-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/fingerprint-processor.html
foreach-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/foreach-processor.html
fuzziness,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/common-options.html#fuzziness
gap-policy,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/search-aggregations-pipeline.html#gap-policy
Expand Down Expand Up @@ -329,6 +331,7 @@ modules-scripting,https://www.elastic.co/guide/en/elasticsearch/reference/{branc
modules-snapshots,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/modules-snapshots.html
monitor-elasticsearch-cluster,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/monitor-elasticsearch-cluster.html
multi-fields,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/multi-fields.html
network-direction-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/network-direction-processor.html
node-roles,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/modules-node.html#node-roles
paginate-search-results,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/paginate-search-results.html
painless-contexts,https://www.elastic.co/guide/en/elasticsearch/painless/{branch}/painless-contexts.html
Expand Down Expand Up @@ -407,6 +410,7 @@ query-dsl,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/query
realtime,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/docs-get.html#realtime
redact-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/redact-processor.html
regexp-syntax,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/regexp-syntax.html
registered-domain-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/registered-domain-processor.html
remove-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/remove-processor.html
rename-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/rename-processor.html
reroute-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/reroute-processor.html
Expand Down
199 changes: 197 additions & 2 deletions specification/ingest/_types/Processors.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ export class ProcessorContainer {
* @doc_id ingest-circle-processor
*/
circle?: CircleProcessor
/**
* Computes the Community ID for network flow data as defined in the
* Community ID Specification. You can use a community ID to correlate network
* events related to a single flow.
* @doc_id community-id-processor
*/
community_id?: CommunityIDProcessor
/**
* Converts a field in the currently ingested document to a different type, such as converting a string to an integer.
* If the field value is an array, all members will be converted.
Expand Down Expand Up @@ -106,6 +113,12 @@ export class ProcessorContainer {
* @doc_id fail-processor
*/
fail?: FailProcessor
/**
* Computes a hash of the document’s content. You can use this hash for
* content fingerprinting.
* @doc_id fingerprint-processor
*/
fingerprint?: FingerprintProcessor
/**
* Runs an ingest processor on each element of an array or object.
* @doc_id foreach-processor
Expand Down Expand Up @@ -169,6 +182,12 @@ export class ProcessorContainer {
* @doc_id lowercase-processor
*/
lowercase?: LowercaseProcessor
/**
* Calculates the network direction given a source IP address, destination IP
* address, and a list of internal networks.
* @doc_id network-direction-processor
*/
network_direction?: NetworkDirectionProcessor
/**
* Executes another pipeline.
* @doc_id pipeline-processor
Expand All @@ -181,6 +200,14 @@ export class ProcessorContainer {
* @doc_id redact-processor
*/
redact?: RedactProcessor
/**
* Extracts the registered domain (also known as the effective top-level
* domain or eTLD), sub-domain, and top-level domain from a fully qualified
* domain name (FQDN). Uses the registered domains defined in the Mozilla
* Public Suffix List.
* @doc_id registered-domain-processor
*/
registered_domain?: RegisteredDomainProcessor
/**
* Removes existing fields.
* If one field doesn’t exist, an exception will be thrown.
Expand Down Expand Up @@ -528,13 +555,77 @@ export class CircleProcessor extends ProcessorBase {
target_field?: Field
}

export class CommunityIDProcessor extends ProcessorBase {
/**
* Field containing the source IP address.
* @server_default source.ip
*/
source_ip?: Field
/**
* Field containing the source port.
* @server_default source.port
*/
source_port?: Field
/**
* Field containing the destination IP address.
* @server_default destination.ip
*/
destination_ip?: Field
/**
* Field containing the destination port.
* @server_default destination.port
*/
destination_port?: Field
/**
* Field containing the IANA number.
* @server_default network.iana_number
*/
iana_number?: Field
/**
* Field containing the ICMP type.
* @server_default icmp.type
*/
icmp_type?: Field
/**
* Field containing the ICMP code.
* @server_default icmp.code
*/
icmp_code?: Field
/**
* Field containing the transport protocol name or number. Used only when the
* iana_number field is not present. The following protocol names are currently
* supported: eigrp, gre, icmp, icmpv6, igmp, ipv6-icmp, ospf, pim, sctp, tcp, udp
* @server_default network.transport
*/
transport?: Field
/**
* Output field for the community ID.
* @server_default network.community_id
*/
target_field?: Field
/**
* Seed for the community ID hash. Must be between 0 and 65535 (inclusive). The
* seed can prevent hash collisions between network domains, such as a staging
* and production network that use the same addressing scheme.
* @server_default 0
*/
seed?: integer
/**
* If true and any required fields are missing, the processor quietly exits
* without modifying the document.
* @server_default true
*/
ignore_missing?: boolean
}

export enum ConvertType {
integer,
long,
float,
double,
string,
float,
boolean,
ip,
string,
auto
}

Expand Down Expand Up @@ -662,6 +753,12 @@ export class DateProcessor extends ProcessorBase {
* @server_default UTC
*/
timezone?: string
/**
* The format to use when writing the date to target_field. Must be a valid
* java time pattern.
* @server_default yyyy-MM-dd'T'HH:mm:ss.SSSXXX
*/
output_format?: string
}

export class DissectProcessor extends ProcessorBase {
Expand Down Expand Up @@ -756,6 +853,44 @@ export class FailProcessor extends ProcessorBase {
message: string
}

export enum FingerprintDigest {
md5 = 'MD5',
sha1 = 'SHA-1',
sha256 = 'SHA-256',
sha512 = 'SHA-512',
murmurHash3 = 'MurmurHash3'
}

export class FingerprintProcessor extends ProcessorBase {
/**
* Array of fields to include in the fingerprint. For objects, the processor
* hashes both the field key and value. For other fields, the processor hashes
* only the field value.
*/
fields: Fields
/**
* Output field for the fingerprint.
* @server_default fingerprint
*/
target_field?: Field
/**
* Salt value for the hash function.
*/
salt?: string
/**
* The hash method used to compute the fingerprint. Must be one of MD5, SHA-1,
* SHA-256, SHA-512, or MurmurHash3.
* @server_default SHA-1
*/
method?: FingerprintDigest
/**
* If true, the processor ignores any missing fields. If all fields are
* missing, the processor silently exits without modifying the document.
* @server_default false
*/
ignore_missing?: boolean
}

export class ForeachProcessor extends ProcessorBase {
/**
* Field containing array or object values.
Expand All @@ -773,6 +908,12 @@ export class ForeachProcessor extends ProcessorBase {
}

export class GrokProcessor extends ProcessorBase {
/**
* Must be disabled or v1. If v1, the processor uses patterns with Elastic
* Common Schema (ECS) field names.
* @server_default disabled
*/
ecs_compatibility?: string
/**
* The field to use for grok expression parsing.
*/
Expand Down Expand Up @@ -1046,6 +1187,42 @@ export class LowercaseProcessor extends ProcessorBase {
target_field?: Field
}

export class NetworkDirectionProcessor extends ProcessorBase {
/**
* Field containing the source IP address.
* @server_default source.ip
*/
source_ip?: Field
/**
* Field containing the destination IP address.
* @server_default destination.ip
*/
destination_ip?: Field
/**
* Output field for the network direction.
* @server_default network.direction
*/
target_field?: Field
/**
* List of internal networks. Supports IPv4 and IPv6 addresses and ranges in
* CIDR notation. Also supports the named ranges listed below. These may be
* constructed with template snippets. Must specify only one of
* internal_networks or internal_networks_field.
*/
internal_networks?: string[]
/**
* A field on the given document to read the internal_networks configuration
* from.
*/
internal_networks_field?: Field
/**
* If true and any required fields are missing, the processor quietly exits
* without modifying the document.
* @server_default true
*/
ignore_missing?: boolean
}

export class PipelineProcessor extends ProcessorBase {
/**
* The name of the pipeline to execute.
Expand Down Expand Up @@ -1102,6 +1279,24 @@ export class RedactProcessor extends ProcessorBase {
trace_redact?: boolean
}

export class RegisteredDomainProcessor extends ProcessorBase {
/**
* Field containing the source FQDN.
*/
field: Field
/**
* Object field containing extracted domain components. If an empty string,
* the processor adds components to the document’s root.
*/
target_field?: Field
/**
* If true and any required fields are missing, the processor quietly exits
* without modifying the document.
* @server_default true
*/
ignore_missing?: boolean
}

export class RemoveProcessor extends ProcessorBase {
/**
* Fields to be removed. Supports template snippets.
Expand Down

0 comments on commit d1857ea

Please sign in to comment.