From 1863bf4094001b43a73a713ac535605c3a781249 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 18 Dec 2018 15:09:24 -0500 Subject: [PATCH] Add code generator for Go (#249) This generates Go code based on the Elastic Common Schema. --- CHANGELOG.md | 1 + Makefile | 16 +- code/README.md | 6 + code/go/ecs/README.md | 8 + code/go/ecs/agent.go | 52 +++++ code/go/ecs/base.go | 49 +++++ code/go/ecs/client.go | 61 ++++++ code/go/ecs/cloud.go | 48 +++++ code/go/ecs/container.go | 43 ++++ code/go/ecs/destination.go | 50 +++++ code/go/ecs/doc.go | 22 ++ code/go/ecs/ecs.go | 31 +++ code/go/ecs/error.go | 34 ++++ code/go/ecs/event.go | 138 +++++++++++++ code/go/ecs/file.go | 74 +++++++ code/go/ecs/geo.go | 52 +++++ code/go/ecs/group.go | 30 +++ code/go/ecs/host.go | 58 ++++++ code/go/ecs/http.go | 55 +++++ code/go/ecs/log.go | 37 ++++ code/go/ecs/network.go | 94 +++++++++ code/go/ecs/observer.go | 57 ++++++ code/go/ecs/organization.go | 31 +++ code/go/ecs/os.go | 41 ++++ code/go/ecs/process.go | 62 ++++++ code/go/ecs/related.go | 32 +++ code/go/ecs/server.go | 61 ++++++ code/go/ecs/service.go | 62 ++++++ code/go/ecs/source.go | 50 +++++ code/go/ecs/url.go | 69 +++++++ code/go/ecs/user.go | 46 +++++ code/go/ecs/user_agent.go | 36 ++++ code/go/ecs/version.go | 23 +++ scripts/cmd/gocodegen/gocodegen.go | 311 +++++++++++++++++++++++++++++ scripts/go.mod | 1 + scripts/go.sum | 2 + 36 files changed, 1842 insertions(+), 1 deletion(-) create mode 100644 code/README.md create mode 100644 code/go/ecs/README.md create mode 100644 code/go/ecs/agent.go create mode 100644 code/go/ecs/base.go create mode 100644 code/go/ecs/client.go create mode 100644 code/go/ecs/cloud.go create mode 100644 code/go/ecs/container.go create mode 100644 code/go/ecs/destination.go create mode 100644 code/go/ecs/doc.go create mode 100644 code/go/ecs/ecs.go create mode 100644 code/go/ecs/error.go create mode 100644 code/go/ecs/event.go create mode 100644 code/go/ecs/file.go create mode 100644 code/go/ecs/geo.go create mode 100644 code/go/ecs/group.go create mode 100644 code/go/ecs/host.go create mode 100644 code/go/ecs/http.go create mode 100644 code/go/ecs/log.go create mode 100644 code/go/ecs/network.go create mode 100644 code/go/ecs/observer.go create mode 100644 code/go/ecs/organization.go create mode 100644 code/go/ecs/os.go create mode 100644 code/go/ecs/process.go create mode 100644 code/go/ecs/related.go create mode 100644 code/go/ecs/server.go create mode 100644 code/go/ecs/service.go create mode 100644 code/go/ecs/source.go create mode 100644 code/go/ecs/url.go create mode 100644 code/go/ecs/user.go create mode 100644 code/go/ecs/user_agent.go create mode 100644 code/go/ecs/version.go create mode 100644 scripts/cmd/gocodegen/gocodegen.go diff --git a/CHANGELOG.md b/CHANGELOG.md index bebbf219a1..3c4bead62c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ All notable changes to this project will be documented in this file based on the * Add fields `source.address`, `destination.address`, `client.address`, and `server.address`. #247 * Add `os.full` to capture full OS name, including version. #259 +* Add generated source code for Go. #249 ### Improvements * Improved the definition of the file fields #196 diff --git a/Makefile b/Makefile index 0205695b42..8498a9f59d 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,10 @@ clean: # Clean all markdown files for use-cases find ./use-cases -type f -name '*.md' -not -name 'README.md' -print0 | xargs -0 rm -- +# Alias to generate source code for all languages. +.PHONY: codegen +codegen: gocodegen + # Build schema.csv from schema files. .PHONY: csv csv: ve @@ -66,7 +70,17 @@ fmt: ve # Alias to generate everything. .PHONY: generate -generate: csv readme template fields +generate: csv readme template fields codegen + +# Generate Go code from the schema. +.PHONY: gocodegen +gocodegen: + find code/go/ecs -name '*.go' -not -name 'doc.go' | xargs rm + cd scripts \ + && $(FORCE_GO_MODULES) go run cmd/gocodegen/gocodegen.go \ + -version=$(VERSION) \ + -schema=../schemas \ + -out=../code/go/ecs # Check Makefile format. .PHONY: makelint diff --git a/code/README.md b/code/README.md new file mode 100644 index 0000000000..59d148f9df --- /dev/null +++ b/code/README.md @@ -0,0 +1,6 @@ +# Generated Source Code + +This directory contains source code that is generated from the Elastic Common +Schema (ECS). + +- [Go](go/ecs/) diff --git a/code/go/ecs/README.md b/code/go/ecs/README.md new file mode 100644 index 0000000000..98339a2539 --- /dev/null +++ b/code/go/ecs/README.md @@ -0,0 +1,8 @@ +# Go - Generated Source Code + +[![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs] + +[godocs]: http://godoc.org/github.com/elastic/ecs/code/go/ecs + +This package contains Go source code that is generated from the Elastic Common +Schema (ECS). diff --git a/code/go/ecs/agent.go b/code/go/ecs/agent.go new file mode 100644 index 0000000000..7f19e5d96e --- /dev/null +++ b/code/go/ecs/agent.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The agent fields contain the data about the software entity, if any, that +// collects, detects, or observes events on a host, or takes measurements on a +// host. Examples include Beats. Agents may also run on observers. ECS agent.* +// fields shall be populated with details of the agent running on the host or +// observer where the event happened or the measurement was taken. +type Agent struct { + // Version of the agent. + Version string `ecs:"version"` + + // Name of the agent. + // This is a name that can be given to an agent. This can be helpful if for + // example two Filebeat instances are running on the same host but a human + // readable separation is needed on which Filebeat instance data is coming + // from. + // If no name is given, the name is often left empty. + Name string `ecs:"name"` + + // Type of the agent. + // The agent type stays always the same and should be given by the agent + // used. In case of Filebeat the agent would always be Filebeat also if two + // Filebeat instances are run on the same machine. + Type string `ecs:"type"` + + // Unique identifier of this agent (if one exists). + // Example: For Beats this would be beat.id. + ID string `ecs:"id"` + + // Ephemeral identifier of this agent (if one exists). + // This id normally changes across restarts, but `agent.id` does not. + EphemeralID string `ecs:"ephemeral_id"` +} diff --git a/code/go/ecs/base.go b/code/go/ecs/base.go new file mode 100644 index 0000000000..11c000fb5e --- /dev/null +++ b/code/go/ecs/base.go @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +import ( + "time" +) + +// The base set contains all fields which are on the top level. These fields +// are common across all types of events. +type Base struct { + // Date/time when the event originated. + // For log events this is the date/time when the event was generated, and + // not when it was read. + // Required field for all events. + Timestamp time.Time `ecs:"@timestamp"` + + // List of keywords used to tag each event. + Tags string `ecs:"tags"` + + // Key/value pairs. + // Can be used to add meta information to events. Should not contain nested + // objects. All values are stored as keyword. + // Example: `docker` and `k8s` labels. + Labels map[string]interface{} `ecs:"labels"` + + // For log events the message field contains the log message. + // In other use cases the message field can be used to concatenate + // different values which are then freely searchable. If multiple messages + // exist, they can be combined into one message. + Message string `ecs:"message"` +} diff --git a/code/go/ecs/client.go b/code/go/ecs/client.go new file mode 100644 index 0000000000..3c4dac9182 --- /dev/null +++ b/code/go/ecs/client.go @@ -0,0 +1,61 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// A client is defined as the initiator of a network connection for events +// regarding sessions, connections, or bidirectional flow records. For TCP +// events, the client is the initiator of the TCP connection that sends the SYN +// packet(s). For other protocols, the client is generally the initiator or +// requestor in the network transaction. Some systems use the term "originator" +// to refer the client in TCP connections. The client fields describe details +// about the system acting as the client in the network event. Client fields +// are usually populated in conjunction with server fields. Client fields are +// generally not populated for packet-level events. +// Client / server representations can add semantic context to an exchange, +// which is helpful to visualize the data in certain situations. If your +// context falls in that category, you should still ensure that source and +// destination are filled appropriately. +type Client struct { + // Some event client addresses are defined ambiguously. The event will + // sometimes list an IP, a domain or a unix socket. You should always + // store the raw address in the `.address` field. + // Then it should be duplicated to `.ip` or `.domain`, depending on which + // one it is. + Address string `ecs:"address"` + + // IP address of the client. + // Can be one or multiple IPv4 or IPv6 addresses. + IP string `ecs:"ip"` + + // Port of the client. + Port int64 `ecs:"port"` + + // MAC address of the client. + MAC string `ecs:"mac"` + + // Client domain. + Domain string `ecs:"domain"` + + // Bytes sent from the client to the server. + Bytes int64 `ecs:"bytes"` + + // Packets sent from the client to the server. + Packets int64 `ecs:"packets"` +} diff --git a/code/go/ecs/cloud.go b/code/go/ecs/cloud.go new file mode 100644 index 0000000000..efbe85dfc1 --- /dev/null +++ b/code/go/ecs/cloud.go @@ -0,0 +1,48 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Fields related to the cloud or infrastructure the events are coming from. +type Cloud struct { + // Name of the cloud provider. Example values are ec2, gce, or + // digitalocean. + Provider string `ecs:"provider"` + + // Availability zone in which this host is running. + AvailabilityZone string `ecs:"availability_zone"` + + // Region in which this host is running. + Region string `ecs:"region"` + + // Instance ID of the host machine. + InstanceID string `ecs:"instance.id"` + + // Instance name of the host machine. + InstanceName string `ecs:"instance.name"` + + // Machine type of the host machine. + MachineType string `ecs:"machine.type"` + + // The cloud account or organization id used to identify different entities + // in a multi-tenant environment. + // Examples: AWS account id, Google Cloud ORG Id, or other unique + // identifier. + AccountID string `ecs:"account.id"` +} diff --git a/code/go/ecs/container.go b/code/go/ecs/container.go new file mode 100644 index 0000000000..a672fc6e3a --- /dev/null +++ b/code/go/ecs/container.go @@ -0,0 +1,43 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Container fields are used for meta information about the specific container +// that is the source of information. These fields help correlate data based +// containers from any runtime. +type Container struct { + // Runtime managing this container. + Runtime string `ecs:"runtime"` + + // Unique container id. + ID string `ecs:"id"` + + // Name of the image the container was built on. + ImageName string `ecs:"image.name"` + + // Container image tag. + ImageTag string `ecs:"image.tag"` + + // Container name. + Name string `ecs:"name"` + + // Image labels. + Labels map[string]interface{} `ecs:"labels"` +} diff --git a/code/go/ecs/destination.go b/code/go/ecs/destination.go new file mode 100644 index 0000000000..26a70e9cb7 --- /dev/null +++ b/code/go/ecs/destination.go @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Destination fields describe details about the destination of a packet/event. +// Destination fields are usually populated in conjunction with source fields. +type Destination struct { + // Some event destination addresses are defined ambiguously. The event will + // sometimes list an IP, a domain or a unix socket. You should always + // store the raw address in the `.address` field. + // Then it should be duplicated to `.ip` or `.domain`, depending on which + // one it is. + Address string `ecs:"address"` + + // IP address of the destination. + // Can be one or multiple IPv4 or IPv6 addresses. + IP string `ecs:"ip"` + + // Port of the destination. + Port int64 `ecs:"port"` + + // MAC address of the destination. + MAC string `ecs:"mac"` + + // Destination domain. + Domain string `ecs:"domain"` + + // Bytes sent from the destination to the source. + Bytes int64 `ecs:"bytes"` + + // Packets sent from the destination to the source. + Packets int64 `ecs:"packets"` +} diff --git a/code/go/ecs/doc.go b/code/go/ecs/doc.go new file mode 100644 index 0000000000..48c8e74d8c --- /dev/null +++ b/code/go/ecs/doc.go @@ -0,0 +1,22 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package ecs contains source code that is generated from the Elastic Common +// Schema (ECS). +// +// https://github.com/elastic/ecs#elastic-common-schema-ecs +package ecs diff --git a/code/go/ecs/ecs.go b/code/go/ecs/ecs.go new file mode 100644 index 0000000000..f6fa12884a --- /dev/null +++ b/code/go/ecs/ecs.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Meta-information specific to ECS. +type ECS struct { + // ECS version this event conforms to. `ecs.version` is a required field + // and must exist in all events. + // When querying across multiple indices -- which may conform to slightly + // different ECS versions -- this field lets integrations adjust to the + // schema version of the events. + // The current version is 1.0.0-beta2 . + Version string `ecs:"version"` +} diff --git a/code/go/ecs/error.go b/code/go/ecs/error.go new file mode 100644 index 0000000000..03151abe7b --- /dev/null +++ b/code/go/ecs/error.go @@ -0,0 +1,34 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// These fields can represent errors of any kind. Use them for errors that +// happen while fetching events or in cases where the event itself contains an +// error. +type Error struct { + // Unique identifier for the error. + ID string `ecs:"id"` + + // Error message. + Message string `ecs:"message"` + + // Error code describing the error. + Code string `ecs:"code"` +} diff --git a/code/go/ecs/event.go b/code/go/ecs/event.go new file mode 100644 index 0000000000..a281154f06 --- /dev/null +++ b/code/go/ecs/event.go @@ -0,0 +1,138 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +import ( + "time" +) + +// The event fields are used for context information about the log or metric +// event itself. A log is defined as an event containing details of something +// that happened. Log events must include the time at which the thing happened. +// Examples of log events include a process starting on a host, a network +// packet being sent from a source to a destination, or a network connection +// between a client and a server being initiated or closed. A metric is defined +// as an event containing one or more numerical or categorical measurements and +// the time at which the measurement was taken. Examples of metric events +// include memory pressure measured on a host, or vulnerabilities measured on a +// scanned host. +type Event struct { + // Unique ID to describe the event. + ID string `ecs:"id"` + + // The kind of the event. + // This gives information about what type of information the event + // contains, without being specific to the contents of the event. Examples + // are `event`, `state`, `alarm`. Warning: In future versions of ECS, we + // plan to provide a list of acceptable values for this field, please use + // with caution. + Kind string `ecs:"kind"` + + // Event category. + // This contains high-level information about the contents of the event. It + // is more generic than `event.action`, in the sense that typically a + // category contains multiple actions. Warning: In future versions of ECS, + // we plan to provide a list of acceptable values for this field, please + // use with caution. + Category string `ecs:"category"` + + // The action captured by the event. + // This describes the information in the event. It is more specific than + // `event.category`. Examples are `group-add`, `process-started`, + // `file-created`. The value is normally defined by the implementer. + Action string `ecs:"action"` + + // The outcome of the event. + // If the event describes an action, this fields contains the outcome of + // that action. Examples outcomes are `success` and `failure`. Warning: In + // future versions of ECS, we plan to provide a list of acceptable values + // for this field, please use with caution. + Outcome string `ecs:"outcome"` + + // Reserved for future usage. + // Please avoid using this field for user data. + Type string `ecs:"type"` + + // Name of the module this data is coming from. + // This information is coming from the modules used in Beats or Logstash. + Module string `ecs:"module"` + + // Name of the dataset. + // The concept of a `dataset` (fileset / metricset) is used in Beats as a + // subset of modules. It contains the information which is currently stored + // in metricset.name and metricset.module or fileset.name. + Dataset string `ecs:"dataset"` + + // Severity describes the severity of the event. What the different + // severity values mean can very different between use cases. It's up to + // the implementer to make sure severities are consistent across events. + Severity int64 `ecs:"severity"` + + // Raw text message of entire event. Used to demonstrate log integrity. + // This field is not indexed and doc_values are disabled. It cannot be + // searched, but it can be retrieved from `_source`. + Original string `ecs:"original"` + + // Hash (perhaps logstash fingerprint) of raw field to be able to + // demonstrate log integrity. + Hash string `ecs:"hash"` + + // Duration of the event in nanoseconds. + // If event.start and event.end are known this value should be the + // difference between the end and start time. + Duration time.Duration `ecs:"duration"` + + // This field should be populated when the event's timestamp does not + // include timezone information already (e.g. default Syslog timestamps). + // It's optional otherwise. + // Acceptable timezone formats are: a canonical ID (e.g. + // "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential + // (e.g. "-05:00"). + Timezone string `ecs:"timezone"` + + // event.created contains the date when the event was created. + // This timestamp is distinct from @timestamp in that @timestamp contains + // the processed timestamp. For logs these two timestamps can be different + // as the timestamp in the log line and when the event is read for example + // by Filebeat are not identical. `@timestamp` must contain the timestamp + // extracted from the log line, event.created when the log line is read. + // The same could apply to package capturing where @timestamp contains the + // timestamp extracted from the network package and event.created when the + // event was created. + // In case the two timestamps are identical, @timestamp should be used. + Created time.Time `ecs:"created"` + + // event.start contains the date when the event started or when the + // activity was first observed. + Start time.Time `ecs:"start"` + + // event.end contains the date when the event ended or when the activity + // was last observed. + End time.Time `ecs:"end"` + + // Risk score or priority of the event (e.g. security solutions). Use your + // system's original value here. + RiskScore float64 `ecs:"risk_score"` + + // Normalized risk score or priority of the event, on a scale of 0 to 100. + // This is mainly useful if you use more than one system that assigns risk + // scores, and you want to see a normalized value across all systems. + RiskScoreNorm float64 `ecs:"risk_score_norm"` +} diff --git a/code/go/ecs/file.go b/code/go/ecs/file.go new file mode 100644 index 0000000000..f876e1a7eb --- /dev/null +++ b/code/go/ecs/file.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +import ( + "time" +) + +// A file is defined as a set of information that has been created on, or has +// existed on a filesystem. File objects can be associated with host events, +// network events, and/or file events (e.g., those produced by File Integrity +// Monitoring [FIM] products or services). File fields provide details about +// the affected file associated with the event or metric. +type File struct { + // Path to the file. + Path string `ecs:"path"` + + // Target path for symlinks. + TargetPath string `ecs:"target_path"` + + // File extension. + // This should allow easy filtering by file extensions. + Extension string `ecs:"extension"` + + // File type (file, dir, or symlink). + Type string `ecs:"type"` + + // Device that is the source of the file. + Device string `ecs:"device"` + + // Inode representing the file in the filesystem. + Inode string `ecs:"inode"` + + // The user ID (UID) or security identifier (SID) of the file owner. + UID string `ecs:"uid"` + + // File owner's username. + Owner string `ecs:"owner"` + + // Primary group ID (GID) of the file. + Gid string `ecs:"gid"` + + // Primary group name of the file. + Group string `ecs:"group"` + + // Mode of the file in octal representation. + Mode string `ecs:"mode"` + + // File size in bytes (field is only added when `type` is `file`). + Size int64 `ecs:"size"` + + // Last time file content was modified. + Mtime time.Time `ecs:"mtime"` + + // Last time file metadata changed. + Ctime time.Time `ecs:"ctime"` +} diff --git a/code/go/ecs/geo.go b/code/go/ecs/geo.go new file mode 100644 index 0000000000..223e996991 --- /dev/null +++ b/code/go/ecs/geo.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Geo fields can carry data about a specific location related to an event or +// geo information derived from an IP field. +type Geo struct { + // Longitude and latitude. + Location string `ecs:"location"` + + // Name of the continent. + ContinentName string `ecs:"continent_name"` + + // Country name. + CountryName string `ecs:"country_name"` + + // Region name. + RegionName string `ecs:"region_name"` + + // City name. + CityName string `ecs:"city_name"` + + // Country ISO code. + CountryIsoCode string `ecs:"country_iso_code"` + + // Region ISO code. + RegionIsoCode string `ecs:"region_iso_code"` + + // User-defined description of a location, at the level of granularity they + // care about. + // Could be the name of their data centers, the floor number, if this + // describes a local physical entity, city names. + // Not typically used in automated geolocation. + Name string `ecs:"name"` +} diff --git a/code/go/ecs/group.go b/code/go/ecs/group.go new file mode 100644 index 0000000000..4ecc3676f0 --- /dev/null +++ b/code/go/ecs/group.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The group fields are meant to represent groups that are relevant to the +// event. +type Group struct { + // Unique identifier for the group on the system/platform. + ID string `ecs:"id"` + + // Name of the group. + Name string `ecs:"name"` +} diff --git a/code/go/ecs/host.go b/code/go/ecs/host.go new file mode 100644 index 0000000000..fbc4dce6a6 --- /dev/null +++ b/code/go/ecs/host.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// A host is defined as a general computing instance. ECS host.* fields should +// be populated with details about the host on which the event happened, or on +// which the measurement was taken. Host types include hardware, virtual +// machines, Docker containers, and Kubernetes nodes. +type Host struct { + // Hostname of the host. + // It normally contains what the `hostname` command returns on the host + // machine. + Hostname string `ecs:"hostname"` + + // Name of the host. + // It can contain what `hostname` returns on Unix systems, the fully + // qualified domain name, or a name specified by the user. The sender + // decides which value to use. + Name string `ecs:"name"` + + // Unique host id. + // As hostname is not always unique, use values that are meaningful in your + // environment. + // Example: The current usage of `beat.name`. + ID string `ecs:"id"` + + // Host ip address. + IP string `ecs:"ip"` + + // Host mac address. + MAC string `ecs:"mac"` + + // Type of host. + // For Cloud providers this can be the machine type like `t2.medium`. If + // vm, this could be the container, for example, or other information + // meaningful in your environment. + Type string `ecs:"type"` + + // Operating system architecture. + Architecture string `ecs:"architecture"` +} diff --git a/code/go/ecs/http.go b/code/go/ecs/http.go new file mode 100644 index 0000000000..19b409d619 --- /dev/null +++ b/code/go/ecs/http.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Fields related to HTTP activity. +type Http struct { + // Http request method. + // The field value must be normalized to lowercase for querying. See + // "Lowercase Capitalization" in the "Implementing ECS" section. + RequestMethod string `ecs:"request.method"` + + // The full http request body. + RequestBodyContent string `ecs:"request.body.content"` + + // Referrer for this HTTP request. + RequestReferrer string `ecs:"request.referrer"` + + // Http response status code. + ResponseStatusCode int64 `ecs:"response.status_code"` + + // The full http response body. + ResponseBodyContent string `ecs:"response.body.content"` + + // Http version. + Version string `ecs:"version"` + + // Total size in bytes of the request (body and headers). + RequestBytes int64 `ecs:"request.bytes"` + + // Size in bytes of the request body. + RequestBodyBytes int64 `ecs:"request.body.bytes"` + + // Total size in bytes of the response (body and headers). + ResponseBytes int64 `ecs:"response.bytes"` + + // Size in bytes of the response body. + ResponseBodyBytes int64 `ecs:"response.body.bytes"` +} diff --git a/code/go/ecs/log.go b/code/go/ecs/log.go new file mode 100644 index 0000000000..e2d0e07d73 --- /dev/null +++ b/code/go/ecs/log.go @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Fields which are specific to log events. +type Log struct { + // Log level of the log event. + // Some examples are `WARN`, `ERR`, `INFO`. + Level string `ecs:"level"` + + // This is the original log message and contains the full log message + // before splitting it up in multiple parts. + // In contrast to the `message` field which can contain an extracted part + // of the log message, this field contains the original, full log message. + // It can have already some modifications applied like encoding or new + // lines removed to clean up the log message. + // This field is not indexed and doc_values are disabled so it can't be + // queried but the value can be retrieved from `_source`. + Original string `ecs:"original"` +} diff --git a/code/go/ecs/network.go b/code/go/ecs/network.go new file mode 100644 index 0000000000..de34708f1c --- /dev/null +++ b/code/go/ecs/network.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The network is defined as the communication path over which a host or +// network event happens. The network.* fields should be populated with details +// about the network activity associated with an event. +type Network struct { + // Name given by operators to sections of their network. + Name string `ecs:"name"` + + // In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, + // pim, etc + // The field value must be normalized to lowercase for querying. See + // "Lowercase Capitalization" in the "Implementing ECS" section. + Type string `ecs:"type"` + + // IANA Protocol Number + // (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). + // Standardized list of protocols. This aligns well with NetFlow and sFlow + // related logs which use the IANA Protocol Number. + IANANumber string `ecs:"iana_number"` + + // Same as network.iana_number, but instead using the Keyword name of the + // transport layer (udp, tcp, ipv6-icmp, etc.) + // The field value must be normalized to lowercase for querying. See + // "Lowercase Capitalization" in the "Implementing ECS" section. + Transport string `ecs:"transport"` + + // A name given to an application. This can be arbitrarily assigned for + // things like microservices, but also apply to things like skype, icq, + // facebook, twitter. This would be used in situations where the vendor or + // service can be decoded such as from the source/dest IP owners, ports, or + // wire format. + // The field value must be normalized to lowercase for querying. See + // "Lowercase Capitalization" in the "Implementing ECS" section. + Application string `ecs:"application"` + + // L7 Network protocol name. ex. http, lumberjack, transport protocol. + // The field value must be normalized to lowercase for querying. See + // "Lowercase Capitalization" in the "Implementing ECS" section. + Protocol string `ecs:"protocol"` + + // Direction of the network traffic. + // Recommended values are: + // * inbound + // * outbound + // * internal + // * external + // * unknown + // + // When mapping events from a host-based monitoring context, populate this + // field from the host's point of view. + // When mapping events from a network or perimeter-based monitoring + // context, populate this field from the point of view of your network + // perimeter. + Direction string `ecs:"direction"` + + // Host IP address when the source IP address is the proxy. + ForwardedIP string `ecs:"forwarded_ip"` + + // A hash of source and destination IPs and ports, as well as the protocol + // used in a communication. This is a tool-agnostic standard to identify + // flows. + // Learn more at https://github.com/corelight/community-id-spec. + CommunityID string `ecs:"community_id"` + + // Total bytes transferred in both directions. + // If `source.bytes` and `destination.bytes` are known, `network.bytes` is + // their sum. + Bytes int64 `ecs:"bytes"` + + // Total packets transferred in both directions. + // If `source.packets` and `destination.packets` are known, + // `network.packets` is their sum. + Packets int64 `ecs:"packets"` +} diff --git a/code/go/ecs/observer.go b/code/go/ecs/observer.go new file mode 100644 index 0000000000..5eaeb82077 --- /dev/null +++ b/code/go/ecs/observer.go @@ -0,0 +1,57 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// An observer is defined as a special network, security, or application device +// used to detect, observe, or create network, security, or application-related +// events and metrics. This could be a custom hardware appliance or a server +// that has been configured to run special network, security, or application +// software. Examples include firewalls, intrusion detection/prevention +// systems, network monitoring sensors, web application firewalls, data loss +// prevention systems, and APM servers. The observer.* fields shall be +// populated with details of the system, if any, that detects, observes and/or +// creates a network, security, or application event or metric. Message queues +// and ETL components used in processing events or metrics are not considered +// observers in ECS. +type Observer struct { + // MAC address of the observer + MAC string `ecs:"mac"` + + // IP address of the observer. + IP string `ecs:"ip"` + + // Hostname of the observer. + Hostname string `ecs:"hostname"` + + // observer vendor information. + Vendor string `ecs:"vendor"` + + // Observer version. + Version string `ecs:"version"` + + // Observer serial number. + SerialNumber string `ecs:"serial_number"` + + // The type of the observer the data is coming from. + // There is no predefined list of observer types. Some examples are + // `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM + // server`. + Type string `ecs:"type"` +} diff --git a/code/go/ecs/organization.go b/code/go/ecs/organization.go new file mode 100644 index 0000000000..ce19626269 --- /dev/null +++ b/code/go/ecs/organization.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The organization fields enrich data with information about the company or +// entity the data is associated with. These fields help you arrange or filter +// data stored in an index by one or multiple organizations. +type Organization struct { + // Organization name. + Name string `ecs:"name"` + + // Unique identifier for the organization. + ID string `ecs:"id"` +} diff --git a/code/go/ecs/os.go b/code/go/ecs/os.go new file mode 100644 index 0000000000..a118950bbf --- /dev/null +++ b/code/go/ecs/os.go @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The OS fields contain information about the operating system. +type Os struct { + // Operating system platform (such centos, ubuntu, windows). + Platform string `ecs:"platform"` + + // Operating system name, without the version. + Name string `ecs:"name"` + + // Operating system name, including the version or code name. + Full string `ecs:"full"` + + // OS family (such as redhat, debian, freebsd, windows). + Family string `ecs:"family"` + + // Operating system version as a raw string. + Version string `ecs:"version"` + + // Operating system kernel version as a raw string. + Kernel string `ecs:"kernel"` +} diff --git a/code/go/ecs/process.go b/code/go/ecs/process.go new file mode 100644 index 0000000000..2b05d0611b --- /dev/null +++ b/code/go/ecs/process.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +import ( + "time" +) + +// These fields contain information about a process. These fields can help you +// correlate metrics information with a process id/name from a log message. +// The `process.pid` often stays in the metric itself and is copied to the +// global field for correlation. +type Process struct { + // Process id. + PID int64 `ecs:"pid"` + + // Process name. + // Sometimes called program name or similar. + Name string `ecs:"name"` + + // Process parent id. + PPID int64 `ecs:"ppid"` + + // Process arguments. + // May be filtered to protect sensitive information. + Args string `ecs:"args"` + + // Absolute path to the process executable. + Executable string `ecs:"executable"` + + // Process title. + // The proctitle, some times the same as process name. Can also be + // different: for example a browser setting its title to the web page + // currently opened. + Title string `ecs:"title"` + + // Thread ID. + ThreadID int64 `ecs:"thread.id"` + + // The time the process started. + Start time.Time `ecs:"start"` + + // The working directory of the process. + WorkingDirectory string `ecs:"working_directory"` +} diff --git a/code/go/ecs/related.go b/code/go/ecs/related.go new file mode 100644 index 0000000000..f4a7f8dc22 --- /dev/null +++ b/code/go/ecs/related.go @@ -0,0 +1,32 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// This field set is meant to facilitate pivoting around a piece of data. Some +// pieces of information can be seen in many places in ECS. To facilitate +// searching for them, append values to their corresponding field in +// `related.`. A concrete example is IP addresses, which can be under host, +// observer, source, destination, client, server, and network.forwarded_ip. If +// you append all IPs to `related.ip`, you can then search for a given IP +// trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. +type Related struct { + // All of the IPs seen on your event. + IP string `ecs:"ip"` +} diff --git a/code/go/ecs/server.go b/code/go/ecs/server.go new file mode 100644 index 0000000000..c05b64b2fc --- /dev/null +++ b/code/go/ecs/server.go @@ -0,0 +1,61 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// A Server is defined as the responder in a network connection for events +// regarding sessions, connections, or bidirectional flow records. For TCP +// events, the server is the receiver of the initial SYN packet(s) of the TCP +// connection. For other protocols, the server is generally the responder in +// the network transaction. Some systems actually use the term "responder" to +// refer the server in TCP connections. The server fields describe details +// about the system acting as the server in the network event. Server fields +// are usually populated in conjunction with client fields. Server fields are +// generally not populated for packet-level events. +// Client / server representations can add semantic context to an exchange, +// which is helpful to visualize the data in certain situations. If your +// context falls in that category, you should still ensure that source and +// destination are filled appropriately. +type Server struct { + // Some event server addresses are defined ambiguously. The event will + // sometimes list an IP, a domain or a unix socket. You should always + // store the raw address in the `.address` field. + // Then it should be duplicated to `.ip` or `.domain`, depending on which + // one it is. + Address string `ecs:"address"` + + // IP address of the server. + // Can be one or multiple IPv4 or IPv6 addresses. + IP string `ecs:"ip"` + + // Port of the server. + Port int64 `ecs:"port"` + + // MAC address of the server. + MAC string `ecs:"mac"` + + // Server domain. + Domain string `ecs:"domain"` + + // Bytes sent from the server to the client. + Bytes int64 `ecs:"bytes"` + + // Packets sent from the server to the client. + Packets int64 `ecs:"packets"` +} diff --git a/code/go/ecs/service.go b/code/go/ecs/service.go new file mode 100644 index 0000000000..cdeedd6b37 --- /dev/null +++ b/code/go/ecs/service.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The service fields describe the service for or from which the data was +// collected. These fields help you find and correlate logs for a specific +// service and version. +type Service struct { + // Unique identifier of the running service. + // This id should uniquely identify this service. This makes it possible to + // correlate logs and metrics for one specific service. + // Example: If you are experiencing issues with one redis instance, you can + // filter on that id to see metrics and logs for that single instance. + ID string `ecs:"id"` + + // Name of the service data is collected from. + // The name of the service is normally user given. This allows if two + // instances of the same service are running on the same machine they can + // be differentiated by the `service.name`. + // Also it allows for distributed services that run on multiple hosts to + // correlate the related instances based on the name. + // In the case of Elasticsearch the service.name could contain the cluster + // name. For Beats the service.name is by default a copy of the + // `service.type` field if no name is specified. + Name string `ecs:"name"` + + // The type of the service data is collected from. + // The type can be used to group and correlate logs and metrics from one + // service type. + // Example: If logs or metrics are collected from Elasticsearch, + // `service.type` would be `elasticsearch`. + Type string `ecs:"type"` + + // Current state of the service. + State string `ecs:"state"` + + // Version of the service the data was collected from. + // This allows to look at a data set only for a specific version of a + // service. + Version string `ecs:"version"` + + // Ephemeral identifier of this service (if one exists). + // This id normally changes across restarts, but `service.id` does not. + EphemeralID string `ecs:"ephemeral_id"` +} diff --git a/code/go/ecs/source.go b/code/go/ecs/source.go new file mode 100644 index 0000000000..35c4a5c05d --- /dev/null +++ b/code/go/ecs/source.go @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Source fields describe details about the source of a packet/event. Source +// fields are usually populated in conjunction with destination fields. +type Source struct { + // Some event source addresses are defined ambiguously. The event will + // sometimes list an IP, a domain or a unix socket. You should always + // store the raw address in the `.address` field. + // Then it should be duplicated to `.ip` or `.domain`, depending on which + // one it is. + Address string `ecs:"address"` + + // IP address of the source. + // Can be one or multiple IPv4 or IPv6 addresses. + IP string `ecs:"ip"` + + // Port of the source. + Port int64 `ecs:"port"` + + // MAC address of the source. + MAC string `ecs:"mac"` + + // Source domain. + Domain string `ecs:"domain"` + + // Bytes sent from the source to the destination. + Bytes int64 `ecs:"bytes"` + + // Packets sent from the source to the destination. + Packets int64 `ecs:"packets"` +} diff --git a/code/go/ecs/url.go b/code/go/ecs/url.go new file mode 100644 index 0000000000..de5d53d0d5 --- /dev/null +++ b/code/go/ecs/url.go @@ -0,0 +1,69 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// URL fields provide a complete URL, with scheme, host, and path. +type Url struct { + // Unmodified original url as seen in the event source. + // Note that in network monitoring, the observed URL may be a full URL, + // whereas in access logs, the URL is often just represented as a path. + // This field is meant to represent the URL as it was observed, complete or + // not. + Original string `ecs:"original"` + + // If full URLs are important to your use case, they should be stored in + // `url.full`, whether this field is reconstructed or present in the event + // source. + Full string `ecs:"full"` + + // Scheme of the request, such as "https". + // Note: The `:` is not part of the scheme. + Scheme string `ecs:"scheme"` + + // Domain of the request, such as "www.elastic.co". + // In some cases a URL may refer to an IP and/or port directly, without a + // domain name. In this case, the IP address would go to the `domain` + // field. + Domain string `ecs:"domain"` + + // Port of the request, such as 443. + Port int32 `ecs:"port"` + + // Path of the request, such as "/search". + Path string `ecs:"path"` + + // The query field describes the query string of the request, such as + // "q=elasticsearch". + // The `?` is excluded from the query string. If a URL contains no `?`, + // there is no query field. If there is a `?` but no query, the query field + // exists with an empty string. The `exists` query can be used to + // differentiate between the two cases. + Query string `ecs:"query"` + + // Portion of the url after the `#`, such as "top". + // The `#` is not part of the fragment. + Fragment string `ecs:"fragment"` + + // Username of the request. + Username string `ecs:"username"` + + // Password of the request. + Password string `ecs:"password"` +} diff --git a/code/go/ecs/user.go b/code/go/ecs/user.go new file mode 100644 index 0000000000..f43f0687c4 --- /dev/null +++ b/code/go/ecs/user.go @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The user fields describe information about the user that is relevant to the +// event. Fields can have one entry or multiple entries. If a user has more +// than one id, provide an array that includes all of them. +type User struct { + // One or multiple unique identifiers of the user. + ID string `ecs:"id"` + + // Short name or login of the user. + Name string `ecs:"name"` + + // User's full name, if available. + FullName string `ecs:"full_name"` + + // User email address. + Email string `ecs:"email"` + + // Unique user hash to correlate information for a user in anonymized form. + // Useful if `user.id` or `user.name` contain confidential information and + // cannot be used. + Hash string `ecs:"hash"` + + // Group the user is a part of. This field can contain a list of groups, if + // necessary. + Group string `ecs:"group"` +} diff --git a/code/go/ecs/user_agent.go b/code/go/ecs/user_agent.go new file mode 100644 index 0000000000..0ee3b280cb --- /dev/null +++ b/code/go/ecs/user_agent.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// The user_agent fields normally come from a browser request. They often show +// up in web service logs coming from the parsed user agent string. +type UserAgent struct { + // Unparsed version of the user_agent. + Original string `ecs:"original"` + + // Name of the user agent. + Name string `ecs:"name"` + + // Version of the user agent. + Version string `ecs:"version"` + + // Name of the device. + DeviceName string `ecs:"device.name"` +} diff --git a/code/go/ecs/version.go b/code/go/ecs/version.go new file mode 100644 index 0000000000..5a22ef62e2 --- /dev/null +++ b/code/go/ecs/version.go @@ -0,0 +1,23 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Version is the Elastic Common Schema version from which this was generated. +const Version = "1.0.0-beta2" diff --git a/scripts/cmd/gocodegen/gocodegen.go b/scripts/cmd/gocodegen/gocodegen.go new file mode 100644 index 0000000000..3f19acdca0 --- /dev/null +++ b/scripts/cmd/gocodegen/gocodegen.go @@ -0,0 +1,311 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "bufio" + "bytes" + "flag" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "text/template" + "unicode" + + wordwrap "github.com/mitchellh/go-wordwrap" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/go-ucfg/yaml" +) + +const license = ` +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License.` + +const typeTmpl = ` +{{.License}} + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +{{if .ImportTime -}} + +import ( + "time" +) + +{{end -}} + +// {{.Description}} +type {{.Name}} struct { +{{- range $field := .Fields}} + // {{$field.Comment}} + {{$field.Name}} {{$field.Type}} \u0060ecs:"{{$field.JSONKey}}"\u0060 +{{ end -}} +} +` + +const versionTmpl = ` +{{.License}} + +// Code generated by scripts/gocodegen.go - DO NOT EDIT. + +package ecs + +// Version is the Elastic Common Schema version from which this was generated. +const Version = "{{.Version}}" +` + +var ( + goFileTemplate = template.Must(template.New("type").Parse( + strings.Replace(typeTmpl[1:], `\u0060`, "`", -1))) + + versionFileTemplate = template.Must(template.New("version").Parse( + versionTmpl[1:])) +) + +type GoType struct { + License string + Description string + Name string + Fields []Field + ImportTime bool +} + +type Field struct { + Comment string + Name string + Type string + JSONKey string +} + +// Flags +var ( + schemaDir string + outputDir string + version string +) + +func init() { + flag.StringVar(&schemaDir, "schema", "schemas/", "Schema directory containing .yml files.") + flag.StringVar(&outputDir, "out", "code/go/ecs", "Output directory for .go files.") + flag.StringVar(&version, "version", "", "ECS Version (required)") +} + +func main() { + log.SetFlags(0) + flag.Parse() + + if version == "" { + log.Fatalf("Error: -version is required") + } + + paths, err := filepath.Glob(filepath.Join(schemaDir, "*.yml")) + if err != nil { + log.Fatalf("Error: %v", err) + } + + // Load schema files. + fields := common.Fields{} + for _, path := range paths { + f := common.Fields{} + + cfg, err := yaml.NewConfigWithFile(path) + if err != nil { + log.Fatalf("Error: %v", err) + } + if err = cfg.Unpack(&f); err != nil { + log.Fatalf("Error: %v", err) + } + + for key := range f { + // The definitions don't have the type group in and the template + // generator assumes otherwise keyword as default. + f[key].Type = "group" + } + + fields = append(fields, f...) + } + + // Generate Go source code. + goFiles := map[string][]byte{} + for _, group := range fields { + if group.Type == "group" { + t := GoType{ + License: license[1:], + Description: descriptionToComment("", group.Description), + Name: goTypeName(group.Name), + } + + for _, field := range group.Fields { + dataType := goDataType(field.Name, field.Type) + if strings.HasPrefix(dataType, "time.") { + t.ImportTime = true + } + + t.Fields = append(t.Fields, Field{ + Comment: descriptionToComment("\t", field.Description), + Name: goTypeName(field.Name), + Type: dataType, + JSONKey: field.Name, + }) + } + + b := new(bytes.Buffer) + err := goFileTemplate.Execute(b, t) + if err != nil { + log.Fatal(err) + } + + goFiles[group.Name+".go"] = b.Bytes() + } + } + + // Create version.go containing a the version as a constant. + b := new(bytes.Buffer) + err = versionFileTemplate.Execute(b, map[string]interface{}{ + "License": license[1:], + "Version": version, + }) + if err != nil { + log.Fatal(err) + } + goFiles["version.go"] = b.Bytes() + + // Output the files if there were no errors. + for name, data := range goFiles { + if err := os.MkdirAll(outputDir, 0755); err != nil { + log.Fatalf("Error: %v", err) + } + if err := ioutil.WriteFile(filepath.Join(outputDir, name), data, 0644); err != nil { + log.Fatalf("Error: %v", err) + } + } +} + +// isSeparate returns true if the character is a field name separator. This is +// used to detect the separators in fields like ephemeral_id or instance.name. +func isSeparator(c rune) bool { + switch c { + case '.', '_': + return true + case '@': + // This effectively filters @ from field names. + return true + default: + return false + } +} + +// descriptionToComment builds a comment string that is wrapped at 80 chars. +func descriptionToComment(indent, desc string) string { + textLength := 80 - len(strings.Replace(indent, "\t", " ", 4)+" // ") + lines := strings.Split(wordwrap.WrapString(desc, uint(textLength)), "\n") + if len(lines) > 0 { + // Remove empty first line. + if strings.TrimSpace(lines[0]) == "" { + lines = lines[1:] + } + } + if len(lines) > 0 { + // Remove empty last line. + if strings.TrimSpace(lines[len(lines)-1]) == "" { + lines = lines[:len(lines)-1] + } + } + for i := 0; i < len(lines); i++ { + + } + return trimTrailingWhitespace(strings.Join(lines, "\n"+indent+"// ")) +} + +func trimTrailingWhitespace(text string) string { + var lines [][]byte + s := bufio.NewScanner(bytes.NewBufferString(text)) + for s.Scan() { + lines = append(lines, bytes.TrimRightFunc(s.Bytes(), unicode.IsSpace)) + } + if err := s.Err(); err != nil { + log.Fatal(err) + } + return string(bytes.Join(lines, []byte("\n"))) +} + +// goDataType returns the Go type to use for Elasticsearch mapping data type. +func goDataType(fieldName, elasticsearchDataType string) string { + // Special cases. + switch { + case fieldName == "duration" && elasticsearchDataType == "long": + return "time.Duration" + } + + switch elasticsearchDataType { + case "keyword", "text", "ip", "geo_point": + return "string" + case "long": + return "int64" + case "integer": + return "int32" + case "float": + return "float64" + case "date": + return "time.Time" + case "object": + return "map[string]interface{}" + default: + log.Fatal("no translation for ", elasticsearchDataType) + return "" + } +} + +// abbreviations capitalizes common abbreviations. +func abbreviations(abv string) string { + switch strings.ToLower(abv) { + case "id", "ppid", "pid", "mac", "ip", "iana", "uid", "ecs": + return strings.ToUpper(abv) + default: + return abv + } +} + +// goTypeName removes special characters ('_', '.', '@') and returns a +// camel-cased name. +func goTypeName(name string) string { + var b strings.Builder + for _, w := range strings.FieldsFunc(name, isSeparator) { + b.WriteString(strings.Title(abbreviations(w))) + } + return b.String() +} diff --git a/scripts/go.mod b/scripts/go.mod index ca3c046007..f1c1c9c239 100644 --- a/scripts/go.mod +++ b/scripts/go.mod @@ -5,6 +5,7 @@ require ( github.com/elastic/go-ucfg v0.6.5 github.com/gofrs/uuid v3.1.0+incompatible // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect + github.com/mitchellh/go-wordwrap v1.0.0 github.com/pkg/errors v0.8.0 // indirect go.uber.org/atomic v1.3.2 // indirect go.uber.org/multierr v1.1.0 // indirect diff --git a/scripts/go.sum b/scripts/go.sum index d28431f823..ea7130c712 100644 --- a/scripts/go.sum +++ b/scripts/go.sum @@ -6,6 +6,8 @@ github.com/gofrs/uuid v3.1.0+incompatible h1:q2rtkjaKT4YEr6E1kamy0Ha4RtepWlQBedy github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=