From 5d62e2e9d5aca5e94b3d2213caaefb0d5fcf61ce Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 11 Dec 2018 09:02:11 -0500 Subject: [PATCH] Introduce size metrics and request.body.content field in HTTP. (#239) --- CHANGELOG.md | 5 ++++- README.md | 7 ++++++- fields.yml | 39 ++++++++++++++++++++++++++++++++++++++- schema.csv | 7 ++++++- schemas/http.yml | 39 ++++++++++++++++++++++++++++++++++++++- template.json | 28 ++++++++++++++++++++++++++-- use-cases/web-logs.md | 2 +- use-cases/web-logs.yml | 2 +- 8 files changed, 120 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0f61b1038..5809769561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ All notable changes to this project will be documented in this file based on the ## Unreleased ### Breaking changes -* Changed `device.*` fields to `observer.*` fields to eliminate user confusion. #238 +* Changed `device.*` fields to `observer.*` fields to eliminate user confusion. #238 * Rename `network.total.bytes` to `network.bytes` and `network.total.packets` to `network.packets`. #179 * Remove `network.inbound.bytes`, `network.inbound.packets`, @@ -30,6 +30,9 @@ All notable changes to this project will be documented in this file based on the * Add `process.executable` field. #209 * Add `process.working_directory` and `process.start`. #215 * Reintroduce `http`. #237 + * Move `http.response.body` to `http.response.body.content`. #239 + * Add `http.request.body.content`. #239 + * Add HTTP size metric fields. #239 * Add `user.full_name` field. #201 * Add `network.community_id` field. #208 * Add fields `geo.country_name` and `geo.region_iso_code`. #214 diff --git a/README.md b/README.md index 5904b3885c..3e07642f3c 100644 --- a/README.md +++ b/README.md @@ -299,10 +299,15 @@ Fields related to HTTP activity. | Field | Description | Level | Type | Example | |---|---|---|---|---| | http.request.method | Http request method.
The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section. | extended | keyword | `get, post, put` | +| http.request.body.content | The full http request body. | extended | keyword | `Hello world` | | http.request.referrer | Referrer for this HTTP request. | extended | keyword | `https://blog.example.com/` | | http.response.status_code | Http response status code. | extended | long | `404` | -| http.response.body | The full http response body. | extended | keyword | `Hello world` | +| http.response.body.content | The full http response body. | extended | keyword | `Hello world` | | http.version | Http version. | extended | keyword | `1.1` | +| http.request.bytes | Total size in bytes of the request (body and headers). | extended | long | `1437` | +| http.request.body.bytes | Size in bytes of the request body. | extended | long | `887` | +| http.response.bytes | Total size in bytes of the response (body and headers). | extended | long | `1437` | +| http.response.body.bytes | Size in bytes of the response body. | extended | long | `887` | ## Log fields diff --git a/fields.yml b/fields.yml index 89be3062ad..1dbf9562ad 100644 --- a/fields.yml +++ b/fields.yml @@ -869,6 +869,13 @@ "Lowercase Capitalization" in the "Implementing ECS" section. example: get, post, put + - name: request.body.content + level: extended + type: keyword + description: > + The full http request body. + example: Hello world + - name: request.referrer level: extended type: keyword @@ -883,7 +890,7 @@ Http response status code. example: 404 - - name: response.body + - name: response.body.content level: extended type: keyword description: > @@ -897,6 +904,36 @@ Http version. example: 1.1 + # Metrics + - name: request.bytes + level: extended + type: long + description: > + Total size in bytes of the request (body and headers). + example: 1437 + + - name: request.body.bytes + level: extended + type: long + description: > + Size in bytes of the request body. + example: 887 + + - name: response.bytes + level: extended + type: long + description: > + Total size in bytes of the response (body and headers). + example: 1437 + + - name: response.body.bytes + level: extended + type: long + description: > + Size in bytes of the response body. + example: 887 + + - name: log title: Log description: > diff --git a/schema.csv b/schema.csv index 129bcaa299..6da2217719 100644 --- a/schema.csv +++ b/schema.csv @@ -88,9 +88,14 @@ host.ip,ip,core, host.mac,keyword,core, host.name,keyword,core, host.type,keyword,core, +http.request.body.bytes,long,extended,887 +http.request.body.content,keyword,extended,Hello world +http.request.bytes,long,extended,1437 http.request.method,keyword,extended,"get, post, put" http.request.referrer,keyword,extended,https://blog.example.com/ -http.response.body,keyword,extended,Hello world +http.response.body.bytes,long,extended,887 +http.response.body.content,keyword,extended,Hello world +http.response.bytes,long,extended,1437 http.response.status_code,long,extended,404 http.version,keyword,extended,1.1 log.level,keyword,core,ERR diff --git a/schemas/http.yml b/schemas/http.yml index 687adc9add..a7ee23348d 100644 --- a/schemas/http.yml +++ b/schemas/http.yml @@ -17,6 +17,13 @@ "Lowercase Capitalization" in the "Implementing ECS" section. example: get, post, put + - name: request.body.content + level: extended + type: keyword + description: > + The full http request body. + example: Hello world + - name: request.referrer level: extended type: keyword @@ -31,7 +38,7 @@ Http response status code. example: 404 - - name: response.body + - name: response.body.content level: extended type: keyword description: > @@ -44,3 +51,33 @@ description: > Http version. example: 1.1 + + # Metrics + - name: request.bytes + level: extended + type: long + description: > + Total size in bytes of the request (body and headers). + example: 1437 + + - name: request.body.bytes + level: extended + type: long + description: > + Size in bytes of the request body. + example: 887 + + - name: response.bytes + level: extended + type: long + description: > + Total size in bytes of the response (body and headers). + example: 1437 + + - name: response.body.bytes + level: extended + type: long + description: > + Size in bytes of the response body. + example: 887 + diff --git a/template.json b/template.json index 1cbd8a1b12..aa565ea373 100644 --- a/template.json +++ b/template.json @@ -412,6 +412,20 @@ "properties": { "request": { "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, "method": { "ignore_above": 1024, "type": "keyword" @@ -425,8 +439,18 @@ "response": { "properties": { "body": { - "ignore_above": 1024, - "type": "keyword" + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" }, "status_code": { "type": "long" diff --git a/use-cases/web-logs.md b/use-cases/web-logs.md index 2d0920d4e0..57f9a96062 100644 --- a/use-cases/web-logs.md +++ b/use-cases/web-logs.md @@ -14,7 +14,7 @@ Using the fields as represented here is not expected to conflict with ECS, but m | [http.request.method](../README.md#http.request.method) | Http request method. | extended | keyword | `GET, POST, PUT` | | [http.request.referrer](../README.md#http.request.referrer) | Referrer for this HTTP request. | extended | keyword | `https://blog.example.com/` | | [http.response.status_code](../README.md#http.response.status_code) | Http response status code. | extended | long | `404` | -| [http.response.body](../README.md#http.response.body) | The full http response body. | extended | keyword | `Hello world` | +| [http.response.body.content](../README.md#http.response.body.content) | The full http response body. | extended | keyword | `Hello world` | | [http.version](../README.md#http.version) | Http version. | extended | keyword | `1.1` | | *user_agent.** | *The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string.
* | | | | | [user_agent.original](../README.md#user_agent.original) | Unparsed version of the user_agent. | extended | keyword | `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` | diff --git a/use-cases/web-logs.yml b/use-cases/web-logs.yml index d05ee53642..06a8f47d9e 100644 --- a/use-cases/web-logs.yml +++ b/use-cases/web-logs.yml @@ -46,7 +46,7 @@ fields: Http response status code. example: 404 - - name: response.body + - name: response.body.content type: keyword description: > The full http response body.