From 9a2070ac7481e622e6832edbf2010752b421740e Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 5 Dec 2018 15:21:18 -0500 Subject: [PATCH 1/5] Introduce total and body size metrics for HTTP. --- README.md | 4 ++++ fields.yml | 30 ++++++++++++++++++++++++++++++ schema.csv | 4 ++++ schemas/http.yml | 30 ++++++++++++++++++++++++++++++ template.json | 20 ++++++++++++++++++++ 5 files changed, 88 insertions(+) diff --git a/README.md b/README.md index 5904b3885c..480f448594 100644 --- a/README.md +++ b/README.md @@ -303,6 +303,10 @@ Fields related to HTTP activity. | http.response.status_code | Http response status code. | extended | long | `404` | | http.response.body | The full http response body. | extended | keyword | `Hello world` | | http.version | Http version. | extended | keyword | `1.1` | +| http.request.bytes.total | Size in bytes of the request body and headers combined. | extended | long | `1437` | +| http.request.bytes.body | Size in bytes of the request body. | extended | long | `887` | +| http.response.bytes.total | Size in bytes of the response body and headers combined. | extended | long | `1437` | +| http.response.bytes.body | Size in bytes of the response body. | extended | long | `887` | ## Log fields diff --git a/fields.yml b/fields.yml index 89be3062ad..6e119f20d9 100644 --- a/fields.yml +++ b/fields.yml @@ -897,6 +897,36 @@ Http version. example: 1.1 + # Metrics + - name: request.bytes.total + level: extended + type: long + description: > + Size in bytes of the request body and headers combined. + example: 1437 + + - name: request.bytes.body + level: extended + type: long + description: > + Size in bytes of the request body. + example: 887 + + - name: response.bytes.total + level: extended + type: long + description: > + Size in bytes of the response body and headers combined. + example: 1437 + + - name: response.bytes.body + level: extended + type: long + description: > + Size in bytes of the response body. + example: 887 + + - name: log title: Log description: > diff --git a/schema.csv b/schema.csv index 129bcaa299..adfef1620f 100644 --- a/schema.csv +++ b/schema.csv @@ -88,9 +88,13 @@ host.ip,ip,core, host.mac,keyword,core, host.name,keyword,core, host.type,keyword,core, +http.request.bytes.body,long,extended,887 +http.request.bytes.total,long,extended,1437 http.request.method,keyword,extended,"get, post, put" http.request.referrer,keyword,extended,https://blog.example.com/ http.response.body,keyword,extended,Hello world +http.response.bytes.body,long,extended,887 +http.response.bytes.total,long,extended,1437 http.response.status_code,long,extended,404 http.version,keyword,extended,1.1 log.level,keyword,core,ERR diff --git a/schemas/http.yml b/schemas/http.yml index 687adc9add..c96e50044d 100644 --- a/schemas/http.yml +++ b/schemas/http.yml @@ -44,3 +44,33 @@ description: > Http version. example: 1.1 + + # Metrics + - name: request.bytes.total + level: extended + type: long + description: > + Size in bytes of the request body and headers combined. + example: 1437 + + - name: request.bytes.body + level: extended + type: long + description: > + Size in bytes of the request body. + example: 887 + + - name: response.bytes.total + level: extended + type: long + description: > + Size in bytes of the response body and headers combined. + example: 1437 + + - name: response.bytes.body + level: extended + type: long + description: > + Size in bytes of the response body. + example: 887 + diff --git a/template.json b/template.json index 1cbd8a1b12..485cf70e10 100644 --- a/template.json +++ b/template.json @@ -412,6 +412,16 @@ "properties": { "request": { "properties": { + "bytes": { + "properties": { + "body": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, "method": { "ignore_above": 1024, "type": "keyword" @@ -428,6 +438,16 @@ "ignore_above": 1024, "type": "keyword" }, + "bytes": { + "properties": { + "body": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, "status_code": { "type": "long" } From 240661eab45e61c7b99f44c7c5671d6542100736 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 7 Dec 2018 16:06:03 -0500 Subject: [PATCH 2/5] Try out the size metrics as discussed in the PR --- README.md | 10 +++++----- fields.yml | 14 +++++++------- schema.csv | 10 +++++----- schemas/http.yml | 14 +++++++------- template.json | 24 ++++++++++++------------ use-cases/web-logs.md | 2 +- 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 480f448594..3cff91bb2d 100644 --- a/README.md +++ b/README.md @@ -301,12 +301,12 @@ Fields related to HTTP activity. | http.request.method | Http request method.
The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section. | extended | keyword | `get, post, put` | | http.request.referrer | Referrer for this HTTP request. | extended | keyword | `https://blog.example.com/` | | http.response.status_code | Http response status code. | extended | long | `404` | -| http.response.body | The full http response body. | extended | keyword | `Hello world` | +| http.response.body.content | The full http response body. | extended | keyword | `Hello world` | | http.version | Http version. | extended | keyword | `1.1` | -| http.request.bytes.total | Size in bytes of the request body and headers combined. | extended | long | `1437` | -| http.request.bytes.body | Size in bytes of the request body. | extended | long | `887` | -| http.response.bytes.total | Size in bytes of the response body and headers combined. | extended | long | `1437` | -| http.response.bytes.body | Size in bytes of the response body. | extended | long | `887` | +| http.request.bytes | Total size in bytes of the request (body and headers). | extended | long | `1437` | +| http.request.body.bytes | Size in bytes of the request body. | extended | long | `887` | +| http.response.bytes | Total size in bytes of the response (body and headers). | extended | long | `1437` | +| http.response.body.bytes | Size in bytes of the response body. | extended | long | `887` | ## Log fields diff --git a/fields.yml b/fields.yml index 6e119f20d9..0f567a8f47 100644 --- a/fields.yml +++ b/fields.yml @@ -883,7 +883,7 @@ Http response status code. example: 404 - - name: response.body + - name: response.body.content level: extended type: keyword description: > @@ -898,28 +898,28 @@ example: 1.1 # Metrics - - name: request.bytes.total + - name: request.bytes level: extended type: long description: > - Size in bytes of the request body and headers combined. + Total size in bytes of the request (body and headers). example: 1437 - - name: request.bytes.body + - name: request.body.bytes level: extended type: long description: > Size in bytes of the request body. example: 887 - - name: response.bytes.total + - name: response.bytes level: extended type: long description: > - Size in bytes of the response body and headers combined. + Total size in bytes of the response (body and headers). example: 1437 - - name: response.bytes.body + - name: response.body.bytes level: extended type: long description: > diff --git a/schema.csv b/schema.csv index adfef1620f..ec846f1d16 100644 --- a/schema.csv +++ b/schema.csv @@ -88,13 +88,13 @@ host.ip,ip,core, host.mac,keyword,core, host.name,keyword,core, host.type,keyword,core, -http.request.bytes.body,long,extended,887 -http.request.bytes.total,long,extended,1437 +http.request.body.bytes,long,extended,887 +http.request.bytes,long,extended,1437 http.request.method,keyword,extended,"get, post, put" http.request.referrer,keyword,extended,https://blog.example.com/ -http.response.body,keyword,extended,Hello world -http.response.bytes.body,long,extended,887 -http.response.bytes.total,long,extended,1437 +http.response.body.bytes,long,extended,887 +http.response.body.content,keyword,extended,Hello world +http.response.bytes,long,extended,1437 http.response.status_code,long,extended,404 http.version,keyword,extended,1.1 log.level,keyword,core,ERR diff --git a/schemas/http.yml b/schemas/http.yml index c96e50044d..341bf154d2 100644 --- a/schemas/http.yml +++ b/schemas/http.yml @@ -31,7 +31,7 @@ Http response status code. example: 404 - - name: response.body + - name: response.body.content level: extended type: keyword description: > @@ -46,28 +46,28 @@ example: 1.1 # Metrics - - name: request.bytes.total + - name: request.bytes level: extended type: long description: > - Size in bytes of the request body and headers combined. + Total size in bytes of the request (body and headers). example: 1437 - - name: request.bytes.body + - name: request.body.bytes level: extended type: long description: > Size in bytes of the request body. example: 887 - - name: response.bytes.total + - name: response.bytes level: extended type: long description: > - Size in bytes of the response body and headers combined. + Total size in bytes of the response (body and headers). example: 1437 - - name: response.bytes.body + - name: response.body.bytes level: extended type: long description: > diff --git a/template.json b/template.json index 485cf70e10..c63c3b9bf3 100644 --- a/template.json +++ b/template.json @@ -412,16 +412,16 @@ "properties": { "request": { "properties": { - "bytes": { + "body": { "properties": { - "body": { - "type": "long" - }, - "total": { + "bytes": { "type": "long" } } }, + "bytes": { + "type": "long" + }, "method": { "ignore_above": 1024, "type": "keyword" @@ -435,19 +435,19 @@ "response": { "properties": { "body": { - "ignore_above": 1024, - "type": "keyword" - }, - "bytes": { "properties": { - "body": { + "bytes": { "type": "long" }, - "total": { - "type": "long" + "content": { + "ignore_above": 1024, + "type": "keyword" } } }, + "bytes": { + "type": "long" + }, "status_code": { "type": "long" } diff --git a/use-cases/web-logs.md b/use-cases/web-logs.md index 2d0920d4e0..e72d14787a 100644 --- a/use-cases/web-logs.md +++ b/use-cases/web-logs.md @@ -14,7 +14,7 @@ Using the fields as represented here is not expected to conflict with ECS, but m | [http.request.method](../README.md#http.request.method) | Http request method. | extended | keyword | `GET, POST, PUT` | | [http.request.referrer](../README.md#http.request.referrer) | Referrer for this HTTP request. | extended | keyword | `https://blog.example.com/` | | [http.response.status_code](../README.md#http.response.status_code) | Http response status code. | extended | long | `404` | -| [http.response.body](../README.md#http.response.body) | The full http response body. | extended | keyword | `Hello world` | +| *http.response.body* | *The full http response body.* | (use case) | keyword | `Hello world` | | [http.version](../README.md#http.version) | Http version. | extended | keyword | `1.1` | | *user_agent.** | *The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string.
* | | | | | [user_agent.original](../README.md#user_agent.original) | Unparsed version of the user_agent. | extended | keyword | `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` | From f82c460e649612ea2eb1f44129d815f2adf0bb57 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 7 Dec 2018 16:08:23 -0500 Subject: [PATCH 3/5] Small tweak to web-logs use case --- use-cases/web-logs.md | 2 +- use-cases/web-logs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/use-cases/web-logs.md b/use-cases/web-logs.md index e72d14787a..57f9a96062 100644 --- a/use-cases/web-logs.md +++ b/use-cases/web-logs.md @@ -14,7 +14,7 @@ Using the fields as represented here is not expected to conflict with ECS, but m | [http.request.method](../README.md#http.request.method) | Http request method. | extended | keyword | `GET, POST, PUT` | | [http.request.referrer](../README.md#http.request.referrer) | Referrer for this HTTP request. | extended | keyword | `https://blog.example.com/` | | [http.response.status_code](../README.md#http.response.status_code) | Http response status code. | extended | long | `404` | -| *http.response.body* | *The full http response body.* | (use case) | keyword | `Hello world` | +| [http.response.body.content](../README.md#http.response.body.content) | The full http response body. | extended | keyword | `Hello world` | | [http.version](../README.md#http.version) | Http version. | extended | keyword | `1.1` | | *user_agent.** | *The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string.
* | | | | | [user_agent.original](../README.md#user_agent.original) | Unparsed version of the user_agent. | extended | keyword | `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` | diff --git a/use-cases/web-logs.yml b/use-cases/web-logs.yml index d05ee53642..06a8f47d9e 100644 --- a/use-cases/web-logs.yml +++ b/use-cases/web-logs.yml @@ -46,7 +46,7 @@ fields: Http response status code. example: 404 - - name: response.body + - name: response.body.content type: keyword description: > The full http response body. From d1135dffa54caa664e26ccdb1c0e8f710c798cee Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Mon, 10 Dec 2018 11:17:43 -0500 Subject: [PATCH 4/5] Changelog all of these changes... Not listing the rename as a breaking change, as `http` was not part of Beta 1. --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0f61b1038..5809769561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ All notable changes to this project will be documented in this file based on the ## Unreleased ### Breaking changes -* Changed `device.*` fields to `observer.*` fields to eliminate user confusion. #238 +* Changed `device.*` fields to `observer.*` fields to eliminate user confusion. #238 * Rename `network.total.bytes` to `network.bytes` and `network.total.packets` to `network.packets`. #179 * Remove `network.inbound.bytes`, `network.inbound.packets`, @@ -30,6 +30,9 @@ All notable changes to this project will be documented in this file based on the * Add `process.executable` field. #209 * Add `process.working_directory` and `process.start`. #215 * Reintroduce `http`. #237 + * Move `http.response.body` to `http.response.body.content`. #239 + * Add `http.request.body.content`. #239 + * Add HTTP size metric fields. #239 * Add `user.full_name` field. #201 * Add `network.community_id` field. #208 * Add fields `geo.country_name` and `geo.region_iso_code`. #214 From 048b343d3bbdb80d09d2821e6f521c9b997f65e5 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Mon, 10 Dec 2018 11:19:52 -0500 Subject: [PATCH 5/5] Introduce the request body field --- README.md | 1 + fields.yml | 7 +++++++ schema.csv | 1 + schemas/http.yml | 7 +++++++ template.json | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/README.md b/README.md index 3cff91bb2d..3e07642f3c 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,7 @@ Fields related to HTTP activity. | Field | Description | Level | Type | Example | |---|---|---|---|---| | http.request.method | Http request method.
The field value must be normalized to lowercase for querying. See "Lowercase Capitalization" in the "Implementing ECS" section. | extended | keyword | `get, post, put` | +| http.request.body.content | The full http request body. | extended | keyword | `Hello world` | | http.request.referrer | Referrer for this HTTP request. | extended | keyword | `https://blog.example.com/` | | http.response.status_code | Http response status code. | extended | long | `404` | | http.response.body.content | The full http response body. | extended | keyword | `Hello world` | diff --git a/fields.yml b/fields.yml index 0f567a8f47..1dbf9562ad 100644 --- a/fields.yml +++ b/fields.yml @@ -869,6 +869,13 @@ "Lowercase Capitalization" in the "Implementing ECS" section. example: get, post, put + - name: request.body.content + level: extended + type: keyword + description: > + The full http request body. + example: Hello world + - name: request.referrer level: extended type: keyword diff --git a/schema.csv b/schema.csv index ec846f1d16..6da2217719 100644 --- a/schema.csv +++ b/schema.csv @@ -89,6 +89,7 @@ host.mac,keyword,core, host.name,keyword,core, host.type,keyword,core, http.request.body.bytes,long,extended,887 +http.request.body.content,keyword,extended,Hello world http.request.bytes,long,extended,1437 http.request.method,keyword,extended,"get, post, put" http.request.referrer,keyword,extended,https://blog.example.com/ diff --git a/schemas/http.yml b/schemas/http.yml index 341bf154d2..a7ee23348d 100644 --- a/schemas/http.yml +++ b/schemas/http.yml @@ -17,6 +17,13 @@ "Lowercase Capitalization" in the "Implementing ECS" section. example: get, post, put + - name: request.body.content + level: extended + type: keyword + description: > + The full http request body. + example: Hello world + - name: request.referrer level: extended type: keyword diff --git a/template.json b/template.json index c63c3b9bf3..aa565ea373 100644 --- a/template.json +++ b/template.json @@ -416,6 +416,10 @@ "properties": { "bytes": { "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" } } },