From 17aae1e98bfcc8847491c6b011a9ac9939b53218 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 12:18:57 -0500 Subject: [PATCH 1/5] Translate fields to ECS in ingest node pipeline and Beat processor --- .../traefik/access/config/traefik-access.yml | 6 +- .../traefik/access/ingest/pipeline.json | 2 +- .../access/test/test.log-expected.json | 76 +++++++++---------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index 1d20f94a29ff..e9a1933b4b2c 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -7,9 +7,9 @@ exclude_files: [".gz$"] processors: - dissect: - tokenizer: '%{traefik.access.remote_ip} %{traefik.access.user_identifier} %{traefik.access.user_name} [%{traefik.access.time}] - "%{traefik.access.method} %{traefik.access.url} HTTP/%{traefik.access.http_version}" - %{traefik.access.response_code} %{traefik.access.message}' + tokenizer: '%{traefik.access.remote_ip} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] + "%{http.request.method} %{url.original} HTTP/%{http.version}" + %{http.response.status_code} %{traefik.access.message}' field: "message" target_prefix: "" diff --git a/filebeat/module/traefik/access/ingest/pipeline.json b/filebeat/module/traefik/access/ingest/pipeline.json index 955cafc1af8c..0d31a50db73c 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.json +++ b/filebeat/module/traefik/access/ingest/pipeline.json @@ -5,7 +5,7 @@ "grok": { "field": "traefik.access.message", "patterns": [ - "(?:%{NUMBER:traefik.access.body_sent.bytes:int}|-)( (?:\"%{DATA:traefik.access.referrer}\"|-)?( (?:\"%{DATA:traefik.access.agent}\"|-)?)?( (?:%{NUMBER:traefik.access.request_count:int}|-)?)?( (?:\"%{DATA:traefik.access.frontend_name}\"|-)?)?( \"%{DATA:traefik.access.backend_url}\")?( %{NUMBER:traefik.access.duration:int}ms)?)?" + "(?:%{NUMBER:traefik.access.body_sent.bytes:int}|-)( (?:\"%{DATA:http.request.referrer}\"|-)?( (?:\"%{DATA:traefik.access.agent}\"|-)?)?( (?:%{NUMBER:traefik.access.request_count:int}|-)?)?( (?:\"%{DATA:traefik.access.frontend_name}\"|-)?)?( \"%{DATA:traefik.access.backend_url}\")?( %{NUMBER:traefik.access.duration:int}ms)?)?" ], "ignore_missing": true } diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json index 9cbe8a785649..d4261d8e6dcd 100644 --- a/filebeat/module/traefik/access/test/test.log-expected.json +++ b/filebeat/module/traefik/access/test/test.log-expected.json @@ -3,19 +3,18 @@ "@timestamp": "2017-10-02T20:22:07.000Z", "event.dataset": "access", "event.module": "traefik", + "http.request.method": "GET", + "http.request.referrer": "http://example.com/login", + "http.response.status_code": "304", + "http.version": "1.1", "input.type": "log", "log.offset": 0, "traefik.access.backend_url": "http://172.19.0.3:5601", "traefik.access.body_sent.bytes": 0, "traefik.access.duration": 2, "traefik.access.frontend_name": "Host-host-1", - "traefik.access.http_version": "1.1", - "traefik.access.method": "GET", - "traefik.access.referrer": "http://example.com/login", "traefik.access.remote_ip": "192.168.33.1", "traefik.access.request_count": 262, - "traefik.access.response_code": "304", - "traefik.access.url": "/ui/favicons/favicon-16x16.png", "traefik.access.user_agent.device": "Other", "traefik.access.user_agent.major": "61", "traefik.access.user_agent.minor": "0", @@ -25,12 +24,17 @@ "traefik.access.user_agent.os_name": "Linux", "traefik.access.user_agent.patch": "3163", "traefik.access.user_identifier": "-", - "traefik.access.user_name": "-" + "url.original": "/ui/favicons/favicon-16x16.png", + "user.name": "-" }, { "@timestamp": "2017-10-02T20:22:08.000Z", "event.dataset": "access", "event.module": "traefik", + "http.request.method": "GET", + "http.request.referrer": "http://example.com/login", + "http.response.status_code": "304", + "http.version": "1.1", "input.type": "log", "log.offset": 280, "traefik.access.backend_url": "http://172.19.0.3:5601", @@ -44,13 +48,8 @@ "traefik.access.geoip.location.lon": 13.3275, "traefik.access.geoip.region_iso_code": "DE-BE", "traefik.access.geoip.region_name": "Land Berlin", - "traefik.access.http_version": "1.1", - "traefik.access.method": "GET", - "traefik.access.referrer": "http://example.com/login", "traefik.access.remote_ip": "85.181.35.98", "traefik.access.request_count": 271, - "traefik.access.response_code": "304", - "traefik.access.url": "/ui/favicons/favicon.ico", "traefik.access.user_agent.device": "Other", "traefik.access.user_agent.major": "61", "traefik.access.user_agent.minor": "0", @@ -60,12 +59,16 @@ "traefik.access.user_agent.os_name": "Linux", "traefik.access.user_agent.patch": "3163", "traefik.access.user_identifier": "-", - "traefik.access.user_name": "-" + "url.original": "/ui/favicons/favicon.ico", + "user.name": "-" }, { "@timestamp": "2018-02-28T17:30:33.000Z", "event.dataset": "access", "event.module": "traefik", + "http.request.method": "GET", + "http.response.status_code": "200", + "http.version": "2.0", "input.type": "log", "log.offset": 553, "traefik.access.backend_url": "http://172.19.0.6:14008", @@ -79,12 +82,8 @@ "traefik.access.geoip.location.lon": -75.7518, "traefik.access.geoip.region_iso_code": "CA-ON", "traefik.access.geoip.region_name": "Ontario", - "traefik.access.http_version": "2.0", - "traefik.access.method": "GET", "traefik.access.remote_ip": "70.29.80.15", "traefik.access.request_count": 13, - "traefik.access.response_code": "200", - "traefik.access.url": "/en/", "traefik.access.user_agent.device": "iPhone", "traefik.access.user_agent.major": "11", "traefik.access.user_agent.minor": "0", @@ -95,25 +94,25 @@ "traefik.access.user_agent.os_minor": "2", "traefik.access.user_agent.os_name": "iOS", "traefik.access.user_identifier": "-", - "traefik.access.user_name": "-" + "url.original": "/en/", + "user.name": "-" }, { "@timestamp": "2018-11-29T15:03:51.000Z", "event.dataset": "access", "event.module": "traefik", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": "404", + "http.version": "1.1", "input.type": "log", "log.offset": 821, "traefik.access.backend_url": "/", "traefik.access.body_sent.bytes": 19, "traefik.access.duration": 0, "traefik.access.frontend_name": "backend not found", - "traefik.access.http_version": "1.1", - "traefik.access.method": "GET", - "traefik.access.referrer": "-", "traefik.access.remote_ip": "::1", "traefik.access.request_count": 10, - "traefik.access.response_code": "404", - "traefik.access.url": "/", "traefik.access.user_agent.device": "Other", "traefik.access.user_agent.major": "7", "traefik.access.user_agent.minor": "62", @@ -123,12 +122,16 @@ "traefik.access.user_agent.os_name": "Other", "traefik.access.user_agent.patch": "0", "traefik.access.user_identifier": "-", - "traefik.access.user_name": "-" + "url.original": "/", + "user.name": "-" }, { "@timestamp": "2018-01-19T10:01:02.000Z", "event.dataset": "access", "event.module": "traefik", + "http.request.method": "GET", + "http.response.status_code": "200", + "http.version": "1.1", "input.type": "log", "log.offset": 931, "traefik.access.backend_url": "http://172.25.0.9:4140", @@ -142,24 +145,24 @@ "traefik.access.geoip.location.lon": 21.0, "traefik.access.geoip.region_iso_code": "PL-MZ", "traefik.access.geoip.region_name": "Mazovia", - "traefik.access.http_version": "1.1", - "traefik.access.method": "GET", "traefik.access.remote_ip": "94.254.131.115", "traefik.access.request_count": 623112, - "traefik.access.response_code": "200", - "traefik.access.url": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", "traefik.access.user_agent.device": "Generic Smartphone", "traefik.access.user_agent.name": "Other", "traefik.access.user_agent.original": "Android", "traefik.access.user_agent.os": "Android", "traefik.access.user_agent.os_name": "Android", "traefik.access.user_identifier": "-", - "traefik.access.user_name": "-" + "url.original": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", + "user.name": "-" }, { "@timestamp": "2018-01-19T10:01:02.000Z", "event.dataset": "access", "event.module": "traefik", + "http.request.method": "GET", + "http.response.status_code": "200", + "http.version": "1.1", "input.type": "log", "log.offset": 1267, "traefik.access.backend_url": "http://172.25.0.6:4140", @@ -173,33 +176,30 @@ "traefik.access.geoip.location.lon": 18.9737, "traefik.access.geoip.region_iso_code": "PL-SL", "traefik.access.geoip.region_name": "Silesia", - "traefik.access.http_version": "1.1", - "traefik.access.method": "GET", "traefik.access.remote_ip": "89.64.35.193", "traefik.access.request_count": 623114, - "traefik.access.response_code": "200", - "traefik.access.url": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", "traefik.access.user_agent.device": "Generic Smartphone", "traefik.access.user_agent.name": "Other", "traefik.access.user_agent.original": "Android", "traefik.access.user_agent.os": "Android", "traefik.access.user_agent.os_name": "Android", "traefik.access.user_identifier": "-", - "traefik.access.user_name": "-" + "url.original": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", + "user.name": "-" }, { "@timestamp": "2000-10-10T20:55:36.000Z", "event.dataset": "access", "event.module": "traefik", + "http.request.method": "GET", + "http.response.status_code": "200", + "http.version": "1.0", "input.type": "log", "log.offset": 1581, "traefik.access.body_sent.bytes": 2326, - "traefik.access.http_version": "1.0", - "traefik.access.method": "GET", "traefik.access.remote_ip": "127.0.0.1", - "traefik.access.response_code": "200", - "traefik.access.url": "/apache_pb.gif", "traefik.access.user_identifier": "-", - "traefik.access.user_name": "frank" + "url.original": "/apache_pb.gif", + "user.name": "frank" } ] \ No newline at end of file From bc8292af2b346935c014257b654a0f903a4308b5 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 12:32:29 -0500 Subject: [PATCH 2/5] Document ECS field transitions in ecs-migration.yml --- dev-tools/ecs-migration.yml | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 0d038792b9c9..73fa25140a1e 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -503,6 +503,72 @@ to: log.level alias: true +## Traefik module + +- from: traefik.access.remote_ip + to: source.address + alias: true + +- from: traefik.access.url + to: url.original + alias: true + +- from: traefik.access.user_name + to: user.name + alias: true + +- from: traefik.access.agent + to: user_agent.original + alias: true + +- from: traefik.access.user_agent.original + to: user_agent.original + alias: true + +- from: traefik.access.user_agent.* + to: user_agent.* + alias: true + +- from: traefik.access.geoip.continent_name + to: source.geo.continent_name + alias: true + +- from: traefik.access.geoip.country_iso_code + to: source.geo.country_iso_code + alias: true + +- from: traefik.access.geoip.location + to: source.geo.location + alias: true + +- from: traefik.access.geoip.region_name + to: source.geo.region_name + alias: true + +- from: traefik.access.geoip.city_name + to: source.geo.city_name + alias: true + +- from: traefik.access.geoip.region_iso_code + to: source.geo.region_iso_code + alias: true + +- from: traefik.access.method + to: http.request.method + alias: true + +- from: traefik.access.response_code + to: http.response.status_code + alias: true + +- from: traefik.access.referrer + to: http.request.referrer + alias: true + +- from: traefik.access.http_version + to: http.version + alias: true + # Auditbeat ## From Auditbeat's auditd module. From 6852079117b8524e718c2859228b2819984c49f9 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 13:21:55 -0500 Subject: [PATCH 3/5] Continue translating fields to ECS --- .../traefik/access/config/traefik-access.yml | 2 +- .../traefik/access/ingest/pipeline.json | 71 ++++++- .../access/test/test.log-expected.json | 198 ++++++++++-------- 3 files changed, 173 insertions(+), 98 deletions(-) diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index e9a1933b4b2c..8b640efa575c 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -7,7 +7,7 @@ exclude_files: [".gz$"] processors: - dissect: - tokenizer: '%{traefik.access.remote_ip} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] + tokenizer: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] "%{http.request.method} %{url.original} HTTP/%{http.version}" %{http.response.status_code} %{traefik.access.message}' diff --git a/filebeat/module/traefik/access/ingest/pipeline.json b/filebeat/module/traefik/access/ingest/pipeline.json index 0d31a50db73c..52b076374a05 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.json +++ b/filebeat/module/traefik/access/ingest/pipeline.json @@ -5,7 +5,7 @@ "grok": { "field": "traefik.access.message", "patterns": [ - "(?:%{NUMBER:traefik.access.body_sent.bytes:int}|-)( (?:\"%{DATA:http.request.referrer}\"|-)?( (?:\"%{DATA:traefik.access.agent}\"|-)?)?( (?:%{NUMBER:traefik.access.request_count:int}|-)?)?( (?:\"%{DATA:traefik.access.frontend_name}\"|-)?)?( \"%{DATA:traefik.access.backend_url}\")?( %{NUMBER:traefik.access.duration:int}ms)?)?" + "(?:%{NUMBER:traefik.access.body_sent.bytes:long}|-)( (?:\"%{DATA:http.request.referrer}\"|-)?( (?:\"%{DATA:traefik.access.agent}\"|-)?)?( (?:%{NUMBER:traefik.access.request_count:long}|-)?)?( (?:\"%{DATA:traefik.access.frontend_name}\"|-)?)?( \"%{DATA:traefik.access.backend_url}\")?( %{NUMBER:traefik.access.duration:long}ms)?)?" ], "ignore_missing": true } @@ -42,24 +42,85 @@ "field": "traefik.access.time" } }, + + { + "convert": { + "field": "http.response.status_code", + "type": "long" + } + }, + + { + "grok": { + "field": "source.address", + "patterns": [ + "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" + ] + } + }, + + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.traefik.access.duration * params.scale)", + "params": { "scale": 1000000 }, + "if": "ctx.traefik.access.containsKey('duration')" + } + }, + { "user_agent": { "field": "traefik.access.agent", - "target_field": "traefik.access.user_agent", "ignore_failure": true } }, { "rename": { "field": "traefik.access.agent", - "target_field": "traefik.access.user_agent.original", + "target_field": "user_agent.original", + "ignore_failure": true + } + }, + { + "rename": { + "field": "user_agent.os", + "target_field": "user_agent.os.full_name", "ignore_failure": true } }, + { + "rename": { + "field": "user_agent.os_name", + "target_field": "user_agent.os.name", + "ignore_failure": true + } + }, + { + "set": { + "field": "user_agent.os.version", + "value": "{{user_agent.os_major}}.{{user_agent.os_minor}}.{{user_agent.os_patch}}", + "ignore_failure": true + } + }, + { "remove": { "field": "user_agent.os_major", "ignore_missing": true } }, + { "remove": { "field": "user_agent.os_minor", "ignore_missing": true } }, + { "remove": { "field": "user_agent.os_patch", "ignore_missing": true } }, + + { + "set": { + "field": "user_agent.version", + "value": "{{user_agent.major}}.{{user_agent.minor}}.{{user_agent.patch}}", + "ignore_failure": true + } + }, + { "remove": { "field": "user_agent.major", "ignore_missing": true } }, + { "remove": { "field": "user_agent.minor", "ignore_missing": true } }, + { "remove": { "field": "user_agent.patch", "ignore_missing": true } }, { "geoip": { - "field": "traefik.access.remote_ip", - "target_field": "traefik.access.geoip" + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true } } ], diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json index d4261d8e6dcd..47a8579ac709 100644 --- a/filebeat/module/traefik/access/test/test.log-expected.json +++ b/filebeat/module/traefik/access/test/test.log-expected.json @@ -2,204 +2,218 @@ { "@timestamp": "2017-10-02T20:22:07.000Z", "event.dataset": "access", + "event.duration": 2000000, "event.module": "traefik", "http.request.method": "GET", "http.request.referrer": "http://example.com/login", - "http.response.status_code": "304", + "http.response.status_code": 304, "http.version": "1.1", "input.type": "log", "log.offset": 0, + "source.address": "192.168.33.1", + "source.ip": "192.168.33.1", "traefik.access.backend_url": "http://172.19.0.3:5601", "traefik.access.body_sent.bytes": 0, "traefik.access.duration": 2, "traefik.access.frontend_name": "Host-host-1", - "traefik.access.remote_ip": "192.168.33.1", "traefik.access.request_count": 262, - "traefik.access.user_agent.device": "Other", - "traefik.access.user_agent.major": "61", - "traefik.access.user_agent.minor": "0", - "traefik.access.user_agent.name": "Chrome", - "traefik.access.user_agent.original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", - "traefik.access.user_agent.os": "Linux", - "traefik.access.user_agent.os_name": "Linux", - "traefik.access.user_agent.patch": "3163", "traefik.access.user_identifier": "-", "url.original": "/ui/favicons/favicon-16x16.png", - "user.name": "-" + "user.name": "-", + "user_agent.device": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", + "user_agent.os.full_name": "Linux", + "user_agent.os.name": "Linux", + "user_agent.os.version": "..", + "user_agent.version": "61.0.3163" }, { "@timestamp": "2017-10-02T20:22:08.000Z", "event.dataset": "access", + "event.duration": 3000000, "event.module": "traefik", "http.request.method": "GET", "http.request.referrer": "http://example.com/login", - "http.response.status_code": "304", + "http.response.status_code": 304, "http.version": "1.1", "input.type": "log", "log.offset": 280, + "source.address": "85.181.35.98", + "source.geo.city_name": "Berlin", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "DE", + "source.geo.location.lat": 52.4908, + "source.geo.location.lon": 13.3275, + "source.geo.region_iso_code": "DE-BE", + "source.geo.region_name": "Land Berlin", + "source.ip": "85.181.35.98", "traefik.access.backend_url": "http://172.19.0.3:5601", "traefik.access.body_sent.bytes": 0, "traefik.access.duration": 3, "traefik.access.frontend_name": "Host-host1", - "traefik.access.geoip.city_name": "Berlin", - "traefik.access.geoip.continent_name": "Europe", - "traefik.access.geoip.country_iso_code": "DE", - "traefik.access.geoip.location.lat": 52.4908, - "traefik.access.geoip.location.lon": 13.3275, - "traefik.access.geoip.region_iso_code": "DE-BE", - "traefik.access.geoip.region_name": "Land Berlin", - "traefik.access.remote_ip": "85.181.35.98", "traefik.access.request_count": 271, - "traefik.access.user_agent.device": "Other", - "traefik.access.user_agent.major": "61", - "traefik.access.user_agent.minor": "0", - "traefik.access.user_agent.name": "Chrome", - "traefik.access.user_agent.original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", - "traefik.access.user_agent.os": "Linux", - "traefik.access.user_agent.os_name": "Linux", - "traefik.access.user_agent.patch": "3163", "traefik.access.user_identifier": "-", "url.original": "/ui/favicons/favicon.ico", - "user.name": "-" + "user.name": "-", + "user_agent.device": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", + "user_agent.os.full_name": "Linux", + "user_agent.os.name": "Linux", + "user_agent.os.version": "..", + "user_agent.version": "61.0.3163" }, { "@timestamp": "2018-02-28T17:30:33.000Z", "event.dataset": "access", + "event.duration": 247000000, "event.module": "traefik", "http.request.method": "GET", - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "2.0", "input.type": "log", "log.offset": 553, + "source.address": "70.29.80.15", + "source.geo.city_name": "Ottawa", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "CA", + "source.geo.location.lat": 45.2691, + "source.geo.location.lon": -75.7518, + "source.geo.region_iso_code": "CA-ON", + "source.geo.region_name": "Ontario", + "source.ip": "70.29.80.15", "traefik.access.backend_url": "http://172.19.0.6:14008", "traefik.access.body_sent.bytes": 2814, "traefik.access.duration": 247, "traefik.access.frontend_name": "Host-host1-com-0", - "traefik.access.geoip.city_name": "Ottawa", - "traefik.access.geoip.continent_name": "North America", - "traefik.access.geoip.country_iso_code": "CA", - "traefik.access.geoip.location.lat": 45.2691, - "traefik.access.geoip.location.lon": -75.7518, - "traefik.access.geoip.region_iso_code": "CA-ON", - "traefik.access.geoip.region_name": "Ontario", - "traefik.access.remote_ip": "70.29.80.15", "traefik.access.request_count": 13, - "traefik.access.user_agent.device": "iPhone", - "traefik.access.user_agent.major": "11", - "traefik.access.user_agent.minor": "0", - "traefik.access.user_agent.name": "Mobile Safari", - "traefik.access.user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1", - "traefik.access.user_agent.os": "iOS 11.2.5", - "traefik.access.user_agent.os_major": "11", - "traefik.access.user_agent.os_minor": "2", - "traefik.access.user_agent.os_name": "iOS", "traefik.access.user_identifier": "-", "url.original": "/en/", - "user.name": "-" + "user.name": "-", + "user_agent.device": "iPhone", + "user_agent.name": "Mobile Safari", + "user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1", + "user_agent.os.full_name": "iOS 11.2.5", + "user_agent.os.name": "iOS", + "user_agent.os.version": "11.2.", + "user_agent.version": "11.0." }, { "@timestamp": "2018-11-29T15:03:51.000Z", "event.dataset": "access", + "event.duration": 0, "event.module": "traefik", "http.request.method": "GET", "http.request.referrer": "-", - "http.response.status_code": "404", + "http.response.status_code": 404, "http.version": "1.1", "input.type": "log", "log.offset": 821, + "source.address": "::1", + "source.ip": "::1", "traefik.access.backend_url": "/", "traefik.access.body_sent.bytes": 19, "traefik.access.duration": 0, "traefik.access.frontend_name": "backend not found", - "traefik.access.remote_ip": "::1", "traefik.access.request_count": 10, - "traefik.access.user_agent.device": "Other", - "traefik.access.user_agent.major": "7", - "traefik.access.user_agent.minor": "62", - "traefik.access.user_agent.name": "curl", - "traefik.access.user_agent.original": "curl/7.62.0", - "traefik.access.user_agent.os": "Other", - "traefik.access.user_agent.os_name": "Other", - "traefik.access.user_agent.patch": "0", "traefik.access.user_identifier": "-", "url.original": "/", - "user.name": "-" + "user.name": "-", + "user_agent.device": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.62.0", + "user_agent.os.full_name": "Other", + "user_agent.os.name": "Other", + "user_agent.os.version": "..", + "user_agent.version": "7.62.0" }, { "@timestamp": "2018-01-19T10:01:02.000Z", "event.dataset": "access", + "event.duration": 13000000, "event.module": "traefik", "http.request.method": "GET", - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 931, + "source.address": "94.254.131.115", + "source.geo.city_name": "Warsaw", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "PL", + "source.geo.location.lat": 52.25, + "source.geo.location.lon": 21.0, + "source.geo.region_iso_code": "PL-MZ", + "source.geo.region_name": "Mazovia", + "source.ip": "94.254.131.115", "traefik.access.backend_url": "http://172.25.0.9:4140", "traefik.access.body_sent.bytes": 85, "traefik.access.duration": 13, "traefik.access.frontend_name": "Host-api-wearerealitygames-com-2", - "traefik.access.geoip.city_name": "Warsaw", - "traefik.access.geoip.continent_name": "Europe", - "traefik.access.geoip.country_iso_code": "PL", - "traefik.access.geoip.location.lat": 52.25, - "traefik.access.geoip.location.lon": 21.0, - "traefik.access.geoip.region_iso_code": "PL-MZ", - "traefik.access.geoip.region_name": "Mazovia", - "traefik.access.remote_ip": "94.254.131.115", "traefik.access.request_count": 623112, - "traefik.access.user_agent.device": "Generic Smartphone", - "traefik.access.user_agent.name": "Other", - "traefik.access.user_agent.original": "Android", - "traefik.access.user_agent.os": "Android", - "traefik.access.user_agent.os_name": "Android", "traefik.access.user_identifier": "-", "url.original": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", - "user.name": "-" + "user.name": "-", + "user_agent.device": "Generic Smartphone", + "user_agent.name": "Other", + "user_agent.original": "Android", + "user_agent.os.full_name": "Android", + "user_agent.os.name": "Android", + "user_agent.os.version": "..", + "user_agent.version": ".." }, { "@timestamp": "2018-01-19T10:01:02.000Z", "event.dataset": "access", + "event.duration": 8000000, "event.module": "traefik", "http.request.method": "GET", - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 1267, + "source.address": "89.64.35.193", + "source.geo.city_name": "Katowice", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "PL", + "source.geo.location.lat": 50.2194, + "source.geo.location.lon": 18.9737, + "source.geo.region_iso_code": "PL-SL", + "source.geo.region_name": "Silesia", + "source.ip": "89.64.35.193", "traefik.access.backend_url": "http://172.25.0.6:4140", "traefik.access.body_sent.bytes": 150, "traefik.access.duration": 8, "traefik.access.frontend_name": "Host-api-wearerealitygames-com-2", - "traefik.access.geoip.city_name": "Katowice", - "traefik.access.geoip.continent_name": "Europe", - "traefik.access.geoip.country_iso_code": "PL", - "traefik.access.geoip.location.lat": 50.2194, - "traefik.access.geoip.location.lon": 18.9737, - "traefik.access.geoip.region_iso_code": "PL-SL", - "traefik.access.geoip.region_name": "Silesia", - "traefik.access.remote_ip": "89.64.35.193", "traefik.access.request_count": 623114, - "traefik.access.user_agent.device": "Generic Smartphone", - "traefik.access.user_agent.name": "Other", - "traefik.access.user_agent.original": "Android", - "traefik.access.user_agent.os": "Android", - "traefik.access.user_agent.os_name": "Android", "traefik.access.user_identifier": "-", "url.original": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", - "user.name": "-" + "user.name": "-", + "user_agent.device": "Generic Smartphone", + "user_agent.name": "Other", + "user_agent.original": "Android", + "user_agent.os.full_name": "Android", + "user_agent.os.name": "Android", + "user_agent.os.version": "..", + "user_agent.version": ".." }, { "@timestamp": "2000-10-10T20:55:36.000Z", "event.dataset": "access", "event.module": "traefik", "http.request.method": "GET", - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.0", "input.type": "log", "log.offset": 1581, + "source.address": "127.0.0.1", + "source.ip": "127.0.0.1", "traefik.access.body_sent.bytes": 2326, - "traefik.access.remote_ip": "127.0.0.1", "traefik.access.user_identifier": "-", "url.original": "/apache_pb.gif", - "user.name": "frank" + "user.name": "frank", + "user_agent.os.version": "..", + "user_agent.version": ".." } ] \ No newline at end of file From 35776d62472d28e50d45e560b8ba1069622ab7a8 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 14:40:36 -0500 Subject: [PATCH 4/5] Alias the migrated fields --- filebeat/docs/fields.asciidoc | 222 +++++++----------- .../module/traefik/access/_meta/fields.yml | 181 ++++++-------- filebeat/module/traefik/fields.go | 2 +- 3 files changed, 163 insertions(+), 242 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index e960e166be05..beee77c21fc8 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -12086,16 +12086,6 @@ Contains fields for the Traefik access logs. -*`traefik.access.remote_ip`*:: -+ --- -type: keyword - -Client IP address. - - --- - *`traefik.access.duration`*:: + -- @@ -12106,314 +12096,272 @@ Duration of the access entry. -- -*`traefik.access.user_name`*:: +*`traefik.access.user_identifier`*:: + -- type: keyword -The user name used when basic authentication is used. +Is the RFC 1413 identity of the client -- -*`traefik.access.user_identifier`*:: +*`traefik.access.body_sent.bytes`*:: + -- -type: keyword +type: long -Is the RFC 1413 identity of the client +format: bytes + +The number of bytes of the server response body. -- -*`traefik.access.method`*:: +*`traefik.access.request_count`*:: + -- -type: keyword - -example: GET +type: long -The request HTTP method. +The number of requests -- -*`traefik.access.url`*:: +*`traefik.access.frontend_name`*:: + -- -type: keyword +type: text -The request HTTP URL. +The name of the frontend used -- -*`traefik.access.http_version`*:: +*`traefik.access.backend_url`*:: + -- -type: keyword - -The HTTP version. +type: text +The url of the backend where request is forwarded -- -*`traefik.access.response_code`*:: +*`traefik.access.remote_ip`*:: + -- -type: long - -The HTTP response code. +type: alias +alias to: source.address -- -*`traefik.access.body_sent.bytes`*:: +*`traefik.access.user_name`*:: + -- -type: long +type: alias -format: bytes +alias to: user.name -The number of bytes of the server response body. +-- +*`traefik.access.method`*:: ++ +-- +type: alias + +alias to: http.request.method -- -*`traefik.access.referrer`*:: +*`traefik.access.url`*:: + -- -type: keyword - -The HTTP referrer. +type: alias +alias to: url.original -- -*`traefik.access.agent`*:: +*`traefik.access.http_version`*:: + -- -type: text +type: alias -Contains the un-parsed user agent string. Only present if the user agent Elasticsearch plugin is not available or not used. +alias to: http.version +-- +*`traefik.access.response_code`*:: ++ -- +type: alias -[float] -== user_agent fields +alias to: http.response.status_code -Contains the parsed User agent field. Only present if the user agent Elasticsearch plugin is available and used. +-- +*`traefik.access.referrer`*:: ++ +-- +type: alias +alias to: http.request.referrer -*`traefik.access.user_agent.device`*:: -+ -- -type: keyword -The name of the physical device. +*`traefik.access.agent`*:: ++ +-- +type: alias +alias to: user_agent.original -- -*`traefik.access.user_agent.build`*:: + +*`traefik.access.user_agent.device`*:: + -- -type: keyword - -The build of the user agent. +type: alias +alias to: user_agent.device -- *`traefik.access.user_agent.major`*:: + -- -type: long - -The major version of the user agent. +type: alias +alias to: user_agent.major -- *`traefik.access.user_agent.minor`*:: + -- -type: long - -The minor version of the user agent. +type: alias +alias to: user_agent.minor -- *`traefik.access.user_agent.patch`*:: + -- -type: keyword - -The patch version of the user agent. +type: alias +alias to: user_agent.patch -- *`traefik.access.user_agent.name`*:: + -- -type: keyword - -example: Chrome - -The name of the user agent. +type: alias +alias to: user_agent.name -- *`traefik.access.user_agent.os`*:: + -- -type: keyword - -The name of the operating system. +type: alias +alias to: user_agent.os.full_name -- *`traefik.access.user_agent.os_major`*:: + -- -type: long - -The major version of the operating system. +type: alias +alias to: user_agent.os.major -- *`traefik.access.user_agent.os_minor`*:: + -- -type: long - -The minor version of the operating system. +type: alias +alias to: user_agent.os.minor -- *`traefik.access.user_agent.os_name`*:: + -- -type: keyword - -The name of the operating system. +type: alias +alias to: user_agent.os.name -- *`traefik.access.user_agent.original`*:: + -- -type: text - -Original user agent value before parsing by ingest-user-agent plugin. - +type: alias -Field is not indexed. +alias to: user_agent.original -- -[float] -== geoip fields - -Contains GeoIP information gathered based on the remote_ip field. Only present if the GeoIP Elasticsearch plugin is available and used. - - *`traefik.access.geoip.continent_name`*:: + -- -type: keyword - -The name of the continent. +type: alias +alias to: source.geo.continent_name -- *`traefik.access.geoip.country_iso_code`*:: + -- -type: keyword - -Country ISO code. +type: alias +alias to: source.geo.country_iso_code -- *`traefik.access.geoip.location`*:: + -- -type: geo_point - -The longitude and latitude. +type: alias +alias to: source.geo.location -- *`traefik.access.geoip.region_name`*:: + -- -type: keyword - -The region name. +type: alias +alias to: source.geo.region_name -- *`traefik.access.geoip.city_name`*:: + -- -type: keyword - -The city name. +type: alias +alias to: source.geo.city_name -- *`traefik.access.geoip.region_iso_code`*:: + -- -type: keyword - -Region ISO code. - - --- - -*`traefik.access.request_count`*:: -+ --- -type: long - -The number of requests - - --- - -*`traefik.access.frontend_name`*:: -+ --- -type: text - -The name of the frontend used - - --- - -*`traefik.access.backend_url`*:: -+ --- -type: text +type: alias -The url of the backend where request is forwarded +alias to: source.geo.region_iso_code -- diff --git a/filebeat/module/traefik/access/_meta/fields.yml b/filebeat/module/traefik/access/_meta/fields.yml index 116ed67d42f4..d1b89ac1b9e6 100644 --- a/filebeat/module/traefik/access/_meta/fields.yml +++ b/filebeat/module/traefik/access/_meta/fields.yml @@ -3,145 +3,118 @@ description: > Contains fields for the Traefik access logs. fields: - - name: remote_ip - type: keyword - description: > - Client IP address. - name: duration type: long description: > Duration of the access entry. - - name: user_name - type: keyword - description: > - The user name used when basic authentication is used. - name: user_identifier type: keyword description: > Is the RFC 1413 identity of the client - - name: method - type: keyword - example: GET - description: > - The request HTTP method. - - name: url - type: keyword - description: > - The request HTTP URL. - - name: http_version - type: keyword - description: > - The HTTP version. - - name: response_code - type: long - description: > - The HTTP response code. - name: body_sent.bytes type: long format: bytes description: > The number of bytes of the server response body. - - name: referrer - type: keyword + - name: request_count + type: long description: > - The HTTP referrer. - - name: agent + The number of requests + - name: frontend_name type: text description: > - Contains the un-parsed user agent string. Only present if the user - agent Elasticsearch plugin is not available or not used. + The name of the frontend used + - name: backend_url + type: text + description: + The url of the backend where request is forwarded + + - name: remote_ip + type: alias + path: source.address + migration: true + - name: user_name + type: alias + path: user.name + migration: true + - name: method + type: alias + path: http.request.method + migration: true + - name: url + type: alias + path: url.original + migration: true + - name: http_version + type: alias + path: http.version + migration: true + - name: response_code + type: alias + path: http.response.status_code + migration: true + - name: referrer + type: alias + path: http.request.referrer + migration: true + - name: agent + type: alias + path: user_agent.original + migration: true + - name: user_agent type: group - description: > - Contains the parsed User agent field. Only present if the user - agent Elasticsearch plugin is available and used. fields: - name: device - type: keyword - description: > - The name of the physical device. - - name: build - type: keyword - description: > - The build of the user agent. + type: alias + path: user_agent.device - name: major - type: long - description: > - The major version of the user agent. + type: alias + path: user_agent.major - name: minor - type: long - description: > - The minor version of the user agent. + type: alias + path: user_agent.minor - name: patch - type: keyword - description: > - The patch version of the user agent. + type: alias + path: user_agent.patch - name: name - type: keyword - example: Chrome - description: > - The name of the user agent. + type: alias + path: user_agent.name - name: os - type: keyword - description: > - The name of the operating system. + type: alias + path: user_agent.os.full_name - name: os_major - type: long - description: > - The major version of the operating system. + type: alias + path: user_agent.os.major - name: os_minor - type: long - description: > - The minor version of the operating system. + type: alias + path: user_agent.os.minor - name: os_name - type: keyword - description: > - The name of the operating system. + type: alias + path: user_agent.os.name - name: original - type: text - index: false - description: > - Original user agent value before parsing by ingest-user-agent plugin. + type: alias + path: user_agent.original + - name: geoip type: group - description: > - Contains GeoIP information gathered based on the remote_ip field. - Only present if the GeoIP Elasticsearch plugin is available and - used. fields: - name: continent_name - type: keyword - description: > - The name of the continent. + type: alias + path: source.geo.continent_name - name: country_iso_code - type: keyword - description: > - Country ISO code. + type: alias + path: source.geo.country_iso_code - name: location - type: geo_point - description: > - The longitude and latitude. + type: alias + path: source.geo.location - name: region_name - type: keyword - description: > - The region name. + type: alias + path: source.geo.region_name - name: city_name - type: keyword - description: > - The city name. + type: alias + path: source.geo.city_name - name: region_iso_code - type: keyword - description: > - Region ISO code. - - name: request_count - type: long - description: > - The number of requests - - name: frontend_name - type: text - description: > - The name of the frontend used - - name: backend_url - type: text - description: - The url of the backend where request is forwarded + type: alias + path: source.geo.region_iso_code + diff --git a/filebeat/module/traefik/fields.go b/filebeat/module/traefik/fields.go index fb141ef270f7..0a6975069a11 100644 --- a/filebeat/module/traefik/fields.go +++ b/filebeat/module/traefik/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy8mEFzqzYQx+/+FDvv7sy8aU8+9JK+95qZdpJJ/c6MjBbYRkh0Jezw7TsSYBMsbOPa4WSD2P9Pq12tliW8YbMCxwIzelsAOHIKV/Bl3d75sgCQaFOmypHRK/htAQDwl5G1QsgMQyXYks7BFQjdS6BMDhkptA8LgIxQSbsK7y1BixKHev5yTYUryNnUVXcnIumv78EUZGzKaT1/DTWHuiJN0dr97Zj0CXl/PRrtBGnbSQQXDFFaBU+0h4kBDaEYS+MwoerD057tDZudYTl6doIwUCpC7eDpBYSUjPYDy0FZ1iy8iaiwMjqfp/p7Zw5MFnzS+QK14yYOUFvkxP+83dTXBQazQcH/krArUMNGWEpB1K5A7ShtQcmGESfgSPrhGSHfDvHJBve8fn+Er79+/QVaDdf0fkvD6kWZSnSFGcudQsF3UVY+o398W893JOO/NVoHf6zXL532hK9Y3XYJPyj/fP0zLls4VyVbZDsVxFfrB93Oclyb0VZGW0xSI+PxOz+D9sq9cfDG4/obI5vEonYPm8ahvZQgM1wKt4LYSxfQ6brcIPtIDQb6kLXIW+QDtoebcluGzLdMp4HTWtNxYZGPk6pXdfg+fnBuf+2rgJ97rZe+CKJs952gA9Yx6fwBnrVqoGL0CwXUOssPOzLZvvZNCesotSg4LaBSdU5hl9LGgdgKUmKjEAyHG2e2rukJj+vd3Bl30/15mG6obzeb7WGmQsvINOP1dOgAiVtKx1kJZ+LsAjdAnwa+uHSxXxWNpVSoTnSMOsjYmlRM80ZMwXwPdQjFaZ5S/GOOl+bE3jEDJtjuN9BZUKTvB+VtXwNVCZcW91u5YP4arsjR6RKs/aHgsWATtXBFGlzCa8YV5xLaK1hMhf4oqnOwjXVYniJKPj0N5tF9dj7MorsyAO+7pEw5aTE+kMKpeu8v0hLfV5AJZa9MiedOeXgQ2ApVI2wwM4z7ZnnTAOkcrVv6kct2ZFv/4gU9RzPRH/6fWv4DzdMLkG7PhD4GcuEKZJS+XUIJRgff7zvUrs4fWYzV/db4RSX+yN41JT812pFG7T4xKPea09GYmtr3vwlZE+sUbgT22KrA09/PkZZhyKNMGuv5Dxw5mqQydHR0nOEivyuRq2V7flPChT/TTIw5GX3ndWtFguKJ1SLX3JnDS5yh6Pxx35B5bf0Rj5gDSWjDkxDFN2xyD21kp2Cj8hkb7VDL6W9E83u3cQ73GmHTiXfaIn3zEFMfOc4wRAlqVj1AZx52fufdf/ig8G1xJ1iiXPwXAAD//wEzgn8=" + return "eJysl8Fu4zgMhu95CqL3Gih2TznspYsCe9jLoHdDtWlHE1v0UFQ7fvuBHDt1DSpx3OgWB/y/n6RE2Y9wxH4PwgYre9wBiJUG9/DwenrysAMo0RdsO7Hk9vDPDgDgfypDg1ARQ2fYW1eDHBDGIGiohso26LMdQGWxKf1+iHsEZ1qc8+KSvsM91EyhG58oyLheBimomNo0L645c841RYHenx9r6Av4uJ7JibHOj4ihBHMrJ0J0dDajGZqbKgObiPry52StIVcv/rjgLq5/RzmganA2OkIn3GeqgeCRc1uiE1tZZNXHEfsP4vI2K//5wcGPl2d4+vvpLzgxpJ+sFY1FJ6qnNyr73KOT7K0X9GtrUxG3RvagBV0x+3pAcKF9Q472BoHJp0d+RwZG35HzOJjTS8n4K6CXvKCwSGx7Q78aGwlexVdMTtCVefyp4gV/L32twZsWp2JMjLhtSr13pjhGE4GbLR5UB4GbycAoDx8HZJzqAXY4ix+Gy5mrRI9aEsxtp5ozjTXLndMZOezBU+ACM1OW/HWIxNXa+nTs4nALmD5nyc6kwTEuU+LWMFuUAy2P7TXgQaTLxsJmqsKqdBPtv5AoNxmxra0zy9A1wGg7f0f2qWF6JWM9dA15mgx5QeWt3R2LfRLIvBgJXtNZ56NC5sQIX9nvhMYavKmX43zd5s6HwFtbnz5jaR/Lmx4Sd/NcssR3Wyy7cTk1Nb2Ezvmkmp+0LPoGii5zhlh3F4gqM0E6I8Xh+xBdZoIo83ADQ1WZEKRJ3Aggn1WhabSxPwfld+o++SsbIKLuswci6uI2IK9nvYV0uXz63NhCWiqpM6ZGSrw8bBkvBTmxDp18p1rju0mNlF3R+8SG+EWQW0/abbMJfEVxQjdUaB88G5BJpc8Lsbbk7lTZS2Lnslrp79XIpNQiu/u1MCW4+xMAAP//WNoXGA==" } From 7145f9728d31cec78ecb2a5e089222bd20ecb6a6 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 14:41:42 -0500 Subject: [PATCH 5/5] Changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a7910878cd5c..f49ac636fdbb 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -24,6 +24,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* - Modify apache/error dataset to follow ECS. {pull}8963[8963] +- Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] *Heartbeat*