RFC 0010 Stage 1 - Add email.* fields to experimental schema (elast…

…ic#1569)

* adding email to exp schema

* artifacts

* changelog entry
# Conflicts:
#	experimental/generated/csv/fields.csv
#	experimental/generated/elasticsearch/template.json

CHANGELOG.next.md

@@ -18,6 +18,7 @@ Thanks, you're awesome :-) -->
 * Added `process.end` field. #1544
 * Introduce container metric fields into experimental schema. #1546
 * Update `user.name` and `user.id` examples for clarity. #1566
+* Add `email.*` field set in the experimental fields. #1569
 
 ### Tooling and Artifact Changes
 

experimental/generated/beats/fields.ecs.yml

@@ -1789,6 +1789,176 @@
       ignore_above: 1024
       description: telfhash symbol hash for ELF file.
       default_field: false
+  - name: email
+    title: Email
+    group: 2
+    description: 'Fields relating to an email transaction.
+
+      This field set focuses on the email message header, body, and attachments. Network
+      protocols that send and receive email messages such as SMTP are outside the
+      scope of the `email.*` fields.'
+    type: group
+    fields:
+    - name: attachments
+      level: extended
+      type: nested
+      description: A list of attachment files sent along with an email message.
+      default_field: false
+    - name: attachments.file.extension
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Attachment file extension, excluding the leading dot.
+      example: txt
+      default_field: false
+    - name: attachments.file.mime_type
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'The MIME media type of the attachment.
+
+        This value will typically be extracted from the `Content-Type` MIME header
+        field.'
+      example: text/plain
+      default_field: false
+    - name: attachments.file.name
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Name of the attachment file including the file extension.
+      example: attachment.txt
+      default_field: false
+    - name: attachments.file.size
+      level: extended
+      type: long
+      description: Attachment file size in bytes.
+      example: 64329
+      default_field: false
+    - name: attachments.hash.md5
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: MD5 hash of the attachment file.
+      example: e25f1c98ffdacf611473af364362ec48
+      default_field: false
+    - name: attachments.hash.sha1
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA-1 hash of the attachment file.
+      example: 8c1cd40f17109b427e61d4e72ca6d9a4fc8175f3
+      default_field: false
+    - name: attachments.hash.sha256
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: SHA-256 hash of the attachment file.
+      example: f0366b3559f577d8732f7e9cc343a4960d202e8137dcc42f9783f3963f6abc6a
+      default_field: false
+    - name: bcc
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: The email address(es) of the blind carbon carbon (BCC) recipients.
+      example: '[''[email protected]'', ''[email protected]'']'
+      default_field: false
+    - name: cc
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: The email address(es) of the carbon carbon (BCC) recipients.
+      example: '[''[email protected]'', ''[email protected]'']'
+      default_field: false
+    - name: content_type
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'Information about how the message is to be displayed.
+
+        Typically a MIME type.'
+      example: text/plain
+      default_field: false
+    - name: delivery_timestamp
+      level: extended
+      type: date
+      description: The date and time when the email message was received by the service
+        or client.
+      example: '2020-11-10T22:12:34.8196921Z'
+      default_field: false
+    - name: direction
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: The direction of the message based on the sending and receiving
+        domains.
+      example: inbound
+      default_field: false
+    - name: from
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: The email address of the sender, typically from the RFC 5322 `From:`
+        header field.
+      example: [email protected]
+      default_field: false
+    - name: local_id
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: 'Unique identifier given to the email by the source that created
+        the event.
+
+        Identifier is not persistent across hops.'
+      example: c26dbea0-80d5-463b-b93c-4e8b708219ce
+      default_field: false
+    - name: message_id
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: Identifier from the RFC 5322 `Message-ID:` email header that refers
+        to a particular email message.
+      example: <[email protected]>
+      default_field: false
+    - name: origination_timestamp
+      level: extended
+      type: date
+      description: The date and time the email message was composed. Many email clients
+        will fill in this value automatically when the message is sent by a user.
+      example: '2020-11-10T22:12:34.8196921Z'
+      default_field: false
+    - name: reply_to
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: The address that replies should be delivered to based on the value
+        in the RFC 5322 `Reply-To:` header.
+      example: [email protected]
+      default_field: false
+    - name: subject
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      multi_fields:
+      - name: text
+        type: match_only_text
+      description: A brief summary of the topic of the message.
+      example: Please see this important message.
+      default_field: false
+    - name: to
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: The email address(es) of the message recipients.
+      example: '[''[email protected]'', ''[email protected]'']'
+      default_field: false
+    - name: x_mailer
+      level: extended
+      type: keyword
+      ignore_above: 1024
+      description: The name of the application that was used to draft and send the
+        original email message.
+      example: Spambot v2.5
+      default_field: false
   - name: error
     title: Error
     group: 2

experimental/generated/csv/fields.csv

@@ -184,6 +184,28 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.12.0-dev+exp,true,dns,dns.response_code,keyword,extended,,NOERROR,The DNS response code.
 1.12.0-dev+exp,true,dns,dns.type,keyword,extended,,answer,"The type of DNS event captured, query or answer."
 1.12.0-dev+exp,true,ecs,ecs.version,keyword,core,,1.0.0,ECS version this event conforms to.
+1.12.0-dev+exp,true,email,email.attachments,nested,extended,,,List of objects describing the attachments.
+1.12.0-dev+exp,true,email,email.attachments.file.extension,keyword,extended,,txt,Attachment file extension.
+1.12.0-dev+exp,true,email,email.attachments.file.mime_type,keyword,extended,,text/plain,MIME type of the attachment file.
+1.12.0-dev+exp,true,email,email.attachments.file.name,keyword,extended,,attachment.txt,Name of the attachment file.
+1.12.0-dev+exp,true,email,email.attachments.file.size,long,extended,,64329,Attachment file size.
+1.12.0-dev+exp,true,email,email.attachments.hash.md5,keyword,extended,,e25f1c98ffdacf611473af364362ec48,MD5 hash of the attachment.
+1.12.0-dev+exp,true,email,email.attachments.hash.sha1,keyword,extended,,8c1cd40f17109b427e61d4e72ca6d9a4fc8175f3,SHA-1 hash of the attachment.
+1.12.0-dev+exp,true,email,email.attachments.hash.sha256,keyword,extended,,f0366b3559f577d8732f7e9cc343a4960d202e8137dcc42f9783f3963f6abc6a,SHA-256 hash of the attachment.
+1.12.0-dev+exp,true,email,email.bcc,keyword,extended,array,"['[email protected]', '[email protected]']",Email address(es) of BCC recipients
+1.12.0-dev+exp,true,email,email.cc,keyword,extended,array,"['[email protected]', '[email protected]']",Email address(es) of CC recipients
+1.12.0-dev+exp,true,email,email.content_type,keyword,extended,,text/plain,MIME type of the email message.
+1.12.0-dev+exp,true,email,email.delivery_timestamp,date,extended,,2020-11-10T22:12:34.8196921Z,Date and time when message was delivered.
+1.12.0-dev+exp,true,email,email.direction,keyword,extended,,inbound,Direction of the message.
+1.12.0-dev+exp,true,email,email.from,keyword,extended,,[email protected],The sender's email address.
+1.12.0-dev+exp,true,email,email.local_id,keyword,extended,,c26dbea0-80d5-463b-b93c-4e8b708219ce,Unique identifier given by the source.
+1.12.0-dev+exp,true,email,email.message_id,keyword,extended,,<[email protected]>,Value from the Message-ID header.
+1.12.0-dev+exp,true,email,email.origination_timestamp,date,extended,,2020-11-10T22:12:34.8196921Z,Date and time the email was composed.
+1.12.0-dev+exp,true,email,email.reply_to,keyword,extended,,[email protected],Address replies should be delivered to.
+1.12.0-dev+exp,true,email,email.subject,keyword,extended,,Please see this important message.,The subject of the email message.
+1.12.0-dev+exp,true,email,email.subject.text,match_only_text,extended,,Please see this important message.,The subject of the email message.
+1.12.0-dev+exp,true,email,email.to,keyword,extended,array,"['[email protected]', '[email protected]']",Email address(es) of the recipients.
+1.12.0-dev+exp,true,email,email.x_mailer,keyword,extended,,Spambot v2.5,Application that drafted email.
 1.12.0-dev+exp,true,error,error.code,keyword,core,,,Error code describing the error.
 1.12.0-dev+exp,true,error,error.id,keyword,core,,,Unique identifier for the error.
 1.12.0-dev+exp,true,error,error.message,match_only_text,core,,,Error message.