Generate outputs for each individual subset with custom options

scripts/generator.py

@@ -2,6 +2,7 @@
 import glob
 import os
 import yaml
+import time
 
 from generators import asciidoc_fields
 from generators import beats
@@ -22,15 +23,6 @@ def main():
     ecs_version = read_version(args.ref)
     print('Running generator. ECS version ' + ecs_version)
 
-    # To debug issues in the gradual building up of the nested structure, insert
-    # statements like this after any step of interest.
-    # ecs_helpers.yaml_dump('ecs.yml', fields)
-
-    fields = loader.load_schemas(ref=args.ref, included_files=args.include)
-    cleaner.clean(fields)
-    finalizer.finalize(fields)
-    fields = subset_filter.filter(fields, args.subset)
-
     # default location to save files
     out_dir = 'generated'
     docs_dir = 'docs'
@@ -44,7 +36,23 @@ def main():
     ecs_helpers.make_dirs(out_dir)
     ecs_helpers.make_dirs(docs_dir)
 
-    nested, flat = intermediate_files.generate(fields, out_dir, default_dirs)
+    # To debug issues in the gradual building up of the nested structure, insert
+    # statements like this after any step of interest.
+    # ecs_helpers.yaml_dump('ecs.yml', fields)
+
+    fields = loader.load_schemas(ref=args.ref, included_files=args.include)
+    cleaner.clean(fields)
+    finalizer.finalize(fields)
+    subsets = subset_filter.load_subset_definitions(args.subset)
+    for subset in subsets:
+        subfields = subset_filter.extract_matching_fields(fields, subset['fields'])
+        intermediate_files.generate(subfields, os.path.join(out_dir, 'ecs', 'subset', subset['name']), default_dirs)
+
+    merged_subset = subset_filter.combine_all_subsets(subsets)
+    if merged_subset:
+        fields = subset_filter.extract_matching_fields(fields, merged_subset)
+
+    nested, flat = intermediate_files.generate(fields, os.path.join(out_dir, 'ecs'), default_dirs)
     if args.intermediate_only:
         exit()
 

scripts/generators/intermediate_files.py

@@ -6,17 +6,16 @@
 
 
 def generate(fields, out_dir, default_dirs):
-    ecs_helpers.make_dirs(join(out_dir, 'ecs'))
+    ecs_helpers.make_dirs(join(out_dir))
 
     # Should only be used for debugging ECS development
     if default_dirs:
-        ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs.yml'), fields)
-
+        ecs_helpers.yaml_dump(join(out_dir, 'ecs.yml'), fields)
     flat = generate_flat_fields(fields)
     nested = generate_nested_fields(fields)
 
-    ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs_flat.yml'), flat)
-    ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs_nested.yml'), nested)
+    ecs_helpers.yaml_dump(join(out_dir, 'ecs_flat.yml'), flat)
+    ecs_helpers.yaml_dump(join(out_dir, 'ecs_nested.yml'), nested)
     return nested, flat
 
 

scripts/schema/loader.py

@@ -195,6 +195,8 @@ def merge_fields(a, b):
                     asd['reusable']['top_level'] = bsd['reusable']['top_level']
                 else:
                     asd['reusable'].setdefault('top_level', True)
+                if 'order' in bsd['reusable']:
+                    asd['reusable']['order'] = bsd['reusable']['order']
                 asd['reusable'].setdefault('expected', [])
                 asd['reusable']['expected'].extend(bsd['reusable']['expected'])
                 bsd.pop('reusable')

scripts/schema/subset_filter.py

@@ -1,28 +1,27 @@
 import glob
 import yaml
+import copy
 
 # This script takes all ECS and custom fields already loaded, and lets users
 # filter out the ones they don't need.
 
 
-def filter(fields, subset_file_globs):
-    '''
-    Takes the deeply nested field structure and the subset file names.
-
-    It returns a copy of the fields that matches the whitelist defined in the subset.
-    '''
-    if not subset_file_globs or subset_file_globs == []:
-        return fields
-    subset_definitions = load_subset_definitions(subset_file_globs)
-    filtered_fields = extract_matching_fields(fields, subset_definitions)
-    return filtered_fields
+def combine_all_subsets(subsets):
+    '''Merges N subsets into one. Strips top level 'name' and 'fields' keys as well as non-ECS field options since we can't know how to merge those.'''
+    merged_subset = {}
+    for subset in subsets:
+        strip_non_ecs_options(subset['fields'])
+        merge_subsets(merged_subset, subset['fields'])
+    return merged_subset
 
 
 def load_subset_definitions(file_globs):
-    subsets = {}
+    if not file_globs:
+        return []
+    subsets = []
     for f in eval_globs(file_globs):
         raw = load_yaml_file(f)
-        merge_subsets(subsets, raw)
+        subsets.append(raw)
     if not subsets:
         raise ValueError('--subset specified, but no subsets found in {}'.format(file_globs))
     return subsets
@@ -50,29 +49,56 @@ def warn(message):
     print(message)
 
 
+ecs_options = ['fields', 'enabled', 'index']
+
+
+def strip_non_ecs_options(subset):
+    for key in subset:
+        subset[key] = {x: subset[key][x] for x in subset[key] if x in ecs_options}
+        if 'fields' in subset[key] and isinstance(subset[key]['fields'], dict):
+            strip_non_ecs_options(subset[key]['fields'])
+
+
 def merge_subsets(a, b):
-    '''Merges field subset definitions together. The b subset is merged into the a subset.'''
+    '''Merges field subset definitions together. The b subset is merged into the a subset. Assumes that subsets have been stripped of non-ecs options.'''
     for key in b:
         if key not in a:
             a[key] = b[key]
-        elif 'fields' not in a[key] or 'fields' not in b[key] or b[key]['fields'] == '*':
-            a[key]['fields'] = '*'
-        elif isinstance(a[key]['fields'], dict) and isinstance(b[key]['fields'], dict):
-            merge_subsets(a[key]['fields'], b[key]['fields'])
+        elif 'fields' in a[key] and 'fields' in b[key]:
+            if b[key]['fields'] == '*':
+                a[key]['fields'] = '*'
+            elif isinstance(a[key]['fields'], dict) and isinstance(b[key]['fields'], dict):
+                merge_subsets(a[key]['fields'], b[key]['fields'])
+        elif 'fields' in a[key] or 'fields' in b[key]:
+            raise ValueError("Subsets unmergeable: 'fields' found in key '{}' in only one subset".format(key))
+        # If both subsets have enabled set to False, this will leave enabled: False in the merged subset
+        # Otherwise, enabled is removed and is implicitly true
+        if a[key].get('enabled', True) or b[key].get('enabled', True):
+            a[key].pop('enabled', None)
+        # Same logic from 'enabled' applies to 'index'
+        if a[key].get('index', True) or b[key].get('index', True):
+            a[key].pop('index', None)
 
 
 def extract_matching_fields(fields, subset_definitions):
-    retained_fields = {}
-    allowed_options = ['fields']
+    '''Removes fields that are not in the subset definition. Returns a copy without modifying the input fields dict.'''
+    retained_fields = {x: fields[x].copy() for x in subset_definitions}
     for key, val in subset_definitions.items():
+        if 'field_details' in fields[key]:
+            retained_fields[key]['field_details'] = fields[key]['field_details'].copy()
         for option in val:
-            if option not in allowed_options:
-                raise ValueError('Unsupported option found in subset: {}'.format(option))
-        # A missing fields key is shorthand for including all subfields
-        if 'fields' not in val or val['fields'] == '*':
-            retained_fields[key] = fields[key]
-        elif isinstance(val['fields'], dict):
-            # Copy the full field over so we get all the options, then replace the 'fields' with the right subset
-            retained_fields[key] = fields[key]
-            retained_fields[key]['fields'] = extract_matching_fields(fields[key]['fields'], val['fields'])
+            if option != 'fields':
+                retained_fields[key].setdefault('field_details', {})
+                retained_fields[key]['field_details'][option] = val[option]
+        # If the field in the schema has a 'fields' key, we expect a 'fields' key in the subset
+        if 'fields' in fields[key]:
+            if 'fields' not in val:
+                raise ValueError("'fields' key expected, not found in subset for {}".format(key))
+            elif isinstance(val['fields'], dict):
+                retained_fields[key]['fields'] = extract_matching_fields(fields[key]['fields'], val['fields'])
+            elif val['fields'] != "*":
+                raise ValueError("Unexpected value '{}' found in 'fields' key".format(val['fields']))
+        # If the field in the schema does not have a 'fields' key, there should not be a 'fields' key in the subset
+        elif 'fields' in val:
+            raise ValueError("'fields' key not expected, found in subset for {}".format(key))
     return retained_fields

scripts/tests/unit/test_schema_subset_filter.py

@@ -39,8 +39,7 @@ def test_basic_merging(self):
 
     def test_merging_superset(self):
         # 'log' is used to test superset with the explicit '{'fields': '*'}' notation
-        # 'process' is used to test superset with the shorhand '{}' notation
-        supersets = {'log': {'fields': '*'}, 'process': {}}
+        supersets = {'log': {'fields': '*'}, 'process': {'fields': '*'}}
         supserseded = {
             'log': {'fields': {'syslog': {'fields': '*'}}},
             'process': {'fields': {'parent': {'fields': '*'}}},
@@ -55,6 +54,50 @@ def test_merging_superset(self):
         subset_filter.merge_subsets(subsets, supersets)
         self.assertEqual(subsets, supersets)
 
+    def test_subset_option_merging(self):
+        subset1 = {
+            'log': {'enabled': False},
+            'network': {'enabled': False, 'fields': '*'},
+            'base': {'fields': {'message': {'index': False}}},
+        }
+        subset2 = {
+            'log': {'enabled': False},
+            'network': {'fields': '*'},
+            'base': {'fields': {'message': {}}},
+        }
+        expected = {
+            'log': {'enabled': False},
+            'network': {'fields': '*'},
+            'base': {'fields': {'message': {}}},
+        }
+        merged = {}
+        subset_filter.merge_subsets(merged, subset1)
+        subset_filter.merge_subsets(merged, subset2)
+        self.assertEqual(merged, expected)
+
+    def test_strip_non_ecs_options(self):
+        subset = {
+            'log': {
+                'custom_option': True,
+                'enabled': False,
+                'fields': {
+                    'syslog': {
+                        'custom_option': True
+                    }
+                }
+            }
+        }
+        expected = {
+            'log': {
+                'enabled': False,
+                'fields': {
+                    'syslog': {}
+                }
+            }
+        }
+        subset_filter.strip_non_ecs_options(subset)
+        self.assertEqual(subset, expected)
+
     def schema_log(self):
         return {
             'log': {
@@ -91,18 +134,13 @@ def schema_log(self):
             }
         }
 
-    def test_extract_matching_fields_shorthand_notation(self):
-        subset = {'log': {}}
-        filtered_fields = subset_filter.extract_matching_fields(self.schema_log(), subset)
-        self.assertEqual(filtered_fields, self.schema_log())
-
     def test_extract_matching_fields_explicit_all_fields_notation(self):
         subset = {'log': {'fields': '*'}}
         filtered_fields = subset_filter.extract_matching_fields(self.schema_log(), subset)
         self.assertEqual(filtered_fields, self.schema_log())
 
     def test_extract_matching_fields_subfields_only_notation(self):
-        subset = {'log': {'fields': {'origin': {}}}}
+        subset = {'log': {'fields': {'origin': {'fields': '*'}}}}
         filtered_fields = subset_filter.extract_matching_fields(self.schema_log(), subset)
         expected_fields = {
             'log': {
@@ -158,3 +196,27 @@ def test_extract_matching_individual_field(self):
             }
         }
         self.assertEqual(filtered_fields, expected_fields)
+
+    def test_extract_field_with_options(self):
+        subset = {'log': {'enabled': False, 'fields': {'level': {'custom_option': True}}}}
+        filtered_fields = subset_filter.extract_matching_fields(self.schema_log(), subset)
+        expected_fields = {
+            'log': {
+                'schema_details': {'root': False},
+                'field_details': {
+                    'name': 'log',
+                    'type': 'group',
+                    'enabled': False
+                },
+                'fields': {
+                    'level': {
+                        'field_details': {
+                            'name': 'level',
+                            'type': 'keyword',
+                            'custom_option': True
+                        }
+                    }
+                }
+            }
+        }
+        self.assertEqual(filtered_fields, expected_fields)