Add build-version option for building specific versions of ECS (#851)

* add build-version option for building specific versions of ECS

* attempt fixing travis build

* address review comments

* changelog and linting

.travis.yml

@@ -11,6 +11,7 @@ go:
 - 1.13.x
 
 install:
+- git fetch --tags --all
 - make setup
 
 addons:

CHANGELOG.next.md

@@ -61,6 +61,7 @@ Thanks, you're awesome :-) -->
 * Add support for reusing offical fieldsets in custom schemas. #751
 * Add full path names to reused fieldsets in `nestings` array in `ecs_nested.yml`. #803
 * Allow shorthand notation for including all subfields in subsets. #805
+* Add `ref` option to generator allowing schemas to be built for a specific ECS version. #851
 
 #### Deprecated
 

scripts/generator.py

@@ -3,6 +3,7 @@
 import os
 import schema_reader
 import yaml
+
 from generators import intermediate_files
 from generators import csv_generator
 from generators import es_template
@@ -17,20 +18,29 @@ def main():
     if args.include and [''] == args.include:
         args.include.clear()
 
-    ecs_version = read_version()
-    print('Running generator. ECS version ' + ecs_version)
+    if args.ref:
+        # Load ECS schemas from a specific git ref
+        print('Loading schemas from git ref ' + args.ref)
+        tree = ecs_helpers.get_tree_by_ref(args.ref)
+        ecs_version = read_version_from_tree(tree)
+        ecs_schemas = schema_reader.load_schemas_from_git(tree)
+    else:
+        # Load the default schemas
+        print('Loading default schemas')
+        ecs_version = read_version()
+        ecs_schemas = schema_reader.load_schemas_from_files()
 
-    # Load the default schemas
-    print('Loading default schemas')
-    intermediate_fields = schema_reader.load_schemas()
+    print('Running generator. ECS version ' + ecs_version)
+    intermediate_fields = schema_reader.create_schema_dicts(ecs_schemas)
 
     # Maybe load user specified directory of schemas
     if args.include:
         include_glob = ecs_helpers.get_glob_files(args.include, ecs_helpers.YAML_EXT)
 
         print('Loading user defined schemas: {0}'.format(include_glob))
 
-        intermediate_custom = schema_reader.load_schemas(include_glob)
+        custom_schemas = schema_reader.load_schemas_from_files(include_glob)
+        intermediate_custom = schema_reader.create_schema_dicts(custom_schemas)
         schema_reader.merge_schema_fields(intermediate_fields, intermediate_custom)
 
     schema_reader.assemble_reusables(intermediate_fields)
@@ -79,6 +89,7 @@ def argument_parser():
     parser.add_argument('--subset', nargs='+',
                         help='render a subset of the schema')
     parser.add_argument('--out', action='store', help='directory to store the generated files')
+    parser.add_argument('--ref', action='store', help='git reference to use when building schemas')
     return parser.parse_args()
 
 
@@ -87,5 +98,9 @@ def read_version(file='version'):
         return infile.read().rstrip()
 
 
+def read_version_from_tree(tree):
+    return tree['version'].data_stream.read().decode('utf-8').rstrip()
+
+
 if __name__ == '__main__':
     main()

scripts/generators/ecs_helpers.py

@@ -1,6 +1,7 @@
 import glob
 import os
 import yaml
+import git
 
 from collections import OrderedDict
 from copy import deepcopy
@@ -115,6 +116,12 @@ def get_glob_files(paths, file_types):
     return sorted(all_files)
 
 
+def get_tree_by_ref(ref):
+    repo = git.Repo(os.getcwd())
+    commit = repo.commit(ref)
+    return commit.tree
+
+
 def ecs_files():
     """Return the schema file list to load"""
     schema_glob = os.path.join(os.path.dirname(__file__), '../../schemas/*.yml')

scripts/requirements.txt

@@ -2,3 +2,4 @@ pip
 PyYAML==5.3b1
 autopep8==1.4.4
 yamllint==1.19.0
+gitpython==3.1.2

scripts/schema_reader.py

@@ -17,31 +17,40 @@
 # Loads schemas and perform cleanup of schema attributes
 
 
-def load_schemas(files=ecs_helpers.ecs_files()):
-    """Loads the list of files and performs schema level cleanup"""
-    fields_intermediate = load_schema_files(files)
-    finalize_schemas(fields_intermediate)
-    return fields_intermediate
-
-
-def load_schema_files(files):
+def create_schema_dicts(schemas):
     fields_nested = {}
-    for f in files:
-        new_fields = read_schema_file(f)
-        fields_nested.update(new_fields)
+    for schema in schemas:
+        raw = yaml.safe_load(schema)
+        fields_nested.update(create_fields_dict(raw))
+    finalize_schemas(fields_nested)
     return fields_nested
 
 
-def read_schema_file(file):
-    """Read a raw schema yml into a map, removing the wrapping array in each file"""
-    with open(file) as f:
-        raw = yaml.safe_load(f.read())
+def create_fields_dict(raw):
     fields = {}
     for field_set in raw:
         fields[field_set['name']] = field_set
     return fields
 
 
+def load_schemas_from_files(files=ecs_helpers.ecs_files()):
+    schemas = []
+    for file in files:
+        with open(file) as f:
+            content = f.read()
+            schemas.append(content)
+    return schemas
+
+
+def load_schemas_from_git(tree):
+    schemas = []
+    for blob in tree['schemas'].blobs:
+        if blob.name.endswith('.yml'):
+            content = blob.data_stream.read().decode('utf-8')
+            schemas.append(content)
+    return schemas
+
+
 def finalize_schemas(fields_nested):
     """Clean up all schema level attributes"""
     for schema_name in fields_nested:

scripts/tests/test_ecs_helpers.py

@@ -234,6 +234,11 @@ def test_get_nested_field(self):
         actual = ecs_helpers.get_nested_field(nested_field_name, fields)
         self.assertEqual(actual, expected)
 
+    def test_get_tree_by_ref(self):
+        ref = 'v1.5.0'
+        tree = ecs_helpers.get_tree_by_ref(ref)
+        self.assertEqual(tree.hexsha, '4449df245f6930d59bcd537a5958891261a9476b')
+
 
 if __name__ == '__main__':
     unittest.main()

scripts/tests/test_ecs_spec.py

@@ -5,20 +5,17 @@
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 
 from scripts import schema_reader
-
-schemas = schema_reader.load_schemas()
-schema_reader.assemble_reusables(schemas)
-(nested, flat) = schema_reader.generate_nested_flat(schemas)
+from generators import ecs_helpers
 
 
 class TestEcsSpec(unittest.TestCase):
     """Sanity check for things that should be true in the ECS spec."""
 
     def setUp(self):
-        global nested
-        global flat
-        self.ecs_nested = nested
-        self.ecs_fields = flat
+        schemas = schema_reader.load_schemas_from_files()
+        intermediate_schemas = schema_reader.create_schema_dicts(schemas)
+        schema_reader.assemble_reusables(intermediate_schemas)
+        (self.ecs_nested, self.ecs_fields) = schema_reader.generate_nested_flat(intermediate_schemas)
 
     def test_base_flat_name(self):
         self.assertIsInstance(self.ecs_fields['@timestamp'], dict)

scripts/tests/test_schema_reader.py

@@ -5,6 +5,7 @@
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 
 from scripts import schema_reader
+from generators import ecs_helpers
 
 
 class TestSchemaReader(unittest.TestCase):
@@ -68,8 +69,14 @@ def test_field_set_multi_field_defaults_missing_name(self):
         self.assertEqual(field, expected)
 
     def test_load_schemas_with_empty_list_loads_nothing(self):
-        result = schema_reader.load_schemas([])
-        self.assertEqual(result, ({}))
+        result = schema_reader.load_schemas_from_files([])
+        self.assertEqual(result, ([]))
+
+    def test_load_schemas_by_git_ref(self):
+        ref = 'v1.5.0'
+        tree = ecs_helpers.get_tree_by_ref(ref)
+        schemas = schema_reader.load_schemas_from_git(tree)
+        self.assertEqual(len(schemas), 42)
 
     def test_flatten_fields(self):
         fields = {