diff --git a/.buildkite/pipelines/ecs-dynamic-template-tests.yml b/.buildkite/pipelines/ecs-dynamic-template-tests.yml index b1fe972b724f1..a8145c61a2d40 100644 --- a/.buildkite/pipelines/ecs-dynamic-template-tests.yml +++ b/.buildkite/pipelines/ecs-dynamic-template-tests.yml @@ -7,3 +7,8 @@ steps: image: family/elasticsearch-ubuntu-2004 diskSizeGb: 350 machineType: custom-32-98304 +notify: + - slack: "#es-delivery" + if: build.state == "failed" + - email: "logs-plus@elastic.co" + if: build.state == "failed" diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs-dynamic-mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs-dynamic-mappings.json index 3df1c1bd7928a..fc29fc98dca96 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ecs-dynamic-mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs-dynamic-mappings.json @@ -83,24 +83,6 @@ }, { "ecs_path_match_keyword_and_match_only_text": { - "mapping": { - "fields": { - "text": { - "type": "match_only_text" - } - }, - "type": "keyword" - }, - "path_match": [ - "*file.path", - "*file.target_path", - "*os.full", - "user_agent.original" - ] - } - }, - { - "ecs_match_keyword_and_match_only_text": { "mapping": { "fields": { "text": { @@ -114,7 +96,13 @@ "*.executable", "*.name", "*.working_directory", - "*.full_name" + "*.full_name", + "*file.path", + "*file.target_path", + "*os.full", + "email.subject", + "vulnerability.description", + "user_agent.original" ] } }, diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index c6cb0e4ab91ff..7e19f988eacc6 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -1,4 +1,5 @@ apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { name 'x-pack-stack' @@ -8,13 +9,25 @@ esplugin { hasNativeController false requiresKeystore true } + base { archivesName = 'x-pack-stack' } dependencies { compileOnly project(path: xpackModule('core')) - testImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation(testArtifact(project(xpackModule('core')))) + javaRestTestImplementation project(path: ':x-pack:plugin:stack') + clusterModules project(':modules:mapper-extras') + clusterModules project(xpackModule('wildcard')) +} + +// These tests are only invoked direclty as part of a dedicated build job +tasks.named('javaRestTest').configure { + onlyIf("E2E test task must be invoked directly") { + gradle.startParameter.getTaskNames().contains(this.path) || + (gradle.startParameter.getTaskNames().contains(this.name) && gradle.startParameter.currentDir == project.projectDir) + } } addQaCheckDependencies(project) diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java new file mode 100644 index 0000000000000..9832c789e2a99 --- /dev/null +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -0,0 +1,413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.stack; + +import org.apache.http.HttpStatus; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.template.TemplateUtils; +import org.junit.BeforeClass; +import org.junit.ClassRule; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +@SuppressWarnings("unchecked") +public class EcsDynamicTemplatesIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("mapper-extras").module("wildcard").build(); + + // The dynamic templates we test against + public static final String ECS_DYNAMIC_TEMPLATES_FILE = "ecs-dynamic-mappings.json"; + + // The current ECS state (branch main) containing all fields in flattened form + private static final String ECS_FLAT_FILE_URL = "https://raw.githubusercontent.com/elastic/ecs/main/generated/ecs/ecs_flat.yml"; + + private static final Set OMIT_FIELD_TYPES = Set.of("object", "nested"); + + private static final Set OMIT_FIELDS = Set.of("data_stream.dataset", "data_stream.namespace", "data_stream.type"); + + private static Map ecsDynamicTemplates; + private static Map> ecsFlatFieldDefinitions; + private static Map ecsFlatMultiFieldDefinitions; + + @BeforeClass + public static void setupSuiteScopeCluster() throws Exception { + prepareEcsDynamicTemplates(); + prepareEcsDefinitions(); + } + + private static void prepareEcsDynamicTemplates() throws IOException { + String rawEcsComponentTemplate = TemplateUtils.loadTemplate( + "/" + ECS_DYNAMIC_TEMPLATES_FILE, + Integer.toString(1), + StackTemplateRegistry.TEMPLATE_VERSION_VARIABLE, + Collections.emptyMap() + ); + Map ecsDynamicTemplatesRaw; + try ( + XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, rawEcsComponentTemplate) + ) { + ecsDynamicTemplatesRaw = parser.map(); + } + + String errorMessage = String.format( + Locale.ENGLISH, + "ECS mappings component template '%s' structure has changed, this test needs to be adjusted", + ECS_DYNAMIC_TEMPLATES_FILE + ); + assertFalse(errorMessage, rawEcsComponentTemplate.isEmpty()); + Object mappings = ecsDynamicTemplatesRaw.get("template"); + assertNotNull(errorMessage, mappings); + assertThat(errorMessage, mappings, instanceOf(Map.class)); + Object dynamicTemplates = ((Map) mappings).get("mappings"); + assertNotNull(errorMessage, dynamicTemplates); + assertThat(errorMessage, dynamicTemplates, instanceOf(Map.class)); + assertEquals(errorMessage, 1, ((Map) dynamicTemplates).size()); + assertTrue(errorMessage, ((Map) dynamicTemplates).containsKey("dynamic_templates")); + ecsDynamicTemplates = (Map) dynamicTemplates; + } + + @SuppressForbidden(reason = "Opening socket connection to read ECS definitions from ECS GitHub repo") + private static void prepareEcsDefinitions() throws IOException { + Map ecsFlatFieldsRawMap; + URL ecsDefinitionsFlatFileUrl = new URL(ECS_FLAT_FILE_URL); + try (InputStream ecsDynamicTemplatesIS = ecsDefinitionsFlatFileUrl.openStream()) { + try ( + XContentParser parser = XContentFactory.xContent(XContentType.YAML) + .createParser(XContentParserConfiguration.EMPTY, ecsDynamicTemplatesIS) + ) { + ecsFlatFieldsRawMap = parser.map(); + } + } + String errorMessage = String.format( + Locale.ENGLISH, + "ECS flat mapping file at %s has changed, this test needs to be adjusted", + ECS_FLAT_FILE_URL + ); + assertFalse(errorMessage, ecsFlatFieldsRawMap.isEmpty()); + Map.Entry fieldEntry = ecsFlatFieldsRawMap.entrySet().iterator().next(); + assertThat(errorMessage, fieldEntry.getValue(), instanceOf(Map.class)); + Map fieldProperties = (Map) fieldEntry.getValue(); + assertFalse(errorMessage, fieldProperties.isEmpty()); + Map.Entry fieldProperty = fieldProperties.entrySet().iterator().next(); + assertThat(errorMessage, fieldProperty.getKey(), instanceOf(String.class)); + + OMIT_FIELDS.forEach(ecsFlatFieldsRawMap::remove); + + // noinspection + ecsFlatFieldDefinitions = (Map>) ecsFlatFieldsRawMap; + ecsFlatMultiFieldDefinitions = new HashMap<>(); + Iterator>> iterator = ecsFlatFieldDefinitions.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + Map definitions = entry.getValue(); + String type = (String) definitions.get("type"); + if (OMIT_FIELD_TYPES.contains(type)) { + iterator.remove(); + } + + List> multiFields = (List>) definitions.get("multi_fields"); + if (multiFields != null) { + multiFields.forEach(multiFieldsDefinitions -> { + String subfieldFlatName = Objects.requireNonNull(multiFieldsDefinitions.get("flat_name")); + String subfieldType = Objects.requireNonNull(multiFieldsDefinitions.get("type")); + ecsFlatMultiFieldDefinitions.put(subfieldFlatName, subfieldType); + }); + } + } + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + public void testFlattenedFields() throws IOException { + String indexName = "test-flattened-fields"; + createTestIndex(indexName); + Map flattenedFieldsMap = createTestDocument(true); + indexDocument(indexName, flattenedFieldsMap); + verifyEcsMappings(indexName); + } + + public void testFlattenedFieldsWithoutSubobjects() throws IOException { + String indexName = "test_flattened_fields_subobjects_false"; + createTestIndex(indexName, Map.of("subobjects", false)); + Map flattenedFieldsMap = createTestDocument(true); + indexDocument(indexName, flattenedFieldsMap); + verifyEcsMappings(indexName); + } + + public void testNestedFields() throws IOException { + String indexName = "test-nested-fields"; + createTestIndex(indexName); + Map nestedFieldsMap = createTestDocument(false); + indexDocument(indexName, nestedFieldsMap); + verifyEcsMappings(indexName); + } + + private static void indexDocument(String indexName, Map flattenedFieldsMap) throws IOException { + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + Request indexRequest = new Request("POST", "/" + indexName + "/_doc"); + indexRequest.setJsonEntity(Strings.toString(bodyBuilder.map(flattenedFieldsMap))); + // noinspection resource + Response response = ESRestTestCase.client().performRequest(indexRequest); + assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); + } + } + + private Map createTestDocument(boolean flattened) { + Map testFieldsMap = new HashMap<>(); + for (Map.Entry> fieldEntry : ecsFlatFieldDefinitions.entrySet()) { + String flattenedFieldName = fieldEntry.getKey(); + Map fieldDefinitions = fieldEntry.getValue(); + String type = (String) fieldDefinitions.get("type"); + assertNotNull( + String.format(Locale.ENGLISH, "Can't find type for field '%s' in %s file", flattenedFieldName, ECS_DYNAMIC_TEMPLATES_FILE), + type + ); + Object testValue = generateTestValue(type); + if (flattened) { + testFieldsMap.put(flattenedFieldName, testValue); + } else { + Map currentField = testFieldsMap; + Iterator fieldPathPartsIterator = Arrays.stream(flattenedFieldName.split("\\.")).iterator(); + String subfield = fieldPathPartsIterator.next(); + while (fieldPathPartsIterator.hasNext()) { + currentField = (Map) currentField.computeIfAbsent(subfield, ignore -> new HashMap<>()); + subfield = fieldPathPartsIterator.next(); + } + currentField.put(subfield, testValue); + } + } + return testFieldsMap; + } + + private static void createTestIndex(String indexName) throws IOException { + createTestIndex(indexName, null); + } + + private static void createTestIndex(String indexName, @Nullable Map customMappings) throws IOException { + final Map indexMappings; + if (customMappings != null) { + indexMappings = Stream.concat(ecsDynamicTemplates.entrySet().stream(), customMappings.entrySet().stream()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + } else { + indexMappings = ecsDynamicTemplates; + } + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + bodyBuilder.startObject(); + bodyBuilder.startObject("settings"); + bodyBuilder.field("index.mapping.total_fields.limit", 10000); + bodyBuilder.endObject(); + bodyBuilder.field("mappings", indexMappings); + bodyBuilder.endObject(); + + Request createIndexRequest = new Request("PUT", "/" + indexName); + createIndexRequest.setJsonEntity(Strings.toString(bodyBuilder)); + // noinspection resource + Response response = ESRestTestCase.client().performRequest(createIndexRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + } + + private Object generateTestValue(String type) { + switch (type) { + case "geo_point" -> { + return new double[] { randomDouble(), randomDouble() }; + } + case "long" -> { + return randomLong(); + } + case "int" -> { + return randomInt(); + } + case "float", "scaled_float" -> { + return randomFloat(); + } + case "keyword", "wildcard", "text", "match_only_text" -> { + return randomAlphaOfLength(20); + } + case "constant_keyword" -> { + return "test"; + } + case "date" -> { + return DateFormatter.forPattern("strict_date_optional_time").formatMillis(System.currentTimeMillis()); + } + case "ip" -> { + return NetworkAddress.format(randomIp(true)); + } + case "boolean" -> { + return randomBoolean(); + } + case "flattened" -> { + // creating multiple subfields + return Map.of("subfield1", randomAlphaOfLength(20), "subfield2", randomAlphaOfLength(20)); + } + } + throw new IllegalArgumentException("Unknown field type: " + type); + } + + private Map getMappings(String indexName) throws IOException { + Request getMappingRequest = new Request("GET", "/" + indexName + "/_mapping"); + // noinspection resource + Response response = ESRestTestCase.client().performRequest(getMappingRequest); + assertEquals(response.getStatusLine().getStatusCode(), HttpStatus.SC_OK); + Map mappingResponse; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, response.getEntity().getContent())) { + mappingResponse = parser.map(); + } + assertThat(mappingResponse.size(), equalTo(1)); + Map indexMap = (Map) mappingResponse.get(indexName); + assertNotNull(indexMap); + Map mappings = (Map) indexMap.get("mappings"); + assertNotNull(mappings); + return (Map) mappings.get("properties"); + } + + private void processRawMappingsSubtree( + final Map fieldSubtrees, + final Map flatFieldMappings, + final Map flatMultiFieldsMappings, + final String subtreePrefix + ) { + fieldSubtrees.forEach((fieldName, fieldMappings) -> { + String fieldFullPath = subtreePrefix + fieldName; + Map fieldMappingsMap = ((Map) fieldMappings); + String type = (String) fieldMappingsMap.get("type"); + if (type != null) { + flatFieldMappings.put(fieldFullPath, type); + } + Map subfields = (Map) fieldMappingsMap.get("properties"); + if (subfields != null) { + processRawMappingsSubtree(subfields, flatFieldMappings, flatMultiFieldsMappings, fieldFullPath + "."); + } + + Map> fields = (Map>) fieldMappingsMap.get("fields"); + if (fields != null) { + fields.forEach((subFieldName, multiFieldMappings) -> { + String subFieldFullPath = fieldFullPath + "." + subFieldName; + String subFieldType = Objects.requireNonNull(multiFieldMappings.get("type")); + flatMultiFieldsMappings.put(subFieldFullPath, subFieldType); + }); + } + }); + } + + private void verifyEcsMappings(String indexName) throws IOException { + final Map rawMappings = getMappings(indexName); + final Map flatFieldMappings = new HashMap<>(); + final Map flatMultiFieldsMappings = new HashMap<>(); + processRawMappingsSubtree(rawMappings, flatFieldMappings, flatMultiFieldsMappings, ""); + + Map> shallowFieldMapCopy = new HashMap<>(ecsFlatFieldDefinitions); + logger.info("Testing mapping of {} ECS fields", shallowFieldMapCopy.size()); + List nonEcsFields = new ArrayList<>(); + Map fieldToWrongMappingType = new HashMap<>(); + flatFieldMappings.forEach((fieldName, actualMappingType) -> { + Map expectedMappings = shallowFieldMapCopy.remove(fieldName); + if (expectedMappings == null) { + nonEcsFields.add(fieldName); + } else { + String expectedType = (String) expectedMappings.get("type"); + if (actualMappingType.equals(expectedType) == false) { + fieldToWrongMappingType.put(fieldName, actualMappingType); + } + } + }); + + Map shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions); + logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size()); + flatMultiFieldsMappings.forEach((fieldName, actualMappingType) -> { + String expectedType = shallowMultiFieldMapCopy.remove(fieldName); + if (expectedType != null) { + // not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to + // ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead + // to multi-field mapping for ECS fields for which such are not defined + if (actualMappingType.equals(expectedType) == false) { + fieldToWrongMappingType.put(fieldName, actualMappingType); + } + } + }); + + shallowFieldMapCopy.forEach( + (fieldName, expectedMappings) -> logger.error( + "ECS field '{}' is not covered by the current dynamic templates. Update {} so that this field is mapped to type '{}'.", + fieldName, + ECS_DYNAMIC_TEMPLATES_FILE, + expectedMappings.get("type") + ) + ); + shallowMultiFieldMapCopy.keySet().forEach(field -> { + int lastDotIndex = field.lastIndexOf('.'); + String parentField = field.substring(0, lastDotIndex); + String subfield = field.substring(lastDotIndex + 1); + logger.error( + "ECS field '{}' is expected to have a multi-field mapping with subfield '{}'. Fix {} accordingly.", + parentField, + subfield, + ECS_DYNAMIC_TEMPLATES_FILE + ); + }); + fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> { + String ecsExpectedType = (String) ecsFlatFieldDefinitions.get(fieldName).get("type"); + logger.error( + "ECS field '{}' should be mapped to type '{}' but is mapped to type '{}'. Update {} accordingly.", + fieldName, + ecsExpectedType, + actualMappingType, + ECS_DYNAMIC_TEMPLATES_FILE + ); + }); + nonEcsFields.forEach(field -> logger.error("The test document contains '{}', which is not an ECS field", field)); + + assertTrue("ECS is not fully covered by the current ECS dynamic templates, see details above", shallowFieldMapCopy.isEmpty()); + assertTrue( + "ECS is not fully covered by the current ECS dynamic templates' multi-fields definitions, see details above", + shallowMultiFieldMapCopy.isEmpty() + ); + assertTrue( + "At least one field was mapped with a type that mismatches the ECS definitions, see details above", + fieldToWrongMappingType.isEmpty() + ); + assertTrue("The test document contains non-ECS fields, see details above", nonEcsFields.isEmpty()); + } +}