From 2585f618f771d0534e3546b7bc728231a35ac4a1 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 28 Feb 2019 12:11:00 +0100 Subject: [PATCH] Don't expect aliases in output dictionaries (#10921) With the creation of aliases for ECS we have found that some features weren't being migrated and were writing to aliases (like in #10757). Consider writing to an alias field an error. --- CHANGELOG-developer.next.asciidoc | 2 ++ libbeat/tests/system/beat/beat.py | 30 ++++++++++++++++++--------- packetbeat/tests/system/packetbeat.py | 2 +- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 323dd72984b..5edff8d9cde 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -19,6 +19,7 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1..master[Check the HEAD dif The list below covers the major changes between 7.0.0-beta1 and master only. ==== Breaking changes + - Remove support for deprecated `GenRootCmd` methods. {pull}10721[10721] ==== Bugfixes @@ -29,3 +30,4 @@ The list below covers the major changes between 7.0.0-beta1 and master only. - Metricset generator generates beta modules by default now. {pull}10657[10657] - Move host name addition to a processor. {pull}10728[10728] - The `beat.Event` accessor methods now support `@metadata` keys. {pull}10761[10761] +- Assertion for documented fields in tests fails if any of the fields in the tested event is documented as an alias. {pull}10921[10921] diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index fa88fb1aba3..452d3ed9287 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -493,9 +493,10 @@ def load_fields(self, fields_doc=None): def extract_fields(doc_list, name): fields = [] dictfields = [] + aliases = [] if doc_list is None: - return fields, dictfields + return fields, dictfields, aliases for field in doc_list: @@ -510,14 +511,19 @@ def extract_fields(doc_list, name): newName = field["name"] if field.get("type") == "group": - subfields, subdictfields = extract_fields(field["fields"], newName) + subfields, subdictfields, subaliases = extract_fields(field["fields"], newName) fields.extend(subfields) dictfields.extend(subdictfields) + aliases.extend(subaliases) else: fields.append(newName) if field.get("type") in ["object", "geo_point"]: dictfields.append(newName) - return fields, dictfields + + if field.get("type") == "alias": + aliases.append(newName) + + return fields, dictfields, aliases global yaml_cache @@ -542,12 +548,14 @@ def extract_fields(doc_list, name): fields = [] dictfields = [] + aliases = [] for item in doc: - subfields, subdictfields = extract_fields(item["fields"], "") + subfields, subdictfields, subaliases = extract_fields(item["fields"], "") fields.extend(subfields) dictfields.extend(subdictfields) - return fields, dictfields + aliases.extend(subaliases) + return fields, dictfields, aliases def flatten_object(self, obj, dict_fields, prefix=""): result = {} @@ -610,7 +618,7 @@ def assert_fields_are_documented(self, evt): Assert that all keys present in evt are documented in fields.yml. This reads from the global fields.yml, means `make collect` has to be run before the check. """ - expected_fields, dict_fields = self.load_fields() + expected_fields, dict_fields, aliases = self.load_fields() flat = self.flatten_object(evt, dict_fields) def field_pattern_match(pattern, key): @@ -625,15 +633,17 @@ def field_pattern_match(pattern, key): return False return True - def is_documented(key): - if key in expected_fields: + def is_documented(key, docs): + if key in docs: return True - for pattern in (f for f in expected_fields if "*" in f): + for pattern in (f for f in docs if "*" in f): if field_pattern_match(pattern, key): return True return False for key in flat.keys(): metaKey = key.startswith('@metadata.') - if not(is_documented(key) or metaKey): + if not(is_documented(key, expected_fields) or metaKey): raise Exception("Key '{}' found in event is not documented!".format(key)) + if is_documented(key, aliases): + raise Exception("Key '{}' found in event is documented as an alias!".format(key)) diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index e6ec4b2559e..24b59a32b9c 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -137,5 +137,5 @@ def read_output(self, def setUp(self): - self.expected_fields, self.dict_fields = self.load_fields() + self.expected_fields, self.dict_fields, _ = self.load_fields() super(BaseTest, self).setUp()