Beta label support (#1051)

Co-authored-by: Mathieu Martin <webmat@gmail.com>

Author: ebeahan

CHANGELOG.next.md

@@ -37,6 +37,7 @@ Thanks, you're awesome :-) -->
 * Added support for `scaled_float`'s mandatory parameter `scaling_factor`. #1042
 * Added ability for --oss flag to fall back `constant_keyword` to `keyword`. #1046
 * Added support in the generated Go source go for `wildcard`, `version`, and `constant_keyword` data types. #1050
+* Added support for marking fields, field sets, or field reuse as beta in the documentation. #1051
 * Added support for `constant_keyword`'s optional parameter `value`. #1112
 
 #### Improvements

schemas/README.md

@@ -32,6 +32,8 @@ Optional field set attributes:
 - type (ignored): at this level, should always be `group`
 - reusable (optional): Used to identify which field sets are expected to be reused in multiple places.
   See "Field set reuse" for details.
+- beta: Adds a beta marker for the entire fieldset. The text provided in this attribute is used as content of the beta marker in the documentation.
+  Beta notices should not have newlines.
 
 ### Field set reuse
 
@@ -104,6 +106,18 @@ The above defines all process fields in both places:
 }
 ```
 
+The `beta` marker can optionally be used along with `at` and `as` to include a beta marker in the field reuses section, marking specific reuse locations as beta.
+Beta notices should not have newlines.
+
+```
+  reusable:
+    top_level: true
+    expected:
+    - at: user
+      as: target
+      beta: Reusing these fields in this location is currently considered beta.
+```
+
 ### List of fields
 
 Array of YAML objects:
@@ -134,6 +148,7 @@ Supported keys to describe fields
 - format: Field format that can be used in a Kibana index template.
 - normalize: Normalization steps that should be applied at ingestion time. Supported values:
   - array: the content of the field should be an array (even when there's only one value).
+- beta (optional): Adds a beta marker for the field to the description. The text provided in this attribute is used as content of the beta marker in the documentation. Note that when a whole field set is marked as beta, it is not necessary nor recommended to mark all fields in the field set as beta. Beta notices should not have newlines.
 
 Supported keys to describe expected values for a field
 

scripts/generators/asciidoc_fields.py

@@ -39,7 +39,8 @@ def render_nestings_reuse_section(fieldset):
         rows.append({
             'flat_nesting': "{}.*".format(reused_here_entry['full']),
             'name': reused_here_entry['schema_name'],
-            'short': reused_here_entry['short']
+            'short': reused_here_entry['short'],
+            'beta': reused_here_entry.get('beta', '')
         })
 
     return sorted(rows, key=lambda x: x['flat_nesting'])

scripts/generators/ecs_helpers.py

@@ -189,5 +189,5 @@ def strict_warning(msg):
     :param msg: custom text which will be displayed with wrapped boilerplate
                 for strict warning messages.
     """
-    warn_message = f"{msg}\n\nThis will cause an exception when running in strict mode."
-    warnings.warn(warn_message)
+    warn_message = f"{msg}\n\nThis will cause an exception when running in strict mode.\nWarning check:"
+    warnings.warn(warn_message, stacklevel=3)

scripts/schema/cleaner.py

@@ -76,6 +76,8 @@ def schema_mandatory_attributes(schema):
 def schema_assertions_and_warnings(schema):
     '''Additional checks on a fleshed out schema'''
     single_line_short_description(schema, strict=strict_mode)
+    if 'beta' in schema['field_details']:
+        single_line_beta_description(schema, strict=strict_mode)
 
 
 def normalize_reuse_notation(schema):
@@ -181,6 +183,8 @@ def field_assertions_and_warnings(field):
         # check short description length if in strict mode
         single_line_short_description(field, strict=strict_mode)
         check_example_value(field, strict=strict_mode)
+        if 'beta' in field['field_details']:
+            single_line_beta_description(field, strict=strict_mode)
         if field['field_details']['level'] not in ACCEPTABLE_FIELD_LEVELS:
             msg = "Invalid level for field '{}'.\nValue: {}\nAcceptable values: {}".format(
                 field['field_details']['name'], field['field_details']['level'],
@@ -220,3 +224,13 @@ def check_example_value(field, strict=True):
             raise ValueError(msg)
         else:
             ecs_helpers.strict_warning(msg)
+
+
+def single_line_beta_description(schema_or_field, strict=True):
+    if "\n" in schema_or_field['field_details']['beta']:
+        msg = "Beta descriptions must be single line.\n"
+        msg += f"Offending field or field set: {schema_or_field['field_details']['name']}"
+        if strict:
+            raise ValueError(msg)
+        else:
+            ecs_helpers.strict_warning(msg)

scripts/schema/finalizer.py

@@ -128,6 +128,9 @@ def append_reused_here(reused_schema, reuse_entry, destination_schema):
         'full': reuse_entry['full'],
         'short': reused_schema['field_details']['short'],
     }
+    # Check for beta attribute
+    if 'beta' in reuse_entry:
+        reused_here_entry['beta'] = reuse_entry['beta']
     destination_schema['schema_details']['reused_here'].extend([reused_here_entry])
 
 

scripts/templates/field_details.j2

@@ -10,6 +10,13 @@ Find additional usage and examples in the {{ fieldset['name'] }} fields <<ecs-{{
 
 {% endif %}
 
+{# Fieldset label beta header -#}
+{% if fieldset['beta'] -%}
+
+beta::[ {{ fieldset['beta'] }}]
+
+{% endif -%}
+
 {# Field Details Table Header -#}
 [discrete]
 ==== {{ fieldset['title'] }} Field Details
@@ -27,7 +34,14 @@ Find additional usage and examples in the {{ fieldset['name'] }} fields <<ecs-{{
 {# `Field` column -#}
 | {{ field['flat_name'] }}
 {# `Description` column -#}
+{#- Beta fields will add the `beta` label -#}
+{% if field['beta'] -%}
+| beta:[ {{ field['beta'] }} ]
+
+{{ field['description']|replace("\n", "\n\n") }}
+{%- else -%}
 | {{ field['description']|replace("\n", "\n\n") }}
+{%- endif %}
 
 type: {{ field['type'] }}
 
@@ -108,8 +122,16 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d
 
 {% for entry in render_nestings_reuse_section -%}
 
+{#- Beta marker on nested fields -#}
 | <<ecs-{{ entry['name'] }},{{ entry['flat_nesting'] }}>>
+{#- Beta marker on nested fields -#}
+{%- if entry['beta'] -%}
+| beta:[ {{ entry['beta'] }}]
+
+{{ entry['short'] }}
+{%- else %}
 | {{ entry['short'] }}
+{%- endif %}
 
 // ===============================================================
 

scripts/tests/unit/test_schema_finalizer.py

@@ -92,7 +92,7 @@ def schema_user(self):
                         'order': 2,
                         'expected': [
                             {'full': 'server.user', 'at': 'server', 'as': 'user'},
-                            {'full': 'user.target', 'at': 'user', 'as': 'target'},
+                            {'full': 'user.target', 'at': 'user', 'as': 'target', 'beta': 'Some beta notice'},
                             {'full': 'user.effective', 'at': 'user', 'as': 'effective'},
                         ]
                     }
@@ -211,7 +211,7 @@ def test_perform_reuse_with_foreign_reuse_and_self_reuse(self):
                       fields['process']['schema_details']['reused_here'])
         self.assertIn({'full': 'user.effective', 'schema_name': 'user', 'short': 'short desc'},
                       fields['user']['schema_details']['reused_here'])
-        self.assertIn({'full': 'user.target', 'schema_name': 'user', 'short': 'short desc'},
+        self.assertIn({'full': 'user.target', 'schema_name': 'user', 'short': 'short desc', 'beta': 'Some beta notice'},
                       fields['user']['schema_details']['reused_here'])
         self.assertIn({'full': 'server.user', 'schema_name': 'user', 'short': 'short desc'},
                       fields['server']['schema_details']['reused_here'])