[1.x] Add example value composite type checking (#966)

CHANGELOG.next.md

@@ -38,6 +38,7 @@ Thanks, you're awesome :-) -->
 #### Added
 
 * Introduced `--strict` flag to perform stricter schema validation when running the generator script. #937
+* Added check under `--strict` that ensures composite types in example fields are quoted. #966
 
 #### Improvements
 

USAGE.md

@@ -308,6 +308,7 @@ $ python/generator.py --strict
 Strict mode requires the following conditions, else the script exits on an exception:
 
 * Short descriptions must be less than or equal to 120 characters.
+* Example values containing arrays or objects must be quoted to avoid unexpected YAML interpretation when the schema files or artifacts are relied on downstream.
 
 The current artifacts generated and published in the ECS repo will always be created using strict mode. However, older ECS versions (pre `v1.5.0`) will cause
 an exception if attempting to generate them using `--strict`. This is due to schema validation checks introduced after that version was released.

docs/field-details.asciidoc

@@ -1211,7 +1211,7 @@ Note: this field should contain an array of values.
 
 
 
-example: `['RD', 'RA']`
+example: `["RD", "RA"]`
 
 | extended
 
@@ -1343,7 +1343,7 @@ Note: this field should contain an array of values.
 
 
 
-example: `['10.10.10.10', '10.10.10.11']`
+example: `["10.10.10.10", "10.10.10.11"]`
 
 | extended
 
@@ -4205,7 +4205,7 @@ Note: this field should contain an array of values.
 
 
 
-example: `['/usr/bin/ssh', '-l', 'user', '10.0.0.16']`
+example: `["/usr/bin/ssh", "-l", "user", "10.0.0.16"]`
 
 | extended
 
@@ -4718,7 +4718,7 @@ Note: this field should contain an array of values.
 
 
 
-example: `['Star-Lord']`
+example: `["Star-Lord"]`
 
 | extended
 
@@ -5624,7 +5624,7 @@ Note: this field should contain an array of values.
 
 
 
-example: `['MII...', 'MII...']`
+example: `["MII...", "MII..."]`
 
 | extended
 
@@ -5757,7 +5757,7 @@ Note: this field should contain an array of values.
 
 
 
-example: `['TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', '...']`
+example: `["TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "..."]`
 
 | extended
 
@@ -5838,7 +5838,7 @@ Note: this field should contain an array of values.
 
 
 
-example: `['MII...', 'MII...']`
+example: `["MII...", "MII..."]`
 
 | extended
 

generated/beats/fields.ecs.yml

@@ -1013,9 +1013,7 @@
       description: 'Array of 2 letter DNS header flags.
 
         Expected values are: AA, TC, RD, RA, AD, CD, DO.'
-      example:
-      - RD
-      - RA
+      example: '["RD", "RA"]'
     - name: id
       level: extended
       type: keyword
@@ -1096,9 +1094,7 @@
         formats it can contain. Extracting all IP addresses seen in there to `dns.resolved_ip`
         makes it possible to index them as IP addresses, and makes them easier to
         visualize and query for.'
-      example:
-      - 10.10.10.10
-      - 10.10.10.11
+      example: '["10.10.10.10", "10.10.10.11"]'
     - name: response_code
       level: extended
       type: keyword
@@ -3229,11 +3225,7 @@
         the executable.
 
         May be filtered to protect sensitive information.'
-      example:
-      - /usr/bin/ssh
-      - -l
-      - user
-      - 10.0.0.16
+      example: '["/usr/bin/ssh", "-l", "user", "10.0.0.16"]'
     - name: args_count
       level: extended
       type: long
@@ -3376,11 +3368,7 @@
         the executable.
 
         May be filtered to protect sensitive information.'
-      example:
-      - /usr/bin/ssh
-      - -l
-      - user
-      - 10.0.0.16
+      example: '["/usr/bin/ssh", "-l", "user", "10.0.0.16"]'
       default_field: false
     - name: parent.args_count
       level: extended
@@ -3884,8 +3872,7 @@
       ignore_above: 1024
       description: Name, organization, or pseudonym of the author or authors who created
         the rule used to generate this event.
-      example:
-      - Star-Lord
+      example: '["Star-Lord"]'
       default_field: false
     - name: category
       level: extended
@@ -4652,9 +4639,7 @@
       description: Array of PEM-encoded certificates that make up the certificate
         chain offered by the client. This is usually mutually-exclusive of `client.certificate`
         since that value should be the first certificate in the chain.
-      example:
-      - MII...
-      - MII...
+      example: '["MII...", "MII..."]'
       default_field: false
     - name: client.hash.md5
       level: extended
@@ -4735,10 +4720,8 @@
       type: keyword
       ignore_above: 1024
       description: Array of ciphers offered by the client during the client hello.
-      example:
-      - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
-      - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
-      - '...'
+      example: '["TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+        "..."]'
       default_field: false
     - name: client.x509.alternative_names
       level: extended
@@ -4955,9 +4938,7 @@
       description: Array of PEM-encoded certificates that make up the certificate
         chain offered by the server. This is usually mutually-exclusive of `server.certificate`
         since that value should be the first certificate in the chain.
-      example:
-      - MII...
-      - MII...
+      example: '["MII...", "MII..."]'
       default_field: false
     - name: server.hash.md5
       level: extended

generated/csv/fields.csv

@@ -117,7 +117,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.7.0-dev,true,dns,dns.answers.name,keyword,extended,,www.example.com,The domain name to which this resource record pertains.
 1.7.0-dev,true,dns,dns.answers.ttl,long,extended,,180,The time interval in seconds that this resource record may be cached before it should be discarded.
 1.7.0-dev,true,dns,dns.answers.type,keyword,extended,,CNAME,The type of data contained in this resource record.
-1.7.0-dev,true,dns,dns.header_flags,keyword,extended,array,"['RD', 'RA']",Array of DNS header flags.
+1.7.0-dev,true,dns,dns.header_flags,keyword,extended,array,"[""RD"", ""RA""]",Array of DNS header flags.
 1.7.0-dev,true,dns,dns.id,keyword,extended,,62111,The DNS packet identifier assigned by the program that generated the query. The identifier is copied to the response.
 1.7.0-dev,true,dns,dns.op_code,keyword,extended,,QUERY,The DNS operation code that specifies the kind of query in the message.
 1.7.0-dev,true,dns,dns.question.class,keyword,extended,,IN,The class of records being queried.
@@ -126,7 +126,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.7.0-dev,true,dns,dns.question.subdomain,keyword,extended,,www,The subdomain of the domain.
 1.7.0-dev,true,dns,dns.question.top_level_domain,keyword,extended,,co.uk,"The effective top level domain (com, org, net, co.uk)."
 1.7.0-dev,true,dns,dns.question.type,keyword,extended,,AAAA,The type of record being queried.
-1.7.0-dev,true,dns,dns.resolved_ip,ip,extended,array,"['10.10.10.10', '10.10.10.11']",Array containing all IPs seen in answers.data
+1.7.0-dev,true,dns,dns.resolved_ip,ip,extended,array,"[""10.10.10.10"", ""10.10.10.11""]",Array containing all IPs seen in answers.data
 1.7.0-dev,true,dns,dns.response_code,keyword,extended,,NOERROR,The DNS response code.
 1.7.0-dev,true,dns,dns.type,keyword,extended,,answer,"The type of DNS event captured, query or answer."
 1.7.0-dev,true,ecs,ecs.version,keyword,core,,1.0.0,ECS version this event conforms to.
@@ -362,7 +362,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.7.0-dev,true,package,package.size,long,extended,,62231,Package size in bytes.
 1.7.0-dev,true,package,package.type,keyword,extended,,rpm,Package type
 1.7.0-dev,true,package,package.version,keyword,extended,,1.12.9,Package version
-1.7.0-dev,true,process,process.args,keyword,extended,array,"['/usr/bin/ssh', '-l', 'user', '10.0.0.16']",Array of process arguments.
+1.7.0-dev,true,process,process.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments.
 1.7.0-dev,true,process,process.args_count,long,extended,,4,Length of the process.args array.
 1.7.0-dev,true,process,process.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present.
 1.7.0-dev,true,process,process.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status.
@@ -381,7 +381,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.7.0-dev,true,process,process.hash.sha512,keyword,extended,,,SHA512 hash.
 1.7.0-dev,true,process,process.name,keyword,extended,,ssh,Process name.
 1.7.0-dev,true,process,process.name.text,text,extended,,ssh,Process name.
-1.7.0-dev,true,process,process.parent.args,keyword,extended,array,"['/usr/bin/ssh', '-l', 'user', '10.0.0.16']",Array of process arguments.
+1.7.0-dev,true,process,process.parent.args,keyword,extended,array,"[""/usr/bin/ssh"", ""-l"", ""user"", ""10.0.0.16""]",Array of process arguments.
 1.7.0-dev,true,process,process.parent.args_count,long,extended,,4,Length of the process.args array.
 1.7.0-dev,true,process,process.parent.code_signature.exists,boolean,core,,true,Boolean to capture if a signature is present.
 1.7.0-dev,true,process,process.parent.code_signature.status,keyword,extended,,ERROR_UNTRUSTED_ROOT,Additional information about the certificate status.
@@ -447,7 +447,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.7.0-dev,true,related,related.hosts,keyword,extended,array,,All the host identifiers seen on your event.
 1.7.0-dev,true,related,related.ip,ip,extended,array,,All of the IPs seen on your event.
 1.7.0-dev,true,related,related.user,keyword,extended,array,,All the user names seen on your event.
-1.7.0-dev,true,rule,rule.author,keyword,extended,array,['Star-Lord'],Rule author
+1.7.0-dev,true,rule,rule.author,keyword,extended,array,"[""Star-Lord""]",Rule author
 1.7.0-dev,true,rule,rule.category,keyword,extended,,Attempted Information Leak,Rule category
 1.7.0-dev,true,rule,rule.description,keyword,extended,,Block requests to public DNS over HTTPS / TLS protocols,Rule description
 1.7.0-dev,true,rule,rule.id,keyword,extended,,101,Rule ID
@@ -547,7 +547,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.7.0-dev,true,threat,threat.technique.subtechnique.reference,keyword,extended,array,https://attack.mitre.org/techniques/T1059/001/,Threat subtechnique URL reference.
 1.7.0-dev,true,tls,tls.cipher,keyword,extended,,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,String indicating the cipher used during the current connection.
 1.7.0-dev,true,tls,tls.client.certificate,keyword,extended,,MII...,PEM-encoded stand-alone certificate offered by the client.
-1.7.0-dev,true,tls,tls.client.certificate_chain,keyword,extended,array,"['MII...', 'MII...']",Array of PEM-encoded certificates that make up the certificate chain offered by the client.
+1.7.0-dev,true,tls,tls.client.certificate_chain,keyword,extended,array,"[""MII..."", ""MII...""]",Array of PEM-encoded certificates that make up the certificate chain offered by the client.
 1.7.0-dev,true,tls,tls.client.hash.md5,keyword,extended,,0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC,Certificate fingerprint using the MD5 digest of DER-encoded version of certificate offered by the client.
 1.7.0-dev,true,tls,tls.client.hash.sha1,keyword,extended,,9E393D93138888D288266C2D915214D1D1CCEB2A,Certificate fingerprint using the SHA1 digest of DER-encoded version of certificate offered by the client.
 1.7.0-dev,true,tls,tls.client.hash.sha256,keyword,extended,,0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0,Certificate fingerprint using the SHA256 digest of DER-encoded version of certificate offered by the client.
@@ -557,7 +557,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.7.0-dev,true,tls,tls.client.not_before,date,extended,,1970-01-01T00:00:00.000Z,Date/Time indicating when client certificate is first considered valid.
 1.7.0-dev,true,tls,tls.client.server_name,keyword,extended,,www.elastic.co,Hostname the client is trying to connect to. Also called the SNI.
 1.7.0-dev,true,tls,tls.client.subject,keyword,extended,,"CN=myclient, OU=Documentation Team, DC=example, DC=com",Distinguished name of subject of the x.509 certificate presented by the client.
-1.7.0-dev,true,tls,tls.client.supported_ciphers,keyword,extended,array,"['TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', '...']",Array of ciphers offered by the client during the client hello.
+1.7.0-dev,true,tls,tls.client.supported_ciphers,keyword,extended,array,"[""TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"", ""TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"", ""...""]",Array of ciphers offered by the client during the client hello.
 1.7.0-dev,true,tls,tls.client.x509.alternative_names,keyword,extended,array,*.elastic.co,List of subject alternative names (SAN).
 1.7.0-dev,true,tls,tls.client.x509.issuer.common_name,keyword,extended,array,Example SHA2 High Assurance Server CA,List of common name (CN) of issuing certificate authority.
 1.7.0-dev,true,tls,tls.client.x509.issuer.country,keyword,extended,array,US,List of country (C) codes
@@ -587,7 +587,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 1.7.0-dev,true,tls,tls.next_protocol,keyword,extended,,http/1.1,String indicating the protocol being tunneled.
 1.7.0-dev,true,tls,tls.resumed,boolean,extended,,,Boolean flag indicating if this TLS connection was resumed from an existing TLS negotiation.
 1.7.0-dev,true,tls,tls.server.certificate,keyword,extended,,MII...,PEM-encoded stand-alone certificate offered by the server.
-1.7.0-dev,true,tls,tls.server.certificate_chain,keyword,extended,array,"['MII...', 'MII...']",Array of PEM-encoded certificates that make up the certificate chain offered by the server.
+1.7.0-dev,true,tls,tls.server.certificate_chain,keyword,extended,array,"[""MII..."", ""MII...""]",Array of PEM-encoded certificates that make up the certificate chain offered by the server.
 1.7.0-dev,true,tls,tls.server.hash.md5,keyword,extended,,0F76C7F2C55BFD7D8E8B8F4BFBF0C9EC,Certificate fingerprint using the MD5 digest of DER-encoded version of certificate offered by the server.
 1.7.0-dev,true,tls,tls.server.hash.sha1,keyword,extended,,9E393D93138888D288266C2D915214D1D1CCEB2A,Certificate fingerprint using the SHA1 digest of DER-encoded version of certificate offered by the server.
 1.7.0-dev,true,tls,tls.server.hash.sha256,keyword,extended,,0687F666A054EF17A08E2F2162EAB4CBC0D265E1D7875BE74BF3C712CA92DAF0,Certificate fingerprint using the SHA256 digest of DER-encoded version of certificate offered by the server.

generated/ecs/ecs_flat.yml

@@ -1384,9 +1384,7 @@ dns.header_flags:
   description: 'Array of 2 letter DNS header flags.
 
     Expected values are: AA, TC, RD, RA, AD, CD, DO.'
-  example:
-  - RD
-  - RA
+  example: '["RD", "RA"]'
   flat_name: dns.header_flags
   ignore_above: 1024
   level: extended
@@ -1514,9 +1512,7 @@ dns.resolved_ip:
     it can contain. Extracting all IP addresses seen in there to `dns.resolved_ip`
     makes it possible to index them as IP addresses, and makes them easier to visualize
     and query for.'
-  example:
-  - 10.10.10.10
-  - 10.10.10.11
+  example: '["10.10.10.10", "10.10.10.11"]'
   flat_name: dns.resolved_ip
   level: extended
   name: resolved_ip
@@ -4777,11 +4773,7 @@ process.args:
     executable.
 
     May be filtered to protect sensitive information.'
-  example:
-  - /usr/bin/ssh
-  - -l
-  - user
-  - 10.0.0.16
+  example: '["/usr/bin/ssh", "-l", "user", "10.0.0.16"]'
   flat_name: process.args
   ignore_above: 1024
   level: extended
@@ -5007,11 +4999,7 @@ process.parent.args:
     executable.
 
     May be filtered to protect sensitive information.'
-  example:
-  - /usr/bin/ssh
-  - -l
-  - user
-  - 10.0.0.16
+  example: '["/usr/bin/ssh", "-l", "user", "10.0.0.16"]'
   flat_name: process.parent.args
   ignore_above: 1024
   level: extended
@@ -5778,8 +5766,7 @@ rule.author:
   dashed_name: rule-author
   description: Name, organization, or pseudonym of the author or authors who created
     the rule used to generate this event.
-  example:
-  - Star-Lord
+  example: '["Star-Lord"]'
   flat_name: rule.author
   ignore_above: 1024
   level: extended
@@ -6998,9 +6985,7 @@ tls.client.certificate_chain:
   description: Array of PEM-encoded certificates that make up the certificate chain
     offered by the client. This is usually mutually-exclusive of `client.certificate`
     since that value should be the first certificate in the chain.
-  example:
-  - MII...
-  - MII...
+  example: '["MII...", "MII..."]'
   flat_name: tls.client.certificate_chain
   ignore_above: 1024
   level: extended
@@ -7126,10 +7111,8 @@ tls.client.subject:
 tls.client.supported_ciphers:
   dashed_name: tls-client-supported-ciphers
   description: Array of ciphers offered by the client during the client hello.
-  example:
-  - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
-  - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
-  - '...'
+  example: '["TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+    "..."]'
   flat_name: tls.client.supported_ciphers
   ignore_above: 1024
   level: extended
@@ -7508,9 +7491,7 @@ tls.server.certificate_chain:
   description: Array of PEM-encoded certificates that make up the certificate chain
     offered by the server. This is usually mutually-exclusive of `server.certificate`
     since that value should be the first certificate in the chain.
-  example:
-  - MII...
-  - MII...
+  example: '["MII...", "MII..."]'
   flat_name: tls.server.certificate_chain
   ignore_above: 1024
   level: extended