Remove duplicated TLS fields from Packetbeat 7.17

- Remove tls.detailed.{client,server}_certificate. - Update dashboards to use tls.{client,server}.x509.* instead of tls.detailed.{client,server}_certificate.* - Remove mappings for 'province'. Those fields was renamed in Packetbeat 7.9 to state_or_province (per ECS) (elastic/beats#19167).
andrewkroh · Apr 13, 2022 · 08a52c9 · 08a52c9
1 parent 260c367
commit 08a52c9
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 222 deletions.
diff --git a/packages/network_traffic/data_stream/tls/elasticsearch/ingest_pipeline/default.yml b/packages/network_traffic/data_stream/tls/elasticsearch/ingest_pipeline/default.yml
@@ -34,13 +34,20 @@ processors:
     type: string
     ignore_missing: true
 
-- convert:
-    field: tls.detailed.client_certificate.version_number
-    type: string
-    ignore_missing: true
-- convert:
-    field: tls.detailed.server_certificate.version_number
-    type: string
+##
+# This handles legacy TLS fields from Packetbeat 7.17.
+##
+- remove:
+    description: Remove legacy fields from Packetbeat 7.17 that are duplicated.
+    field:
+      - tls.client.x509.issuer.province  # Duplicated as tls.client.x509.issuer.state_or_province.
+      - tls.client.x509.subject.province # Duplicated as tls.client.x509.subject.state_or_province.
+      - tls.client.x509.version          # Duplicated as tls.client.x509.version_number.
+      - tls.detailed.client_certificate  # Duplicated as tls.client.x509.
+      - tls.detailed.server_certificate  # Duplicated as tls.server.x509.
+      - tls.server.x509.issuer.province  # Duplicated as tls.server.x509.issuer.state_or_province.
+      - tls.server.x509.subject.province # Duplicated as tls.server.x509.subject.state_or_province.
+      - tls.server.x509.version          # Duplicated as tls.server.x509.version_number.
     ignore_missing: true
 on_failure:
 - set:

diff --git a/packages/network_traffic/data_stream/tls/fields/protocol.yml b/packages/network_traffic/data_stream/tls/fields/protocol.yml
@@ -1,32 +1,6 @@
 - name: tls
   type: group
   fields:
-    # get rid of this when we upgrade to ECS 1.6
-    - name: client
-      type: group
-      fields:
-        - name: x509
-          type: group
-          fields:
-            - name: issuer.province
-              type: keyword
-              description: Province or region within country.
-            - name: subject.province
-              type: keyword
-              description: Province or region within country.
-    # get rid of this when we upgrade to ECS 1.6
-    - name: server
-      type: group
-      fields:
-        - name: x509
-          type: group
-          fields:
-            - name: issuer.province
-              type: keyword
-              description: Province or region within country.
-            - name: subject.province
-              type: keyword
-              description: Province or region within country.
     - name: detailed
       type: group
       fields:
@@ -186,188 +160,6 @@
                   description: >
                     List of extensions that were left unparsed by Packetbeat.
 
-        - name: client_certificate
-          type: group
-          description: Certificate provided by the client for authentication.
-          fields:
-            - name: version
-              type: long
-              description: X509 format version.
-            - name: version_number
-              type: keyword
-              description: Version of x509 format.
-              example: 3
-            - name: serial_number
-              type: keyword
-              description: The certificate's serial number.
-            - name: not_before
-              type: date
-              description: Date before which the certificate is not valid.
-            - name: not_after
-              type: date
-              description: Date after which the certificate expires.
-            - name: public_key_algorithm
-              type: keyword
-              description: >
-                The algorithm used for this certificate's public key. One of RSA, DSA or ECDSA.
-
-            - name: public_key_size
-              type: long
-              description: Size of the public key.
-            - name: signature_algorithm
-              type: keyword
-              description: >
-                The algorithm used for the certificate's signature.
-
-            - name: alternative_names
-              type: keyword
-              description: Subject Alternative Names for this certificate.
-            - name: subject
-              type: group
-              description: Subject represented by this certificate.
-              fields:
-                - name: country
-                  type: keyword
-                  description: Country code.
-                - name: organization
-                  type: keyword
-                  description: Organization name.
-                - name: organizational_unit
-                  type: keyword
-                  description: Unit within organization.
-                - name: province
-                  type: keyword
-                  description: Province or region within country.
-                - name: common_name
-                  type: keyword
-                  description: Name or host name identified by the certificate.
-                - name: locality
-                  type: keyword
-                  description: Locality.
-                - name: distinguished_name
-                  type: keyword
-                  description: Distinguished name (DN) of the certificate subject entity.
-                  example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net
-            - name: issuer
-              type: group
-              description: Entity that issued and signed this certificate.
-              fields:
-                - name: country
-                  type: keyword
-                  description: Country code.
-                - name: organization
-                  type: keyword
-                  description: Organization name.
-                - name: organizational_unit
-                  type: keyword
-                  description: Unit within organization.
-                - name: province
-                  type: keyword
-                  description: Province or region within country.
-                - name: common_name
-                  type: keyword
-                  description: Name or host name identified by the certificate.
-                - name: locality
-                  type: keyword
-                  description: Locality.
-                - name: distinguished_name
-                  type: keyword
-                  description: Distinguished name (DN) of the certificate issuer entity.
-                  example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net
-        - name: server_certificate
-          type: group
-          description: Certificate provided by the server for authentication.
-          fields:
-            - name: version
-              type: long
-              description: X509 format version.
-            - name: version_number
-              type: keyword
-              description: Version of x509 format.
-              example: 3
-            - name: serial_number
-              type: keyword
-              description: The certificate's serial number.
-            - name: not_before
-              type: date
-              description: Date before which the certificate is not valid.
-            - name: not_after
-              type: date
-              description: Date after which the certificate expires.
-            - name: public_key_algorithm
-              type: keyword
-              description: >
-                The algorithm used for this certificate's public key. One of RSA, DSA or ECDSA.
-
-            - name: public_key_size
-              type: long
-              description: Size of the public key.
-            - name: signature_algorithm
-              type: keyword
-              description: >
-                The algorithm used for the certificate's signature.
-
-            - name: alternative_names
-              type: keyword
-              description: Subject Alternative Names for this certificate.
-            - name: subject
-              type: group
-              description: Subject represented by this certificate.
-              fields:
-                - name: country
-                  type: keyword
-                  description: Country code.
-                - name: organization
-                  type: keyword
-                  description: Organization name.
-                - name: organizational_unit
-                  type: keyword
-                  description: Unit within organization.
-                - name: province
-                  type: keyword
-                  description: Province or region within country.
-                - name: state_or_province
-                  type: keyword
-                  description: Province or region within country.
-                - name: common_name
-                  type: keyword
-                  description: Name or host name identified by the certificate.
-                - name: locality
-                  type: keyword
-                  description: Locality.
-                - name: distinguished_name
-                  type: keyword
-                  description: Distinguished name (DN) of the certificate subject entity.
-                  example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net
-            - name: issuer
-              type: group
-              description: Entity that issued and signed this certificate.
-              fields:
-                - name: country
-                  type: keyword
-                  description: Country code.
-                - name: organization
-                  type: keyword
-                  description: Organization name.
-                - name: organizational_unit
-                  type: keyword
-                  description: Unit within organization.
-                - name: province
-                  type: keyword
-                  description: Province or region within country.
-                - name: state_or_province
-                  type: keyword
-                  description: Province or region within country.
-                - name: common_name
-                  type: keyword
-                  description: Name or host name identified by the certificate.
-                - name: locality
-                  type: keyword
-                  description: Locality.
-                - name: distinguished_name
-                  type: keyword
-                  description: Distinguished name (DN) of the certificate issuer entity.
-                  example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net
         - name: server_certificate_chain
           type: array
           description: Chain of trust for the server certificate.

diff --git a/...s/network_traffic/kibana/search/network_traffic-8f0ff590-d37d-11e7-9914-4982455b3063.json b/...s/network_traffic/kibana/search/network_traffic-8f0ff590-d37d-11e7-9914-4982455b3063.json
@@ -13,13 +13,13 @@
                             "store": "appState"
                         },
                         "exists": {
-                            "field": "tls.detailed.server_certificate.public_key_size"
+                            "field": "tls.server.x509.public_key_size"
                         },
                         "meta": {
                             "alias": null,
                             "disabled": false,
                             "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
-                            "key": "tls.detailed.server_certificate.public_key_size",
+                            "key": "tls.server.x509.public_key_size",
                             "negate": false,
                             "type": "exists",
                             "value": "exists"

diff --git a/...rk_traffic/kibana/visualization/network_traffic-0958a910-d396-11e7-8fa0-232aa9259081.json b/...rk_traffic/kibana/visualization/network_traffic-0958a910-d396-11e7-8fa0-232aa9259081.json
@@ -39,7 +39,7 @@
                     "id": "4",
                     "params": {
                         "customLabel": "Subject Common Name",
-                        "field": "tls.detailed.server_certificate.subject.common_name",
+                        "field": "tls.server.x509.subject.common_name",
                         "order": "desc",
                         "orderBy": "1",
                         "size": 10
@@ -52,7 +52,7 @@
                     "id": "3",
                     "params": {
                         "customLabel": "Organization",
-                        "field": "tls.detailed.server_certificate.subject.organization",
+                        "field": "tls.server.x509.subject.organization",
                         "order": "desc",
                         "orderBy": "1",
                         "size": 5

diff --git a/...rk_traffic/kibana/visualization/network_traffic-86743f90-d396-11e7-8fa0-232aa9259081.json b/...rk_traffic/kibana/visualization/network_traffic-86743f90-d396-11e7-8fa0-232aa9259081.json
@@ -37,7 +37,7 @@
                     "id": "2",
                     "params": {
                         "customLabel": "Subject Common Name",
-                        "field": "tls.detailed.client_certificate.subject.common_name",
+                        "field": "tls.client.x509.subject.common_name",
                         "order": "desc",
                         "orderBy": "1",
                         "size": 5
@@ -50,7 +50,7 @@
                     "id": "3",
                     "params": {
                         "customLabel": "Signature Algorithm",
-                        "field": "tls.detailed.client_certificate.signature_algorithm",
+                        "field": "tls.client.x509.signature_algorithm",
                         "json": "{ \"missing\": \"N/A\" }",
                         "order": "desc",
                         "orderBy": "1",

diff --git a/...rk_traffic/kibana/visualization/network_traffic-ae6e33c0-d37d-11e7-9914-4982455b3063.json b/...rk_traffic/kibana/visualization/network_traffic-ae6e33c0-d37d-11e7-9914-4982455b3063.json
@@ -28,7 +28,7 @@
                     "id": "2",
                     "params": {
                         "customLabel": "Public Key Size",
-                        "field": "tls.detailed.server_certificate.public_key_size",
+                        "field": "tls.server.x509.public_key_size",
                         "order": "desc",
                         "orderBy": "1",
                         "size": 5