Migrate obs-infraobs-integrations to package-spec v3 #1

packages/activemq/_dev/deploy/variants.yml

@@ -1,4 +1,4 @@
 variants:
-  v5.17.1:
+  "v5.17.1":
     SERVICE_VERSION: 5.17.1
 default: v5.17.1

packages/activemq/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 0.14.0
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8170
 - version: "0.13.1"
   changes:
     - description: Remove forwarded tag from metrics data streams.

packages/activemq/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json

@@ -11,7 +11,7 @@
                 "version": "8.5.1"
             },
             "event": {
-                "ingested": "2022-12-08T15:06:10.692324051Z",
+                "ingested": "2023-10-11T20:53:28.142058918Z",
                 "kind": "event",
                 "module": "activemq",
                 "original": "INFO  | anonymous called org.apache.activemq.broker.jmx.QueueView.retryMessages[] at 27-11-2019 08:45:57,213 | qtp443290224-47",
@@ -41,7 +41,7 @@
                 "version": "8.5.1"
             },
             "event": {
-                "ingested": "2022-12-08T15:06:10.692336343Z",
+                "ingested": "2023-10-11T20:53:28.142069543Z",
                 "kind": "event",
                 "module": "activemq",
                 "original": "INFO  | admin called org.apache.activemq.broker.jmx.QueueView.retryMessages[] at 27-11-2019 08:45:57,229 | qtp443290224-45",
@@ -71,7 +71,7 @@
                 "version": "8.5.1"
             },
             "event": {
-                "ingested": "2022-12-08T15:06:10.692337760Z",
+                "ingested": "2023-10-11T20:53:28.142070835Z",
                 "kind": "event",
                 "module": "activemq",
                 "original": "WARN  | admin requested /admin/createDestination.action [JMSDestination='test' JMSDestinationType='queue' secret='4eb0bc3e-9d7a-4256-844c-24f40fda98f1' ] from 127.0.0.1 | qtp12205619-39",
@@ -101,7 +101,7 @@
                 "version": "8.5.1"
             },
             "event": {
-                "ingested": "2022-12-08T15:06:10.692338926Z",
+                "ingested": "2023-10-11T20:53:28.142071877Z",
                 "kind": "event",
                 "module": "activemq",
                 "original": "INFO  | guest requested /admin/purgeDestination.action [JMSDestination='test' JMSDestinationType='queue' secret='eff6a932-1b58-45da-a64a-1b30b246cfc9' ] from 127.0.0.1 | qtp12205619-36",

packages/activemq/data_stream/audit/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,5 @@
 dynamic_fields:
-  event.ingested: ".*"
+  "event.ingested": ".*"
 fields:
   "@timestamp": "2020-04-28T11:07:58.223Z"
   tags:

packages/activemq/data_stream/log/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,5 @@
 dynamic_fields:
-  event.ingested: ".*"
+  "event.ingested": ".*"
 fields:
   "@timestamp": "2020-04-28T11:07:58.223Z"
   tags:

packages/activemq/data_stream/topic/fields/ecs.yml

@@ -23,8 +23,6 @@
   name: service.type
 - external: ecs
   name: tags
-- external: ecs
-  name: tags
 - external: ecs
   name: agent.id
   dimension: true

packages/activemq/manifest.yml

@@ -1,20 +1,22 @@
 name: activemq
 title: ActiveMQ
-version: "0.13.1"
+version: "0.14.0"
 description: Collect logs and metrics from ActiveMQ instances with Elastic Agent.
 type: integration
 icons:
   - src: /img/activemq.svg
     title: activemq
     size: 32x32
     type: image/svg+xml
-format_version: 1.0.0
-license: basic
+format_version: "3.0.0"
 categories:
   - message_queue
   - observability
 conditions:
-  kibana.version: ^8.8.0
+  kibana:
+    version: ^8.8.0
+  elastic:
+    subscription: basic
 screenshots:
   - src: /img/filebeat-activemq-log.png
     title: filebeat ActiveMQ application events
@@ -116,3 +118,4 @@ policy_templates:
         description: Collecting broker, queue and topic metrics from ActiveMQ instances.
 owner:
   github: elastic/obs-infraobs-integrations
+  type: elastic

packages/activemq/validation.yml

@@ -0,0 +1,4 @@
+errors:
+  exclude_checks:
+    - SVR00004
+    - SVR00002

packages/airflow/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 0.5.0
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8170
 - version: "0.4.0"
   changes:
     - description: Enable time series data streams for the metrics datasets. This dramatically reduces storage for metrics and is expected to progressively improve query performance. For more details, see https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds.html.

packages/airflow/data_stream/statsd/fields/agent.yml

@@ -57,33 +57,6 @@
     - name: image.id
       type: keyword
       description: Image ID for the cloud instance.
-- name: container
-  title: Container
-  group: 2
-  description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime."
-  type: group
-  fields:
-    - name: id
-      level: core
-      type: keyword
-      ignore_above: 1024
-      description: Unique container id.
-      dimension: true
-    - name: image.name
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      description: Name of the image the container was built on.
-    - name: labels
-      level: extended
-      type: object
-      object_type: keyword
-      description: Image labels.
-    - name: name
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      description: Container name.
 - name: host
   title: Host
   group: 2

packages/airflow/data_stream/statsd/fields/ecs.yml

@@ -14,5 +14,7 @@
   external: ecs
 - name: container.image.name
   external: ecs
+- name: container.labels
+  external: ecs
 - name: host
   external: ecs

packages/airflow/manifest.yml

@@ -1,14 +1,16 @@
 name: airflow
 title: Airflow
-version: "0.4.0"
+version: "0.5.0"
 description: Airflow Integration.
 type: integration
-format_version: 1.0.0
-license: basic
+format_version: "3.0.0"
 categories:
   - observability
 conditions:
-  kibana.version: "^8.9.0"
+  kibana:
+    version: "^8.9.0"
+  elastic:
+    subscription: basic
 icons:
   - src: /img/airflow.svg
     title: Airflow logo
@@ -42,3 +44,4 @@ policy_templates:
         input_group: metrics
 owner:
   github: elastic/obs-infraobs-integrations
+  type: elastic

packages/airflow/validation.yml

@@ -0,0 +1,3 @@
+errors:
+  exclude_checks:
+    - SVR00002

packages/apache_spark/_dev/deploy/variants.yml

@@ -1,4 +1,4 @@
 variants:
-  v3.2.0:
+  "v3.2.0":
     SERVICE_VERSION: 3.2.0
 default: v3.2.0

packages/apache_spark/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.8.0"
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8170
 - version: "0.7.9"
   changes:
     - description: Add filters in visualizations.

packages/apache_spark/data_stream/application/elasticsearch/ingest_pipeline/default.yml

@@ -10,7 +10,7 @@ processors:
       ignore_missing: true
   - set:
       field: event.type
-      value: info
+      value: [info]
   - set:
       field: event.kind
       value: metric

packages/apache_spark/data_stream/application/sample_event.json

@@ -34,14 +34,18 @@
         "ingested": "2023-09-28T09:24:37Z",
         "kind": "metric",
         "module": "apache_spark",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "host": {
         "architecture": "x86_64",
         "containerized": true,
         "hostname": "docker-fleet-agent",
         "id": "e8978f2086c14e13b7a0af9ed0011d19",
-        "ip": "172.20.0.7",
+        "ip": [
+            "172.20.0.7"
+        ],
         "mac": "02-42-AC-14-00-07",
         "name": "docker-fleet-agent",
         "os": {

packages/apache_spark/data_stream/driver/elasticsearch/ingest_pipeline/default.yml

@@ -10,7 +10,7 @@ processors:
       ignore_missing: true
   - set:
       field: event.type
-      value: info
+      value: [info]
   - set:
       field: event.kind
       value: metric

packages/apache_spark/data_stream/driver/sample_event.json

@@ -38,14 +38,18 @@
         "ingested": "2023-09-29T12:04:41Z",
         "kind": "metric",
         "module": "apache_spark",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "host": {
         "architecture": "x86_64",
         "containerized": true,
         "hostname": "docker-fleet-agent",
         "id": "e8978f2086c14e13b7a0af9ed0011d19",
-        "ip": "172.26.0.7",
+        "ip": [
+            "172.26.0.7"
+        ],
         "mac": "02-42-AC-1A-00-07",
         "name": "docker-fleet-agent",
         "os": {

packages/apache_spark/data_stream/executor/elasticsearch/ingest_pipeline/default.yml

@@ -10,7 +10,7 @@ processors:
       ignore_missing: true
   - set:
       field: event.type
-      value: info
+      value: [info]
   - set:
       field: event.kind
       value: metric

packages/apache_spark/data_stream/executor/sample_event.json

@@ -37,14 +37,18 @@
         "ingested": "2023-09-28T09:26:49Z",
         "kind": "metric",
         "module": "apache_spark",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "host": {
         "architecture": "x86_64",
         "containerized": true,
         "hostname": "docker-fleet-agent",
         "id": "e8978f2086c14e13b7a0af9ed0011d19",
-        "ip": "172.20.0.7",
+        "ip": [
+            "172.20.0.7"
+        ],
         "mac": "02-42-AC-14-00-07",
         "name": "docker-fleet-agent",
         "os": {

packages/apache_spark/data_stream/node/elasticsearch/ingest_pipeline/default.yml

@@ -10,7 +10,7 @@ processors:
       ignore_missing: true
   - set:
       field: event.type
-      value: info
+      value: [info]
   - set:
       field: event.kind
       value: metric

packages/apache_spark/data_stream/node/sample_event.json

@@ -41,7 +41,9 @@
         "ingested": "2022-04-12T04:42:53Z",
         "kind": "metric",
         "module": "apache_spark",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "host": {
         "architecture": "x86_64",

packages/apache_spark/docs/README.md

@@ -112,14 +112,18 @@ An example event for `application` looks as following:
         "ingested": "2023-09-28T09:24:37Z",
         "kind": "metric",
         "module": "apache_spark",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "host": {
         "architecture": "x86_64",
         "containerized": true,
         "hostname": "docker-fleet-agent",
         "id": "e8978f2086c14e13b7a0af9ed0011d19",
-        "ip": "172.20.0.7",
+        "ip": [
+            "172.20.0.7"
+        ],
         "mac": "02-42-AC-14-00-07",
         "name": "docker-fleet-agent",
         "os": {
@@ -223,14 +227,18 @@ An example event for `driver` looks as following:
         "ingested": "2023-09-29T12:04:41Z",
         "kind": "metric",
         "module": "apache_spark",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "host": {
         "architecture": "x86_64",
         "containerized": true,
         "hostname": "docker-fleet-agent",
         "id": "e8978f2086c14e13b7a0af9ed0011d19",
-        "ip": "172.26.0.7",
+        "ip": [
+            "172.26.0.7"
+        ],
         "mac": "02-42-AC-1A-00-07",
         "name": "docker-fleet-agent",
         "os": {
@@ -400,14 +408,18 @@ An example event for `executor` looks as following:
         "ingested": "2023-09-28T09:26:49Z",
         "kind": "metric",
         "module": "apache_spark",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "host": {
         "architecture": "x86_64",
         "containerized": true,
         "hostname": "docker-fleet-agent",
         "id": "e8978f2086c14e13b7a0af9ed0011d19",
-        "ip": "172.20.0.7",
+        "ip": [
+            "172.20.0.7"
+        ],
         "mac": "02-42-AC-14-00-07",
         "name": "docker-fleet-agent",
         "os": {
@@ -579,7 +591,9 @@ An example event for `node` looks as following:
         "ingested": "2022-04-12T04:42:53Z",
         "kind": "metric",
         "module": "apache_spark",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "host": {
         "architecture": "x86_64",