Skip to content

Add support for Array Processor in Logs Pipelines specs #30168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .apigentools-info
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
"spec_versions": {
"v1": {
"apigentools_version": "1.6.6",
"regenerated": "2025-06-26 17:58:30.290427",
"spec_repo_commit": "76086f13"
"regenerated": "2025-06-27 13:28:17.946946",
"spec_repo_commit": "a435f1bd"
},
"v2": {
"apigentools_version": "1.6.6",
"regenerated": "2025-06-26 17:58:39.398706",
"spec_repo_commit": "76086f13"
"regenerated": "2025-06-27 13:28:27.141603",
"spec_repo_commit": "a435f1bd"
}
}
}
12 changes: 6 additions & 6 deletions content/en/api/v1/logs-pipelines/examples.json

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"filter": {
"query": "source:python"
},
"name": "testPipelineArrayAppend",
"processors": [
{
"type": "array-processor",
"is_enabled": true,
"name": "append_ip_to_array",
"operation": {
"type": "append",
"source": "network.client.ip",
"target": "sourceIps"
}
}
],
"tags": []
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"filter": {
"query": "source:python"
},
"name": "testPipelineArraySelect",
"processors": [
{
"type": "array-processor",
"is_enabled": true,
"name": "extract_referrer",
"operation": {
"type": "select",
"source": "httpRequest.headers",
"target": "referrer",
"filter": "name:Referrer",
"value_to_extract": "value"
}
}
],
"tags": []
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"filter": {
"query": "source:python"
},
"name": "testPipelineArrayLength",
"processors": [
{
"type": "array-processor",
"is_enabled": true,
"name": "count_tags",
"operation": {
"type": "length",
"source": "tags",
"target": "tagCount"
}
}
],
"tags": []
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"filter": {
"query": "source:python"
},
"name": "testPipelineArrayAppendPreserve",
"processors": [
{
"type": "array-processor",
"is_enabled": true,
"name": "append_ip_and_keep_source",
"operation": {
"type": "append",
"source": "network.client.ip",
"target": "sourceIps",
"preserve_source": true
}
}
],
"tags": []
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"filter": {
"query": "source:python"
},
"name": "testPipelineArrayAppendNoPreserve",
"processors": [
{
"type": "array-processor",
"is_enabled": true,
"name": "append_ip_and_remove_source",
"operation": {
"type": "append",
"source": "network.client.ip",
"target": "sourceIps",
"preserve_source": false
}
}
],
"tags": []
}
25 changes: 25 additions & 0 deletions data/api/v1/CodeExamples.json
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,31 @@
}
],
"CreateLogsPipeline": [
{
"group": "logs_pipelines",
"suffix": "_1248402480",
"description": "Create a pipeline with Array Processor Append Operation returns \"OK\" response"
},
{
"group": "logs_pipelines",
"suffix": "_3934594739",
"description": "Create a pipeline with Array Processor Append Operation with preserve_source false returns \"OK\" response"
},
{
"group": "logs_pipelines",
"suffix": "_3314493032",
"description": "Create a pipeline with Array Processor Append Operation with preserve_source true returns \"OK\" response"
},
{
"group": "logs_pipelines",
"suffix": "_1271012410",
"description": "Create a pipeline with Array Processor Length Operation returns \"OK\" response"
},
{
"group": "logs_pipelines",
"suffix": "_1267211320",
"description": "Create a pipeline with Array Processor Select Operation returns \"OK\" response"
},
{
"group": "logs_pipelines",
"suffix": "_2707101123",
Expand Down
138 changes: 138 additions & 0 deletions data/api/v1/full_spec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5263,6 +5263,143 @@ components:
type: string
x-enum-varnames:
- ARITHMETIC_PROCESSOR
LogsArrayProcessor:
description: 'A processor for extracting, aggregating, or transforming values
from JSON arrays within your logs.

Supported operations are:

- Select value from matching element

- Compute array length

- Append a value to an array'
properties:
is_enabled:
default: false
description: Whether or not the processor is enabled.
type: boolean
name:
description: Name of the processor.
type: string
operation:
$ref: '#/components/schemas/LogsArrayProcessorOperation'
type:
$ref: '#/components/schemas/LogsArrayProcessorType'
required:
- operation
- type
type: object
LogsArrayProcessorOperation:
description: Configuration of the array processor operation to perform.
oneOf:
- $ref: '#/components/schemas/LogsArrayProcessorOperationAppend'
- $ref: '#/components/schemas/LogsArrayProcessorOperationLength'
- $ref: '#/components/schemas/LogsArrayProcessorOperationSelect'
LogsArrayProcessorOperationAppend:
description: Operation that appends a value to a target array attribute.
properties:
preserve_source:
default: true
description: Remove or preserve the remapped source element.
type: boolean
source:
description: Attribute path containing the value to append.
example: network.client.ip
type: string
target:
description: Attribute path of the array to append to.
example: sourceIps
type: string
type:
$ref: '#/components/schemas/LogsArrayProcessorOperationAppendType'
required:
- type
- source
- target
type: object
LogsArrayProcessorOperationAppendType:
description: Operation type.
enum:
- append
example: append
type: string
x-enum-varnames:
- APPEND
LogsArrayProcessorOperationLength:
description: Operation that computes the length of a `source` array and stores
the result in the `target` attribute.
properties:
source:
description: Attribute path of the array to measure.
example: tags
type: string
target:
description: Attribute that receives the computed length.
example: tagCount
type: string
type:
$ref: '#/components/schemas/LogsArrayProcessorOperationLengthType'
required:
- type
- source
- target
type: object
LogsArrayProcessorOperationLengthType:
description: Operation type.
enum:
- length
example: length
type: string
x-enum-varnames:
- LENGTH
LogsArrayProcessorOperationSelect:
description: Operation that finds an object in a `source` array using a `filter`,
and then extracts a specific value into the `target` attribute.
properties:
filter:
description: Filter condition expressed as `key:value` used to find the
matching element.
example: name:Referrer
type: string
source:
description: Attribute path of the array to search into.
example: httpRequest.headers
type: string
target:
description: Attribute that receives the extracted value.
example: referrer
type: string
type:
$ref: '#/components/schemas/LogsArrayProcessorOperationSelectType'
value_to_extract:
description: Key of the value to extract from the matching element.
example: value
type: string
required:
- type
- source
- target
- filter
- value_to_extract
type: object
LogsArrayProcessorOperationSelectType:
description: Operation type.
enum:
- select
example: select
type: string
x-enum-varnames:
- SELECT
LogsArrayProcessorType:
default: array-processor
description: Type of logs array processor.
enum:
- array-processor
example: array-processor
type: string
x-enum-varnames:
- ARRAY_PROCESSOR
LogsAttributeRemapper:
description: 'The remapper processor remaps any source attribute(s) or tag to
another target attribute or tag.
Expand Down Expand Up @@ -6160,6 +6297,7 @@ components:
- $ref: '#/components/schemas/ReferenceTableLogsLookupProcessor'
- $ref: '#/components/schemas/LogsTraceRemapper'
- $ref: '#/components/schemas/LogsSpanRemapper'
- $ref: '#/components/schemas/LogsArrayProcessor'
LogsQueryCompute:
description: Define computation for a log query.
properties:
Expand Down
Loading
Loading