Skip to content

Adding custom mapper support to Observability Pipelines OCSF Mapper #3478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
api-clients-generation-pipeline wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Adding custom mapper support to Observability Pipelines OCSF Mapper #3478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 111 additions & 4 deletions .generator/schemas/v2/openapi.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40869,8 +40869,6 @@ components:
type: integer
type:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptionsMemoryType'
when_full:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptionsWhenFull'
type: object
ObservabilityPipelineMemoryBufferSizeOptions:
description: Options for configuring a memory buffer by queue length.
Expand All @@ -40882,8 +40880,6 @@ components:
type: integer
type:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptionsMemoryType'
when_full:
$ref: '#/components/schemas/ObservabilityPipelineBufferOptionsWhenFull'
type: object
ObservabilityPipelineMetadataEntry:
description: A custom metadata entry.
Expand Down Expand Up @@ -41112,6 +41108,7 @@ components:
example: CloudTrail Account Change
oneOf:
- $ref: '#/components/schemas/ObservabilityPipelineOcsfMappingLibrary'
- $ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustom'
ObservabilityPipelineOcsfMapperProcessorType:
default: ocsf_mapper
description: The processor type. The value should always be `ocsf_mapper`.
Expand All @@ -41121,6 +41118,116 @@ components:
type: string
x-enum-varnames:
- OCSF_MAPPER
ObservabilityPipelineOcsfMappingCustom:
description: Custom OCSF mapping configuration for transforming logs.
properties:
mapping:
description: A list of field mapping rules for transforming log fields to
OCSF schema fields.
items:
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomFieldMapping'
type: array
metadata:
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomMetadata'
version:
description: The version of the custom mapping configuration.
example: 1
format: int64
type: integer
required:
- mapping
- metadata
- version
type: object
ObservabilityPipelineOcsfMappingCustomFieldMapping:
description: Defines a single field mapping rule for transforming a source field
to an OCSF destination field.
properties:
default:
description: The default value to use if the source field is missing or
empty.
example: ''
dest:
description: The destination OCSF field path.
example: device.type
type: string
lookup:
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomLookup'
source:
description: The source field path from the log event.
example: host.type
sources:
description: Multiple source field paths for combined mapping.
example:
- field1
- field2
value:
description: A static value to use for the destination field.
example: static_value
required:
- dest
type: object
ObservabilityPipelineOcsfMappingCustomLookup:
description: Lookup table configuration for mapping source values to destination
values.
properties:
default:
description: The default value to use if no lookup match is found.
example: unknown
table:
description: A list of lookup table entries for value transformation.
items:
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomLookupTableEntry'
type: array
type: object
ObservabilityPipelineOcsfMappingCustomLookupTableEntry:
description: A single entry in a lookup table for value transformation.
properties:
contains:
description: The substring to match in the source value.
example: Desktop
type: string
equals:
description: The exact value to match in the source.
example: desktop
equals_source:
description: The source field whose value to match.
example: device_type
type: string
matches:
description: A regex pattern to match in the source value.
example: ^Desktop.*
type: string
not_matches:
description: A regex pattern that must not match in the source value.
example: ^Mobile.*
type: string
value:
description: The value to use when the match is found.
example: desktop
type: object
ObservabilityPipelineOcsfMappingCustomMetadata:
description: Metadata for the custom OCSF mapping.
properties:
class:
description: The OCSF event class name.
example: Device Inventory Info
type: string
profiles:
description: A list of OCSF profiles to apply.
example:
- container
items:
type: string
type: array
version:
description: The OCSF schema version.
example: 1.3.0
type: string
required:
- class
- version
type: object
ObservabilityPipelineOcsfMappingLibrary:
description: Predefined library mappings for common log formats.
enum:
Expand Down
151 changes: 151 additions & 0 deletions examples/v2/observability-pipelines/ValidatePipeline_3024756866.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
// Validate an observability pipeline with OCSF mapper custom mapping returns "OK" response

import com.datadog.api.client.ApiClient;
import com.datadog.api.client.ApiException;
import com.datadog.api.client.v2.api.ObservabilityPipelinesApi;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfig;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigDestinationItem;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigProcessorGroup;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigProcessorItem;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigSourceItem;
import com.datadog.api.client.v2.model.ObservabilityPipelineDataAttributes;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogAgentSource;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogAgentSourceType;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogLogsDestination;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogLogsDestinationType;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMapperProcessor;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMapperProcessorMapping;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMapperProcessorMappingMapping;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMapperProcessorType;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustom;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustomFieldMapping;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustomLookup;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustomLookupTableEntry;
import com.datadog.api.client.v2.model.ObservabilityPipelineOcsfMappingCustomMetadata;
import com.datadog.api.client.v2.model.ObservabilityPipelineSpec;
import com.datadog.api.client.v2.model.ObservabilityPipelineSpecData;
import com.datadog.api.client.v2.model.ValidationResponse;
import java.util.Arrays;
import java.util.Collections;

public class Example {
public static void main(String[] args) {
ApiClient defaultClient = ApiClient.getDefaultApiClient();
ObservabilityPipelinesApi apiInstance = new ObservabilityPipelinesApi(defaultClient);

ObservabilityPipelineSpec body =
new ObservabilityPipelineSpec()
.data(
new ObservabilityPipelineSpecData()
.attributes(
new ObservabilityPipelineDataAttributes()
.config(
new ObservabilityPipelineConfig()
.destinations(
Collections.singletonList(
new ObservabilityPipelineConfigDestinationItem(
new ObservabilityPipelineDatadogLogsDestination()
.id("datadog-logs-destination")
.inputs(
Collections.singletonList(
"my-processor-group"))
.type(
ObservabilityPipelineDatadogLogsDestinationType
.DATADOG_LOGS))))
.processorGroups(
Collections.singletonList(
new ObservabilityPipelineConfigProcessorGroup()
.enabled(true)
.id("my-processor-group")
.include("service:my-service")
.inputs(
Collections.singletonList(
"datadog-agent-source"))
.processors(
Collections.singletonList(
new ObservabilityPipelineConfigProcessorItem(
new ObservabilityPipelineOcsfMapperProcessor()
.enabled(true)
.id("ocsf-mapper-processor")
.include("service:my-service")
.type(
ObservabilityPipelineOcsfMapperProcessorType
.OCSF_MAPPER)
.mappings(
Collections.singletonList(
new ObservabilityPipelineOcsfMapperProcessorMapping()
.include(
"source:custom")
.mapping(
new ObservabilityPipelineOcsfMapperProcessorMappingMapping(
new ObservabilityPipelineOcsfMappingCustom()
.version(1L)
.metadata(
new ObservabilityPipelineOcsfMappingCustomMetadata()
._class(
"Device"
+ " Inventory"
+ " Info")
.profiles(
Collections
.singletonList(
"container"))
.version(
"1.3.0"))
.mapping(
Arrays
.asList(
new ObservabilityPipelineOcsfMappingCustomFieldMapping()
.dest(
"time")
.source(
"timestamp")
._default(
""),
new ObservabilityPipelineOcsfMappingCustomFieldMapping()
.dest(
"severity")
.source(
"level")
._default(
""),
new ObservabilityPipelineOcsfMappingCustomFieldMapping()
.dest(
"device.type")
.source(
"host.type")
._default(
"")
.lookup(
new ObservabilityPipelineOcsfMappingCustomLookup()
.table(
Collections
.singletonList(
new ObservabilityPipelineOcsfMappingCustomLookupTableEntry()
.contains(
"Desktop")
.value(
"desktop")))))))))))))))
.sources(
Collections.singletonList(
new ObservabilityPipelineConfigSourceItem(
new ObservabilityPipelineDatadogAgentSource()
.id("datadog-agent-source")
.type(
ObservabilityPipelineDatadogAgentSourceType
.DATADOG_AGENT)))))
.name("OCSF Custom Mapper Pipeline"))
.type("pipelines"));

try {
ValidationResponse result = apiInstance.validatePipeline(body);
System.out.println(result);
} catch (ApiException e) {
System.err.println("Exception when calling ObservabilityPipelinesApi#validatePipeline");
System.err.println("Status code: " + e.getCode());
System.err.println("Reason: " + e.getResponseBody());
System.err.println("Response headers: " + e.getResponseHeaders());
e.printStackTrace();
}
}
}
Loading
Loading