From 06545b5766fec14b12b4f445127f88168e149cd9 Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Wed, 25 Sep 2024 12:47:21 -0700 Subject: [PATCH 1/5] added span-link normalization and fixed errors in getTransaction --- packages/kbn-apm-types/src/es_fields/apm.ts | 6 +- .../apm/common/correlations/types.ts | 1 - .../apm/common/es_fields/infra_metrics.ts | 1 + .../get_error_sample_details.ts | 131 +++++++++++++++++- .../transactions/get_transaction/index.ts | 58 +++++++- .../apm/server/utils/es_fields_mappings.ts | 38 +++-- 6 files changed, 208 insertions(+), 27 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index eaf6e4e0282f86..ea5e89f91d71fb 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -56,12 +56,14 @@ export const USER_AGENT_DEVICE_NAME = 'user_agent.device.name'; export const USER_AGENT_VERSION = 'user_agent.version'; export const OBSERVER_HOSTNAME = 'observer.hostname'; -export const OBSERVER_LISTENING = 'observer.listening'; +export const OBSERVER_ID = 'observer.id'; export const OBSERVER_TYPE = 'observer.type'; +export const OBSERVER_NAME = 'observer.name'; export const OBSERVER_VERSION = 'observer.version'; +export const OBSERVER_LISTENING = 'observer.listening'; export const OBSERVER_VERSION_MAJOR = 'observer.version_major'; export const PROCESSOR_EVENT = 'processor.event'; -export const PROCESOR_NAME = 'processor.name'; +export const PROCESSOR_NAME = 'processor.name'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; diff --git a/x-pack/plugins/observability_solution/apm/common/correlations/types.ts b/x-pack/plugins/observability_solution/apm/common/correlations/types.ts index 6555e8a5d953d8..08b345b2461d54 100644 --- a/x-pack/plugins/observability_solution/apm/common/correlations/types.ts +++ b/x-pack/plugins/observability_solution/apm/common/correlations/types.ts @@ -27,7 +27,6 @@ export interface ResponseHitSource { } export interface ResponseHit { - _source: ResponseHitSource; fields: ResponseHitSource; } diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts index fbf09e1887dfbc..bfd8d6aa91f9b9 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts @@ -7,6 +7,7 @@ // Kubernetes export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; +export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment'; export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name'; export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts index 28f8a1d4573d15..f008f8d6e44fbc 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_sample_details.ts @@ -6,7 +6,61 @@ */ import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server'; -import { ERROR_ID, SERVICE_NAME } from '../../../../common/es_fields/apm'; +import { OBSERVER_VERSION_MAJOR } from '@kbn/observability-shared-plugin/common'; +import { + ERROR_EXCEPTION_STACKTRACE, + ERROR_LOG_STACKTRACE, + ERROR_STACK_TRACE, +} from '@kbn/discover-utils/src/field_constants'; +import { + AGENT_NAME, + AGENT_VERSION, + CONTAINER_ID, + CONTAINER_IMAGE, + ERROR_ID, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_NAME, + HTTP_RESPONSE_STATUS_CODE, + KUBERNETES_POD_UID, + OBSERVER_HOSTNAME, + OBSERVER_ID, + OBSERVER_TYPE, + OBSERVER_NAME, + OBSERVER_VERSION, + PARENT_ID, + PROCESSOR_EVENT, + PROCESSOR_NAME, + SERVICE_NAME, + TRACE_ID, + SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, + SERVICE_VERSION, + HOST_OS_PLATFORM, + TRANSACTION_ID, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + ERROR_CULPRIT, + ERROR_PAGE_URL, + ERROR_LOG_MESSAGE, + URL_FULL, + USER_ID, + PROCESS_PID, + PROCESS_ARGS, + PROCESS_TITLE, + AT_TIMESTAMP, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ERROR_EXC_MESSAGE, + URL_DOMAIN, + URL_ORIGINAL, +} from '../../../../common/es_fields/apm'; import { environmentQuery } from '../../../../common/utils/environment_query'; import { ApmDocumentType } from '../../../../common/document_type'; import { RollupInterval } from '../../../../common/rollup'; @@ -15,6 +69,13 @@ import { getTransaction } from '../../transactions/get_transaction'; import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; import { errorSampleDetailsMapping } from '../../../utils/es_fields_mappings'; +import { + KUBERNETES_CONTAINER_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_CONTAINER_ID, +} from '../../../../common/es_fields/infra_metrics'; export interface ErrorSampleDetailsResponse { transaction: Transaction | undefined; @@ -61,7 +122,73 @@ export async function getErrorSampleDetails({ ], }, }, - fields: ['*'], + fields: [ + AGENT_VERSION, + AGENT_NAME, + PARENT_ID, + TRACE_ID, + // AGENT_EPHEMERAL_ID, + // OBSERVER_EPHEMERAL_ID, + OBSERVER_HOSTNAME, + OBSERVER_ID, + OBSERVER_TYPE, + OBSERVER_NAME, + OBSERVER_VERSION, + OBSERVER_VERSION_MAJOR, + CONTAINER_ID, + CONTAINER_IMAGE, + PROCESSOR_NAME, + PROCESSOR_EVENT, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_NAME, + 'host.ip', + 'http.request.method', + HTTP_RESPONSE_STATUS_CODE, + 'http.version', + KUBERNETES_POD_UID, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + SERVICE_NAME, + SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, + SERVICE_VERSION, + PROCESS_ARGS, + PROCESS_PID, + PROCESS_TITLE, + HOST_OS_PLATFORM, + AT_TIMESTAMP, + TRANSACTION_ID, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + ERROR_ID, + ERROR_CULPRIT, + 'error.exception.attributes.response', + // ERROR_EXC_ATTRIBUTES_RESPONSE, + // ERROR_EXC_CODE, + ERROR_EXC_MESSAGE, + ERROR_EXC_TYPE, + // ERROR_EXCEPTION_MODULE, + ERROR_EXC_HANDLED, + ERROR_EXCEPTION_STACKTRACE, // todo: fix me + ERROR_PAGE_URL, + ERROR_LOG_MESSAGE, + ERROR_LOG_STACKTRACE, // todo: fixme + ERROR_STACK_TRACE, + URL_DOMAIN, + URL_FULL, + URL_ORIGINAL, + USER_ID, + ], }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts index e5f08762f6d6b8..b3ce38f1df4a30 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/transactions/get_transaction/index.ts @@ -7,7 +7,34 @@ import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; import { transactionMapping } from '../../../utils/es_fields_mappings'; -import { TRACE_ID, TRANSACTION_ID } from '../../../../common/es_fields/apm'; +import { + AGENT_NAME, + AGENT_VERSION, + DATA_STEAM_TYPE, + DATA_STREAM_DATASET, + DATA_STREAM_NAMESPACE, + EVENT_OUTCOME, + EVENT_SUCCESS_COUNT, + LABEL_SOME_RESOURCE_ATTRIBUTE, + OBSERVER_HOSTNAME, + OBSERVER_TYPE, + OBSERVER_VERSION, + PROCESSOR_EVENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_NAME, + SERVICE_NODE_NAME, + TIMESTAMP, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_RESULT, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, + SPAN_ID, + AT_TIMESTAMP, + TRANSACTION_REPRESENTATIVE_COUNT, +} from '../../../../common/es_fields/apm'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { ApmDocumentType } from '../../../../common/document_type'; @@ -48,7 +75,34 @@ export async function getTransaction({ ]), }, }, - fields: ['*'], + fields: [ + TRANSACTION_REPRESENTATIVE_COUNT, + TRANSACTION_RESULT, + TRANSACTION_SAMPLED, + TRANSACTION_ID, + TRANSACTION_DURATION, + TRANSACTION_TYPE, + TRANSACTION_NAME, + SERVICE_NODE_NAME, + SERVICE_NAME, + SERVICE_FRAMEWORK_NAME, + TRACE_ID, + AGENT_NAME, + AGENT_VERSION, + EVENT_SUCCESS_COUNT, + EVENT_OUTCOME, + PROCESSOR_EVENT, + DATA_STREAM_NAMESPACE, + DATA_STEAM_TYPE, + DATA_STREAM_DATASET, + SPAN_ID, + OBSERVER_HOSTNAME, + OBSERVER_TYPE, + OBSERVER_VERSION, + TIMESTAMP, + AT_TIMESTAMP, + LABEL_SOME_RESOURCE_ATTRIBUTE, + ], }, }); diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index b8b57b5f36b021..8b500b11b5838e 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -29,7 +29,7 @@ import { OBSERVER_VERSION, OBSERVER_VERSION_MAJOR, PARENT_ID, - PROCESOR_NAME, + PROCESSOR_NAME, PROCESSOR_EVENT, SERVICE_ENVIRONMENT, SERVICE_FRAMEWORK_NAME, @@ -48,7 +48,6 @@ import { SPAN_SUBTYPE, SPAN_SYNC, SPAN_TYPE, - SPAN_LINKS, TIMESTAMP, TRACE_ID, TRANSACTION_DURATION, @@ -197,7 +196,7 @@ export const spanMapping = (fields: Partial>) => { success_count: normalizeValue(fields[EVENT_SUCCESS_COUNT]), }, processor: { - name: normalizeValue<'transaction'>(fields[PROCESOR_NAME]), + name: normalizeValue<'transaction'>(fields[PROCESSOR_NAME]), event: normalizeValue<'span'>(fields[PROCESSOR_EVENT]), }, transaction: { @@ -274,20 +273,16 @@ export const spanLinksDetailsMapping = (fields: Partial>) => { if (!fields) return undefined; - return { - span: { - links: [ - { - trace: { - id: normalizeValue(fields[SPAN_LINKS_TRACE_ID]), - }, - span: { - id: normalizeValue(fields[SPAN_LINKS_SPAN_ID]), - }, - }, - ], - }, - }; + return (fields[SPAN_LINKS_TRACE_ID] as string[]).map((v, index) => { + return { + trace: { + id: v, + }, + span: { + id: fields[SPAN_LINKS_SPAN_ID]?.[index] ?? '', + }, + }; + }) as SpanLink[]; }; export const transactionMapping = (fields: Partial>) => { @@ -329,7 +324,8 @@ export const transactionMapping = (fields: Partial>) = outcome: normalizeValue(fields[EVENT_OUTCOME]), }, processor: { - event: normalizeValue(fields[PROCESSOR_EVENT]), + event: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), + name: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), }, data_stream: { namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), @@ -338,11 +334,13 @@ export const transactionMapping = (fields: Partial>) = }, span: { id: normalizeValue(fields[SPAN_ID]), + links: linkedParentsOfSpanMapping(fields), }, observer: { hostname: normalizeValue(fields[OBSERVER_HOSTNAME]), type: normalizeValue(fields[OBSERVER_TYPE]), version: normalizeValue(fields[OBSERVER_VERSION]), + version_major: normalizeValue(fields[OBSERVER_VERSION_MAJOR]), }, timestamp: { us: normalizeValue(fields[TIMESTAMP]), @@ -403,7 +401,7 @@ export const traceDocMapping = ( us: normalizeValue(fields[SPAN_DURATION]), }, action: normalizeValue(fields[SPAN_ACTION]), - links: normalizeValue(fields[SPAN_LINKS]), + links: linkedParentsOfSpanMapping(fields) as SpanLink[], composite: { count: normalizeValue(fields[SPAN_COMPOSITE_COUNT]), sum: { @@ -517,7 +515,7 @@ export const errorSampleDetailsMapping = ( grouping_key: normalizeValue(fields[ERROR_GROUP_ID]), }, processor: { - name: normalizeValue<'error'>(fields[PROCESOR_NAME]), + name: normalizeValue<'error'>(fields[PROCESSOR_NAME]), event: normalizeValue<'error'>(fields[PROCESSOR_EVENT]), }, transaction: { From a08ef938215db02e87e1dedb0979d4766eeadcf6 Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Thu, 26 Sep 2024 13:15:23 -0700 Subject: [PATCH 2/5] fix service metadata endpoints --- packages/kbn-apm-types/src/es_fields/apm.ts | 6 ++ .../apm/common/es_fields/infra_metrics.ts | 2 + .../services/get_service_metadata_details.ts | 87 +++++++++++++++++-- .../services/get_service_metadata_icons.ts | 45 +++++++++- .../apm/server/utils/es_fields_mappings.ts | 35 ++++++-- 5 files changed, 158 insertions(+), 17 deletions(-) diff --git a/packages/kbn-apm-types/src/es_fields/apm.ts b/packages/kbn-apm-types/src/es_fields/apm.ts index ea5e89f91d71fb..6c7276317894cf 100644 --- a/packages/kbn-apm-types/src/es_fields/apm.ts +++ b/packages/kbn-apm-types/src/es_fields/apm.ts @@ -21,7 +21,11 @@ export const CLOUD_AVAILABILITY_ZONE = 'cloud.availability_zone'; export const CLOUD_PROVIDER = 'cloud.provider'; export const CLOUD_REGION = 'cloud.region'; export const CLOUD_MACHINE_TYPE = 'cloud.machine.type'; +export const CLOUD_PROJECT_ID = 'cloud.project.id'; +export const CLOUD_PROJECT_NAME = 'cloud.project.name'; +export const CLOUD_IMAGE_ID = 'cloud.image.id'; export const CLOUD_ACCOUNT_ID = 'cloud.account.id'; +export const CLOUD_ACCOUNT_NAME = 'cloud.account.name'; export const CLOUD_INSTANCE_ID = 'cloud.instance.id'; export const CLOUD_INSTANCE_NAME = 'cloud.instance.name'; export const CLOUD_SERVICE_NAME = 'cloud.service.name'; @@ -168,6 +172,8 @@ export const HOST_IP = 'host.ip'; export const CONTAINER_ID = 'container.id'; export const CONTAINER = 'container'; export const CONTAINER_IMAGE = 'container.image.name'; +export const CONTAINER_OS = 'container.os'; +export const CONTAINER_TTL_INSTANCES = 'container.totalNumberInstances'; export const KUBERNETES = 'kubernetes'; export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; diff --git a/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts b/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts index bfd8d6aa91f9b9..7d7081a0e1900d 100644 --- a/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts +++ b/x-pack/plugins/observability_solution/apm/common/es_fields/infra_metrics.ts @@ -7,6 +7,7 @@ // Kubernetes export const KUBERNETES_CONTAINER_NAME = 'kubernetes.container.name'; +export const KUBERNETES_NODE_NAME = 'kubernetes.node.name'; export const KUBERNETES_CONTAINER_ID = 'kubernetes.container.id'; export const KUBERNETES_DEPLOYMENT = 'kubernetes.deployment'; export const KUBERNETES_DEPLOYMENT_NAME = 'kubernetes.deployment.name'; @@ -14,3 +15,4 @@ export const KUBERNETES_NAMESPACE_NAME = 'kubernetes.namespace.name'; export const KUBERNETES_NAMESPACE = 'kubernetes.namespace'; export const KUBERNETES_REPLICASET = 'kubernetes.replicaset'; export const KUBERNETES_REPLICASET_NAME = 'kubernetes.replicaset.name'; +export const KUBERNETES_POD_NAME = 'kubernetes.pod.name'; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index 3609905e2d28f2..7ce433bd6f1add 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -9,23 +9,50 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; import { environmentQuery } from '../../../common/utils/environment_query'; import { - AGENT, - CONTAINER, - CLOUD, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_NODE_NAME, + KUBERNETES_POD_NAME, +} from '../../../common/es_fields/infra_metrics'; +import { + AGENT_NAME, + AGENT_VERSION, + CONTAINER_ID, + CONTAINER_IMAGE, CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, CLOUD_REGION, + CLOUD_PROJECT_NAME, CLOUD_MACHINE_TYPE, CLOUD_SERVICE_NAME, - CONTAINER_ID, - HOST, - KUBERNETES, - SERVICE, - SERVICE_NAME, + CLOUD_INSTANCE_NAME, + CLOUD_PROVIDER, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, + CLOUD_PROJECT_ID, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_NAME, + HOST_IP, + SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, SERVICE_VERSION, + SERVICE_NAME, FAAS_ID, FAAS_TRIGGER_TYPE, LABEL_TELEMETRY_AUTO_VERSION, + CONTAINER_TTL_INSTANCES, + HOST_OS_PLATFORM, } from '../../../common/es_fields/apm'; import { ContainerType } from '../../../common/service_metadata'; @@ -113,6 +140,7 @@ export async function getServiceMetadataDetails({ body: { track_total_hits: 1, size: 1, + _source: false, query: { bool: { filter, should } }, aggs: { serviceVersions: { @@ -166,7 +194,48 @@ export async function getServiceMetadataDetails({ }, totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, - fields: [SERVICE, AGENT, HOST, CONTAINER, KUBERNETES, CLOUD, LABEL_TELEMETRY_AUTO_VERSION], + fields: [ + SERVICE_ENVIRONMENT, + SERVICE_FRAMEWORK_NAME, + SERVICE_FRAMEWORK_VERSION, + SERVICE_NODE_NAME, + SERVICE_RUNTIME_NAME, + SERVICE_RUNTIME_VERSION, + SERVICE_LANGUAGE_NAME, + SERVICE_LANGUAGE_VERSION, + SERVICE_VERSION, + SERVICE_NAME, + AGENT_NAME, + AGENT_VERSION, + CONTAINER_ID, + CONTAINER_IMAGE, + HOST_ARCHITECTURE, + HOST_HOSTNAME, + HOST_NAME, + HOST_IP, + HOST_OS_PLATFORM, + CONTAINER_TTL_INSTANCES, + KUBERNETES_NAMESPACE, + KUBERNETES_NODE_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_NAME, + CLOUD_INSTANCE_ID, + CLOUD_MACHINE_TYPE, + CLOUD_PROJECT_ID, + CLOUD_PROJECT_NAME, + CLOUD_PROVIDER, + CLOUD_REGION, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, + CLOUD_SERVICE_NAME, + LABEL_TELEMETRY_AUTO_VERSION, + ], }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 8ba26cf5a55c5a..42a0e8f40d3f47 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -13,18 +13,35 @@ import { CLOUD_PROVIDER, CLOUD_SERVICE_NAME, CONTAINER_ID, - KUBERNETES, SERVICE_NAME, KUBERNETES_POD_NAME, HOST_OS_PLATFORM, LABEL_TELEMETRY_AUTO_VERSION, AGENT_VERSION, SERVICE_FRAMEWORK_NAME, + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_NAME, + CLOUD_INSTANCE_ID, + CLOUD_MACHINE_TYPE, + CLOUD_PROJECT_ID, + CLOUD_PROJECT_NAME, + CLOUD_REGION, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, } from '../../../common/es_fields/apm'; import { ContainerType } from '../../../common/service_metadata'; import { getProcessorEventForTransactions } from '../../lib/helpers/transactions'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { ServerlessType, getServerlessTypeFromCloudData } from '../../../common/serverless'; +import { + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_NAMESPACE, + KUBERNETES_NODE_NAME, + KUBERNETES_REPLICASET_NAME, +} from '../../../common/es_fields/infra_metrics'; export interface ServiceMetadataIcons { agentName?: string; @@ -70,8 +87,32 @@ export async function getServiceMetadataIcons({ body: { track_total_hits: 1, size: 1, + _source: false, query: { bool: { filter, should } }, - fields: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME, CLOUD_SERVICE_NAME], + fields: [ + KUBERNETES_NAMESPACE, + KUBERNETES_NODE_NAME, + KUBERNETES_POD_NAME, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_CONTAINER_ID, + KUBERNETES_CONTAINER_NAME, + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_NAME, + CLOUD_INSTANCE_ID, + CLOUD_MACHINE_TYPE, + CLOUD_PROJECT_ID, + CLOUD_PROJECT_NAME, + CLOUD_PROVIDER, + CLOUD_REGION, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, + CLOUD_SERVICE_NAME, + CONTAINER_ID, + AGENT_NAME, + CLOUD_SERVICE_NAME, + ], }, }; diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 8b500b11b5838e..bf511a0864a518 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -63,11 +63,9 @@ import { ERROR_ID, ERROR_LOG_MESSAGE, HOST_NAME, - Container, - Kubernetes, + CONTAINER_ID, + CONTAINER_IMAGE, CLOUD_PROVIDER, - CONTAINER, - KUBERNETES, AGENT_ACTIVATION_METHOD, HOST_ARCHITECTURE, HOST_HOSTNAME, @@ -87,6 +85,12 @@ import { SERVICE_TARGET_TYPE, SPAN_REPRESENTATIVE_COUNT, } from '@kbn/apm-types'; +import { + KUBERNETES_CONTAINER_NAME, + KUBERNETES_DEPLOYMENT_NAME, + KUBERNETES_REPLICASET_NAME, + KUBERNETES_CONTAINER_ID, +} from '../../common/es_fields/infra_metrics'; import { Transaction } from '../../typings/es_schemas/ui/transaction'; import { TransactionRaw } from '../../typings/es_schemas/raw/transaction_raw'; import { @@ -545,8 +549,27 @@ export const serviceMetadataIconsMapping = ( name: normalizeValue(fields[CLOUD_PROVIDER]), }, }, - container: normalizeValue(fields[CONTAINER]), - kubernetes: normalizeValue(fields[KUBERNETES]), + container: { + id: normalizeValue(fields[CONTAINER_ID]), + image: normalizeValue(fields[CONTAINER_IMAGE]), + }, + kubernetes: { + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + id: normalizeValue(fields[KUBERNETES_POD_UID]), + }, + namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), + replicaset: { + name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), + }, + deployment: { + name: normalizeValue(fields[KUBERNETES_DEPLOYMENT_NAME]), + }, + container: { + id: normalizeValue(fields[KUBERNETES_CONTAINER_ID]), + name: normalizeValue(fields[KUBERNETES_CONTAINER_NAME]), + }, + }, }; }; From f15655962dc056250360e9eea3c1a1aa359a1d4b Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Fri, 27 Sep 2024 10:49:55 -0700 Subject: [PATCH 3/5] PR feedback fix --- .../apm/server/utils/es_fields_mappings.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index bf511a0864a518..773d2ae9c21470 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -329,7 +329,7 @@ export const transactionMapping = (fields: Partial>) = }, processor: { event: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), - name: normalizeValue<'transaction'>(fields[PROCESSOR_EVENT]), + name: normalizeValue<'transaction'>(fields[PROCESSOR_NAME]), }, data_stream: { namespace: normalizeValue(fields[DATA_STREAM_NAMESPACE]), From 2ef8527280340aad562b8f7e9465d2f6bf807bc4 Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Fri, 27 Sep 2024 13:43:43 -0700 Subject: [PATCH 4/5] added mapping for metadata_details --- .../services/get_service_metadata_details.ts | 4 +- .../services/get_service_metadata_icons.ts | 5 +- .../apm/server/utils/es_fields_mappings.ts | 189 +++++++++++++++--- 3 files changed, 166 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts index 7ce433bd6f1add..2c0064e0782c3a 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_details.ts @@ -7,6 +7,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { serviceMetadataDetailsMapping } from '../../utils/es_fields_mappings'; import { environmentQuery } from '../../../common/utils/environment_query'; import { KUBERNETES_CONTAINER_NAME, @@ -59,7 +60,6 @@ import { ContainerType } from '../../../common/service_metadata'; import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client'; import { should } from './get_service_metadata_icons'; -import { normalizeFields } from '../../utils/normalize_fields'; import { isOpenTelemetryAgentName, hasOpenTelemetryPrefix } from '../../../common/agent_name'; type ServiceMetadataDetailsRaw = Pick< @@ -243,7 +243,7 @@ export async function getServiceMetadataDetails({ const fields = response.hits.hits[0]?.fields; // todo: missing `fields` property? - const fieldsNorm = (fields ? normalizeFields(fields) : undefined) as + const fieldsNorm = (fields ? serviceMetadataDetailsMapping(fields) : undefined) as | ServiceMetadataDetailsRaw | undefined; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts index 42a0e8f40d3f47..3108b07524ec27 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/services/get_service_metadata_icons.ts @@ -7,6 +7,7 @@ import { rangeQuery } from '@kbn/observability-plugin/server'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { TransactionRaw } from '@kbn/apm-types'; import { serviceMetadataIconsMapping } from '../../utils/es_fields_mappings'; import { AGENT_NAME, @@ -43,6 +44,8 @@ import { KUBERNETES_REPLICASET_NAME, } from '../../../common/es_fields/infra_metrics'; +type ServiceMetadataIconsRaw = Pick; + export interface ServiceMetadataIcons { agentName?: string; containerType?: ContainerType; @@ -129,7 +132,7 @@ export async function getServiceMetadataIcons({ const { kubernetes, cloud, container, agent } = serviceMetadataIconsMapping( response.hits.hits[0].fields - ); + ) as ServiceMetadataIconsRaw; let containerType: ContainerType; if (!!kubernetes) { diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 773d2ae9c21470..81fadf4424fbd1 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -66,6 +66,7 @@ import { CONTAINER_ID, CONTAINER_IMAGE, CLOUD_PROVIDER, + CLOUD_INSTANCE_NAME, AGENT_ACTIVATION_METHOD, HOST_ARCHITECTURE, HOST_HOSTNAME, @@ -84,6 +85,16 @@ import { SERVICE_TARGET_NAME, SERVICE_TARGET_TYPE, SPAN_REPRESENTATIVE_COUNT, + SERVICE_LANGUAGE_VERSION, + CLOUD_AVAILABILITY_ZONE, + CLOUD_INSTANCE_ID, + CLOUD_MACHINE_TYPE, + CLOUD_PROJECT_ID, + CLOUD_PROJECT_NAME, + CLOUD_REGION, + CLOUD_ACCOUNT_ID, + CLOUD_ACCOUNT_NAME, + CLOUD_IMAGE_ID, } from '@kbn/apm-types'; import { KUBERNETES_CONTAINER_NAME, @@ -102,7 +113,10 @@ import { EventOutcome } from '../../common/event_outcome'; import { Exception } from '../../typings/es_schemas/raw/error_raw'; type ServiceMetadataIconsRaw = Pick; - +type ServiceMetadataDetailsRaw = Pick< + TransactionRaw, + 'service' | 'agent' | 'host' | 'container' | 'kubernetes' | 'cloud' | 'labels' +>; const normalizeValue = (field: unknown[] | unknown): T => { return (Array.isArray(field) && field.length > 0 ? field[0] : field) as T; }; @@ -533,43 +547,160 @@ export const errorSampleDetailsMapping = ( }; }; -export const serviceMetadataIconsMapping = ( +export const serviceMetadataDetailsMapping = ( fields: Partial> -): ServiceMetadataIconsRaw | undefined => { +): ServiceMetadataDetailsRaw | undefined => { if (!fields) return undefined; - + const kubernetesNamespace = normalizeValue(fields[KUBERNETES_NAMESPACE]); + const containerId = normalizeValue(fields[CONTAINER_ID]); + const cloudServiceName = normalizeValue(fields[CLOUD_PROVIDER]); return { - agent: { - name: normalizeValue(fields[AGENT_NAME]), - version: '', - }, - cloud: { - provider: normalizeValue(fields[CLOUD_PROVIDER]), - service: { - name: normalizeValue(fields[CLOUD_PROVIDER]), + service: { + name: normalizeValue(fields[SERVICE_NAME]), + version: normalizeValue(fields[SERVICE_VERSION]), + environment: normalizeValue(fields[SERVICE_ENVIRONMENT]), + framework: { + name: normalizeValue(fields[SERVICE_FRAMEWORK_NAME]), + version: normalizeValue(fields[SERVICE_FRAMEWORK_VERSION]), }, - }, - container: { - id: normalizeValue(fields[CONTAINER_ID]), - image: normalizeValue(fields[CONTAINER_IMAGE]), - }, - kubernetes: { - pod: { - name: normalizeValue(fields[KUBERNETES_POD_NAME]), - id: normalizeValue(fields[KUBERNETES_POD_UID]), + node: { + name: normalizeValue(fields[SERVICE_NODE_NAME]), }, - namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), - replicaset: { - name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), + runtime: { + name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), + version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), }, - deployment: { - name: normalizeValue(fields[KUBERNETES_DEPLOYMENT_NAME]), + language: { + name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), + version: normalizeValue(fields[SERVICE_LANGUAGE_VERSION]), }, - container: { - id: normalizeValue(fields[KUBERNETES_CONTAINER_ID]), - name: normalizeValue(fields[KUBERNETES_CONTAINER_NAME]), + }, + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: normalizeValue(fields[AGENT_VERSION]), + }, + host: { + architecture: normalizeValue(fields[HOST_ARCHITECTURE]), + hostname: normalizeValue(fields[HOST_HOSTNAME]), + name: normalizeValue(fields[HOST_NAME]), + ip: normalizeValue(fields[HOST_IP]), + os: { + platform: normalizeValue(fields[HOST_OS_PLATFORM]), }, }, + ...(containerId + ? { + container: { + id: containerId, + image: normalizeValue(fields[CONTAINER_IMAGE]), + }, + } + : undefined), + ...(kubernetesNamespace + ? { + kubernetes: { + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + uid: normalizeValue(fields[KUBERNETES_POD_UID]), + }, + namespace: kubernetesNamespace, + replicaset: { + name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), + }, + deployment: { + name: normalizeValue(fields[KUBERNETES_DEPLOYMENT_NAME]), + }, + container: { + id: normalizeValue(fields[KUBERNETES_CONTAINER_ID]), + name: normalizeValue(fields[KUBERNETES_CONTAINER_NAME]), + }, + }, + } + : undefined), + ...(cloudServiceName + ? { + cloud: { + availability_zone: normalizeValue(fields[CLOUD_AVAILABILITY_ZONE]), + instance: { + name: normalizeValue(fields[CLOUD_INSTANCE_NAME]), + id: normalizeValue(fields[CLOUD_INSTANCE_ID]), + }, + machine: { + type: normalizeValue(fields[CLOUD_MACHINE_TYPE]), + }, + project: { + id: normalizeValue(fields[CLOUD_PROJECT_ID]), + name: normalizeValue(fields[CLOUD_PROJECT_NAME]), + }, + provider: normalizeValue(fields[CLOUD_PROVIDER]), + region: normalizeValue(fields[CLOUD_REGION]), + account: { + id: normalizeValue(fields[CLOUD_ACCOUNT_ID]), + name: normalizeValue(fields[CLOUD_ACCOUNT_NAME]), + }, + image: { + id: normalizeValue(fields[CLOUD_IMAGE_ID]), + }, + service: { + name: normalizeValue(fields[CLOUD_PROVIDER]), + }, + }, + } + : undefined), + }; +}; + +export const serviceMetadataIconsMapping = ( + fields: Partial> +): ServiceMetadataIconsRaw | undefined => { + if (!fields) return undefined; + const kubernetesNamespace = normalizeValue(fields[KUBERNETES_NAMESPACE]); + const containerId = normalizeValue(fields[CONTAINER_ID]); + const cloudServiceName = normalizeValue(fields[CLOUD_PROVIDER]); + return { + agent: { + name: normalizeValue(fields[AGENT_NAME]), + version: '', + }, + ...(cloudServiceName + ? { + cloud: { + provider: normalizeValue(fields[CLOUD_PROVIDER]), + service: { + name: normalizeValue(fields[CLOUD_PROVIDER]), + }, + }, + } + : undefined), + ...(containerId + ? { + container: { + id: containerId, + image: normalizeValue(fields[CONTAINER_IMAGE]), + }, + } + : undefined), + ...(kubernetesNamespace + ? { + kubernetes: { + pod: { + name: normalizeValue(fields[KUBERNETES_POD_NAME]), + uid: normalizeValue(fields[KUBERNETES_POD_UID]), + }, + namespace: kubernetesNamespace, + replicaset: { + name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), + }, + deployment: { + name: normalizeValue(fields[KUBERNETES_DEPLOYMENT_NAME]), + }, + container: { + id: normalizeValue(fields[KUBERNETES_CONTAINER_ID]), + name: normalizeValue(fields[KUBERNETES_CONTAINER_NAME]), + }, + }, + } + : undefined), }; }; From 004f58730ea35f65f726d2aae16dbdec09f2bfb0 Mon Sep 17 00:00:00 2001 From: Bryce Buchanan Date: Fri, 27 Sep 2024 14:24:26 -0700 Subject: [PATCH 5/5] fixed kubernetes check in metadata mappings --- .../apm/server/utils/es_fields_mappings.ts | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts index 81fadf4424fbd1..26887fa08ee362 100644 --- a/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts +++ b/x-pack/plugins/observability_solution/apm/server/utils/es_fields_mappings.ts @@ -95,6 +95,7 @@ import { CLOUD_ACCOUNT_ID, CLOUD_ACCOUNT_NAME, CLOUD_IMAGE_ID, + CLOUD_SERVICE_NAME, } from '@kbn/apm-types'; import { KUBERNETES_CONTAINER_NAME, @@ -554,6 +555,7 @@ export const serviceMetadataDetailsMapping = ( const kubernetesNamespace = normalizeValue(fields[KUBERNETES_NAMESPACE]); const containerId = normalizeValue(fields[CONTAINER_ID]); const cloudServiceName = normalizeValue(fields[CLOUD_PROVIDER]); + const serviceRuntimeName = normalizeValue(fields[SERVICE_RUNTIME_NAME]); return { service: { name: normalizeValue(fields[SERVICE_NAME]), @@ -566,10 +568,14 @@ export const serviceMetadataDetailsMapping = ( node: { name: normalizeValue(fields[SERVICE_NODE_NAME]), }, - runtime: { - name: normalizeValue(fields[SERVICE_RUNTIME_NAME]), - version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), - }, + ...(serviceRuntimeName + ? { + runtime: { + name: serviceRuntimeName, + version: normalizeValue(fields[SERVICE_RUNTIME_VERSION]), + }, + } + : undefined), language: { name: normalizeValue(fields[SERVICE_LANGUAGE_NAME]), version: normalizeValue(fields[SERVICE_LANGUAGE_VERSION]), @@ -642,7 +648,7 @@ export const serviceMetadataDetailsMapping = ( id: normalizeValue(fields[CLOUD_IMAGE_ID]), }, service: { - name: normalizeValue(fields[CLOUD_PROVIDER]), + name: normalizeValue(fields[CLOUD_SERVICE_NAME]), }, }, } @@ -654,20 +660,40 @@ export const serviceMetadataIconsMapping = ( fields: Partial> ): ServiceMetadataIconsRaw | undefined => { if (!fields) return undefined; - const kubernetesNamespace = normalizeValue(fields[KUBERNETES_NAMESPACE]); + const kubernetesPodName = normalizeValue(fields[KUBERNETES_POD_NAME]); const containerId = normalizeValue(fields[CONTAINER_ID]); - const cloudServiceName = normalizeValue(fields[CLOUD_PROVIDER]); + const cloudProvider = normalizeValue(fields[CLOUD_PROVIDER]); return { agent: { name: normalizeValue(fields[AGENT_NAME]), version: '', }, - ...(cloudServiceName + ...(cloudProvider ? { cloud: { - provider: normalizeValue(fields[CLOUD_PROVIDER]), + availability_zone: normalizeValue(fields[CLOUD_AVAILABILITY_ZONE]), + instance: { + name: normalizeValue(fields[CLOUD_INSTANCE_NAME]), + id: normalizeValue(fields[CLOUD_INSTANCE_ID]), + }, + machine: { + type: normalizeValue(fields[CLOUD_MACHINE_TYPE]), + }, + project: { + id: normalizeValue(fields[CLOUD_PROJECT_ID]), + name: normalizeValue(fields[CLOUD_PROJECT_NAME]), + }, + provider: cloudProvider, + region: normalizeValue(fields[CLOUD_REGION]), + account: { + id: normalizeValue(fields[CLOUD_ACCOUNT_ID]), + name: normalizeValue(fields[CLOUD_ACCOUNT_NAME]), + }, + image: { + id: normalizeValue(fields[CLOUD_IMAGE_ID]), + }, service: { - name: normalizeValue(fields[CLOUD_PROVIDER]), + name: normalizeValue(fields[CLOUD_SERVICE_NAME]), }, }, } @@ -680,14 +706,14 @@ export const serviceMetadataIconsMapping = ( }, } : undefined), - ...(kubernetesNamespace + ...(kubernetesPodName ? { kubernetes: { pod: { name: normalizeValue(fields[KUBERNETES_POD_NAME]), uid: normalizeValue(fields[KUBERNETES_POD_UID]), }, - namespace: kubernetesNamespace, + namespace: normalizeValue(fields[KUBERNETES_NAMESPACE]), replicaset: { name: normalizeValue(fields[KUBERNETES_REPLICASET_NAME]), },