Explore Metrics: Get OTel resources and filter metrics and labels (gr…

…afana#91221)

* add OTel filter in metric select scene

* add resource query to get matching OTEL job&instance

* filter metrics by OTEL resources

* only add otel select if DS has OTEL matching job and instance

* add folder for otel resources

* upate metric select for new otel folder

* move otel api call

* get otel resources for labels for single series job/instance target_info

* add otel resources to adhoc variable dropdown

* update otel api to check for standardization and return labels

* label types for api

* check standardization, show otel variable, select depenv, update other variables

* remove otel target list from metric select scene

* load resources if dep_env label has already been selected

* exclude previously used filters

* do not check standardization if there are already otel filters

* drop filters when switching data sources

* add experience var for switching to otel experience

* remove otel from variables and place near settings

* add error for non-standard prom with otel resources

* fix typescript errors, remove ts-ignores

* add custom variable for deployment environment like app-olly

* fix name of otel variable

* add function for getting otel resources from variables

* add otel join query const

* update standard check to be simpler

* allow for unstandard otel data sources but give warning

* add otelJoinQuery to the base query and clean up variables when state changes

* refactor otel functions to return filters for targets, use targets to filter metrics

* update metric names on otel target filter change

* when no otel targets for otel resource filter, show no metrics

* move switch to settings, default to use experience, refactor otel checks

* clean code

* fix refactor to add hasOtelResources for showing the switch in settings

* sort otel resources by blessed list

* reset otel when data source is changed

* move otel experience toggle back outside settings

* move showPreviews into settings

* do not re-add otel resources from blessed list to filters when already selected

* add otel join query variable to histogram base query

* only show settings for appropriate scenes

* show info tooltip the same but show error on hover for disabling otel exp for unstandard DS

* refactor tagKeys and tagValues for otel resources variable, fix promoted list ordering, fix dep env state bug

* default dep env value

* apply var filters only where they are using VAR_FILTER_EXPR in queryies

* change copy for labels to attributes

* do not group_left job label when already joining by job

* update copy for label variable when using otel

* remove isStandard check for now because of data staleness in Prometheus

* default to showing heatmap for histograms

* add trail history for selecting dep env and otel resources

* add otel resource attributes tests for DataTrail

* move otel functions to utils

* write tests for otel api calls

* write tests for otel utils functions

* fix history

* standard otel has target_info metric and deployment_environment resource attributes

* fix tests

* refactor otel functions for updating state and variables

* clean code

* fix tests

* fix tests

* mock checkDataSourceForOtelResources

* fix tests

* update query tests with otelJoinQuery and default to heatmap for _bucket metrics

* fix tests for otel api

* fix trail history test

* fix trail store tests for missing otel variables

* make i18n-extract

* handle target_info with inconsistent job and instance labels

* fix otel copy and <Trans> component

* fix custom variable deployment environment bug when switchiing data sources from non otel to otel

* fix linting error for trans component

* format i18nKey correctly

* clean up old comments

* add frontend hardening for OTel job and instance metric list filtering

* fix test for deployment environment custom variable to use changeValueTo

* fix i18n

* remove comments for fixed bug

* edit skipped tests

.betterer.results

@@ -5279,8 +5279,7 @@ exports[`better eslint`] = {
       [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "1"]
     ],
     "public/app/features/trails/DataTrailSettings.tsx:5381": [
-      [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"],
-      [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "1"]
+      [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"]
     ],
     "public/app/features/trails/DataTrailsHistory.tsx:5381": [
       [0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"]

public/app/features/trails/ActionTabs/BreakdownScene.tsx

@@ -211,13 +211,15 @@ export class BreakdownScene extends SceneObjectBase<BreakdownSceneState> {
     const { labels, body, loading, value, blockingMessage } = model.useState();
     const styles = useStyles2(getStyles);
 
+    const { useOtelExperience } = getTrailFor(model).useState();
+
     return (
       <div className={styles.container}>
         <StatusWrapper {...{ isLoading: loading, blockingMessage }}>
           <div className={styles.controls}>
             {!loading && labels.length && (
               <div className={styles.controlsLeft}>
-                <Field label="By label">
+                <Field label={useOtelExperience ? 'By metric attribute' : 'By label'}>
                   <BreakdownLabelSelector options={labels} value={value} onChange={model.onChange} />
                 </Field>
               </div>

public/app/features/trails/ActionTabs/MetricOverviewScene.tsx

@@ -68,6 +68,8 @@ export class MetricOverviewScene extends SceneObjectBase<MetricOverviewSceneStat
     const variable = model.getVariable();
     const { loading: labelsLoading, options: labelOptions } = variable.useState();
 
+    const { useOtelExperience } = getTrailFor(model).useState();
+
     // Get unit name from the metric name
     const metricScene = getMetricSceneFor(model);
     const metric = metricScene.state.metric;
@@ -110,7 +112,11 @@ export class MetricOverviewScene extends SceneObjectBase<MetricOverviewSceneStat
             </Stack>
             <Stack direction="column" gap={0.5}>
               <Text weight={'medium'}>
-                <Trans i18nKey="trails.metric-overview.labels-label">Labels</Trans>
+                {useOtelExperience ? (
+                  <Trans i18nKey="trails.metric-overview.metric-attributes">Metric attributes</Trans>
+                ) : (
+                  <Trans i18nKey="trails.metric-overview.labels">Labels</Trans>
+                )}
               </Text>
               {labelOptions.length === 0 && 'Unable to fetch labels.'}
               {labelOptions.map((l) => (

public/app/features/trails/AutomaticMetricQueries/query-generators/common/baseQuery.ts

@@ -1,8 +1,10 @@
-import { VAR_METRIC_EXPR, VAR_FILTERS_EXPR } from 'app/features/trails/shared';
+import { VAR_METRIC_EXPR, VAR_FILTERS_EXPR, VAR_OTEL_JOIN_QUERY_EXPR } from 'app/features/trails/shared';
 
 const GENERAL_BASE_QUERY = `${VAR_METRIC_EXPR}${VAR_FILTERS_EXPR}`;
 const GENERAL_RATE_BASE_QUERY = `rate(${GENERAL_BASE_QUERY}[$__rate_interval])`;
 
 export function getGeneralBaseQuery(rate: boolean) {
-  return rate ? GENERAL_RATE_BASE_QUERY : GENERAL_BASE_QUERY;
+  return rate
+    ? `${GENERAL_RATE_BASE_QUERY} ${VAR_OTEL_JOIN_QUERY_EXPR}`
+    : `${GENERAL_BASE_QUERY} ${VAR_OTEL_JOIN_QUERY_EXPR}`;
 }

public/app/features/trails/AutomaticMetricQueries/query-generators/histogram.ts

@@ -1,6 +1,6 @@
 import { PromQuery } from '@grafana/prometheus';
 
-import { VAR_FILTERS_EXPR, VAR_GROUP_BY_EXP, VAR_METRIC_EXPR } from '../../shared';
+import { VAR_FILTERS_EXPR, VAR_GROUP_BY_EXP, VAR_METRIC_EXPR, VAR_OTEL_JOIN_QUERY_EXPR } from '../../shared';
 import { heatmapGraphBuilder } from '../graph-builders/heatmap';
 import { percentilesGraphBuilder } from '../graph-builders/percentiles';
 import { simpleGraphBuilder } from '../graph-builders/simple';
@@ -47,10 +47,10 @@ export function createHistogramMetricQueryDefs(metricParts: string[]) {
     vizBuilder: () => heatmapGraphBuilder(heatmap),
   };
 
-  return { preview: p50, main: percentiles, variants: [percentiles, heatmap], breakdown: breakdown };
+  return { preview: heatmap, main: heatmap, variants: [percentiles, heatmap], breakdown: breakdown };
 }
 
-const BASE_QUERY = `rate(${VAR_METRIC_EXPR}${VAR_FILTERS_EXPR}[$__rate_interval])`;
+const BASE_QUERY = `rate(${VAR_METRIC_EXPR}${VAR_FILTERS_EXPR}[$__rate_interval])${VAR_OTEL_JOIN_QUERY_EXPR}`;
 
 function baseQuery(groupings: string[] = []) {
   const sumByList = ['le', ...groupings];

public/app/features/trails/DataTrail.test.tsx

@@ -1,5 +1,6 @@
+import { VariableHide } from '@grafana/data';
 import { locationService, setDataSourceSrv } from '@grafana/runtime';
-import { AdHocFiltersVariable, sceneGraph } from '@grafana/scenes';
+import { AdHocFiltersVariable, ConstantVariable, CustomVariable, sceneGraph } from '@grafana/scenes';
 import { DataSourceType } from 'app/features/alerting/unified/utils/datasource';
 
 import { MockDataSourceSrv, mockDataSource } from '../alerting/unified/mocks';
@@ -8,10 +9,23 @@ import { activateFullSceneTree } from '../dashboard-scene/utils/test-utils';
 import { DataTrail } from './DataTrail';
 import { MetricScene } from './MetricScene';
 import { MetricSelectScene } from './MetricSelect/MetricSelectScene';
-import { MetricSelectedEvent, VAR_FILTERS } from './shared';
+import {
+  MetricSelectedEvent,
+  VAR_FILTERS,
+  VAR_OTEL_DEPLOYMENT_ENV,
+  VAR_OTEL_JOIN_QUERY,
+  VAR_OTEL_RESOURCES,
+} from './shared';
+
+jest.mock('./otel/api', () => ({
+  totalOtelResources: jest.fn(() => ({ job: 'oteldemo', instance: 'instance' })),
+  getDeploymentEnvironments: jest.fn(() => ['production', 'staging']),
+  isOtelStandardization: jest.fn(() => true),
+}));
 
 describe('DataTrail', () => {
   beforeAll(() => {
+    jest.spyOn(DataTrail.prototype, 'checkDataSourceForOTelResources').mockImplementation(() => Promise.resolve());
     setDataSourceSrv(
       new MockDataSourceSrv({
         prom: mockDataSource({
@@ -22,6 +36,10 @@ describe('DataTrail', () => {
     );
   });
 
+  afterAll(() => {
+    jest.restoreAllMocks();
+  });
+
   describe('Given starting non-embedded trail with url sync and no url state', () => {
     let trail: DataTrail;
     const preTrailUrl = '/';
@@ -459,4 +477,74 @@ describe('DataTrail', () => {
       });
     });
   });
+
+  describe('OTel resources attributes', () => {
+    let trail: DataTrail;
+    const preTrailUrl =
+      '/trail?from=now-1h&to=now&var-ds=edwxqcebl0cg0c&var-deployment_environment=oteldemo01&var-otel_resources=k8s_cluster_name%7C%3D%7Cappo11ydev01&var-filters=&refresh=&metricPrefix=all&metricSearch=http&actionView=breakdown&var-groupby=$__all&metric=http_client_duration_milliseconds_bucket';
+
+    function getOtelDepEnvVar(trail: DataTrail) {
+      const variable = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, trail);
+      if (variable instanceof CustomVariable) {
+        return variable;
+      }
+      throw new Error('getDepEnvVar failed');
+    }
+
+    function getOtelJoinQueryVar(trail: DataTrail) {
+      const variable = sceneGraph.lookupVariable(VAR_OTEL_JOIN_QUERY, trail);
+      if (variable instanceof ConstantVariable) {
+        return variable;
+      }
+      throw new Error('getDepEnvVar failed');
+    }
+
+    function getOtelResourcesVar(trail: DataTrail) {
+      const variable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, trail);
+      if (variable instanceof AdHocFiltersVariable) {
+        return variable;
+      }
+      throw new Error('getOtelResourcesVar failed');
+    }
+
+    beforeEach(() => {
+      trail = new DataTrail({});
+      locationService.push(preTrailUrl);
+      activateFullSceneTree(trail);
+      getOtelResourcesVar(trail).setState({ filters: [{ key: 'service_name', operator: '=', value: 'adservice' }] });
+      getOtelDepEnvVar(trail).changeValueTo('production');
+    });
+
+    it('should start with hidden dep env variable', () => {
+      const depEnvVarHide = getOtelDepEnvVar(trail).state.hide;
+      expect(depEnvVarHide).toBe(VariableHide.hideVariable);
+    });
+
+    it('should start with hidden otel resources variable', () => {
+      const resourcesVarHide = getOtelResourcesVar(trail).state.hide;
+      expect(resourcesVarHide).toBe(VariableHide.hideVariable);
+    });
+
+    it('should start with hidden otel join query variable', () => {
+      const joinQueryVarHide = getOtelJoinQueryVar(trail).state.hide;
+      expect(joinQueryVarHide).toBe(VariableHide.hideVariable);
+    });
+
+    it('should add history step for when updating the otel resource variable', () => {
+      expect(trail.state.history.state.steps[2].type).toBe('resource');
+    });
+
+    it('Should have otel resource attribute selected as "service_name=adservice"', () => {
+      expect(getOtelResourcesVar(trail).state.filters[0].key).toBe('service_name');
+      expect(getOtelResourcesVar(trail).state.filters[0].value).toBe('adservice');
+    });
+
+    it('Should have deployment environment selected as "production"', () => {
+      expect(getOtelDepEnvVar(trail).getValue()).toBe('production');
+    });
+
+    it('should add history step for when updating the dep env variable', () => {
+      expect(trail.state.history.state.steps[3].type).toBe('dep_env');
+    });
+  });
 });