Merge branch 'main' into renovate/main-oas

Author: kgeller

.github/CODEOWNERS

@@ -1204,6 +1204,7 @@ x-pack/platform/test/serverless/api_integration/test_suites/platform_security @e
 /src/platform/test/functional/fixtures/kbn_archiver/discover.json @elastic/kibana-data-discovery
 /src/platform/test/functional/fixtures/kbn_archiver/date_nested.json @elastic/kibana-data-discovery
 /src/platform/test/functional/fixtures/kbn_archiver/date_* @elastic/kibana-data-discovery
+/src/platform/test/functional/fixtures/kbn_archiver/managed_data_view.json @elastic/kibana-data-discovery
 /src/platform/test/functional/fixtures/es_archiver/unmapped_fields @elastic/kibana-data-discovery # Assigned per the only use: https://github.com/elastic/kibana/blob/main/test/functional/apps/discover/group7/_indexpattern_with_unmapped_fields.ts#L26
 /src/platform/test/functional/fixtures/es_archiver/message_with_newline @elastic/kibana-data-discovery # Assigned per the only use: https://github.com/elastic/kibana/blob/main/test/functional/apps/discover/classic/_doc_table_newline.ts#L24
 /src/platform/test/functional/fixtures/es_archiver/hamlet @elastic/kibana-data-discovery # Assigned per the only use: https://github.com/elastic/kibana/blob/main/test/functional/apps/discover/group5/_large_string.ts#L30

packages/kbn-optimizer/limits.yml

@@ -47,7 +47,7 @@ pageLoadAssetSize:
   embeddableAlertsTable: 6596
   embeddableEnhanced: 10332
   enterpriseSearch: 38494
-  esql: 12323
+  esql: 13323
   esqlDataGrid: 10209
   esUiShared: 100338
   eventAnnotation: 22361

src/platform/packages/private/kbn-esql-editor/src/custom_commands/use_lookup_index_editor.test.tsx

@@ -70,14 +70,14 @@ describe('getMonacoCommandString', () => {
       canEditIndex: true,
     });
     expect(result).toBe(
-      '[Create lookup index](command:esql.lookup_index.create?%7B%22indexName%22%3A%22test-index%22%2C%22doesIndexExist%22%3Afalse%2C%22canEditIndex%22%3Atrue%7D)'
+      '[Create lookup index](command:esql.lookup_index.create?%7B%22indexName%22%3A%22test-index%22%2C%22doesIndexExist%22%3Afalse%2C%22canEditIndex%22%3Atrue%2C%22triggerSource%22%3A%22esql_hover%22%7D)'
     );
   });
 
   it('should return edit command for existing index with edit permissions', () => {
     const result = getMonacoCommandString('test-index', true, { canEditIndex: true });
     expect(result).toBe(
-      '[Edit lookup index](command:esql.lookup_index.create?%7B%22indexName%22%3A%22test-index%22%2C%22doesIndexExist%22%3Atrue%2C%22canEditIndex%22%3Atrue%7D)'
+      '[Edit lookup index](command:esql.lookup_index.create?%7B%22indexName%22%3A%22test-index%22%2C%22doesIndexExist%22%3Atrue%2C%22canEditIndex%22%3Atrue%2C%22triggerSource%22%3A%22esql_hover%22%7D)'
     );
   });
 
@@ -87,7 +87,7 @@ describe('getMonacoCommandString', () => {
       canEditIndex: false,
     });
     expect(result).toBe(
-      '[View lookup index](command:esql.lookup_index.create?%7B%22indexName%22%3A%22test-index%22%2C%22doesIndexExist%22%3Atrue%2C%22canEditIndex%22%3Afalse%7D)'
+      '[View lookup index](command:esql.lookup_index.create?%7B%22indexName%22%3A%22test-index%22%2C%22doesIndexExist%22%3Atrue%2C%22canEditIndex%22%3Afalse%2C%22triggerSource%22%3A%22esql_hover%22%7D)'
     );
   });
 
@@ -252,7 +252,7 @@ describe('useLookupIndexCommand', () => {
           options: expect.objectContaining({
             hoverMessage: expect.objectContaining({
               value:
-                '[Create lookup index](command:esql.lookup_index.create?%7B%22indexName%22%3A%22test-index%22%2C%22doesIndexExist%22%3Afalse%2C%22canEditIndex%22%3Afalse%7D)',
+                '[Create lookup index](command:esql.lookup_index.create?%7B%22indexName%22%3A%22test-index%22%2C%22doesIndexExist%22%3Afalse%2C%22canEditIndex%22%3Afalse%2C%22triggerSource%22%3A%22esql_hover%22%7D)',
             }),
           }),
         }),

src/platform/packages/private/kbn-esql-editor/src/custom_commands/use_lookup_index_editor.tsx

@@ -79,6 +79,7 @@ export function getMonacoCommandString(
       indexName,
       doesIndexExist: isExistingIndex,
       canEditIndex,
+      triggerSource: 'esql_hover',
     })
   )})`;
 }
@@ -254,11 +255,17 @@ export const useLookupIndexCommand = (
   );
 
   const openFlyout = useCallback(
-    async (indexName: string, doesIndexExist?: boolean, canEditIndex = true) => {
+    async (
+      indexName: string,
+      doesIndexExist?: boolean,
+      canEditIndex = true,
+      triggerSource = 'esql_autocomplete'
+    ) => {
       await uiActions.getTrigger('EDIT_LOOKUP_INDEX_CONTENT_TRIGGER_ID').exec({
         indexName,
         doesIndexExist,
         canEditIndex,
+        triggerSource,
         onClose: async ({ indexName: resultIndexName, indexCreatedDuringFlyout }) => {
           await onFlyoutClose(indexName, resultIndexName, indexCreatedDuringFlyout);
         },
@@ -275,9 +282,17 @@ export const useLookupIndexCommand = (
   useEffect(function registerCommandOnMount() {
     const disposable = monaco.editor.registerCommand(
       COMMAND_ID,
-      async (_, args: { indexName: string; doesIndexExist?: boolean; canEditIndex?: boolean }) => {
-        const { indexName, doesIndexExist, canEditIndex } = args;
-        await openFlyoutRef.current(indexName, doesIndexExist, canEditIndex);
+      async (
+        _,
+        args: {
+          indexName: string;
+          doesIndexExist?: boolean;
+          canEditIndex?: boolean;
+          triggerSource?: string;
+        }
+      ) => {
+        const { indexName, doesIndexExist, canEditIndex, triggerSource } = args;
+        await openFlyoutRef.current(indexName, doesIndexExist, canEditIndex, triggerSource);
       }
     );
     return () => {

src/platform/packages/private/kbn-index-editor/README.md

@@ -45,6 +45,10 @@ import {
 } from '@kbn/index-editor';
 import type { EditLookupIndexFlyoutDeps } from '@kbn/index-editor';
 
+// Register the index editor telemetry events inside setup
+registerIndexEditorAnalyticsEvents(core.analytics);
+
+
 // Register the index editor actions with required dependencies
 const deps: EditLookupIndexFlyoutDeps = {
   coreStart,
@@ -69,9 +73,10 @@ import type { EditLookupIndexContentContext } from '@kbn/index-editor';
 
 // Create context for the index editor
 const context: EditLookupIndexContentContext = {
-  indexName: 'my-lookup-index', // Set to 
+  indexName: 'my-lookup-index',
   doesIndexExist: true,
   canEditIndex: true,
+  triggerSource: 'your_source', // Used only for telemetry
   onClose: (result) => {
     console.log('Index editor closed:', result);
   }

src/platform/packages/private/kbn-index-editor/index.ts

@@ -13,4 +13,7 @@ export {
   ACTION_EDIT_LOOKUP_INDEX,
   registerIndexEditorActions,
 } from './src/ui_action';
+
+export { registerIndexEditorAnalyticsEvents } from './src/telemetry/events_registration';
+
 export type { EditLookupIndexContentContext, EditLookupIndexFlyoutDeps } from './src/types';

src/platform/packages/private/kbn-index-editor/src/components/create_flyout.test.tsx

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { createFlyout } from './create_flyout';
+import { BehaviorSubject, of, Subject } from 'rxjs';
+import {
+  httpServiceMock,
+  notificationServiceMock,
+  overlayServiceMock,
+} from '@kbn/core/public/mocks';
+import type { FlyoutDeps } from '../types';
+import { IndexEditorTelemetryService } from '../telemetry/telemetry_service';
+import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
+
+describe('createFlyout', () => {
+  const coreStart = {
+    http: httpServiceMock.createStartContract(),
+    overlays: overlayServiceMock.createStartContract(),
+    notifications: notificationServiceMock.createStartContract(),
+    data: dataPluginMock.createStartContract(),
+    application: {
+      currentAppId$: new BehaviorSubject('my-app'),
+    },
+  };
+
+  const indexEditorTelemetryService = new IndexEditorTelemetryService(
+    {
+      reportEvent: jest.fn(),
+    } as any,
+    true,
+    true,
+    'test-source'
+  );
+
+  const indexUpdateService = {
+    setIndexName: jest.fn(),
+    setIndexCreated: jest.fn(),
+    exit: jest.fn(),
+    completed$: of(),
+    totalHits$: new BehaviorSubject<number>(0),
+    dataTableColumns$: new Subject<any[]>(),
+  };
+
+  const deps: FlyoutDeps = {
+    coreStart: coreStart as any,
+    data: coreStart.data,
+    indexUpdateService: indexUpdateService as any,
+    indexEditorTelemetryService,
+    uiActions: {} as any,
+    fieldFormats: {} as any,
+    share: {} as any,
+    fileUpload: {} as any,
+    storage: {} as any,
+    fileManager: {} as any,
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    indexUpdateService.totalHits$.next(0);
+  });
+
+  it('should call trackFlyoutOpened with correct data when index exists', async () => {
+    const trackFlyoutOpenedSpy = jest.spyOn(deps.indexEditorTelemetryService, 'trackFlyoutOpened');
+
+    createFlyout(deps, {
+      doesIndexExist: true,
+      indexName: 'test-index',
+      canEditIndex: true,
+      triggerSource: 'test-source',
+    });
+
+    indexUpdateService.totalHits$.next(10);
+    indexUpdateService.dataTableColumns$.next([{ name: 'field1' }, { name: 'field2' }]);
+
+    expect(trackFlyoutOpenedSpy).toHaveBeenCalledWith({
+      docCount: 10,
+      fieldCount: 2,
+    });
+  });
+
+  it('should call trackFlyoutOpened with zero counts when index does not exist', () => {
+    const trackFlyoutOpenedSpy = jest.spyOn(deps.indexEditorTelemetryService, 'trackFlyoutOpened');
+
+    createFlyout(deps, {
+      doesIndexExist: false,
+      canEditIndex: true,
+      triggerSource: 'test-source',
+    });
+
+    expect(trackFlyoutOpenedSpy).toHaveBeenCalledWith({
+      docCount: 0,
+      fieldCount: 0,
+    });
+  });
+});

src/platform/packages/private/kbn-index-editor/src/components/create_flyout.tsx

@@ -12,8 +12,9 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import { toMountPoint } from '@kbn/react-kibana-mount';
 import type { FC } from 'react';
 import React, { Suspense, lazy } from 'react';
-import { distinctUntilChanged, from, skip, takeUntil } from 'rxjs';
+import { combineLatest, distinctUntilChanged, first, from, skip, takeUntil } from 'rxjs';
 import type { EditLookupIndexContentContext, FlyoutDeps } from '../types';
+import { isPlaceholderColumn } from '../utils';
 
 export function createFlyout(deps: FlyoutDeps, props: EditLookupIndexContentContext) {
   const {
@@ -31,6 +32,23 @@ export function createFlyout(deps: FlyoutDeps, props: EditLookupIndexContentCont
   }
   indexUpdateService.setIndexCreated(props.doesIndexExist);
 
+  // Track telemetry event when flyout is opened
+  if (props.doesIndexExist) {
+    combineLatest([
+      indexUpdateService.totalHits$.pipe(skip(1)), // skip initial value
+      indexUpdateService.dataTableColumns$,
+    ])
+      .pipe(first())
+      .subscribe(([docCount, columns]) => {
+        deps.indexEditorTelemetryService.trackFlyoutOpened({
+          docCount,
+          fieldCount: columns.filter((c) => !isPlaceholderColumn(c.name)).length,
+        });
+      });
+  } else {
+    deps.indexEditorTelemetryService.trackFlyoutOpened({ docCount: 0, fieldCount: 0 });
+  }
+
   const LazyFlyoutContent = lazy(async () => {
     const { FlyoutContent } = await import('./flyout_content');
     return {

src/platform/packages/private/kbn-index-editor/src/index_update_service.test.ts

@@ -14,20 +14,48 @@ import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import { ROW_PLACEHOLDER_PREFIX } from './constants';
 import { IndexUpdateService } from './index_update_service';
 import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
-import { httpServiceMock, notificationServiceMock } from '@kbn/core/public/mocks';
+import {
+  analyticsServiceMock,
+  httpServiceMock,
+  notificationServiceMock,
+} from '@kbn/core/public/mocks';
+import { IndexEditorTelemetryService } from './telemetry/telemetry_service';
+import type { AnalyticsServiceStart } from '@kbn/core/server';
+import { getESQLAdHocDataview } from '@kbn/esql-utils';
+
+jest.mock('@kbn/esql-utils', () => ({
+  getESQLAdHocDataview: jest.fn(),
+}));
 
 describe('IndexUpdateService', () => {
   let http: HttpStart;
   let data: DataPublicPluginStart;
   let service: IndexUpdateService;
   let notifications: NotificationsStart;
+  let analytics: AnalyticsServiceStart;
+  let indexEditorTelemetryService: IndexEditorTelemetryService;
 
   beforeEach(() => {
     http = httpServiceMock.createStartContract();
     data = dataPluginMock.createStartContract();
     notifications = notificationServiceMock.createStartContract();
+    analytics = analyticsServiceMock.createAnalyticsServiceStart();
+    indexEditorTelemetryService = new IndexEditorTelemetryService(
+      analytics,
+      true,
+      true,
+      'esql_hover'
+    );
+
+    (getESQLAdHocDataview as jest.Mock).mockResolvedValue({
+      fields: {
+        getByName: () => {},
+        create: () => ({}),
+        concat: () => [],
+      },
+    });
 
-    service = new IndexUpdateService(http, data, notifications, true);
+    service = new IndexUpdateService(http, data, notifications, indexEditorTelemetryService, true);
   });
 
   afterEach(() => {
@@ -179,4 +207,51 @@ describe('IndexUpdateService', () => {
     expect(rowsAfterDeletion.length).toBe(1); // An empty placeholder row should always be visible
     expect(rowsAfterDeletion[0].raw).toMatchObject({ _id: expect.anything() });
   });
+
+  describe('flush operations', () => {
+    it('should call telemetry on successful flush', async () => {
+      (http.post as jest.Mock).mockResolvedValue({
+        errors: false,
+        items: [],
+        took: 0,
+      } satisfies BulkResponse);
+      const telemetrySpy = jest.spyOn(indexEditorTelemetryService, 'trackSaveSubmitted');
+
+      service.setIndexName('my-index');
+      service.setIndexCreated(true);
+      await firstValueFrom(service.dataView$);
+
+      // Adding and modifying a new row counts as 1 row added and 0 cells edited
+      service.addEmptyRow();
+      const placeholderRow = (await firstValueFrom(service.rows$))[0];
+      service.updateDoc(placeholderRow.id, { field: 'value' });
+
+      // Adding a column and editing its name counts as 1 col added
+      service.addNewColumn();
+      const newColumn = (await firstValueFrom(service.pendingColumnsToBeSaved$))[0];
+      service.editColumn('newColumn', newColumn.name);
+
+      // Counts as 2 cell edited
+      service.updateDoc('123', { field: 'value' });
+      service.updateDoc('123', { newColumn: 'value' });
+
+      // Wait for the changes to be registered
+      await firstValueFrom(service.hasUnsavedChanges$);
+
+      // Save the changes
+      service.flush();
+
+      // wait for async operations to complete
+      await new Promise((resolve) => process.nextTick(resolve));
+
+      expect(telemetrySpy).toHaveBeenCalledWith({
+        pendingRowsAdded: 1,
+        pendingColsAdded: 1,
+        pendingCellsEdited: 2,
+        action: 'save',
+        outcome: 'success',
+        latency: expect.any(Number),
+      });
+    });
+  });
 });