[Index Management] Added docs count and size for serverless (#191985)

## Summary Fixes #190131 This PR adds size and documents count to indices and data streams tables in Index Management on serverless. ### Screenrecording https://github.com/user-attachments/assets/51a933e2-e4ef-42a0-9c82-39bf6e194ee0 ### Screenshots <img width="1047" alt="Screenshot 2024-09-06 at 19 20 59" src="https://github.com/user-attachments/assets/8c6d0378-116f-44e3-aaca-0c95f135b1bf"> <img width="1045" alt="Screenshot 2024-09-06 at 19 21 06" src="https://github.com/user-attachments/assets/37477f7b-e229-4400-9a28-8382f7d1155e"> <img width="1036" alt="Screenshot 2024-09-06 at 19 27 59" src="https://github.com/user-attachments/assets/0fd2ef1b-3b0c-4d3e-8b56-984281436898"> <img width="506" alt="Screenshot 2024-09-06 at 19 28 12" src="https://github.com/user-attachments/assets/1823a964-6f6f-464b-9910-e586ccd2e9bb"> ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
elastic · Sep 16, 2024 · 3de252e · 3de252e
1 parent f168fb4
commit 3de252e
Show file tree

Hide file tree

Showing 26 changed files with 531 additions and 215 deletions.
diff --git a/test/plugin_functional/test_suites/core_plugins/rendering.ts b/test/plugin_functional/test_suites/core_plugins/rendering.ts
@@ -295,6 +295,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.index_management.enableLegacyTemplates (boolean?|never)',
         'xpack.index_management.enableIndexStats (boolean?|never)',
         'xpack.index_management.enableDataStreamStats (boolean?|never)',
+        'xpack.index_management.enableSizeAndDocCount (boolean?|never)',
         'xpack.index_management.editableIndexSettings (all?|limited?|never)',
         'xpack.index_management.enableMappingsSourceFieldSection (boolean?|never)',
         'xpack.index_management.dev.enableSemanticText (boolean?)',

diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts
@@ -24,7 +24,7 @@ export interface DataStreamsTabTestBed extends TestBed<TestSubjects> {
   actions: {
     goToDataStreamsList: () => void;
     clickEmptyPromptIndexTemplateLink: () => void;
-    clickIncludeStatsSwitch: () => void;
+    clickIncludeStatsSwitch: () => Promise<void>;
     toggleViewFilterAt: (index: number) => void;
     sortTableOnStorageSize: () => void;
     sortTableOnName: () => void;
@@ -90,9 +90,13 @@ export const setup = async (
     component.update();
   };
 
-  const clickIncludeStatsSwitch = () => {
-    const { find } = testBed;
-    find('includeStatsSwitch').simulate('click');
+  const clickIncludeStatsSwitch = async () => {
+    const { find, component } = testBed;
+
+    await act(async () => {
+      find('includeStatsSwitch').simulate('click');
+    });
+    component.update();
   };
 
   const toggleViewFilterAt = (index: number) => {

diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts
@@ -156,6 +156,10 @@ describe('Data Streams tab', () => {
         name: 'dataStream1',
         storageSize: '5b',
         storageSizeBytes: 5,
+        // metering API mock
+        meteringStorageSize: '156kb',
+        meteringStorageSizeBytes: 156000,
+        meteringDocsCount: 10000,
       });
 
       setLoadDataStreamsResponse([
@@ -164,6 +168,10 @@ describe('Data Streams tab', () => {
           name: 'dataStream2',
           storageSize: '1kb',
           storageSizeBytes: 1000,
+          // metering API mock
+          meteringStorageSize: '156kb',
+          meteringStorageSizeBytes: 156000,
+          meteringDocsCount: 10000,
           lifecycle: {
             enabled: true,
             data_retention: '7d',
@@ -224,15 +232,12 @@ describe('Data Streams tab', () => {
     });
 
     test('has a switch that will reload the data streams with additional stats when clicked', async () => {
-      const { exists, actions, table, component } = testBed;
+      const { exists, actions, table } = testBed;
 
       expect(exists('includeStatsSwitch')).toBe(true);
 
       // Changing the switch will automatically reload the data streams.
-      await act(async () => {
-        actions.clickIncludeStatsSwitch();
-      });
-      component.update();
+      await actions.clickIncludeStatsSwitch();
 
       expect(httpSetup.get).toHaveBeenLastCalledWith(
         `${API_BASE_PATH}/data_streams`,
@@ -267,12 +272,9 @@ describe('Data Streams tab', () => {
 
     test('sorting on stats sorts by bytes value instead of human readable value', async () => {
       // Guards against regression of #86122.
-      const { actions, table, component } = testBed;
+      const { actions, table } = testBed;
 
-      await act(async () => {
-        actions.clickIncludeStatsSwitch();
-      });
-      component.update();
+      await actions.clickIncludeStatsSwitch();
 
       actions.sortTableOnStorageSize();
 
@@ -306,7 +308,7 @@ describe('Data Streams tab', () => {
       actions.sortTableOnName();
     });
 
-    test('hides stats toggle if enableDataStreamStats===false', async () => {
+    test(`doesn't hide stats toggle if enableDataStreamStats===false`, async () => {
       testBed = await setup(httpSetup, {
         config: {
           enableDataStreamStats: false,
@@ -321,14 +323,82 @@ describe('Data Streams tab', () => {
 
       component.update();
 
-      expect(exists('includeStatsSwitch')).toBeFalsy();
+      expect(exists('includeStatsSwitch')).toBeTruthy();
+    });
+
+    test('shows storage size and documents count if enableSizeAndDocCount===true, enableDataStreamStats==false', async () => {
+      testBed = await setup(httpSetup, {
+        config: {
+          enableSizeAndDocCount: true,
+          enableDataStreamStats: false,
+        },
+      });
+
+      const { actions, component, table } = testBed;
+
+      await act(async () => {
+        actions.goToDataStreamsList();
+      });
+
+      component.update();
+
+      await actions.clickIncludeStatsSwitch();
+
+      const { tableCellsValues } = table.getMetaData('dataStreamTable');
+      expect(tableCellsValues).toEqual([
+        ['', 'dataStream1', 'green', '156kb', '10000', '1', '7 days', 'Delete'],
+        ['', 'dataStream2', 'green', '156kb', '10000', '1', '5 days ', 'Delete'],
+      ]);
+    });
+
+    test('shows last updated and storage size if enableDataStreamStats===true, enableSizeAndDocCount===false', async () => {
+      testBed = await setup(httpSetup, {
+        config: {
+          enableDataStreamStats: true,
+          enableSizeAndDocCount: false,
+        },
+      });
+
+      const { actions, component, table } = testBed;
+
+      await act(async () => {
+        actions.goToDataStreamsList();
+      });
+
+      component.update();
+
+      await actions.clickIncludeStatsSwitch();
+
+      const { tableCellsValues } = table.getMetaData('dataStreamTable');
+      expect(tableCellsValues).toEqual([
+        [
+          '',
+          'dataStream1',
+          'green',
+          'December 31st, 1969 7:00:00 PM',
+          '5b',
+          '1',
+          '7 days',
+          'Delete',
+        ],
+        [
+          '',
+          'dataStream2',
+          'green',
+          'December 31st, 1969 7:00:00 PM',
+          '1kb',
+          '1',
+          '5 days ',
+          'Delete',
+        ],
+      ]);
     });
 
     test('clicking the indices count navigates to the backing indices', async () => {
       const { table, actions } = testBed;
       await actions.clickIndicesAt(0);
       expect(table.getMetaData('indexTable').tableCellsValues).toEqual([
-        ['', 'data-stream-index', '', '', '', '', '', '', 'dataStream1'],
+        ['', 'data-stream-index', '', '', '', '', '0', '', 'dataStream1'],
       ]);
     });
 
@@ -707,7 +777,7 @@ describe('Data Streams tab', () => {
         const { table, actions } = testBed;
         await actions.clickIndicesAt(0);
         expect(table.getMetaData('indexTable').tableCellsValues).toEqual([
-          ['', 'data-stream-index', '', '', '', '', '', '', '%dataStream'],
+          ['', 'data-stream-index', '', '', '', '', '0', '', '%dataStream'],
         ]);
       });
     });

diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.test.tsx b/x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.test.tsx
@@ -394,7 +394,7 @@ describe('<IndexManagementHome />', () => {
       component.update();
     });
 
-    test('renders the table column with index stats by default', () => {
+    test('renders the table column with all index stats when enableIndexStats is true', () => {
       const { table } = testBed;
       const { tableCellsValues } = table.getMetaData('indexTable');
 
@@ -403,14 +403,15 @@ describe('<IndexManagementHome />', () => {
       ]);
     });
 
-    describe('Disabled', () => {
+    describe('renders only size and docs count when enableIndexStats is false, enableSizeAndDocCount is true', () => {
       beforeEach(async () => {
         await act(async () => {
           testBed = await setup(httpSetup, {
             config: {
               enableLegacyTemplates: true,
               enableIndexActions: true,
               enableIndexStats: false,
+              enableSizeAndDocCount: true,
             },
           });
         });
@@ -420,7 +421,33 @@ describe('<IndexManagementHome />', () => {
         component.update();
       });
 
-      test('hides index stats information from table', async () => {
+      test('hides some index stats information from table', async () => {
+        const { table } = testBed;
+        const { tableCellsValues } = table.getMetaData('indexTable');
+
+        expect(tableCellsValues).toEqual([['', 'test', '10,000', '156kb', '']]);
+      });
+    });
+
+    describe('renders no index stats when enableIndexStats is false, enableSizeAndDocCount is false', () => {
+      beforeEach(async () => {
+        await act(async () => {
+          testBed = await setup(httpSetup, {
+            config: {
+              enableLegacyTemplates: true,
+              enableIndexActions: true,
+              enableIndexStats: false,
+              enableSizeAndDocCount: false,
+            },
+          });
+        });
+
+        const { component } = testBed;
+
+        component.update();
+      });
+
+      test('hides all index stats information from table', async () => {
         const { table } = testBed;
         const { tableCellsValues } = table.getMetaData('indexTable');
 

diff --git a/...mon/lib/data_stream_serialization.test.ts → ...ment/common/lib/data_stream_utils.test.ts b/...mon/lib/data_stream_serialization.test.ts → ...ment/common/lib/data_stream_utils.test.ts
@@ -5,9 +5,9 @@
  * 2.0.
  */
 
-import { splitSizeAndUnits } from './data_stream_serialization';
+import { splitSizeAndUnits } from './data_stream_utils';
 
-describe('Data stream serialization', () => {
+describe('Data stream utils', () => {
   test('can split size and units from lifecycle string', () => {
     expect(splitSizeAndUnits('1h')).toEqual({ size: '1', unit: 'h' });
     expect(splitSizeAndUnits('20micron')).toEqual({ size: '20', unit: 'micron' });

diff --git a/x-pack/plugins/index_management/common/lib/data_stream_utils.ts b/x-pack/plugins/index_management/common/lib/data_stream_utils.ts
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataStream, DataRetention } from '../types';
+
+export const splitSizeAndUnits = (field: string): { size: string; unit: string } => {
+  let size = '';
+  let unit = '';
+
+  const result = /(\d+)(\w+)/.exec(field);
+  if (result) {
+    size = result[1];
+    unit = result[2];
+  }
+
+  return {
+    size,
+    unit,
+  };
+};
+
+export const serializeAsESLifecycle = (lifecycle?: DataRetention): DataStream['lifecycle'] => {
+  if (!lifecycle || !lifecycle?.enabled) {
+    return undefined;
+  }
+
+  const { infiniteDataRetention, value, unit } = lifecycle;
+
+  if (infiniteDataRetention) {
+    return {
+      enabled: true,
+    };
+  }
+
+  return {
+    enabled: true,
+    data_retention: `${value}${unit}`,
+  };
+};
+
+export const deserializeESLifecycle = (lifecycle?: DataStream['lifecycle']): DataRetention => {
+  if (!lifecycle || !lifecycle?.enabled) {
+    return { enabled: false };
+  }
+
+  if (!lifecycle.data_retention) {
+    return {
+      enabled: true,
+      infiniteDataRetention: true,
+    };
+  }
+
+  const { size, unit } = splitSizeAndUnits(lifecycle.data_retention as string);
+
+  return {
+    enabled: true,
+    value: Number(size),
+    unit,
+  };
+};
diff --git a/x-pack/plugins/index_management/common/lib/index.ts b/x-pack/plugins/index_management/common/lib/index.ts
@@ -6,10 +6,10 @@
  */
 
 export {
-  deserializeDataStream,
-  deserializeDataStreamList,
   splitSizeAndUnits,
-} from './data_stream_serialization';
+  serializeAsESLifecycle,
+  deserializeESLifecycle,
+} from './data_stream_utils';
 
 export {
   deserializeTemplate,

diff --git a/x-pack/plugins/index_management/common/lib/template_serialization.ts b/x-pack/plugins/index_management/common/lib/template_serialization.ts
@@ -12,7 +12,7 @@ import {
   TemplateListItem,
   TemplateType,
 } from '../types';
-import { deserializeESLifecycle } from './data_stream_serialization';
+import { deserializeESLifecycle } from './data_stream_utils';
 import { allowAutoCreateRadioValues, allowAutoCreateRadioIds } from '../constants';
 
 const hasEntries = (data: object = {}) => Object.entries(data).length > 0;

diff --git a/x-pack/plugins/index_management/common/types/data_streams.ts b/x-pack/plugins/index_management/common/types/data_streams.ts
@@ -37,6 +37,8 @@ export interface EnhancedDataStreamFromEs extends IndicesDataStream {
   store_size?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size'];
   store_size_bytes?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size_bytes'];
   maximum_timestamp?: IndicesDataStreamsStatsDataStreamsStatsItem['maximum_timestamp'];
+  metering_size_in_bytes?: number;
+  metering_doc_count?: number;
   indices: DataStreamIndexFromEs[];
   privileges: {
     delete_index: boolean;
@@ -55,6 +57,9 @@ export interface DataStream {
   storageSize?: ByteSize;
   storageSizeBytes?: number;
   maxTimeStamp?: number;
+  meteringStorageSizeBytes?: number;
+  meteringStorageSize?: string;
+  meteringDocsCount?: number;
   _meta?: Metadata;
   privileges: Privileges;
   hidden: boolean;

diff --git a/...onents/component_templates/__jest__/client_integration/component_template_create.test.tsx b/...onents/component_templates/__jest__/client_integration/component_template_create.test.tsx
@@ -12,7 +12,7 @@ import { breadcrumbService, IndexManagementBreadcrumb } from '../../../../servic
 import { setupEnvironment } from './helpers';
 import { API_BASE_PATH } from './helpers/constants';
 import { setup, ComponentTemplateCreateTestBed } from './helpers/component_template_create.helpers';
-import { serializeAsESLifecycle } from '../../../../../../common/lib/data_stream_serialization';
+import { serializeAsESLifecycle } from '../../../../../../common/lib';
 
 jest.mock('@kbn/code-editor', () => {
   const original = jest.requireActual('@kbn/code-editor');

diff --git a/...t_templates/component_template_wizard/component_template_form/component_template_form.tsx b/...t_templates/component_template_wizard/component_template_form/component_template_form.tsx
@@ -19,10 +19,7 @@ import {
   StepMappingsContainer,
   StepAliasesContainer,
 } from '../../shared_imports';
-import {
-  serializeAsESLifecycle,
-  deserializeESLifecycle,
-} from '../../../../../../common/lib/data_stream_serialization';
+import { serializeAsESLifecycle, deserializeESLifecycle } from '../../../../../../common/lib';
 import { useComponentTemplatesContext } from '../../component_templates_context';
 import { StepLogisticsContainer, StepReviewContainer } from './steps';