diff --git a/test/plugin_functional/test_suites/core_plugins/rendering.ts b/test/plugin_functional/test_suites/core_plugins/rendering.ts index 9bd43504d7db1e..a8c9fef0cfb96c 100644 --- a/test/plugin_functional/test_suites/core_plugins/rendering.ts +++ b/test/plugin_functional/test_suites/core_plugins/rendering.ts @@ -295,6 +295,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { 'xpack.index_management.enableLegacyTemplates (boolean?|never)', 'xpack.index_management.enableIndexStats (boolean?|never)', 'xpack.index_management.enableDataStreamStats (boolean?|never)', + 'xpack.index_management.enableSizeAndDocCount (boolean?|never)', 'xpack.index_management.editableIndexSettings (all?|limited?|never)', 'xpack.index_management.enableMappingsSourceFieldSection (boolean?|never)', 'xpack.index_management.dev.enableSemanticText (boolean?)', diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts index 8b97b24eadb7a2..bd119a77378af5 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts @@ -24,7 +24,7 @@ export interface DataStreamsTabTestBed extends TestBed { actions: { goToDataStreamsList: () => void; clickEmptyPromptIndexTemplateLink: () => void; - clickIncludeStatsSwitch: () => void; + clickIncludeStatsSwitch: () => Promise; toggleViewFilterAt: (index: number) => void; sortTableOnStorageSize: () => void; sortTableOnName: () => void; @@ -90,9 +90,13 @@ export const setup = async ( component.update(); }; - const clickIncludeStatsSwitch = () => { - const { find } = testBed; - find('includeStatsSwitch').simulate('click'); + const clickIncludeStatsSwitch = async () => { + const { find, component } = testBed; + + await act(async () => { + find('includeStatsSwitch').simulate('click'); + }); + component.update(); }; const toggleViewFilterAt = (index: number) => { diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts index 26eb9eab172b38..a4ea7b9296e285 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts @@ -156,6 +156,10 @@ describe('Data Streams tab', () => { name: 'dataStream1', storageSize: '5b', storageSizeBytes: 5, + // metering API mock + meteringStorageSize: '156kb', + meteringStorageSizeBytes: 156000, + meteringDocsCount: 10000, }); setLoadDataStreamsResponse([ @@ -164,6 +168,10 @@ describe('Data Streams tab', () => { name: 'dataStream2', storageSize: '1kb', storageSizeBytes: 1000, + // metering API mock + meteringStorageSize: '156kb', + meteringStorageSizeBytes: 156000, + meteringDocsCount: 10000, lifecycle: { enabled: true, data_retention: '7d', @@ -224,15 +232,12 @@ describe('Data Streams tab', () => { }); test('has a switch that will reload the data streams with additional stats when clicked', async () => { - const { exists, actions, table, component } = testBed; + const { exists, actions, table } = testBed; expect(exists('includeStatsSwitch')).toBe(true); // Changing the switch will automatically reload the data streams. - await act(async () => { - actions.clickIncludeStatsSwitch(); - }); - component.update(); + await actions.clickIncludeStatsSwitch(); expect(httpSetup.get).toHaveBeenLastCalledWith( `${API_BASE_PATH}/data_streams`, @@ -267,12 +272,9 @@ describe('Data Streams tab', () => { test('sorting on stats sorts by bytes value instead of human readable value', async () => { // Guards against regression of #86122. - const { actions, table, component } = testBed; + const { actions, table } = testBed; - await act(async () => { - actions.clickIncludeStatsSwitch(); - }); - component.update(); + await actions.clickIncludeStatsSwitch(); actions.sortTableOnStorageSize(); @@ -306,7 +308,7 @@ describe('Data Streams tab', () => { actions.sortTableOnName(); }); - test('hides stats toggle if enableDataStreamStats===false', async () => { + test(`doesn't hide stats toggle if enableDataStreamStats===false`, async () => { testBed = await setup(httpSetup, { config: { enableDataStreamStats: false, @@ -321,14 +323,82 @@ describe('Data Streams tab', () => { component.update(); - expect(exists('includeStatsSwitch')).toBeFalsy(); + expect(exists('includeStatsSwitch')).toBeTruthy(); + }); + + test('shows storage size and documents count if enableSizeAndDocCount===true, enableDataStreamStats==false', async () => { + testBed = await setup(httpSetup, { + config: { + enableSizeAndDocCount: true, + enableDataStreamStats: false, + }, + }); + + const { actions, component, table } = testBed; + + await act(async () => { + actions.goToDataStreamsList(); + }); + + component.update(); + + await actions.clickIncludeStatsSwitch(); + + const { tableCellsValues } = table.getMetaData('dataStreamTable'); + expect(tableCellsValues).toEqual([ + ['', 'dataStream1', 'green', '156kb', '10000', '1', '7 days', 'Delete'], + ['', 'dataStream2', 'green', '156kb', '10000', '1', '5 days ', 'Delete'], + ]); + }); + + test('shows last updated and storage size if enableDataStreamStats===true, enableSizeAndDocCount===false', async () => { + testBed = await setup(httpSetup, { + config: { + enableDataStreamStats: true, + enableSizeAndDocCount: false, + }, + }); + + const { actions, component, table } = testBed; + + await act(async () => { + actions.goToDataStreamsList(); + }); + + component.update(); + + await actions.clickIncludeStatsSwitch(); + + const { tableCellsValues } = table.getMetaData('dataStreamTable'); + expect(tableCellsValues).toEqual([ + [ + '', + 'dataStream1', + 'green', + 'December 31st, 1969 7:00:00 PM', + '5b', + '1', + '7 days', + 'Delete', + ], + [ + '', + 'dataStream2', + 'green', + 'December 31st, 1969 7:00:00 PM', + '1kb', + '1', + '5 days ', + 'Delete', + ], + ]); }); test('clicking the indices count navigates to the backing indices', async () => { const { table, actions } = testBed; await actions.clickIndicesAt(0); expect(table.getMetaData('indexTable').tableCellsValues).toEqual([ - ['', 'data-stream-index', '', '', '', '', '', '', 'dataStream1'], + ['', 'data-stream-index', '', '', '', '', '0', '', 'dataStream1'], ]); }); @@ -707,7 +777,7 @@ describe('Data Streams tab', () => { const { table, actions } = testBed; await actions.clickIndicesAt(0); expect(table.getMetaData('indexTable').tableCellsValues).toEqual([ - ['', 'data-stream-index', '', '', '', '', '', '', '%dataStream'], + ['', 'data-stream-index', '', '', '', '', '0', '', '%dataStream'], ]); }); }); diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.test.tsx b/x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.test.tsx index 8a8a2fc23d54df..3af9b4b9a2f627 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.test.tsx +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.test.tsx @@ -394,7 +394,7 @@ describe('', () => { component.update(); }); - test('renders the table column with index stats by default', () => { + test('renders the table column with all index stats when enableIndexStats is true', () => { const { table } = testBed; const { tableCellsValues } = table.getMetaData('indexTable'); @@ -403,7 +403,7 @@ describe('', () => { ]); }); - describe('Disabled', () => { + describe('renders only size and docs count when enableIndexStats is false, enableSizeAndDocCount is true', () => { beforeEach(async () => { await act(async () => { testBed = await setup(httpSetup, { @@ -411,6 +411,7 @@ describe('', () => { enableLegacyTemplates: true, enableIndexActions: true, enableIndexStats: false, + enableSizeAndDocCount: true, }, }); }); @@ -420,7 +421,33 @@ describe('', () => { component.update(); }); - test('hides index stats information from table', async () => { + test('hides some index stats information from table', async () => { + const { table } = testBed; + const { tableCellsValues } = table.getMetaData('indexTable'); + + expect(tableCellsValues).toEqual([['', 'test', '10,000', '156kb', '']]); + }); + }); + + describe('renders no index stats when enableIndexStats is false, enableSizeAndDocCount is false', () => { + beforeEach(async () => { + await act(async () => { + testBed = await setup(httpSetup, { + config: { + enableLegacyTemplates: true, + enableIndexActions: true, + enableIndexStats: false, + enableSizeAndDocCount: false, + }, + }); + }); + + const { component } = testBed; + + component.update(); + }); + + test('hides all index stats information from table', async () => { const { table } = testBed; const { tableCellsValues } = table.getMetaData('indexTable'); diff --git a/x-pack/plugins/index_management/common/lib/data_stream_serialization.test.ts b/x-pack/plugins/index_management/common/lib/data_stream_utils.test.ts similarity index 81% rename from x-pack/plugins/index_management/common/lib/data_stream_serialization.test.ts rename to x-pack/plugins/index_management/common/lib/data_stream_utils.test.ts index 334e6bbf97de0b..afbcf7835f7641 100644 --- a/x-pack/plugins/index_management/common/lib/data_stream_serialization.test.ts +++ b/x-pack/plugins/index_management/common/lib/data_stream_utils.test.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { splitSizeAndUnits } from './data_stream_serialization'; +import { splitSizeAndUnits } from './data_stream_utils'; -describe('Data stream serialization', () => { +describe('Data stream utils', () => { test('can split size and units from lifecycle string', () => { expect(splitSizeAndUnits('1h')).toEqual({ size: '1', unit: 'h' }); expect(splitSizeAndUnits('20micron')).toEqual({ size: '20', unit: 'micron' }); diff --git a/x-pack/plugins/index_management/common/lib/data_stream_utils.ts b/x-pack/plugins/index_management/common/lib/data_stream_utils.ts new file mode 100644 index 00000000000000..443373f8a6b4b2 --- /dev/null +++ b/x-pack/plugins/index_management/common/lib/data_stream_utils.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DataStream, DataRetention } from '../types'; + +export const splitSizeAndUnits = (field: string): { size: string; unit: string } => { + let size = ''; + let unit = ''; + + const result = /(\d+)(\w+)/.exec(field); + if (result) { + size = result[1]; + unit = result[2]; + } + + return { + size, + unit, + }; +}; + +export const serializeAsESLifecycle = (lifecycle?: DataRetention): DataStream['lifecycle'] => { + if (!lifecycle || !lifecycle?.enabled) { + return undefined; + } + + const { infiniteDataRetention, value, unit } = lifecycle; + + if (infiniteDataRetention) { + return { + enabled: true, + }; + } + + return { + enabled: true, + data_retention: `${value}${unit}`, + }; +}; + +export const deserializeESLifecycle = (lifecycle?: DataStream['lifecycle']): DataRetention => { + if (!lifecycle || !lifecycle?.enabled) { + return { enabled: false }; + } + + if (!lifecycle.data_retention) { + return { + enabled: true, + infiniteDataRetention: true, + }; + } + + const { size, unit } = splitSizeAndUnits(lifecycle.data_retention as string); + + return { + enabled: true, + value: Number(size), + unit, + }; +}; diff --git a/x-pack/plugins/index_management/common/lib/index.ts b/x-pack/plugins/index_management/common/lib/index.ts index d46d3d8b6a1d49..da29f3289bcd42 100644 --- a/x-pack/plugins/index_management/common/lib/index.ts +++ b/x-pack/plugins/index_management/common/lib/index.ts @@ -6,10 +6,10 @@ */ export { - deserializeDataStream, - deserializeDataStreamList, splitSizeAndUnits, -} from './data_stream_serialization'; + serializeAsESLifecycle, + deserializeESLifecycle, +} from './data_stream_utils'; export { deserializeTemplate, diff --git a/x-pack/plugins/index_management/common/lib/template_serialization.ts b/x-pack/plugins/index_management/common/lib/template_serialization.ts index aacbc15aab3b83..f8b4ed47a22f77 100644 --- a/x-pack/plugins/index_management/common/lib/template_serialization.ts +++ b/x-pack/plugins/index_management/common/lib/template_serialization.ts @@ -12,7 +12,7 @@ import { TemplateListItem, TemplateType, } from '../types'; -import { deserializeESLifecycle } from './data_stream_serialization'; +import { deserializeESLifecycle } from './data_stream_utils'; import { allowAutoCreateRadioValues, allowAutoCreateRadioIds } from '../constants'; const hasEntries = (data: object = {}) => Object.entries(data).length > 0; diff --git a/x-pack/plugins/index_management/common/types/data_streams.ts b/x-pack/plugins/index_management/common/types/data_streams.ts index 4e20252792d716..c44305edb9d8f9 100644 --- a/x-pack/plugins/index_management/common/types/data_streams.ts +++ b/x-pack/plugins/index_management/common/types/data_streams.ts @@ -37,6 +37,8 @@ export interface EnhancedDataStreamFromEs extends IndicesDataStream { store_size?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size']; store_size_bytes?: IndicesDataStreamsStatsDataStreamsStatsItem['store_size_bytes']; maximum_timestamp?: IndicesDataStreamsStatsDataStreamsStatsItem['maximum_timestamp']; + metering_size_in_bytes?: number; + metering_doc_count?: number; indices: DataStreamIndexFromEs[]; privileges: { delete_index: boolean; @@ -55,6 +57,9 @@ export interface DataStream { storageSize?: ByteSize; storageSizeBytes?: number; maxTimeStamp?: number; + meteringStorageSizeBytes?: number; + meteringStorageSize?: string; + meteringDocsCount?: number; _meta?: Metadata; privileges: Privileges; hidden: boolean; diff --git a/x-pack/plugins/index_management/public/application/components/component_templates/__jest__/client_integration/component_template_create.test.tsx b/x-pack/plugins/index_management/public/application/components/component_templates/__jest__/client_integration/component_template_create.test.tsx index 729ca7680ad6cd..787aa689077307 100644 --- a/x-pack/plugins/index_management/public/application/components/component_templates/__jest__/client_integration/component_template_create.test.tsx +++ b/x-pack/plugins/index_management/public/application/components/component_templates/__jest__/client_integration/component_template_create.test.tsx @@ -12,7 +12,7 @@ import { breadcrumbService, IndexManagementBreadcrumb } from '../../../../servic import { setupEnvironment } from './helpers'; import { API_BASE_PATH } from './helpers/constants'; import { setup, ComponentTemplateCreateTestBed } from './helpers/component_template_create.helpers'; -import { serializeAsESLifecycle } from '../../../../../../common/lib/data_stream_serialization'; +import { serializeAsESLifecycle } from '../../../../../../common/lib'; jest.mock('@kbn/code-editor', () => { const original = jest.requireActual('@kbn/code-editor'); diff --git a/x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_form/component_template_form.tsx b/x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_form/component_template_form.tsx index 1cbc33fc051359..ca791311d2408b 100644 --- a/x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_form/component_template_form.tsx +++ b/x-pack/plugins/index_management/public/application/components/component_templates/component_template_wizard/component_template_form/component_template_form.tsx @@ -19,10 +19,7 @@ import { StepMappingsContainer, StepAliasesContainer, } from '../../shared_imports'; -import { - serializeAsESLifecycle, - deserializeESLifecycle, -} from '../../../../../../common/lib/data_stream_serialization'; +import { serializeAsESLifecycle, deserializeESLifecycle } from '../../../../../../common/lib'; import { useComponentTemplatesContext } from '../../component_templates_context'; import { StepLogisticsContainer, StepReviewContainer } from './steps'; diff --git a/x-pack/plugins/index_management/public/application/components/template_form/template_form.tsx b/x-pack/plugins/index_management/public/application/components/template_form/template_form.tsx index ac247500e32481..2da3eef609a65d 100644 --- a/x-pack/plugins/index_management/public/application/components/template_form/template_form.tsx +++ b/x-pack/plugins/index_management/public/application/components/template_form/template_form.tsx @@ -22,7 +22,7 @@ import { } from '../shared'; import { documentationService } from '../../services/documentation'; import { SectionError } from '../section_error'; -import { serializeAsESLifecycle } from '../../../../common/lib/data_stream_serialization'; +import { serializeAsESLifecycle } from '../../../../common/lib'; import { SimulateTemplateFlyoutContent, SimulateTemplateProps, diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx index da53b0241095dd..974ba6f082042d 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx @@ -52,13 +52,14 @@ import { useAppContext } from '../../../../app_context'; import { DataStreamsBadges } from '../data_stream_badges'; import { useIlmLocator } from '../../../../services/use_ilm_locator'; +interface Detail { + name: string; + toolTip: string; + content: any; + dataTestSubj: string; +} interface DetailsListProps { - details: Array<{ - name: string; - toolTip: string; - content: any; - dataTestSubj: string; - }>; + details: Detail[]; } const DetailsList: React.FunctionComponent = ({ details }) => { @@ -162,6 +163,8 @@ export const DataStreamDetailPanel: React.FunctionComponent = ({ ilmPolicyName, storageSize, maxTimeStamp, + meteringStorageSize, + meteringDocsCount, lifecycle, } = dataStream; @@ -222,7 +225,7 @@ export const DataStreamDetailPanel: React.FunctionComponent = ({ ); - const defaultDetails = [ + const defaultDetails: Detail[] = [ { name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.healthTitle', { defaultMessage: 'Health', @@ -233,34 +236,67 @@ export const DataStreamDetailPanel: React.FunctionComponent = ({ content: , dataTestSubj: 'healthDetail', }, - { - name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampTitle', { - defaultMessage: 'Last updated', - }), - toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampToolTip', { - defaultMessage: 'The most recent document to be added to the data stream.', - }), - content: maxTimeStamp ? ( - humanizeTimeStamp(maxTimeStamp) - ) : ( - - {i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampNoneMessage', { - defaultMessage: `Never`, - })} - - ), - dataTestSubj: 'lastUpdatedDetail', - }, - { - name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeTitle', { - defaultMessage: 'Storage size', - }), - toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeToolTip', { - defaultMessage: `The total size of all shards in the data stream’s backing indices.`, - }), - content: storageSize, - dataTestSubj: 'storageSizeDetail', - }, + ]; + + // add either documents count and size or last updated and size + if (config.enableSizeAndDocCount) { + defaultDetails.push( + { + name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.meteringDocsCountTitle', { + defaultMessage: 'Documents count', + }), + toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.meteringDocsCountToolTip', { + defaultMessage: 'The number of documents in this data stream.', + }), + content: meteringDocsCount, + dataTestSubj: 'docsCountDetail', + }, + { + name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeTitle', { + defaultMessage: 'Storage size', + }), + toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeToolTip', { + defaultMessage: `The total size of all shards in the data stream’s backing indices.`, + }), + content: meteringStorageSize, + dataTestSubj: 'meteringStorageSizeDetail', + } + ); + } + if (config.enableDataStreamStats) { + defaultDetails.push( + { + name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampTitle', { + defaultMessage: 'Last updated', + }), + toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampToolTip', { + defaultMessage: 'The most recent document to be added to the data stream.', + }), + content: maxTimeStamp ? ( + humanizeTimeStamp(maxTimeStamp) + ) : ( + + {i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.maxTimeStampNoneMessage', { + defaultMessage: `Never`, + })} + + ), + dataTestSubj: 'lastUpdatedDetail', + }, + { + name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeTitle', { + defaultMessage: 'Storage size', + }), + toolTip: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.storageSizeToolTip', { + defaultMessage: `The total size of all shards in the data stream’s backing indices.`, + }), + content: storageSize, + dataTestSubj: 'storageSizeDetail', + } + ); + } + + defaultDetails.push( { name: i18n.translate('xpack.idxMgmt.dataStreamDetailPanel.indicesTitle', { defaultMessage: 'Indices', @@ -328,8 +364,8 @@ export const DataStreamDetailPanel: React.FunctionComponent = ({ ), dataTestSubj: 'dataRetentionDetail', - }, - ]; + } + ); // If both rentention types are available, we wanna surface to the user both if (lifecycle?.effective_retention && lifecycle?.data_retention) { diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx index 0103b51f1f51d8..125f676897ffb3 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx @@ -60,11 +60,8 @@ export const DataStreamList: React.FunctionComponent - {isDataStreamStatsEnabled && ( - - - - setIsIncludeStatsChecked(e.target.checked)} - data-test-subj="includeStatsSwitch" - /> - + + + + setIsIncludeStatsChecked(e.target.checked)} + data-test-subj="includeStatsSwitch" + /> + - - - - - - )} + + + + + filters={filters} onChange={setFilters} /> diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx index b4dbb663e08596..47b170babc5a61 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx @@ -104,31 +104,55 @@ export const DataStreamTable: React.FunctionComponent = ({ }); if (includeStats) { - columns.push({ - field: 'maxTimeStamp', - name: i18n.translate('xpack.idxMgmt.dataStreamList.table.maxTimeStampColumnTitle', { - defaultMessage: 'Last updated', - }), - truncateText: true, - sortable: true, - render: (maxTimeStamp: DataStream['maxTimeStamp']) => - maxTimeStamp - ? humanizeTimeStamp(maxTimeStamp) - : i18n.translate('xpack.idxMgmt.dataStreamList.table.maxTimeStampColumnNoneMessage', { - defaultMessage: 'Never', - }), - }); - - columns.push({ - field: 'storageSizeBytes', - name: i18n.translate('xpack.idxMgmt.dataStreamList.table.storageSizeColumnTitle', { - defaultMessage: 'Storage size', - }), - truncateText: true, - sortable: true, - render: (storageSizeBytes: DataStream['storageSizeBytes'], dataStream: DataStream) => - dataStream.storageSize, - }); + if (config.enableSizeAndDocCount) { + // datastreams stats from metering API on serverless + columns.push({ + field: 'meteringStorageSizeBytes', + name: i18n.translate('xpack.idxMgmt.dataStreamList.table.storageSizeColumnTitle', { + defaultMessage: 'Storage size', + }), + truncateText: true, + sortable: true, + render: ( + meteringStorageSizeBytes: DataStream['meteringStorageSizeBytes'], + dataStream: DataStream + ) => dataStream.meteringStorageSize, + }); + columns.push({ + field: 'meteringDocsCount', + name: i18n.translate('xpack.idxMgmt.dataStreamList.table.docsCountColumnTitle', { + defaultMessage: 'Documents count', + }), + truncateText: true, + sortable: true, + }); + } + if (config.enableDataStreamStats) { + columns.push({ + field: 'maxTimeStamp', + name: i18n.translate('xpack.idxMgmt.dataStreamList.table.maxTimeStampColumnTitle', { + defaultMessage: 'Last updated', + }), + truncateText: true, + sortable: true, + render: (maxTimeStamp: DataStream['maxTimeStamp']) => + maxTimeStamp + ? humanizeTimeStamp(maxTimeStamp) + : i18n.translate('xpack.idxMgmt.dataStreamList.table.maxTimeStampColumnNoneMessage', { + defaultMessage: 'Never', + }), + }); + columns.push({ + field: 'storageSizeBytes', + name: i18n.translate('xpack.idxMgmt.dataStreamList.table.storageSizeColumnTitle', { + defaultMessage: 'Storage size', + }), + truncateText: true, + sortable: true, + render: (storageSizeBytes: DataStream['storageSizeBytes'], dataStream: DataStream) => + dataStream.storageSize, + }); + } } columns.push({ diff --git a/x-pack/plugins/index_management/public/application/sections/home/index_list/details_page/details_page_overview/details_page_overview.tsx b/x-pack/plugins/index_management/public/application/sections/home/index_list/details_page/details_page_overview/details_page_overview.tsx index c3e985a638fd55..a5fcb4a5c24a83 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/index_list/details_page/details_page_overview/details_page_overview.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/index_list/details_page/details_page_overview/details_page_overview.tsx @@ -26,14 +26,15 @@ import { getLanguageDefinitionCodeSnippet, getConsoleRequest, } from '@kbn/search-api-panels'; -import { StatusDetails } from './status_details'; import type { Index } from '../../../../../../../common'; import { useAppContext } from '../../../../../app_context'; import { documentationService } from '../../../../../services'; import { languageDefinitions, curlDefinition } from './languages'; +import { StatusDetails } from './status_details'; import { DataStreamDetails } from './data_stream_details'; import { StorageDetails } from './storage_details'; import { AliasesDetails } from './aliases_details'; +import { SizeDocCountDetails } from './size_doc_count_details'; interface Props { indexDetails: Index; @@ -85,6 +86,8 @@ export const DetailsPageOverview: React.FunctionComponent = ({ indexDetai health={health} /> + + {dataStream && } diff --git a/x-pack/plugins/index_management/public/application/sections/home/index_list/details_page/details_page_overview/size_doc_count_details.tsx b/x-pack/plugins/index_management/public/application/sections/home/index_list/details_page/details_page_overview/size_doc_count_details.tsx new file mode 100644 index 00000000000000..40294d76b26988 --- /dev/null +++ b/x-pack/plugins/index_management/public/application/sections/home/index_list/details_page/details_page_overview/size_doc_count_details.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FunctionComponent } from 'react'; +import { css } from '@emotion/react'; +import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiText, EuiTextColor } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { euiThemeVars } from '@kbn/ui-theme'; +import type { Index } from '../../../../../../../common'; +import { useAppContext } from '../../../../../app_context'; +import { OverviewCard } from './overview_card'; + +export const SizeDocCountDetails: FunctionComponent<{ + size: Index['size']; + documents: Index['documents']; +}> = ({ size, documents }) => { + const { config } = useAppContext(); + if (!config.enableSizeAndDocCount) { + return null; + } + return ( + + + + {size} + + + + + {i18n.translate('xpack.idxMgmt.indexDetails.overviewTab.storage.totalSizeLabel', { + defaultMessage: 'Total', + })} + + + + ), + right: null, + }} + footer={{ + left: ( + + + + + {documents} + + + {i18n.translate( + 'xpack.idxMgmt.indexDetails.overviewTab.status.meteringDocumentsLabel', + { + defaultMessage: '{documents, plural, one {Document} other {Documents}}', + values: { + documents, + }, + } + )} + + + + ), + }} + /> + ); +}; diff --git a/x-pack/plugins/index_management/public/application/sections/home/index_list/index_table/index_table.js b/x-pack/plugins/index_management/public/application/sections/home/index_list/index_table/index_table.js index 89b14d3db05c9e..bdc245fe577033 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/index_list/index_table/index_table.js +++ b/x-pack/plugins/index_management/public/application/sections/home/index_list/index_table/index_table.js @@ -52,6 +52,7 @@ import { IndexTablePagination, PAGE_SIZE_OPTIONS } from './index_table_paginatio const getColumnConfigs = ({ showIndexStats, + showSizeAndDocCount, history, filterChanged, extensionsService, @@ -100,6 +101,28 @@ const getColumnConfigs = ({ }, ]; + // size and docs count enabled by either "enableIndexStats" or "enableSizeAndDocCount" configs + if (showIndexStats || showSizeAndDocCount) { + columns.push( + { + fieldName: 'documents', + label: i18n.translate('xpack.idxMgmt.indexTable.headers.documentsHeader', { + defaultMessage: 'Documents count', + }), + order: 60, + render: (index) => { + return Number(index.documents ?? 0).toLocaleString(); + }, + }, + { + fieldName: 'size', + label: i18n.translate('xpack.idxMgmt.indexTable.headers.storageSizeHeader', { + defaultMessage: 'Storage size', + }), + order: 70, + } + ); + } if (showIndexStats) { columns.push( { @@ -130,25 +153,6 @@ const getColumnConfigs = ({ defaultMessage: 'Replicas', }), order: 50, - }, - { - fieldName: 'documents', - label: i18n.translate('xpack.idxMgmt.indexTable.headers.documentsHeader', { - defaultMessage: 'Docs count', - }), - order: 60, - render: (index) => { - if (index.documents) { - return Number(index.documents).toLocaleString(); - } - }, - }, - { - fieldName: 'size', - label: i18n.translate('xpack.idxMgmt.indexTable.headers.storageSizeHeader', { - defaultMessage: 'Storage size', - }), - order: 70, } ); } @@ -533,6 +537,7 @@ export class IndexTable extends Component { const { extensionsService } = services; const columnConfigs = getColumnConfigs({ showIndexStats: config.enableIndexStats, + showSizeAndDocCount: config.enableSizeAndDocCount, extensionsService, filterChanged, history, diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx index c2aa548100b577..513377714ffe0d 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_summary.tsx @@ -22,7 +22,7 @@ import { } from '@elastic/eui'; import { reactRouterNavigate } from '../../../../../../shared_imports'; import { useAppContext } from '../../../../../app_context'; -import { serializeAsESLifecycle } from '../../../../../../../common/lib/data_stream_serialization'; +import { serializeAsESLifecycle } from '../../../../../../../common/lib'; import { getLifecycleValue } from '../../../../../lib/data_streams'; import { TemplateDeserialized } from '../../../../../../../common'; import { ILM_PAGES_POLICY_EDIT } from '../../../../../constants'; diff --git a/x-pack/plugins/index_management/public/plugin.ts b/x-pack/plugins/index_management/public/plugin.ts index 49693bb0d9aa9e..4efe613fc2a045 100644 --- a/x-pack/plugins/index_management/public/plugin.ts +++ b/x-pack/plugins/index_management/public/plugin.ts @@ -44,8 +44,8 @@ export class IndexMgmtUIPlugin enableIndexActions: boolean; enableLegacyTemplates: boolean; enableIndexStats: boolean; - enableSizeAndDocCount: boolean; enableDataStreamStats: boolean; + enableSizeAndDocCount: boolean; editableIndexSettings: 'all' | 'limited'; isIndexManagementUiEnabled: boolean; enableMappingsSourceFieldSection: boolean; @@ -63,8 +63,8 @@ export class IndexMgmtUIPlugin enableIndexActions, enableLegacyTemplates, enableIndexStats, - enableSizeAndDocCount, enableDataStreamStats, + enableSizeAndDocCount, editableIndexSettings, enableMappingsSourceFieldSection, enableTogglingDataRetention, @@ -75,8 +75,8 @@ export class IndexMgmtUIPlugin enableIndexActions: enableIndexActions ?? true, enableLegacyTemplates: enableLegacyTemplates ?? true, enableIndexStats: enableIndexStats ?? true, - enableSizeAndDocCount: enableSizeAndDocCount ?? true, enableDataStreamStats: enableDataStreamStats ?? true, + enableSizeAndDocCount: enableSizeAndDocCount ?? false, editableIndexSettings: editableIndexSettings ?? 'all', enableMappingsSourceFieldSection: enableMappingsSourceFieldSection ?? true, enableTogglingDataRetention: enableTogglingDataRetention ?? true, diff --git a/x-pack/plugins/index_management/server/config.ts b/x-pack/plugins/index_management/server/config.ts index 3480e380281e59..9bddc6417cc1b3 100644 --- a/x-pack/plugins/index_management/server/config.ts +++ b/x-pack/plugins/index_management/server/config.ts @@ -83,6 +83,7 @@ const configLatest: PluginConfigDescriptor = { enableLegacyTemplates: true, enableIndexStats: true, enableDataStreamStats: true, + enableSizeAndDocCount: true, editableIndexSettings: true, enableMappingsSourceFieldSection: true, enableTogglingDataRetention: true, diff --git a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts b/x-pack/plugins/index_management/server/lib/data_stream_serialization.ts similarity index 58% rename from x-pack/plugins/index_management/common/lib/data_stream_serialization.ts rename to x-pack/plugins/index_management/server/lib/data_stream_serialization.ts index ceedd072139aa7..ffe058907e000f 100644 --- a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts +++ b/x-pack/plugins/index_management/server/lib/data_stream_serialization.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { DataStream, EnhancedDataStreamFromEs, Health, DataRetention } from '../types'; +import { ByteSizeValue } from '@kbn/config-schema'; +import type { DataStream, EnhancedDataStreamFromEs, Health } from '../../common'; export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs): DataStream { const { @@ -19,12 +20,18 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs store_size: storageSize, store_size_bytes: storageSizeBytes, maximum_timestamp: maxTimeStamp, + metering_size_in_bytes: meteringStorageSizeBytes, + metering_doc_count: meteringDocsCount, _meta, privileges, hidden, lifecycle, next_generation_managed_by: nextGenerationManagedBy, } = dataStreamFromEs; + const meteringStorageSize = + meteringStorageSizeBytes !== undefined + ? new ByteSizeValue(meteringStorageSizeBytes).toString() + : undefined; return { name, @@ -54,6 +61,9 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs storageSize, storageSizeBytes, maxTimeStamp, + meteringStorageSize, + meteringStorageSizeBytes, + meteringDocsCount, _meta, privileges, hidden, @@ -67,59 +77,3 @@ export function deserializeDataStreamList( ): DataStream[] { return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream)); } - -export const splitSizeAndUnits = (field: string): { size: string; unit: string } => { - let size = ''; - let unit = ''; - - const result = /(\d+)(\w+)/.exec(field); - if (result) { - size = result[1]; - unit = result[2]; - } - - return { - size, - unit, - }; -}; - -export const serializeAsESLifecycle = (lifecycle?: DataRetention): DataStream['lifecycle'] => { - if (!lifecycle || !lifecycle?.enabled) { - return undefined; - } - - const { infiniteDataRetention, value, unit } = lifecycle; - - if (infiniteDataRetention) { - return { - enabled: true, - }; - } - - return { - enabled: true, - data_retention: `${value}${unit}`, - }; -}; - -export const deserializeESLifecycle = (lifecycle?: DataStream['lifecycle']): DataRetention => { - if (!lifecycle || !lifecycle?.enabled) { - return { enabled: false }; - } - - if (!lifecycle.data_retention) { - return { - enabled: true, - infiniteDataRetention: true, - }; - } - - const { size, unit } = splitSizeAndUnits(lifecycle.data_retention as string); - - return { - enabled: true, - value: Number(size), - unit, - }; -}; diff --git a/x-pack/plugins/index_management/server/lib/fetch_indices.ts b/x-pack/plugins/index_management/server/lib/fetch_indices.ts index 1df453d1042cd5..0e82f03f7308ff 100644 --- a/x-pack/plugins/index_management/server/lib/fetch_indices.ts +++ b/x-pack/plugins/index_management/server/lib/fetch_indices.ts @@ -10,13 +10,10 @@ import { IScopedClusterClient } from '@kbn/core/server'; import { IndexDataEnricher } from '../services'; import { Index } from '..'; import { RouteDependencies } from '../types'; +import type { MeteringStats } from './types'; interface MeteringStatsResponse { - indices: Array<{ - name: string; - num_docs: number; - size_in_bytes: number; - }>; + indices: MeteringStats[]; } async function fetchIndicesCall( diff --git a/x-pack/plugins/index_management/server/lib/types.ts b/x-pack/plugins/index_management/server/lib/types.ts new file mode 100644 index 00000000000000..1657c4bbbb6e02 --- /dev/null +++ b/x-pack/plugins/index_management/server/lib/types.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface MeteringStats { + name: string; + num_docs: number; + size_in_bytes: number; +} diff --git a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts index 11db019eacf6aa..78c0328f526174 100644 --- a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts +++ b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts @@ -13,18 +13,27 @@ import { IndicesDataStreamsStatsDataStreamsStatsItem, SecurityHasPrivilegesResponse, } from '@elastic/elasticsearch/lib/api/types'; -import { deserializeDataStream, deserializeDataStreamList } from '../../../../common/lib'; +import type { MeteringStats } from '../../../lib/types'; +import { + deserializeDataStream, + deserializeDataStreamList, +} from '../../../lib/data_stream_serialization'; import { EnhancedDataStreamFromEs } from '../../../../common/types'; import { RouteDependencies } from '../../../types'; import { addBasePath } from '..'; +interface MeteringStatsResponse { + datastreams: MeteringStats[]; +} const enhanceDataStreams = ({ dataStreams, dataStreamsStats, + meteringStats, dataStreamsPrivileges, }: { dataStreams: IndicesDataStream[]; dataStreamsStats?: IndicesDataStreamsStatsDataStreamsStatsItem[]; + meteringStats?: MeteringStats[]; dataStreamsPrivileges?: SecurityHasPrivilegesResponse; }): EnhancedDataStreamFromEs[] => { return dataStreams.map((dataStream) => { @@ -51,6 +60,14 @@ const enhanceDataStreams = ({ } } + if (meteringStats) { + const datastreamMeteringStats = meteringStats.find((s) => s.name === dataStream.name); + if (datastreamMeteringStats) { + enhancedDataStream.metering_size_in_bytes = datastreamMeteringStats.size_in_bytes; + enhancedDataStream.metering_doc_count = datastreamMeteringStats.num_docs; + } + } + return enhancedDataStream; }); }; @@ -70,6 +87,17 @@ const getDataStreamsStats = (client: IScopedClusterClient, name = '*') => { }); }; +const getMeteringStats = (client: IScopedClusterClient, name?: string) => { + let path = `/_metering/stats`; + if (name) { + path = `${path}/${name}`; + } + return client.asSecondaryAuthUser.transport.request({ + method: 'GET', + path, + }); +}; + const getDataStreamsPrivileges = (client: IScopedClusterClient, names: string[]) => { return client.asCurrentUser.security.hasPrivileges({ body: { @@ -99,10 +127,14 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }: let dataStreamsStats; let dataStreamsPrivileges; + let meteringStats; if (includeStats && config.isDataStreamStatsEnabled !== false) { ({ data_streams: dataStreamsStats } = await getDataStreamsStats(client)); } + if (includeStats && config.isSizeAndDocCountEnabled !== false) { + ({ datastreams: meteringStats } = await getMeteringStats(client)); + } if (config.isSecurityEnabled() && dataStreams.length > 0) { dataStreamsPrivileges = await getDataStreamsPrivileges( @@ -114,6 +146,7 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }: const enhancedDataStreams = enhanceDataStreams({ dataStreams, dataStreamsStats, + meteringStats, dataStreamsPrivileges, }); @@ -138,6 +171,7 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }: const { name } = request.params as TypeOf; const { client } = (await context.core).elasticsearch; let dataStreamsStats; + let meteringStats; try { const { data_streams: dataStreams } = await getDataStreams(client, name); @@ -146,6 +180,10 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }: ({ data_streams: dataStreamsStats } = await getDataStreamsStats(client, name)); } + if (config.isSizeAndDocCountEnabled !== false) { + ({ datastreams: meteringStats } = await getMeteringStats(client, name)); + } + if (dataStreams[0]) { let dataStreamsPrivileges; @@ -156,6 +194,7 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }: const enhancedDataStreams = enhanceDataStreams({ dataStreams, dataStreamsStats, + meteringStats, dataStreamsPrivileges, }); const body = deserializeDataStream(enhancedDataStreams[0]); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts index 961c9a73bdf477..de3a92587d6b96 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts @@ -118,6 +118,9 @@ export default function ({ getService }: FtrProviderContext) { lifecycle: { enabled: true, }, + meteringDocsCount: 0, + meteringStorageSize: '0b', + meteringStorageSizeBytes: 0, }); }); });