[8.x] [Dataset quality] remove usage of dataStreamStats API in server…

…less (#192839) (#193468)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Dataset quality] remove usage of dataStreamStats API in serverless
(#192839)](#192839)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Yngrid
Coello","email":"yngrid.coello@elastic.co"},"sourceCommit":{"committedDate":"2024-09-19T18:47:45Z","message":"[Dataset
quality] remove usage of dataStreamStats API in serverless
(#192839)\n\nCloses
https://github.com/elastic/kibana/issues/192166.\n\nThis PR aims to
remove the usage of `DataStreamStats` API in serverless.\nAditionally we
now make use of the new verbose mode for `DataStream` API\nwhich allow
us to still show the `lastActivity` for serverless projects.\nThe change
was implemented for both flavours.\n\nVisually it won't represent any
changes at the moment, but here are the\ntwo flavours still working as
before\n\n\nhttps://github.com/user-attachments/assets/18cc9b14-24f1-438a-ac07-565c3e728991","sha":"444f95ed0b4a2d251851f40d62b126544a6c17c0","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","ci:project-deploy-observability"],"title":"[Dataset
quality] remove usage of dataStreamStats API in
serverless","number":192839,"url":"https://github.com/elastic/kibana/pull/192839","mergeCommit":{"message":"[Dataset
quality] remove usage of dataStreamStats API in serverless
(#192839)\n\nCloses
https://github.com/elastic/kibana/issues/192166.\n\nThis PR aims to
remove the usage of `DataStreamStats` API in serverless.\nAditionally we
now make use of the new verbose mode for `DataStream` API\nwhich allow
us to still show the `lastActivity` for serverless projects.\nThe change
was implemented for both flavours.\n\nVisually it won't represent any
changes at the moment, but here are the\ntwo flavours still working as
before\n\n\nhttps://github.com/user-attachments/assets/18cc9b14-24f1-438a-ac07-565c3e728991","sha":"444f95ed0b4a2d251851f40d62b126544a6c17c0"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/192839","number":192839,"mergeCommit":{"message":"[Dataset
quality] remove usage of dataStreamStats API in serverless
(#192839)\n\nCloses
https://github.com/elastic/kibana/issues/192166.\n\nThis PR aims to
remove the usage of `DataStreamStats` API in serverless.\nAditionally we
now make use of the new verbose mode for `DataStream` API\nwhich allow
us to still show the `lastActivity` for serverless projects.\nThe change
was implemented for both flavours.\n\nVisually it won't represent any
changes at the moment, but here are the\ntwo flavours still working as
before\n\n\nhttps://github.com/user-attachments/assets/18cc9b14-24f1-438a-ac07-565c3e728991","sha":"444f95ed0b4a2d251851f40d62b126544a6c17c0"}}]}]
BACKPORT-->

Co-authored-by: Yngrid Coello <yngrid.coello@elastic.co>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/observability_solution/dataset_quality/public/components/common/insufficient_privileges.tsx

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React, { useCallback } from 'react';
+import React from 'react';
 import useToggle from 'react-use/lib/useToggle';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
@@ -54,7 +54,7 @@ export const PrivilegesWarningIconWrapper = ({
 }) => {
   const [isPopoverOpen, togglePopover] = useToggle(false);
 
-  const handleButtonClick = useCallback(() => togglePopover(true), [togglePopover]);
+  const handleButtonClick = togglePopover;
 
   if (hasPrivileges) {
     return <>{children}</>;

x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/summary_panel/datasets_activity.tsx

@@ -16,8 +16,7 @@ import {
 import { DataPlaceholder } from './data_placeholder';
 
 export function DatasetsActivity() {
-  const { datasetsActivity, isDatasetsActivityLoading, isUserAuthorizedForDataset } =
-    useSummaryPanelContext();
+  const { datasetsActivity, isDatasetsActivityLoading } = useSummaryPanelContext();
   const text = `${datasetsActivity.active} ${tableSummaryOfText} ${datasetsActivity.total}`;
 
   return (
@@ -26,7 +25,7 @@ export function DatasetsActivity() {
       tooltip={summaryPanelDatasetsActivityTooltipText}
       value={text}
       isLoading={isDatasetsActivityLoading}
-      isUserAuthorizedForDataset={isUserAuthorizedForDataset}
+      isUserAuthorizedForDataset={true}
     />
   );
 }

x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx

@@ -300,46 +300,37 @@ export const getDatasetQualityTableColumns = ({
       ),
       width: '140px',
     },
-    ...(canUserMonitorDataset && canUserMonitorAnyDataStream
-      ? [
-          {
-            name: (
-              <EuiTableHeader data-test-subj="datasetQualityLastActivityColumn">
-                {lastActivityColumnName}
-              </EuiTableHeader>
-            ),
-            field: 'lastActivity',
-            render: (timestamp: number, { userPrivileges, title }: DataStreamStat) => (
-              <PrivilegesWarningIconWrapper
-                title={`lastActivity-${title}`}
-                hasPrivileges={userPrivileges?.canMonitor ?? true}
-              >
-                <EuiSkeletonRectangle
-                  width="200px"
-                  height="20px"
-                  borderRadius="m"
-                  isLoading={loadingDataStreamStats}
-                >
-                  {!isActiveDataset(timestamp) ? (
-                    <EuiFlexGroup gutterSize="xs" alignItems="center">
-                      <EuiText size="s">{inactiveDatasetActivityColumnDescription}</EuiText>
-                      <EuiToolTip position="top" content={inactiveDatasetActivityColumnTooltip}>
-                        <EuiIcon tabIndex={0} type="iInCircle" size="s" />
-                      </EuiToolTip>
-                    </EuiFlexGroup>
-                  ) : (
-                    fieldFormats
-                      .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE])
-                      .convert(timestamp)
-                  )}
-                </EuiSkeletonRectangle>
-              </PrivilegesWarningIconWrapper>
-            ),
-            width: '300px',
-            sortable: true,
-          },
-        ]
-      : []),
+    {
+      name: (
+        <EuiTableHeader data-test-subj="datasetQualityLastActivityColumn">
+          {lastActivityColumnName}
+        </EuiTableHeader>
+      ),
+      field: 'lastActivity',
+      render: (timestamp: number) => (
+        <EuiSkeletonRectangle
+          width="200px"
+          height="20px"
+          borderRadius="m"
+          isLoading={loadingDataStreamStats}
+        >
+          {!isActiveDataset(timestamp) ? (
+            <EuiFlexGroup gutterSize="xs" alignItems="center">
+              <EuiText size="s">{inactiveDatasetActivityColumnDescription}</EuiText>
+              <EuiToolTip position="top" content={inactiveDatasetActivityColumnTooltip}>
+                <EuiIcon tabIndex={0} type="iInCircle" size="s" />
+              </EuiToolTip>
+            </EuiFlexGroup>
+          ) : (
+            fieldFormats
+              .getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE])
+              .convert(timestamp)
+          )}
+        </EuiSkeletonRectangle>
+      ),
+      width: '300px',
+      sortable: true,
+    },
     {
       name: actionsColumnName,
       render: (dataStreamStat: DataStreamStat) => (

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts

@@ -19,7 +19,7 @@ import { _IGNORED } from '../../../../common/es_fields';
 import { DataStreamDetails, DataStreamSettings } from '../../../../common/api_types';
 import { createDatasetQualityESClient } from '../../../utils';
 import { dataStreamService, datasetQualityPrivileges } from '../../../services';
-import { getDataStreamsStats } from '../get_data_streams_stats';
+import { getDataStreams } from '../get_data_streams';
 
 export async function getDataStreamSettings({
   esClient,
@@ -49,28 +49,27 @@ export async function getDataStreamDetails({
   dataStream,
   start,
   end,
-  sizeStatsAvailable = true,
+  isServerless,
 }: {
   esClient: ElasticsearchClient;
   dataStream: string;
   start: number;
   end: number;
-  sizeStatsAvailable?: boolean; // Only Needed to determine whether `_stats` endpoint is available https://github.com/elastic/kibana/issues/178954
+  isServerless: boolean;
 }): Promise<DataStreamDetails> {
   throwIfInvalidDataStreamParams(dataStream);
 
   const hasAccessToDataStream = (
     await datasetQualityPrivileges.getHasIndexPrivileges(esClient, [dataStream], ['monitor'])
   )[dataStream];
 
-  const lastActivity = hasAccessToDataStream
+  const esDataStream = hasAccessToDataStream
     ? (
-        await getDataStreamsStats({
+        await getDataStreams({
           esClient,
-          dataStreams: [dataStream],
-          sizeStatsAvailable,
+          datasetQuery: dataStream,
         })
-      ).items[0]?.lastActivity
+      ).dataStreams[0]
     : undefined;
 
   try {
@@ -82,17 +81,17 @@ export async function getDataStreamDetails({
     );
 
     const whenSizeStatsNotAvailable = NaN; // This will indicate size cannot be calculated
-    const avgDocSizeInBytes = sizeStatsAvailable
-      ? hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0
-        ? await getAvgDocSizeInBytes(esClient, dataStream)
-        : 0
-      : whenSizeStatsNotAvailable;
+    const avgDocSizeInBytes = isServerless
+      ? whenSizeStatsNotAvailable
+      : hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0
+      ? await getAvgDocSizeInBytes(esClient, dataStream)
+      : 0;
     const sizeBytes = Math.ceil(avgDocSizeInBytes * dataStreamSummaryStats.docsCount);
 
     return {
       ...dataStreamSummaryStats,
       sizeBytes,
-      lastActivity,
+      lastActivity: esDataStream?.lastActivity,
       userPrivileges: {
         canMonitor: hasAccessToDataStream,
       },

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts

@@ -40,8 +40,22 @@ describe('getDataStreams', () => {
     const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
     const result = await getDataStreams({
       esClient: esClientMock,
-      types: ['logs'],
-      datasetQuery: 'nginx-*',
+      types: ['logs', 'metrics'],
+      uncategorisedOnly: true,
+    });
+    expect(dataStreamService.getMatchingDataStreams).toHaveBeenCalledWith(
+      expect.anything(),
+      'logs-*-*,metrics-*-*'
+    );
+
+    expect(result.datasetUserPrivileges.canMonitor).toBe(true);
+  });
+
+  it('Passes datasetQuery parameter to the DataStreamService', async () => {
+    const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
+    const result = await getDataStreams({
+      esClient: esClientMock,
+      datasetQuery: 'logs-nginx-*',
       uncategorisedOnly: true,
     });
     expect(dataStreamService.getMatchingDataStreams).toHaveBeenCalledWith(
@@ -58,20 +72,18 @@ describe('getDataStreams', () => {
       const results = await getDataStreams({
         esClient: esClientMock,
         types: ['logs'],
-        datasetQuery: 'nginx-*',
         uncategorisedOnly: true,
       });
-      expect(results.items.length).toBe(1);
+      expect(results.dataStreams.length).toBe(1);
     });
     it('Returns the correct number of results when false', async () => {
       const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
       const results = await getDataStreams({
         esClient: esClientMock,
         types: ['logs'],
-        datasetQuery: 'nginx-*',
         uncategorisedOnly: false,
       });
-      expect(results.items.length).toBe(5);
+      expect(results.dataStreams.length).toBe(5);
     });
   });
 });

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams/index.ts

@@ -12,18 +12,20 @@ import { dataStreamService, datasetQualityPrivileges } from '../../../services';
 
 export async function getDataStreams(options: {
   esClient: ElasticsearchClient;
-  types: DataStreamType[];
+  types?: DataStreamType[];
   datasetQuery?: string;
-  uncategorisedOnly: boolean;
+  uncategorisedOnly?: boolean;
 }) {
-  const { esClient, types, datasetQuery, uncategorisedOnly } = options;
+  const { esClient, types = [], datasetQuery, uncategorisedOnly } = options;
 
-  const datasetNames = types.map((type) =>
-    streamPartsToIndexPattern({
-      typePattern: type,
-      datasetPattern: datasetQuery ? `${datasetQuery}` : '*-*',
-    })
-  );
+  const datasetNames = datasetQuery
+    ? [datasetQuery]
+    : types.map((type) =>
+        streamPartsToIndexPattern({
+          typePattern: type,
+          datasetPattern: '*-*',
+        })
+      );
 
   const datasetUserPrivileges = await datasetQualityPrivileges.getDatasetPrivileges(
     esClient,
@@ -32,7 +34,7 @@ export async function getDataStreams(options: {
 
   if (!datasetUserPrivileges.canMonitor) {
     return {
-      items: [],
+      dataStreams: [],
       datasetUserPrivileges,
     };
   }
@@ -59,13 +61,15 @@ export async function getDataStreams(options: {
   const mappedDataStreams = filteredDataStreams.map((dataStream) => ({
     name: dataStream.name,
     integration: dataStream._meta?.package?.name,
+    // @ts-expect-error
+    lastActivity: dataStream.maximum_timestamp,
     userPrivileges: {
       canMonitor: dataStreamsPrivileges[dataStream.name],
     },
   }));
 
   return {
-    items: mappedDataStreams,
+    dataStreams: mappedDataStreams,
     datasetUserPrivileges,
   };
 }

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts

@@ -12,42 +12,31 @@ import { indexStatsService } from '../../../services';
 export async function getDataStreamsStats({
   esClient,
   dataStreams,
-  sizeStatsAvailable = true,
 }: {
   esClient: ElasticsearchClient;
   dataStreams: string[];
-  sizeStatsAvailable?: boolean; // Only Needed to determine whether `_stats` endpoint is available https://github.com/elastic/kibana/issues/178954
-}) {
+}): Promise<Record<string, { size: string; sizeBytes: number; totalDocs: number }>> {
   if (!dataStreams.length) {
-    return {
-      items: [],
-    };
+    return {};
   }
 
   const matchingDataStreamsStats = dataStreamService.getStreamsStats(esClient, dataStreams);
-
-  const indicesDocsCount = sizeStatsAvailable
-    ? indexStatsService.getIndicesDocCounts(esClient, dataStreams)
-    : Promise.resolve(null);
+  const indicesDocsCount = indexStatsService.getIndicesDocCounts(esClient, dataStreams);
 
   const [indicesDocsCountStats, dataStreamsStats] = await Promise.all([
     indicesDocsCount,
     matchingDataStreamsStats,
   ]);
 
-  const mappedDataStreams = dataStreamsStats.map((dataStream) => {
-    return {
-      name: dataStream.data_stream,
-      size: dataStream.store_size?.toString(),
-      sizeBytes: dataStream.store_size_bytes,
-      lastActivity: dataStream.maximum_timestamp,
-      totalDocs: sizeStatsAvailable
-        ? indicesDocsCountStats!.docsCountPerDataStream[dataStream.data_stream] || 0
-        : null,
-    };
-  });
-
-  return {
-    items: mappedDataStreams,
-  };
+  return dataStreamsStats.reduce(
+    (acc, dataStream) => ({
+      ...acc,
+      [dataStream.data_stream]: {
+        size: dataStream.store_size!.toString(),
+        sizeBytes: dataStream.store_size_bytes,
+        totalDocs: indicesDocsCountStats!.docsCountPerDataStream[dataStream.data_stream],
+      },
+    }),
+    {}
+  );
 }