[Fleet] Improve performance of data stream API (#97058) (#97075)

* Improve performance of data stream API

* Remove extra logger, replace filter with reduce

* Remove unused import
# Conflicts:
#	x-pack/plugins/fleet/server/routes/data_streams/handlers.ts
#	x-pack/plugins/fleet/server/services/epm/packages/get.ts

Author: jen-huang

x-pack/plugins/fleet/server/routes/data_streams/handlers.ts

@@ -5,11 +5,12 @@
  * 2.0.
  */
 
-import { RequestHandler, SavedObjectsClientContract } from 'src/core/server';
+import { RequestHandler, SavedObjectsBulkGetObject } from 'src/core/server';
 import { keyBy, keys, merge } from 'lodash';
 import { DataStream } from '../../types';
-import { GetDataStreamsResponse, KibanaAssetType, KibanaSavedObjectType } from '../../../common';
-import { getPackageSavedObjects, getKibanaSavedObject } from '../../services/epm/packages/get';
+import { GetDataStreamsResponse, KibanaSavedObjectType } from '../../../common';
+import { getPackageSavedObjects } from '../../services/epm/packages/get';
+
 import { defaultIngestErrorHandler } from '../../errors';
 
 const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*,traces-*-*';
@@ -80,6 +81,40 @@ export const getListHandler: RequestHandler = async (context, request, response)
     const packageSavedObjectsByName = keyBy(packageSavedObjects.saved_objects, 'id');
     const packageMetadata: any = {};
 
+    // Get dashboard information for all packages
+    const dashboardIdsByPackageName = packageSavedObjects.saved_objects.reduce<
+      Record<string, string[]>
+    >((allDashboards, pkgSavedObject) => {
+      const dashboards: string[] = [];
+      (pkgSavedObject.attributes?.installed_kibana || []).forEach((o) => {
+        if (o.type === KibanaSavedObjectType.dashboard) {
+          dashboards.push(o.id);
+        }
+      });
+      allDashboards[pkgSavedObject.id] = dashboards;
+      return allDashboards;
+    }, {});
+    const allDashboardSavedObjects = await context.core.savedObjects.client.bulkGet<{
+      title?: string;
+    }>(
+      Object.values(dashboardIdsByPackageName).reduce<SavedObjectsBulkGetObject[]>(
+        (allDashboards, dashboardIds) => {
+          return allDashboards.concat(
+            dashboardIds.map((id) => ({
+              id,
+              type: KibanaSavedObjectType.dashboard,
+              fields: ['title'],
+            }))
+          );
+        },
+        []
+      )
+    );
+    const allDashboardSavedObjectsById = keyBy(
+      allDashboardSavedObjects.saved_objects,
+      (dashboardSavedObject) => dashboardSavedObject.id
+    );
+
     // Query additional information for each data stream
     const dataStreamPromises = dataStreamNames.map(async (dataStreamName) => {
       const dataStream = dataStreams[dataStreamName];
@@ -157,19 +192,23 @@ export const getListHandler: RequestHandler = async (context, request, response)
         // - and we didn't pick the metadata in an earlier iteration of this map()
         if (!packageMetadata[pkgName]) {
           // then pick the dashboards from the package saved object
-          const dashboards =
-            pkgSavedObject.attributes?.installed_kibana?.filter(
-              (o) => o.type === KibanaSavedObjectType.dashboard
-            ) || [];
-          // and then pick the human-readable titles from the dashboard saved objects
-          const enhancedDashboards = await getEnhancedDashboards(
-            context.core.savedObjects.client,
-            dashboards
-          );
+          const packageDashboardIds = dashboardIdsByPackageName[pkgName] || [];
+          const packageDashboards = packageDashboardIds.reduce<
+            Array<{ id: string; title: string }>
+          >((dashboards, dashboardId) => {
+            const dashboard = allDashboardSavedObjectsById[dashboardId];
+            if (dashboard) {
+              dashboards.push({
+                id: dashboard.id,
+                title: dashboard.attributes.title || dashboard.id,
+              });
+            }
+            return dashboards;
+          }, []);
 
           packageMetadata[pkgName] = {
             version: pkgSavedObject.attributes?.version || '',
-            dashboards: enhancedDashboards,
+            dashboards: packageDashboards,
           };
         }
 
@@ -194,21 +233,3 @@ export const getListHandler: RequestHandler = async (context, request, response)
     return defaultIngestErrorHandler({ error, response });
   }
 };
-
-const getEnhancedDashboards = async (
-  savedObjectsClient: SavedObjectsClientContract,
-  dashboards: any[]
-) => {
-  const dashboardsPromises = dashboards.map(async (db) => {
-    const dbSavedObject: any = await getKibanaSavedObject(
-      savedObjectsClient,
-      KibanaAssetType.dashboard,
-      db.id
-    );
-    return {
-      id: db.id,
-      title: dbSavedObject.attributes?.title || db.id,
-    };
-  });
-  return await Promise.all(dashboardsPromises);
-};

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -15,7 +15,7 @@ import {
 } from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import { ArchivePackage, RegistryPackage, EpmPackageAdditions } from '../../../../common/types';
-import { Installation, PackageInfo, KibanaAssetType } from '../../../types';
+import { Installation, PackageInfo } from '../../../types';
 import { IngestManagerError } from '../../../errors';
 import { appContextService } from '../../';
 import * as Registry from '../registry';
@@ -254,11 +254,3 @@ function sortByName(a: { name: string }, b: { name: string }) {
     return 0;
   }
 }
-
-export async function getKibanaSavedObject(
-  savedObjectsClient: SavedObjectsClientContract,
-  type: KibanaAssetType,
-  id: string
-) {
-  return savedObjectsClient.get(type, id);
-}