Fleet Usage telemetry extension

x-pack/plugins/fleet/server/collectors/agent_collectors.ts

@@ -8,7 +8,9 @@
 import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server';
 
 import type { FleetConfigType } from '../../common/types';
+import { AGENTS_INDEX } from '../../common';
 import * as AgentService from '../services/agents';
+import { appContextService } from '../services';
 
 export interface AgentUsage {
   total_enrolled: number;
@@ -47,3 +49,97 @@ export const getAgentUsage = async (
     updating,
   };
 };
+
+export interface AgentData {
+  agent_versions: string[];
+  agent_checkin_status: {
+    error: number;
+    degraded: number;
+  };
+  agent_checkin_status_last_1h: {
+    error: number;
+    degraded: number;
+  };
+}
+
+const DEFAULT_AGENT_DATA = {
+  agent_versions: [],
+  agent_checkin_status: { error: 0, degraded: 0 },
+  agent_checkin_status_last_1h: { error: 0, degraded: 0 },
+};
+
+export const getAgentData = async (esClient?: ElasticsearchClient): Promise<AgentData> => {
+  if (!esClient) {
+    return DEFAULT_AGENT_DATA;
+  }
+  try {
+    const transformLastCheckinStatusBuckets = (resp: any) =>
+      ((resp?.aggregations?.last_checkin_status as any).buckets ?? []).reduce(
+        (acc: any, bucket: any) => {
+          if (acc[bucket.key] !== undefined) acc[bucket.key] = bucket.doc_count;
+          return acc;
+        },
+        { error: 0, degraded: 0 }
+      );
+    const response = await esClient.search({
+      index: AGENTS_INDEX,
+      size: 0,
+      aggs: {
+        versions: {
+          terms: { field: 'agent.version' },
+        },
+        last_checkin_status: {
+          terms: { field: 'last_checkin_status' },
+        },
+      },
+    });
+    const versions = ((response?.aggregations?.versions as any).buckets ?? []).map(
+      (bucket: any) => bucket.key
+    );
+    const statuses = transformLastCheckinStatusBuckets(response);
+
+    const responseLast1h = await esClient.search({
+      index: AGENTS_INDEX,
+      size: 0,
+      query: {
+        bool: {
+          filter: [
+            {
+              bool: {
+                must: [
+                  {
+                    range: {
+                      last_checkin: {
+                        gte: 'now-1h/h',
+                        lt: 'now/h',
+                      },
+                    },
+                  },
+                ],
+              },
+            },
+          ],
+        },
+      },
+      aggs: {
+        last_checkin_status: {
+          terms: { field: 'last_checkin_status' },
+        },
+      },
+    });
+    const statusesLast1h = transformLastCheckinStatusBuckets(responseLast1h);
+
+    return {
+      agent_versions: versions,
+      agent_checkin_status: statuses,
+      agent_checkin_status_last_1h: statusesLast1h,
+    };
+  } catch (error) {
+    if (error.statusCode === 404) {
+      appContextService.getLogger().debug('Index .fleet-agents does not exist yet.');
+    } else {
+      throw error;
+    }
+    return DEFAULT_AGENT_DATA;
+  }
+};

x-pack/plugins/fleet/server/collectors/agent_policies.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server';
+
+import { AGENT_POLICY_INDEX } from '../../common';
+import { ES_SEARCH_LIMIT } from '../../common/constants';
+
+export const getAgentPoliciesUsage = async (
+  soClient?: SavedObjectsClient,
+  esClient?: ElasticsearchClient
+): Promise<any> => {
+  if (!soClient || !esClient) {
+    return {};
+  }
+
+  const res = await esClient.search({
+    index: AGENT_POLICY_INDEX,
+    size: ES_SEARCH_LIMIT,
+    track_total_hits: true,
+    rest_total_hits_as_int: true,
+  });
+
+  const agentPolicies = res.hits.hits;
+
+  const outputTypes = new Set<string>();
+  agentPolicies.forEach((item) => {
+    const source = (item._source as any) ?? {};
+    Object.keys(source.data.outputs).forEach((output) => {
+      outputTypes.add(source.data.outputs[output].type);
+    });
+  });
+
+  return {
+    count: res.hits.total,
+    output_types: Array.from(outputTypes),
+  };
+};

x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts

@@ -7,6 +7,8 @@
 
 import type { SavedObjectsClient, ElasticsearchClient } from '@kbn/core/server';
 
+import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../constants';
+
 import { packagePolicyService } from '../services';
 import { getAgentStatusForAgentPolicy } from '../services/agents';
 import { listFleetServerHosts } from '../services/fleet_server_host';
@@ -84,3 +86,32 @@ export const getFleetServerUsage = async (
     num_host_urls: numHostsUrls,
   };
 };
+
+export const getFleetServerConfig = async (
+  soClient?: SavedObjectsClient,
+  esClient?: ElasticsearchClient
+): Promise<any> => {
+  if (!soClient || !esClient) {
+    return {};
+  }
+  const fleetServerHosts = await listFleetServerHosts(soClient);
+  const hosts = fleetServerHosts.items.map((item) => ({
+    ...item,
+    proxy_id: (item as any).proxy_id ?? '',
+  }));
+
+  const res = await packagePolicyService.list(soClient, {
+    page: 1,
+    perPage: SO_SEARCH_LIMIT,
+    kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:fleet_server`,
+  });
+  const policies = res.items.map((item) => ({
+    id: item.id,
+    name: item.name,
+    enabled: item.enabled,
+    policy_id: item.policy_id,
+    input_config: (item.inputs[0] ?? {}).compiled_input,
+  }));
+
+  return { hosts, policies };
+};

x-pack/plugins/fleet/server/collectors/register.ts

@@ -11,13 +11,14 @@ import type { CoreSetup } from '@kbn/core/server';
 import type { FleetConfigType } from '..';
 
 import { getIsAgentsEnabled } from './config_collectors';
-import { getAgentUsage } from './agent_collectors';
+import { getAgentUsage, getAgentData } from './agent_collectors';
 import type { AgentUsage } from './agent_collectors';
 import { getInternalClients } from './helpers';
 import { getPackageUsage } from './package_collectors';
 import type { PackageUsage } from './package_collectors';
-import { getFleetServerUsage } from './fleet_server_collector';
+import { getFleetServerUsage, getFleetServerConfig } from './fleet_server_collector';
 import type { FleetServerUsage } from './fleet_server_collector';
+import { getAgentPoliciesUsage } from './agent_policies';
 
 export interface Usage {
   agents_enabled: boolean;
@@ -26,7 +27,22 @@ export interface Usage {
   fleet_server: FleetServerUsage;
 }
 
-export const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => {
+export const fetchFleetUsage = async (core: CoreSetup, config: FleetConfigType) => {
+  const [soClient, esClient] = await getInternalClients(core);
+  const usage = {
+    agents_enabled: getIsAgentsEnabled(config),
+    agents: await getAgentUsage(config, soClient, esClient),
+    fleet_server: await getFleetServerUsage(soClient, esClient),
+    packages: await getPackageUsage(soClient),
+    ...(await getAgentData(esClient)),
+    fleet_server_config: await getFleetServerConfig(soClient, esClient),
+    agent_policies: await getAgentPoliciesUsage(soClient, esClient),
+  };
+  return usage;
+};
+
+// used by kibana daily collector
+const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => {
   const [soClient, esClient] = await getInternalClients(core);
   const usage = {
     agents_enabled: getIsAgentsEnabled(config),

x-pack/plugins/fleet/server/plugin.ts

@@ -88,7 +88,11 @@ import {
   AgentServiceImpl,
   PackageServiceImpl,
 } from './services';
-import { registerFleetUsageCollector, fetchUsage, fetchAgentsUsage } from './collectors/register';
+import {
+  registerFleetUsageCollector,
+  fetchAgentsUsage,
+  fetchFleetUsage,
+} from './collectors/register';
 import { getAuthzFromRequest, makeRouterWithFleetAuthz } from './routes/security';
 import { FleetArtifactsClient } from './services/artifacts';
 import type { FleetRouter } from './types/request_context';
@@ -370,14 +374,8 @@ export class FleetPlugin
 
     // Register usage collection
     registerFleetUsageCollector(core, config, deps.usageCollection);
-    const fetch = async () => fetchUsage(core, config);
-    this.fleetUsageSender = new FleetUsageSender(
-      deps.taskManager,
-      core,
-      fetch,
-      this.kibanaVersion,
-      this.isProductionMode
-    );
+    const fetch = async () => fetchFleetUsage(core, config);
+    this.fleetUsageSender = new FleetUsageSender(deps.taskManager, core, fetch);
     registerFleetUsageLogger(deps.taskManager, async () => fetchAgentsUsage(core, config));
 
     const router: FleetRouter = core.http.createRouter<FleetRequestHandlerContext>();