[Fleet] Support browsing granular integrations (#99866) (#100707)

* Manual cherry pick of work to support integration tiles and package-level vars

* Fix types

* Remove registry input group typings

* Show integration-specific readme, title, and icon in package details page

* Revert unnecessary changes

* Add package-level `vars` field to package policy SO mappings

* Fix types

* Fix test

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Jen Huang <its.jenetic@gmail.com>

x-pack/plugins/fleet/common/services/index.ts

@@ -16,3 +16,4 @@ export { isValidNamespace } from './is_valid_namespace';
 export { isDiffPathProtocol } from './is_diff_path_protocol';
 export { LicenseService } from './license';
 export { isAgentUpgradeable } from './is_agent_upgradeable';
+export { doesPackageHaveIntegrations } from './packages_with_integrations';

x-pack/plugins/fleet/common/services/package_to_package_policy.ts

@@ -40,6 +40,21 @@ const getStreamsForInputType = (
   return streams;
 };
 
+// Reduces registry var def into config object entry
+const varsReducer = (
+  configObject: PackagePolicyConfigRecord,
+  registryVar: RegistryVarsEntry
+): PackagePolicyConfigRecord => {
+  const configEntry: PackagePolicyConfigRecordEntry = {
+    value: !registryVar.default && registryVar.multi ? [] : registryVar.default,
+  };
+  if (registryVar.type) {
+    configEntry.type = registryVar.type;
+  }
+  configObject![registryVar.name] = configEntry;
+  return configObject;
+};
+
 /*
  * This service creates a package policy inputs definition from defaults provided in package info
  */
@@ -58,21 +73,6 @@ export const packageToPackagePolicyInputs = (
   if (packagePolicyTemplate?.inputs?.length) {
     // Map each package package policy input to agent policy package policy input
     packagePolicyTemplate.inputs.forEach((packageInput) => {
-      // Reduces registry var def into config object entry
-      const varsReducer = (
-        configObject: PackagePolicyConfigRecord,
-        registryVar: RegistryVarsEntry
-      ): PackagePolicyConfigRecord => {
-        const configEntry: PackagePolicyConfigRecordEntry = {
-          value: !registryVar.default && registryVar.multi ? [] : registryVar.default,
-        };
-        if (registryVar.type) {
-          configEntry.type = registryVar.type;
-        }
-        configObject![registryVar.name] = configEntry;
-        return configObject;
-      };
-
       // Map each package input stream into package policy input stream
       const streams: NewPackagePolicyInputStream[] = getStreamsForInputType(
         packageInput.type,
@@ -121,7 +121,7 @@ export const packageToPackagePolicy = (
   packagePolicyName?: string,
   description?: string
 ): NewPackagePolicy => {
-  return {
+  const packagePolicy: NewPackagePolicy = {
     name: packagePolicyName || `${packageInfo.name}-1`,
     namespace,
     description,
@@ -135,4 +135,10 @@ export const packageToPackagePolicy = (
     output_id: outputId,
     inputs: packageToPackagePolicyInputs(packageInfo),
   };
+
+  if (packageInfo.vars?.length) {
+    packagePolicy.vars = packageInfo.vars.reduce(varsReducer, {});
+  }
+
+  return packagePolicy;
 };

x-pack/plugins/fleet/common/services/packages_with_integrations.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { PackageInfo, PackageListItem } from '../types';
+
+export const doesPackageHaveIntegrations = (pkgInfo: PackageInfo | PackageListItem) => {
+  return (pkgInfo.policy_templates || []).length > 1;
+};

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -19,7 +19,12 @@ import type {
 } from '../../constants';
 import type { ValueOf } from '../../types';
 
-import type { PackageSpecManifest, PackageSpecScreenshot } from './package_spec';
+import type {
+  PackageSpecManifest,
+  PackageSpecIcon,
+  PackageSpecScreenshot,
+  PackageSpecCategory,
+} from './package_spec';
 
 export type InstallationStatus = typeof installationStatuses;
 
@@ -118,27 +123,33 @@ interface RegistryOverridePropertyValue {
 }
 
 export type RegistryRelease = PackageSpecManifest['release'];
-export interface RegistryImage {
-  src: string;
+export interface RegistryImage extends PackageSpecIcon {
   path: string;
-  title?: string;
-  size?: string;
-  type?: string;
 }
 
 export enum RegistryPolicyTemplateKeys {
   name = 'name',
   title = 'title',
   description = 'description',
+  icons = 'icons',
+  screenshots = 'screenshots',
+  categories = 'categories',
+  data_streams = 'data_streams',
   inputs = 'inputs',
+  readme = 'readme',
   multiple = 'multiple',
 }
 
 export interface RegistryPolicyTemplate {
   [RegistryPolicyTemplateKeys.name]: string;
   [RegistryPolicyTemplateKeys.title]: string;
   [RegistryPolicyTemplateKeys.description]: string;
+  [RegistryPolicyTemplateKeys.icons]?: RegistryImage[];
+  [RegistryPolicyTemplateKeys.screenshots]?: RegistryImage[];
+  [RegistryPolicyTemplateKeys.categories]?: Array<PackageSpecCategory | undefined>;
+  [RegistryPolicyTemplateKeys.data_streams]?: string[];
   [RegistryPolicyTemplateKeys.inputs]?: RegistryInput[];
+  [RegistryPolicyTemplateKeys.readme]?: string;
   [RegistryPolicyTemplateKeys.multiple]?: boolean;
 }
 
@@ -148,15 +159,19 @@ export enum RegistryInputKeys {
   description = 'description',
   template_path = 'template_path',
   condition = 'condition',
+  input_group = 'input_group',
   vars = 'vars',
 }
 
+export type RegistryInputGroup = 'logs' | 'metrics';
+
 export interface RegistryInput {
   [RegistryInputKeys.type]: string;
   [RegistryInputKeys.title]: string;
   [RegistryInputKeys.description]: string;
   [RegistryInputKeys.template_path]?: string;
   [RegistryInputKeys.condition]?: string;
+  [RegistryInputKeys.input_group]?: RegistryInputGroup;
   [RegistryInputKeys.vars]?: RegistryVarsEntry[];
 }
 
@@ -273,7 +288,7 @@ export interface RegistryDataStream {
   [RegistryDataStreamKeys.streams]?: RegistryStream[];
   [RegistryDataStreamKeys.package]: string;
   [RegistryDataStreamKeys.path]: string;
-  [RegistryDataStreamKeys.ingest_pipeline]: string;
+  [RegistryDataStreamKeys.ingest_pipeline]?: string;
   [RegistryDataStreamKeys.elasticsearch]?: RegistryElasticsearch;
   [RegistryDataStreamKeys.dataset_is_prefix]?: boolean;
 }
@@ -307,7 +322,7 @@ export interface RegistryVarsEntry {
   [RegistryVarsEntryKeys.required]?: boolean;
   [RegistryVarsEntryKeys.show_user]?: boolean;
   [RegistryVarsEntryKeys.multi]?: boolean;
-  [RegistryVarsEntryKeys.default]?: string | string[];
+  [RegistryVarsEntryKeys.default]?: string | string[] | boolean;
   [RegistryVarsEntryKeys.os]?: {
     [key: string]: {
       default: string | string[];
@@ -329,8 +344,11 @@ type Merge<FirstType, SecondType> = Omit<FirstType, Extract<keyof FirstType, key
 
 // Managers public HTTP response types
 export type PackageList = PackageListItem[];
+export type PackageListItem = Installable<RegistrySearchResult> & {
+  integration?: string;
+  id: string;
+};
 
-export type PackageListItem = Installable<RegistrySearchResult>;
 export type PackagesGroupedByStatus = Record<ValueOf<InstallationStatus>, PackageList>;
 export type PackageInfo =
   | Installable<Merge<RegistryPackage, EpmPackageAdditions>>

x-pack/plugins/fleet/common/types/models/package_policy.ts

@@ -58,6 +58,7 @@ export interface NewPackagePolicy {
   output_id: string;
   package?: PackagePolicyPackage;
   inputs: NewPackagePolicyInput[];
+  vars?: PackagePolicyConfigRecord;
 }
 
 export interface UpdatePackagePolicy extends NewPackagePolicy {

x-pack/plugins/fleet/common/types/models/package_spec.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { RegistryPolicyTemplate } from './epm';
+import type { RegistryPolicyTemplate, RegistryVarsEntry } from './epm';
 
 // Based on https://github.com/elastic/package-spec/blob/master/versions/1/manifest.spec.yml#L8
 export interface PackageSpecManifest {
@@ -22,6 +22,7 @@ export interface PackageSpecManifest {
   icons?: PackageSpecIcon[];
   screenshots?: PackageSpecScreenshot[];
   policy_templates?: RegistryPolicyTemplate[];
+  vars?: RegistryVarsEntry[];
   owner: { github: string };
 }
 

x-pack/plugins/fleet/common/types/rest_spec/epm.ts

@@ -8,8 +8,7 @@
 import type {
   AssetReference,
   CategorySummaryList,
-  Installable,
-  RegistrySearchResult,
+  PackageList,
   PackageInfo,
   PackageUsageStats,
   InstallType,
@@ -18,6 +17,7 @@ import type {
 export interface GetCategoriesRequest {
   query: {
     experimental?: boolean;
+    include_policy_templates?: boolean;
   };
 }
 
@@ -33,7 +33,7 @@ export interface GetPackagesRequest {
 }
 
 export interface GetPackagesResponse {
-  response: Array<Installable<RegistrySearchResult>>;
+  response: PackageList;
 }
 
 export interface GetLimitedPackagesResponse {

x-pack/plugins/fleet/public/applications/fleet/components/package_icon.tsx

@@ -14,7 +14,7 @@ import { usePackageIconType } from '../hooks';
 
 export const PackageIcon: React.FunctionComponent<
   UsePackageIconType & Omit<EuiIconProps, 'type'>
-> = ({ packageName, version, icons, tryApi, ...euiIconProps }) => {
-  const iconType = usePackageIconType({ packageName, version, icons, tryApi });
+> = ({ packageName, integrationName, version, icons, tryApi, ...euiIconProps }) => {
+  const iconType = usePackageIconType({ packageName, integrationName, version, icons, tryApi });
   return <EuiIcon size="s" type={iconType} {...euiIconProps} />;
 };

x-pack/plugins/fleet/public/applications/fleet/constants/page_paths.ts

@@ -56,7 +56,7 @@ export const PAGE_ROUTING_PATHS = {
   policy_details: '/policies/:policyId/:tabId?',
   policy_details_settings: '/policies/:policyId/settings',
   add_integration_from_policy: '/policies/:policyId/add-integration',
-  add_integration_to_policy: '/integrations/:pkgkey/add-integration',
+  add_integration_to_policy: '/integrations/:pkgkey/add-integration/:integration?',
   edit_integration: '/policies/:policyId/edit-integration/:packagePolicyId',
   fleet: '/fleet',
   fleet_agent_list: '/fleet/agents',
@@ -77,17 +77,22 @@ export const pagePathGetters: {
   integrations: () => '/integrations',
   integrations_all: () => '/integrations',
   integrations_installed: () => '/integrations/installed',
-  integration_details_overview: ({ pkgkey }) => `/integrations/detail/${pkgkey}/overview`,
-  integration_details_policies: ({ pkgkey }) => `/integrations/detail/${pkgkey}/policies`,
-  integration_details_settings: ({ pkgkey }) => `/integrations/detail/${pkgkey}/settings`,
-  integration_details_custom: ({ pkgkey }) => `/integrations/detail/${pkgkey}/custom`,
+  integration_details_overview: ({ pkgkey, integration }) =>
+    `/integrations/detail/${pkgkey}/overview${integration ? `?integration=${integration}` : ''}`,
+  integration_details_policies: ({ pkgkey, integration }) =>
+    `/integrations/detail/${pkgkey}/policies${integration ? `?integration=${integration}` : ''}`,
+  integration_details_settings: ({ pkgkey, integration }) =>
+    `/integrations/detail/${pkgkey}/settings${integration ? `?integration=${integration}` : ''}`,
+  integration_details_custom: ({ pkgkey, integration }) =>
+    `/integrations/detail/${pkgkey}/custom${integration ? `?integration=${integration}` : ''}`,
   integration_policy_edit: ({ packagePolicyId }) =>
     `/integrations/edit-integration/${packagePolicyId}`,
   policies: () => '/policies',
   policies_list: () => '/policies',
   policy_details: ({ policyId, tabId }) => `/policies/${policyId}${tabId ? `/${tabId}` : ''}`,
   add_integration_from_policy: ({ policyId }) => `/policies/${policyId}/add-integration`,
-  add_integration_to_policy: ({ pkgkey }) => `/integrations/${pkgkey}/add-integration`,
+  add_integration_to_policy: ({ pkgkey, integration }) =>
+    `/integrations/${pkgkey}/add-integration${integration ? `/${integration}` : ''}`,
   edit_integration: ({ policyId, packagePolicyId }) =>
     `/policies/${policyId}/edit-integration/${packagePolicyId}`,
   fleet: () => '/fleet',

x-pack/plugins/fleet/public/applications/fleet/hooks/use_package_icon_type.ts

@@ -16,7 +16,8 @@ import { sendGetPackageInfoByKey } from './index';
 type Package = PackageInfo | PackageListItem;
 
 export interface UsePackageIconType {
-  packageName: Package['name'];
+  packageName: string;
+  integrationName?: string;
   version: Package['version'];
   icons?: Package['icons'];
   tryApi?: boolean; // should it call API to try to find missing icons?
@@ -26,20 +27,21 @@ const CACHED_ICONS = new Map<string, string>();
 
 export const usePackageIconType = ({
   packageName,
+  integrationName,
   version,
   icons: paramIcons,
   tryApi = false,
 }: UsePackageIconType) => {
   const { toPackageImage } = useLinks();
   const [iconList, setIconList] = useState<UsePackageIconType['icons']>();
   const [iconType, setIconType] = useState<string>(''); // FIXME: use `empty` icon during initialization - see: https://github.com/elastic/kibana/issues/60622
-  const pkgKey = `${packageName}-${version}`;
+  const cacheKey = `${packageName}-${version}${integrationName ? `-${integrationName}` : ''}`;
 
   // Generates an icon path or Eui Icon name based on an icon list from the package
   // or by using the package name against logo icons from Eui
   useEffect(() => {
-    if (CACHED_ICONS.has(pkgKey)) {
-      setIconType(CACHED_ICONS.get(pkgKey) || '');
+    if (CACHED_ICONS.has(cacheKey)) {
+      setIconType(CACHED_ICONS.get(cacheKey) || '');
       return;
     }
     const svgIcons = (paramIcons || iconList)?.filter(
@@ -48,29 +50,29 @@ export const usePackageIconType = ({
     const localIconSrc =
       Array.isArray(svgIcons) && toPackageImage(svgIcons[0], packageName, version);
     if (localIconSrc) {
-      CACHED_ICONS.set(pkgKey, localIconSrc);
-      setIconType(CACHED_ICONS.get(pkgKey) || '');
+      CACHED_ICONS.set(cacheKey, localIconSrc);
+      setIconType(CACHED_ICONS.get(cacheKey) || '');
       return;
     }
 
     const euiLogoIcon = ICON_TYPES.find((key) => key.toLowerCase() === `logo${packageName}`);
     if (euiLogoIcon) {
-      CACHED_ICONS.set(pkgKey, euiLogoIcon);
+      CACHED_ICONS.set(cacheKey, euiLogoIcon);
       setIconType(euiLogoIcon);
       return;
     }
 
     if (tryApi && !paramIcons && !iconList) {
-      sendGetPackageInfoByKey(pkgKey)
+      sendGetPackageInfoByKey(cacheKey)
         .catch((error) => undefined) // Ignore API errors
         .then((res) => {
-          CACHED_ICONS.delete(pkgKey);
+          CACHED_ICONS.delete(cacheKey);
           setIconList(res?.data?.response?.icons);
         });
     }
 
-    CACHED_ICONS.set(pkgKey, 'package');
+    CACHED_ICONS.set(cacheKey, 'package');
     setIconType('package');
-  }, [paramIcons, pkgKey, toPackageImage, iconList, packageName, iconType, tryApi, version]);
+  }, [paramIcons, cacheKey, toPackageImage, iconList, packageName, iconType, tryApi, version]);
   return iconType;
 };

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/index.tsx

@@ -333,6 +333,7 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => {
             packagePolicy={packagePolicy}
             updatePackagePolicy={updatePackagePolicy}
             validationResults={validationResults!}
+            submitAttempted={formState === 'INVALID'}
           />
 
           {/* Only show the out-of-box configuration step if a UI extension is NOT registered */}