[EPM]: Assign data source to policy in UI (elastic#53597)

* Let ES generate source ids. Refactor along the way.
* Datasource.id isn't optional. It's just missing before we send to Ingest
* Delete EPM's mapping of datasources saved object. Ingest handles that.
* Keep datasource object-related work in constructDatasource
* Move asset installation into own function. Keep entry point high-level.
* More descriptive (less ambiguous) names for these two functions
* Use enum values from Ingest instead of plain strings
* Limit the 'type' key of references to known asset types.
* Update variable names to clarify that we're merging arrays of references
* Use [].flat instead .reduce + .concat to avoid error on empty arrays.
* Pass PackageInfo value directly to component vs pulling off n properties
* Name handlers/options based on the data, not the UI element
* Populate policy combo box based on values from Ingest policy API
* Mark Dataset.vars as optional.
* Add TODOs

x-pack/legacy/plugins/epm/common/constants.ts

@@ -18,4 +18,6 @@ export const PLUGIN = {
 };
 
 export const SAVED_OBJECT_TYPE_PACKAGES = 'epm-package';
-export const SAVED_OBJECT_TYPE_DATASOURCES = 'epm-datasource';
+// This is actually controled by Ingest
+// TODO: Ultimately, EPM should a) import this or b) not know about it at all
+export const SAVED_OBJECT_TYPE_DATASOURCES = 'datasources';

x-pack/legacy/plugins/epm/common/routes.ts

@@ -42,3 +42,4 @@ export function getRemovePath(pkgkey: string) {
 }
 
 export const getInstallDatasourcePath = () => API_INSTALL_DATASOURCE_PATTERN;
+export const getListPoliciesPath = () => '/api/ingest/policies';

x-pack/legacy/plugins/epm/common/types.ts

@@ -9,6 +9,7 @@ import {
   SavedObjectAttributes,
   SavedObjectReference,
 } from '../../../../../src/core/server';
+import { AssetType as IngestAssetType } from '../../ingest/server/libs/types';
 
 export enum InstallationStatus {
   installed = 'installed',
@@ -124,7 +125,7 @@ export interface Dataset {
   name: string;
   release: string;
   ingest_pipeline: string;
-  vars: VarsEntry[];
+  vars?: VarsEntry[];
   type: string;
   // This is for convenience and not in the output from the registry. When creating a dataset, this info should be added.
   package: string;
@@ -164,10 +165,13 @@ export type NotInstalled<T = {}> = T & {
   status: InstallationStatus.notInstalled;
 };
 
-export type AssetReference = Pick<SavedObjectReference, 'id' | 'type'>;
+export type AssetReference = Pick<SavedObjectReference, 'id'> & {
+  type: AssetType | IngestAssetType;
+};
 
 export interface DatasourcePayload {
   pkgkey: string;
   datasourceName: string;
   datasets: Dataset[];
+  policyIds: string[];
 }

x-pack/legacy/plugins/epm/public/data.ts

@@ -12,6 +12,7 @@ import {
   getInstallDatasourcePath,
   getInstallPath,
   getListPath,
+  getListPoliciesPath,
   getRemovePath,
   ListParams,
 } from '../common/routes';
@@ -23,6 +24,7 @@ import {
   PackagesGroupedByStatus,
   DatasourcePayload,
 } from '../common/types';
+import { ReturnTypeList } from '../../ingest/common/types/std_return_format';
 
 const defaultClient: HttpHandler = (path, options?) => fetch(path, options).then(res => res.json());
 
@@ -88,3 +90,16 @@ export async function installDatasource(datasource: DatasourcePayload): Promise<
   const body = JSON.stringify(datasource);
   return _fetch(path, { body, method: 'POST' });
 }
+
+// TODO: This should come from the shared Ingest types
+// However, they are in a /server directory so /public/* cannot access them
+// Using this partial/placeholder type until we can figure out how to share
+interface PlaceholderPolicy {
+  name: string;
+  id: string;
+}
+
+export async function getPolicies(): Promise<ReturnTypeList<PlaceholderPolicy>> {
+  const path = getListPoliciesPath();
+  return _fetch(path);
+}

x-pack/legacy/plugins/epm/public/screens/add_data_source/add_data_source_form.tsx

@@ -5,18 +5,19 @@
  */
 import {
   EuiButton,
+  EuiComboBoxOptionProps,
   EuiHorizontalRule,
   EuiPanel,
   EuiSteps,
   EuiCheckboxGroupIdToSelectedMap,
 } from '@elastic/eui';
-import React, { Fragment, useState } from 'react';
+import React, { Fragment, useEffect, useState } from 'react';
 import { Redirect } from 'react-router-dom';
 import styled from 'styled-components';
-import { installDatasource } from '../../data';
+import { installDatasource, getPolicies } from '../../data';
 import { useCore, useLinks } from '../../hooks';
 import { StepOne } from './step_one';
-import { Dataset } from '../../../common/types';
+import { PackageInfo } from '../../../common/types';
 
 const StyledSteps = styled.div`
   .euiStep__titleWrapper {
@@ -28,60 +29,78 @@ const StyledSteps = styled.div`
   }
 `;
 interface AddDataSourceStepsProps {
-  pkgName: string;
-  pkgTitle: string;
-  pkgVersion: string;
-  datasets: Dataset[];
+  package: PackageInfo;
+}
+interface PolicyOption {
+  label: string;
+  value: string;
 }
 export interface FormState {
   datasourceName: string;
   datasets: EuiCheckboxGroupIdToSelectedMap;
+  policies: Array<EuiComboBoxOptionProps<string>>;
 }
+
 const FormNav = styled.div`
   text-align: right;
 `;
 
-export const AddDataSourceForm = (props: AddDataSourceStepsProps) => {
+export const AddDataSourceForm = ({ package: pkg }: AddDataSourceStepsProps) => {
+  const defaultPolicyOption: PolicyOption = { label: 'Default policy', value: 'default' };
+  const [policyOptions, setPolicyOptions] = useState<PolicyOption[]>([defaultPolicyOption]);
+  useEffect(() => {
+    getPolicies()
+      .then(response => response.list)
+      .then(policies => policies.map(policy => ({ label: policy.name, value: policy.id })))
+      .then(setPolicyOptions);
+  }, []);
+
   const [addDataSourceSuccess, setAddDataSourceSuccess] = useState<boolean>(false);
-  const [formState, setFormState] = useState<FormState>({ datasourceName: '', datasets: {} });
+  const [datasourceName, setDatasourceName] = useState<FormState['datasourceName']>('');
+  const [selectedDatasets, setSelectedDatasets] = useState<FormState['datasets']>({});
+  const [selectedPolicies, setSelectedPolicies] = useState<FormState['policies']>([
+    defaultPolicyOption,
+  ]);
+
+  const formState: FormState = {
+    datasourceName,
+    datasets: selectedDatasets,
+    policies: selectedPolicies,
+  };
+
   const { notifications } = useCore();
   const { toDetailView } = useLinks();
-  const { pkgName, pkgTitle, pkgVersion, datasets } = props;
-
+  const datasets = pkg?.datasets || [];
   const handleRequestInstallDatasource = async () => {
     try {
       await installDatasource({
-        pkgkey: `${pkgName}-${pkgVersion}`,
+        pkgkey: `${pkg.name}-${pkg.version}`,
         datasets: datasets.filter(d => formState.datasets[d.name] === true),
         datasourceName: formState.datasourceName,
+        // @ts-ignore not sure where/how to enforce a `value` key on options
+        policyIds: formState.policies.map(({ value }) => value),
       });
       setAddDataSourceSuccess(true);
       notifications.toasts.addSuccess({
-        title: `Added ${pkgTitle} data source`,
+        title: `Added ${pkg.title} data source`,
       });
       return;
     } catch (err) {
       notifications.toasts.addWarning({
-        title: `Failed to add data source to ${pkgTitle}`,
+        title: `Failed to add data source to ${pkg.title}`,
         iconType: 'alert',
       });
     }
   };
 
-  const onCheckboxChange = (name: string) => {
-    const newCheckboxStateMap = {
-      ...formState,
-      datasets: {
-        ...formState.datasets,
-        [name]: !formState.datasets[name],
-      },
-    };
-    setFormState(newCheckboxStateMap);
-  };
+  const onDatasetChange = (id: string) =>
+    setSelectedDatasets({
+      ...selectedDatasets,
+      [id]: !selectedDatasets[id],
+    });
 
-  const onTextChange = (evt: React.ChangeEvent<HTMLInputElement>) => {
-    setFormState({ ...formState, [evt.target.name]: evt.target.value });
-  };
+  const onNameChange = (evt: React.ChangeEvent<HTMLInputElement>) =>
+    setDatasourceName(evt.target.value);
 
   // create checkbox items from datasets for EuiCheckboxGroup
   const checkboxes = datasets.map(dataset => ({
@@ -95,8 +114,10 @@ export const AddDataSourceForm = (props: AddDataSourceStepsProps) => {
       children: (
         <StepOne
           datasetCheckboxes={checkboxes}
-          onCheckboxChange={onCheckboxChange}
-          onTextChange={onTextChange}
+          onDatasetChange={onDatasetChange}
+          onNameChange={onNameChange}
+          policyOptions={policyOptions}
+          onPolicyChange={setSelectedPolicies}
           formState={formState}
         />
       ),
@@ -108,8 +129,8 @@ export const AddDataSourceForm = (props: AddDataSourceStepsProps) => {
       {addDataSourceSuccess ? (
         <Redirect
           to={toDetailView({
-            name: pkgName,
-            version: pkgVersion,
+            name: pkg.name,
+            version: pkg.version,
             panel: 'data-sources',
             withAppRoot: false,
           })}

x-pack/legacy/plugins/epm/public/screens/add_data_source/index.tsx

@@ -76,14 +76,7 @@ export function AddDataSource({ pkgkey }: AddDataSourceProps) {
               <h1>Add {title} data source</h1>
             </EuiTitle>
           </EuiPageHeader>
-          {datasets && (
-            <AddDataSourceForm
-              pkgName={name}
-              pkgVersion={version}
-              pkgTitle={title}
-              datasets={datasets}
-            />
-          )}
+          {datasets && <AddDataSourceForm package={info} />}
         </PageBody>
       </EuiFlexGroup>
     </PageContainer>

x-pack/legacy/plugins/epm/public/screens/add_data_source/step_one.tsx

@@ -5,6 +5,7 @@
  */
 import React, { Fragment } from 'react';
 import {
+  EuiComboBox,
   EuiDescribedFormGroup,
   EuiFieldText,
   EuiForm,
@@ -17,16 +18,20 @@ import { FormState } from './add_data_source_form';
 
 interface AddDataSourceFormProps {
   formState: FormState;
-  onCheckboxChange: (name: string) => void;
-  onTextChange: (evt: React.ChangeEvent<HTMLInputElement>) => void;
+  onDatasetChange: (name: string) => void;
+  onNameChange: (evt: React.ChangeEvent<HTMLInputElement>) => void;
   datasetCheckboxes: EuiCheckboxGroupOption[];
+  policyOptions: FormState['policies'];
+  onPolicyChange: (selectedOptions: AddDataSourceFormProps['policyOptions']) => unknown;
 }
 
 export const StepOne = ({
   formState,
-  onCheckboxChange,
-  onTextChange,
+  onDatasetChange,
+  onNameChange,
   datasetCheckboxes,
+  onPolicyChange,
+  policyOptions,
 }: AddDataSourceFormProps) => {
   return (
     <Fragment>
@@ -45,7 +50,7 @@ export const StepOne = ({
             <EuiFieldText
               name="datasourceName"
               value={formState.datasourceName}
-              onChange={onTextChange}
+              onChange={onNameChange}
             />
           </EuiFormRow>
         </EuiDescribedFormGroup>
@@ -61,7 +66,26 @@ export const StepOne = ({
             <EuiCheckboxGroup
               options={datasetCheckboxes}
               idToSelectedMap={formState.datasets}
-              onChange={onCheckboxChange}
+              onChange={onDatasetChange}
+            />
+          </EuiFormRow>
+        </EuiDescribedFormGroup>
+        <EuiHorizontalRule />
+        <EuiDescribedFormGroup
+          idAria="data-source-policy"
+          title={<h3>Assign data source to policy</h3>}
+          description={
+            <Fragment>
+              Policies can help you maintain a group of data sources across a fleet of agents.
+            </Fragment>
+          }
+        >
+          <EuiFormRow label="Policy name" describedByIds={['policy-name']}>
+            <EuiComboBox
+              placeholder="Select a policy"
+              options={policyOptions}
+              selectedOptions={formState.policies}
+              onChange={onPolicyChange}
             />
           </EuiFormRow>
         </EuiDescribedFormGroup>