WIP. Add datasource to policy

 * Combobox on add datasource page (values are static, need to get from Ingest API)
 * POSTs to Ingest API to add datasource to policy

x-pack/legacy/plugins/epm/common/types.ts

@@ -170,4 +170,5 @@ export interface DatasourcePayload {
   pkgkey: string;
   datasourceName: string;
   datasets: Dataset[];
+  policyIds: string[];
 }

x-pack/legacy/plugins/epm/public/screens/add_data_source/add_data_source_form.tsx

@@ -5,6 +5,7 @@
  */
 import {
   EuiButton,
+  EuiComboBoxOptionProps,
   EuiHorizontalRule,
   EuiPanel,
   EuiSteps,
@@ -36,14 +37,28 @@ interface AddDataSourceStepsProps {
 export interface FormState {
   datasourceName: string;
   datasets: EuiCheckboxGroupIdToSelectedMap;
+  policies: Array<EuiComboBoxOptionProps<string>>;
 }
+
 const FormNav = styled.div`
   text-align: right;
 `;
 
 export const AddDataSourceForm = (props: AddDataSourceStepsProps) => {
+  const defaultPolicyOption = { label: 'Default policy', value: 'default' };
   const [addDataSourceSuccess, setAddDataSourceSuccess] = useState<boolean>(false);
-  const [formState, setFormState] = useState<FormState>({ datasourceName: '', datasets: {} });
+  const [datasourceName, setDatasourceName] = useState<FormState['datasourceName']>('');
+  const [selectedDatasets, setSelectedDatasets] = useState<FormState['datasets']>({});
+  const [selectedPolicies, setSelectedPolicies] = useState<FormState['policies']>([
+    defaultPolicyOption,
+  ]);
+
+  const formState: FormState = {
+    datasourceName,
+    datasets: selectedDatasets,
+    policies: selectedPolicies,
+  };
+
   const { notifications } = useCore();
   const { toDetailView } = useLinks();
   const { pkgName, pkgTitle, pkgVersion, datasets } = props;
@@ -54,6 +69,8 @@ export const AddDataSourceForm = (props: AddDataSourceStepsProps) => {
         pkgkey: `${pkgName}-${pkgVersion}`,
         datasets: datasets.filter(d => formState.datasets[d.name] === true),
         datasourceName: formState.datasourceName,
+        // @ts-ignore not sure where/how to enforce a `value` key
+        policyIds: formState.policies.map(({ value }) => value),
       });
       setAddDataSourceSuccess(true);
       notifications.toasts.addSuccess({
@@ -68,20 +85,14 @@ export const AddDataSourceForm = (props: AddDataSourceStepsProps) => {
     }
   };
 
-  const onCheckboxChange = (name: string) => {
-    const newCheckboxStateMap = {
-      ...formState,
-      datasets: {
-        ...formState.datasets,
-        [name]: !formState.datasets[name],
-      },
-    };
-    setFormState(newCheckboxStateMap);
-  };
+  const onCheckboxChange = (id: string) =>
+    setSelectedDatasets({
+      ...selectedDatasets,
+      [id]: !selectedDatasets[id],
+    });
 
-  const onTextChange = (evt: React.ChangeEvent<HTMLInputElement>) => {
-    setFormState({ ...formState, [evt.target.name]: evt.target.value });
-  };
+  const onTextChange = (evt: React.ChangeEvent<HTMLInputElement>) =>
+    setDatasourceName(evt.target.value);
 
   // create checkbox items from datasets for EuiCheckboxGroup
   const checkboxes = datasets.map(dataset => ({
@@ -97,6 +108,11 @@ export const AddDataSourceForm = (props: AddDataSourceStepsProps) => {
           datasetCheckboxes={checkboxes}
           onCheckboxChange={onCheckboxChange}
           onTextChange={onTextChange}
+          policyOptions={[
+            defaultPolicyOption,
+            { label: 'Foo policy', value: 'd09bbe00-21e0-11ea-9786-4545a9e62b25' },
+          ]}
+          onPolicyChange={setSelectedPolicies}
           formState={formState}
         />
       ),

x-pack/legacy/plugins/epm/public/screens/add_data_source/step_one.tsx

@@ -5,6 +5,7 @@
  */
 import React, { Fragment } from 'react';
 import {
+  EuiComboBox,
   EuiDescribedFormGroup,
   EuiFieldText,
   EuiForm,
@@ -20,13 +21,17 @@ interface AddDataSourceFormProps {
   onCheckboxChange: (name: string) => void;
   onTextChange: (evt: React.ChangeEvent<HTMLInputElement>) => void;
   datasetCheckboxes: EuiCheckboxGroupOption[];
+  policyOptions: FormState['policies'];
+  onPolicyChange: (selectedOptions: AddDataSourceFormProps['policyOptions']) => unknown;
 }
 
 export const StepOne = ({
   formState,
   onCheckboxChange,
   onTextChange,
   datasetCheckboxes,
+  onPolicyChange,
+  policyOptions,
 }: AddDataSourceFormProps) => {
   return (
     <Fragment>
@@ -65,6 +70,25 @@ export const StepOne = ({
             />
           </EuiFormRow>
         </EuiDescribedFormGroup>
+        <EuiHorizontalRule />
+        <EuiDescribedFormGroup
+          idAria="data-source-policy"
+          title={<h3>Assign data source to policy</h3>}
+          description={
+            <Fragment>
+              Policies can help you maintain a group of data sources across a fleet of agents.
+            </Fragment>
+          }
+        >
+          <EuiFormRow label="Policy name" describedByIds={['policy-name']}>
+            <EuiComboBox
+              placeholder="Select a policy"
+              options={policyOptions}
+              selectedOptions={formState.policies}
+              onChange={onPolicyChange}
+            />
+          </EuiFormRow>
+        </EuiDescribedFormGroup>
       </EuiForm>
     </Fragment>
   );

x-pack/legacy/plugins/epm/server/datasources/create.ts

@@ -4,20 +4,20 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import fetch from 'node-fetch';
 import yaml from 'js-yaml';
+import fetch from 'node-fetch';
 import { SavedObjectsClientContract } from 'src/core/server/';
 import { Asset, Datasource, Stream } from '../../../ingest/server/libs/types';
 import { SAVED_OBJECT_TYPE_DATASOURCES } from '../../common/constants';
 import { AssetReference, Dataset, InstallationStatus, RegistryPackage } from '../../common/types';
+import { createInput } from '../lib/agent/agent';
 import { CallESAsCurrentUser } from '../lib/cluster_access';
 import { installILMPolicy, policyExists } from '../lib/elasticsearch/ilm/install';
 import { installPipelinesForDataset } from '../lib/elasticsearch/ingest_pipeline/ingest_pipelines';
 import { installTemplateForDataset } from '../lib/elasticsearch/template/install';
 import { getPackageInfo, PackageNotInstalledError } from '../packages';
 import * as Registry from '../registry';
 import { Request } from '../types';
-import { createInput } from '../lib/agent/agent';
 
 export async function createDatasource(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -26,8 +26,17 @@ export async function createDatasource(options: {
   pkgkey: string;
   datasourceName: string;
   datasets: Dataset[];
+  policyIds: string[];
 }) {
-  const { savedObjectsClient, callCluster, pkgkey, datasets, datasourceName, request } = options;
+  const {
+    savedObjectsClient,
+    callCluster,
+    pkgkey,
+    datasets,
+    datasourceName,
+    request,
+    policyIds,
+  } = options;
 
   const epmPackageInfo = await getPackageInfo({ savedObjectsClient, pkgkey });
   if (epmPackageInfo.status !== InstallationStatus.installed) {
@@ -91,6 +100,12 @@ export async function createDatasource(options: {
     streams,
   });
 
+  await Promise.all(
+    policyIds.map(policyId =>
+      ingestAddDatasourcesToPolicy({ datasources: [pkgkey], policyId, request })
+    )
+  );
+
   return toSave;
 }
 
@@ -167,7 +182,7 @@ async function getDatasource(options: {
   const { savedObjectsClient, pkg } = options;
   const datasource = await savedObjectsClient
     .get<Datasource>(SAVED_OBJECT_TYPE_DATASOURCES, Registry.pkgToPkgKey(pkg))
-    .catch(e => undefined);
+    .catch(() => undefined);
 
   return datasource?.attributes;
 }
@@ -183,7 +198,6 @@ interface CreateFakeDatasource {
 function createFakeDatasource({
   pkg,
   datasourceName,
-  datasets,
   assets = [],
   streams,
 }: CreateFakeDatasource): Datasource {
@@ -202,37 +216,6 @@ function createFakeDatasource({
   };
 }
 
-async function ingestDatasourceCreate({
-  request,
-  datasource,
-}: {
-  request: Request;
-  datasource: Datasource;
-}) {
-  // OMG, so gross! Will not keep
-  // if we end up keeping the "make another HTTP request" method,
-  // we'll clean this up via proxy or something else which prevents these functions from needing to know this.
-  // The key here is to show the Saved Object we create being stored/retrieved by Ingest
-
-  // node-fetch requires absolute urls because there isn't an origin on Node
-  const origin = request.server.info.uri; // e.g. http://localhost:5601
-  const basePath = request.getBasePath(); // e.g. /abc
-  const apiPath = '/api/ingest/datasources';
-  const url = `${origin}${basePath}${apiPath}`;
-  const body = { datasource };
-  delete request.headers['transfer-encoding'];
-  await fetch(url, {
-    method: 'post',
-    body: JSON.stringify(body),
-    headers: {
-      'kbn-xsrf': 'some value, any value',
-      'Content-Type': 'application/json',
-      // the main (only?) one we want is `authorization`
-      ...request.headers,
-    },
-  });
-}
-
 async function getConfig(pkgkey: string, dataset: Dataset): Promise<string> {
   const vars = dataset.vars;
 
@@ -241,7 +224,7 @@ async function getConfig(pkgkey: string, dataset: Dataset): Promise<string> {
     isDatasetInput(entry, dataset.name)
   );
 
-  if (paths.length === 1) {
+  if (paths.length === 1 && Array.isArray(vars)) {
     const buffer = Registry.getAsset(paths[0]);
     // Load input template from path
     return createInput(vars, buffer.toString());
@@ -255,3 +238,66 @@ const isDatasetInput = ({ path }: Registry.ArchiveEntry, datasetName: string) =>
 };
 
 const isDirectory = ({ path }: Registry.ArchiveEntry) => path.endsWith('/');
+
+async function ingestAddDatasourcesToPolicy({
+  request,
+  datasources,
+  policyId,
+}: {
+  request: Request;
+  datasources: Array<Datasource['id']>;
+  policyId: string;
+}) {
+  await ingestAPI({
+    method: 'post',
+    path: `/api/ingest/policies/${policyId}/addDatasources`,
+    body: { datasources },
+    request,
+  });
+}
+
+async function ingestDatasourceCreate({
+  request,
+  datasource,
+}: {
+  request: Request;
+  datasource: Datasource;
+}) {
+  await ingestAPI({
+    path: '/api/ingest/datasources',
+    method: 'post',
+    body: { datasource },
+    request,
+  });
+}
+
+// Temporary while we're iterating.
+// If we end up keeping the "make another HTTP request" method,
+// we'll clean this up via proxy or something else
+async function ingestAPI({
+  path,
+  method,
+  body,
+  request,
+}: {
+  path: string;
+  method: string;
+  body: Record<string, any>;
+  request: Request;
+}) {
+  // node-fetch requires absolute urls because there isn't an origin on Node
+  const origin = request.server.info.uri; // e.g. http://localhost:5601
+  const basePath = request.getBasePath(); // e.g. /abc
+  const url = `${origin}${basePath}${path}`;
+
+  return fetch(url, {
+    method,
+    body: JSON.stringify(body),
+    headers: {
+      'kbn-xsrf': 'some value, any value',
+      'Content-Type': 'application/json',
+      // the main (only?) one we want is `authorization`
+      ...request.headers,
+    },
+  });
+}

x-pack/legacy/plugins/epm/server/datasources/handlers.ts

@@ -26,14 +26,15 @@ export async function handleRequestInstallDatasource(
   request: CreateDatasourceRequest,
   extra: Extra
 ) {
-  const { pkgkey, datasets, datasourceName } = request.payload;
+  const { pkgkey, datasets, datasourceName, policyIds } = request.payload;
   const user = await request.server.plugins.security?.getUser(request);
   if (!user) return Boom.unauthorized('Must be logged in to perform this operation');
   if (!pkgkey) return Boom.badRequest('Please supply a value for pkgkey in the POST payload');
   if (!datasets) return Boom.badRequest('Please supply a value for datasets in the POST payload');
   if (!datasourceName) {
     return Boom.badRequest('Please supply a value for datasourceName in the POST payload');
   }
+
   const savedObjectsClient = getClient(request);
   const callCluster = getClusterAccessor(extra.context.esClient, request);
   try {
@@ -46,6 +47,7 @@ export async function handleRequestInstallDatasource(
       // long-term, I don't want to pass `request` through
       // but this was the fastest/least invasive change way to make the change
       request,
+      policyIds,
     });
 
     return result;

x-pack/legacy/plugins/ingest/server/libs/datasources.ts

@@ -25,7 +25,7 @@ export class DatasourcesLib {
       throw new Error('Could not get version information about Kibana from xpack');
     }
 
-    return await this.adapter.create(withUser, datasource);
+    return await this.adapter.create(withUser, datasource, { id: datasource.id });
   }
 
   public async get(user: FrameworkUser, id: string): Promise<Datasource | null> {