[Ingest] Edit datasource UI (#64727)

* Adjust NewDatasource type to exclude stream `agent_stream` property, add additional datasource hooks

* Initial pass at edit datasource UI

* Clean up dupe code, fix submit button not enabled after re-selecting a package

* Remove delete config functionality from list page

* Show validation errors for data source name and description fields

* Fix types

* Add success toasts

* Review fixes, clean up i18n

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Author: jen-huang

x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts

@@ -3,23 +3,19 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { Datasource, NewDatasource, DatasourceInput } from '../types';
+import { Datasource, DatasourceInput } from '../types';
 import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource';
 
 describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
-  const mockNewDatasource: NewDatasource = {
+  const mockDatasource: Datasource = {
+    id: 'some-uuid',
     name: 'mock-datasource',
     description: '',
     config_id: '',
     enabled: true,
     output_id: '',
     namespace: 'default',
     inputs: [],
-  };
-
-  const mockDatasource: Datasource = {
-    ...mockNewDatasource,
-    id: 'some-uuid',
     revision: 1,
   };
 
@@ -107,17 +103,6 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
     });
   });
 
-  it('uses name for id when id is not provided in case of new datasource', () => {
-    expect(storedDatasourceToAgentDatasource(mockNewDatasource)).toEqual({
-      id: 'mock-datasource',
-      name: 'mock-datasource',
-      namespace: 'default',
-      enabled: true,
-      use_output: 'default',
-      inputs: [],
-    });
-  });
-
   it('returns agent datasource config with flattened input and package stream', () => {
     expect(storedDatasourceToAgentDatasource({ ...mockDatasource, inputs: [mockInput] })).toEqual({
       id: 'some-uuid',

x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts

@@ -3,16 +3,16 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { Datasource, NewDatasource, FullAgentConfigDatasource } from '../types';
+import { Datasource, FullAgentConfigDatasource } from '../types';
 import { DEFAULT_OUTPUT } from '../constants';
 
 export const storedDatasourceToAgentDatasource = (
-  datasource: Datasource | NewDatasource
+  datasource: Datasource
 ): FullAgentConfigDatasource => {
-  const { name, namespace, enabled, package: pkg, inputs } = datasource;
+  const { id, name, namespace, enabled, package: pkg, inputs } = datasource;
 
   const fullDatasource: FullAgentConfigDatasource = {
-    id: 'id' in datasource ? datasource.id : name,
+    id: id || name,
     name,
     namespace,
     enabled,

x-pack/plugins/ingest_manager/common/types/models/agent_config.ts

@@ -3,8 +3,6 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-
-import { SavedObjectAttributes } from 'src/core/public';
 import {
   Datasource,
   DatasourcePackage,
@@ -26,7 +24,7 @@ export interface NewAgentConfig {
   monitoring_enabled?: Array<'logs' | 'metrics'>;
 }
 
-export interface AgentConfig extends NewAgentConfig, SavedObjectAttributes {
+export interface AgentConfig extends NewAgentConfig {
   id: string;
   status: AgentConfigStatus;
   datasources: string[] | Datasource[];

x-pack/plugins/ingest_manager/common/types/models/datasource.ts

@@ -17,22 +17,29 @@ export interface DatasourceConfigRecordEntry {
 
 export type DatasourceConfigRecord = Record<string, DatasourceConfigRecordEntry>;
 
-export interface DatasourceInputStream {
+export interface NewDatasourceInputStream {
   id: string;
   enabled: boolean;
   dataset: string;
   processors?: string[];
   config?: DatasourceConfigRecord;
   vars?: DatasourceConfigRecord;
+}
+
+export interface DatasourceInputStream extends NewDatasourceInputStream {
   agent_stream?: any;
 }
 
-export interface DatasourceInput {
+export interface NewDatasourceInput {
   type: string;
   enabled: boolean;
   processors?: string[];
   config?: DatasourceConfigRecord;
   vars?: DatasourceConfigRecord;
+  streams: NewDatasourceInputStream[];
+}
+
+export interface DatasourceInput extends Omit<NewDatasourceInput, 'streams'> {
   streams: DatasourceInputStream[];
 }
 
@@ -44,10 +51,11 @@ export interface NewDatasource {
   enabled: boolean;
   package?: DatasourcePackage;
   output_id: string;
-  inputs: DatasourceInput[];
+  inputs: NewDatasourceInput[];
 }
 
-export type Datasource = NewDatasource & {
+export interface Datasource extends Omit<NewDatasource, 'inputs'> {
   id: string;
+  inputs: DatasourceInput[];
   revision: number;
-};
+}

x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/datasource.ts

@@ -5,12 +5,18 @@
  */
 import { sendRequest, useRequest } from './use_request';
 import { datasourceRouteService } from '../../services';
-import { CreateDatasourceRequest, CreateDatasourceResponse } from '../../types';
+import {
+  CreateDatasourceRequest,
+  CreateDatasourceResponse,
+  UpdateDatasourceRequest,
+  UpdateDatasourceResponse,
+} from '../../types';
 import {
   DeleteDatasourcesRequest,
   DeleteDatasourcesResponse,
   GetDatasourcesRequest,
   GetDatasourcesResponse,
+  GetOneDatasourceResponse,
 } from '../../../../../common/types/rest_spec';
 
 export const sendCreateDatasource = (body: CreateDatasourceRequest['body']) => {
@@ -21,6 +27,17 @@ export const sendCreateDatasource = (body: CreateDatasourceRequest['body']) => {
   });
 };
 
+export const sendUpdateDatasource = (
+  datasourceId: string,
+  body: UpdateDatasourceRequest['body']
+) => {
+  return sendRequest<UpdateDatasourceResponse>({
+    path: datasourceRouteService.getUpdatePath(datasourceId),
+    method: 'put',
+    body: JSON.stringify(body),
+  });
+};
+
 export const sendDeleteDatasource = (body: DeleteDatasourcesRequest['body']) => {
   return sendRequest<DeleteDatasourcesResponse>({
     path: datasourceRouteService.getDeletePath(),
@@ -36,3 +53,10 @@ export function useGetDatasources(query: GetDatasourcesRequest['query']) {
     query,
   });
 }
+
+export const sendGetOneDatasource = (datasourceId: string) => {
+  return sendRequest<GetOneDatasourceResponse>({
+    path: datasourceRouteService.getInfoPath(datasourceId),
+    method: 'get',
+  });
+};

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/layout.tsx

@@ -39,17 +39,29 @@ export const CreateDatasourcePageLayout: React.FunctionComponent<{
       <EuiFlexItem>
         <EuiText>
           <h1>
-            <FormattedMessage
-              id="xpack.ingestManager.createDatasource.pageTitle"
-              defaultMessage="Add data source"
-            />
+            {from === 'edit' ? (
+              <FormattedMessage
+                id="xpack.ingestManager.editDatasource.pageTitle"
+                defaultMessage="Edit data source"
+              />
+            ) : (
+              <FormattedMessage
+                id="xpack.ingestManager.createDatasource.pageTitle"
+                defaultMessage="Add data source"
+              />
+            )}
           </h1>
         </EuiText>
       </EuiFlexItem>
       <EuiFlexItem>
         <EuiSpacer size="s" />
         <EuiText color="subdued" size="s">
-          {from === 'config' ? (
+          {from === 'edit' ? (
+            <FormattedMessage
+              id="xpack.ingestManager.editDatasource.pageDescription"
+              defaultMessage="Follow the instructions below to edit this data source."
+            />
+          ) : from === 'config' ? (
             <FormattedMessage
               id="xpack.ingestManager.createDatasource.pageDescriptionfromConfig"
               defaultMessage="Follow the instructions below to add an integration to this agent configuration."
@@ -68,7 +80,7 @@ export const CreateDatasourcePageLayout: React.FunctionComponent<{
     <EuiFlexGroup justifyContent="flexEnd" direction={'row'} gutterSize="xl">
       <EuiFlexItem grow={false}>
         <EuiSpacer size="s" />
-        {agentConfig && from === 'config' ? (
+        {agentConfig && (from === 'config' || from === 'edit') ? (
           <EuiDescriptionList style={{ textAlign: 'right' }} textStyle="reverse">
             <EuiDescriptionListTitle>
               <FormattedMessage

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/index.tsx

@@ -28,7 +28,7 @@ import {
 } from '../../../hooks';
 import { useLinks as useEPMLinks } from '../../epm/hooks';
 import { CreateDatasourcePageLayout, ConfirmCreateDatasourceModal } from './components';
-import { CreateDatasourceFrom } from './types';
+import { CreateDatasourceFrom, DatasourceFormState } from './types';
 import { DatasourceValidationResults, validateDatasource, validationHasErrors } from './services';
 import { StepSelectPackage } from './step_select_package';
 import { StepSelectConfig } from './step_select_config';
@@ -85,6 +85,7 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
   const updatePackageInfo = (updatedPackageInfo: PackageInfo | undefined) => {
     if (updatedPackageInfo) {
       setPackageInfo(updatedPackageInfo);
+      setFormState('VALID');
     } else {
       setFormState('INVALID');
       setPackageInfo(undefined);
@@ -152,9 +153,7 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
   const cancelUrl = from === 'config' ? CONFIG_URL : PACKAGE_URL;
 
   // Save datasource
-  const [formState, setFormState] = useState<
-    'VALID' | 'INVALID' | 'CONFIRM' | 'LOADING' | 'SUBMITTED'
-  >('INVALID');
+  const [formState, setFormState] = useState<DatasourceFormState>('INVALID');
   const saveDatasource = async () => {
     setFormState('LOADING');
     const result = await sendCreateDatasource(datasource);
@@ -174,6 +173,23 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
     const { error } = await saveDatasource();
     if (!error) {
       history.push(`${AGENT_CONFIG_DETAILS_PATH}${agentConfig ? agentConfig.id : configId}`);
+      notifications.toasts.addSuccess({
+        title: i18n.translate('xpack.ingestManager.createDatasource.addedNotificationTitle', {
+          defaultMessage: `Successfully added '{datasourceName}'`,
+          values: {
+            datasourceName: datasource.name,
+          },
+        }),
+        text:
+          agentCount && agentConfig
+            ? i18n.translate('xpack.ingestManager.createDatasource.addedNotificationMessage', {
+                defaultMessage: `Fleet will deploy updates to all agents that use the '{agentConfigName}' configuration`,
+                values: {
+                  agentConfigName: agentConfig.name,
+                },
+              })
+            : undefined,
+      });
     } else {
       notifications.toasts.addError(error, {
         title: 'Error',
@@ -229,6 +245,7 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
             packageInfo={packageInfo}
             datasource={datasource}
             updateDatasource={updateDatasource}
+            validationResults={validationResults!}
           />
         ) : null,
     },
@@ -240,7 +257,6 @@ export const CreateDatasourcePage: React.FunctionComponent = () => {
       children:
         agentConfig && packageInfo ? (
           <StepConfigureDatasource
-            agentConfig={agentConfig}
             packageInfo={packageInfo}
             datasource={datasource}
             updateDatasource={updateDatasource}

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/step_configure_datasource.tsx

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import React, { useEffect } from 'react';
+import React from 'react';
 import { FormattedMessage } from '@kbn/i18n/react';
 import {
   EuiPanel,
@@ -15,75 +15,21 @@ import {
   EuiCallOut,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import {
-  AgentConfig,
-  PackageInfo,
-  Datasource,
-  NewDatasource,
-  DatasourceInput,
-} from '../../../types';
+import { PackageInfo, NewDatasource, DatasourceInput } from '../../../types';
 import { Loading } from '../../../components';
-import { packageToConfigDatasourceInputs } from '../../../services';
 import { DatasourceValidationResults, validationHasErrors } from './services';
 import { DatasourceInputPanel } from './components';
 
 export const StepConfigureDatasource: React.FunctionComponent<{
-  agentConfig: AgentConfig;
   packageInfo: PackageInfo;
   datasource: NewDatasource;
   updateDatasource: (fields: Partial<NewDatasource>) => void;
   validationResults: DatasourceValidationResults;
   submitAttempted: boolean;
-}> = ({
-  agentConfig,
-  packageInfo,
-  datasource,
-  updateDatasource,
-  validationResults,
-  submitAttempted,
-}) => {
-  // Form show/hide states
-
+}> = ({ packageInfo, datasource, updateDatasource, validationResults, submitAttempted }) => {
   const hasErrors = validationResults ? validationHasErrors(validationResults) : false;
 
-  // Update datasource's package and config info
-  useEffect(() => {
-    const dsPackage = datasource.package;
-    const currentPkgKey = dsPackage ? `${dsPackage.name}-${dsPackage.version}` : '';
-    const pkgKey = `${packageInfo.name}-${packageInfo.version}`;
-
-    // If package has changed, create shell datasource with input&stream values based on package info
-    if (currentPkgKey !== pkgKey) {
-      // Existing datasources on the agent config using the package name, retrieve highest number appended to datasource name
-      const dsPackageNamePattern = new RegExp(`${packageInfo.name}-(\\d+)`);
-      const dsWithMatchingNames = (agentConfig.datasources as Datasource[])
-        .filter(ds => Boolean(ds.name.match(dsPackageNamePattern)))
-        .map(ds => parseInt(ds.name.match(dsPackageNamePattern)![1], 10))
-        .sort();
-
-      updateDatasource({
-        name: `${packageInfo.name}-${
-          dsWithMatchingNames.length ? dsWithMatchingNames[dsWithMatchingNames.length - 1] + 1 : 1
-        }`,
-        package: {
-          name: packageInfo.name,
-          title: packageInfo.title,
-          version: packageInfo.version,
-        },
-        inputs: packageToConfigDatasourceInputs(packageInfo),
-      });
-    }
-
-    // If agent config has changed, update datasource's config ID and namespace
-    if (datasource.config_id !== agentConfig.id) {
-      updateDatasource({
-        config_id: agentConfig.id,
-        namespace: agentConfig.namespace,
-      });
-    }
-  }, [datasource.package, datasource.config_id, agentConfig, packageInfo, updateDatasource]);
-
-  // Step B, configure inputs (and their streams)
+  // Configure inputs (and their streams)
   // Assume packages only export one datasource for now
   const renderConfigureInputs = () =>
     packageInfo.datasources &&