Skip to content

Commit cd1e182

Browse files
jen-huangjen-huang
authored
[Ingest] Add support for yaml field types (#60440) (#60579)
* Support yaml var type: * Change stream config model to save type and value, instead of just value * Add code editor for configuring yaml vars * Adjust tests * Account for empty yaml value * Better account for invalid yaml parsing
1 parent 4ee34e6 commit cd1e182

File tree

9 files changed

+158
-57
lines changed

9 files changed

+158
-57
lines changed

‎x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* or more contributor license agreements. Licensed under the Elastic License;
44
* you may not use this file except in compliance with the Elastic License.
55
*/
6-
import { NewDatasource } from '../types';
6+
import { NewDatasource, DatasourceInput } from '../types';
77
import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource';
88

99
describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
@@ -17,21 +17,37 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
1717
inputs: [],
1818
};
1919

20-
const mockInput = {
20+
const mockInput: DatasourceInput = {
2121
type: 'test-logs',
2222
enabled: true,
2323
streams: [
2424
{
2525
id: 'test-logs-foo',
2626
enabled: true,
2727
dataset: 'foo',
28-
config: { fooVar: 'foo-value', fooVar2: [1, 2] },
28+
config: { fooVar: { value: 'foo-value' }, fooVar2: { value: [1, 2] } },
2929
},
3030
{
3131
id: 'test-logs-bar',
3232
enabled: false,
3333
dataset: 'bar',
34-
config: { barVar: 'bar-value', barVar2: [1, 2] },
34+
config: {
35+
barVar: { value: 'bar-value' },
36+
barVar2: { value: [1, 2] },
37+
barVar3: {
38+
type: 'yaml',
39+
value:
40+
'- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2',
41+
},
42+
barVar4: {
43+
type: 'yaml',
44+
value: '',
45+
},
46+
barVar5: {
47+
type: 'yaml',
48+
value: 'testField: test\n invalidSpacing: foo',
49+
},
50+
},
3551
},
3652
],
3753
};
@@ -91,6 +107,16 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
91107
dataset: 'bar',
92108
barVar: 'bar-value',
93109
barVar2: [1, 2],
110+
barVar3: [
111+
{
112+
namespace: 'mockNamespace',
113+
anotherProp: 'test',
114+
},
115+
{
116+
namespace: 'mockNamespace2',
117+
anotherProp: 'test2',
118+
},
119+
],
94120
},
95121
],
96122
},

‎x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* or more contributor license agreements. Licensed under the Elastic License;
44
* you may not use this file except in compliance with the Elastic License.
55
*/
6+
import { safeLoad } from 'js-yaml';
67
import { Datasource, NewDatasource, FullAgentConfigDatasource } from '../types';
78
import { DEFAULT_OUTPUT } from '../constants';
89

@@ -23,12 +24,26 @@ export const storedDatasourceToAgentDatasource = (
2324
if (stream.config) {
2425
const fullStream = {
2526
...stream,
26-
...Object.entries(stream.config).reduce((acc, [configName, configValue]) => {
27-
if (configValue !== undefined) {
28-
acc[configName] = configValue;
29-
}
30-
return acc;
31-
}, {} as { [key: string]: any }),
27+
...Object.entries(stream.config).reduce(
28+
(acc, [configName, { type: configType, value: configValue }]) => {
29+
if (configValue !== undefined) {
30+
if (configType === 'yaml') {
31+
try {
32+
const yamlValue = safeLoad(configValue);
33+
if (yamlValue) {
34+
acc[configName] = yamlValue;
35+
}
36+
} catch (e) {
37+
// Silently swallow parsing error
38+
}
39+
} else {
40+
acc[configName] = configValue;
41+
}
42+
}
43+
return acc;
44+
},
45+
{} as { [key: string]: any }
46+
),
3247
};
3348
delete fullStream.config;
3449
return fullStream;

‎x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,14 @@ describe('Ingest Manager - packageToConfig', () => {
108108
{
109109
type: 'bar',
110110
streams: [
111-
{ dataset: 'bar', vars: [{ default: 'bar-var-value', name: 'var-name' }] },
112-
{ dataset: 'bar2', vars: [{ default: 'bar2-var-value', name: 'var-name' }] },
111+
{
112+
dataset: 'bar',
113+
vars: [{ default: 'bar-var-value', name: 'var-name', type: 'text' }],
114+
},
115+
{
116+
dataset: 'bar2',
117+
vars: [{ default: 'bar2-var-value', name: 'var-name', type: 'yaml' }],
118+
},
113119
],
114120
},
115121
],
@@ -125,7 +131,7 @@ describe('Ingest Manager - packageToConfig', () => {
125131
id: 'foo-foo',
126132
enabled: true,
127133
dataset: 'foo',
128-
config: { 'var-name': 'foo-var-value' },
134+
config: { 'var-name': { value: 'foo-var-value' } },
129135
},
130136
],
131137
},
@@ -137,13 +143,13 @@ describe('Ingest Manager - packageToConfig', () => {
137143
id: 'bar-bar',
138144
enabled: true,
139145
dataset: 'bar',
140-
config: { 'var-name': 'bar-var-value' },
146+
config: { 'var-name': { type: 'text', value: 'bar-var-value' } },
141147
},
142148
{
143149
id: 'bar-bar2',
144150
enabled: true,
145151
dataset: 'bar2',
146-
config: { 'var-name': 'bar2-var-value' },
152+
config: { 'var-name': { type: 'yaml', value: 'bar2-var-value' } },
147153
},
148154
],
149155
},
@@ -204,10 +210,10 @@ describe('Ingest Manager - packageToConfig', () => {
204210
enabled: true,
205211
dataset: 'foo',
206212
config: {
207-
'var-name': 'foo-var-value',
208-
'foo-input-var-name': 'foo-input-var-value',
209-
'foo-input2-var-name': 'foo-input2-var-value',
210-
'foo-input3-var-name': undefined,
213+
'var-name': { value: 'foo-var-value' },
214+
'foo-input-var-name': { value: 'foo-input-var-value' },
215+
'foo-input2-var-name': { value: 'foo-input2-var-value' },
216+
'foo-input3-var-name': { value: undefined },
211217
},
212218
},
213219
],
@@ -221,19 +227,19 @@ describe('Ingest Manager - packageToConfig', () => {
221227
enabled: true,
222228
dataset: 'bar',
223229
config: {
224-
'var-name': 'bar-var-value',
225-
'bar-input-var-name': ['value1', 'value2'],
226-
'bar-input2-var-name': 123456,
230+
'var-name': { value: 'bar-var-value' },
231+
'bar-input-var-name': { value: ['value1', 'value2'] },
232+
'bar-input2-var-name': { value: 123456 },
227233
},
228234
},
229235
{
230236
id: 'bar-bar2',
231237
enabled: true,
232238
dataset: 'bar2',
233239
config: {
234-
'var-name': 'bar2-var-value',
235-
'bar-input-var-name': ['value1', 'value2'],
236-
'bar-input2-var-name': 123456,
240+
'var-name': { value: 'bar2-var-value' },
241+
'bar-input-var-name': { value: ['value1', 'value2'] },
242+
'bar-input2-var-name': { value: 123456 },
237243
},
238244
},
239245
],
@@ -247,7 +253,7 @@ describe('Ingest Manager - packageToConfig', () => {
247253
enabled: false,
248254
dataset: 'disabled',
249255
config: {
250-
'var-name': [],
256+
'var-name': { value: [] },
251257
},
252258
},
253259
{

‎x-pack/plugins/ingest_manager/common/services/package_to_config.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,9 @@ export const packageToConfigDatasourceInputs = (packageInfo: PackageInfo): Datas
4141
streamVar: RegistryVarsEntry
4242
): DatasourceInputStream['config'] => {
4343
if (!streamVar.default && streamVar.multi) {
44-
configObject![streamVar.name] = [];
44+
configObject![streamVar.name] = { type: streamVar.type, value: [] };
4545
} else {
46-
configObject![streamVar.name] = streamVar.default;
46+
configObject![streamVar.name] = { type: streamVar.type, value: streamVar.default };
4747
}
4848
return configObject;
4949
};

‎x-pack/plugins/ingest_manager/common/types/models/datasource.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,13 @@ export interface DatasourceInputStream {
1515
enabled: boolean;
1616
dataset: string;
1717
processors?: string[];
18-
config?: Record<string, any>;
18+
config?: Record<
19+
string,
20+
{
21+
type?: string;
22+
value: any;
23+
}
24+
>;
1925
}
2026

2127
export interface DatasourceInput {

‎x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_config.tsx‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ export const DatasourceInputConfig: React.FunctionComponent<{
6363
<EuiFlexItem>
6464
<EuiFlexGroup direction="column" gutterSize="m">
6565
{requiredVars.map(varDef => {
66-
const varName = varDef.name;
67-
const value = datasourceInput.streams[0].config![varName];
66+
const { name: varName, type: varType } = varDef;
67+
const value = datasourceInput.streams[0].config![varName].value;
6868
return (
6969
<EuiFlexItem key={varName}>
7070
<DatasourceInputVarField
@@ -76,7 +76,10 @@ export const DatasourceInputConfig: React.FunctionComponent<{
7676
...stream,
7777
config: {
7878
...stream.config,
79-
[varName]: newValue,
79+
[varName]: {
80+
type: varType,
81+
value: newValue,
82+
},
8083
},
8184
})),
8285
});
@@ -105,8 +108,8 @@ export const DatasourceInputConfig: React.FunctionComponent<{
105108
</EuiFlexItem>
106109
{isShowingAdvanced
107110
? advancedVars.map(varDef => {
108-
const varName = varDef.name;
109-
const value = datasourceInput.streams[0].config![varName];
111+
const { name: varName, type: varType } = varDef;
112+
const value = datasourceInput.streams[0].config![varName].value;
110113
return (
111114
<EuiFlexItem key={varName}>
112115
<DatasourceInputVarField
@@ -118,7 +121,10 @@ export const DatasourceInputConfig: React.FunctionComponent<{
118121
...stream,
119122
config: {
120123
...stream.config,
121-
[varName]: newValue,
124+
[varName]: {
125+
type: varType,
126+
value: newValue,
127+
},
122128
},
123129
})),
124130
});

‎x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_stream_config.tsx‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ export const DatasourceInputStreamConfig: React.FunctionComponent<{
6565
<EuiFlexItem>
6666
<EuiFlexGroup direction="column" gutterSize="m">
6767
{requiredVars.map(varDef => {
68-
const varName = varDef.name;
69-
const value = datasourceInputStream.config![varName];
68+
const { name: varName, type: varType } = varDef;
69+
const value = datasourceInputStream.config![varName].value;
7070
return (
7171
<EuiFlexItem key={varName}>
7272
<DatasourceInputVarField
@@ -76,7 +76,10 @@ export const DatasourceInputStreamConfig: React.FunctionComponent<{
7676
updateDatasourceInputStream({
7777
config: {
7878
...datasourceInputStream.config,
79-
[varName]: newValue,
79+
[varName]: {
80+
type: varType,
81+
value: newValue,
82+
},
8083
},
8184
});
8285
}}
@@ -104,8 +107,8 @@ export const DatasourceInputStreamConfig: React.FunctionComponent<{
104107
</EuiFlexItem>
105108
{isShowingAdvanced
106109
? advancedVars.map(varDef => {
107-
const varName = varDef.name;
108-
const value = datasourceInputStream.config![varName];
110+
const { name: varName, type: varType } = varDef;
111+
const value = datasourceInputStream.config![varName].value;
109112
return (
110113
<EuiFlexItem key={varName}>
111114
<DatasourceInputVarField
@@ -115,7 +118,10 @@ export const DatasourceInputStreamConfig: React.FunctionComponent<{
115118
updateDatasourceInputStream({
116119
config: {
117120
...datasourceInputStream.config,
118-
[varName]: newValue,
121+
[varName]: {
122+
type: varType,
123+
value: newValue,
124+
},
119125
},
120126
});
121127
}}

‎x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/create_datasource_page/components/datasource_input_var_field.tsx‎

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,57 @@
66
import React from 'react';
77
import ReactMarkdown from 'react-markdown';
88
import { FormattedMessage } from '@kbn/i18n/react';
9-
import { EuiFormRow, EuiFieldText, EuiComboBox, EuiText } from '@elastic/eui';
9+
import { EuiFormRow, EuiFieldText, EuiComboBox, EuiText, EuiCodeEditor } from '@elastic/eui';
1010
import { RegistryVarsEntry } from '../../../../types';
1111

12+
import 'brace/mode/yaml';
13+
import 'brace/theme/textmate';
14+
1215
export const DatasourceInputVarField: React.FunctionComponent<{
1316
varDef: RegistryVarsEntry;
1417
value: any;
1518
onChange: (newValue: any) => void;
1619
}> = ({ varDef, value, onChange }) => {
20+
const renderField = () => {
21+
if (varDef.multi) {
22+
return (
23+
<EuiComboBox
24+
noSuggestions
25+
selectedOptions={value.map((val: string) => ({ label: val }))}
26+
onCreateOption={(newVal: any) => {
27+
onChange([...value, newVal]);
28+
}}
29+
onChange={(newVals: any[]) => {
30+
onChange(newVals.map(val => val.label));
31+
}}
32+
/>
33+
);
34+
}
35+
if (varDef.type === 'yaml') {
36+
return (
37+
<EuiCodeEditor
38+
width="100%"
39+
mode="yaml"
40+
theme="textmate"
41+
setOptions={{
42+
minLines: 10,
43+
maxLines: 30,
44+
tabSize: 2,
45+
showGutter: false,
46+
}}
47+
value={value}
48+
onChange={newVal => onChange(newVal)}
49+
/>
50+
);
51+
}
52+
return (
53+
<EuiFieldText
54+
value={value === undefined ? '' : value}
55+
onChange={e => onChange(e.target.value)}
56+
/>
57+
);
58+
};
59+
1760
return (
1861
<EuiFormRow
1962
label={varDef.title || varDef.name}
@@ -29,20 +72,7 @@ export const DatasourceInputVarField: React.FunctionComponent<{
2972
}
3073
helpText={<ReactMarkdown source={varDef.description} />}
3174
>
32-
{varDef.multi ? (
33-
<EuiComboBox
34-
noSuggestions
35-
selectedOptions={value.map((val: string) => ({ label: val }))}
36-
onCreateOption={(newVal: any) => {
37-
onChange([...value, newVal]);
38-
}}
39-
onChange={(newVals: any[]) => {
40-
onChange(newVals.map(val => val.label));
41-
}}
42-
/>
43-
) : (
44-
<EuiFieldText value={value} onChange={e => onChange(e.target.value)} />
45-
)}
75+
{renderField()}
4676
</EuiFormRow>
4777
);
4878
};

0 commit comments

Comments
 (0)