Skip to content

Commit 4370b44

Browse files
qn895qn895
committed
Fix payload size too big 413, num of requests
1 parent e48b539 commit 4370b44

File tree

12 files changed

+136
-106
lines changed

12 files changed

+136
-106
lines changed

‎x-pack/plugins/data_visualizer/public/application/common/components/stats_table/types/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export type {
1010
FileBasedFieldVisConfig,
1111
MetricFieldVisStats,
1212
} from '../../../../../../common/types/field_vis_config';
13-
export type {
13+
export {
1414
isFileBasedFieldVisConfig,
1515
isIndexBasedFieldVisConfig,
1616
} from '../../../../../../common/types/field_vis_config';

‎x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -250,32 +250,32 @@ export const IndexDataVisualizerView: FC<IndexDataVisualizerViewProps> = (dataVi
250250
} = useDataVisualizerGridData(input, dataVisualizerListState, setGlobalState);
251251

252252
useEffect(() => {
253-
// Force refresh on index pattern change
254-
setLastRefresh(Date.now());
255-
256253
return () => {
257254
// When navigating away from the index pattern
258255
// Reset all previously set filters
259256
// to make sure new page doesn't have unrelated filters
260257
data.query.filterManager.removeAll();
261258
};
262-
}, [currentIndexPattern.id, data.query.filterManager, setLastRefresh]);
259+
}, [currentIndexPattern.id, data.query.filterManager]);
260+
261+
useEffect(() => {
262+
// Force refresh on index pattern change
263+
setLastRefresh(Date.now());
264+
}, [currentIndexPattern.id, setLastRefresh]);
263265

264266
useEffect(() => {
265267
if (globalState?.time !== undefined) {
266268
timefilter.setTime({
267269
from: globalState.time.from,
268270
to: globalState.time.to,
269271
});
270-
setLastRefresh(Date.now());
271272
}
272273
// eslint-disable-next-line react-hooks/exhaustive-deps
273274
}, [JSON.stringify(globalState?.time), timefilter]);
274275

275276
useEffect(() => {
276277
if (globalState?.refreshInterval !== undefined) {
277278
timefilter.setRefreshInterval(globalState.refreshInterval);
278-
setLastRefresh(Date.now());
279279
}
280280
// eslint-disable-next-line react-hooks/exhaustive-deps
281281
}, [JSON.stringify(globalState?.refreshInterval), timefilter]);

‎x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,6 @@ export const useDataVisualizerGridData = (
202202
runtimeFieldMap: currentIndexPattern.getComputedFields().runtimeFields,
203203
aggregatableFields,
204204
nonAggregatableFields,
205-
lastRefresh,
206205
};
207206
},
208207
// eslint-disable-next-line react-hooks/exhaustive-deps
@@ -218,7 +217,10 @@ export const useDataVisualizerGridData = (
218217
]
219218
);
220219

221-
const { overallStats, progress: overallStatsProgress } = useOverallStats(fieldStatsRequest);
220+
const { overallStats, progress: overallStatsProgress } = useOverallStats(
221+
fieldStatsRequest,
222+
lastRefresh
223+
);
222224

223225
const configsWithoutStats = useMemo(() => {
224226
if (overallStatsProgress.loaded < 100) return;

‎x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_field_stats.ts‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -158,15 +158,14 @@ export function useFieldStatsSearchStrategy(
158158
10
159159
);
160160

161+
const statsMap$ = new Subject();
162+
const fieldsToRetry$ = new Subject<Field[]>();
163+
161164
const fieldStatsSub = combineLatest(
162165
batches
163-
.map((batch) => getFieldsStats(data, params, batch, searchOptions))
166+
.map((batch) => getFieldsStats(data.search, params, batch, searchOptions))
164167
.filter((obs) => obs !== undefined) as Array<Observable<FieldStats[] | FieldStatsError>>
165168
);
166-
167-
const statsMap$ = new Subject();
168-
const fieldsToRetry$ = new Subject<Field[]>();
169-
170169
const onError = (error: any) => {
171170
toasts.addError(error, {
172171
title: i18n.translate('xpack.dataVisualizer.index.errorFetchingFieldStatisticsMessage', {
@@ -212,8 +211,10 @@ export function useFieldStatsSearchStrategy(
212211

213212
setFieldStats(statsMap);
214213

215-
statsMap$.next(statsMap);
216-
fieldsToRetry$.next(failedFields);
214+
if (failedFields.length > 0) {
215+
statsMap$.next(statsMap);
216+
fieldsToRetry$.next(failedFields);
217+
}
217218
}
218219
},
219220
error: onError,
@@ -227,7 +228,9 @@ export function useFieldStatsSearchStrategy(
227228
switchMap((failedFields) => {
228229
return combineLatest(
229230
failedFields
230-
.map((failedField) => getFieldsStats(data, params, [failedField], searchOptions))
231+
.map((failedField) =>
232+
getFieldsStats(data.search, params, [failedField], searchOptions)
233+
)
231234
.filter((obs) => obs !== undefined)
232235
);
233236
})
@@ -253,7 +256,8 @@ export function useFieldStatsSearchStrategy(
253256
error: onError,
254257
complete: onComplete,
255258
});
256-
}, [data, toasts, searchStrategyParams, fieldStatsParams, initialDataVisualizerListState]);
259+
// eslint-disable-next-line react-hooks/exhaustive-deps
260+
}, [data.search, toasts, fieldStatsParams, initialDataVisualizerListState]);
257261

258262
const cancelFetch = useCallback(() => {
259263
searchSubscription$.current?.unsubscribe();

‎x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts‎

Lines changed: 40 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ import { forkJoin, of, Subscription } from 'rxjs';
1010
import { switchMap } from 'rxjs/operators';
1111
import { i18n } from '@kbn/i18n';
1212
import type { ToastsStart } from 'kibana/public';
13+
import { chunk } from 'lodash';
1314
import { useDataVisualizerKibana } from '../../kibana_context';
1415
import {
16+
AggregatableFieldOverallStats,
1517
checkAggregatableFieldsExistRequest,
1618
checkNonAggregatableFieldExistsRequest,
1719
processAggregatableFieldsExistResponse,
@@ -62,7 +64,8 @@ function displayError(toastNotifications: ToastsStart, indexPattern: string, err
6264
}
6365

6466
export function useOverallStats<TParams extends OverallStatsSearchStrategyParams>(
65-
searchStrategyParams: TParams | undefined
67+
searchStrategyParams: TParams | undefined,
68+
lastRefresh: number
6669
): {
6770
progress: DataStatsFetchProgress;
6871
overallStats: OverallStats;
@@ -87,7 +90,8 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
8790
searchSubscription$.current?.unsubscribe();
8891
abortCtrl.current.abort();
8992
abortCtrl.current = new AbortController();
90-
if (!searchStrategyParams) return;
93+
94+
if (!searchStrategyParams || lastRefresh === 0) return;
9195

9296
setFetchState({
9397
...getInitialProgress(),
@@ -142,25 +146,40 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
142146
)
143147
: of(undefined);
144148

145-
const aggregatableOverallStats$ =
149+
// Have to divide into smaller requests to avoid 413 payload too large
150+
const aggregatableFieldsChunks = chunk(aggregatableFields, 30);
151+
152+
const aggregatableOverallStats$ = forkJoin(
146153
aggregatableFields.length > 0
147-
? data.search.search(
148-
{
149-
params: checkAggregatableFieldsExistRequest(
150-
index,
151-
searchQuery,
152-
aggregatableFields,
153-
samplerShardSize,
154-
timeFieldName,
155-
earliest,
156-
latest,
157-
undefined,
158-
runtimeFieldMap
159-
),
160-
},
161-
searchOptions
154+
? aggregatableFieldsChunks.map((aggregatableFieldsChunk) =>
155+
data.search
156+
.search(
157+
{
158+
params: checkAggregatableFieldsExistRequest(
159+
index,
160+
searchQuery,
161+
aggregatableFieldsChunk,
162+
samplerShardSize,
163+
timeFieldName,
164+
earliest,
165+
latest,
166+
undefined,
167+
runtimeFieldMap
168+
),
169+
},
170+
searchOptions
171+
)
172+
.pipe(
173+
switchMap((resp) => {
174+
return of({
175+
...resp,
176+
aggregatableFields: aggregatableFieldsChunk,
177+
} as AggregatableFieldOverallStats);
178+
})
179+
)
162180
)
163-
: of(undefined);
181+
: of(undefined)
182+
);
164183

165184
const documentCountStats$ =
166185
timeFieldName !== undefined && intervalMs !== undefined && intervalMs > 0
@@ -183,7 +202,7 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
183202
aggregatableOverallStatsResp,
184203
}) => {
185204
const aggregatableOverallStats = processAggregatableFieldsExistResponse(
186-
aggregatableOverallStatsResp?.rawResponse,
205+
aggregatableOverallStatsResp,
187206
aggregatableFields,
188207
samplerShardSize
189208
);
@@ -224,7 +243,7 @@ export function useOverallStats<TParams extends OverallStatsSearchStrategyParams
224243
});
225244
},
226245
});
227-
}, [data.search, searchStrategyParams, toasts]);
246+
}, [data.search, searchStrategyParams, toasts, lastRefresh]);
228247

229248
const cancelFetch = useCallback(() => {
230249
searchSubscription$.current?.unsubscribe();

‎x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_boolean_field_stats.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@ import type {
2020
FieldStatsCommonRequestParams,
2121
} from '../../../../../common/types/field_stats';
2222
import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats';
23-
import {
24-
DataPublicPluginStart,
23+
import type {
2524
IKibanaSearchRequest,
2625
IKibanaSearchResponse,
2726
ISearchOptions,
27+
ISearchStart,
2828
} from '../../../../../../../../src/plugins/data/public';
2929
import { extractErrorProperties } from '../../utils/error_utils';
3030

@@ -62,14 +62,14 @@ export const getBooleanFieldsStatsRequest = (
6262
};
6363

6464
export const fetchBooleanFieldsStats = (
65-
data: DataPublicPluginStart,
65+
dataSearch: ISearchStart,
6666
params: FieldStatsCommonRequestParams,
6767
fields: Field[],
6868
options: ISearchOptions
6969
): Observable<BooleanFieldStats[] | FieldStatsError> => {
7070
const { samplerShardSize } = params;
7171
const request: estypes.SearchRequest = getBooleanFieldsStatsRequest(params, fields);
72-
return data.search
72+
return dataSearch
7373
.search<IKibanaSearchRequest, IKibanaSearchResponse>({ params: request }, options)
7474
.pipe(
7575
catchError((e) =>

‎x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_date_field_stats.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@ import {
1616
import { isPopulatedObject } from '../../../../../common/utils/object_utils';
1717
import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats';
1818
import type { Field, DateFieldStats, Aggs } from '../../../../../common/types/field_stats';
19-
import {
20-
DataPublicPluginStart,
19+
import type {
2120
IKibanaSearchRequest,
2221
IKibanaSearchResponse,
2322
ISearchOptions,
23+
ISearchStart,
2424
} from '../../../../../../../../src/plugins/data/public';
2525
import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats';
2626
import { extractErrorProperties } from '../../utils/error_utils';
@@ -59,15 +59,15 @@ export const getDateFieldsStatsRequest = (
5959
};
6060

6161
export const fetchDateFieldsStats = (
62-
data: DataPublicPluginStart,
62+
dataSearch: ISearchStart,
6363
params: FieldStatsCommonRequestParams,
6464
fields: Field[],
6565
options: ISearchOptions
6666
): Observable<DateFieldStats[] | FieldStatsError> => {
6767
const { samplerShardSize } = params;
6868

6969
const request: estypes.SearchRequest = getDateFieldsStatsRequest(params, fields);
70-
return data.search
70+
return dataSearch
7171
.search<IKibanaSearchRequest, IKibanaSearchResponse>({ params: request }, options)
7272
.pipe(
7373
catchError((e) =>

‎x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_field_examples.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@ import type {
1515
FieldExamples,
1616
FieldStatsCommonRequestParams,
1717
} from '../../../../../common/types/field_stats';
18-
import {
19-
DataPublicPluginStart,
18+
import type {
2019
IKibanaSearchRequest,
2120
IKibanaSearchResponse,
2221
ISearchOptions,
22+
ISearchStart,
2323
} from '../../../../../../../../src/plugins/data/public';
2424
import { FieldStatsError, isIKibanaSearchResponse } from '../../../../../common/types/field_stats';
2525
import { extractErrorProperties } from '../../utils/error_utils';
@@ -60,7 +60,7 @@ export const getFieldExamplesRequest = (params: FieldStatsCommonRequestParams, f
6060
};
6161

6262
export const fetchFieldsExamples = (
63-
data: DataPublicPluginStart,
63+
dataSearch: ISearchStart,
6464
params: FieldStatsCommonRequestParams,
6565
fields: Field[],
6666
options: ISearchOptions
@@ -70,7 +70,7 @@ export const fetchFieldsExamples = (
7070
fields.map((field) => {
7171
const request: estypes.SearchRequest = getFieldExamplesRequest(params, field);
7272

73-
return data.search
73+
return dataSearch
7474
.search<IKibanaSearchRequest, IKibanaSearchResponse>({ params: request }, options)
7575
.pipe(
7676
catchError((e) =>

‎x-pack/plugins/data_visualizer/public/application/index_data_visualizer/search_strategy/requests/get_fields_stats.ts‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@
55
* 2.0.
66
*/
77

8-
import { Observable } from 'rxjs';
8+
import type { Observable } from 'rxjs';
99
import type { FieldStatsCommonRequestParams } from '../../../../../common/types/field_stats';
1010
import type { FieldStatsError } from '../../../../../common/types/field_stats';
11-
import { ISearchOptions } from '../../../../../../../../src/plugins/data/common';
12-
import { DataPublicPluginStart } from '../../../../../../../../src/plugins/data/public';
13-
import { FieldStats } from '../../../../../common/types/field_stats';
11+
import type { ISearchOptions } from '../../../../../../../../src/plugins/data/common';
12+
import { ISearchStart } from '../../../../../../../../src/plugins/data/public';
13+
import type { FieldStats } from '../../../../../common/types/field_stats';
1414
import { JOB_FIELD_TYPES } from '../../../../../common';
1515
import { fetchDateFieldsStats } from './get_date_field_stats';
1616
import { fetchBooleanFieldsStats } from './get_boolean_field_stats';
@@ -19,7 +19,7 @@ import { fetchNumericFieldsStats } from './get_numeric_field_stats';
1919
import { fetchStringFieldsStats } from './get_string_field_stats';
2020

2121
export const getFieldsStats = (
22-
dataPlugin: DataPublicPluginStart,
22+
dataSearch: ISearchStart,
2323
params: FieldStatsCommonRequestParams,
2424
fields: Array<{
2525
fieldName: string;
@@ -32,19 +32,19 @@ export const getFieldsStats = (
3232
const fieldType = fields[0].type;
3333
switch (fieldType) {
3434
case JOB_FIELD_TYPES.NUMBER:
35-
return fetchNumericFieldsStats(dataPlugin, params, fields, options);
35+
return fetchNumericFieldsStats(dataSearch, params, fields, options);
3636
case JOB_FIELD_TYPES.KEYWORD:
3737
case JOB_FIELD_TYPES.IP:
38-
return fetchStringFieldsStats(dataPlugin, params, fields, options);
38+
return fetchStringFieldsStats(dataSearch, params, fields, options);
3939
case JOB_FIELD_TYPES.DATE:
40-
return fetchDateFieldsStats(dataPlugin, params, fields, options);
40+
return fetchDateFieldsStats(dataSearch, params, fields, options);
4141
case JOB_FIELD_TYPES.BOOLEAN:
42-
return fetchBooleanFieldsStats(dataPlugin, params, fields, options);
42+
return fetchBooleanFieldsStats(dataSearch, params, fields, options);
4343
case JOB_FIELD_TYPES.TEXT:
44-
return fetchFieldsExamples(dataPlugin, params, fields, options);
44+
return fetchFieldsExamples(dataSearch, params, fields, options);
4545
default:
4646
// Use an exists filter on the the field name to get
4747
// examples of the field, so cannot batch up.
48-
return fetchFieldsExamples(dataPlugin, params, fields, options);
48+
return fetchFieldsExamples(dataSearch, params, fields, options);
4949
}
5050
};

0 commit comments

Comments
 (0)