Skip to content

[ML] Replace use of rest_total_hits_as_int with track_total_hits #78423

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 30, 2020
Merged

[ML] Replace use of rest_total_hits_as_int with track_total_hits #78423

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b3594cb
[ML] Replace use of rest_total_hits_as_int with track_total_hits
peteharverson Sep 24, 2020
ecd38dd
[ML] Replace use of lodash get for obtaining hits total
peteharverson Sep 29, 2020
d19ff31
[ML] Update mocks to use new total hits format
peteharverson Sep 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,11 +296,10 @@ export function getTestUrl(job, customUrl) {
return new Promise((resolve, reject) => {
ml.results
.anomalySearch({
rest_total_hits_as_int: true,
body,
})
.then((resp) => {
if (resp.hits.total > 0) {
if (resp.hits.total.value > 0) {
const record = resp.hits.hits[0]._source;
testUrl = replaceTokensInUrlValue(customUrl, bucketSpanSecs, record, 'timestamp');
resolve(testUrl);
Expand Down
6 changes: 2 additions & 4 deletions x-pack/plugins/ml/public/application/services/forecast_service.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ function getForecastsSummary(job, query, earliestMs, maxResults) {
ml.results
.anomalySearch({
size: maxResults,
rest_total_hits_as_int: true,
body: {
query: {
bool: {
Expand All @@ -61,7 +60,7 @@ function getForecastsSummary(job, query, earliestMs, maxResults) {
},
})
.then((resp) => {
if (resp.hits.total !== 0) {
if (resp.hits.total.value > 0) {
obj.forecasts = resp.hits.hits.map((hit) => hit._source);
}

Expand Down Expand Up @@ -344,7 +343,6 @@ function getForecastRequestStats(job, forecastId) {
ml.results
.anomalySearch({
size: 1,
rest_total_hits_as_int: true,
body: {
query: {
bool: {
Expand All @@ -354,7 +352,7 @@ function getForecastRequestStats(job, forecastId) {
},
})
.then((resp) => {
if (resp.hits.total !== 0) {
if (resp.hits.total.value > 0) {
obj.stats = resp.hits.hits[0]._source;
}
resolve(obj);
Expand Down
3 changes: 1 addition & 2 deletions x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,6 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
return mlApiServices.results
.anomalySearch$({
index: ML_RESULTS_INDEX_PATTERN,
rest_total_hits_as_int: true,
size: maxResults !== undefined ? maxResults : 100,
body: {
query: {
Expand All @@ -427,7 +426,7 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
})
.pipe(
map((resp) => {
if (resp.hits.total !== 0) {
if (resp.hits.total.value > 0) {
each(resp.hits.hits, (hit: any) => {
obj.records.push(hit._source);
});
Expand Down
16 changes: 6 additions & 10 deletions x-pack/plugins/ml/public/application/services/results_service/results_service.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,6 @@ export function resultsServiceProvider(mlApiServices) {
mlApiServices.results
.anomalySearch({
size: maxResults !== undefined ? maxResults : 100,
rest_total_hits_as_int: true,
body: {
_source: ['job_id', 'detector_index', 'influencers', 'record_score'],
query: {
Expand All @@ -749,7 +748,7 @@ export function resultsServiceProvider(mlApiServices) {
},
})
.then((resp) => {
if (resp.hits.total !== 0) {
if (resp.hits.total.value > 0) {
each(resp.hits.hits, (hit) => {
obj.records.push(hit._source);
});
Expand Down Expand Up @@ -857,7 +856,6 @@ export function resultsServiceProvider(mlApiServices) {
mlApiServices.results
.anomalySearch({
size: maxResults !== undefined ? maxResults : 100,
rest_total_hits_as_int: true,
body: {
query: {
bool: {
Expand All @@ -880,7 +878,7 @@ export function resultsServiceProvider(mlApiServices) {
},
})
.then((resp) => {
if (resp.hits.total !== 0) {
if (resp.hits.total.value > 0) {
each(resp.hits.hits, (hit) => {
obj.records.push(hit._source);
});
Expand Down Expand Up @@ -982,7 +980,6 @@ export function resultsServiceProvider(mlApiServices) {
mlApiServices.results
.anomalySearch({
size: maxResults !== undefined ? maxResults : 100,
rest_total_hits_as_int: true,
body: {
query: {
bool: {
Expand All @@ -1005,7 +1002,7 @@ export function resultsServiceProvider(mlApiServices) {
},
})
.then((resp) => {
if (resp.hits.total !== 0) {
if (resp.hits.total.value > 0) {
each(resp.hits.hits, (hit) => {
obj.records.push(hit._source);
});
Expand Down Expand Up @@ -1058,7 +1055,6 @@ export function resultsServiceProvider(mlApiServices) {
mlApiServices
.esSearch({
index,
rest_total_hits_as_int: true,
size: 0,
body: {
query: {
Expand Down Expand Up @@ -1090,7 +1086,7 @@ export function resultsServiceProvider(mlApiServices) {
const time = dataForTime.key;
obj.results[time] = dataForTime.doc_count;
});
obj.total = resp.hits.total;
obj.total = resp.hits.total.value;

resolve(obj);
})
Expand Down Expand Up @@ -1227,13 +1223,13 @@ export function resultsServiceProvider(mlApiServices) {
.esSearch({
index,
body,
rest_total_hits_as_int: true,
track_total_hits: true,
})
.then((resp) => {
// Because of the sampling, results of metricFunctions which use sum or count
// can be significantly skewed. Taking into account totalHits we calculate a
// a factor to normalize results for these metricFunctions.
const totalHits = get(resp, ['hits', 'total'], 0);
const totalHits = resp.hits.total.value;
const successfulShards = get(resp, ['_shards', 'successful'], 0);

let normalizeFactor = 1;
Expand Down
3 changes: 1 addition & 2 deletions x-pack/plugins/ml/server/models/data_frame_analytics/analytics_audit_messages.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ export function analyticsAuditMessagesProvider({ asInternalUser }: IScopedCluste
const { body } = await asInternalUser.search({
index: ML_NOTIFICATION_INDEX_PATTERN,
ignore_unavailable: true,
rest_total_hits_as_int: true,
size: SIZE,
body: {
sort: [{ timestamp: { order: 'desc' } }, { job_id: { order: 'asc' } }],
Expand All @@ -80,7 +79,7 @@ export function analyticsAuditMessagesProvider({ asInternalUser }: IScopedCluste
});

let messages = [];
if (body.hits.total !== 0) {
if (body.hits.total.value > 0) {
messages = body.hits.hits.map((hit: Message) => hit._source);
messages.reverse();
}
Expand Down
3 changes: 1 addition & 2 deletions x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,12 +248,11 @@ export class DataRecognizer {

const { body } = await this._asCurrentUser.search({
index,
rest_total_hits_as_int: true,
size,
body: searchBody,
});

return body.hits.total !== 0;
return body.hits.total.value > 0;
}

async listModules() {
Expand Down
10 changes: 4 additions & 6 deletions x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -627,12 +627,12 @@ export class DataVisualizer {

const { body } = await this._asCurrentUser.search({
index,
rest_total_hits_as_int: true,
track_total_hits: true,
size,
body: searchBody,
});
const aggregations = body.aggregations;
const totalCount = get(body, ['hits', 'total'], 0);
const totalCount = body.hits.total.value;
const stats = {
totalCount,
aggregatableExistsFields: [] as FieldData[],
Expand Down Expand Up @@ -694,11 +694,10 @@ export class DataVisualizer {

const { body } = await this._asCurrentUser.search({
index,
rest_total_hits_as_int: true,
size,
body: searchBody,
});
return body.hits.total > 0;
return body.hits.total.value > 0;
}

async getDocumentCountStats(
Expand Down Expand Up @@ -1164,15 +1163,14 @@ export class DataVisualizer {

const { body } = await this._asCurrentUser.search({
index,
rest_total_hits_as_int: true,
size,
body: searchBody,
});
const stats = {
fieldName: field,
examples: [] as any[],
};
if (body.hits.total !== 0) {
if (body.hits.total.value > 0) {
const hits = body.hits.hits;
for (let i = 0; i < hits.length; i++) {
// Look in the _source for the field value.
Expand Down
6 changes: 2 additions & 4 deletions x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ export function jobAuditMessagesProvider({ asInternalUser }) {
const { body } = await asInternalUser.search({
index: ML_NOTIFICATION_INDEX_PATTERN,
ignore_unavailable: true,
rest_total_hits_as_int: true,
size: SIZE,
body: {
sort: [{ timestamp: { order: 'desc' } }, { job_id: { order: 'asc' } }],
Expand All @@ -111,7 +110,7 @@ export function jobAuditMessagesProvider({ asInternalUser }) {
});

let messages = [];
if (body.hits.total !== 0) {
if (body.hits.total.value > 0) {
messages = body.hits.hits.map((hit) => hit._source);
}
return messages;
Expand Down Expand Up @@ -153,7 +152,6 @@ export function jobAuditMessagesProvider({ asInternalUser }) {
const { body } = await asInternalUser.search({
index: ML_NOTIFICATION_INDEX_PATTERN,
ignore_unavailable: true,
rest_total_hits_as_int: true,
size: 0,
body: {
query,
Expand Down Expand Up @@ -196,7 +194,7 @@ export function jobAuditMessagesProvider({ asInternalUser }) {
let messagesPerJob = [];
const jobMessages = [];
if (
body.hits.total !== 0 &&
body.hits.total.value > 0 &&
body.aggregations &&
body.aggregations.levelsPerJob &&
body.aggregations.levelsPerJob.buckets &&
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugins/ml/server/models/job_service/new_job/line_chart.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ function processSearchResults(resp: any, fields: string[]): ProcessedResults {
return {
success: true,
results: tempResults,
totalResults: resp.hits.total,
totalResults: resp.hits.total.value,
};
}

Expand All @@ -107,7 +107,7 @@ function getSearchJsonFromConfig(
const json = {
index: indexPatternTitle,
size: 0,
rest_total_hits_as_int: true,
track_total_hits: true,
body: {
query: {},
aggs: {
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugins/ml/server/models/job_service/new_job/population_chart.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ function processSearchResults(resp: any, fields: string[]): ProcessedResults {
return {
success: true,
results: tempResults,
totalResults: resp.hits.total,
totalResults: resp.hits.total.value,
};
}

Expand All @@ -135,7 +135,7 @@ function getPopulationSearchJsonFromConfig(
const json = {
index: indexPatternTitle,
size: 0,
rest_total_hits_as_int: true,
track_total_hits: true,
body: {
query: {},
aggs: {
Expand Down
7 changes: 5 additions & 2 deletions x-pack/plugins/ml/server/models/job_validation/__mocks__/mock_farequote_cardinality.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
"took": 0,
"timed_out": false,
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": { "total": 86274, "max_score": 0, "hits": [] },
"aggregations": { "airline_cardinality": { "value": 19 }, "airline_count": { "doc_count": 86274 } }
"hits": { "total": { "value": 86274, "relation": "eq" }, "max_score": 0, "hits": [] },
"aggregations": {
"airline_cardinality": { "value": 19 },
"airline_count": { "doc_count": 86274 }
}
}
2 changes: 1 addition & 1 deletion x-pack/plugins/ml/server/models/job_validation/__mocks__/mock_farequote_search_response.json
View file
Open in desktop

Large diffs are not rendered by default.

Loading