[Canvas][tech-debt] Ensure cursor is called until full results are received (#73347)

* Ensure cursor is called until full results are receeived

* Fix Typecheck

* Convert dependencies to typescript

* Fix typings

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Author: Corey Robertson

x-pack/plugins/canvas/canvas_plugin_src/functions/server/esdocs.ts

@@ -7,7 +7,6 @@
 import squel from 'squel';
 import { ExpressionFunctionDefinition } from 'src/plugins/expressions';
 /* eslint-disable */
-// @ts-expect-error untyped local
 import { queryEsSQL } from '../../../server/lib/query_es_sql';
 /* eslint-enable */
 import { ExpressionValueFilter } from '../../../types';

x-pack/plugins/canvas/canvas_plugin_src/functions/server/essql.ts

@@ -6,7 +6,6 @@
 
 import { ExpressionFunctionDefinition } from 'src/plugins/expressions/common';
 /* eslint-disable */
-// @ts-expect-error untyped local
 import { queryEsSQL } from '../../../server/lib/query_es_sql';
 /* eslint-enable */
 import { ExpressionValueFilter } from '../../../types';

x-pack/plugins/canvas/server/lib/build_bool_array.js renamed to x-pack/plugins/canvas/server/lib/build_bool_array.ts

@@ -5,10 +5,11 @@
  */
 
 import { getESFilter } from './get_es_filter';
+import { ExpressionValueFilter } from '../../types';
 
-const compact = (arr) => (Array.isArray(arr) ? arr.filter((val) => Boolean(val)) : []);
+const compact = <T>(arr: T[]) => (Array.isArray(arr) ? arr.filter((val) => Boolean(val)) : []);
 
-export function buildBoolArray(canvasQueryFilterArray) {
+export function buildBoolArray(canvasQueryFilterArray: ExpressionValueFilter[]) {
   return compact(
     canvasQueryFilterArray.map((clause) => {
       try {

x-pack/plugins/canvas/server/lib/filters.js

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  FilterType,
+  ExpressionValueFilter,
+  CanvasTimeFilter,
+  CanvasLuceneFilter,
+  CanvasExactlyFilter,
+} from '../../types';
+
+/*
+ TODO: This could be pluggable
+*/
+
+const isTimeFilter = (
+  maybeTimeFilter: ExpressionValueFilter
+): maybeTimeFilter is CanvasTimeFilter => {
+  return maybeTimeFilter.filterType === FilterType.time;
+};
+const isLuceneFilter = (
+  maybeLuceneFilter: ExpressionValueFilter
+): maybeLuceneFilter is CanvasLuceneFilter => {
+  return maybeLuceneFilter.filterType === FilterType.luceneQueryString;
+};
+const isExactlyFilter = (
+  maybeExactlyFilter: ExpressionValueFilter
+): maybeExactlyFilter is CanvasExactlyFilter => {
+  return maybeExactlyFilter.filterType === FilterType.exactly;
+};
+
+export function time(filter: ExpressionValueFilter) {
+  if (!isTimeFilter(filter) || !filter.column) {
+    throw new Error('column is required for Elasticsearch range filters');
+  }
+  return {
+    range: {
+      [filter.column]: { gte: filter.from, lte: filter.to },
+    },
+  };
+}
+
+export function luceneQueryString(filter: ExpressionValueFilter) {
+  if (!isLuceneFilter(filter)) {
+    throw new Error('Filter is not a lucene filter');
+  }
+  return {
+    query_string: {
+      query: filter.query || '*',
+    },
+  };
+}
+
+export function exactly(filter: ExpressionValueFilter) {
+  if (!isExactlyFilter(filter)) {
+    throw new Error('Filter is not an exactly filter');
+  }
+  return {
+    term: {
+      [filter.column]: {
+        value: filter.value,
+      },
+    },
+  };
+}
+
+export const filters: Record<string, any> = {
+  exactly,
+  time,
+  luceneQueryString,
+};

x-pack/plugins/canvas/server/lib/filters.ts

@@ -10,11 +10,11 @@
   filter is the abstracted canvas filter.
 */
 
-/*eslint import/namespace: ['error', { allowComputed: true }]*/
-import * as filters from './filters';
+import { filters } from './filters';
+import { ExpressionValueFilter } from '../../types';
 
-export function getESFilter(filter) {
-  if (!filters[filter.filterType]) {
+export function getESFilter(filter: ExpressionValueFilter) {
+  if (!filter.filterType || !filters[filter.filterType]) {
     throw new Error(`Unknown filter type: ${filter.filterType}`);
   }
 

x-pack/plugins/canvas/server/lib/get_es_filter.js renamed to x-pack/plugins/canvas/server/lib/get_es_filter.ts

@@ -4,8 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export function normalizeType(type) {
-  const normalTypes = {
+export function normalizeType(type: string) {
+  const normalTypes: Record<string, string[]> = {
     string: ['string', 'text', 'keyword', '_type', '_id', '_index', 'geo_point'],
     number: [
       'float',

x-pack/plugins/canvas/server/lib/normalize_type.js renamed to x-pack/plugins/canvas/server/lib/normalize_type.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { zipObject } from 'lodash';
+import { queryEsSQL } from './query_es_sql';
+// @ts-expect-error
+import { buildBoolArray } from './build_bool_array';
+
+const response = {
+  columns: [
+    { name: 'One', type: 'keyword' },
+    { name: 'Two', type: 'keyword' },
+  ],
+  rows: [
+    ['foo', 'bar'],
+    ['buz', 'baz'],
+  ],
+  cursor: 'cursor-value',
+};
+
+const baseArgs = {
+  count: 1,
+  query: 'query',
+  filter: [],
+  timezone: 'timezone',
+};
+
+const getApi = (resp = response) => {
+  const api = jest.fn();
+  api.mockResolvedValue(resp);
+  return api;
+};
+
+describe('query_es_sql', () => {
+  it('should call the api with the given args', async () => {
+    const api = getApi();
+
+    queryEsSQL(api, baseArgs);
+
+    expect(api).toHaveBeenCalled();
+    const givenArgs = api.mock.calls[0][1];
+
+    expect(givenArgs.body.fetch_size).toBe(baseArgs.count);
+    expect(givenArgs.body.query).toBe(baseArgs.query);
+    expect(givenArgs.body.time_zone).toBe(baseArgs.timezone);
+  });
+
+  it('formats the response', async () => {
+    const api = getApi();
+
+    const result = await queryEsSQL(api, baseArgs);
+
+    const expectedColumns = response.columns.map((c) => ({ name: c.name, type: 'string' }));
+    const columnNames = expectedColumns.map((c) => c.name);
+    const expectedRows = response.rows.map((r) => zipObject(columnNames, r));
+
+    expect(result.type).toBe('datatable');
+    expect(result.columns).toEqual(expectedColumns);
+    expect(result.rows).toEqual(expectedRows);
+  });
+
+  it('fetches pages until it has the requested count', async () => {
+    const pageOne = {
+      columns: [
+        { name: 'One', type: 'keyword' },
+        { name: 'Two', type: 'keyword' },
+      ],
+      rows: [['foo', 'bar']],
+      cursor: 'cursor-value',
+    };
+
+    const pageTwo = {
+      rows: [['buz', 'baz']],
+    };
+
+    const api = getApi(pageOne);
+    api.mockReturnValueOnce(pageOne).mockReturnValueOnce(pageTwo);
+
+    const result = await queryEsSQL(api, { ...baseArgs, count: 2 });
+    expect(result.rows).toHaveLength(2);
+  });
+
+  it('closes any cursors that remain open', async () => {
+    const api = getApi();
+
+    await queryEsSQL(api, baseArgs);
+    expect(api.mock.calls[1][1].body.cursor).toBe(response.cursor);
+  });
+
+  it('throws on errors', async () => {
+    const api = getApi();
+    api.mockRejectedValueOnce(new Error('parsing_exception'));
+    api.mockRejectedValueOnce(new Error('generic es error'));
+
+    expect(queryEsSQL(api, baseArgs)).rejects.toThrowErrorMatchingInlineSnapshot(
+      `"Couldn't parse Elasticsearch SQL query. You may need to add double quotes to names containing special characters. Check your query and try again. Error: parsing_exception"`
+    );
+
+    expect(queryEsSQL(api, baseArgs)).rejects.toThrowErrorMatchingInlineSnapshot(
+      `"Unexpected error from Elasticsearch: generic es error"`
+    );
+  });
+});