[Discover] Fix formatting and sorting for custom ES|QL vars (#209360)

- Closes #208020

## Summary

By default the data view (which is used in ES|QL mode) has only mapped
fields as per field caps for the current index pattern. This PR
dynamically extends with additional fields based on ES|QL meta
information.

This change helps to fix the formatting for ES|QL var values and fixes
sorting on them.

<img width="1673" alt="Screenshot 2025-02-03 at 18 42 14"
src="https://github.com/user-attachments/assets/3647a375-f0f5-43e6-815d-d5a4292c637a"
/>
<img width="643" alt="Screenshot 2025-02-03 at 18 42 50"
src="https://github.com/user-attachments/assets/9d84bc23-7665-43c1-8ac2-d67174b68c31"
/>

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
(cherry picked from commit 9635cfa)

src/platform/packages/shared/kbn-data-view-utils/index.ts

@@ -9,5 +9,6 @@
 
 export * from './src/constants';
 export { convertDatatableColumnToDataViewFieldSpec } from './src/utils/convert_to_data_view_field_spec';
+export { getDataViewFieldOrCreateFromColumnMeta } from './src/utils/get_data_view_field_or_create';
 export { createRegExpPatternFrom } from './src/utils/create_regexp_pattern_from';
 export { testPatternAgainstAllowedList } from './src/utils/test_pattern_against_allowed_list';

src/platform/packages/shared/kbn-data-view-utils/src/utils/get_data_view_field_or_create.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { isEqual } from 'lodash';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import type { DatatableColumnMeta } from '@kbn/expressions-plugin/common';
+import { convertDatatableColumnToDataViewFieldSpec } from './convert_to_data_view_field_spec';
+
+export const getDataViewFieldOrCreateFromColumnMeta = ({
+  dataView,
+  fieldName,
+  columnMeta,
+}: {
+  dataView: DataView;
+  fieldName: string;
+  columnMeta?: DatatableColumnMeta; // based on ES|QL query
+}) => {
+  const dataViewField = dataView.fields.getByName(fieldName);
+
+  if (!columnMeta) {
+    return dataViewField;
+  }
+
+  const fieldSpecFromColumnMeta = convertDatatableColumnToDataViewFieldSpec({
+    name: fieldName,
+    id: fieldName,
+    meta: columnMeta,
+  });
+
+  if (
+    !dataViewField ||
+    dataViewField.type !== fieldSpecFromColumnMeta.type ||
+    !isEqual(dataViewField.esTypes, fieldSpecFromColumnMeta.esTypes)
+  ) {
+    return dataView.fields.create(fieldSpecFromColumnMeta);
+  }
+
+  return dataViewField;
+};

src/platform/packages/shared/kbn-discover-utils/src/__mocks__/data_view.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { DataView, DataViewField } from '@kbn/data-views-plugin/public';
+import { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/public';
 
 export const shallowMockedFields = [
   {
@@ -101,6 +101,10 @@ export const buildDataViewMock = ({
     return dataViewFields;
   };
 
+  dataViewFields.create = (spec: FieldSpec) => {
+    return new DataViewField(spec);
+  };
+
   const dataView = {
     id: `${name}-id`,
     title: `${name}-title`,

src/platform/packages/shared/kbn-unified-data-table/__mocks__/table_context.ts

@@ -36,6 +36,7 @@ const buildTableContext = (dataView: DataView, rows: DataTableRecord[]): DataTab
         fieldFormats: servicesMock.fieldFormats,
         rows,
         dataView,
+        columnsMeta: undefined,
         options,
       }),
   };