[Lens] Fix existence for dotted paths in _source (elastic#63752) (elastic#63893)

Author: Wylie Conlon

x-pack/plugins/lens/server/routes/existing_fields.test.ts

@@ -57,6 +57,24 @@ describe('existingFields', () => {
     expect(result).toEqual(['geo.coordinates']);
   });
 
+  it('should handle objects with dotted fields', () => {
+    const result = existingFields(
+      [indexPattern({ 'geo.country_name': 'US' })],
+      [field('geo.country_name')]
+    );
+
+    expect(result).toEqual(['geo.country_name']);
+  });
+
+  it('should handle arrays with dotted fields on both sides', () => {
+    const result = existingFields(
+      [indexPattern({ 'process.cpu': [{ 'user.pct': 50 }] })],
+      [field('process.cpu.user.pct')]
+    );
+
+    expect(result).toEqual(['process.cpu.user.pct']);
+  });
+
   it('should be false if it hits a positive leaf before the end of the path', () => {
     const result = existingFields(
       [indexPattern({ geo: { coordinates: 32 } })],

x-pack/plugins/lens/server/routes/existing_fields.ts

@@ -258,6 +258,8 @@ async function fetchIndexPatternStats({
   return result.hits.hits;
 }
 
+// Recursive function to determine if the _source of a document
+// contains a known path.
 function exists(obj: unknown, path: string[], i = 0): boolean {
   if (obj == null) {
     return false;
@@ -272,6 +274,22 @@ function exists(obj: unknown, path: string[], i = 0): boolean {
   }
 
   if (typeof obj === 'object') {
+    // Because Elasticsearch flattens paths, dots in the field name are allowed
+    // as JSON keys. For example, { 'a.b': 10 }
+    const partialKeyMatches = Object.getOwnPropertyNames(obj)
+      .map(key => key.split('.'))
+      .filter(keyPaths => keyPaths.every((key, keyIndex) => key === path[keyIndex + i]));
+
+    if (partialKeyMatches.length) {
+      return partialKeyMatches.every(keyPaths => {
+        return exists(
+          (obj as Record<string, unknown>)[keyPaths.join('.')],
+          path,
+          i + keyPaths.length
+        );
+      });
+    }
+
     return exists((obj as Record<string, unknown>)[path[i]], path, i + 1);
   }