Fix filter loading issues by searching over more rows and caching (#786)

Fixes: HDX-1562
Fixes: HDX-1575

Author: knudtty

packages/app/src/components/DBSearchPageFilters.tsx

@@ -1,4 +1,5 @@
 import { useEffect, useMemo, useState } from 'react';
+import { DEFAULT_MAX_ROWS_TO_READ } from '@hyperdx/common-utils/dist/metadata';
 import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
 import {
   Box,
@@ -18,6 +19,7 @@ import {
 } from '@mantine/core';
 import { IconSearch } from '@tabler/icons-react';
 
+import { useExplainQuery } from '@/hooks/useExplainQuery';
 import { useAllFields, useGetKeyValues } from '@/hooks/useMetadata';
 import useResizable from '@/hooks/useResizable';
 import { FilterStateHook, usePinnedFilters } from '@/searchFilters';
@@ -343,6 +345,9 @@ export const DBSearchPageFilters = ({
   );
   const { width, startResize } = useResizable(16, 'left');
 
+  const { data: countData } = useExplainQuery(chartConfig);
+  const numRows: number = countData?.[0]?.rows ?? 0;
+
   const { data, isLoading } = useAllFields({
     databaseName: chartConfig.from.databaseName,
     tableName: chartConfig.from.tableName,
@@ -390,19 +395,33 @@ export const DBSearchPageFilters = ({
   useEffect(() => {
     if (!isLive) {
       setDateRange(chartConfig.dateRange);
+      setDisableRowLimit(false);
     }
   }, [chartConfig.dateRange, isLive]);
 
   const showRefreshButton = isLive && dateRange !== chartConfig.dateRange;
 
+  const [disableRowLimit, setDisableRowLimit] = useState(false);
+  const keyLimit = 100;
   const {
     data: facets,
     isLoading: isFacetsLoading,
     isFetching: isFacetsFetching,
   } = useGetKeyValues({
     chartConfigs: { ...chartConfig, dateRange },
+    limit: keyLimit,
     keys: datum,
+    disableRowLimit,
   });
+  useEffect(() => {
+    if (
+      numRows > DEFAULT_MAX_ROWS_TO_READ &&
+      facets &&
+      facets.length < keyLimit
+    ) {
+      setDisableRowLimit(true);
+    }
+  }, [numRows, keyLimit, facets]);
 
   const shownFacets = useMemo(() => {
     const _facets: { key: string; value: string[] }[] = [];

packages/app/src/hooks/useMetadata.tsx

@@ -116,6 +116,7 @@ export function useGetKeyValues(
       'useMetadata.useGetKeyValues',
       ...chartConfigsArr.map(cc => ({ ...cc })),
       ...keys,
+      disableRowLimit,
     ],
     queryFn: async () =>
       (

packages/common-utils/src/metadata.ts

@@ -11,7 +11,7 @@ import {
 import { renderChartConfig } from '@/renderChartConfig';
 import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types';
 
-const DEFAULT_SAMPLE_SIZE = 1e6;
+export const DEFAULT_MAX_ROWS_TO_READ = 1e6;
 
 export class MetadataCache {
   private cache = new Map<string, any>();
@@ -268,7 +268,7 @@ export class Metadata {
           query_params: sql.params,
           connectionId,
           clickhouse_settings: {
-            max_rows_to_read: DEFAULT_SAMPLE_SIZE,
+            max_rows_to_read: DEFAULT_MAX_ROWS_TO_READ,
             read_overflow_mode: 'break',
           },
         })
@@ -341,7 +341,7 @@ export class Metadata {
             query_params: sql.params,
             connectionId,
             clickhouse_settings: {
-              max_rows_to_read: DEFAULT_SAMPLE_SIZE,
+              max_rows_to_read: DEFAULT_MAX_ROWS_TO_READ,
               read_overflow_mode: 'break',
             },
           })
@@ -438,36 +438,41 @@ export class Metadata {
     limit?: number;
     disableRowLimit?: boolean;
   }) {
-    const sql = await renderChartConfig(
-      {
-        ...chartConfig,
-        select: keys
-          .map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
-          .join(', '),
+    return this.cache.getOrFetch(
+      `${chartConfig.from.databaseName}.${keys.join(',')}.${chartConfig.dateRange.toString()}.${disableRowLimit}.values`,
+      async () => {
+        const sql = await renderChartConfig(
+          {
+            ...chartConfig,
+            select: keys
+              .map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
+              .join(', '),
+          },
+          this,
+        );
+
+        const json = await this.clickhouseClient
+          .query<'JSON'>({
+            query: sql.sql,
+            query_params: sql.params,
+            connectionId: chartConfig.connection,
+            clickhouse_settings: !disableRowLimit
+              ? {
+                  max_rows_to_read: DEFAULT_MAX_ROWS_TO_READ,
+                  read_overflow_mode: 'break',
+                }
+              : undefined,
+          })
+          .then(res => res.json<any>());
+
+        // TODO: Fix type issues mentioned in HDX-1548. value is not acually a
+        // string[], sometimes it's { [key: string]: string; }
+        return Object.entries(json?.data?.[0]).map(([key, value]) => ({
+          key: keys[parseInt(key.replace('param', ''))],
+          value: (value as string[])?.filter(Boolean), // remove nulls
+        }));
       },
-      this,
     );
-
-    const json = await this.clickhouseClient
-      .query<'JSON'>({
-        query: sql.sql,
-        query_params: sql.params,
-        connectionId: chartConfig.connection,
-        clickhouse_settings: !disableRowLimit
-          ? {
-              max_rows_to_read: DEFAULT_SAMPLE_SIZE,
-              read_overflow_mode: 'break',
-            }
-          : undefined,
-      })
-      .then(res => res.json<any>());
-
-    // TODO: Fix type issues mentioned in HDX-1548. value is not acually a
-    // string[], sometimes it's { [key: string]: string; }
-    return Object.entries(json.data[0]).map(([key, value]) => ({
-      key: keys[parseInt(key.replace('param', ''))],
-      value: (value as string[])?.filter(Boolean), // remove nulls
-    }));
   }
 }