feat: granular date filtering in explorer (#4118)

.eslintrc.cjs

@@ -179,6 +179,7 @@ module.exports = {
         'jsx-a11y/click-events-have-key-events': 'off',
         'jsx-a11y/no-static-element-interactions': 'off',
         '@next/next/no-html-link-for-pages': 'off',
+        'unicorn/no-negated-condition': 'off',
       },
     },
     {

packages/services/api/src/modules/operations/providers/operations-reader.ts

@@ -62,6 +62,20 @@ function ensureNumber(value: number | string): number {
   return parseFloat(value);
 }
 
+const msMinute = 60 * 1_000;
+const msHour = msMinute * 60;
+const msDay = msHour * 24;
+
+// How long rows are kept in the database, per table.
+const tableTTLInHours = {
+  daily: 365 * 24,
+  hourly: 30 * 24,
+  minutely: 24,
+};
+
+const thresholdDataPointPerDay = 28;
+const thresholdDataPointPerHour = 24;
+
 @Injectable({
   global: true,
 })
@@ -99,108 +113,67 @@ export class OperationsReader {
       };
     }
 
-    const dataPoints = resolution;
-    const timeDifference = period.to.getTime() - period.from.getTime();
-    const timeDifferenceInHours = timeDifference / 1000 / 60 / 60;
-    const timeDifferenceInDays = timeDifference / 1000 / 60 / 60 / 24;
+    if (resolution && (resolution < 1 || resolution > 90)) {
+      throw new Error('Invalid resolution provided.');
+    } else {
+      // default value :shrug:
+      resolution = 30;
+    }
 
-    // How long rows are kept in the database, per table.
-    const tableTTLInHours = {
-      daily: 365 * 24,
-      hourly: 30 * 24,
-      minutely: 24,
-    };
+    const now = new Date();
 
     // The oldest data point we can fetch from the database, per table.
     // ! We subtract 2 minutes as we round the date to the nearest minute on UI
     //   and there's also a chance that request will be made at 59th second of the minute
     //   and by the time it this function is called the minute will change.
     //   That's why we use 2 minutes as a buffer.
     const tableOldestDateTimePoint = {
-      daily: subMinutes(startOfDay(subHours(new Date(), tableTTLInHours.daily)), 2),
-      hourly: subMinutes(startOfHour(subHours(new Date(), tableTTLInHours.hourly)), 2),
-      minutely: subMinutes(startOfMinute(subHours(new Date(), tableTTLInHours.minutely)), 2),
+      daily: subMinutes(startOfDay(subHours(now, tableTTLInHours.daily)), 2),
+      hourly: subMinutes(startOfHour(subHours(now, tableTTLInHours.hourly)), 2),
+      minutely: subMinutes(startOfMinute(subHours(now, tableTTLInHours.minutely)), 2),
     };
 
-    let selectedQueryType: 'daily' | 'hourly' | 'minutely';
+    const daysDifference = (period.to.getTime() - period.from.getTime()) / msDay;
 
-    // If the user requested a specific resolution, we need to pick the right table.
-    //
-    if (dataPoints) {
-      const interval = calculateTimeWindow({ period, resolution });
+    if (
+      daysDifference > thresholdDataPointPerDay ||
+      /** if we are outside this range, we always need to get daily data */
+      period.to.getTime() <= tableOldestDateTimePoint.daily.getTime() ||
+      period.from.getTime() <= tableOldestDateTimePoint.daily.getTime()
+    ) {
+      return {
+        ...queryMap['daily'],
+        queryType: 'daily',
+      };
+    }
 
-      if (timeDifferenceInDays >= dataPoints) {
-        // If user selected a date range of 60 days or more and requested 30 data points
-        // we can use daily table as each data point represents at least 1 day.
-        selectedQueryType = 'daily';
-
-        if (interval.unit !== 'd') {
-          this.logger.error(
-            `Calculated interval ${interval.value}${
-              interval.unit
-            } for the requested date range ${formatDate(period.from)} - ${formatDate(
-              period.to,
-            )} does not satisfy the daily table.`,
-          );
-          throw new Error(`Invalid number of data points for the requested date range`);
-        }
-      } else if (timeDifferenceInHours >= dataPoints) {
-        // Same as for daily table, but for hourly table.
-        // If data point represents at least 1 full hour, use hourly table.
-        selectedQueryType = 'hourly';
-
-        if (interval.unit === 'm') {
-          this.logger.error(
-            `Calculated interval ${interval.value}${
-              interval.unit
-            } for the requested date range ${formatDate(period.from)} - ${formatDate(
-              period.to,
-            )} does not satisfy the hourly table.`,
-          );
-          throw new Error(`Invalid number of data points for the requested date range`);
-        }
-      } else {
-        selectedQueryType = 'minutely';
-      }
-    } else if (timeDifferenceInHours > 24) {
-      selectedQueryType = 'daily';
-    } else if (timeDifferenceInHours > 1) {
-      selectedQueryType = 'hourly';
-    } else {
-      selectedQueryType = 'minutely';
+    const hoursDifference = (period.to.getTime() - period.from.getTime()) / msHour;
+
+    if (
+      hoursDifference > thresholdDataPointPerHour ||
+      /** if we are outside this range, we always need to get hourly data */
+      period.to.getTime() <= tableOldestDateTimePoint.hourly.getTime() ||
+      period.from.getTime() <= tableOldestDateTimePoint.hourly.getTime()
+    ) {
+      return {
+        ...queryMap['hourly'],
+        queryType: 'hourly',
+      };
     }
 
-    if (tableOldestDateTimePoint[selectedQueryType].getTime() > period.from.getTime()) {
-      if (dataPoints) {
-        // If the oldest data point is newer than the requested data range,
-        // and the user requested a specific resolution
-        const interval = calculateTimeWindow({ period, resolution });
-        this.logger.error(
-          `Requested date range ${formatDate(period.from)} - ${formatDate(
-            period.to,
-          )} is older than the oldest available data point ${formatDate(
-            tableOldestDateTimePoint[selectedQueryType],
-          )} for the selected query type ${selectedQueryType}. The requested resolution is ${
-            interval.value
-          } ${interval.unit}.`,
-        );
-        throw new Error(`The requested date range is too old for the selected query type.`);
-      } else {
-        // If the oldest data point is newer than the requested data range,
-        // but the user didn't request a specific resolution, it's fine.
-        this.logger.warn(
-          `[OPERATIONS_READER_TTL_DATE_RANGE_WARN] The requested date range is too old for the selected query type. Requested date range ${formatDate(
-            period.from,
-          )} - ${formatDate(period.to)} is older than the oldest available data point ${formatDate(
-            tableOldestDateTimePoint[selectedQueryType],
-          )}`,
-        );
-      }
+    if (
+      period.to.getTime() <= tableOldestDateTimePoint.minutely.getTime() ||
+      period.from.getTime() <= tableOldestDateTimePoint.minutely.getTime()
+    ) {
+      this.logger.error(
+        `Requested date range ${formatDate(period.from)} - ${formatDate(period.to)} is too old.`,
+      );
+      throw new Error(`The requested date range is too old for the selected query type.`);
     }
 
     return {
-      ...queryMap[selectedQueryType],
-      queryType: selectedQueryType,
+      ...queryMap['minutely'],
+      queryType: 'minutely',
     };
   }
 

packages/web/app/package.json

@@ -77,6 +77,7 @@
     "pino": "8.19.0",
     "pino-http": "9.0.0",
     "react": "18.2.0",
+    "react-day-picker": "8.10.0",
     "react-dom": "18.2.0",
     "react-highlight-words": "0.20.0",
     "react-hook-form": "7.51.0",

packages/web/app/pages/[organizationId]/[projectId]/[targetId]/explorer.tsx

@@ -1,4 +1,4 @@
-import { ReactElement, useEffect } from 'react';
+import { ReactElement, useEffect, useRef } from 'react';
 import Link from 'next/link';
 import { AlertCircleIcon } from 'lucide-react';
 import { useQuery } from 'urql';
@@ -140,14 +140,15 @@ const TargetExplorerPageQuery = graphql(`
 
 function ExplorerPageContent() {
   const router = useRouteSelector();
-  const { period, dataRetentionInDays, setDataRetentionInDays } = useSchemaExplorerContext();
+  const { resolvedPeriod, dataRetentionInDays, setDataRetentionInDays } =
+    useSchemaExplorerContext();
   const [query] = useQuery({
     query: TargetExplorerPageQuery,
     variables: {
       organizationId: router.organizationId,
       projectId: router.projectId,
       targetId: router.targetId,
-      period,
+      period: resolvedPeriod,
     },
   });
 
@@ -172,6 +173,13 @@ function ExplorerPageContent() {
   const latestSchemaVersion = currentTarget?.latestSchemaVersion;
   const latestValidSchemaVersion = currentTarget?.latestValidSchemaVersion;
 
+  /* to avoid janky behaviour we keep track if the version has a successful explorer once, and in that case always show the filter bar. */
+  const isFilterVisible = useRef(false);
+
+  if (latestValidSchemaVersion?.explorer) {
+    isFilterVisible.current = true;
+  }
+
   return (
     <TargetLayout
       page={Page.Explorer}
@@ -186,12 +194,12 @@ function ExplorerPageContent() {
           <Title>Explore</Title>
           <Subtitle>Insights from the latest version.</Subtitle>
         </div>
-        {!query.fetching && latestValidSchemaVersion?.explorer && (
+        {isFilterVisible.current && (
           <SchemaExplorerFilter
             organization={{ cleanId: router.organizationId }}
             project={{ cleanId: router.projectId }}
             target={{ cleanId: router.targetId }}
-            period={period}
+            period={resolvedPeriod}
           >
             <Button variant="outline" asChild>
               <Link

packages/web/app/pages/[organizationId]/[projectId]/[targetId]/explorer/[typename].tsx

@@ -164,14 +164,15 @@ const TargetExplorerTypenamePageQuery = graphql(`
 
 function TypeExplorerPageContent({ typename }: { typename: string }) {
   const router = useRouteSelector();
-  const { period, dataRetentionInDays, setDataRetentionInDays } = useSchemaExplorerContext();
+  const { resolvedPeriod, dataRetentionInDays, setDataRetentionInDays } =
+    useSchemaExplorerContext();
   const [query] = useQuery({
     query: TargetExplorerTypenamePageQuery,
     variables: {
       organizationId: router.organizationId,
       projectId: router.projectId,
       targetId: router.targetId,
-      period,
+      period: resolvedPeriod,
       typename,
     },
   });
@@ -216,7 +217,7 @@ function TypeExplorerPageContent({ typename }: { typename: string }) {
             organization={{ cleanId: router.organizationId }}
             project={{ cleanId: router.projectId }}
             target={{ cleanId: router.targetId }}
-            period={period}
+            period={resolvedPeriod}
           />
         ) : null}
       </div>

packages/web/app/pages/[organizationId]/[projectId]/[targetId]/explorer/unused.tsx

@@ -1,19 +1,13 @@
 import { memo, ReactElement, useEffect, useMemo, useState } from 'react';
-import { AlertCircleIcon } from 'lucide-react';
+import { AlertCircleIcon, RefreshCw } from 'lucide-react';
 import { useQuery } from 'urql';
 import { authenticated } from '@/components/authenticated-container';
 import { Page, TargetLayout } from '@/components/layouts/target';
 import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
 import { Button } from '@/components/ui/button';
+import { DateRangePicker, presetLast7Days } from '@/components/ui/date-range-picker';
 import { Subtitle, Title } from '@/components/ui/page';
 import { QueryError } from '@/components/ui/query-error';
-import {
-  Select,
-  SelectContent,
-  SelectItem,
-  SelectTrigger,
-  SelectValue,
-} from '@/components/ui/select';
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip';
 import { Link, MetaTitle } from '@/components/v2';
 import { EmptyList, noSchemaVersion } from '@/components/v2/empty-list';
@@ -185,27 +179,27 @@ function UnusedSchemaExplorer(props: {
   projectCleanId: string;
   targetCleanId: string;
 }) {
-  const {
-    updateDateRangeByKey,
-    dateRangeKey,
-    displayDateRangeLabel,
-    availableDateRangeOptions,
-    dateRange,
-  } = useDateRangeController({
+  const dateRangeController = useDateRangeController({
     dataRetentionInDays: props.dataRetentionInDays,
-    minKey: '7d',
+    defaultPreset: presetLast7Days,
   });
 
-  const [query] = useQuery({
+  const [query, refresh] = useQuery({
     query: UnusedSchemaExplorer_UnusedSchemaQuery,
     variables: {
       organizationId: props.organizationCleanId,
       projectId: props.projectCleanId,
       targetId: props.targetCleanId,
-      period: dateRange,
+      period: dateRangeController.resolvedRange,
     },
   });
 
+  useEffect(() => {
+    if (!query.fetching) {
+      refresh({ requestPolicy: 'network-only' });
+    }
+  }, [dateRangeController.resolvedRange]);
+
   if (query.error) {
     return <QueryError error={query.error} />;
   }
@@ -223,22 +217,16 @@ function UnusedSchemaExplorer(props: {
           </Subtitle>
         </div>
         <div className="flex justify-end gap-x-2">
-          <Select
-            onValueChange={updateDateRangeByKey}
-            defaultValue={dateRangeKey}
-            disabled={query.fetching}
-          >
-            <SelectTrigger className="w-[180px]">
-              <SelectValue placeholder={displayDateRangeLabel(dateRangeKey)} />
-            </SelectTrigger>
-            <SelectContent>
-              {availableDateRangeOptions.map(key => (
-                <SelectItem key={key} value={key}>
-                  {displayDateRangeLabel(key)}
-                </SelectItem>
-              ))}
-            </SelectContent>
-          </Select>
+          <DateRangePicker
+            validUnits={['y', 'M', 'w', 'd']}
+            selectedRange={dateRangeController.selectedPreset.range}
+            startDate={dateRangeController.startDate}
+            align="end"
+            onUpdate={args => dateRangeController.setSelectedPreset(args.preset)}
+          />
+          <Button variant="outline" onClick={() => dateRangeController.refreshResolvedRange()}>
+            <RefreshCw className="size-4" />
+          </Button>
           <Button variant="outline" asChild>
             <Link
               href={{