21855 – Paginate and filter Queries on the server, update platform …

…filtering from compatible to targeted platforms (#24446) ## Addresses #21855 and all of its subtasks **Frontend:** - Update list queries API call to include pagination and filter-related query params, including new `platform` param for filtering queries by platforms they've been set to target - Convert all filtering, sorting, and pagination functionality of the Manage queries page from client-side to server-side - Remove unneeded variable declarations / logic - Various typing and naming improvements **Server:** - Add new `platform` `ListQueryOption` - Update service and datastore level list queries logic to handle filtering queries by targeted platform - Update service and datastore level list queries logic to include `meta` and `count` fields in addition to filtered/paginated queries - [x] Changes file added for user-visible changes in `changes/`, ` - [x] Added/updated tests - [x] update DB, integration - [x] add integration (pagination) - [x] add integration (platform filter) - [x] add DB (pagination) - [x] add DB (platform filter) - [x] Manual QA for all new/changed functionality --------- Co-authored-by: Jacob Shandling <jacob@fleetdm.com>
fleetdm · Dec 11, 2024 · 2118616 · 2118616
1 parent bf7876a
commit 2118616
Show file tree

Hide file tree

Showing 40 changed files with 774 additions and 382 deletions.
diff --git a/changes/21855-paginate-queries b/changes/21855-paginate-queries
@@ -0,0 +1,5 @@
+- Fleshed out server response from `queries` endpoint to include `count` and `meta` pagination information.
+- Updated UI queries page to filter, sort, paginate, etc. via query params in call to server.
+- Updated platform filtering on queries page to refer to targeted platforms instead of compatible
+  platforms
+- Updated queries API to support above targeted platform filtering
diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go
@@ -1250,7 +1250,7 @@ func TestGetQueries(t *testing.T) {
 		}
 		return nil, &notFoundError{}
 	}
-	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
+	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
 		if opt.TeamID == nil { //nolint:gocritic // ignore ifElseChain
 			return []*fleet.Query{
 				{
@@ -1282,7 +1282,7 @@ func TestGetQueries(t *testing.T) {
 					Saved:              true, // ListQueries always returns the saved ones.
 					ObserverCanRun:     true,
 				},
-			}, nil
+			}, 3, nil, nil
 		} else if *opt.TeamID == 1 {
 			return []*fleet.Query{
 				{
@@ -1299,11 +1299,11 @@ func TestGetQueries(t *testing.T) {
 					TeamID:             ptr.Uint(1),
 					ObserverCanRun:     true,
 				},
-			}, nil
+			}, 1, nil, nil
 		} else if *opt.TeamID == 2 {
-			return []*fleet.Query{}, nil
+			return []*fleet.Query{}, 0, nil, nil
 		}
-		return nil, errors.New("invalid team ID")
+		return nil, 0, nil, errors.New("invalid team ID")
 	}
 
 	expectedGlobal := `+--------+-------------+-----------+-----------+--------------------------------+
@@ -1563,7 +1563,7 @@ func TestGetQueriesAsObserver(t *testing.T) {
 		}
 	}
 
-	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
+	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
 		return []*fleet.Query{
 			{
 				ID:             42,
@@ -1586,7 +1586,7 @@ func TestGetQueriesAsObserver(t *testing.T) {
 				Query:          "select 3;",
 				ObserverCanRun: false,
 			},
-		}, nil
+		}, 3, nil, nil
 	}
 
 	for _, tc := range []struct {
@@ -1794,7 +1794,7 @@ spec:
 		GlobalRole: nil,
 		Teams:      []fleet.UserTeam{{Role: fleet.RoleObserver}},
 	})
-	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
+	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
 		return []*fleet.Query{
 			{
 				ID:             42,
@@ -1810,12 +1810,12 @@ spec:
 				Query:          "select 2;",
 				ObserverCanRun: false,
 			},
-		}, nil
+		}, 2, nil, nil
 	}
 	assert.Equal(t, "", runAppForTest(t, []string{"get", "queries"}))
 
 	// No filtering is performed if all are observer_can_run.
-	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
+	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
 		return []*fleet.Query{
 			{
 				ID:             42,
@@ -1831,7 +1831,7 @@ spec:
 				Query:          "select 2;",
 				ObserverCanRun: true,
 			},
-		}, nil
+		}, 2, nil, nil
 	}
 	expected = `+--------+-------------+-----------+-----------+----------------------------+
 |  NAME  | DESCRIPTION |   QUERY   |   TEAM    |          SCHEDULE          |

diff --git a/cmd/fleetctl/gitops_test.go b/cmd/fleetctl/gitops_test.go
@@ -69,7 +69,9 @@ func TestGitOpsBasicGlobalFree(t *testing.T) {
 		return nil
 	}
 	ds.ListGlobalPoliciesFunc = func(ctx context.Context, opts fleet.ListOptions) ([]*fleet.Policy, error) { return nil, nil }
-	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, error) { return nil, nil }
+	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return nil, 0, nil, nil
+	}
 
 	// Mock appConfig
 	savedAppConfig := &fleet.AppConfig{}
@@ -224,7 +226,9 @@ func TestGitOpsBasicGlobalPremium(t *testing.T) {
 		return nil
 	}
 	ds.ListGlobalPoliciesFunc = func(ctx context.Context, opts fleet.ListOptions) ([]*fleet.Policy, error) { return nil, nil }
-	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, error) { return nil, nil }
+	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return nil, 0, nil, nil
+	}
 
 	// Mock appConfig
 	savedAppConfig := &fleet.AppConfig{}
@@ -364,7 +368,9 @@ func TestGitOpsBasicTeam(t *testing.T) {
 	) (teamPolicies []*fleet.Policy, inheritedPolicies []*fleet.Policy, err error) {
 		return nil, nil, nil
 	}
-	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, error) { return nil, nil }
+	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return nil, 0, nil, nil
+	}
 	team := &fleet.Team{
 		ID:        1,
 		CreatedAt: time.Now(),
@@ -600,8 +606,8 @@ func TestGitOpsFullGlobal(t *testing.T) {
 	query.ID = 1
 	query.Name = "Query to delete"
 	queryDeleted := false
-	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, error) {
-		return []*fleet.Query{&query}, nil
+	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return []*fleet.Query{&query}, 1, nil, nil
 	}
 	ds.DeleteQueriesFunc = func(ctx context.Context, ids []uint) (uint, error) {
 		queryDeleted = true
@@ -861,8 +867,8 @@ func TestGitOpsFullTeam(t *testing.T) {
 	query.TeamID = ptr.Uint(teamID)
 	query.Name = "Query to delete"
 	queryDeleted := false
-	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, error) {
-		return []*fleet.Query{&query}, nil
+	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return []*fleet.Query{&query}, 1, nil, nil
 	}
 	ds.DeleteQueriesFunc = func(ctx context.Context, ids []uint) (uint, error) {
 		queryDeleted = true
@@ -1154,7 +1160,9 @@ func TestGitOpsBasicGlobalAndTeam(t *testing.T) {
 		}
 		return nil, nil
 	}
-	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, error) { return nil, nil }
+	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return nil, 0, nil, nil
+	}
 	ds.NewActivityFunc = func(
 		ctx context.Context, user *fleet.User, activity fleet.ActivityDetails, details []byte, createdAt time.Time,
 	) error {
@@ -1485,7 +1493,9 @@ func TestGitOpsBasicGlobalAndNoTeam(t *testing.T) {
 	ds.ListTeamsFunc = func(ctx context.Context, filter fleet.TeamFilter, opt fleet.ListOptions) ([]*fleet.Team, error) {
 		return nil, nil
 	}
-	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, error) { return nil, nil }
+	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return nil, 0, nil, nil
+	}
 	ds.NewActivityFunc = func(
 		ctx context.Context, user *fleet.User, activity fleet.ActivityDetails, details []byte, createdAt time.Time,
 	) error {
@@ -2441,7 +2451,9 @@ func setupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig,
 		}
 		return nil, nil
 	}
-	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, error) { return nil, nil }
+	ds.ListQueriesFunc = func(ctx context.Context, opts fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return nil, 0, nil, nil
+	}
 	ds.NewActivityFunc = func(
 		ctx context.Context, user *fleet.User, activity fleet.ActivityDetails, details []byte, createdAt time.Time,
 	) error {

diff --git a/cmd/fleetctl/query_test.go b/cmd/fleetctl/query_test.go
@@ -60,11 +60,11 @@ func TestSavedLiveQuery(t *testing.T) {
 	ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
 		return &fleet.AppConfig{}, nil
 	}
-	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
+	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
 		if opt.MatchQuery == queryName {
-			return []*fleet.Query{&query}, nil
+			return []*fleet.Query{&query}, 1, nil, nil
 		}
-		return []*fleet.Query{}, nil
+		return []*fleet.Query{}, 0, nil, nil
 	}
 	ds.NewDistributedQueryCampaignFunc = func(ctx context.Context, camp *fleet.DistributedQueryCampaign) (*fleet.DistributedQueryCampaign, error) {
 		camp.ID = 321

diff --git a/cmd/fleetctl/session_test.go b/cmd/fleetctl/session_test.go
@@ -14,8 +14,8 @@ import (
 
 func TestEarlySessionCheck(t *testing.T) {
 	_, ds := runServerWithMockedDS(t)
-	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
-		return nil, nil
+	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return nil, 0, nil, nil
 	}
 	ds.SessionByKeyFunc = func(ctx context.Context, key string) (*fleet.Session, error) {
 		return nil, errors.New("invalid session")

diff --git a/cmd/fleetctl/upgrade_packs_test.go b/cmd/fleetctl/upgrade_packs_test.go
@@ -247,8 +247,8 @@ func TestFleetctlUpgradePacks_EmptyPacks(t *testing.T) {
 		return fleet.TargetMetrics{}, nil
 	}
 
-	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
-		return nil, nil
+	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
+		return nil, 0, nil, nil
 	}
 
 	tempDir := t.TempDir()
@@ -314,12 +314,12 @@ func TestFleetctlUpgradePacks_NonEmpty(t *testing.T) {
 		return fleet.TargetMetrics{}, nil
 	}
 
-	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, error) {
+	ds.ListQueriesFunc = func(ctx context.Context, opt fleet.ListQueryOptions) ([]*fleet.Query, int, *fleet.PaginationMetadata, error) {
 		return []*fleet.Query{
 			{Name: "q1", Query: "select 1"},
 			{Name: "q2", Query: "select 2"},
 			{Name: "q3", Query: "select 3"},
-		}, nil
+		}, 3, nil, nil
 	}
 
 	const expected = `

diff --git a/frontend/components/PlatformCompatibility/PlatformCompatibility.tsx b/frontend/components/PlatformCompatibility/PlatformCompatibility.tsx
@@ -88,8 +88,13 @@ const PlatformCompatibility = ({
           tipContent={
             <>
               Estimated compatibility based on the <br />
-              tables used in the query. Querying <br />
-              iPhones & iPads is not supported.
+              tables used in the query.
+              <br />
+              <br />
+              Only live queries are supported on ChromeOS.
+              <br />
+              <br />
+              Querying iPhones & iPads is not supported.
             </>
           }
         >

diff --git a/frontend/hooks/usePlatformCompatibility.tsx b/frontend/hooks/usePlatformCompatibility.tsx
@@ -1,7 +1,7 @@
 import React, { useCallback, useState } from "react";
 import { useDebouncedCallback } from "use-debounce";
 
-import { QueryablePlatform, SUPPORTED_PLATFORMS } from "interfaces/platform";
+import { QueryablePlatform, QUERYABLE_PLATFORMS } from "interfaces/platform";
 import { checkPlatformCompatibility } from "utilities/sql_tools";
 
 import PlatformCompatibility from "components/PlatformCompatibility";
@@ -37,7 +37,7 @@ const usePlatformCompatibility = (): IPlatformCompatibility => {
   );
 
   const getCompatiblePlatforms = useCallback(
-    () => SUPPORTED_PLATFORMS.filter((p) => compatiblePlatforms?.includes(p)),
+    () => QUERYABLE_PLATFORMS.filter((p) => compatiblePlatforms?.includes(p)),
     [compatiblePlatforms]
   );
 

diff --git a/frontend/hooks/usePlatformSelector.tsx b/frontend/hooks/usePlatformSelector.tsx
@@ -3,7 +3,7 @@ import { forEach } from "lodash";
 
 import {
   SelectedPlatformString,
-  SUPPORTED_PLATFORMS,
+  QUERYABLE_PLATFORMS,
   QueryablePlatform,
 } from "interfaces/platform";
 
@@ -48,7 +48,7 @@ const usePlatformSelector = (
   };
 
   const getSelectedPlatforms = useCallback(() => {
-    return SUPPORTED_PLATFORMS.filter((p) => checksByPlatform[p]);
+    return QUERYABLE_PLATFORMS.filter((p) => checksByPlatform[p]);
   }, [checksByPlatform]);
 
   const isAnyPlatformSelected = Object.values(checksByPlatform).includes(true);

diff --git a/frontend/interfaces/platform.ts b/frontend/interfaces/platform.ts
@@ -22,13 +22,18 @@ export type QueryableDisplayPlatform = Exclude<
 >;
 export type QueryablePlatform = Exclude<Platform, "ios" | "ipados">;
 
-export const SUPPORTED_PLATFORMS: QueryablePlatform[] = [
+export const QUERYABLE_PLATFORMS: QueryablePlatform[] = [
   "darwin",
   "windows",
   "linux",
   "chrome",
 ];
 
+export const isQueryablePlatform = (
+  platform: string | undefined
+): platform is QueryablePlatform =>
+  QUERYABLE_PLATFORMS.includes(platform as QueryablePlatform);
+
 // TODO - add "iOS" and "iPadOS" once we support them
 export const VULN_SUPPORTED_PLATFORMS: Platform[] = ["darwin", "windows"];
 

diff --git a/frontend/interfaces/query.ts b/frontend/interfaces/query.ts
@@ -14,10 +14,6 @@ export interface IStoredQueryResponse {
   query: ISchedulableQuery;
 }
 
-export interface IFleetQueriesResponse {
-  queries: ISchedulableQuery[];
-}
-
 export interface IQuery {
   created_at: string;
   updated_at: string;

diff --git a/frontend/interfaces/schedulable_query.ts b/frontend/interfaces/schedulable_query.ts
@@ -3,7 +3,11 @@ import PropTypes from "prop-types";
 
 import { IFormField } from "./form_field";
 import { IPack } from "./pack";
-import { SelectedPlatformString, QueryablePlatform } from "./platform";
+import {
+  SelectedPlatformString,
+  QueryablePlatform,
+  SelectedPlatform,
+} from "./platform";
 
 // Query itself
 export interface ISchedulableQuery {
@@ -15,7 +19,7 @@ export interface ISchedulableQuery {
   query: string;
   team_id: number | null;
   interval: number;
-  platform: SelectedPlatformString; // Might more accurately be called `platforms_to_query` – comma-sepparated string of platforms to query, default all platforms if ommitted
+  platform: SelectedPlatformString; // Might more accurately be called `platforms_to_query` or `targeted_platforms` – comma-sepparated string of platforms to query, default all platforms if ommitted
   min_osquery_version: string;
   automations_enabled: boolean;
   logging: QueryLoggingOption;
@@ -32,7 +36,7 @@ export interface ISchedulableQuery {
 
 export interface IEnhancedQuery extends ISchedulableQuery {
   performance: string;
-  platforms: QueryablePlatform[];
+  targetedPlatforms: QueryablePlatform[];
 }
 export interface ISchedulableQueryStats {
   user_time_p50?: number | null;
@@ -67,7 +71,14 @@ export interface IListQueriesResponse {
 
 export interface IQueryKeyQueriesLoadAll {
   scope: "queries";
-  teamId: number | undefined;
+  teamId?: number;
+  page?: number;
+  perPage?: number;
+  query?: string;
+  orderDirection?: "asc" | "desc";
+  orderKey?: string;
+  mergeInherited?: boolean;
+  targetedPlatform?: SelectedPlatform;
 }
 // Create a new query
 /** POST /api/v1/fleet/queries */

diff --git a/frontend/pages/DashboardPage/DashboardPage.tsx b/frontend/pages/DashboardPage/DashboardPage.tsx
@@ -460,7 +460,7 @@ const DashboardPage = ({ router, location }: IDashboardProps): JSX.Element => {
     setShowAddHostsModal(!showAddHostsModal);
   };
 
-  // NOTE: this is called once on the initial rendering. The initial render of
+  // This is called once on the initial rendering. The initial render of
   // the TableContainer child component will call this handler.
   const onSoftwareQueryChange = async ({
     pageIndex: newPageIndex,