refactor(gatsby): new dirty tracking implementation for queries (#27504)

* Remove dead code

* Consistently use job hash for static-query identification on the frontend (and queryId in node)

* New reducer to track the state of queries

* tmp: output newly calculated dirty queries (still using old calculation for actual query running to compare)

* Add new `QUERY_START` action

* Remove redundant component-data-dependencies reducer (now handled in the queries reducer)

* Actually use the new query tracking (and remove the old one)

* Fix data-tracking test

* Shape of tracked component state should match component reducer

* remove page-component machine (as we track query state in `queries` reducer now)

* Remove DELETE_COMPONENTS_DEPENDENCIES action

* Cleanup

* Cleanup

* Re-enable previously skipped test

* Cleanup

* Do-not re-run queries with babel extraction errors

* WIP: tests for the queries reducer

* Track babel errors per component (not per page/static query)

* tests for the queries reducer

* rename test

* Cleanup / update snapshots

* Add missing snapshot

* fix integration tests?

* Revert "fix integration tests?"

This reverts commit 066a4e1

* Restore DELETE_COMPONENTS_DEPENDENCIES as a no-op for BC

* Take into account deletePage/createPage pattern in onCreatePage

* Update test snapshot

* Do not mark page query as dirty when component has babel errors

* Use flag constants vs. literal values in tests

* Rename FLAG_ERROR_BABEL to FLAG_ERROR_EXTRACTION

Author: vladar

packages/gatsby/src/bootstrap/page-hot-reloader.ts

@@ -1,7 +1,7 @@
 import { emitter, store } from "../redux"
 import apiRunnerNode from "../utils/api-runner-node"
 import { boundActionCreators } from "../redux/actions"
-const { deletePage, deleteComponentsDependencies } = boundActionCreators
+const { deletePage } = boundActionCreators
 import report from "gatsby-cli/lib/reporter"
 import {
   ICreateNodeAction,
@@ -39,7 +39,6 @@ const runCreatePages = async (): Promise<void> => {
       page.updatedAt < timestamp &&
       page.path !== `/404.html`
     ) {
-      deleteComponentsDependencies([page.path])
       deletePage(page)
     }
   })

packages/gatsby/src/query/__tests__/data-tracking.js

@@ -200,14 +200,18 @@ const setup = async ({ restart = isFirstRun, clearCache = false } = {}) => {
   })
 
   Object.entries(staticQueries).forEach(([id, query]) => {
-    store.dispatch({
-      type: `REPLACE_STATIC_QUERY`,
-      payload: {
-        id: `sq--${id}`,
-        hash: `sq--${id}`,
-        query,
-      },
-    })
+    // Mimic real code behavior by only calling this action when static query text changes
+    const lastQuery = mockPersistedState.staticQueryComponents?.get(`sq--${id}`)
+    if (lastQuery?.query !== query) {
+      store.dispatch({
+        type: `REPLACE_STATIC_QUERY`,
+        payload: {
+          id: `sq--${id}`,
+          hash: `sq--${id}`,
+          query,
+        },
+      })
+    }
   })
 
   const queryIds = queryUtil.calcInitialDirtyQueryIds(store.getState())
@@ -1238,9 +1242,7 @@ describe(`query caching between builds`, () => {
       expect(staticQueriesThatRan).toEqual([])
     }, 999999)
 
-    // TO-DO: this is known issue - we always rerun queries for pages with no dependencies
-    // this mean that we will retry to rerun them every time we restart gatsby
-    it.skip(`rerunning should not run any queries (with restart)`, async () => {
+    it(`rerunning should not run any queries (with restart)`, async () => {
       const {
         pathsOfPagesWithQueriesThatRan,
         staticQueriesThatRan,

packages/gatsby/src/query/index.js

@@ -1,152 +1,36 @@
-// @flow
-
 const _ = require(`lodash`)
-const Queue = require(`better-queue`)
-// const convertHrtime = require(`convert-hrtime`)
-const { store, emitter } = require(`../redux`)
-const { boundActionCreators } = require(`../redux/actions`)
-const report = require(`gatsby-cli/lib/reporter`)
+const { store } = require(`../redux`)
+const { hasFlag, FLAG_ERROR_EXTRACTION } = require(`../redux/reducers/queries`)
 const queryQueue = require(`./queue`)
-const { GraphQLRunner } = require(`./graphql-runner`)
-const pageDataUtil = require(`../utils/page-data`)
-
-const seenIdsWithoutDataDependencies = new Set()
-let queuedDirtyActions = []
-const extractedQueryIds = new Set()
-
-// Remove pages from seenIdsWithoutDataDependencies when they're deleted
-// so their query will be run again if they're created again.
-emitter.on(`DELETE_PAGE`, action => {
-  seenIdsWithoutDataDependencies.delete(action.payload.path)
-})
-
-emitter.on(`CREATE_NODE`, action => {
-  queuedDirtyActions.push(action)
-})
-
-emitter.on(`DELETE_NODE`, action => {
-  queuedDirtyActions.push({ payload: action.payload })
-})
 
-// ///////////////////////////////////////////////////////////////////
-// Calculate dirty static/page queries
-
-const popExtractedQueries = () => {
-  const queries = [...extractedQueryIds]
-  extractedQueryIds.clear()
-  return queries
-}
-
-const findIdsWithoutDataDependencies = state => {
-  const allTrackedIds = new Set()
-  const boundAddToTrackedIds = allTrackedIds.add.bind(allTrackedIds)
-  state.componentDataDependencies.nodes.forEach(dependenciesOnNode => {
-    dependenciesOnNode.forEach(boundAddToTrackedIds)
-  })
-  state.componentDataDependencies.connections.forEach(
-    dependenciesOnConnection => {
-      dependenciesOnConnection.forEach(boundAddToTrackedIds)
+/**
+ * Calculates the set of dirty query IDs (page.paths, or staticQuery.id's).
+ *
+ * Dirty state is tracked in `queries` reducer, here we simply filter
+ * them from all tracked queries.
+ */
+const calcDirtyQueryIds = state => {
+  const { trackedQueries, trackedComponents, deletedQueries } = state.queries
+
+  const queriesWithBabelErrors = new Set()
+  for (const component of trackedComponents.values()) {
+    if (hasFlag(component.errors, FLAG_ERROR_EXTRACTION)) {
+      for (const queryId of component.pages) {
+        queriesWithBabelErrors.add(queryId)
+      }
     }
-  )
-
-  // Get list of paths not already tracked and run the queries for these
-  // paths.
-  const notTrackedIds = new Set(
-    [
-      ...Array.from(state.pages.values(), p => p.path),
-      ...[...state.staticQueryComponents.values()].map(c => c.id),
-    ].filter(
-      x => !allTrackedIds.has(x) && !seenIdsWithoutDataDependencies.has(x)
-    )
-  )
-
-  // Add new IDs to our seen array so we don't keep trying to run queries for them.
-  // Pages without queries can't be tracked.
-  for (const notTrackedId of notTrackedIds) {
-    seenIdsWithoutDataDependencies.add(notTrackedId)
   }
-
-  return notTrackedIds
-}
-
-const popNodeQueries = state => {
-  const actions = _.uniq(queuedDirtyActions, a => a.payload.id)
-  const uniqDirties = actions.reduce((dirtyIds, action) => {
-    const node = action.payload
-
-    if (!node || !node.id || !node.internal.type) return dirtyIds
-
-    // Find components that depend on this node so are now dirty.
-    if (state.componentDataDependencies.nodes.has(node.id)) {
-      state.componentDataDependencies.nodes.get(node.id).forEach(n => {
-        if (n) {
-          dirtyIds.add(n)
-        }
-      })
+  // Note: trackedQueries contains both - page and static query ids
+  const dirtyQueryIds = []
+  for (const [queryId, query] of trackedQueries) {
+    if (deletedQueries.has(queryId)) {
+      continue
     }
-
-    // Find connections that depend on this node so are now invalid.
-    if (state.componentDataDependencies.connections.has(node.internal.type)) {
-      state.componentDataDependencies.connections
-        .get(node.internal.type)
-        .forEach(n => {
-          if (n) {
-            dirtyIds.add(n)
-          }
-        })
+    if (query.dirty > 0 && !queriesWithBabelErrors.has(queryId)) {
+      dirtyQueryIds.push(queryId)
     }
-
-    return dirtyIds
-  }, new Set())
-
-  boundActionCreators.deleteComponentsDependencies([...uniqDirties])
-
-  queuedDirtyActions = []
-  return uniqDirties
-}
-
-const popNodeAndDepQueries = state => {
-  const nodeQueries = popNodeQueries(state)
-
-  const noDepQueries = findIdsWithoutDataDependencies(state)
-
-  return _.uniq([...nodeQueries, ...noDepQueries])
-}
-
-/**
- * Calculates the set of dirty query IDs (page.paths, or
- * staticQuery.hash's). These are queries that:
- *
- * - depend on nodes or node collections (via
- *   `actions.createPageDependency`) that have changed.
- * - do NOT have node dependencies. Since all queries should return
- *   data, then this implies that node dependencies have not been
- *   tracked, and therefore these queries haven't been run before
- * - have been recently extracted (see `./query-watcher.js`)
- *
- * Note, this function pops queries off internal queues, so it's up
- * to the caller to reference the results
- */
-
-const calcDirtyQueryIds = state =>
-  _.union(popNodeAndDepQueries(state), popExtractedQueries())
-
-/**
- * Same as `calcDirtyQueryIds`, except that we only include extracted
- * queries that depend on nodes or haven't been run yet. We do this
- * because the page component reducer/machine always enqueues
- * extractedQueryIds but during bootstrap we may not want to run those
- * page queries if their data hasn't changed since the last time we
- * ran Gatsby.
- */
-const calcInitialDirtyQueryIds = state => {
-  const nodeAndNoDepQueries = popNodeAndDepQueries(state)
-
-  const extractedQueriesThatNeedRunning = _.intersection(
-    popExtractedQueries(),
-    nodeAndNoDepQueries
-  )
-  return _.union(extractedQueriesThatNeedRunning, nodeAndNoDepQueries)
+  }
+  return dirtyQueryIds
 }
 
 /**
@@ -176,37 +60,14 @@ const createStaticQueryJob = (state, queryId) => {
   const component = state.staticQueryComponents.get(queryId)
   const { hash, id, query, componentPath } = component
   return {
-    id: hash,
+    id: queryId,
     hash,
     query,
     componentPath,
     context: { path: id },
   }
 }
 
-/**
- * Creates activity object which:
- *  - creates actual progress activity if there are any queries that need to be run
- *  - creates activity-like object that just cancels pending activity if there are no queries to run
- */
-const createQueryRunningActivity = (queryJobsCount, parentSpan) => {
-  if (queryJobsCount) {
-    const activity = report.createProgress(`run queries`, queryJobsCount, 0, {
-      id: `query-running`,
-      parentSpan,
-    })
-    activity.start()
-    return activity
-  } else {
-    return {
-      done: () => {
-        report.completeActivity(`query-running`)
-      },
-      tick: () => {},
-    }
-  }
-}
-
 const processStaticQueries = async (
   queryIds,
   { state, activity, graphqlRunner, graphqlTracing }
@@ -258,120 +119,10 @@ const createPageQueryJob = (state, page) => {
   }
 }
 
-// ///////////////////////////////////////////////////////////////////
-// Listener for gatsby develop
-
-// Initialized via `startListening`
-let listenerQueue
-
-/**
- * Run any dirty queries. See `calcQueries` for what constitutes a
- * dirty query
- */
-const runQueuedQueries = () => {
-  if (listenerQueue) {
-    const state = store.getState()
-    const { staticQueryIds, pageQueryIds } = groupQueryIds(
-      calcDirtyQueryIds(state)
-    )
-    const pages = _.filter(pageQueryIds.map(id => state.pages.get(id)))
-    const queryJobs = [
-      ...staticQueryIds.map(id => createStaticQueryJob(state, id)),
-      ...pages.map(page => createPageQueryJob(state, page)),
-    ]
-    listenerQueue.push(queryJobs)
-  }
-}
-
-/**
- * Starts a background process that processes any dirty queries
- * whenever one of the following occurs:
- *
- * 1. A node has changed (but only after the api call has finished
- * running)
- * 2. A component query (e.g. by editing a React Component) has
- * changed
- *
- * For what constitutes a dirty query, see `calcQueries`
- */
-
-const startListeningToDevelopQueue = ({ graphqlTracing } = {}) => {
-  // We use a queue to process batches of queries so that they are
-  // processed consecutively
-  let graphqlRunner = null
-  const developQueue = queryQueue.createDevelopQueue(() => {
-    if (!graphqlRunner) {
-      graphqlRunner = new GraphQLRunner(store, { graphqlTracing })
-    }
-    return graphqlRunner
-  })
-  listenerQueue = new Queue((queryJobs, callback) => {
-    const activity = createQueryRunningActivity(queryJobs.length)
-
-    const onFinish = (...arg) => {
-      pageDataUtil.enqueueFlush()
-      activity.done()
-      return callback(...arg)
-    }
-
-    return queryQueue
-      .processBatch(developQueue, queryJobs, activity)
-      .then(() => onFinish(null))
-      .catch(onFinish)
-  })
-
-  emitter.on(`API_RUNNING_START`, () => {
-    report.pendingActivity({ id: `query-running` })
-  })
-
-  emitter.on(`API_RUNNING_QUEUE_EMPTY`, runQueuedQueries)
-  ;[
-    `DELETE_CACHE`,
-    `CREATE_NODE`,
-    `DELETE_NODE`,
-    `DELETE_NODES`,
-    `SET_SCHEMA_COMPOSER`,
-    `SET_SCHEMA`,
-    `ADD_FIELD_TO_NODE`,
-    `ADD_CHILD_NODE_TO_PARENT_NODE`,
-  ].forEach(eventType => {
-    emitter.on(eventType, event => {
-      graphqlRunner = null
-    })
-  })
-}
-
-const enqueueExtractedQueryId = pathname => {
-  extractedQueryIds.add(pathname)
-}
-
-const getPagesForComponent = componentPath => {
-  const state = store.getState()
-  return [...state.pages.values()].filter(
-    p => p.componentPath === componentPath
-  )
-}
-
-const enqueueExtractedPageComponent = componentPath => {
-  const pages = getPagesForComponent(componentPath)
-  // Remove page data dependencies before re-running queries because
-  // the changing of the query could have changed the data dependencies.
-  // Re-running the queries will add back data dependencies.
-  boundActionCreators.deleteComponentsDependencies(
-    pages.map(p => p.path || p.id)
-  )
-  pages.forEach(page => enqueueExtractedQueryId(page.path))
-  runQueuedQueries()
-}
-
 module.exports = {
-  calcInitialDirtyQueryIds,
+  calcInitialDirtyQueryIds: calcDirtyQueryIds,
   calcDirtyQueryIds,
   processPageQueries,
   processStaticQueries,
   groupQueryIds,
-  startListeningToDevelopQueue,
-  runQueuedQueries,
-  enqueueExtractedQueryId,
-  enqueueExtractedPageComponent,
 }