Ongoing / Maximum Usage Queue

[janwirth]

I have found that the best model to reliably process queues is to store the work in a DB and then fetch it with every run of my queue. For this I am using a supervisor (CRON) that runs once every minute to enqueue necessary syncs for resources with pending work.

To maximize the amount of work done within the 60s of the supervisor resolution I have created the following

import { differenceInSeconds } from "date-fns";
import { type QuirrelOptions, type JobMeta } from "quirrel/dist/esm/src/client";
import { Queue } from "quirrel/next-pages";
import { logToSlackErrorsChannel } from "~/server/help";

/**
 *
 * logging and maximum exhaustion
 */
export const OngoingQueue = <T>(
  path: string,
  run: (args: T, jobMeta: JobMeta) => Promise<boolean | void>,
  options?: QuirrelOptions
) => {
  const runBetter = async (args: T, meta: JobMeta) => {
    try {
      const startTime = new Date();
      let lastRunStart;
      let lastRunEnd;
      let totalTime = 0;
      let lastRunDuration = 0;
      let runs = 0;
      while (totalTime + lastRunDuration * 2 < 50) {
        console.log("run", runs);
        lastRunStart = new Date();
        const res = await run(args, meta);
        console.log("result", res);
        if (typeof res == "undefined") {
          break;
        }
        if (res == true) {
          // completed
          break;
        }
        lastRunEnd = new Date();
        lastRunDuration = differenceInSeconds(lastRunStart, lastRunEnd);
        totalTime = differenceInSeconds(startTime, lastRunEnd);
        runs++;
      }
    } catch (e) {
      void logToSlackErrorsChannel((e as Error).stack || (e as Error).message);
      throw e;
    }
  };
  return Queue(path, runBetter, options);
};