feat: improve cache cleanup

Author: LouisHaftmann

docs/content/1.getting-started/1.index.md

@@ -102,11 +102,15 @@ variant: subtle
 ---
 ::
 
-#### `CACHE_CLEANUP_OLDER_THAN_DAYS`
+#### `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS`
 
-- Default: `90`
+- Default: `30`
 
-The number of days to keep stale cache data and metadata before deleting it. Set to `0` to disable cache cleanup.
+Cache entries which have not been accessed for `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` days will be automatically deleted. Set to `0` to disable.
+
+#### `CACHE_CLEANUP_TTL_DAYS`
+
+Cache entries which have been created `CACHE_CLEANUP_TTL_DAYS` days ago will be automatically deleted. Set to `0` to disable.
 
 #### `CACHE_CLEANUP_CRON`
 

lib/db/index.ts

@@ -18,6 +18,7 @@ export interface CacheKeysTable {
   version: string
   updated_at: string
   accessed_at: string
+  created_at: string
 }
 export interface UploadsTable {
   created_at: string
@@ -191,17 +192,26 @@ export async function touchKey(
 
 export async function findStaleKeys(
   db: DB,
-  { olderThanDays, date }: { olderThanDays?: number; date?: Date },
+  opts?: {
+    untouchedTTLDays?: number
+    ttlDays?: number
+    date?: Date
+  },
 ) {
-  if (olderThanDays === undefined) return db.selectFrom('cache_keys').selectAll().execute()
+  const now = opts?.date ?? new Date()
+  let query = db.selectFrom('cache_keys')
 
-  const now = date ?? new Date()
-  const threshold = new Date(now.getTime() - olderThanDays * 24 * 60 * 60 * 1000)
-  return db
-    .selectFrom('cache_keys')
-    .where('accessed_at', '<', threshold.toISOString())
-    .selectAll()
-    .execute()
+  if (opts?.ttlDays !== undefined) {
+    const threshold = new Date(now.getTime() - opts.ttlDays * 24 * 60 * 60 * 1000)
+    query = query.where('created_at', '<', threshold.toISOString())
+  }
+
+  if (opts?.untouchedTTLDays !== undefined) {
+    const untouchedThreshold = new Date(now.getTime() - opts.untouchedTTLDays * 24 * 60 * 60 * 1000)
+    query = query.where('accessed_at', '<', untouchedThreshold.toISOString())
+  }
+
+  return query.selectAll().execute()
 }
 
 export async function createKey(
@@ -217,6 +227,7 @@ export async function createKey(
       version,
       updated_at: now.toISOString(),
       accessed_at: now.toISOString(),
+      created_at: now.toISOString(),
     })
     .execute()
 }

lib/db/migrations.ts

@@ -1,23 +1,33 @@
-import type { Migration } from 'kysely'
+import type { Kysely, Migration } from 'kysely'
 
 import type { DatabaseDriverName } from '~/lib/db/drivers'
 import { sql } from 'kysely'
 
+async function createCacheKeysTable({
+  db,
+  dbType,
+}: {
+  db: Kysely<any>
+  dbType: DatabaseDriverName
+}) {
+  let query = db.schema
+    .createTable('cache_keys')
+    .addColumn('id', 'varchar(255)', (col) => col.notNull().primaryKey())
+    .addColumn('key', 'text', (col) => col.notNull())
+    .addColumn('version', 'text', (col) => col.notNull())
+    .addColumn('updated_at', 'text', (col) => col.notNull())
+    .addColumn('accessed_at', 'text', (col) => col.notNull())
+
+  if (dbType === 'mysql') query = query.modifyEnd(sql`engine=InnoDB CHARSET=latin1`)
+
+  await query.ifNotExists().execute()
+}
+
 export function migrations(dbType: DatabaseDriverName) {
   return {
     $0_cache_keys_table: {
       async up(db) {
-        let query = db.schema
-          .createTable('cache_keys')
-          .addColumn('id', 'varchar(255)', (col) => col.notNull().primaryKey())
-          .addColumn('key', 'text', (col) => col.notNull())
-          .addColumn('version', 'text', (col) => col.notNull())
-          .addColumn('updated_at', 'text', (col) => col.notNull())
-          .addColumn('accessed_at', 'text', (col) => col.notNull())
-
-        if (dbType === 'mysql') query = query.modifyEnd(sql`engine=InnoDB CHARSET=latin1`)
-
-        await query.ifNotExists().execute()
+        await createCacheKeysTable({ db, dbType })
       },
       async down(db) {
         await db.schema.dropTable('cache_keys').ifExists().execute()
@@ -76,5 +86,58 @@ export function migrations(dbType: DatabaseDriverName) {
         await db.schema.alterTable('upload_parts').dropColumn('e_tag').execute()
       },
     },
+    $4_cache_entry_created_at: {
+      async up(db) {
+        await db
+          .insertInto('cache_keys')
+          .values({
+            id: '',
+            key: '',
+            version: '',
+            updated_at: new Date().toISOString(),
+            accessed_at: new Date().toISOString(),
+          })
+          .execute()
+        await db.schema.alterTable('cache_keys').addColumn('created_at', 'text').execute()
+
+        await db
+          .updateTable('cache_keys')
+          .set({
+            created_at: new Date().toISOString(),
+          })
+          .execute()
+
+        if (dbType === 'mysql')
+          await db.schema
+            .alterTable('cache_keys')
+            .modifyColumn('created_at', 'text', (c) => c.notNull())
+            .execute()
+        else if (dbType === 'postgres')
+          await db.schema
+            .alterTable('cache_keys')
+            .alterColumn('created_at', (c) => c.setNotNull())
+            .execute()
+        else {
+          // rename old table
+          await db.schema.alterTable('cache_keys').renameTo('old_cache_keys').execute()
+          // recreate table
+          await createCacheKeysTable({ db, dbType })
+
+          // add not null column
+          await db.schema
+            .alterTable('cache_keys')
+            .addColumn('created_at', 'text', (c) => c.notNull())
+            .execute()
+
+          // migrate old data
+          await db
+            .insertInto('cache_keys')
+            .expression((e) => e.selectFrom('old_cache_keys').selectAll())
+            .execute()
+
+          await db.schema.dropTable('old_cache_keys').execute()
+        }
+      },
+    },
   } satisfies Record<string, Migration>
 }

lib/env.ts

@@ -6,7 +6,12 @@ const portSchema = z.coerce.number().int().min(1).max(65_535)
 
 const envSchema = z.object({
   ENABLE_DIRECT_DOWNLOADS: booleanSchema.default('false'),
+  /**
+   * @deprecated use `CACHE_CLEANUP_UNTOUCHED_TTL_DAYS` or `CACHE_CLEANUP_TTL_DAYS` instead
+   */
   CACHE_CLEANUP_OLDER_THAN_DAYS: z.coerce.number().int().min(0).default(90),
+  CACHE_CLEANUP_UNTOUCHED_TTL_DAYS: z.coerce.number().int().min(0).default(30),
+  CACHE_CLEANUP_TTL_DAYS: z.coerce.number().int().min(0).optional(),
   CACHE_CLEANUP_CRON: z.string().default('0 0 * * *'),
   UPLOAD_CLEANUP_CRON: z.string().default('*/10 * * * *'),
   API_BASE_URL: z.string().url(),

lib/storage/index.ts

@@ -184,12 +184,10 @@ export const useStorageAdapter = createSingletonPromise(async () => {
         logger.debug('Download:', objectName)
         return driver.createReadStream(objectName)
       },
-      async pruneCaches(olderThanDays?: number) {
-        logger.debug('Prune:', {
-          olderThanDays,
-        })
+      async pruneCaches(opts?: { untouchedTTLDays?: number; ttlDays?: number }) {
+        logger.debug('Prune:', opts)
 
-        const keys = await findStaleKeys(db, { olderThanDays })
+        const keys = await findStaleKeys(db, opts)
         if (keys.length === 0) {
           logger.debug('Prune: No caches to prune')
           return
@@ -198,9 +196,7 @@ export const useStorageAdapter = createSingletonPromise(async () => {
         await driver.delete(keys.map((key) => getObjectNameFromKey(key.key, key.version)))
         await pruneKeys(db, keys)
 
-        logger.debug('Prune: Caches pruned', {
-          olderThanDays,
-        })
+        logger.debug('Prune: Caches pruned', opts)
       },
       async pruneUploads(olderThanDate: Date) {
         logger.debug('Prune uploads')

plugins/cleanup.ts

@@ -11,15 +11,22 @@ export default defineNitroPlugin(() => {
   if (!cluster.isPrimary) return
 
   // cache cleanup
-  if (ENV.CACHE_CLEANUP_OLDER_THAN_DAYS > 0) {
+  if (
+    ENV.CACHE_CLEANUP_OLDER_THAN_DAYS ||
+    ENV.CACHE_CLEANUP_TTL_DAYS ||
+    ENV.CACHE_CLEANUP_UNTOUCHED_TTL_DAYS
+  ) {
     const job = new Cron(ENV.CACHE_CLEANUP_CRON)
     const nextRun = job.nextRun()
     logger.info(
-      `Cleaning up cache entries older than ${colorize('blue', `${ENV.CACHE_CLEANUP_OLDER_THAN_DAYS}d`)} with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`,
+      `Cleaning up cache entries with schedule ${colorize('blue', job.getPattern() ?? '')}${nextRun ? ` (next run: ${nextRun.toLocaleString()})` : ''}`,
     )
     job.schedule(async () => {
       const adapter = await useStorageAdapter()
-      await adapter.pruneCaches(ENV.CACHE_CLEANUP_OLDER_THAN_DAYS)
+      await adapter.pruneCaches({
+        ttlDays: ENV.CACHE_CLEANUP_TTL_DAYS,
+        untouchedTTLDays: ENV.CACHE_CLEANUP_UNTOUCHED_TTL_DAYS ?? ENV.CACHE_CLEANUP_OLDER_THAN_DAYS,
+      })
     })
   }
 

tests/cleanup.test.ts

@@ -61,14 +61,14 @@ describe('getting stale keys', async () => {
   beforeEach(() => pruneKeys(db))
 
   const version = '0577ec58bee6d5415625'
-  test('returns stale keys if threshold is passed', async () => {
+  test('returns untouched stale keys if threshold is passed', async () => {
     const referenceDate = new Date('2024-04-01T00:00:00Z')
     await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') })
     await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') })
     await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') })
     await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') })
 
-    const match = await findStaleKeys(db, { olderThanDays: 30, date: referenceDate })
+    const match = await findStaleKeys(db, { untouchedTTLDays: 30, date: referenceDate })
     expect(match.length).toBe(2)
 
     const matchA = match.find((m) => m.key === 'cache-a')
@@ -80,6 +80,29 @@ describe('getting stale keys', async () => {
     expect(matchB?.accessed_at).toBe('2024-02-01T00:00:00.000Z')
   })
 
+  test('returns stale keys if threshold is passed', async () => {
+    const referenceDate = new Date('2024-04-01T00:00:00Z')
+    await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') })
+    await touchKey(db, { key: 'cache-a', version, date: referenceDate })
+    await updateOrCreateKey(db, { key: 'cache-b', version, date: new Date('2024-02-01T00:00:00Z') })
+    await touchKey(db, { key: 'cache-b', version, date: referenceDate })
+    await updateOrCreateKey(db, { key: 'cache-c', version, date: new Date('2024-03-15T00:00:00Z') })
+    await updateOrCreateKey(db, { key: 'cache-d', version, date: new Date('2024-03-20T00:00:00Z') })
+
+    const match = await findStaleKeys(db, { ttlDays: 30, date: referenceDate })
+    expect(match.length).toBe(2)
+
+    const matchA = match.find((m) => m.key === 'cache-a')
+    expect(matchA).toBeDefined()
+    expect(matchA?.created_at).toBe('2024-01-01T00:00:00.000Z')
+    expect(matchA?.accessed_at).toBe('2024-04-01T00:00:00.000Z')
+
+    const matchB = match.find((m) => m.key === 'cache-b')
+    expect(matchB).toBeDefined()
+    expect(matchB?.created_at).toBe('2024-02-01T00:00:00.000Z')
+    expect(matchB?.accessed_at).toBe('2024-04-01T00:00:00.000Z')
+  })
+
   test('returns all keys if threshold is not passed', async () => {
     const referenceDate = new Date('2024-04-01T00:00:00Z')
     await updateOrCreateKey(db, { key: 'cache-a', version, date: new Date('2024-01-01T00:00:00Z') })