Skip to content

Commit 0250f74

Browse files
authored
Merge pull request #76 from supabase-community/chore/cache-eviction-strategy
feat: cache eviction strategy
2 parents 6b25b59 + 4831dea commit 0250f74

File tree

7 files changed

+167
-11
lines changed

7 files changed

+167
-11
lines changed

apps/proxy/.env.example

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,14 @@ AWS_ENDPOINT_URL_S3=http://host.docker.internal:54321/storage/v1/s3
33
AWS_S3_BUCKET=s3fs
44
AWS_SECRET_ACCESS_KEY=850181e4652dd023b7a98c58ae0d2d34bd487ee0cc3254aed6eda37307425907
55
AWS_REGION=local
6+
# Cache disk usage threshold in percentage of the total disk space
7+
CACHE_DISK_USAGE_THRESHOLD=90
8+
CACHE_PATH=/var/lib/postgres-new/cache
9+
# Cache schedule interval in hours
10+
CACHE_SCHEDULE_INTERVAL=1
11+
CACHE_TIMESTAMP_FILE=/var/lib/postgres-new/delete_cache_last_run
12+
# Cache time to live in hours
13+
CACHE_TTL=24
614
S3FS_MOUNT=/mnt/s3
715
SUPABASE_SERVICE_ROLE_KEY="<service-role-key>"
816
SUPABASE_URL="<supabase-url>"

apps/proxy/fly.toml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,15 @@ primary_region = 'iad'
66
dockerfile = "Dockerfile"
77
ignorefile = ".dockerignore"
88

9+
[env]
10+
CACHE_DISK_USAGE_THRESHOLD = "90"
11+
CACHE_SCHEDULE_INTERVAL = "1"
12+
CACHE_TIMESTAMP_FILE = "/var/lib/postgres-new/delete_cache_last_run"
13+
CACHE_TTL = "24"
14+
CACHE_PATH = "/var/lib/postgres-new/cache"
15+
S3FS_MOUNT = "/mnt/s3"
16+
WILDCARD_DOMAIN = "db.postgres.new"
17+
918
[[services]]
1019
internal_port = 5432
1120
protocol = "tcp"

apps/proxy/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@
1414
"@supabase/supabase-js": "^2.45.1",
1515
"find-up": "^7.0.0",
1616
"pg-gateway": "0.3.0-alpha.6",
17-
"tar": "^7.4.3"
17+
"tar": "^7.4.3",
18+
"zod": "^3.23.8"
1819
},
1920
"devDependencies": {
2021
"@postgres-new/supabase": "*",

apps/proxy/src/delete-cache.ts

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import * as fs from 'node:fs/promises'
2+
import * as path from 'node:path'
3+
import { exec } from 'node:child_process'
4+
import { promisify } from 'node:util'
5+
import { env } from './env.js'
6+
const execAsync = promisify(exec)
7+
8+
async function deleteOldFolders() {
9+
const now = Date.now()
10+
const ttlInMillis = env.CACHE_TTL * 60 * 60 * 1000
11+
12+
try {
13+
const folders = await fs.readdir(env.CACHE_PATH)
14+
for (const folder of folders) {
15+
const folderPath = path.join(env.CACHE_PATH, folder)
16+
const stats = await fs.stat(folderPath)
17+
18+
if (stats.isDirectory() && now - stats.mtimeMs > ttlInMillis) {
19+
await fs.rm(folderPath, { recursive: true, force: true })
20+
console.log(`Deleted folder: ${folderPath}`)
21+
}
22+
}
23+
} catch (err) {
24+
console.error('Failed to delete old folders:', err)
25+
}
26+
}
27+
28+
async function scriptAlreadyRan() {
29+
try {
30+
const lastRun = parseInt(await fs.readFile(env.CACHE_TIMESTAMP_FILE, 'utf8'))
31+
const now = Math.floor(Date.now() / 1000)
32+
const diff = now - lastRun
33+
return diff < env.CACHE_SCHEDULE_INTERVAL * 60 * 60 * 1000
34+
} catch (err) {
35+
// File does not exist
36+
if (err instanceof Error && 'code' in err && err.code === 'ENOENT') {
37+
return false
38+
}
39+
throw err
40+
}
41+
}
42+
43+
async function updateTimestampFile() {
44+
const now = Math.floor(Date.now() / 1000).toString()
45+
await fs.writeFile(env.CACHE_TIMESTAMP_FILE, now)
46+
}
47+
48+
/**
49+
* Get the disk usage of the root directory
50+
*/
51+
async function getDiskUsage() {
52+
// awk 'NR==2 {print $5}' prints the 5th column of the df command which contains the percentage of the total disk space used
53+
// sed 's/%//' removes the % from the output
54+
const command = `df / | awk 'NR==2 {print $5}' | sed 's/%//'`
55+
const { stdout } = await execAsync(command)
56+
return parseInt(stdout.trim(), 10)
57+
}
58+
59+
async function getFoldersByModificationTime() {
60+
const folders = await fs.readdir(env.CACHE_PATH, { withFileTypes: true })
61+
const folderStats = await Promise.all(
62+
folders
63+
.filter((dirent) => dirent.isDirectory())
64+
.map(async (dirent) => {
65+
const fullPath = path.join(env.CACHE_PATH, dirent.name)
66+
const stats = await fs.stat(fullPath)
67+
return { path: fullPath, mtime: stats.mtime.getTime() }
68+
})
69+
)
70+
return folderStats.sort((a, b) => a.mtime - b.mtime).map((folder) => folder.path)
71+
}
72+
73+
export async function deleteCache() {
74+
if (await scriptAlreadyRan()) {
75+
console.log(`Script already ran in the last ${env.CACHE_SCHEDULE_INTERVAL} hours, skipping.`)
76+
return
77+
}
78+
79+
await updateTimestampFile()
80+
81+
// Always delete old folders based on TTL
82+
await deleteOldFolders()
83+
84+
let diskUsage = await getDiskUsage()
85+
86+
// If disk usage exceeds the threshold, delete additional old folders
87+
if (diskUsage >= env.CACHE_DISK_USAGE_THRESHOLD) {
88+
console.log(
89+
`Disk usage is at ${diskUsage}%, which is above the threshold of ${env.CACHE_DISK_USAGE_THRESHOLD}%.`
90+
)
91+
92+
const folders = await getFoldersByModificationTime()
93+
94+
// Loop through the folders and delete them one by one until disk usage is below the threshold
95+
for (const folder of folders) {
96+
console.log(`Deleting folder: ${folder}`)
97+
await fs.rm(folder, { recursive: true, force: true })
98+
99+
diskUsage = await getDiskUsage()
100+
if (diskUsage < env.CACHE_DISK_USAGE_THRESHOLD) {
101+
console.log(`Disk usage is now at ${diskUsage}%, which is below the threshold.`)
102+
break
103+
}
104+
}
105+
} else {
106+
console.log(
107+
`Disk usage is at ${diskUsage}%, which is below the threshold of ${env.CACHE_DISK_USAGE_THRESHOLD}%.`
108+
)
109+
}
110+
}

apps/proxy/src/env.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { z } from 'zod'
2+
3+
export const env = z
4+
.object({
5+
AWS_ACCESS_KEY_ID: z.string(),
6+
AWS_ENDPOINT_URL_S3: z.string(),
7+
AWS_S3_BUCKET: z.string(),
8+
AWS_SECRET_ACCESS_KEY: z.string(),
9+
AWS_REGION: z.string(),
10+
CACHE_DISK_USAGE_THRESHOLD: z.string().transform((val) => parseInt(val, 10)),
11+
CACHE_PATH: z.string(),
12+
CACHE_SCHEDULE_INTERVAL: z.string().transform((val) => parseInt(val, 10)),
13+
CACHE_TIMESTAMP_FILE: z.string(),
14+
CACHE_TTL: z.string().transform((val) => parseInt(val, 10)),
15+
DATA_MOUNT: z.string(),
16+
S3FS_MOUNT: z.string(),
17+
SUPABASE_SERVICE_ROLE_KEY: z.string(),
18+
SUPABASE_URL: z.string(),
19+
WILDCARD_DOMAIN: z.string(),
20+
})
21+
.parse(process.env)

apps/proxy/src/index.ts

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,15 @@ import { PostgresConnection, ScramSha256Data, TlsOptions } from 'pg-gateway'
1010
import { createClient } from '@supabase/supabase-js'
1111
import type { Database } from '@postgres-new/supabase'
1212
import { findUp } from 'find-up'
13+
import { env } from './env.js'
14+
import { deleteCache } from './delete-cache.js'
15+
import path from 'node:path'
16+
17+
const supabaseUrl = env.SUPABASE_URL
18+
const supabaseKey = env.SUPABASE_SERVICE_ROLE_KEY
19+
const s3fsMount = env.S3FS_MOUNT
20+
const wildcardDomain = env.WILDCARD_DOMAIN
1321

14-
const supabaseUrl = process.env.SUPABASE_URL ?? 'http://127.0.0.1:54321'
15-
const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY ?? ''
16-
const dataMount = process.env.DATA_MOUNT ?? './data'
17-
const s3fsMount = process.env.S3FS_MOUNT ?? './s3'
18-
const wildcardDomain = process.env.WILDCARD_DOMAIN ?? 'db.example.com'
1922
const packageLockJsonPath = await findUp('package-lock.json')
2023
if (!packageLockJsonPath) {
2124
throw new Error('package-lock.json not found')
@@ -31,10 +34,9 @@ const pgliteVersion = `(PGlite ${packageLockJson.packages['node_modules/@electri
3134

3235
const dumpDir = `${s3fsMount}/dbs`
3336
const tlsDir = `${s3fsMount}/tls`
34-
const dbDir = `${dataMount}/dbs`
3537

3638
await mkdir(dumpDir, { recursive: true })
37-
await mkdir(dbDir, { recursive: true })
39+
await mkdir(env.CACHE_PATH, { recursive: true })
3840
await mkdir(tlsDir, { recursive: true })
3941

4042
const tls: TlsOptions = {
@@ -77,6 +79,10 @@ const supabase = createClient<Database>(supabaseUrl, supabaseKey)
7779
const server = net.createServer((socket) => {
7880
let db: PGliteInterface
7981

82+
deleteCache().catch((err) => {
83+
console.error(`Error deleting cache: ${err}`)
84+
})
85+
8086
const connection = new PostgresConnection(socket, {
8187
serverVersion: async () => {
8288
const {
@@ -161,12 +167,12 @@ const server = net.createServer((socket) => {
161167

162168
console.log(`Serving database '${databaseId}'`)
163169

164-
const dbPath = `${dbDir}/${databaseId}`
170+
const dbPath = path.join(env.CACHE_PATH, databaseId)
165171

166172
if (!(await fileExists(dbPath))) {
167173
console.log(`Database '${databaseId}' is not cached, downloading...`)
168174

169-
const dumpPath = `${dumpDir}/${databaseId}.tar.gz`
175+
const dumpPath = path.join(dumpDir, `${databaseId}.tar.gz`)
170176

171177
if (!(await fileExists(dumpPath))) {
172178
connection.sendError({

package-lock.json

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)