Add support for LibSQL remote

.changeset/healthy-boxes-poke.md

@@ -0,0 +1,14 @@
+---
+'@astrojs/db': minor
+---
+
+Adds support for connecting Astro DB to any remote LibSQL server. This allows Astro DB to be used with self-hosting and air-gapped deployments.
+
+To connect Astro DB to a remote LibSQL server instead of Studio, set the following environment variables:
+
+- `ASTRO_STUDIO_REMOTE_DB_URL`: the connection URL to your LibSQL server
+- `ASTRO_STUDIO_APP_TOKEN`: the auth token to your LibSQL server
+
+Details of the LibSQL connection can be configured using the connection URL.
+
+For more details, please visit [the Astro DB documentation](https://docs.astro.build/en/guides/astro-db/#libsql)

packages/db/src/core/cli/commands/push/index.ts

@@ -1,11 +1,13 @@
 import { getManagedAppTokenOrExit } from '@astrojs/studio';
 import type { AstroConfig } from 'astro';
+import { sql } from 'drizzle-orm';
 import prompts from 'prompts';
 import type { Arguments } from 'yargs-parser';
+import { createRemoteDatabaseClient } from '../../../../runtime/index.js';
 import { safeFetch } from '../../../../runtime/utils.js';
 import { MIGRATION_VERSION } from '../../../consts.js';
 import { type DBConfig, type DBSnapshot } from '../../../types.js';
-import { type Result, getRemoteDatabaseUrl } from '../../../utils.js';
+import { type Result, getRemoteDatabaseUrl, isRemoteStudio } from '../../../utils.js';
 import {
 	createCurrentSnapshot,
 	createEmptySnapshot,
@@ -87,7 +89,7 @@ async function pushSchema({
 	isDryRun: boolean;
 	currentSnapshot: DBSnapshot;
 }) {
-	const requestBody = {
+	const requestBody: RequestBody = {
 		snapshot: currentSnapshot,
 		sql: statements,
 		version: MIGRATION_VERSION,
@@ -96,7 +98,43 @@ async function pushSchema({
 		console.info('[DRY RUN] Batch query:', JSON.stringify(requestBody, null, 2));
 		return new Response(null, { status: 200 });
 	}
-	const url = new URL('/db/push', getRemoteDatabaseUrl());
+
+	const remoteUrl = getRemoteDatabaseUrl();
+
+	return isRemoteStudio(remoteUrl)
+		? pushToStudio(requestBody, appToken, remoteUrl)
+		: pushToDb(requestBody, appToken, remoteUrl);
+}
+
+type RequestBody = {
+	snapshot: DBSnapshot;
+	sql: string[];
+	version: string;
+};
+
+async function pushToDb(requestBody: RequestBody, appToken: string, remoteUrl: string) {
+	const client = createRemoteDatabaseClient(appToken, remoteUrl);
+
+	await client.run(sql`create table if not exists _astro_db_snapshot (
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		version TEXT,
+		snapshot BLOB
+	);`);
+
+	await client.transaction(async (tx) => {
+		for (const stmt of requestBody.sql) {
+			await tx.run(sql.raw(stmt));
+		}
+
+		await tx.run(sql`insert into _astro_db_snapshot (version, snapshot) values (
+			${requestBody.version},
+			${JSON.stringify(requestBody.snapshot)}
+		)`);
+	});
+}
+
+async function pushToStudio(requestBody: RequestBody, appToken: string, remoteUrl: string) {
+	const url = new URL('/db/push', remoteUrl);
 	const response = await safeFetch(
 		url,
 		{

packages/db/src/core/cli/migration-queries.ts

@@ -1,9 +1,11 @@
 import deepDiff from 'deep-diff';
+import { sql } from 'drizzle-orm';
 import { SQLiteAsyncDialect } from 'drizzle-orm/sqlite-core';
 import * as color from 'kleur/colors';
 import { customAlphabet } from 'nanoid';
 import stripAnsi from 'strip-ansi';
 import { hasPrimaryKey } from '../../runtime/index.js';
+import { createRemoteDatabaseClient } from '../../runtime/index.js';
 import { isSerializedSQL } from '../../runtime/types.js';
 import { safeFetch } from '../../runtime/utils.js';
 import { MIGRATION_VERSION } from '../consts.js';
@@ -33,7 +35,7 @@ import {
 	type ResolvedIndexes,
 	type TextColumn,
 } from '../types.js';
-import { type Result, getRemoteDatabaseUrl } from '../utils.js';
+import { type Result, getRemoteDatabaseUrl, isRemoteStudio } from '../utils.js';
 
 const sqlite = new SQLiteAsyncDialect();
 const genTempTableName = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10);
@@ -422,12 +424,46 @@ function hasRuntimeDefault(column: DBColumn): column is DBColumnWithDefault {
 	return !!(column.schema.default && isSerializedSQL(column.schema.default));
 }
 
-export async function getProductionCurrentSnapshot({
-	appToken,
-}: {
+export function getProductionCurrentSnapshot(options: {
 	appToken: string;
 }): Promise<DBSnapshot | undefined> {
-	const url = new URL('/db/schema', getRemoteDatabaseUrl());
+	const remoteUrl = getRemoteDatabaseUrl();
+
+	return isRemoteStudio(remoteUrl)
+		? getStudioCurrentSnapshot(options.appToken, remoteUrl)
+		: getDbCurrentSnapshot(options.appToken, remoteUrl);
+}
+
+async function getDbCurrentSnapshot(
+	appToken: string,
+	remoteUrl: string
+): Promise<DBSnapshot | undefined> {
+	const client = createRemoteDatabaseClient(appToken, remoteUrl);
+
+	try {
+		const res = await client.get<{ snapshot: string }>(
+			// Latest snapshot
+			sql`select snapshot from _astro_db_snapshot order by id desc limit 1;`
+		);
+
+		console.log('DB res:', res);
+
+		return JSON.parse(res.snapshot);
+	} catch (error: any) {
+		if (error.code === 'SQLITE_UNKNOWN') {
+			// Snapshots table was not created yet,
+			return;
+		}
+
+		throw error;
+	}
+}
+
+async function getStudioCurrentSnapshot(
+	appToken: string,
+	remoteUrl: string
+): Promise<DBSnapshot | undefined> {
+	const url = new URL('/db/schema', remoteUrl);
 
 	const response = await safeFetch(
 		url,

packages/db/src/core/utils.ts

@@ -15,6 +15,11 @@ export function getRemoteDatabaseUrl(): string {
 	return env.ASTRO_STUDIO_REMOTE_DB_URL || 'https://db.services.astro.build';
 }
 
+export function isRemoteStudio(url: string): boolean {
+	const protocol = new URL(url).protocol;
+	return protocol === 'http:' || protocol === 'https:';
+}
+
 export function getDbDirectoryUrl(root: URL | string) {
 	return new URL('db/', root);
 }

packages/db/src/runtime/db-client.ts

@@ -1,5 +1,5 @@
 import type { InStatement } from '@libsql/client';
-import { createClient } from '@libsql/client';
+import { type Config as LibSQLConfig, createClient } from '@libsql/client';
 import type { LibSQLDatabase } from 'drizzle-orm/libsql';
 import { drizzle as drizzleLibsql } from 'drizzle-orm/libsql';
 import { type SqliteRemoteDatabase, drizzle as drizzleProxy } from 'drizzle-orm/sqlite-proxy';
@@ -23,7 +23,9 @@ export function createLocalDatabaseClient({ dbUrl }: { dbUrl: string }): LibSQLD
 	const client = createClient({ url });
 	const db = drizzleLibsql(client);
 
-	applyTransactionNotSupported(db);
+	if (!process.env.ASTRO_DB_ENABLE_TRANSACTIONS) {
+		applyTransactionNotSupported(db);
+	}
 	return db;
 }
 
@@ -36,6 +38,44 @@ const remoteResultSchema = z.object({
 });
 
 export function createRemoteDatabaseClient(appToken: string, remoteDbURL: string) {
+	const remoteUrl = new URL(remoteDbURL);
+
+	switch (remoteUrl.protocol) {
+		case 'http:':
+		case 'https:':
+			return createStudioDatabaseClient(appToken, remoteUrl);
+		case 'libsql+http:':
+			remoteUrl.protocol = 'http:';
+			return createRemoteLibSQLClient(appToken, remoteUrl);
+		case 'libsql+https:':
+			remoteUrl.protocol = 'https:';
+			return createRemoteLibSQLClient(appToken, remoteUrl);
+		case 'ws:':
+		case 'wss:':
+		case 'file:':
+		case 'memory:':
+		case 'libsql:':
+			return createRemoteLibSQLClient(appToken, remoteUrl);
+		default:
+			throw new Error(`Unsupported remote DB: ${remoteDbURL}`);
+	}
+}
+
+function createRemoteLibSQLClient(appToken: string, remoteDbURL: URL) {
+	const options: Partial<LibSQLConfig> = Object.fromEntries(remoteDbURL.searchParams.entries());
+	remoteDbURL.search = '';
+
+	const client = createClient({
+		...options,
+		authToken: appToken,
+		url: remoteDbURL.protocol === 'memory:' ? ':memory:' : remoteDbURL.toString(),
+	});
+	const db = drizzleLibsql(client);
+
+	return db;
+}
+
+function createStudioDatabaseClient(appToken: string, remoteDbURL: URL) {
 	if (appToken == null) {
 		throw new Error(`Cannot create a remote client: missing app token.`);
 	}

packages/db/test/static-remote.test.js

@@ -39,4 +39,31 @@ describe('astro:db', () => {
 			assert.match($('#row').text(), /1/);
 		});
 	});
+
+	describe('static build --remote with custom LibSQL', () => {
+		let remoteDbServer;
+
+		before(async () => {
+			process.env.ASTRO_STUDIO_REMOTE_DB_URL = `memory:`;
+			await fixture.build();
+		});
+
+		after(async () => {
+			await remoteDbServer?.stop();
+		});
+
+		it('Can render page', async () => {
+			const html = await fixture.readFile('/index.html');
+			const $ = cheerioLoad(html);
+
+			assert.equal($('li').length, 1);
+		});
+
+		it('Returns correct shape from db.run()', async () => {
+			const html = await fixture.readFile('/run/index.html');
+			const $ = cheerioLoad(html);
+
+			assert.match($('#row').text(), /1/);
+		});
+	});
 });