Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for LibSQL remote #11385

Merged
merged 13 commits into from
Aug 28, 2024
14 changes: 14 additions & 0 deletions .changeset/healthy-boxes-poke.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
'@astrojs/db': minor
---

Adds support for connecting Astro DB to any remote LibSQL server. This allows Astro DB to be used with self-hosting and air-gapped deployments.

To connect Astro DB to a remote LibSQL server instead of Studio, set the following environment variables:

- `ASTRO_DB_REMOTE_URL`: the connection URL to your LibSQL server
- `ASTRO_DB_APP_TOKEN`: the auth token to your LibSQL server

Details of the LibSQL connection can be configured using the connection URL.
ematipico marked this conversation as resolved.
Show resolved Hide resolved

For more details, please visit [the Astro DB documentation](https://docs.astro.build/en/guides/astro-db/#libsql)
48 changes: 45 additions & 3 deletions packages/db/src/core/cli/commands/push/index.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { getManagedAppTokenOrExit } from '@astrojs/studio';
import type { AstroConfig } from 'astro';
import { sql } from 'drizzle-orm';
import prompts from 'prompts';
import type { Arguments } from 'yargs-parser';
import { createRemoteDatabaseClient } from '../../../../runtime/index.js';
import { safeFetch } from '../../../../runtime/utils.js';
import { MIGRATION_VERSION } from '../../../consts.js';
import { type DBConfig, type DBSnapshot } from '../../../types.js';
import { type Result, getRemoteDatabaseUrl } from '../../../utils.js';
import { type Result, getRemoteDatabaseInfo } from '../../../utils.js';
import {
createCurrentSnapshot,
createEmptySnapshot,
Expand Down Expand Up @@ -87,7 +89,7 @@ async function pushSchema({
isDryRun: boolean;
currentSnapshot: DBSnapshot;
}) {
const requestBody = {
const requestBody: RequestBody = {
snapshot: currentSnapshot,
sql: statements,
version: MIGRATION_VERSION,
Expand All @@ -96,7 +98,47 @@ async function pushSchema({
console.info('[DRY RUN] Batch query:', JSON.stringify(requestBody, null, 2));
return new Response(null, { status: 200 });
}
const url = new URL('/db/push', getRemoteDatabaseUrl());

const dbInfo = getRemoteDatabaseInfo();

return dbInfo.type === 'studio'
? pushToStudio(requestBody, appToken, dbInfo.url)
: pushToDb(requestBody, appToken, dbInfo.url);
}

type RequestBody = {
snapshot: DBSnapshot;
sql: string[];
version: string;
};

async function pushToDb(requestBody: RequestBody, appToken: string, remoteUrl: string) {
const client = createRemoteDatabaseClient({
dbType: 'libsql',
appToken,
remoteUrl,
});

await client.run(sql`create table if not exists _astro_db_snapshot (
id INTEGER PRIMARY KEY AUTOINCREMENT,
version TEXT,
snapshot BLOB
);`);

await client.transaction(async (tx) => {
for (const stmt of requestBody.sql) {
await tx.run(sql.raw(stmt));
}

await tx.run(sql`insert into _astro_db_snapshot (version, snapshot) values (
${requestBody.version},
${JSON.stringify(requestBody.snapshot)}
)`);
});
}

async function pushToStudio(requestBody: RequestBody, appToken: string, remoteUrl: string) {
const url = new URL('/db/push', remoteUrl);
const response = await safeFetch(
url,
{
Expand Down
11 changes: 8 additions & 3 deletions packages/db/src/core/cli/commands/shell/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { normalizeDatabaseUrl } from '../../../../runtime/index.js';
import { DB_PATH } from '../../../consts.js';
import { SHELL_QUERY_MISSING_ERROR } from '../../../errors.js';
import type { DBConfigInput } from '../../../types.js';
import { getAstroEnv, getRemoteDatabaseUrl } from '../../../utils.js';
import { getAstroEnv, getRemoteDatabaseInfo } from '../../../utils.js';

export async function cmd({
flags,
Expand All @@ -25,9 +25,14 @@ export async function cmd({
console.error(SHELL_QUERY_MISSING_ERROR);
process.exit(1);
}
const dbInfo = getRemoteDatabaseInfo();
if (flags.remote) {
const appToken = await getManagedAppTokenOrExit(flags.token);
const db = createRemoteDatabaseClient(appToken.token, getRemoteDatabaseUrl());
const db = createRemoteDatabaseClient({
dbType: dbInfo.type,
remoteUrl: dbInfo.url,
appToken: appToken.token,
});
const result = await db.run(sql.raw(query));
await appToken.destroy();
console.log(result);
Expand All @@ -37,7 +42,7 @@ export async function cmd({
ASTRO_DATABASE_FILE,
new URL(DB_PATH, astroConfig.root).href,
);
const db = createLocalDatabaseClient({ dbUrl });
const db = createLocalDatabaseClient({ dbUrl, enableTransations: dbInfo.type === 'libsql' });
const result = await db.run(sql.raw(query));
console.log(result);
}
Expand Down
49 changes: 44 additions & 5 deletions packages/db/src/core/cli/migration-queries.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import deepDiff from 'deep-diff';
import { sql } from 'drizzle-orm';
import { SQLiteAsyncDialect } from 'drizzle-orm/sqlite-core';
import * as color from 'kleur/colors';
import { customAlphabet } from 'nanoid';
import stripAnsi from 'strip-ansi';
import { hasPrimaryKey } from '../../runtime/index.js';
import { createRemoteDatabaseClient } from '../../runtime/index.js';
import { isSerializedSQL } from '../../runtime/types.js';
import { safeFetch } from '../../runtime/utils.js';
import { MIGRATION_VERSION } from '../consts.js';
Expand Down Expand Up @@ -33,7 +35,7 @@ import {
type ResolvedIndexes,
type TextColumn,
} from '../types.js';
import { type Result, getRemoteDatabaseUrl } from '../utils.js';
import { type Result, getRemoteDatabaseInfo } from '../utils.js';

const sqlite = new SQLiteAsyncDialect();
const genTempTableName = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10);
Expand Down Expand Up @@ -422,12 +424,49 @@ function hasRuntimeDefault(column: DBColumn): column is DBColumnWithDefault {
return !!(column.schema.default && isSerializedSQL(column.schema.default));
}

export async function getProductionCurrentSnapshot({
appToken,
}: {
export function getProductionCurrentSnapshot(options: {
appToken: string;
}): Promise<DBSnapshot | undefined> {
const url = new URL('/db/schema', getRemoteDatabaseUrl());
const dbInfo = getRemoteDatabaseInfo();

return dbInfo.type === 'studio'
? getStudioCurrentSnapshot(options.appToken, dbInfo.url)
: getDbCurrentSnapshot(options.appToken, dbInfo.url);
}

async function getDbCurrentSnapshot(
appToken: string,
remoteUrl: string
): Promise<DBSnapshot | undefined> {
const client = createRemoteDatabaseClient({
dbType: 'libsql',
appToken,
remoteUrl,
});

try {
const res = await client.get<{ snapshot: string }>(
// Latest snapshot
sql`select snapshot from _astro_db_snapshot order by id desc limit 1;`
);

return JSON.parse(res.snapshot);
} catch (error: any) {
if (error.code === 'SQLITE_UNKNOWN') {
// If the schema was never pushed to the database yet the table won't exist.
// Treat a missing snapshot table as an empty table.
return;
}

throw error;
}
}

async function getStudioCurrentSnapshot(
appToken: string,
remoteUrl: string
): Promise<DBSnapshot | undefined> {
const url = new URL('/db/schema', remoteUrl);

const response = await safeFetch(
url,
Expand Down
28 changes: 20 additions & 8 deletions packages/db/src/core/integration/vite-plugin-db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { DB_PATH, RUNTIME_IMPORT, RUNTIME_VIRTUAL_IMPORT, VIRTUAL_MODULE_ID } fr
import { getResolvedFileUrl } from '../load-file.js';
import { SEED_DEV_FILE_NAME, getCreateIndexQueries, getCreateTableQuery } from '../queries.js';
import type { DBTables } from '../types.js';
import { type VitePlugin, getAstroEnv, getDbDirectoryUrl, getRemoteDatabaseUrl } from '../utils.js';
import { type VitePlugin, getAstroEnv, getDbDirectoryUrl, getRemoteDatabaseInfo } from '../utils.js';

export const resolved = {
module: '\0' + VIRTUAL_MODULE_ID,
Expand Down Expand Up @@ -119,12 +119,13 @@ export function getLocalVirtualModContents({
tables: DBTables;
root: URL;
}) {
const dbInfo = getRemoteDatabaseInfo();
const dbUrl = new URL(DB_PATH, root);
return `
import { asDrizzleTable, createLocalDatabaseClient, normalizeDatabaseUrl } from ${RUNTIME_IMPORT};

const dbUrl = normalizeDatabaseUrl(import.meta.env.ASTRO_DATABASE_FILE, ${JSON.stringify(dbUrl)});
export const db = createLocalDatabaseClient({ dbUrl });
export const db = createLocalDatabaseClient({ dbUrl, enableTransactions: ${dbInfo.url === 'libsql'} });

export * from ${RUNTIME_VIRTUAL_IMPORT};

Expand All @@ -142,30 +143,40 @@ export function getStudioVirtualModContents({
isBuild: boolean;
output: AstroConfig['output'];
}) {
const dbInfo = getRemoteDatabaseInfo();

function appTokenArg() {
if (isBuild) {
const envPrefix = dbInfo.type === 'studio' ? 'ASTRO_STUDIO' : 'ASTRO_DB';
if (output === 'server') {
// In production build, always read the runtime environment variable.
return 'process.env.ASTRO_STUDIO_APP_TOKEN';
return `process.env.${envPrefix}_APP_TOKEN`;
} else {
// Static mode or prerendering needs the local app token.
return `process.env.ASTRO_STUDIO_APP_TOKEN ?? ${JSON.stringify(appToken)}`;
return `process.env.${envPrefix}_APP_TOKEN ?? ${JSON.stringify(appToken)}`;
}
} else {
return JSON.stringify(appToken);
}
}

function dbUrlArg() {
const dbStr = JSON.stringify(getRemoteDatabaseUrl());
const dbStr = JSON.stringify(dbInfo.url);

// Allow overriding, mostly for testing
return `import.meta.env.ASTRO_STUDIO_REMOTE_DB_URL ?? ${dbStr}`;
return dbInfo.type === 'studio'
? `import.meta.env.ASTRO_STUDIO_REMOTE_DB_URL ?? ${dbStr}`
: `import.meta.env.ASTRO_DB_REMOTE_URL ?? ${dbStr}`;
}

return `
import {asDrizzleTable, createRemoteDatabaseClient} from ${RUNTIME_IMPORT};

export const db = await createRemoteDatabaseClient(${appTokenArg()}, ${dbUrlArg()});
export const db = await createRemoteDatabaseClient({
dbType: ${JSON.stringify(dbInfo.type)},
remoteUrl: ${dbUrlArg()},
appToken: ${appTokenArg()},
});

export * from ${RUNTIME_VIRTUAL_IMPORT};

Expand All @@ -187,9 +198,10 @@ function getStringifiedTableExports(tables: DBTables) {
const sqlite = new SQLiteAsyncDialect();

async function recreateTables({ tables, root }: { tables: LateTables; root: URL }) {
const dbInfo = getRemoteDatabaseInfo();
const { ASTRO_DATABASE_FILE } = getAstroEnv();
const dbUrl = normalizeDatabaseUrl(ASTRO_DATABASE_FILE, new URL(DB_PATH, root).href);
const db = createLocalDatabaseClient({ dbUrl });
const db = createLocalDatabaseClient({ dbUrl, enableTransations: dbInfo.type === 'libsql' });
const setupQueries: SQL[] = [];
for (const [name, table] of Object.entries(tables.get() ?? {})) {
const dropQuery = sql.raw(`DROP TABLE IF EXISTS ${sqlite.escapeName(name)}`);
Expand Down
1 change: 1 addition & 0 deletions packages/db/src/core/load-file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ export async function bundleFile({
metafile: true,
define: {
'import.meta.env.ASTRO_STUDIO_REMOTE_DB_URL': 'undefined',
'import.meta.env.ASTRO_DB_REMOTE_DB_URL': 'undefined',
'import.meta.env.ASTRO_DATABASE_FILE': JSON.stringify(ASTRO_DATABASE_FILE ?? ''),
},
plugins: [
Expand Down
26 changes: 23 additions & 3 deletions packages/db/src/core/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,29 @@ export function getAstroEnv(envMode = ''): Record<`ASTRO_${string}`, string> {
return env;
}

export function getRemoteDatabaseUrl(): string {
const env = getAstroStudioEnv();
return env.ASTRO_STUDIO_REMOTE_DB_URL || 'https://db.services.astro.build';
export type RemoteDatabaseInfo = {
type: 'libsql' | 'studio';
url: string;
};

export function getRemoteDatabaseInfo(): RemoteDatabaseInfo {
const astroEnv = getAstroEnv();
const studioEnv = getAstroStudioEnv();

if (studioEnv.ASTRO_STUDIO_REMOTE_DB_URL) return {
type: 'studio',
url: studioEnv.ASTRO_STUDIO_REMOTE_DB_URL,
};

if (astroEnv.ASTRO_DB_REMOTE_URL) return {
type: 'libsql',
url: astroEnv.ASTRO_DB_REMOTE_URL,
};

return {
type: 'studio',
url: 'https://db.services.astro.build',
};
}

export function getDbDirectoryUrl(root: URL | string) {
Expand Down
43 changes: 38 additions & 5 deletions packages/db/src/runtime/db-client.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { InStatement } from '@libsql/client';
import { createClient } from '@libsql/client';
import { type Config as LibSQLConfig, createClient } from '@libsql/client';
import type { LibSQLDatabase } from 'drizzle-orm/libsql';
import { drizzle as drizzleLibsql } from 'drizzle-orm/libsql';
import { type SqliteRemoteDatabase, drizzle as drizzleProxy } from 'drizzle-orm/sqlite-proxy';
Expand All @@ -18,12 +18,19 @@ function applyTransactionNotSupported(db: SqliteRemoteDatabase) {
});
}

export function createLocalDatabaseClient({ dbUrl }: { dbUrl: string }): LibSQLDatabase {
const url = isWebContainer ? 'file:content.db' : dbUrl;
type LocalDbClientOptions = {
dbUrl: string;
enableTransations: boolean;
};

export function createLocalDatabaseClient(options: LocalDbClientOptions): LibSQLDatabase {
const url = isWebContainer ? 'file:content.db' : options.dbUrl;
const client = createClient({ url });
const db = drizzleLibsql(client);

applyTransactionNotSupported(db);
if (!options.enableTransations) {
applyTransactionNotSupported(db);
}
return db;
}

Expand All @@ -35,7 +42,33 @@ const remoteResultSchema = z.object({
lastInsertRowid: z.unknown().optional(),
});

export function createRemoteDatabaseClient(appToken: string, remoteDbURL: string) {
type RemoteDbClientOptions = {
dbType: 'studio' | 'libsql',
appToken: string,
remoteUrl: string | URL,
}

export function createRemoteDatabaseClient(options: RemoteDbClientOptions) {
const remoteUrl = new URL(options.remoteUrl);

return options.dbType === 'studio'
? createStudioDatabaseClient(options.appToken, remoteUrl)
: createRemoteLibSQLClient(options.appToken, remoteUrl);
}

function createRemoteLibSQLClient(appToken: string, remoteDbURL: URL) {
const options: Partial<LibSQLConfig> = Object.fromEntries(remoteDbURL.searchParams.entries());
remoteDbURL.search = '';

const client = createClient({
...options,
authToken: appToken,
url: remoteDbURL.protocol === 'memory:' ? ':memory:' : remoteDbURL.toString(),
Fryuni marked this conversation as resolved.
Show resolved Hide resolved
});
return drizzleLibsql(client);
}

function createStudioDatabaseClient(appToken: string, remoteDbURL: URL) {
if (appToken == null) {
throw new Error(`Cannot create a remote client: missing app token.`);
}
Expand Down
Loading
Loading