diff --git a/.changeset/healthy-boxes-poke.md b/.changeset/healthy-boxes-poke.md new file mode 100644 index 0000000000..d4c1e43024 --- /dev/null +++ b/.changeset/healthy-boxes-poke.md @@ -0,0 +1,14 @@ +--- +'@astrojs/db': minor +--- + +Adds support for connecting Astro DB to any remote LibSQL server. This allows Astro DB to be used with self-hosting and air-gapped deployments. + +To connect Astro DB to a remote LibSQL server instead of Studio, set the following environment variables: + +- `ASTRO_DB_REMOTE_URL`: the connection URL to your LibSQL server +- `ASTRO_DB_APP_TOKEN`: the auth token to your LibSQL server + +Details of the LibSQL connection can be configured using the connection URL. For example, `memory:?syncUrl=libsql%3A%2F%2Fdb-server.example.com` would create an in-memory embedded replica for the LibSQL DB on `libsql://db-server.example.com`. + +For more details, please visit [the Astro DB documentation](https://docs.astro.build/en/guides/astro-db/#libsql) diff --git a/packages/db/src/core/cli/commands/push/index.ts b/packages/db/src/core/cli/commands/push/index.ts index 2d19daee58..0efa7ebcbf 100644 --- a/packages/db/src/core/cli/commands/push/index.ts +++ b/packages/db/src/core/cli/commands/push/index.ts @@ -1,11 +1,13 @@ import { getManagedAppTokenOrExit } from '@astrojs/studio'; import type { AstroConfig } from 'astro'; +import { sql } from 'drizzle-orm'; import prompts from 'prompts'; import type { Arguments } from 'yargs-parser'; +import { createRemoteDatabaseClient } from '../../../../runtime/index.js'; import { safeFetch } from '../../../../runtime/utils.js'; import { MIGRATION_VERSION } from '../../../consts.js'; import type { DBConfig, DBSnapshot } from '../../../types.js'; -import { type Result, getRemoteDatabaseUrl } from '../../../utils.js'; +import { type Result, getRemoteDatabaseInfo } from '../../../utils.js'; import { createCurrentSnapshot, createEmptySnapshot, @@ -87,7 +89,7 @@ async function pushSchema({ isDryRun: boolean; currentSnapshot: DBSnapshot; }) { - const requestBody = { + const requestBody: RequestBody = { snapshot: currentSnapshot, sql: statements, version: MIGRATION_VERSION, @@ -96,7 +98,47 @@ async function pushSchema({ console.info('[DRY RUN] Batch query:', JSON.stringify(requestBody, null, 2)); return new Response(null, { status: 200 }); } - const url = new URL('/db/push', getRemoteDatabaseUrl()); + + const dbInfo = getRemoteDatabaseInfo(); + + return dbInfo.type === 'studio' + ? pushToStudio(requestBody, appToken, dbInfo.url) + : pushToDb(requestBody, appToken, dbInfo.url); +} + +type RequestBody = { + snapshot: DBSnapshot; + sql: string[]; + version: string; +}; + +async function pushToDb(requestBody: RequestBody, appToken: string, remoteUrl: string) { + const client = createRemoteDatabaseClient({ + dbType: 'libsql', + appToken, + remoteUrl, + }); + + await client.run(sql`create table if not exists _astro_db_snapshot ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + version TEXT, + snapshot BLOB + );`); + + await client.transaction(async (tx) => { + for (const stmt of requestBody.sql) { + await tx.run(sql.raw(stmt)); + } + + await tx.run(sql`insert into _astro_db_snapshot (version, snapshot) values ( + ${requestBody.version}, + ${JSON.stringify(requestBody.snapshot)} + )`); + }); +} + +async function pushToStudio(requestBody: RequestBody, appToken: string, remoteUrl: string) { + const url = new URL('/db/push', remoteUrl); const response = await safeFetch( url, { diff --git a/packages/db/src/core/cli/commands/shell/index.ts b/packages/db/src/core/cli/commands/shell/index.ts index e0a1a6086c..0b5c2c2f06 100644 --- a/packages/db/src/core/cli/commands/shell/index.ts +++ b/packages/db/src/core/cli/commands/shell/index.ts @@ -10,7 +10,7 @@ import { normalizeDatabaseUrl } from '../../../../runtime/index.js'; import { DB_PATH } from '../../../consts.js'; import { SHELL_QUERY_MISSING_ERROR } from '../../../errors.js'; import type { DBConfigInput } from '../../../types.js'; -import { getAstroEnv, getRemoteDatabaseUrl } from '../../../utils.js'; +import { getAstroEnv, getRemoteDatabaseInfo } from '../../../utils.js'; export async function cmd({ flags, @@ -25,9 +25,14 @@ export async function cmd({ console.error(SHELL_QUERY_MISSING_ERROR); process.exit(1); } + const dbInfo = getRemoteDatabaseInfo(); if (flags.remote) { const appToken = await getManagedAppTokenOrExit(flags.token); - const db = createRemoteDatabaseClient(appToken.token, getRemoteDatabaseUrl()); + const db = createRemoteDatabaseClient({ + dbType: dbInfo.type, + remoteUrl: dbInfo.url, + appToken: appToken.token, + }); const result = await db.run(sql.raw(query)); await appToken.destroy(); console.log(result); @@ -37,7 +42,7 @@ export async function cmd({ ASTRO_DATABASE_FILE, new URL(DB_PATH, astroConfig.root).href, ); - const db = createLocalDatabaseClient({ dbUrl }); + const db = createLocalDatabaseClient({ dbUrl, enableTransations: dbInfo.type === 'libsql' }); const result = await db.run(sql.raw(query)); console.log(result); } diff --git a/packages/db/src/core/cli/migration-queries.ts b/packages/db/src/core/cli/migration-queries.ts index da5d70be10..1a16001dff 100644 --- a/packages/db/src/core/cli/migration-queries.ts +++ b/packages/db/src/core/cli/migration-queries.ts @@ -1,9 +1,11 @@ import deepDiff from 'deep-diff'; +import { sql } from 'drizzle-orm'; import { SQLiteAsyncDialect } from 'drizzle-orm/sqlite-core'; import * as color from 'kleur/colors'; import { customAlphabet } from 'nanoid'; import stripAnsi from 'strip-ansi'; import { hasPrimaryKey } from '../../runtime/index.js'; +import { createRemoteDatabaseClient } from '../../runtime/index.js'; import { isSerializedSQL } from '../../runtime/types.js'; import { safeFetch } from '../../runtime/utils.js'; import { MIGRATION_VERSION } from '../consts.js'; @@ -33,7 +35,7 @@ import type { ResolvedIndexes, TextColumn, } from '../types.js'; -import { type Result, getRemoteDatabaseUrl } from '../utils.js'; +import { type Result, getRemoteDatabaseInfo } from '../utils.js'; const sqlite = new SQLiteAsyncDialect(); const genTempTableName = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10); @@ -422,12 +424,49 @@ function hasRuntimeDefault(column: DBColumn): column is DBColumnWithDefault { return !!(column.schema.default && isSerializedSQL(column.schema.default)); } -export async function getProductionCurrentSnapshot({ - appToken, -}: { +export function getProductionCurrentSnapshot(options: { appToken: string; }): Promise { - const url = new URL('/db/schema', getRemoteDatabaseUrl()); + const dbInfo = getRemoteDatabaseInfo(); + + return dbInfo.type === 'studio' + ? getStudioCurrentSnapshot(options.appToken, dbInfo.url) + : getDbCurrentSnapshot(options.appToken, dbInfo.url); +} + +async function getDbCurrentSnapshot( + appToken: string, + remoteUrl: string +): Promise { + const client = createRemoteDatabaseClient({ + dbType: 'libsql', + appToken, + remoteUrl, + }); + + try { + const res = await client.get<{ snapshot: string }>( + // Latest snapshot + sql`select snapshot from _astro_db_snapshot order by id desc limit 1;` + ); + + return JSON.parse(res.snapshot); + } catch (error: any) { + if (error.code === 'SQLITE_UNKNOWN') { + // If the schema was never pushed to the database yet the table won't exist. + // Treat a missing snapshot table as an empty table. + return; + } + + throw error; + } +} + +async function getStudioCurrentSnapshot( + appToken: string, + remoteUrl: string +): Promise { + const url = new URL('/db/schema', remoteUrl); const response = await safeFetch( url, diff --git a/packages/db/src/core/integration/vite-plugin-db.ts b/packages/db/src/core/integration/vite-plugin-db.ts index 05cfa3ef0e..c8e273151f 100644 --- a/packages/db/src/core/integration/vite-plugin-db.ts +++ b/packages/db/src/core/integration/vite-plugin-db.ts @@ -9,7 +9,7 @@ import { DB_PATH, RUNTIME_IMPORT, RUNTIME_VIRTUAL_IMPORT, VIRTUAL_MODULE_ID } fr import { getResolvedFileUrl } from '../load-file.js'; import { SEED_DEV_FILE_NAME, getCreateIndexQueries, getCreateTableQuery } from '../queries.js'; import type { DBTables } from '../types.js'; -import { type VitePlugin, getAstroEnv, getDbDirectoryUrl, getRemoteDatabaseUrl } from '../utils.js'; +import { type VitePlugin, getAstroEnv, getDbDirectoryUrl, getRemoteDatabaseInfo } from '../utils.js'; export const resolved = { module: '\0' + VIRTUAL_MODULE_ID, @@ -119,12 +119,13 @@ export function getLocalVirtualModContents({ tables: DBTables; root: URL; }) { + const dbInfo = getRemoteDatabaseInfo(); const dbUrl = new URL(DB_PATH, root); return ` import { asDrizzleTable, createLocalDatabaseClient, normalizeDatabaseUrl } from ${RUNTIME_IMPORT}; const dbUrl = normalizeDatabaseUrl(import.meta.env.ASTRO_DATABASE_FILE, ${JSON.stringify(dbUrl)}); -export const db = createLocalDatabaseClient({ dbUrl }); +export const db = createLocalDatabaseClient({ dbUrl, enableTransactions: ${dbInfo.url === 'libsql'} }); export * from ${RUNTIME_VIRTUAL_IMPORT}; @@ -142,14 +143,17 @@ export function getStudioVirtualModContents({ isBuild: boolean; output: AstroConfig['output']; }) { + const dbInfo = getRemoteDatabaseInfo(); + function appTokenArg() { if (isBuild) { + const envPrefix = dbInfo.type === 'studio' ? 'ASTRO_STUDIO' : 'ASTRO_DB'; if (output === 'server') { // In production build, always read the runtime environment variable. - return 'process.env.ASTRO_STUDIO_APP_TOKEN'; + return `process.env.${envPrefix}_APP_TOKEN`; } else { // Static mode or prerendering needs the local app token. - return `process.env.ASTRO_STUDIO_APP_TOKEN ?? ${JSON.stringify(appToken)}`; + return `process.env.${envPrefix}_APP_TOKEN ?? ${JSON.stringify(appToken)}`; } } else { return JSON.stringify(appToken); @@ -157,15 +161,22 @@ export function getStudioVirtualModContents({ } function dbUrlArg() { - const dbStr = JSON.stringify(getRemoteDatabaseUrl()); + const dbStr = JSON.stringify(dbInfo.url); + // Allow overriding, mostly for testing - return `import.meta.env.ASTRO_STUDIO_REMOTE_DB_URL ?? ${dbStr}`; + return dbInfo.type === 'studio' + ? `import.meta.env.ASTRO_STUDIO_REMOTE_DB_URL ?? ${dbStr}` + : `import.meta.env.ASTRO_DB_REMOTE_URL ?? ${dbStr}`; } return ` import {asDrizzleTable, createRemoteDatabaseClient} from ${RUNTIME_IMPORT}; -export const db = await createRemoteDatabaseClient(${appTokenArg()}, ${dbUrlArg()}); +export const db = await createRemoteDatabaseClient({ + dbType: ${JSON.stringify(dbInfo.type)}, + remoteUrl: ${dbUrlArg()}, + appToken: ${appTokenArg()}, +}); export * from ${RUNTIME_VIRTUAL_IMPORT}; @@ -187,9 +198,10 @@ function getStringifiedTableExports(tables: DBTables) { const sqlite = new SQLiteAsyncDialect(); async function recreateTables({ tables, root }: { tables: LateTables; root: URL }) { + const dbInfo = getRemoteDatabaseInfo(); const { ASTRO_DATABASE_FILE } = getAstroEnv(); const dbUrl = normalizeDatabaseUrl(ASTRO_DATABASE_FILE, new URL(DB_PATH, root).href); - const db = createLocalDatabaseClient({ dbUrl }); + const db = createLocalDatabaseClient({ dbUrl, enableTransations: dbInfo.type === 'libsql' }); const setupQueries: SQL[] = []; for (const [name, table] of Object.entries(tables.get() ?? {})) { const dropQuery = sql.raw(`DROP TABLE IF EXISTS ${sqlite.escapeName(name)}`); diff --git a/packages/db/src/core/load-file.ts b/packages/db/src/core/load-file.ts index b9fb0136c0..027deaa601 100644 --- a/packages/db/src/core/load-file.ts +++ b/packages/db/src/core/load-file.ts @@ -146,6 +146,7 @@ export async function bundleFile({ metafile: true, define: { 'import.meta.env.ASTRO_STUDIO_REMOTE_DB_URL': 'undefined', + 'import.meta.env.ASTRO_DB_REMOTE_DB_URL': 'undefined', 'import.meta.env.ASTRO_DATABASE_FILE': JSON.stringify(ASTRO_DATABASE_FILE ?? ''), }, plugins: [ diff --git a/packages/db/src/core/utils.ts b/packages/db/src/core/utils.ts index 6229cb070d..9797992b86 100644 --- a/packages/db/src/core/utils.ts +++ b/packages/db/src/core/utils.ts @@ -10,9 +10,29 @@ export function getAstroEnv(envMode = ''): Record<`ASTRO_${string}`, string> { return env; } -export function getRemoteDatabaseUrl(): string { - const env = getAstroStudioEnv(); - return env.ASTRO_STUDIO_REMOTE_DB_URL || 'https://db.services.astro.build'; +export type RemoteDatabaseInfo = { + type: 'libsql' | 'studio'; + url: string; +}; + +export function getRemoteDatabaseInfo(): RemoteDatabaseInfo { + const astroEnv = getAstroEnv(); + const studioEnv = getAstroStudioEnv(); + + if (studioEnv.ASTRO_STUDIO_REMOTE_DB_URL) return { + type: 'studio', + url: studioEnv.ASTRO_STUDIO_REMOTE_DB_URL, + }; + + if (astroEnv.ASTRO_DB_REMOTE_URL) return { + type: 'libsql', + url: astroEnv.ASTRO_DB_REMOTE_URL, + }; + + return { + type: 'studio', + url: 'https://db.services.astro.build', + }; } export function getDbDirectoryUrl(root: URL | string) { diff --git a/packages/db/src/runtime/db-client.ts b/packages/db/src/runtime/db-client.ts index 55c1dd0873..08a68e2e83 100644 --- a/packages/db/src/runtime/db-client.ts +++ b/packages/db/src/runtime/db-client.ts @@ -1,5 +1,5 @@ import type { InStatement } from '@libsql/client'; -import { createClient } from '@libsql/client'; +import { type Config as LibSQLConfig, createClient } from '@libsql/client'; import type { LibSQLDatabase } from 'drizzle-orm/libsql'; import { drizzle as drizzleLibsql } from 'drizzle-orm/libsql'; import { type SqliteRemoteDatabase, drizzle as drizzleProxy } from 'drizzle-orm/sqlite-proxy'; @@ -18,12 +18,19 @@ function applyTransactionNotSupported(db: SqliteRemoteDatabase) { }); } -export function createLocalDatabaseClient({ dbUrl }: { dbUrl: string }): LibSQLDatabase { - const url = isWebContainer ? 'file:content.db' : dbUrl; +type LocalDbClientOptions = { + dbUrl: string; + enableTransations: boolean; +}; + +export function createLocalDatabaseClient(options: LocalDbClientOptions): LibSQLDatabase { + const url = isWebContainer ? 'file:content.db' : options.dbUrl; const client = createClient({ url }); const db = drizzleLibsql(client); - applyTransactionNotSupported(db); + if (!options.enableTransations) { + applyTransactionNotSupported(db); + } return db; } @@ -35,7 +42,33 @@ const remoteResultSchema = z.object({ lastInsertRowid: z.unknown().optional(), }); -export function createRemoteDatabaseClient(appToken: string, remoteDbURL: string) { +type RemoteDbClientOptions = { + dbType: 'studio' | 'libsql', + appToken: string, + remoteUrl: string | URL, +} + +export function createRemoteDatabaseClient(options: RemoteDbClientOptions) { + const remoteUrl = new URL(options.remoteUrl); + + return options.dbType === 'studio' + ? createStudioDatabaseClient(options.appToken, remoteUrl) + : createRemoteLibSQLClient(options.appToken, remoteUrl); +} + +function createRemoteLibSQLClient(appToken: string, remoteDbURL: URL) { + const options: Partial = Object.fromEntries(remoteDbURL.searchParams.entries()); + remoteDbURL.search = ''; + + const client = createClient({ + ...options, + authToken: appToken, + url: remoteDbURL.protocol === 'memory:' ? ':memory:' : remoteDbURL.toString(), + }); + return drizzleLibsql(client); +} + +function createStudioDatabaseClient(appToken: string, remoteDbURL: URL) { if (appToken == null) { throw new Error(`Cannot create a remote client: missing app token.`); } diff --git a/packages/db/test/static-remote.test.js b/packages/db/test/static-remote.test.js index 39e4eda0cd..ddfe735e55 100644 --- a/packages/db/test/static-remote.test.js +++ b/packages/db/test/static-remote.test.js @@ -39,4 +39,31 @@ describe('astro:db', () => { assert.match($('#row').text(), /1/); }); }); + + describe('static build --remote with custom LibSQL', () => { + let remoteDbServer; + + before(async () => { + process.env.ASTRO_DB_REMOTE_URL = `memory:`; + await fixture.build(); + }); + + after(async () => { + await remoteDbServer?.stop(); + }); + + it('Can render page', async () => { + const html = await fixture.readFile('/index.html'); + const $ = cheerioLoad(html); + + assert.equal($('li').length, 1); + }); + + it('Returns correct shape from db.run()', async () => { + const html = await fixture.readFile('/run/index.html'); + const $ = cheerioLoad(html); + + assert.match($('#row').text(), /1/); + }); + }); });