Skip to content

Commit

Permalink
feat(adapters): cloudflare-d1 - mapRow (#22750)
Browse files Browse the repository at this point in the history 
Co-authored-by: Serhii Tatarintsev <tatarintsev@prisma.io>
Co-authored-by: Joël Galeran <Jolg42@users.noreply.github.com>
  • Loading branch information
@Druue @SevInf @Jolg42
3 people authored Feb 16, 2024
1 parent 55077f8 commit 2daf967
Show file tree
Hide file tree
Showing 14 changed files with 463 additions and 177 deletions.
156 changes: 85 additions & 71 deletions packages/adapter-d1/src/conversion.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,43 +28,70 @@ export function getColumnTypes(columnNames: string[], rows: Object[]): ColumnTyp
return columnTypes as ColumnType[]
}

// JavaScript D1
// null NULL
// Number REAL
// Number 1 INTEGER
// String TEXT
// Boolean 2 INTEGER
// ArrayBuffer BLOB
// undefined Not supported. Queries with undefined values will return a D1_TYPE_ERROR
//
// 1 D1 supports 64-bit signed INTEGER values internally, however BigInts
// are not currently supported in the API yet. JavaScript integers are safe up to Number.MAX_SAFE_INTEGER
//
// 2 Booleans will be cast to an INTEGER type where 1 is TRUE and 0 is FALSE.

/**
* Default mapping between JS and D1 types.
* | JavaScript | D1 |
* | :--------------: | :---------: |
* | null | NULL |
* | Number | REAL |
* | Number¹ | INTEGER |
* | null | TEXT |
* | Boolean² | INTEGER |
* | ArrayBuffer | BLOB |
*
* ¹ - D1 supports 64-bit signed INTEGER values internally, however BigInts are not currently supported in the API yet. JavaScript integers are safe up to Number.MAX_SAFE_INTEGER.
*
* ² - Booleans will be cast to an INTEGER type where 1 is TRUE and 0 is FALSE.
*/
function inferColumnType(value: NonNullable<Value>): ColumnType {
switch (typeof value) {
case 'string':
return ColumnTypeEnum.Text
// case 'bigint':
// return ColumnTypeEnum.Int64
// case 'boolean':
// return ColumnTypeEnum.Boolean
return inferStringType(value)
case 'number':
// Hack - TODO change this when we have type metadata
if (Number.isInteger(value) && Math.abs(value) < Number.MAX_SAFE_INTEGER) {
return ColumnTypeEnum.Int32
} else {
return ColumnTypeEnum.UnknownNumber
}
return inferNumberType(value)
case 'object':
return inferObjectType(value)
default:
throw new UnexpectedTypeError(value)
}
}

function inferObjectType(value: {}): ColumnType {
// See https://stackoverflow.com/a/3143231/1345244
const isoDateRegex = new RegExp(
/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,
)
function isISODate(str) {
return isoDateRegex.test(str)
}

function inferStringType(value: string): ColumnType {
if (['true', 'false'].includes(value)) {
return ColumnTypeEnum.Boolean
}

if (isISODate(value)) {
return ColumnTypeEnum.DateTime
}

return ColumnTypeEnum.Text
}

function inferNumberType(value: number): ColumnType {
if (!Number.isInteger(value)) {
return ColumnTypeEnum.Float
// Note: returning "Numeric" makes is better for our Decimal type
// But we can't tell what is a float or a decimal here
// return ColumnTypeEnum.Numeric
}
// Hack - TODO change this when we have type metadata
else if (Number.isInteger(value) && Math.abs(value) < Number.MAX_SAFE_INTEGER) {
return ColumnTypeEnum.Int32
} else {
return ColumnTypeEnum.UnknownNumber
}
}

function inferObjectType(value: Object): ColumnType {
if (value instanceof Array) {
return ColumnTypeEnum.Bytes
}
Expand All @@ -80,48 +107,35 @@ class UnexpectedTypeError extends Error {
}
}

// TODO
// export function mapRow(row: Row, columnTypes: ColumnType[]): unknown[] {
// // `Row` doesn't have map, so we copy the array once and modify it in-place
// // to avoid allocating and copying twice if we used `Array.from(row).map(...)`.
// const result: unknown[] = Array.from(row)

// for (let i = 0; i < result.length; i++) {
// const value = result[i]

// // Convert array buffers to arrays of bytes.
// // Base64 would've been more efficient but would collide with the existing
// // logic that treats string values of type Bytes as raw UTF-8 bytes that was
// // implemented for other adapters.
// if (value instanceof ArrayBuffer) {
// result[i] = Array.from(new Uint8Array(value))
// continue
// }

// // If an integer is required and the current number isn't one,
// // discard the fractional part.
// if (
// typeof value === 'number' &&
// (columnTypes[i] === ColumnTypeEnum.Int32 || columnTypes[i] === ColumnTypeEnum.Int64) &&
// !Number.isInteger(value)
// ) {
// result[i] = Math.trunc(value)
// continue
// }

// // Decode DateTime values saved as numeric timestamps which is the
// // format used by the native quaint sqlite connector.
// if (['number', 'bigint'].includes(typeof value) && columnTypes[i] === ColumnTypeEnum.DateTime) {
// result[i] = new Date(Number(value)).toISOString()
// continue
// }

// // Convert bigint to string as we can only use JSON-encodable types here.
// if (typeof value === 'bigint') {
// result[i] = value.toString()
// continue
// }
// }

// return result
// }
export function mapRow(obj: Object, columnTypes: ColumnType[]): unknown[] {
const result: unknown[] = Object.values(obj)

for (let i = 0; i < result.length; i++) {
const value = result[i]

if (value instanceof ArrayBuffer) {
result[i] = Array.from(new Uint8Array(value))
continue
}

if (
typeof value === 'number' &&
(columnTypes[i] === ColumnTypeEnum.Int32 || columnTypes[i] === ColumnTypeEnum.Int64) &&
!Number.isInteger(value)
) {
result[i] = Math.trunc(value)
continue
}

if (typeof value === 'bigint') {
result[i] = value.toString()
continue
}

if (columnTypes[i] === ColumnTypeEnum.Boolean) {
result[i] = JSON.parse(value as any)
}
}

return result
}
101 changes: 44 additions & 57 deletions packages/adapter-d1/src/d1.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { D1Database, D1Result } from '@cloudflare/workers-types'
import {
// Debug,
Debug,
DriverAdapter,
err,
ok,
Expand All @@ -13,18 +13,13 @@ import {
} from '@prisma/driver-adapter-utils'
import { blue, cyan, red, yellow } from 'kleur/colors'

import { getColumnTypes } from './conversion'
import { getColumnTypes, mapRow } from './conversion'

// TODO? Env var works differently in D1 so `debug` does not work.
// const debug = Debug('prisma:driver-adapter:d1')
const debug = Debug('prisma:driver-adapter:d1')

type PerformIOResult = D1Result
// type ExecIOResult = D1ExecResult

type StdClient = D1Database

// const LOCK_TAG = Symbol()

class D1Queryable<ClientT extends StdClient> implements Queryable {
readonly provider = 'sqlite'

Expand All @@ -34,15 +29,13 @@ class D1Queryable<ClientT extends StdClient> implements Queryable {
* Execute a query given as SQL, interpolating the given parameters.
*/
async queryRaw(query: Query): Promise<Result<ResultSet>> {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const tag = '[js::query_raw]'
// console.debug(`${tag} %O`, query)
debug(`${tag} %O`, query)

const ioResult = await this.performIO(query)

return ioResult.map((data) => {
const convertedData = this.convertData(data)
// console.debug({ convertedData })
return convertedData
})
}
Expand All @@ -58,73 +51,42 @@ class D1Queryable<ClientT extends StdClient> implements Queryable {

const results = ioResult.results as Object[]
const columnNames = Object.keys(results[0])
const columnTypes = getColumnTypes(columnNames, results)
const rows = this.mapD1ToRows(results)
const columnTypes = Object.values(getColumnTypes(columnNames, results))
const rows = ioResult.results.map((value) => mapRow(value as Object, columnTypes))

return {
columnNames,
// Note: without Object.values the array looks like
// columnTypes: [ id: 128 ],
// and errors with:
// ✘ [ERROR] A hanging Promise was canceled. This happens when the worker runtime is waiting for a Promise from JavaScript to resolve, but has detected that the Promise cannot possibly ever resolve because all code and events related to the Promise's I/O context have already finished.
columnTypes: Object.values(columnTypes),
// * Note: without Object.values the array looks like
// * columnTypes: [ id: 128 ],
// * and errors with:
// * ✘ [ERROR] A hanging Promise was canceled. This happens when the worker runtime is waiting for a Promise from JavaScript to resolve, but has detected that the Promise cannot possibly ever resolve because all code and events related to the Promise's I/O context have already finished.
columnTypes,
rows,
}
}

private mapD1ToRows(results: any) {
const rows: unknown[][] = []
for (const row of results) {
const entry = Object.keys(row).map((k) => row[k])
rows.push(entry)
}
return rows
}

/**
* Execute a query given as SQL, interpolating the given parameters and
* returning the number of affected rows.
* Note: Queryable expects a u64, but napi.rs only supports u32.
*/
async executeRaw(query: Query): Promise<Result<number>> {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const tag = '[js::execute_raw]'
// console.debug(`${tag} %O`, query)
debug(`${tag} %O`, query)

return (await this.performIO(query)).map(({ meta }) => meta.rows_written ?? 0)
}

private async performIO(query: Query): Promise<Result<PerformIOResult>> {
// console.debug({ query })

try {
// Hack for booleans, we must convert them to 0/1.
// ✘ [ERROR] Error in performIO: Error: D1_TYPE_ERROR: Type 'boolean' not supported for value 'true'
query.args = query.args.map((arg) => {
if (arg === true) {
return 1
} else if (arg === false) {
return 0
}
// Temporary unblock for "D1_TYPE_ERROR: Type 'bigint' not supported for value '20'"
// For 0-legacy-ports.query-raw tests
// https://github.com/prisma/team-orm/issues/878
else if (typeof arg === 'bigint') {
return Number(arg)
} else if (arg instanceof Uint8Array) {
return Array.from(arg)
}

return arg
})
query.args = query.args.map((arg) => this.cleanArg(arg))

const result = await this.client
.prepare(query.sql)
.bind(...query.args)
// TODO use .raw({ columnNames: true }) later
.all()

// console.debug({ result })

return ok(result)
} catch (e) {
const error = e as Error
Expand All @@ -149,6 +111,28 @@ class D1Queryable<ClientT extends StdClient> implements Queryable {
})
}
}

cleanArg(arg: unknown): unknown {
// * Hack for booleans, we must convert them to 0/1.
// * ✘ [ERROR] Error in performIO: Error: D1_TYPE_ERROR: Type 'boolean' not supported for value 'true'
if (arg === true) {
return 1
}

if (arg === false) {
return 0
}

if (arg instanceof Uint8Array) {
return Array.from(arg)
}

if (typeof arg === 'bigint') {
return String(arg)
}

return arg
}
}

class D1Transaction extends D1Queryable<StdClient> implements Transaction {
Expand All @@ -158,14 +142,14 @@ class D1Transaction extends D1Queryable<StdClient> implements Transaction {

// eslint-disable-next-line @typescript-eslint/require-await
async commit(): Promise<Result<void>> {
// console.debug(`[js::commit]`)
debug(`[js::commit]`)

return ok(undefined)
}

// eslint-disable-next-line @typescript-eslint/require-await
async rollback(): Promise<Result<void>> {
// console.debug(`[js::rollback]`)
debug(`[js::rollback]`)

return ok(undefined)
}
Expand All @@ -181,8 +165,12 @@ export class PrismaD1 extends D1Queryable<StdClient> implements DriverAdapter {

alreadyWarned = new Set()

constructor(client: StdClient) {
// TODO: decide what we want to do for "debug"
constructor(client: StdClient, debug?: string) {
super(client)
if (debug) {
globalThis.DEBUG = debug
}
}

/**
Expand All @@ -209,9 +197,8 @@ export class PrismaD1 extends D1Queryable<StdClient> implements DriverAdapter {
usePhantomQuery: true,
}

// eslint-disable-next-line @typescript-eslint/no-unused-vars
const tag = '[js::startTransaction]'
// console.debug(`${tag} options: %O`, options)
debug(`${tag} options: %O`, options)

this.warnOnce(
'D1 Transaction',
Expand Down
7 changes: 7 additions & 0 deletions packages/client/helpers/functional-test/run-tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import fs from 'fs'
import { setupQueryEngine } from '../../tests/_utils/setupQueryEngine'
import { AdapterProviders, isDriverAdapterProviderLabel, Providers } from '../../tests/functional/_utils/providers'
import { JestCli } from './JestCli'
import path from 'path'

const allProviders = new Set(Object.values(Providers))
const allAdapterProviders = new Set(Object.values(AdapterProviders))
Expand Down Expand Up @@ -157,6 +158,12 @@ async function main(): Promise<number | void> {
}

if (adapterProviders.some(isDriverAdapterProviderLabel)) {
// Locally, running D1 tests accumulates a lot of data in the .wrangler directory.
// Because we cannot reset the database contents programmatically at the moment,
// deleting it is the easy way
// It makes local tests consistently fast and clean
fs.rmSync(path.join(__dirname, '..', '..', '.wrangler'), { recursive: true, force: true })

jestCli = jestCli.withArgs(['--runInBand'])
jestCli = jestCli.withEnv({ PRISMA_DISABLE_QUAINT_EXECUTORS: 'true' })
jestCli = jestCli.withEnv({ TEST_REUSE_DATABASE: 'true' })
Expand Down
Loading

0 comments on commit 2daf967

Please sign in to comment.