From a2ca5198d760c7c838ad654eb236043ebb2c43ae Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Thu, 10 Oct 2024 17:24:31 -0400 Subject: [PATCH] feat: use load methods from sql package in duckdb server, add createSchema method --- packages/core/src/DataCubeIndexer.js | 4 ++-- packages/duckdb/package.json | 3 ++- packages/duckdb/src/load/create-table.js | 6 ------ packages/duckdb/src/load/csv.js | 9 +++------ packages/duckdb/src/load/json.js | 9 +++------ packages/duckdb/src/load/parameters.js | 11 ----------- packages/duckdb/src/load/parquet.js | 7 +++---- packages/spec/src/ast/DataNode.js | 6 +++--- packages/sql/src/index.js | 2 +- packages/sql/src/load/create.js | 10 +++++++++- packages/sql/src/load/load.js | 6 +++--- packages/sql/test/create.test.js | 24 ++++++++++++++++++++++++ packages/vgplot/src/api.js | 2 +- 13 files changed, 54 insertions(+), 45 deletions(-) delete mode 100644 packages/duckdb/src/load/create-table.js delete mode 100644 packages/duckdb/src/load/parameters.js create mode 100644 packages/sql/test/create.test.js diff --git a/packages/core/src/DataCubeIndexer.js b/packages/core/src/DataCubeIndexer.js index c83086ba..02d610e8 100644 --- a/packages/core/src/DataCubeIndexer.js +++ b/packages/core/src/DataCubeIndexer.js @@ -1,5 +1,5 @@ import { - Query, and, asColumn, create, isBetween, scaleTransform, sql + Query, and, asColumn, createTable, isBetween, scaleTransform, sql } from '@uwdata/mosaic-sql'; import { indexColumns } from './util/index-columns.js'; import { fnv_hash } from './util/hash.js'; @@ -185,7 +185,7 @@ export class DataCubeIndexer { info = dataCubeInfo(client.query(filter), active, indexCols, schema); info.result = mc.exec([ `CREATE SCHEMA IF NOT EXISTS ${schema}`, - create(info.table, info.create, { temp: false }) + createTable(info.table, info.create, { temp: false }) ]); info.result.catch(e => mc.logger().error(e)); } diff --git a/packages/duckdb/package.json b/packages/duckdb/package.json index 52a03cd6..604a77aa 100644 --- a/packages/duckdb/package.json +++ b/packages/duckdb/package.json @@ -31,6 +31,7 @@ }, "dependencies": { "duckdb": "^1.0.0", - "ws": "^8.18.0" + "ws": "^8.18.0", + "@uwdata/mosaic-sql": "^0.11.0" } } diff --git a/packages/duckdb/src/load/create-table.js b/packages/duckdb/src/load/create-table.js deleted file mode 100644 index bebf238e..00000000 --- a/packages/duckdb/src/load/create-table.js +++ /dev/null @@ -1,6 +0,0 @@ -export function createTable(db, name, as, options = {}) { - const { temp, replace } = options; - const create = `CREATE${replace ? ' OR REPLACE' : ''}`; - const type = `${temp ? 'TEMP ' : ''}TABLE${replace ? '' : ' IF NOT EXISTS'}`; - return db.exec(`${create} ${type} ${name} AS ${as}`); -} diff --git a/packages/duckdb/src/load/csv.js b/packages/duckdb/src/load/csv.js index 0208b5fc..5b5479b1 100644 --- a/packages/duckdb/src/load/csv.js +++ b/packages/duckdb/src/load/csv.js @@ -1,9 +1,6 @@ -import { createTable } from './create-table.js'; -import { parameters } from './parameters.js'; +import { loadCSV as loadCSVSQL } from "@uwdata/mosaic-sql"; export function loadCSV(db, tableName, fileName, options = {}) { - const { select = ['*'], temp, replace, ...csvOptions } = options; - const params = parameters({ auto_detect: true, sample_size: -1, ...csvOptions }); - const query = `SELECT ${select.join(', ')} FROM read_csv('${fileName}', ${params})`; - return createTable(db, tableName, query, { temp, replace }); + const query = loadCSVSQL(tableName, fileName, options) + return db.exec(query) } diff --git a/packages/duckdb/src/load/json.js b/packages/duckdb/src/load/json.js index 274dcf3f..cb252e39 100644 --- a/packages/duckdb/src/load/json.js +++ b/packages/duckdb/src/load/json.js @@ -1,9 +1,6 @@ -import { createTable } from './create-table.js'; -import { parameters } from './parameters.js'; +import { loadJSON as loadJSONSQL } from "@uwdata/mosaic-sql"; export function loadJSON(db, tableName, fileName, options = {}) { - const { select = ['*'], temp, replace, ...jsonOptions } = options; - const params = parameters({ auto_detect: true, ...jsonOptions }); - const query = `SELECT ${select.join(', ')} FROM read_json('${fileName}', ${params})`; - return createTable(db, tableName, query, { temp, replace }); + const query = loadJSONSQL(tableName, fileName, options) + return db.exec(query) } diff --git a/packages/duckdb/src/load/parameters.js b/packages/duckdb/src/load/parameters.js deleted file mode 100644 index bb407de3..00000000 --- a/packages/duckdb/src/load/parameters.js +++ /dev/null @@ -1,11 +0,0 @@ -export function parameters(options) { - return Object.entries(options) - .map(([key, value]) => { - const t = typeof value; - const v = t === 'boolean' ? String(value) - : t === 'string' ? `'${value}'` - : value; - return `${key}=${v}`; - }) - .join(', '); -} diff --git a/packages/duckdb/src/load/parquet.js b/packages/duckdb/src/load/parquet.js index bc4569d9..12466b6e 100644 --- a/packages/duckdb/src/load/parquet.js +++ b/packages/duckdb/src/load/parquet.js @@ -1,7 +1,6 @@ -import { createTable } from './create-table.js'; +import { loadParquet as loadParquetSQL } from "@uwdata/mosaic-sql"; export function loadParquet(db, tableName, fileName, options = {}) { - const { select = ['*'], ...tableOptions } = options; - const query = `SELECT ${select.join(', ')} FROM read_parquet('${fileName}')`; - return createTable(db, tableName, query, tableOptions); + const query = loadParquetSQL(tableName, fileName, options) + return db.exec(query) } diff --git a/packages/spec/src/ast/DataNode.js b/packages/spec/src/ast/DataNode.js index 889b812b..139fd05a 100644 --- a/packages/spec/src/ast/DataNode.js +++ b/packages/spec/src/ast/DataNode.js @@ -1,4 +1,4 @@ -import { create } from '@uwdata/mosaic-sql'; +import { createTable } from '@uwdata/mosaic-sql'; import { DATA } from '../constants.js'; import { isArray, isString } from '../util.js'; import { ASTNode } from './ASTNode.js'; @@ -162,7 +162,7 @@ export class TableDataNode extends QueryDataNode { instantiateQuery(ctx) { const { name, query, options } = this; if (query) { - return ctx.api.create(name, query, options.instantiate(ctx)); + return ctx.api.createTable(name, query, options.instantiate(ctx)); } } @@ -174,7 +174,7 @@ export class TableDataNode extends QueryDataNode { codegenQuery(ctx) { const { name, query, options } = this; if (query) { - return `\`${create(name, query, options.instantiate(ctx))}\``; + return `\`${createTable(name, query, options.instantiate(ctx))}\``; } } diff --git a/packages/sql/src/index.js b/packages/sql/src/index.js index ecf76118..57eb6a73 100644 --- a/packages/sql/src/index.js +++ b/packages/sql/src/index.js @@ -145,7 +145,7 @@ export { scaleTransform } from './scales.js'; -export { create } from './load/create.js'; +export { createTable, createSchema } from './load/create.js'; export { loadExtension } from './load/extension.js'; export { loadCSV, diff --git a/packages/sql/src/load/create.js b/packages/sql/src/load/create.js index e00bf696..dcddc14d 100644 --- a/packages/sql/src/load/create.js +++ b/packages/sql/src/load/create.js @@ -1,4 +1,4 @@ -export function create(name, query, { +export function createTable(name, query, { replace = false, temp = true, view = false @@ -10,3 +10,11 @@ export function create(name, query, { + (replace ? ' ' : ' IF NOT EXISTS ') + name + ' AS ' + query; } + +export function createSchema(name, { + strict = false +} = {}) { + return 'CREATE SCHEMA ' + + (strict ? '' : 'IF NOT EXISTS ') + + name; +} diff --git a/packages/sql/src/load/load.js b/packages/sql/src/load/load.js index 354c1695..e9632a30 100644 --- a/packages/sql/src/load/load.js +++ b/packages/sql/src/load/load.js @@ -1,4 +1,4 @@ -import { create } from './create.js'; +import { createTable } from './create.js'; import { sqlFrom } from './sql-from.js'; export function load(method, tableName, fileName, options = {}, defaults = {}) { @@ -7,7 +7,7 @@ export function load(method, tableName, fileName, options = {}, defaults = {}) { const read = `${method}('${fileName}'${params ? ', ' + params : ''})`; const filter = where ? ` WHERE ${where}` : ''; const query = `SELECT ${select.join(', ')} FROM ${read}${filter}`; - return create(tableName, query, { view, temp, replace }); + return createTable(tableName, query, { view, temp, replace }); } export function loadCSV(tableName, fileName, options) { @@ -51,7 +51,7 @@ export function loadObjects(tableName, data, options = {}) { const query = select.length === 1 && select[0] === '*' ? values : `SELECT ${select} FROM ${values}`; - return create(tableName, query, opt); + return createTable(tableName, query, opt); } function parameters(options) { diff --git a/packages/sql/test/create.test.js b/packages/sql/test/create.test.js new file mode 100644 index 00000000..fe897b1c --- /dev/null +++ b/packages/sql/test/create.test.js @@ -0,0 +1,24 @@ +import { expect, describe, it } from 'vitest'; +import { createSchema, createTable } from '../src/load/create.js'; + +describe('createTable', () => { + it('creates a table', () => { + expect(createTable('table', 'SELECT 1')).toBe( + `CREATE TABLE IF NOT EXISTS table AS SELECT 1` + ); + }); +}); + +describe('createSchema', () => { + it('creates a schema strict', () => { + expect(createSchema('s1', { strict: true })).toBe( + `CREATE SCHEMA s1` + ); + }); + + it('creates a schema if it does not exist', () => { + expect(createSchema('s1')).toBe( + `CREATE SCHEMA IF NOT EXISTS s1` + ); + }); +}); diff --git a/packages/vgplot/src/api.js b/packages/vgplot/src/api.js index ee610377..a0f4988b 100644 --- a/packages/vgplot/src/api.js +++ b/packages/vgplot/src/api.js @@ -61,7 +61,7 @@ export { isDistinct, isNotDistinct, isNull, isNotNull, centroid, centroidX, centroidY, geojson, - create, loadExtension, + createTable, createSchema, loadExtension, loadCSV, loadJSON, loadObjects, loadParquet, loadSpatial } from '@uwdata/mosaic-sql';