From ab5f50d2b8db2be4be57cf54eb64d77473e9d2e8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 23 Aug 2024 18:00:52 +0100 Subject: [PATCH 01/31] Implementing a JSON aggregate method of selecting relationships. --- packages/backend-core/src/sql/sql.ts | 130 +++++++++++++++++++-------- 1 file changed, 95 insertions(+), 35 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 6a8518ea7b9..1c87abf5aef 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -126,12 +126,20 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { resource, meta } = this.query + const { endpoint, resource, meta, tableAliases } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } + // no relationships - select everything in SQLite + if (this.client === SqlClient.SQL_LITE) { + const alias = tableAliases?.[endpoint.entityId] + ? tableAliases?.[endpoint.entityId] + : endpoint.entityId + return [this.knex.raw(`${this.quote(alias)}.*`)] + } + const schema = meta.table.schema return resource.fields.map(field => { const parts = field.split(/\./g) @@ -745,16 +753,83 @@ class InternalBuilder { return withSchema } + addJsonRelationships( + query: Knex.QueryBuilder, + fromTable: string, + relationships: RelationshipsJson[] + ): Knex.QueryBuilder { + const { resource, tableAliases: aliases, endpoint } = this.query + const fields = resource?.fields || [] + const jsonField = (field: string) => { + const unAliased = field.split(".").slice(1).join(".") + return `'${unAliased}',${field}` + } + for (let relationship of relationships) { + const { + tableName: toTable, + through: throughTable, + to: toKey, + from: fromKey, + fromPrimary, + toPrimary, + } = relationship + // skip invalid relationships + if (!toTable || !fromTable || !fromPrimary || !toPrimary) { + continue + } + if (!throughTable) { + throw new Error("Only many-to-many implemented for JSON relationships") + } + const toAlias = aliases?.[toTable] || toTable, + throughAlias = aliases?.[throughTable] || throughTable, + fromAlias = aliases?.[fromTable] || fromTable + let toTableWithSchema = this.tableNameWithSchema(toTable, { + alias: toAlias, + schema: endpoint.schema, + }) + let throughTableWithSchema = this.tableNameWithSchema(throughTable, { + alias: throughAlias, + schema: endpoint.schema, + }) + const relationshipFields = fields.filter( + field => field.split(".")[0] === toAlias + ) + const fieldList: string = relationshipFields + .map(field => jsonField(field)) + .join(",") + let rawJsonArray: Knex.Raw + switch (this.client) { + case SqlClient.SQL_LITE: + rawJsonArray = this.knex.raw( + `json_group_array(json_object(${fieldList}))` + ) + break + default: + throw new Error(`JSON relationships not implement for ${this.client}`) + } + const subQuery = this.knex + .select(rawJsonArray) + .from(toTableWithSchema) + .join(throughTableWithSchema, function () { + this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) + }) + .where( + `${throughAlias}.${fromKey}`, + "=", + this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) + ) + query = query.select({ [relationship.column]: subQuery }) + } + return query + } + addRelationships( query: Knex.QueryBuilder, fromTable: string, - relationships: RelationshipsJson[] | undefined, - schema: string | undefined, + relationships: RelationshipsJson[], + schema?: string, aliases?: Record ): Knex.QueryBuilder { - if (!relationships) { - return query - } const tableSets: Record = {} // aggregate into table sets (all the same to tables) for (let relationship of relationships) { @@ -957,42 +1032,27 @@ class InternalBuilder { if (foundOffset != null) { query = query.offset(foundOffset) } - // add sorting to pre-query - // no point in sorting when counting - query = this.addSorting(query) } - // add filters to the query (where) - query = this.addFilters(query, filters) - const alias = tableAliases?.[tableName] || tableName - let preQuery: Knex.QueryBuilder = this.knex({ - // the typescript definition for the knex constructor doesn't support this - // syntax, but it is the only way to alias a pre-query result as part of - // a query - there is an alias dictionary type, but it assumes it can only - // be a table name, not a pre-query - [alias]: query as any, - }) // if counting, use distinct count, else select - preQuery = !counting - ? preQuery.select(this.generateSelectStatement()) - : this.addDistinctCount(preQuery) + query = !counting + ? query.select(this.generateSelectStatement()) + : this.addDistinctCount(query) // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { - preQuery = this.addSorting(preQuery) + query = this.addSorting(query) } // handle joins - query = this.addRelationships( - preQuery, - tableName, - relationships, - endpoint.schema, - tableAliases - ) - - // add a base limit over the whole query - // if counting we can't set this limit - if (limits?.base) { - query = query.limit(limits.base) + if (relationships && this.client === SqlClient.SQL_LITE) { + query = this.addJsonRelationships(query, tableName, relationships) + } else if (relationships) { + query = this.addRelationships( + query, + tableName, + relationships, + endpoint.schema, + tableAliases + ) } return this.addFilters(query, filters, { relationship: true }) From 80f3e5954bf5ecb84a8559789fcc50f5b9f5e474 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 23 Aug 2024 18:30:29 +0100 Subject: [PATCH 02/31] Getting processing of SQS relationships working. --- .../src/api/controllers/row/utils/basic.ts | 11 +++++++--- .../src/api/controllers/row/utils/utils.ts | 20 ++++++++++--------- .../src/sdk/app/rows/search/internal/sqs.ts | 13 +++++++++++- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index f28f6504229..8f3607bc73c 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,5 +1,5 @@ // need to handle table name + field or just field, depending on if relationships used -import { FieldType, Row, Table } from "@budibase/types" +import { FieldSchema, FieldType, Row, Table } from "@budibase/types" import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" import { generateRowIdField } from "../../../../integrations/utils" @@ -82,7 +82,7 @@ export function basicProcessing({ value = value.toString() } // all responses include "select col as table.col" so that overlaps are handled - if (value != null) { + else if (value != null) { thisRow[fieldName] = value } } @@ -93,12 +93,17 @@ export function basicProcessing({ } else { const columns = Object.keys(table.schema) for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { - thisRow[internalColumn] = extractFieldValue({ + const schema: FieldSchema | undefined = table.schema[internalColumn] + let value = extractFieldValue({ row, tableName: table._id!, fieldName: internalColumn, isLinked, }) + if (sqs && schema?.type === FieldType.LINK && typeof value === "string") { + value = JSON.parse(value) + } + thisRow[internalColumn] = value } } return thisRow diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 911cfe8d5bc..fdf05baf3c2 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -147,7 +147,7 @@ export async function sqlOutputProcessing( row._id = rowId } // this is a relationship of some sort - if (finalRows[rowId]) { + if (!opts?.sqs && finalRows[rowId]) { finalRows = await updateRelationshipColumns( table, tables, @@ -174,14 +174,16 @@ export async function sqlOutputProcessing( finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table }) // do this at end once its been added to the final rows - finalRows = await updateRelationshipColumns( - table, - tables, - row, - finalRows, - relationships, - opts - ) + if (!opts?.sqs) { + finalRows = await updateRelationshipColumns( + table, + tables, + row, + finalRows, + relationships, + opts + ) + } } // make sure all related rows are correct diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 6736ff6abfd..fb140e3c14c 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -37,9 +37,9 @@ import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" import { processRowCountResponse } from "../../utils" import { - updateFilterKeys, getRelationshipColumns, getTableIDList, + updateFilterKeys, } from "../filters" import { dataFilters, @@ -368,6 +368,17 @@ export async function search( }) ) + // make sure relationships have columns reversed correctly + for (let columnName of Object.keys(table.schema)) { + if (table.schema[columnName].type !== FieldType.LINK) { + continue + } + // process the relationships (JSON generated by SQS) + for (let row of processed) { + row[columnName] = reverseUserColumnMapping(row[columnName]) + } + } + // check for pagination final row let nextRow: boolean = false if (paginate && params.limit && rows.length > params.limit) { From 5d53e64360f9832cfee92521d6305b94aff42299 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 23 Aug 2024 18:45:13 +0100 Subject: [PATCH 03/31] Getting fields from all relationships loading correctly. --- packages/backend-core/src/sql/sql.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 1c87abf5aef..74678a79b56 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -761,8 +761,17 @@ class InternalBuilder { const { resource, tableAliases: aliases, endpoint } = this.query const fields = resource?.fields || [] const jsonField = (field: string) => { - const unAliased = field.split(".").slice(1).join(".") - return `'${unAliased}',${field}` + const parts = field.split(".") + let tableField: string, unaliased: string + if (parts.length > 1) { + const alias = parts.shift()! + unaliased = parts.join(".") + tableField = `${this.quote(alias)}.${this.quote(unaliased)}` + } else { + unaliased = parts.join(".") + tableField = this.quote(unaliased) + } + return `'${unaliased}',${tableField}` } for (let relationship of relationships) { const { From b11ee56a38ca8b056f74794c97b5a3366409d3c4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 23 Aug 2024 18:54:46 +0100 Subject: [PATCH 04/31] Adding limit in for wide tables to be related correctly. --- packages/backend-core/src/sql/sql.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 74678a79b56..5f8a8e1a9bc 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -39,6 +39,7 @@ import { dataFilters, helpers } from "@budibase/shared-core" import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any +const MAX_SQS_RELATIONSHIP_FIELDS = 63 function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS @@ -800,9 +801,15 @@ class InternalBuilder { alias: throughAlias, schema: endpoint.schema, }) - const relationshipFields = fields.filter( + let relationshipFields = fields.filter( field => field.split(".")[0] === toAlias ) + if (this.client === SqlClient.SQL_LITE) { + relationshipFields = relationshipFields.slice( + 0, + MAX_SQS_RELATIONSHIP_FIELDS + ) + } const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") From 0c604b7310a96aba482dab0b25296b5f428fabe3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 27 Aug 2024 18:34:05 +0100 Subject: [PATCH 05/31] Moving things around, making join logic more accessible. --- packages/backend-core/src/sql/sql.ts | 169 ++++++++++++++++++--------- 1 file changed, 112 insertions(+), 57 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 5f8a8e1a9bc..9fbb02a69ec 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -337,6 +337,40 @@ class InternalBuilder { return filters } + addRelationshipForFilter( + query: Knex.QueryBuilder, + filterKey: string, + whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder + ): Knex.QueryBuilder { + const mainKnex = this.knex + const { relationships, endpoint } = this.query + const tableName = endpoint.entityId + if (!relationships) { + return query + } + for (const relationship of relationships) { + // this is the relationship which is being filtered + if (filterKey.startsWith(relationship.column) && relationship.to) { + const subQuery = query.whereExists(function () { + this.select(mainKnex.raw(1)).from(relationship.tableName!) + }) + query = whereCb( + this.addJoin( + subQuery, + { + from: relationship.tableName, + to: tableName, + through: relationship.through, + }, + [relationship] + ) + ) + break + } + } + return query + } + // right now we only do filters on the specific table being queried addFilters( query: Knex.QueryBuilder, @@ -387,6 +421,7 @@ class InternalBuilder { fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) } if (opts?.relationship && isRelationshipField) { + // TODO: need to update fn to take the query const [filterTableName, property] = updatedKey.split(".") const alias = getTableAlias(filterTableName) fn(alias ? `${alias}.${property}` : property, value) @@ -839,12 +874,75 @@ class InternalBuilder { return query } + addJoin( + query: Knex.QueryBuilder, + tables: { from: string; to: string; through?: string }, + columns: { + from?: string + to?: string + fromPrimary?: string + toPrimary?: string + }[] + ): Knex.QueryBuilder { + const { tableAliases: aliases, endpoint } = this.query + const schema = endpoint.schema + const toTable = tables.to, + fromTable = tables.from, + throughTable = tables.through + const toAlias = aliases?.[toTable] || toTable, + throughAlias = (throughTable && aliases?.[throughTable]) || throughTable, + fromAlias = aliases?.[fromTable] || fromTable + let toTableWithSchema = this.tableNameWithSchema(toTable, { + alias: toAlias, + schema, + }) + let throughTableWithSchema = throughTable + ? this.tableNameWithSchema(throughTable, { + alias: throughAlias, + schema, + }) + : undefined + if (!throughTable) { + // @ts-ignore + query = query.leftJoin(toTableWithSchema, function () { + for (let relationship of columns) { + const from = relationship.from, + to = relationship.to + // @ts-ignore + this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`) + } + }) + } else { + query = query + // @ts-ignore + .leftJoin(throughTableWithSchema, function () { + for (let relationship of columns) { + const fromPrimary = relationship.fromPrimary + const from = relationship.from + // @ts-ignore + this.orOn( + `${fromAlias}.${fromPrimary}`, + "=", + `${throughAlias}.${from}` + ) + } + }) + .leftJoin(toTableWithSchema, function () { + for (let relationship of columns) { + const toPrimary = relationship.toPrimary + const to = relationship.to + // @ts-ignore + this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`) + } + }) + } + return query + } + addRelationships( query: Knex.QueryBuilder, fromTable: string, - relationships: RelationshipsJson[], - schema?: string, - aliases?: Record + relationships: RelationshipsJson[] ): Knex.QueryBuilder { const tableSets: Record = {} // aggregate into table sets (all the same to tables) @@ -865,51 +963,15 @@ class InternalBuilder { } for (let [key, relationships] of Object.entries(tableSets)) { const { toTable, throughTable } = JSON.parse(key) - const toAlias = aliases?.[toTable] || toTable, - throughAlias = aliases?.[throughTable] || throughTable, - fromAlias = aliases?.[fromTable] || fromTable - let toTableWithSchema = this.tableNameWithSchema(toTable, { - alias: toAlias, - schema, - }) - let throughTableWithSchema = this.tableNameWithSchema(throughTable, { - alias: throughAlias, - schema, - }) - if (!throughTable) { - // @ts-ignore - query = query.leftJoin(toTableWithSchema, function () { - for (let relationship of relationships) { - const from = relationship.from, - to = relationship.to - // @ts-ignore - this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`) - } - }) - } else { - query = query - // @ts-ignore - .leftJoin(throughTableWithSchema, function () { - for (let relationship of relationships) { - const fromPrimary = relationship.fromPrimary - const from = relationship.from - // @ts-ignore - this.orOn( - `${fromAlias}.${fromPrimary}`, - "=", - `${throughAlias}.${from}` - ) - } - }) - .leftJoin(toTableWithSchema, function () { - for (let relationship of relationships) { - const toPrimary = relationship.toPrimary - const to = relationship.to - // @ts-ignore - this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`) - } - }) - } + query = this.addJoin( + query, + { + from: fromTable, + to: toTable, + through: throughTable, + }, + relationships + ) } return query } @@ -1015,8 +1077,7 @@ class InternalBuilder { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { endpoint, filters, paginate, relationships, tableAliases } = - this.query + let { endpoint, filters, paginate, relationships } = this.query const { limits } = opts const counting = endpoint.operation === Operation.COUNT @@ -1062,13 +1123,7 @@ class InternalBuilder { if (relationships && this.client === SqlClient.SQL_LITE) { query = this.addJsonRelationships(query, tableName, relationships) } else if (relationships) { - query = this.addRelationships( - query, - tableName, - relationships, - endpoint.schema, - tableAliases - ) + query = this.addRelationships(query, tableName, relationships) } return this.addFilters(query, filters, { relationship: true }) From 49c1f34b5d5fe78fbfd21c5bb80a934149c8e81a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 28 Aug 2024 18:41:02 +0100 Subject: [PATCH 06/31] Saving at this point - got exists working. --- packages/backend-core/src/sql/sql.ts | 173 ++++++++++++++++----------- 1 file changed, 101 insertions(+), 72 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 9fbb02a69ec..1243eaea8aa 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -343,18 +343,24 @@ class InternalBuilder { whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex - const { relationships, endpoint } = this.query + const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId if (!relationships) { return query } for (const relationship of relationships) { // this is the relationship which is being filtered - if (filterKey.startsWith(relationship.column) && relationship.to) { - const subQuery = query.whereExists(function () { - this.select(mainKnex.raw(1)).from(relationship.tableName!) - }) - query = whereCb( + if ( + filterKey.startsWith(`${relationship.tableName}.`) && + relationship.to && + relationship.tableName + ) { + const relatedTableName = relationship.tableName + const alias = aliases?.[relatedTableName] || relatedTableName + let subQuery = mainKnex + .select(mainKnex.raw(1)) + .from({ [alias]: relatedTableName }) + subQuery = whereCb( this.addJoin( subQuery, { @@ -365,6 +371,7 @@ class InternalBuilder { [relationship] ) ) + query = query.whereExists(subQuery) break } } @@ -382,12 +389,13 @@ class InternalBuilder { if (!filters) { return query } + const builder = this filters = this.parseFilters({ ...filters }) const aliases = this.query.tableAliases // if all or specified in filters, then everything is an or const allOr = filters.allOr - const tableName = - this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name + const isSqlite = this.client === SqlClient.SQL_LITE + const tableName = isSqlite ? this.table._id! : this.table.name function getTableAlias(name: string) { const alias = aliases?.[name] @@ -395,13 +403,33 @@ class InternalBuilder { } function iterate( structure: AnySearchFilter, - fn: (key: string, value: any) => void, - complexKeyFn?: (key: string[], value: any) => void + fn: ( + query: Knex.QueryBuilder, + key: string, + value: any + ) => Knex.QueryBuilder, + complexKeyFn?: ( + query: Knex.QueryBuilder, + key: string[], + value: any + ) => Knex.QueryBuilder ) { + const handleRelationship = ( + q: Knex.QueryBuilder, + key: string, + value: any + ) => { + const [filterTableName, ...otherProperties] = key.split(".") + const property = otherProperties.join(".") + const alias = getTableAlias(filterTableName) + return fn(q, alias ? `${alias}.${property}` : property, value) + } for (const key in structure) { const value = structure[key] const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") + const shouldProcessRelationship = + opts?.relationship && isRelationshipField let castedTypeValue if ( @@ -410,7 +438,8 @@ class InternalBuilder { complexKeyFn ) { const alias = getTableAlias(tableName) - complexKeyFn( + query = complexKeyFn( + query, castedTypeValue.id.map((x: string) => alias ? `${alias}.${x}` : x ), @@ -418,27 +447,31 @@ class InternalBuilder { ) } else if (!isRelationshipField) { const alias = getTableAlias(tableName) - fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) - } - if (opts?.relationship && isRelationshipField) { - // TODO: need to update fn to take the query - const [filterTableName, property] = updatedKey.split(".") - const alias = getTableAlias(filterTableName) - fn(alias ? `${alias}.${property}` : property, value) + query = fn( + query, + alias ? `${alias}.${updatedKey}` : updatedKey, + value + ) + } else if (isSqlite && shouldProcessRelationship) { + query = builder.addRelationshipForFilter(query, updatedKey, q => { + return handleRelationship(q, updatedKey, value) + }) + } else if (shouldProcessRelationship) { + query = handleRelationship(query, updatedKey, value) } } } - const like = (key: string, value: any) => { + const like = (q: Knex.QueryBuilder, key: string, value: any) => { const fuzzyOr = filters?.fuzzyOr const fnc = fuzzyOr || allOr ? "orWhere" : "where" // postgres supports ilike, nothing else does if (this.client === SqlClient.POSTGRES) { - query = query[fnc](key, "ilike", `%${value}%`) + return q[fnc](key, "ilike", `%${value}%`) } else { const rawFnc = `${fnc}Raw` // @ts-ignore - query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ `%${value.toLowerCase()}%`, ]) } @@ -456,13 +489,13 @@ class InternalBuilder { return `[${value.join(",")}]` } if (this.client === SqlClient.POSTGRES) { - iterate(mode, (key, value) => { + iterate(mode, (q, key, value) => { const wrap = any ? "" : "'" const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) const table = fieldNames[0] const col = fieldNames[1] - query = query[rawFnc]( + return q[rawFnc]( `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray( value, any ? "'" : '"' @@ -471,8 +504,8 @@ class InternalBuilder { }) } else if (this.client === SqlClient.MY_SQL) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" - iterate(mode, (key, value) => { - query = query[rawFnc]( + iterate(mode, (q, key, value) => { + return q[rawFnc]( `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( value )}'), FALSE)` @@ -480,7 +513,7 @@ class InternalBuilder { }) } else { const andOr = mode === filters?.containsAny ? " OR " : " AND " - iterate(mode, (key, value) => { + iterate(mode, (q, key, value) => { let statement = "" const identifier = this.quotedIdentifier(key) for (let i in value) { @@ -495,16 +528,16 @@ class InternalBuilder { } if (statement === "") { - return + return q } if (not) { - query = query[rawFnc]( + return q[rawFnc]( `(NOT (${statement}) OR ${identifier} IS NULL)`, value ) } else { - query = query[rawFnc](statement, value) + return q[rawFnc](statement, value) } }) } @@ -534,39 +567,39 @@ class InternalBuilder { const fnc = allOr ? "orWhereIn" : "whereIn" iterate( filters.oneOf, - (key: string, array) => { + (q, key: string, array) => { if (this.client === SqlClient.ORACLE) { key = this.convertClobs(key) array = Array.isArray(array) ? array : [array] const binding = new Array(array.length).fill("?").join(",") - query = query.whereRaw(`${key} IN (${binding})`, array) + return q.whereRaw(`${key} IN (${binding})`, array) } else { - query = query[fnc](key, Array.isArray(array) ? array : [array]) + return q[fnc](key, Array.isArray(array) ? array : [array]) } }, - (key: string[], array) => { + (q, key: string[], array) => { if (this.client === SqlClient.ORACLE) { const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})` const binding = `(${array .map((a: any) => `(${new Array(a.length).fill("?").join(",")})`) .join(",")})` - query = query.whereRaw(`${keyStr} IN ${binding}`, array.flat()) + return q.whereRaw(`${keyStr} IN ${binding}`, array.flat()) } else { - query = query[fnc](key, Array.isArray(array) ? array : [array]) + return q[fnc](key, Array.isArray(array) ? array : [array]) } } ) } if (filters.string) { - iterate(filters.string, (key, value) => { + iterate(filters.string, (q, key, value) => { const fnc = allOr ? "orWhere" : "where" // postgres supports ilike, nothing else does if (this.client === SqlClient.POSTGRES) { - query = query[fnc](key, "ilike", `${value}%`) + return q[fnc](key, "ilike", `${value}%`) } else { const rawFnc = `${fnc}Raw` // @ts-ignore - query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ `${value.toLowerCase()}%`, ]) } @@ -576,7 +609,7 @@ class InternalBuilder { iterate(filters.fuzzy, like) } if (filters.range) { - iterate(filters.range, (key, value) => { + iterate(filters.range, (q, key, value) => { const isEmptyObject = (val: any) => { return ( val && @@ -605,97 +638,93 @@ class InternalBuilder { schema?.type === FieldType.BIGINT && this.client === SqlClient.SQL_LITE ) { - query = query.whereRaw( + return q.whereRaw( `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, [value.low, value.high] ) } else { const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) + return q[fnc](key, [value.low, value.high]) } } else if (lowValid) { if ( schema?.type === FieldType.BIGINT && this.client === SqlClient.SQL_LITE ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, - [value.low] - ) + return q.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ + value.low, + ]) } else { const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, ">=", value.low) + return q[fnc](key, ">=", value.low) } } else if (highValid) { if ( schema?.type === FieldType.BIGINT && this.client === SqlClient.SQL_LITE ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, - [value.high] - ) + return q.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ + value.high, + ]) } else { const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "<=", value.high) + return q[fnc](key, "<=", value.high) } } + return q }) } if (filters.equal) { - iterate(filters.equal, (key, value) => { + iterate(filters.equal, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { - query = query[fnc]( + return q[fnc]( `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`, [value] ) } else if (this.client === SqlClient.ORACLE) { const identifier = this.convertClobs(key) - query = query[fnc]( - `(${identifier} IS NOT NULL AND ${identifier} = ?)`, - [value] - ) + return q[fnc](`(${identifier} IS NOT NULL AND ${identifier} = ?)`, [ + value, + ]) } else { - query = query[fnc]( - `COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, - [value] - ) + return q[fnc](`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [ + value, + ]) } }) } if (filters.notEqual) { - iterate(filters.notEqual, (key, value) => { + iterate(filters.notEqual, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { - query = query[fnc]( + return q[fnc]( `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`, [value] ) } else if (this.client === SqlClient.ORACLE) { const identifier = this.convertClobs(key) - query = query[fnc]( + return q[fnc]( `(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`, [value] ) } else { - query = query[fnc]( - `COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, - [value] - ) + return q[fnc](`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [ + value, + ]) } }) } if (filters.empty) { - iterate(filters.empty, key => { + iterate(filters.empty, (q, key) => { const fnc = allOr ? "orWhereNull" : "whereNull" - query = query[fnc](key) + return q[fnc](key) }) } if (filters.notEmpty) { - iterate(filters.notEmpty, key => { + iterate(filters.notEmpty, (q, key) => { const fnc = allOr ? "orWhereNotNull" : "whereNotNull" - query = query[fnc](key) + return q[fnc](key) }) } if (filters.contains) { From 628964364a3ce5e7c6a988656b8f0fa535d15c07 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 28 Aug 2024 18:55:15 +0100 Subject: [PATCH 07/31] Getting through join working as expected. --- packages/backend-core/src/sql/sql.ts | 40 +++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 1243eaea8aa..84231288a28 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -345,6 +345,7 @@ class InternalBuilder { const mainKnex = this.knex const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId + const fromAlias = aliases?.[tableName] || tableName if (!relationships) { return query } @@ -356,22 +357,37 @@ class InternalBuilder { relationship.tableName ) { const relatedTableName = relationship.tableName - const alias = aliases?.[relatedTableName] || relatedTableName + const toAlias = aliases?.[relatedTableName] || relatedTableName let subQuery = mainKnex .select(mainKnex.raw(1)) - .from({ [alias]: relatedTableName }) - subQuery = whereCb( - this.addJoin( - subQuery, - { - from: relationship.tableName, - to: tableName, - through: relationship.through, - }, - [relationship] + .from({ [toAlias]: relatedTableName }) + let mainTableRelatesTo = toAlias + if (relationship.through) { + const throughAlias = + aliases?.[relationship.through] || relationship.through + let throughTable = this.tableNameWithSchema(relationship.through, { + alias: throughAlias, + schema: endpoint.schema, + }) + subQuery = subQuery.innerJoin(throughTable, function () { + // @ts-ignore + this.orOn( + `${toAlias}.${relationship.toPrimary}`, + "=", + `${throughAlias}.${relationship.to}` + ) + }) + mainTableRelatesTo = throughAlias + } + // "join" to the main table, making sure the ID matches that of the main + subQuery = subQuery.where( + `${mainTableRelatesTo}.${relationship.from}`, + "=", + mainKnex.raw( + this.quotedIdentifier(`${fromAlias}.${relationship.fromPrimary}`) ) ) - query = query.whereExists(subQuery) + query = query.whereExists(whereCb(subQuery)) break } } From 3e51dde6d2be09351faaed3e6c79f214977d65ff Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 29 Aug 2024 17:58:11 +0100 Subject: [PATCH 08/31] Check for alias as well when deciding whether filter requires relationship addition. --- packages/backend-core/src/sql/sql.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 84231288a28..9b6819d433b 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -346,21 +346,25 @@ class InternalBuilder { const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId const fromAlias = aliases?.[tableName] || tableName + const matches = (possibleTable: string) => + filterKey.startsWith(`${possibleTable}`) if (!relationships) { return query } for (const relationship of relationships) { + const relatedTableName = relationship.tableName + const toAlias = aliases?.[relatedTableName] || relatedTableName // this is the relationship which is being filtered if ( - filterKey.startsWith(`${relationship.tableName}.`) && + (matches(relatedTableName) || matches(toAlias)) && relationship.to && relationship.tableName ) { - const relatedTableName = relationship.tableName - const toAlias = aliases?.[relatedTableName] || relatedTableName let subQuery = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) + // relationships should never have more than the base limit + .limit(getBaseLimit()) let mainTableRelatesTo = toAlias if (relationship.through) { const throughAlias = From a9b1a2240332e7ea337ed0e2c6625b97228ab2ad Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 29 Aug 2024 18:56:14 +0100 Subject: [PATCH 09/31] Some improvements to get SQS tests passing. --- packages/backend-core/src/sql/sql.ts | 20 +++++++++++++++---- .../src/api/routes/tests/search.spec.ts | 3 ++- .../src/utilities/rowProcessor/index.ts | 7 +++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 9b6819d433b..e6738d4b362 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -337,6 +337,11 @@ class InternalBuilder { return filters } + addJoinFieldCheck(query: Knex.QueryBuilder, relationship: RelationshipsJson) { + const document = relationship.from?.split(".")[0] || "" + return query.andWhere(`${document}.fieldName`, "=", relationship.column) + } + addRelationshipForFilter( query: Knex.QueryBuilder, filterKey: string, @@ -363,8 +368,6 @@ class InternalBuilder { let subQuery = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) - // relationships should never have more than the base limit - .limit(getBaseLimit()) let mainTableRelatesTo = toAlias if (relationship.through) { const throughAlias = @@ -375,12 +378,15 @@ class InternalBuilder { }) subQuery = subQuery.innerJoin(throughTable, function () { // @ts-ignore - this.orOn( + this.on( `${toAlias}.${relationship.toPrimary}`, "=", `${throughAlias}.${relationship.to}` ) }) + if (this.client === SqlClient.SQL_LITE) { + subQuery = this.addJoinFieldCheck(subQuery, relationship) + } mainTableRelatesTo = throughAlias } // "join" to the main table, making sure the ID matches that of the main @@ -907,7 +913,7 @@ class InternalBuilder { default: throw new Error(`JSON relationships not implement for ${this.client}`) } - const subQuery = this.knex + let subQuery = this.knex .select(rawJsonArray) .from(toTableWithSchema) .join(throughTableWithSchema, function () { @@ -918,6 +924,12 @@ class InternalBuilder { "=", this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) ) + // relationships should never have more than the base limit + .limit(getBaseLimit()) + // need to check the junction table document is to the right column + if (this.client === SqlClient.SQL_LITE) { + subQuery = this.addJoinFieldCheck(subQuery, relationship) + } query = query.select({ [relationship.column]: subQuery }) } return query diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index bac9b6f7740..44fe18d0eff 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2687,7 +2687,8 @@ describe.each([ }) }) - isSql && + // TODO: when all SQL databases use the same mechanism - remove this test, new relationship system doesn't have this problem + !isInternal && describe("pagination edge case with relationships", () => { let mainRows: Row[] = [] diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 795f6970ab3..6028b2d2486 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -336,6 +336,13 @@ export async function outputProcessing( row[property] = `${hours}:${minutes}:${seconds}` } } + } else if (column.type === FieldType.LINK) { + for (let row of enriched) { + // if relationship is empty - remove the array, this has been part of the API for some time + if (Array.isArray(row[property]) && row[property].length === 0) { + delete row[property] + } + } } } From 67301054913024ecab6e0bc5650ee464c73cd34d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 13:44:23 +0100 Subject: [PATCH 10/31] Adding the option to disable user sync, always importing large apps which are problematic. --- packages/server/src/environment.ts | 1 + packages/server/src/sdk/app/applications/sync.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 05ebbe31511..585eb6a7f2d 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -112,6 +112,7 @@ const environment = { parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || DEFAULTS.JS_RUNNER_MEMORY_LIMIT, LOG_JS_ERRORS: process.env.LOG_JS_ERRORS, + DISABLE_USER_SYNC: process.env.DISABLE_USER_SYNC, // old CLIENT_ID: process.env.CLIENT_ID, _set(key: string, value: any) { diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 44e44a5aaa0..37450acf1d6 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -8,6 +8,10 @@ import { generateUserMetadataID, InternalTables } from "../../../db/utils" type DeletedUser = { _id: string; deleted: boolean } +function userSyncEnabled() { + return !env.DISABLE_USER_SYNC +} + async function syncUsersToApp( appId: string, users: (User | DeletedUser)[], @@ -56,7 +60,7 @@ async function syncUsersToApp( // the user doesn't exist, or doesn't have a role anymore // get rid of their metadata - if (deletedUser || !roleId) { + if (userSyncEnabled() && (deletedUser || !roleId)) { await db.remove(metadata) continue } @@ -149,7 +153,9 @@ export async function syncApp( } // sync the users - kept for safe keeping - await sdk.users.syncGlobalUsers() + if (userSyncEnabled()) { + await sdk.users.syncGlobalUsers() + } if (error) { throw error From ac7838f80dd45735a4808afe888da236f7c7272e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Sep 2024 12:09:33 +0100 Subject: [PATCH 11/31] Fixing an issue with inconsistent relationship order. --- packages/backend-core/src/sql/sql.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e6738d4b362..626ab3bf8e1 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -926,6 +926,8 @@ class InternalBuilder { ) // relationships should never have more than the base limit .limit(getBaseLimit()) + // add sorting to get consistent order + .orderBy(`${toAlias}.${toPrimary}`) // need to check the junction table document is to the right column if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, relationship) From b29a4e2b37020683139c19524c489a5a8b65f14b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Sep 2024 18:24:50 +0100 Subject: [PATCH 12/31] Work to support all SQL DBs across the board using the aggregation method. --- packages/backend-core/src/sql/sql.ts | 297 ++++++++++-------- packages/backend-core/src/sql/utils.ts | 30 +- .../src/api/controllers/row/utils/basic.ts | 61 +++- .../src/api/controllers/row/utils/sqlUtils.ts | 67 ---- .../src/api/controllers/row/utils/utils.ts | 27 +- .../src/api/routes/tests/search.spec.ts | 281 ++++++----------- packages/types/src/sdk/search.ts | 11 + 7 files changed, 370 insertions(+), 404 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 626ab3bf8e1..6181d180467 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -7,6 +7,7 @@ import { isValidFilter, isValidISODateString, sqlLog, + validateManyToMany, } from "./utils" import SqlTableQueryBuilder from "./sqlTable" import { @@ -133,80 +134,78 @@ class InternalBuilder { return "*" } - // no relationships - select everything in SQLite - if (this.client === SqlClient.SQL_LITE) { - const alias = tableAliases?.[endpoint.entityId] - ? tableAliases?.[endpoint.entityId] - : endpoint.entityId - return [this.knex.raw(`${this.quote(alias)}.*`)] - } - - const schema = meta.table.schema - return resource.fields.map(field => { - const parts = field.split(/\./g) - let table: string | undefined = undefined - let column: string | undefined = undefined - - // Just a column name, e.g.: "column" - if (parts.length === 1) { - column = parts[0] - } - - // A table name and a column name, e.g.: "table.column" - if (parts.length === 2) { - table = parts[0] - column = parts[1] - } - - // A link doc, e.g.: "table.doc1.fieldName" - if (parts.length > 2) { - table = parts[0] - column = parts.slice(1).join(".") - } - - if (!column) { - throw new Error(`Invalid field name: ${field}`) - } - - const columnSchema = schema[column] - - if ( - this.client === SqlClient.POSTGRES && - columnSchema?.externalType?.includes("money") - ) { - return this.knex.raw( - `${this.quotedIdentifier( - [table, column].join(".") - )}::money::numeric as ${this.quote(field)}` - ) - } - - if ( - this.client === SqlClient.MS_SQL && - columnSchema?.type === FieldType.DATETIME && - columnSchema.timeOnly - ) { - // Time gets returned as timestamp from mssql, not matching the expected - // HH:mm format - return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) - } - - // There's at least two edge cases being handled in the expression below. - // 1. The column name could start/end with a space, and in that case we - // want to preseve that space. - // 2. Almost all column names are specified in the form table.column, except - // in the case of relationships, where it's table.doc1.column. In that - // case, we want to split it into `table`.`doc1.column` for reasons that - // aren't actually clear to me, but `table`.`doc1` breaks things with the - // sample data tests. - if (table) { - return this.knex.raw( - `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}` - ) - } else { - return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`) - } - }) + const alias = tableAliases?.[endpoint.entityId] + ? tableAliases?.[endpoint.entityId] + : endpoint.entityId + return [this.knex.raw(`${this.quote(alias)}.*`)] + // + // + // const schema = meta.table.schema + // return resource.fields.map(field => { + // const parts = field.split(/\./g) + // let table: string | undefined = undefined + // let column: string | undefined = undefined + // + // // Just a column name, e.g.: "column" + // if (parts.length === 1) { + // column = parts[0] + // } + // + // // A table name and a column name, e.g.: "table.column" + // if (parts.length === 2) { + // table = parts[0] + // column = parts[1] + // } + // + // // A link doc, e.g.: "table.doc1.fieldName" + // if (parts.length > 2) { + // table = parts[0] + // column = parts.slice(1).join(".") + // } + // + // if (!column) { + // throw new Error(`Invalid field name: ${field}`) + // } + // + // const columnSchema = schema[column] + // + // if ( + // this.client === SqlClient.POSTGRES && + // columnSchema?.externalType?.includes("money") + // ) { + // return this.knex.raw( + // `${this.quotedIdentifier( + // [table, column].join(".") + // )}::money::numeric as ${this.quote(field)}` + // ) + // } + // + // if ( + // this.client === SqlClient.MS_SQL && + // columnSchema?.type === FieldType.DATETIME && + // columnSchema.timeOnly + // ) { + // // Time gets returned as timestamp from mssql, not matching the expected + // // HH:mm format + // return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) + // } + // + // // There's at least two edge cases being handled in the expression below. + // // 1. The column name could start/end with a space, and in that case we + // // want to preseve that space. + // // 2. Almost all column names are specified in the form table.column, except + // // in the case of relationships, where it's table.doc1.column. In that + // // case, we want to split it into `table`.`doc1.column` for reasons that + // // aren't actually clear to me, but `table`.`doc1` breaks things with the + // // sample data tests. + // if (table) { + // return this.knex.raw( + // `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}` + // ) + // } else { + // return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`) + // } + // }) } // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, @@ -368,35 +367,47 @@ class InternalBuilder { let subQuery = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) - let mainTableRelatesTo = toAlias - if (relationship.through) { + const manyToMany = validateManyToMany(relationship) + if (manyToMany) { const throughAlias = - aliases?.[relationship.through] || relationship.through - let throughTable = this.tableNameWithSchema(relationship.through, { + aliases?.[manyToMany.through] || relationship.through + let throughTable = this.tableNameWithSchema(manyToMany.through, { alias: throughAlias, schema: endpoint.schema, }) - subQuery = subQuery.innerJoin(throughTable, function () { - // @ts-ignore - this.on( - `${toAlias}.${relationship.toPrimary}`, + subQuery = subQuery + // add a join through the junction table + .innerJoin(throughTable, function () { + // @ts-ignore + this.on( + `${toAlias}.${manyToMany.toPrimary}`, + "=", + `${throughAlias}.${manyToMany.to}` + ) + }) + // check the document in the junction table points to the main table + .where( + `${throughAlias}.${manyToMany.from}`, "=", - `${throughAlias}.${relationship.to}` + mainKnex.raw( + this.quotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`) + ) ) - }) + // in SQS the same junction table is used for different many-to-many relationships between the + // two same tables, this is needed to avoid rows ending up in all columns if (this.client === SqlClient.SQL_LITE) { - subQuery = this.addJoinFieldCheck(subQuery, relationship) + subQuery = this.addJoinFieldCheck(subQuery, manyToMany) } - mainTableRelatesTo = throughAlias - } - // "join" to the main table, making sure the ID matches that of the main - subQuery = subQuery.where( - `${mainTableRelatesTo}.${relationship.from}`, - "=", - mainKnex.raw( - this.quotedIdentifier(`${fromAlias}.${relationship.fromPrimary}`) + } else { + // "join" to the main table, making sure the ID matches that of the main + subQuery = subQuery.where( + `${toAlias}.${relationship.to}`, + "=", + mainKnex.raw( + this.quotedIdentifier(`${fromAlias}.${relationship.from}`) + ) ) - ) + } query = query.whereExists(whereCb(subQuery)) break } @@ -478,12 +489,10 @@ class InternalBuilder { alias ? `${alias}.${updatedKey}` : updatedKey, value ) - } else if (isSqlite && shouldProcessRelationship) { + } else if (shouldProcessRelationship) { query = builder.addRelationshipForFilter(query, updatedKey, q => { return handleRelationship(q, updatedKey, value) }) - } else if (shouldProcessRelationship) { - query = handleRelationship(query, updatedKey, value) } } } @@ -849,6 +858,7 @@ class InternalBuilder { fromTable: string, relationships: RelationshipsJson[] ): Knex.QueryBuilder { + const sqlClient = this.client const { resource, tableAliases: aliases, endpoint } = this.query const fields = resource?.fields || [] const jsonField = (field: string) => { @@ -862,7 +872,15 @@ class InternalBuilder { unaliased = parts.join(".") tableField = this.quote(unaliased) } - return `'${unaliased}',${tableField}` + let separator = "," + switch (sqlClient) { + case SqlClient.ORACLE: + separator = " VALUE " + break + case SqlClient.MS_SQL: + separator = ":" + } + return `'${unaliased}'${separator}${tableField}` } for (let relationship of relationships) { const { @@ -874,23 +892,15 @@ class InternalBuilder { toPrimary, } = relationship // skip invalid relationships - if (!toTable || !fromTable || !fromPrimary || !toPrimary) { + if (!toTable || !fromTable) { continue } - if (!throughTable) { - throw new Error("Only many-to-many implemented for JSON relationships") - } const toAlias = aliases?.[toTable] || toTable, - throughAlias = aliases?.[throughTable] || throughTable, fromAlias = aliases?.[fromTable] || fromTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, }) - let throughTableWithSchema = this.tableNameWithSchema(throughTable, { - alias: throughAlias, - schema: endpoint.schema, - }) let relationshipFields = fields.filter( field => field.split(".")[0] === toAlias ) @@ -903,32 +913,75 @@ class InternalBuilder { const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") - let rawJsonArray: Knex.Raw - switch (this.client) { + let rawJsonArray: Knex.Raw, limit: number + switch (sqlClient) { case SqlClient.SQL_LITE: rawJsonArray = this.knex.raw( `json_group_array(json_object(${fieldList}))` ) + limit = getBaseLimit() + break + case SqlClient.POSTGRES: + rawJsonArray = this.knex.raw( + `json_agg(json_build_object(${fieldList}))` + ) + limit = 1 + break + case SqlClient.MY_SQL: + case SqlClient.ORACLE: + rawJsonArray = this.knex.raw( + `json_arrayagg(json_object(${fieldList}))` + ) + limit = getBaseLimit() + break + case SqlClient.MS_SQL: + rawJsonArray = this.knex.raw(`json_array(json_object(${fieldList}))`) + limit = 1 break default: throw new Error(`JSON relationships not implement for ${this.client}`) } + // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax + // it reduces the result set rather than limiting how much data it filters over + const primaryKey = `${toAlias}.${toPrimary || toKey}` let subQuery = this.knex .select(rawJsonArray) .from(toTableWithSchema) - .join(throughTableWithSchema, function () { - this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) + .limit(limit) + // add sorting to get consistent order + .orderBy(primaryKey) + + if (sqlClient === SqlClient.POSTGRES) { + subQuery = subQuery.groupBy(primaryKey) + } + + // many-to-many relationship with junction table + if (throughTable && toPrimary && fromPrimary) { + const throughAlias = aliases?.[throughTable] || throughTable + let throughTableWithSchema = this.tableNameWithSchema(throughTable, { + alias: throughAlias, + schema: endpoint.schema, }) - .where( - `${throughAlias}.${fromKey}`, + subQuery = subQuery + .join(throughTableWithSchema, function () { + this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) + }) + .where( + `${throughAlias}.${fromKey}`, + "=", + this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) + ) + } + // one-to-many relationship with foreign key + else { + subQuery = subQuery.where( + `${toAlias}.${toKey}`, "=", - this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) + this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) ) - // relationships should never have more than the base limit - .limit(getBaseLimit()) - // add sorting to get consistent order - .orderBy(`${toAlias}.${toPrimary}`) - // need to check the junction table document is to the right column + } + + // need to check the junction table document is to the right column, this is just for SQS if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, relationship) } @@ -1179,14 +1232,12 @@ class InternalBuilder { ? query.select(this.generateSelectStatement()) : this.addDistinctCount(query) // have to add after as well (this breaks MS-SQL) - if (this.client !== SqlClient.MS_SQL && !counting) { + if (!counting) { query = this.addSorting(query) } // handle joins - if (relationships && this.client === SqlClient.SQL_LITE) { + if (relationships) { query = this.addJsonRelationships(query, tableName, relationships) - } else if (relationships) { - query = this.addRelationships(query, tableName, relationships) } return this.addFilters(query, filters, { relationship: true }) diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 1b32cc6da7f..1b80ff337df 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -1,4 +1,11 @@ -import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types" +import { + DocumentType, + ManyToManyRelationshipJson, + RelationshipsJson, + SqlQuery, + Table, + TableSourceType, +} from "@budibase/types" import { DEFAULT_BB_DATASOURCE_ID } from "../constants" import { Knex } from "knex" import { SEPARATOR } from "../db" @@ -163,3 +170,24 @@ export function sqlLog(client: string, query: string, values?: any[]) { } console.log(string) } + +function isValidManyToManyRelationship( + relationship: RelationshipsJson +): relationship is ManyToManyRelationshipJson { + return ( + !!relationship.through && + !!relationship.fromPrimary && + !!relationship.from && + !!relationship.toPrimary && + !!relationship.to + ) +} + +export function validateManyToMany( + relationship: RelationshipsJson +): ManyToManyRelationshipJson | undefined { + if (isValidManyToManyRelationship(relationship)) { + return relationship + } + return undefined +} diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 8f3607bc73c..9d5a315628e 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,6 +1,10 @@ // need to handle table name + field or just field, depending on if relationships used import { FieldSchema, FieldType, Row, Table } from "@budibase/types" -import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" +import { + helpers, + PROTECTED_EXTERNAL_COLUMNS, + PROTECTED_INTERNAL_COLUMNS, +} from "@budibase/shared-core" import { generateRowIdField } from "../../../../integrations/utils" function extractFieldValue({ @@ -61,11 +65,13 @@ export function generateIdForRow( export function basicProcessing({ row, table, + tables, isLinked, sqs, }: { row: Row table: Table + tables: Table[] isLinked: boolean sqs?: boolean }): Row { @@ -86,24 +92,65 @@ export function basicProcessing({ thisRow[fieldName] = value } } + let columns: string[] = Object.keys(table.schema) if (!sqs) { thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) } else { - const columns = Object.keys(table.schema) + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { - const schema: FieldSchema | undefined = table.schema[internalColumn] - let value = extractFieldValue({ + thisRow[internalColumn] = extractFieldValue({ row, tableName: table._id!, fieldName: internalColumn, isLinked, }) - if (sqs && schema?.type === FieldType.LINK && typeof value === "string") { - value = JSON.parse(value) + } + } + for (let col of columns) { + const schema: FieldSchema | undefined = table.schema[col] + if (schema?.type !== FieldType.LINK) { + continue + } + const relatedTable = tables.find(tbl => tbl._id === schema.tableId) + if (!relatedTable) { + continue + } + const value = extractFieldValue({ + row, + tableName: table._id!, + fieldName: col, + isLinked, + }) + const array: Row[] = Array.isArray(value) + ? value + : typeof value === "string" + ? JSON.parse(value) + : undefined + if (array) { + thisRow[col] = array + // make sure all of them have an _id + if (Array.isArray(thisRow[col])) { + const sortField = + relatedTable.primaryDisplay || relatedTable.primary![0]! + thisRow[col] = (thisRow[col] as Row[]) + .map(relatedRow => { + relatedRow._id = relatedRow._id + ? relatedRow._id + : generateIdForRow(relatedRow, relatedTable) + return relatedRow + }) + .sort((a, b) => { + if (!a?.[sortField]) { + return 1 + } else if (!b?.[sortField]) { + return -1 + } + return a[sortField].localeCompare(b[sortField]) + }) } - thisRow[internalColumn] = value } } return thisRow diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index a24ec17c263..d6e5c3e8f16 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -36,73 +36,6 @@ function isCorrectRelationship( return !!possibleColumns.find(col => row[col] === relationship.column) } -/** - * This iterates through the returned rows and works out what elements of the rows - * actually match up to another row (based on primary keys) - this is pretty specific - * to SQL and the way that SQL relationships are returned based on joins. - * This is complicated, but the idea is that when a SQL query returns all the relations - * will be separate rows, with all of the data in each row. We have to decipher what comes - * from where (which tables) and how to convert that into budibase columns. - */ -export async function updateRelationshipColumns( - table: Table, - tables: TableMap, - row: Row, - rows: { [key: string]: Row }, - relationships: RelationshipsJson[], - opts?: { sqs?: boolean } -) { - const columns: { [key: string]: any } = {} - for (let relationship of relationships) { - const linkedTable = tables[relationship.tableName] - if (!linkedTable) { - continue - } - const fromColumn = `${table.name}.${relationship.from}` - const toColumn = `${linkedTable.name}.${relationship.to}` - // this is important when working with multiple relationships - // between the same tables, don't want to overlap/multiply the relations - if ( - !relationship.through && - row[fromColumn]?.toString() !== row[toColumn]?.toString() - ) { - continue - } - - let linked = basicProcessing({ - row, - table: linkedTable, - isLinked: true, - sqs: opts?.sqs, - }) - if (!linked._id) { - continue - } - if ( - !opts?.sqs || - isCorrectRelationship(relationship, table, linkedTable, row) - ) { - columns[relationship.column] = linked - } - } - for (let [column, related] of Object.entries(columns)) { - if (!row._id) { - continue - } - const rowId: string = row._id - if (!Array.isArray(rows[rowId][column])) { - rows[rowId][column] = [] - } - // make sure relationship hasn't been found already - if ( - !rows[rowId][column].find((relation: Row) => relation._id === related._id) - ) { - rows[rowId][column].push(related) - } - } - return rows -} - /** * Gets the list of relationship JSON structures based on the columns in the table, * this will be used by the underlying library to build whatever relationship mechanism diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index e52ee63e420..45f0cef0851 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -13,7 +13,7 @@ import { processDates, processFormulas, } from "../../../../utilities/rowProcessor" -import { isKnexEmptyReadResponse, updateRelationshipColumns } from "./sqlUtils" +import { isKnexEmptyReadResponse } from "./sqlUtils" import { basicProcessing, generateIdForRow, @@ -149,22 +149,11 @@ export async function sqlOutputProcessing( rowId = generateIdForRow(row, table) row._id = rowId } - // this is a relationship of some sort - if (!opts?.sqs && finalRows[rowId]) { - finalRows = await updateRelationshipColumns( - table, - tables, - row, - finalRows, - relationships, - opts - ) - continue - } const thisRow = fixArrayTypes( basicProcessing({ row, table, + tables: Object.values(tables), isLinked: false, sqs: opts?.sqs, }), @@ -175,18 +164,6 @@ export async function sqlOutputProcessing( } finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table }) - - // do this at end once its been added to the final rows - if (!opts?.sqs) { - finalRows = await updateRelationshipColumns( - table, - tables, - row, - finalRows, - relationships, - opts - ) - } } // make sure all related rows are correct diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index de7fac4f1b1..86d86b11612 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2126,81 +2126,76 @@ describe.each([ }) }) - // This will never work for Lucene. - !isLucene && - // It also can't work for in-memory searching because the related table name - // isn't available. - !isInMemory && - describe("relations", () => { - let productCategoryTable: Table, productCatRows: Row[] + describe("relations", () => { + let productCategoryTable: Table, productCatRows: Row[] - beforeAll(async () => { - productCategoryTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - }, - "productCategory" - ) - table = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - productCat: { - type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, - name: "productCat", - fieldName: "product", - tableId: productCategoryTable._id!, - constraints: { - type: "array", - }, + beforeAll(async () => { + productCategoryTable = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + }, + "productCategory" + ) + table = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + productCat: { + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + name: "productCat", + fieldName: "product", + tableId: productCategoryTable._id!, + constraints: { + type: "array", }, }, - "product" - ) + }, + "product" + ) - productCatRows = await Promise.all([ - config.api.row.save(productCategoryTable._id!, { name: "foo" }), - config.api.row.save(productCategoryTable._id!, { name: "bar" }), - ]) + productCatRows = await Promise.all([ + config.api.row.save(productCategoryTable._id!, { name: "foo" }), + config.api.row.save(productCategoryTable._id!, { name: "bar" }), + ]) - await Promise.all([ - config.api.row.save(table._id!, { - name: "foo", - productCat: [productCatRows[0]._id], - }), - config.api.row.save(table._id!, { - name: "bar", - productCat: [productCatRows[1]._id], - }), - config.api.row.save(table._id!, { - name: "baz", - productCat: [], - }), - ]) - }) + await Promise.all([ + config.api.row.save(table._id!, { + name: "foo", + productCat: [productCatRows[0]._id], + }), + config.api.row.save(table._id!, { + name: "bar", + productCat: [productCatRows[1]._id], + }), + config.api.row.save(table._id!, { + name: "baz", + productCat: [], + }), + ]) + }) - it("should be able to filter by relationship using column name", async () => { - await expectQuery({ - equal: { ["productCat.name"]: "foo" }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) + it("should be able to filter by relationship using column name", async () => { + await expectQuery({ + equal: { ["productCat.name"]: "foo" }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) - it("should be able to filter by relationship using table name", async () => { - await expectQuery({ - equal: { ["productCategory.name"]: "foo" }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) + it("should be able to filter by relationship using table name", async () => { + await expectQuery({ + equal: { ["productCategory.name"]: "foo" }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) - it("shouldn't return any relationship for last row", async () => { - await expectQuery({ - equal: { ["name"]: "baz" }, - }).toContainExactly([{ name: "baz", productCat: undefined }]) - }) + it("shouldn't return any relationship for last row", async () => { + await expectQuery({ + equal: { ["name"]: "baz" }, + }).toContainExactly([{ name: "baz", productCat: undefined }]) }) + }) ;(isSqs || isLucene) && describe("relations to same table", () => { let relatedTable: Table, relatedRows: Row[] @@ -2610,50 +2605,50 @@ describe.each([ }) }) - !isInMemory && - describe("search by _id", () => { - let row: Row + // !isInMemory && + describe("search by _id", () => { + let row: Row - beforeAll(async () => { - const toRelateTable = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - table = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - rel: { - name: "rel", - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_MANY, - tableId: toRelateTable._id!, - fieldName: "rel", - }, - }) - const [row1, row2] = await Promise.all([ - config.api.row.save(toRelateTable._id!, { name: "tag 1" }), - config.api.row.save(toRelateTable._id!, { name: "tag 2" }), - ]) - row = await config.api.row.save(table._id!, { - name: "product 1", - rel: [row1._id, row2._id], - }) + beforeAll(async () => { + const toRelateTable = await createTable({ + name: { + name: "name", + type: FieldType.STRING, + }, }) - - it("can filter by the row ID with limit 1", async () => { - await expectSearch({ - query: { - equal: { _id: row._id }, - }, - limit: 1, - }).toContainExactly([row]) + table = await createTable({ + name: { + name: "name", + type: FieldType.STRING, + }, + rel: { + name: "rel", + type: FieldType.LINK, + relationshipType: RelationshipType.MANY_TO_MANY, + tableId: toRelateTable._id!, + fieldName: "rel", + }, + }) + const [row1, row2] = await Promise.all([ + config.api.row.save(toRelateTable._id!, { name: "tag 1" }), + config.api.row.save(toRelateTable._id!, { name: "tag 2" }), + ]) + row = await config.api.row.save(table._id!, { + name: "product 1", + rel: [row1._id, row2._id], }) }) + it("can filter by the row ID with limit 1", async () => { + await expectSearch({ + query: { + equal: { _id: row._id }, + }, + limit: 1, + }).toContainExactly([row]) + }) + }) + !isInternal && describe("search by composite key", () => { beforeAll(async () => { @@ -2690,82 +2685,6 @@ describe.each([ }) }) - // TODO: when all SQL databases use the same mechanism - remove this test, new relationship system doesn't have this problem - !isInternal && - describe("pagination edge case with relationships", () => { - let mainRows: Row[] = [] - - beforeAll(async () => { - const toRelateTable = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - table = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - rel: { - name: "rel", - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_ONE, - tableId: toRelateTable._id!, - fieldName: "rel", - }, - }) - const relatedRows = await Promise.all([ - config.api.row.save(toRelateTable._id!, { name: "tag 1" }), - config.api.row.save(toRelateTable._id!, { name: "tag 2" }), - config.api.row.save(toRelateTable._id!, { name: "tag 3" }), - config.api.row.save(toRelateTable._id!, { name: "tag 4" }), - config.api.row.save(toRelateTable._id!, { name: "tag 5" }), - config.api.row.save(toRelateTable._id!, { name: "tag 6" }), - ]) - mainRows = await Promise.all([ - config.api.row.save(table._id!, { - name: "product 1", - rel: relatedRows.map(row => row._id), - }), - config.api.row.save(table._id!, { - name: "product 2", - rel: [], - }), - config.api.row.save(table._id!, { - name: "product 3", - rel: [], - }), - ]) - }) - - it("can still page when the hard limit is hit", async () => { - await withCoreEnv( - { - SQL_MAX_ROWS: "6", - }, - async () => { - const params: Omit = { - query: {}, - paginate: true, - limit: 3, - sort: "name", - sortType: SortType.STRING, - sortOrder: SortOrder.ASCENDING, - } - const page1 = await expectSearch(params).toContain([mainRows[0]]) - expect(page1.hasNextPage).toBe(true) - expect(page1.bookmark).toBeDefined() - const page2 = await expectSearch({ - ...params, - bookmark: page1.bookmark, - }).toContain([mainRows[1], mainRows[2]]) - expect(page2.hasNextPage).toBe(false) - } - ) - }) - }) - isSql && describe("primaryDisplay", () => { beforeAll(async () => { diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6feea407666..7d61aebdfb6 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -134,6 +134,17 @@ export interface RelationshipsJson { column: string } +// TODO - this can be combined with the above type +export interface ManyToManyRelationshipJson { + through: string + from: string + to: string + fromPrimary: string + toPrimary: string + tableName: string + column: string +} + export interface QueryJson { endpoint: { datasourceId: string From 2a24a3dda683cafbbb65b0893c102240ea8c7b1e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Sep 2024 18:40:20 +0100 Subject: [PATCH 13/31] Correcting test cases. --- .../src/api/routes/tests/search.spec.ts | 205 +++++++++--------- 1 file changed, 105 insertions(+), 100 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 86d86b11612..f5c5ade2f84 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2126,76 +2126,81 @@ describe.each([ }) }) - describe("relations", () => { - let productCategoryTable: Table, productCatRows: Row[] + // This will never work for Lucene. + !isLucene && + // It also can't work for in-memory searching because the related table name + // isn't available. + !isInMemory && + describe("relations", () => { + let productCategoryTable: Table, productCatRows: Row[] - beforeAll(async () => { - productCategoryTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - }, - "productCategory" - ) - table = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - productCat: { - type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, - name: "productCat", - fieldName: "product", - tableId: productCategoryTable._id!, - constraints: { - type: "array", + beforeAll(async () => { + productCategoryTable = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + }, + "productCategory" + ) + table = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + productCat: { + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + name: "productCat", + fieldName: "product", + tableId: productCategoryTable._id!, + constraints: { + type: "array", + }, }, }, - }, - "product" - ) + "product" + ) - productCatRows = await Promise.all([ - config.api.row.save(productCategoryTable._id!, { name: "foo" }), - config.api.row.save(productCategoryTable._id!, { name: "bar" }), - ]) + productCatRows = await Promise.all([ + config.api.row.save(productCategoryTable._id!, { name: "foo" }), + config.api.row.save(productCategoryTable._id!, { name: "bar" }), + ]) - await Promise.all([ - config.api.row.save(table._id!, { - name: "foo", - productCat: [productCatRows[0]._id], - }), - config.api.row.save(table._id!, { - name: "bar", - productCat: [productCatRows[1]._id], - }), - config.api.row.save(table._id!, { - name: "baz", - productCat: [], - }), - ]) - }) + await Promise.all([ + config.api.row.save(table._id!, { + name: "foo", + productCat: [productCatRows[0]._id], + }), + config.api.row.save(table._id!, { + name: "bar", + productCat: [productCatRows[1]._id], + }), + config.api.row.save(table._id!, { + name: "baz", + productCat: [], + }), + ]) + }) - it("should be able to filter by relationship using column name", async () => { - await expectQuery({ - equal: { ["productCat.name"]: "foo" }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) + it("should be able to filter by relationship using column name", async () => { + await expectQuery({ + equal: { ["productCat.name"]: "foo" }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) - it("should be able to filter by relationship using table name", async () => { - await expectQuery({ - equal: { ["productCategory.name"]: "foo" }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) + it("should be able to filter by relationship using table name", async () => { + await expectQuery({ + equal: { ["productCategory.name"]: "foo" }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) - it("shouldn't return any relationship for last row", async () => { - await expectQuery({ - equal: { ["name"]: "baz" }, - }).toContainExactly([{ name: "baz", productCat: undefined }]) + it("shouldn't return any relationship for last row", async () => { + await expectQuery({ + equal: { ["name"]: "baz" }, + }).toContainExactly([{ name: "baz", productCat: undefined }]) + }) }) - }) ;(isSqs || isLucene) && describe("relations to same table", () => { let relatedTable: Table, relatedRows: Row[] @@ -2605,49 +2610,49 @@ describe.each([ }) }) - // !isInMemory && - describe("search by _id", () => { - let row: Row + !isInMemory && + describe("search by _id", () => { + let row: Row - beforeAll(async () => { - const toRelateTable = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - table = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - rel: { - name: "rel", - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_MANY, - tableId: toRelateTable._id!, - fieldName: "rel", - }, - }) - const [row1, row2] = await Promise.all([ - config.api.row.save(toRelateTable._id!, { name: "tag 1" }), - config.api.row.save(toRelateTable._id!, { name: "tag 2" }), - ]) - row = await config.api.row.save(table._id!, { - name: "product 1", - rel: [row1._id, row2._id], + beforeAll(async () => { + const toRelateTable = await createTable({ + name: { + name: "name", + type: FieldType.STRING, + }, + }) + table = await createTable({ + name: { + name: "name", + type: FieldType.STRING, + }, + rel: { + name: "rel", + type: FieldType.LINK, + relationshipType: RelationshipType.MANY_TO_MANY, + tableId: toRelateTable._id!, + fieldName: "rel", + }, + }) + const [row1, row2] = await Promise.all([ + config.api.row.save(toRelateTable._id!, { name: "tag 1" }), + config.api.row.save(toRelateTable._id!, { name: "tag 2" }), + ]) + row = await config.api.row.save(table._id!, { + name: "product 1", + rel: [row1._id, row2._id], + }) }) - }) - it("can filter by the row ID with limit 1", async () => { - await expectSearch({ - query: { - equal: { _id: row._id }, - }, - limit: 1, - }).toContainExactly([row]) + it("can filter by the row ID with limit 1", async () => { + await expectSearch({ + query: { + equal: { _id: row._id }, + }, + limit: 1, + }).toContainExactly([row]) + }) }) - }) !isInternal && describe("search by composite key", () => { From 2d6a8d9ff51c224177d7dcee2df44b4a920edfd5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Sep 2024 18:50:01 +0100 Subject: [PATCH 14/31] Fix for sorting, didn't account for some primitive types. --- packages/server/src/api/controllers/row/utils/basic.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 9d5a315628e..b754e288ed2 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -143,12 +143,16 @@ export function basicProcessing({ return relatedRow }) .sort((a, b) => { - if (!a?.[sortField]) { + const aField = a?.[sortField], + bField = b?.[sortField] + if (!aField) { return 1 - } else if (!b?.[sortField]) { + } else if (!bField) { return -1 } - return a[sortField].localeCompare(b[sortField]) + return aField.localeCompare + ? aField.localeCompare(bField) + : aField - bField }) } } From fed82dffaf0e62997d6d030e0df134b4e6071248 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 13:11:03 +0100 Subject: [PATCH 15/31] Linting. --- packages/backend-core/src/sql/sql.ts | 2 +- .../src/api/controllers/row/utils/sqlUtils.ts | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 6181d180467..0bc91269243 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -128,7 +128,7 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { endpoint, resource, meta, tableAliases } = this.query + const { endpoint, resource, tableAliases } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index d6e5c3e8f16..249bb43bbce 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -7,11 +7,9 @@ import { ManyToManyRelationshipFieldMetadata, RelationshipFieldMetadata, RelationshipsJson, - Row, Table, } from "@budibase/types" import { breakExternalTableId } from "../../../../integrations/utils" -import { basicProcessing } from "./basic" import { generateJunctionTableID } from "../../../../db/utils" type TableMap = Record @@ -22,20 +20,6 @@ export function isManyToMany( return !!(field as ManyToManyRelationshipFieldMetadata).through } -function isCorrectRelationship( - relationship: RelationshipsJson, - table1: Table, - table2: Table, - row: Row -): boolean { - const junctionTableId = generateJunctionTableID(table1._id!, table2._id!) - const possibleColumns = [ - `${junctionTableId}.doc1.fieldName`, - `${junctionTableId}.doc2.fieldName`, - ] - return !!possibleColumns.find(col => row[col] === relationship.column) -} - /** * Gets the list of relationship JSON structures based on the columns in the table, * this will be used by the underlying library to build whatever relationship mechanism From eefb1f01a3521a052f8b0c16ef6366053cbb6319 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 13:18:54 +0100 Subject: [PATCH 16/31] Fix for generic sql test. --- .../src/api/routes/tests/queries/generic-sql.spec.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index a84b243e2d7..4bd1951d674 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -832,10 +832,12 @@ describe.each( }, }) expect(res).toHaveLength(1) - expect(res[0]).toEqual({ - id: 2, - name: "two", - }) + expect(res[0]).toEqual( + expect.objectContaining({ + id: 2, + name: "two", + }) + ) }) // this parameter really only impacts SQL queries From 79de7b2c4578d82eba71e3643c48422e849548b0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 16:17:25 +0100 Subject: [PATCH 17/31] Updating to use a sub-query with a wrapper to get the JSON aggregations. --- packages/backend-core/src/sql/sql.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 0bc91269243..7b11f4ce37f 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -913,30 +913,26 @@ class InternalBuilder { const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") - let rawJsonArray: Knex.Raw, limit: number + let rawJsonArray: Knex.Raw switch (sqlClient) { case SqlClient.SQL_LITE: rawJsonArray = this.knex.raw( `json_group_array(json_object(${fieldList}))` ) - limit = getBaseLimit() break case SqlClient.POSTGRES: rawJsonArray = this.knex.raw( `json_agg(json_build_object(${fieldList}))` ) - limit = 1 break case SqlClient.MY_SQL: case SqlClient.ORACLE: rawJsonArray = this.knex.raw( `json_arrayagg(json_object(${fieldList}))` ) - limit = getBaseLimit() break case SqlClient.MS_SQL: rawJsonArray = this.knex.raw(`json_array(json_object(${fieldList}))`) - limit = 1 break default: throw new Error(`JSON relationships not implement for ${this.client}`) @@ -945,16 +941,12 @@ class InternalBuilder { // it reduces the result set rather than limiting how much data it filters over const primaryKey = `${toAlias}.${toPrimary || toKey}` let subQuery = this.knex - .select(rawJsonArray) + .select(`${toAlias}.*`) .from(toTableWithSchema) - .limit(limit) + .limit(getBaseLimit()) // add sorting to get consistent order .orderBy(primaryKey) - if (sqlClient === SqlClient.POSTGRES) { - subQuery = subQuery.groupBy(primaryKey) - } - // many-to-many relationship with junction table if (throughTable && toPrimary && fromPrimary) { const throughAlias = aliases?.[throughTable] || throughTable @@ -985,7 +977,11 @@ class InternalBuilder { if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, relationship) } - query = query.select({ [relationship.column]: subQuery }) + // @ts-ignore - the from alias syntax isn't in Knex typing + const wrapperQuery = this.knex.select(rawJsonArray).from({ + [toAlias]: subQuery, + }) + query = query.select({ [relationship.column]: wrapperQuery }) } return query } From 12db64513ba2597397daf2db9fedcab71d875c71 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 4 Sep 2024 16:21:32 +0100 Subject: [PATCH 18/31] Revert to testing against mssql 2017, attempt to get relationship aggreggation working. --- packages/backend-core/src/sql/sql.ts | 30 ++++++++++--------- packages/server/datasource-sha.env | 2 +- .../src/integrations/microsoftSqlServer.ts | 4 +-- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 0bc91269243..b51079ac2d9 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -913,30 +913,25 @@ class InternalBuilder { const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") - let rawJsonArray: Knex.Raw, limit: number + let select: Knex.Raw, limit: number switch (sqlClient) { case SqlClient.SQL_LITE: - rawJsonArray = this.knex.raw( - `json_group_array(json_object(${fieldList}))` - ) + select = this.knex.raw(`json_group_array(json_object(${fieldList}))`) limit = getBaseLimit() break case SqlClient.POSTGRES: - rawJsonArray = this.knex.raw( - `json_agg(json_build_object(${fieldList}))` - ) + select = this.knex.raw(`json_agg(json_build_object(${fieldList}))`) limit = 1 break case SqlClient.MY_SQL: case SqlClient.ORACLE: - rawJsonArray = this.knex.raw( - `json_arrayagg(json_object(${fieldList}))` - ) + select = this.knex.raw(`json_arrayagg(json_object(${fieldList}))`) limit = getBaseLimit() break case SqlClient.MS_SQL: - rawJsonArray = this.knex.raw(`json_array(json_object(${fieldList}))`) - limit = 1 + // Cursed, needs some code later instead + select = this.knex.raw(`*`) + limit = getBaseLimit() break default: throw new Error(`JSON relationships not implement for ${this.client}`) @@ -944,8 +939,8 @@ class InternalBuilder { // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over const primaryKey = `${toAlias}.${toPrimary || toKey}` - let subQuery = this.knex - .select(rawJsonArray) + let subQuery: Knex.QueryBuilder | Knex.Raw = this.knex + .select(select) .from(toTableWithSchema) .limit(limit) // add sorting to get consistent order @@ -985,6 +980,13 @@ class InternalBuilder { if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, relationship) } + + if (this.client === SqlClient.MS_SQL) { + subQuery = this.knex.raw( + `(SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH)` + ) + } + query = query.select({ [relationship.column]: subQuery }) } return query diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 9b935ed8eba..61249d530cd 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,4 +1,4 @@ -MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 +MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 88c75891e68..0a07371cd39 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -343,9 +343,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { err.number ) if (readableMessage) { - throw new Error(readableMessage) + throw new Error(readableMessage, { cause: err }) } else { - throw new Error(err.message as string) + throw new Error(err.message as string, { cause: err }) } } } From cda778598d8e4e9cb77f7a34bfc932b0ee4939ca Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 4 Sep 2024 16:41:36 +0100 Subject: [PATCH 19/31] Fix some MSSQL test cases. --- packages/backend-core/src/sql/sql.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 60cfdea9d98..b283ed21069 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -973,16 +973,18 @@ class InternalBuilder { subQuery = this.addJoinFieldCheck(subQuery, relationship) } + let wrapperQuery: Knex.QueryBuilder | Knex.Raw if (this.client === SqlClient.MS_SQL) { - subQuery = this.knex.raw( - `(SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH)` + wrapperQuery = this.knex.raw( + `(SELECT [${toAlias}] = (SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH))` ) + } else { + // @ts-ignore - the from alias syntax isn't in Knex typing + wrapperQuery = this.knex.select(select).from({ + [toAlias]: subQuery, + }) } - // @ts-ignore - the from alias syntax isn't in Knex typing - const wrapperQuery = this.knex.select(select).from({ - [toAlias]: subQuery, - }) query = query.select({ [relationship.column]: wrapperQuery }) } return query From 637ac55a9f87f10bf2f9e954667888cd68f81383 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 17:42:30 +0100 Subject: [PATCH 20/31] Slight refactor. --- packages/backend-core/src/sql/sql.ts | 73 +++++++++++++++------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b283ed21069..5dd94456f44 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -859,6 +859,7 @@ class InternalBuilder { relationships: RelationshipsJson[] ): Knex.QueryBuilder { const sqlClient = this.client + const knex = this.knex const { resource, tableAliases: aliases, endpoint } = this.query const fields = resource?.fields || [] const jsonField = (field: string) => { @@ -913,29 +914,10 @@ class InternalBuilder { const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") - let select: Knex.Raw - switch (sqlClient) { - case SqlClient.SQL_LITE: - select = this.knex.raw(`json_group_array(json_object(${fieldList}))`) - break - case SqlClient.POSTGRES: - select = this.knex.raw(`json_agg(json_build_object(${fieldList}))`) - break - case SqlClient.MY_SQL: - case SqlClient.ORACLE: - select = this.knex.raw(`json_arrayagg(json_object(${fieldList}))`) - break - case SqlClient.MS_SQL: - // Cursed, needs some code later instead - select = this.knex.raw(`*`) - break - default: - throw new Error(`JSON relationships not implement for ${this.client}`) - } // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over const primaryKey = `${toAlias}.${toPrimary || toKey}` - let subQuery: Knex.QueryBuilder | Knex.Raw = this.knex + let subQuery: Knex.QueryBuilder | Knex.Raw = knex .select(`${toAlias}.*`) .from(toTableWithSchema) .limit(getBaseLimit()) @@ -956,7 +938,7 @@ class InternalBuilder { .where( `${throughAlias}.${fromKey}`, "=", - this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) + knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) ) } // one-to-many relationship with foreign key @@ -964,26 +946,49 @@ class InternalBuilder { subQuery = subQuery.where( `${toAlias}.${toKey}`, "=", - this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) + knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) ) } - // need to check the junction table document is to the right column, this is just for SQS - if (this.client === SqlClient.SQL_LITE) { - subQuery = this.addJoinFieldCheck(subQuery, relationship) - } - - let wrapperQuery: Knex.QueryBuilder | Knex.Raw - if (this.client === SqlClient.MS_SQL) { - wrapperQuery = this.knex.raw( - `(SELECT [${toAlias}] = (SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH))` - ) - } else { + const standardWrap = (select: string): Knex.QueryBuilder => { // @ts-ignore - the from alias syntax isn't in Knex typing - wrapperQuery = this.knex.select(select).from({ + return knex.select(knex.raw(select)).from({ [toAlias]: subQuery, }) } + let wrapperQuery: Knex.QueryBuilder | Knex.Raw + switch (sqlClient) { + case SqlClient.SQL_LITE: + // need to check the junction table document is to the right column, this is just for SQS + subQuery = this.addJoinFieldCheck(subQuery, relationship) + wrapperQuery = standardWrap( + `json_group_array(json_object(${fieldList}))` + ) + break + case SqlClient.POSTGRES: + wrapperQuery = standardWrap( + `json_agg(json_build_object(${fieldList}))` + ) + break + case SqlClient.MY_SQL: + case SqlClient.ORACLE: + wrapperQuery = standardWrap( + `json_arrayagg(json_object(${fieldList}))` + ) + break + case SqlClient.MS_SQL: + wrapperQuery = knex.raw( + `(SELECT ${this.quote(toAlias)} = (${knex + .select(`${fromAlias}.*`) + // @ts-ignore - from alias syntax not TS supported + .from({ + [fromAlias]: subQuery, + })} FOR JSON PATH))` + ) + break + default: + throw new Error(`JSON relationships not implement for ${sqlClient}`) + } query = query.select({ [relationship.column]: wrapperQuery }) } From e30469ce16fec403b4ba1b2881842b5e913de890 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 18:14:24 +0100 Subject: [PATCH 21/31] Getting MariaDB to work again. --- packages/backend-core/src/sql/sql.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 5dd94456f44..ec8ebe01069 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -917,8 +917,7 @@ class InternalBuilder { // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over const primaryKey = `${toAlias}.${toPrimary || toKey}` - let subQuery: Knex.QueryBuilder | Knex.Raw = knex - .select(`${toAlias}.*`) + let subQuery: Knex.QueryBuilder = knex .from(toTableWithSchema) .limit(getBaseLimit()) // add sorting to get consistent order @@ -951,6 +950,7 @@ class InternalBuilder { } const standardWrap = (select: string): Knex.QueryBuilder => { + subQuery = subQuery.select(`${toAlias}.*`) // @ts-ignore - the from alias syntax isn't in Knex typing return knex.select(knex.raw(select)).from({ [toAlias]: subQuery, @@ -971,6 +971,10 @@ class InternalBuilder { ) break case SqlClient.MY_SQL: + wrapperQuery = subQuery.select( + knex.raw(`json_arrayagg(json_object(${fieldList}))`) + ) + break case SqlClient.ORACLE: wrapperQuery = standardWrap( `json_arrayagg(json_object(${fieldList}))` @@ -982,7 +986,7 @@ class InternalBuilder { .select(`${fromAlias}.*`) // @ts-ignore - from alias syntax not TS supported .from({ - [fromAlias]: subQuery, + [fromAlias]: subQuery.select(`${toAlias}.*`), })} FOR JSON PATH))` ) break From 7cdf8137c52d42419af2db13a344e6c256c7d9d5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Sep 2024 17:57:15 +0100 Subject: [PATCH 22/31] Fixing aliasing test cases. --- .../src/integrations/tests/sqlAlias.spec.ts | 91 ++++++++++--------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 6f34f4eb890..1ba37018dc3 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -32,8 +32,8 @@ function multiline(sql: string) { } describe("Captures of real examples", () => { - const limit = 5000 - const relationshipLimit = 100 + const baseLimit = 5000 + const primaryLimit = 100 function getJson(name: string): QueryJson { return require(join(__dirname, "sqlQueryJson", name)) as QueryJson @@ -42,7 +42,7 @@ describe("Captures of real examples", () => { describe("create", () => { it("should create a row with relationships", () => { const queryJson = getJson("createWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: ["A Street", 34, "London", "A", "B", "designer", 1990], sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") @@ -54,40 +54,42 @@ describe("Captures of real examples", () => { describe("read", () => { it("should handle basic retrieval with relationships", () => { const queryJson = getJson("basicFetchWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, limit], + bindings: [baseLimit, baseLimit, primaryLimit], sql: expect.stringContaining( - multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", - "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", - "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", - "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", - "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", - "b"."completed" as "b.completed", "b"."qaid" as "b.qaid"`) + multiline( + `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` + ) ), }) }) it("should handle filtering by relationship", () => { const queryJson = getJson("filterByRelationship.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, "assembling", limit], + bindings: [baseLimit, "assembling", primaryLimit], sql: expect.stringContaining( - multiline(`where COALESCE("b"."taskname" = $2, FALSE) - order by "a"."productname" asc nulls first, "a"."productid" asc limit $3`) + multiline( + `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" + where "c"."productid" = "a"."productid" and COALESCE("b"."taskname" = $2, FALSE)` + ) ), }) }) it("should handle fetching many to many relationships", () => { const queryJson = getJson("fetchManyToMany.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, limit], + bindings: [baseLimit, primaryLimit], sql: expect.stringContaining( - multiline(`left join "products_tasks" as "c" on "a"."productid" = "c"."productid" - left join "tasks" as "b" on "b"."taskid" = "c"."taskid" `) + multiline( + `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) + from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" + where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $1` + ) ), }) }) @@ -95,22 +97,21 @@ describe("Captures of real examples", () => { it("should handle enrichment of rows", () => { const queryJson = getJson("enrichRelationship.json") const filters = queryJson.filters?.oneOf?.taskid as number[] - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [...filters, limit, ...filters, limit], + bindings: [baseLimit, ...filters, baseLimit], sql: multiline( - `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", - "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid" - from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a" - left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid" - where "a"."taskid" in ($4, $5) order by "a"."taskid" asc limit $6` + `select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) + from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" + where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $1) as "b") as "products" + from "tasks" as "a" where "a"."taskid" in ($2, $3) order by "a"."taskid" asc limit $4` ), }) }) it("should manage query with many relationship filters", () => { const queryJson = getJson("manyRelationshipFilters.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) const filters = queryJson.filters const notEqualsValue = Object.values(filters?.notEqual!)[0] const rangeValue: { high?: string | number; low?: string | number } = @@ -119,17 +120,18 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [ - notEqualsValue, - relationshipLimit, + baseLimit, + baseLimit, + baseLimit, rangeValue.low, rangeValue.high, equalValue, - true, - limit, + notEqualsValue, + primaryLimit, ], sql: expect.stringContaining( multiline( - `where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE)` + `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $4 and $5)` ) ), }) @@ -139,17 +141,19 @@ describe("Captures of real examples", () => { describe("update", () => { it("should handle performing a simple update", () => { const queryJson = getJson("updateSimple.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], - sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, - "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`), + sql: multiline( + `update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, + "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *` + ), }) }) it("should handle performing an update of relationships", () => { const queryJson = getJson("updateRelationship.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, @@ -161,12 +165,12 @@ describe("Captures of real examples", () => { describe("delete", () => { it("should handle deleting with relationships", () => { const queryJson = getJson("deleteSimple.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: ["ddd", ""], sql: multiline(`delete from "compositetable" as "a" where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE) - returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`), + returning "a".*`), }) }) }) @@ -174,7 +178,7 @@ describe("Captures of real examples", () => { describe("returning (everything bar Postgres)", () => { it("should be able to handle row returning", () => { const queryJson = getJson("createSimple.json") - const SQL = new Sql(SqlClient.MS_SQL, limit) + const SQL = new Sql(SqlClient.MS_SQL, baseLimit) let query = SQL._query(queryJson, { disableReturning: true }) expect(query).toEqual({ sql: "insert into [people] ([age], [name]) values (@p0, @p1)", @@ -187,10 +191,11 @@ describe("Captures of real examples", () => { returningQuery = input }, queryJson) expect(returningQuery).toEqual({ - sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2 - THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people] - where CASE WHEN [people].[name] = @p4 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p5 THEN 1 ELSE 0 END = 1`), - bindings: [5000, 1, "Test", 22, "Test", 22], + sql: multiline( + `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 + and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc` + ), + bindings: [1, "Test", 22], }) }) }) From 888c4214bdbef1c5202144ff1c9ca0449e07072e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Sep 2024 18:12:53 +0100 Subject: [PATCH 23/31] Fixing SQL unit tests. --- .../server/src/integrations/tests/sql.spec.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index a6e63c434d9..c434ec42cb6 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -160,16 +160,16 @@ describe("SQL query builder", () => { it("should add the schema to the LEFT JOIN", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" order by "test"."id" asc limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`, }) }) it("should handle if the schema is not present when doing a LEFT JOIN", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" order by "test"."id" asc limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`, }) }) @@ -178,8 +178,8 @@ describe("SQL query builder", () => { generateManyRelationshipJson({ schema: "production" }) ) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" order by "test"."id" asc limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`, }) }) @@ -194,8 +194,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: ["john%", limit, "john%", 5000], - sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`, + bindings: ["john%", limit], + sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -210,8 +210,8 @@ describe("SQL query builder", () => { ) const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`] expect(query).toEqual({ - bindings: [...filterSet, limit, ...filterSet, 5000], - sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" where COALESCE(LOWER("test"."age"), '') LIKE :6 AND COALESCE(LOWER("test"."age"), '') LIKE :7 and COALESCE(LOWER("test"."name"), '') LIKE :8 AND COALESCE(LOWER("test"."name"), '') LIKE :9 order by "test"."id" asc) where rownum <= :10`, + bindings: [...filterSet, limit], + sql: `select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -224,8 +224,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: [`%jo%`, limit, `%jo%`, 5000], - sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`, + bindings: [`%jo%`, limit], + sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, }) }) @@ -242,8 +242,8 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ - bindings: ["John", limit, "John", 5000], - sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :3) order by "test"."id" asc) where rownum <= :4`, + bindings: ["John", limit], + sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`, }) }) @@ -260,8 +260,8 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ - bindings: ["John", limit, "John", 5000], - sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :3) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :4`, + bindings: ["John", limit], + sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2`, }) }) }) From f7d9b8a9b31da5ee8357e42398c5bf733d738e9c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Sep 2024 19:04:45 +0100 Subject: [PATCH 24/31] Updating select statement generation. --- packages/backend-core/src/sql/sql.ts | 142 ++++++++++++++------------- 1 file changed, 72 insertions(+), 70 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index ec8ebe01069..b1a2bc060ab 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -94,6 +94,23 @@ class InternalBuilder { }) } + // states the various situations in which we need a full mapped select statement + private readonly SPECIAL_SELECT_CASES = { + POSTGRES_MONEY: (field: FieldSchema | undefined) => { + return ( + this.client === SqlClient.POSTGRES && + field?.externalType?.includes("money") + ) + }, + MSSQL_DATES: (field: FieldSchema | undefined) => { + return ( + this.client === SqlClient.MS_SQL && + field?.type === FieldType.DATETIME && + field.timeOnly + ) + }, + } + get table(): Table { return this.query.meta.table } @@ -127,8 +144,20 @@ class InternalBuilder { .join(".") } + private isFullSelectStatementRequired(): boolean { + const { meta } = this.query + for (let column of Object.values(meta.table.schema)) { + if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) { + return true + } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) { + return true + } + } + return false + } + private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { endpoint, resource, tableAliases } = this.query + const { meta, endpoint, resource, tableAliases } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" @@ -137,75 +166,48 @@ class InternalBuilder { const alias = tableAliases?.[endpoint.entityId] ? tableAliases?.[endpoint.entityId] : endpoint.entityId - return [this.knex.raw(`${this.quote(alias)}.*`)] - // - // - // const schema = meta.table.schema - // return resource.fields.map(field => { - // const parts = field.split(/\./g) - // let table: string | undefined = undefined - // let column: string | undefined = undefined - // - // // Just a column name, e.g.: "column" - // if (parts.length === 1) { - // column = parts[0] - // } - // - // // A table name and a column name, e.g.: "table.column" - // if (parts.length === 2) { - // table = parts[0] - // column = parts[1] - // } - // - // // A link doc, e.g.: "table.doc1.fieldName" - // if (parts.length > 2) { - // table = parts[0] - // column = parts.slice(1).join(".") - // } - // - // if (!column) { - // throw new Error(`Invalid field name: ${field}`) - // } - // - // const columnSchema = schema[column] - // - // if ( - // this.client === SqlClient.POSTGRES && - // columnSchema?.externalType?.includes("money") - // ) { - // return this.knex.raw( - // `${this.quotedIdentifier( - // [table, column].join(".") - // )}::money::numeric as ${this.quote(field)}` - // ) - // } - // - // if ( - // this.client === SqlClient.MS_SQL && - // columnSchema?.type === FieldType.DATETIME && - // columnSchema.timeOnly - // ) { - // // Time gets returned as timestamp from mssql, not matching the expected - // // HH:mm format - // return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) - // } - // - // // There's at least two edge cases being handled in the expression below. - // // 1. The column name could start/end with a space, and in that case we - // // want to preseve that space. - // // 2. Almost all column names are specified in the form table.column, except - // // in the case of relationships, where it's table.doc1.column. In that - // // case, we want to split it into `table`.`doc1.column` for reasons that - // // aren't actually clear to me, but `table`.`doc1` breaks things with the - // // sample data tests. - // if (table) { - // return this.knex.raw( - // `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}` - // ) - // } else { - // return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`) - // } - // }) + const schema = meta.table.schema + if (!this.isFullSelectStatementRequired()) { + return [this.knex.raw(`${this.quote(alias)}.*`)] + } + // get just the fields for this table + return resource.fields + .map(field => { + const parts = field.split(/\./g) + let table: string | undefined = undefined + let column = parts[0] + + // Just a column name, e.g.: "column" + if (parts.length > 1) { + table = parts[0] + column = parts.slice(1).join(".") + } + + return { table, column, field } + }) + .filter(({ table }) => !table || table === alias) + .map(({ table, column, field }) => { + const columnSchema = schema[column] + + if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) { + return this.knex.raw( + `${this.quotedIdentifier( + [table, column].join(".") + )}::money::numeric as ${this.quote(field)}` + ) + } + + if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) { + // Time gets returned as timestamp from mssql, not matching the expected + // HH:mm format + return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) + } + + const quoted = table + ? `${this.quote(table)}.${this.quote(column)}` + : this.quote(field) + return this.knex.raw(quoted) + }) } // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, From f63c95e44c282743b31b80a0306feeeb90e30452 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 13:59:21 +0100 Subject: [PATCH 25/31] Adding SQL_MAX_RELATED_ROWS environment variable, defaults to 500, allows for 500 rows per relationship. --- packages/backend-core/src/environment.ts | 1 + packages/backend-core/src/sql/sql.ts | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 3d931a8c67c..6bef6efeb33 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -171,6 +171,7 @@ const environment = { // Couch/search SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE, SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, + SQL_MAX_RELATED_ROWS: process.env.MAX_RELATED_ROWS, // smtp SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, SMTP_USER: process.env.SMTP_USER, diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b1a2bc060ab..d8ad7918295 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -49,6 +49,13 @@ function getBaseLimit() { return envLimit || 5000 } +function getRelationshipLimit() { + const envLimit = environment.SQL_MAX_RELATED_ROWS + ? parseInt(environment.SQL_MAX_RELATED_ROWS) + : null + return envLimit || 500 +} + function getTableName(table?: Table): string | undefined { // SQS uses the table ID rather than the table name if ( @@ -921,7 +928,7 @@ class InternalBuilder { const primaryKey = `${toAlias}.${toPrimary || toKey}` let subQuery: Knex.QueryBuilder = knex .from(toTableWithSchema) - .limit(getBaseLimit()) + .limit(getRelationshipLimit()) // add sorting to get consistent order .orderBy(primaryKey) From 2fd5c1a99f9b976793bb1830f1fa7ee031db1bd5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 15:45:32 +0100 Subject: [PATCH 26/31] Handling JSON types within relationships, they need to be parsed as well. --- .../src/api/controllers/row/utils/basic.ts | 54 +++++++++---------- .../src/api/controllers/row/utils/utils.ts | 24 +++------ .../src/sdk/app/rows/search/internal/sqs.ts | 25 +++++---- 3 files changed, 47 insertions(+), 56 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index b754e288ed2..596cc90b208 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,5 +1,5 @@ // need to handle table name + field or just field, depending on if relationships used -import { FieldSchema, FieldType, Row, Table } from "@budibase/types" +import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -62,6 +62,22 @@ export function generateIdForRow( return generateRowIdField(idParts) } +function fixJsonTypes(row: Row, table: Table) { + for (let [fieldName, schema] of Object.entries(table.schema)) { + if (JsonTypes.includes(schema.type) && typeof row[fieldName] === "string") { + try { + row[fieldName] = JSON.parse(row[fieldName]) + } catch (err) { + if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) { + // couldn't convert back to array, ignore + delete row[fieldName] + } + } + } + } + return row +} + export function basicProcessing({ row, table, @@ -136,12 +152,15 @@ export function basicProcessing({ const sortField = relatedTable.primaryDisplay || relatedTable.primary![0]! thisRow[col] = (thisRow[col] as Row[]) - .map(relatedRow => { - relatedRow._id = relatedRow._id - ? relatedRow._id - : generateIdForRow(relatedRow, relatedTable) - return relatedRow - }) + .map(relatedRow => + basicProcessing({ + row: relatedRow, + table: relatedTable, + tables, + isLinked: false, + sqs, + }) + ) .sort((a, b) => { const aField = a?.[sortField], bField = b?.[sortField] @@ -157,24 +176,5 @@ export function basicProcessing({ } } } - return thisRow -} - -export function fixArrayTypes(row: Row, table: Table) { - for (let [fieldName, schema] of Object.entries(table.schema)) { - if ( - [FieldType.ARRAY, FieldType.BB_REFERENCE].includes(schema.type) && - typeof row[fieldName] === "string" - ) { - try { - row[fieldName] = JSON.parse(row[fieldName]) - } catch (err) { - if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) { - // couldn't convert back to array, ignore - delete row[fieldName] - } - } - } - } - return row + return fixJsonTypes(thisRow, table) } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 45f0cef0851..ac305e70b6d 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -14,12 +14,7 @@ import { processFormulas, } from "../../../../utilities/rowProcessor" import { isKnexEmptyReadResponse } from "./sqlUtils" -import { - basicProcessing, - generateIdForRow, - fixArrayTypes, - getInternalRowId, -} from "./basic" +import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic" import sdk from "../../../../sdk" import { processStringSync } from "@budibase/string-templates" import validateJs from "validate.js" @@ -149,16 +144,13 @@ export async function sqlOutputProcessing( rowId = generateIdForRow(row, table) row._id = rowId } - const thisRow = fixArrayTypes( - basicProcessing({ - row, - table, - tables: Object.values(tables), - isLinked: false, - sqs: opts?.sqs, - }), - table - ) + const thisRow = basicProcessing({ + row, + table, + tables: Object.values(tables), + isLinked: false, + sqs: opts?.sqs, + }) if (thisRow._id == null) { throw new Error("Unable to generate row ID for SQL rows") } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index b5e25e02ead..cf540a244b5 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -18,6 +18,7 @@ import { } from "@budibase/types" import { buildInternalRelationships, + fixJsonTypes, sqlOutputProcessing, } from "../../../../../api/controllers/row/utils" import sdk from "../../../../index" @@ -182,11 +183,20 @@ function buildTableMap(tables: Table[]) { return tableMap } -function reverseUserColumnMapping(rows: Row[]) { +// table is only needed to handle relationships +function reverseUserColumnMapping(rows: Row[], table?: Table) { const prefixLength = USER_COLUMN_PREFIX.length return rows.map(row => { const finalRow: Row = {} for (let key of Object.keys(row)) { + // handle relationships + if ( + table?.schema[key]?.type === FieldType.LINK && + typeof row[key] === "string" + ) { + // no table required, relationship rows don't contain relationships + row[key] = reverseUserColumnMapping(JSON.parse(row[key])) + } // it should be the first prefix const index = key.indexOf(USER_COLUMN_PREFIX) if (index !== -1) { @@ -261,7 +271,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { return processRowCountResponse(response) } else if (Array.isArray(response)) { - return reverseUserColumnMapping(response) + return reverseUserColumnMapping(response, json.meta.table) } return response } @@ -368,17 +378,6 @@ export async function search( }) ) - // make sure relationships have columns reversed correctly - for (let columnName of Object.keys(table.schema)) { - if (table.schema[columnName].type !== FieldType.LINK) { - continue - } - // process the relationships (JSON generated by SQS) - for (let row of processed) { - row[columnName] = reverseUserColumnMapping(row[columnName]) - } - } - // check for pagination final row let nextRow: boolean = false if (paginate && params.limit && rows.length > params.limit) { From d1b12b8d4a5c7a0c74c589597bb9aa73e0f7d163 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 15:52:41 +0100 Subject: [PATCH 27/31] Linting. --- packages/server/src/sdk/app/rows/search/internal/sqs.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index cf540a244b5..560a39e70aa 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -18,7 +18,6 @@ import { } from "@budibase/types" import { buildInternalRelationships, - fixJsonTypes, sqlOutputProcessing, } from "../../../../../api/controllers/row/utils" import sdk from "../../../../index" From 86a6664c84445861b76477b90871cfaf3e199843 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 16:06:53 +0100 Subject: [PATCH 28/31] Updating test case. --- .../src/integrations/tests/sqlAlias.spec.ts | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 1ba37018dc3..528e782543b 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -32,7 +32,7 @@ function multiline(sql: string) { } describe("Captures of real examples", () => { - const baseLimit = 5000 + const relationshipLimit = 500 const primaryLimit = 100 function getJson(name: string): QueryJson { @@ -42,7 +42,9 @@ describe("Captures of real examples", () => { describe("create", () => { it("should create a row with relationships", () => { const queryJson = getJson("createWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ bindings: ["A Street", 34, "London", "A", "B", "designer", 1990], sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") @@ -54,9 +56,11 @@ describe("Captures of real examples", () => { describe("read", () => { it("should handle basic retrieval with relationships", () => { const queryJson = getJson("basicFetchWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ - bindings: [baseLimit, baseLimit, primaryLimit], + bindings: [relationshipLimit, relationshipLimit, primaryLimit], sql: expect.stringContaining( multiline( `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` @@ -67,9 +71,11 @@ describe("Captures of real examples", () => { it("should handle filtering by relationship", () => { const queryJson = getJson("filterByRelationship.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ - bindings: [baseLimit, "assembling", primaryLimit], + bindings: [relationshipLimit, "assembling", primaryLimit], sql: expect.stringContaining( multiline( `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" @@ -81,9 +87,11 @@ describe("Captures of real examples", () => { it("should handle fetching many to many relationships", () => { const queryJson = getJson("fetchManyToMany.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ - bindings: [baseLimit, primaryLimit], + bindings: [relationshipLimit, primaryLimit], sql: expect.stringContaining( multiline( `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) @@ -97,9 +105,11 @@ describe("Captures of real examples", () => { it("should handle enrichment of rows", () => { const queryJson = getJson("enrichRelationship.json") const filters = queryJson.filters?.oneOf?.taskid as number[] - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ - bindings: [baseLimit, ...filters, baseLimit], + bindings: [relationshipLimit, ...filters, relationshipLimit], sql: multiline( `select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" @@ -111,7 +121,9 @@ describe("Captures of real examples", () => { it("should manage query with many relationship filters", () => { const queryJson = getJson("manyRelationshipFilters.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) const filters = queryJson.filters const notEqualsValue = Object.values(filters?.notEqual!)[0] const rangeValue: { high?: string | number; low?: string | number } = @@ -120,9 +132,9 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [ - baseLimit, - baseLimit, - baseLimit, + relationshipLimit, + relationshipLimit, + relationshipLimit, rangeValue.low, rangeValue.high, equalValue, @@ -141,7 +153,9 @@ describe("Captures of real examples", () => { describe("update", () => { it("should handle performing a simple update", () => { const queryJson = getJson("updateSimple.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline( @@ -153,7 +167,9 @@ describe("Captures of real examples", () => { it("should handle performing an update of relationships", () => { const queryJson = getJson("updateRelationship.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, @@ -165,7 +181,9 @@ describe("Captures of real examples", () => { describe("delete", () => { it("should handle deleting with relationships", () => { const queryJson = getJson("deleteSimple.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ bindings: ["ddd", ""], sql: multiline(`delete from "compositetable" as "a" @@ -178,7 +196,7 @@ describe("Captures of real examples", () => { describe("returning (everything bar Postgres)", () => { it("should be able to handle row returning", () => { const queryJson = getJson("createSimple.json") - const SQL = new Sql(SqlClient.MS_SQL, baseLimit) + const SQL = new Sql(SqlClient.MS_SQL, relationshipLimit) let query = SQL._query(queryJson, { disableReturning: true }) expect(query).toEqual({ sql: "insert into [people] ([age], [name]) values (@p0, @p1)", From 1582e3221faa8f7eac1d87e9eb9eb3e5a11622fa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 17:04:59 +0100 Subject: [PATCH 29/31] Adding test case for getting related array column in a JS formula. --- .../src/api/routes/tests/search.spec.ts | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index f5c5ade2f84..edac4c6662c 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -9,10 +9,10 @@ import { db as dbCore, MAX_VALID_DATE, MIN_VALID_DATE, + setEnv as setCoreEnv, SQLITE_DESIGN_DOC_ID, utils, withEnv as withCoreEnv, - setEnv as setCoreEnv, } from "@budibase/backend-core" import * as setup from "./utilities" @@ -1937,6 +1937,67 @@ describe.each([ }) }) + isSql && + describe("related formulas", () => { + beforeAll(async () => { + const arrayTable = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + array: { + name: "array", + type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["option 1", "option 2"], + }, + }, + }, + "array" + ) + table = await createTable( + { + relationship: { + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + name: "relationship", + fieldName: "relate", + tableId: arrayTable._id!, + constraints: { + type: "array", + }, + }, + formula: { + type: FieldType.FORMULA, + name: "formula", + formula: encodeJSBinding( + `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.join(",")` + ), + }, + }, + "main" + ) + const arrayRows = await Promise.all([ + config.api.row.save(arrayTable._id!, { + name: "foo", + array: ["option 1"], + }), + config.api.row.save(arrayTable._id!, { + name: "bar", + array: ["option 2"], + }), + ]) + await Promise.all([ + config.api.row.save(table._id!, { + relationship: [arrayRows[0]._id, arrayRows[1]._id], + }), + ]) + }) + + it("formula is correct with relationship arrays", async () => { + await expectQuery({}).toContain([{ formula: "option 1,option 2" }]) + }) + }) + describe("user", () => { let user1: User let user2: User From 9a61ec5950ecb63e89f586d59c73be14408a1d6b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 17:07:31 +0100 Subject: [PATCH 30/31] More incorrect limits. --- packages/server/src/integrations/tests/sql.spec.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index c434ec42cb6..f2bcc0bde0f 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -149,6 +149,7 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) { } describe("SQL query builder", () => { + const relationshipLimit = 500 const limit = 500 const client = SqlClient.POSTGRES let sql: any @@ -160,7 +161,7 @@ describe("SQL query builder", () => { it("should add the schema to the LEFT JOIN", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ - bindings: [5000, limit], + bindings: [relationshipLimit, limit], sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`, }) }) @@ -168,7 +169,7 @@ describe("SQL query builder", () => { it("should handle if the schema is not present when doing a LEFT JOIN", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ - bindings: [5000, limit], + bindings: [relationshipLimit, limit], sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`, }) }) @@ -178,7 +179,7 @@ describe("SQL query builder", () => { generateManyRelationshipJson({ schema: "production" }) ) expect(query).toEqual({ - bindings: [5000, limit], + bindings: [relationshipLimit, limit], sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`, }) }) From 595dd7ea6d31cd092f2adb7ccb0f54e3ffd86d3b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 17:21:36 +0100 Subject: [PATCH 31/31] Fix for test case. --- packages/server/src/api/routes/tests/search.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index edac4c6662c..26bb1582dd1 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1958,7 +1958,7 @@ describe.each([ { relationship: { type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, + relationshipType: RelationshipType.MANY_TO_ONE, name: "relationship", fieldName: "relate", tableId: arrayTable._id!, @@ -1970,7 +1970,7 @@ describe.each([ type: FieldType.FORMULA, name: "formula", formula: encodeJSBinding( - `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.join(",")` + `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")` ), }, },