From 50f0dd0b40f38eb5634263788010ed864c47d83e Mon Sep 17 00:00:00 2001 From: Etienne Deladonchamps Date: Mon, 29 Jan 2024 18:55:37 +0100 Subject: [PATCH] Add expr in subquery --- src/Expr.ts | 61 +++++++++++++-- src/TableQuery.ts | 51 ++++++------- src/utils/dependencies.ts | 46 ++++++++++++ src/utils/isStateEmpty.ts | 5 ++ tests/advanced.test.ts | 154 ++++++++++++++++++++++++++++++++++---- 5 files changed, 266 insertions(+), 51 deletions(-) create mode 100644 src/utils/dependencies.ts create mode 100644 src/utils/isStateEmpty.ts diff --git a/src/Expr.ts b/src/Expr.ts index bcdca7e..1ac622f 100644 --- a/src/Expr.ts +++ b/src/Expr.ts @@ -2,7 +2,9 @@ import type { Ast } from '@dldc/sqlite'; import { builder, Utils } from '@dldc/sqlite'; import { Datatype } from './Datatype'; import { Random } from './Random'; +import type { ITableQuery, ITableQueryDependency } from './TableQuery.types'; import { PRIV, TYPES } from './utils/constants'; +import { appendDependencies } from './utils/dependencies'; import { expectNever, mapObject } from './utils/functions'; import type { ExprResultFrom, ExprsNullables } from './utils/types'; @@ -32,6 +34,8 @@ export interface IExprInternal { // JsonExpr is transformed to JsonRef when converted to a ref // JsonRef is wrapped in a json() function when unsed in other json functions readonly jsonMode?: JsonMode; + // Used for X in (select X from ...) where the target is a CTE that needs to be defined + readonly dependencies?: Array; } export const Expr = (() => { @@ -64,6 +68,8 @@ export const Expr = (() => { isNull, inList, notInList, + inSubquery, + notInSubquery, compare, @@ -80,6 +86,7 @@ export const Expr = (() => { return create(builder.Expr.AggregateFunctions.count({ params: expr.ast }), { parse: Datatype.number.parse, nullable: false, + dependencies: expr[PRIV].dependencies, }); }, // Note: for the following functions, the result is always nullable because the result is null when the input is empty @@ -87,24 +94,28 @@ export const Expr = (() => { return create(builder.Expr.AggregateFunctions.avg({ params: expr.ast }), { parse: Datatype.number.parse, nullable: true, + dependencies: expr[PRIV].dependencies, }); }, sum: >(expr: Expr): IExpr => { return create(builder.Expr.AggregateFunctions.sum({ params: expr.ast }), { parse: Datatype.number.parse, nullable: true, + dependencies: expr[PRIV].dependencies, }); }, min: (expr: Expr): IExpr => { return create(builder.Expr.AggregateFunctions.min({ params: expr.ast }), { parse: expr[PRIV].parse, nullable: true, + dependencies: expr[PRIV].dependencies, }); }, max: (expr: Expr): IExpr => { return create(builder.Expr.AggregateFunctions.max({ params: expr.ast }), { parse: expr[PRIV].parse, nullable: true, + dependencies: expr[PRIV].dependencies, }); }, }, @@ -130,6 +141,7 @@ export const Expr = (() => { return create(builder.Expr.add(left.ast, right.ast), { parse: Datatype.number.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -140,6 +152,7 @@ export const Expr = (() => { return create(builder.Expr.equal(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -174,6 +187,7 @@ export const Expr = (() => { return create(builder.Expr.different(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -184,6 +198,7 @@ export const Expr = (() => { return create(builder.Expr.like(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -191,6 +206,7 @@ export const Expr = (() => { return create(builder.Expr.or(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -201,6 +217,7 @@ export const Expr = (() => { return create(builder.Expr.and(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -215,6 +232,7 @@ export const Expr = (() => { return create(builder.Expr.lowerThan(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -225,6 +243,7 @@ export const Expr = (() => { return create(builder.Expr.lowerThanOrEqual(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -235,6 +254,7 @@ export const Expr = (() => { return create(builder.Expr.greaterThan(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -245,6 +265,7 @@ export const Expr = (() => { return create(builder.Expr.greaterThanOrEqual(left.ast, right.ast), { parse: Datatype.boolean.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } @@ -255,17 +276,23 @@ export const Expr = (() => { return create(builder.Expr.concatenate(left.ast, right.ast), { parse: Datatype.text.parse, nullable: someNullable(left, right), + dependencies: mergeExprDependencies(left, right), }); } function isNull(expr: IExprUnknow): IExpr { - return create(builder.Expr.isNull(expr.ast), { parse: Datatype.boolean.parse, nullable: false }); + return create(builder.Expr.isNull(expr.ast), { + parse: Datatype.boolean.parse, + nullable: false, + dependencies: expr[PRIV].dependencies, + }); } function inList(left: IExprUnknow, items: IExprUnknow[]): IExpr { return create(builder.Expr.In.list(left.ast, Utils.arrayToNonEmptyArray(items.map((item) => item.ast))), { nullable: false, parse: Datatype.boolean.parse, + dependencies: mergeExprDependencies(left, ...items), }); } @@ -273,15 +300,31 @@ export const Expr = (() => { return create(builder.Expr.NotIn.list(left.ast, Utils.arrayToNonEmptyArray(items.map((item) => item.ast))), { nullable: false, parse: Datatype.boolean.parse, + dependencies: mergeExprDependencies(left, ...items), + }); + } + + function inSubquery>( + expr: IExprUnknow, + subquery: RTable, + ): IExpr { + return create(builder.Expr.In.tableName(expr.ast, subquery[PRIV].name), { + nullable: false, + parse: Datatype.boolean.parse, + dependencies: appendDependencies(expr[PRIV].dependencies ?? [], subquery[PRIV]), }); } - // function inSubquery>(expr: IExprUnknow, subquery: RTable): IExpr { - // return create(builder.Expr.In.select(left.ast, {}), { - // nullable: false, - // parse: Datatype.boolean.parse, - // }); - // } + function notInSubquery>( + expr: IExprUnknow, + subquery: RTable, + ): IExpr { + return create(builder.Expr.NotIn.tableName(expr.ast, subquery[PRIV].name), { + nullable: false, + parse: Datatype.boolean.parse, + dependencies: [subquery[PRIV]], + }); + } function external( val: Val, @@ -373,4 +416,8 @@ export const Expr = (() => { function someNullable(...exprs: IExprUnknow[]): boolean { return exprs.some((expr) => expr[PRIV].nullable); } + + function mergeExprDependencies(...exprs: IExprUnknow[]): ITableQueryDependency[] { + return exprs.flatMap((expr) => expr[PRIV].dependencies ?? []); + } })(); diff --git a/src/TableQuery.ts b/src/TableQuery.ts index 5b62f80..69f881d 100644 --- a/src/TableQuery.ts +++ b/src/TableQuery.ts @@ -15,13 +15,14 @@ import type { ITableQuery, ITableQueryDependency, ITableQueryInternal, - ITableQueryState, OrderingTerms, SelectFn, } from './TableQuery.types'; import { ZendbErreur } from './ZendbErreur'; import { PRIV, TYPES } from './utils/constants'; +import { appendDependencies, mergeDependencies } from './utils/dependencies'; import { mapObject } from './utils/functions'; +import { isStateEmpty } from './utils/isStateEmpty'; import { extractParams } from './utils/params'; import type { AnyRecord, ExprRecord, ExprRecordNested, ExprRecordOutput, FilterEqualCols } from './utils/types'; import { whereEqual } from './utils/whereEqual'; @@ -87,11 +88,12 @@ export const TableQuery = (() => { function where(whereFn: ColsFn): ITableQuery { const result = resolveColFn(whereFn)(internal.inputColsRefs); + const nextDependencies = mergeDependencies(internal.dependencies, result[PRIV].dependencies); if (internal.state.where) { const whereAnd = Expr.and(internal.state.where, result); - return create({ ...internal, state: { ...internal.state, where: whereAnd } }); + return create({ ...internal, dependencies: nextDependencies, state: { ...internal.state, where: whereAnd } }); } - return create({ ...internal, state: { ...internal.state, where: result } }); + return create({ ...internal, dependencies: nextDependencies, state: { ...internal.state, where: result } }); } function groupBy(groupFn: ColsFn>): ITableQuery { @@ -104,7 +106,11 @@ export const TableQuery = (() => { if (having === internal.state.having) { return self; } - return create({ ...internal, state: { ...internal.state, having } }); + return create({ + ...internal, + dependencies: mergeDependencies(internal.dependencies, having[PRIV].dependencies), + state: { ...internal.state, having }, + }); } function select( @@ -115,11 +121,12 @@ export const TableQuery = (() => { if ((nextOutputColsExprs as any) === internal.outputColsExprs) { return self as any; } - const { select, columnsRef } = resolvedColumns(internal.from, nextOutputColsExprs); + const { select, columnsRef, dependencies } = resolvedColumns(internal.from, nextOutputColsExprs); return create({ ...internal, outputColsRefs: nextOutputColsExprs, outputColsExprs: columnsRef as any, + dependencies: mergeDependencies(internal.dependencies, dependencies), state: { ...internal.state, select }, }); } @@ -190,7 +197,7 @@ export const TableQuery = (() => { ...internal.state, joins: [...(internal.state.joins ?? []), joinItem], }, - dependencies: mergeDependencies(internal.dependencies, table[PRIV]), + dependencies: appendDependencies(internal.dependencies, table[PRIV]), }); } @@ -222,7 +229,7 @@ export const TableQuery = (() => { ...internal.state, joins: [...(internal.state.joins ?? []), joinItem], }, - dependencies: mergeDependencies(internal.dependencies, table[PRIV]), + dependencies: appendDependencies(internal.dependencies, table[PRIV]), }); } @@ -385,7 +392,9 @@ export const TableQuery = (() => { : builder.From.Table(internal.from.name), resultColumns: state.select ? Utils.arrayToNonEmptyArray(state.select) : [builder.ResultColumn.Star()], where: state.where?.ast, - groupBy: state.groupBy ? { exprs: Utils.arrayToNonEmptyArray(state.groupBy.map((e) => e.ast)) } : undefined, + groupBy: state.groupBy + ? { exprs: Utils.arrayToNonEmptyArray(state.groupBy.map((e) => e.ast)), having: state.having?.ast } + : undefined, }, orderBy: state.orderBy ? Utils.arrayToNonEmptyArray(state.orderBy) : undefined, limit: state.limit @@ -401,12 +410,14 @@ export const TableQuery = (() => { function resolvedColumns( table: Ast.Identifier, selected: ExprRecord, - ): { select: Array>; columnsRef: ExprRecord } { + ): { select: Array>; columnsRef: ExprRecord; dependencies: ITableQueryDependency[] } { + let dependencies: ITableQueryDependency[] = []; const select = Object.entries(selected).map(([key, expr]): Ast.Node<'ResultColumn'> => { + dependencies = mergeDependencies(dependencies, expr[PRIV].dependencies); return builder.ResultColumn.Expr(expr.ast, key); }); const columnsRef = exprsToRefs(table, selected); - return { select, columnsRef }; + return { select, columnsRef, dependencies }; } function exprsToRefs(table: Ast.Identifier, exprs: ExprRecord): ExprRecord { @@ -444,24 +455,4 @@ export const TableQuery = (() => { state: {}, }); } - - function mergeDependencies( - prevDeps: Array, - table: ITableQueryInternal, - ): Array { - if (isStateEmpty(table.state)) { - if (table.dependencies.length === 0) { - // No state and no dependencies, this is a base table, we can skip it - return prevDeps; - } - // No state but has dependencies, we can just keep the dependencies - return [...prevDeps, ...table.dependencies]; - } - // Has state, we need to add it to the dependencies as well as all its dependencies - return [...prevDeps, ...table.dependencies, table]; - } - - function isStateEmpty(state: ITableQueryState): boolean { - return Object.values(state).every((v) => v === undefined); - } })(); diff --git a/src/utils/dependencies.ts b/src/utils/dependencies.ts new file mode 100644 index 0000000..5e77d2b --- /dev/null +++ b/src/utils/dependencies.ts @@ -0,0 +1,46 @@ +import type { ITableQueryDependency, ITableQueryInternal } from '../TableQuery.types'; +import { isStateEmpty } from './isStateEmpty'; + +export function appendDependencies( + prevDeps: Array, + table: ITableQueryInternal, +): Array { + if (isStateEmpty(table.state)) { + if (table.dependencies.length === 0) { + // No state and no dependencies, this is a base table, we can skip it + return prevDeps; + } + // No state but has dependencies, we can just keep the dependencies + return [...prevDeps, ...table.dependencies]; + } + // Has state, we need to add it to the dependencies as well as all its dependencies + return [...prevDeps, ...table.dependencies, table]; +} + +export function asTableDependency(table: ITableQueryInternal): Array { + if (isStateEmpty(table.state)) { + if (table.dependencies.length === 0) { + // No state and no dependencies, this is a base table, we can skip it + return []; + } + // No state but has dependencies, we can just keep the dependencies + return [...table.dependencies]; + } + return [...table.dependencies, table]; +} + +export function mergeDependencies( + left: Array | undefined, + right: Array | undefined, +): Array { + if (!left && !right) { + return []; + } + if (!left) { + return right!; + } + if (!right) { + return left; + } + return [...left, ...right]; +} diff --git a/src/utils/isStateEmpty.ts b/src/utils/isStateEmpty.ts new file mode 100644 index 0000000..728e0a1 --- /dev/null +++ b/src/utils/isStateEmpty.ts @@ -0,0 +1,5 @@ +import type { ITableQueryState } from '../TableQuery.types'; + +export function isStateEmpty(state: ITableQueryState): boolean { + return Object.values(state).every((v) => v === undefined); +} diff --git a/tests/advanced.test.ts b/tests/advanced.test.ts index 61bc58f..a6aa6b6 100644 --- a/tests/advanced.test.ts +++ b/tests/advanced.test.ts @@ -12,16 +12,7 @@ const db = TestDatabase.create(); beforeAll(() => { // disable random suffix for testing Random.setCreateId(() => `id${nextRandomId++}`); -}); - -beforeEach(() => { - nextRandomId = 0; -}); - -type UserInput = (typeof tasksDb)['users'] extends ITable ? Val : never; -type TaksInput = (typeof tasksDb)['tasks'] extends ITable ? Val : never; -test('Select ', () => { db.execMany(Database.schema(tasksDb)); const users: UserInput[] = [ @@ -46,6 +37,13 @@ test('Select ', () => { displayName: 'Jack', updatedAt: new Date('2023-12-24T22:30:12.250Z'), }, + { + id: '4', + name: 'Jill Doe', + email: 'jill@example.com', + displayName: 'Jill', + updatedAt: new Date('2023-12-24T22:30:12.250Z'), + }, ]; users.forEach((user) => db.exec(tasksDb.users.insert(user))); @@ -63,7 +61,19 @@ test('Select ', () => { db.exec(tasksDb.users_tasks.insert({ user_id: '1', task_id: '1' })); db.exec(tasksDb.users_tasks.insert({ user_id: '1', task_id: '2' })); db.exec(tasksDb.users_tasks.insert({ user_id: '2', task_id: '3' })); + db.exec(tasksDb.users_tasks.insert({ user_id: '3', task_id: '1' })); + nextRandomId = 0; +}); + +beforeEach(() => { + nextRandomId = 0; +}); + +type UserInput = (typeof tasksDb)['users'] extends ITable ? Val : never; +type TaksInput = (typeof tasksDb)['tasks'] extends ITable ? Val : never; + +test('Find all user with their linked tasks', () => { const allUsers = tasksDb.users.query(); const tasksByUserId = tasksDb.users_tasks .query() @@ -99,13 +109,14 @@ test('Select ', () => { expect(tasksByUserIdResult).toEqual([ { + userId: '1', tasks: [ { completed: false, description: 'First Task', id: '1', title: 'First Task' }, { completed: true, description: 'Second Task', id: '2', title: 'Second Task' }, ], - userId: '1', }, - { tasks: [{ completed: true, description: 'Third Task', id: '3', title: 'Third Task' }], userId: '2' }, + { userId: '2', tasks: [{ completed: true, description: 'Third Task', id: '3', title: 'Third Task' }] }, + { userId: '3', tasks: [{ completed: false, description: 'First Task', id: '1', title: 'First Task' }] }, ]); const query = allUsers @@ -115,7 +126,7 @@ test('Select ', () => { expect(format(query.sql)).toEqual(sql` WITH - cte_id43 AS ( + cte_id2 AS ( SELECT users_tasks.user_id AS userId, json_group_array( @@ -142,10 +153,10 @@ test('Select ', () => { users.email AS email, users.displayName AS displayName, users.updatedAt AS updatedAt, - cte_id43.tasks AS tasks + cte_id2.tasks AS tasks FROM users - LEFT JOIN cte_id43 ON users.id == cte_id43.userId + LEFT JOIN cte_id2 ON users.id == cte_id2.userId `); const result = db.exec(query); @@ -175,8 +186,123 @@ test('Select ', () => { email: 'jack@example.com', id: '3', name: 'Jack Doe', + tasks: [{ completed: false, description: 'First Task', id: '1', title: 'First Task' }], + updatedAt: new Date('2023-12-24T22:30:12.250Z'), + }, + { + displayName: 'Jill', + email: 'jill@example.com', + id: '4', + name: 'Jill Doe', tasks: null, updatedAt: new Date('2023-12-24T22:30:12.250Z'), }, ]); }); + +test('Find all users with only task 1 & 2 using subquery in expression', () => { + const subQuery = tasksDb.users_tasks + .query() + .where((c) => Expr.inList(c.task_id, [Expr.literal('1'), Expr.literal('2')])) + .groupBy((c) => [c.user_id]) + .select((c) => ({ id: c.user_id })) + .having((c) => Expr.equal(Expr.AggregateFunctions.count(c.task_id), Expr.literal(2))); + + const subQueryOp = subQuery.all(); + + expect(format(subQueryOp.sql)).toEqual(sql` + SELECT + users_tasks.user_id AS id + FROM + users_tasks + WHERE + users_tasks.task_id IN ('1', '2') + GROUP BY + users_tasks.user_id + HAVING + count(users_tasks.task_id) == 2 + `); + + const subQueryRes = db.exec(subQueryOp); + expect(subQueryRes).toEqual([{ id: '1' }]); + + const filteredUsers = tasksDb.users + .query() + .where((c) => Expr.inSubquery(c.id, subQuery)) + .all(); + + expect(format(filteredUsers.sql)).toEqual(sql` + WITH + cte_id3 AS ( + SELECT + users_tasks.user_id AS id + FROM + users_tasks + WHERE + users_tasks.task_id IN ('1', '2') + GROUP BY + users_tasks.user_id + HAVING + count(users_tasks.task_id) == 2 + ) + SELECT + * + FROM + users + WHERE + users.id IN cte_id3 + `); + + const result = db.exec(filteredUsers); + expect(result).toEqual([ + { + displayName: null, + email: 'john@exmaple.com', + id: '1', + name: 'John Doe', + updatedAt: new Date('2023-12-24T22:30:12.250Z'), + }, + ]); +}); + +test('Find all users with no tasks', () => { + const usersWithTasks = tasksDb.users_tasks + .query() + .groupBy((c) => [c.user_id]) + .select((c) => ({ id: c.user_id })); + + const usersWithNoTasks = tasksDb.users + .query() + .where((c) => Expr.notInSubquery(c.id, usersWithTasks)) + .all(); + + expect(format(usersWithNoTasks.sql)).toEqual(sql` + WITH + cte_id1 AS ( + SELECT + users_tasks.user_id AS id + FROM + users_tasks + GROUP BY + users_tasks.user_id + ) + SELECT + * + FROM + users + WHERE + users.id NOT IN cte_id1 + `); + + const result = db.exec(usersWithNoTasks); + + expect(result).toEqual([ + { + displayName: 'Jill', + email: 'jill@example.com', + id: '4', + name: 'Jill Doe', + updatedAt: new Date('2023-12-24T22:30:12.250Z'), + }, + ]); +});