diff --git a/internal-packages/tsql/NOTICE.md b/internal-packages/tsql/NOTICE.md new file mode 100644 index 000000000..efb4563d7 --- /dev/null +++ b/internal-packages/tsql/NOTICE.md @@ -0,0 +1,22 @@ +Portions of this package are derived from PostHog (MIT License). +Copyright (c) 2020-2025 PostHog Inc. + +The original license is reproduced below: + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index bc8c521db..84a17bc91 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -1,5 +1,4 @@ import { Logger } from "@trigger.dev/core/logger"; -export { TQuery } from "./query/TQuery.js"; -export type { TQueryOptions } from "./query/TQuery.js"; -export type { QueryConfig } from "./query/QueryConfig.js"; +export { TQuery } from "./query/query.js"; +export type { TQueryOptions } from "./query/query.js"; diff --git a/internal-packages/tsql/src/query/ClickHouseQueryVisitor.ts b/internal-packages/tsql/src/query/ClickHouseQueryVisitor.ts deleted file mode 100644 index 0cdf26748..000000000 --- a/internal-packages/tsql/src/query/ClickHouseQueryVisitor.ts +++ /dev/null @@ -1,536 +0,0 @@ -import { ParserRuleContext } from "antlr4ts/ParserRuleContext"; -import { ErrorNode } from "antlr4ts/tree/ErrorNode"; -import { ParseTree } from "antlr4ts/tree/ParseTree"; -import { TerminalNode } from "antlr4ts/tree/TerminalNode"; -import { - ArrayJoinClauseContext, - ColumnExprContext, - ColumnExprListContext, - FromClauseContext, - GroupByClauseContext, - HavingClauseContext, - JoinExprContext, - LimitAndOffsetClauseContext, - LimitByClauseContext, - OffsetOnlyClauseContext, - OrderByClauseContext, - PlaceholderContext, - PrewhereClauseContext, - SelectContext, - SelectSetStmtContext, - SelectStmtContext, - SelectStmtWithParensContext, - SettingsClauseContext, - TopClauseContext, - WhereClauseContext, - WindowClauseContext, - WithClauseContext, -} from "../grammar/TSQLParser.js"; -import { TSQLParserVisitor } from "../grammar/TSQLParserVisitor.js"; -import { QueryConfig } from "./QueryConfig.js"; - -/** - * Visitor that converts TSQL AST to a QueryConfig - * The QueryConfig can then be used to build a ClickhouseQueryBuilder - */ -export class ClickHouseQueryVisitor implements TSQLParserVisitor { - visitSelect(ctx: SelectContext): QueryConfig { - const selectSetStmt = ctx.selectSetStmt(); - if (selectSetStmt) { - return this.visitSelectSetStmt(selectSetStmt); - } - - const selectStmt = ctx.selectStmt(); - if (selectStmt) { - return this.visitSelectStmt(selectStmt); - } - - // Handle tSQLxTagElement if needed - return empty config - return { - baseQuery: "", - whereClauses: [], - }; - } - - visitSelectSetStmt(ctx: SelectSetStmtContext): QueryConfig { - const selectStmtWithParens = ctx.selectStmtWithParens(); - let config = this.visitSelectStmtWithParens(selectStmtWithParens); - - // Handle subsequent select set clauses (UNION, EXCEPT, INTERSECT) - // For now, we'll convert the config back to SQL for UNION operations - // This is a limitation - UNION queries can't use the query builder pattern easily - const subsequentClauses = ctx.subsequentSelectSetClause(); - if (subsequentClauses.length > 0) { - // Convert config to SQL for UNION operations - let query = this.configToSql(config); - for (const clause of subsequentClauses) { - const op = clause.EXCEPT() - ? "EXCEPT" - : clause.UNION() - ? clause.ALL() - ? "UNION ALL" - : clause.DISTINCT() - ? "UNION DISTINCT" - : "UNION" - : clause.INTERSECT() - ? clause.DISTINCT() - ? "INTERSECT DISTINCT" - : "INTERSECT" - : ""; - - if (op) { - const nextConfig = this.visitSelectStmtWithParens(clause.selectStmtWithParens()); - query += ` ${op} ${this.configToSql(nextConfig)}`; - } - } - // Return as a base query (can't use query builder features with UNION) - return { - baseQuery: query, - whereClauses: [], - }; - } - - return config; - } - - visitSelectStmtWithParens(ctx: SelectStmtWithParensContext): QueryConfig { - const selectStmt = ctx.selectStmt(); - if (selectStmt) { - return this.visitSelectStmt(selectStmt); - } - - const selectSetStmt = ctx.selectSetStmt(); - if (selectSetStmt) { - const config = this.visitSelectSetStmt(selectSetStmt); - // Wrap in parentheses for subquery - return { - ...config, - baseQuery: `(${this.configToSql(config)})`, - whereClauses: [], // Subqueries don't contribute to outer WHERE - }; - } - - // Handle placeholder if needed - const placeholder = ctx.placeholder(); - if (placeholder) { - const placeholderConfig = this.visitPlaceholder(placeholder); - return placeholderConfig; - } - - return { - baseQuery: this.getTextFromContext(ctx), - whereClauses: [], - }; - } - - visitPlaceholder(ctx: PlaceholderContext): QueryConfig { - return { - baseQuery: this.getTextFromContext(ctx), - whereClauses: [], - }; - } - - visitSelectStmt(ctx: SelectStmtContext): QueryConfig { - const config: QueryConfig = { - baseQuery: "", - whereClauses: [], - }; - - const parts: string[] = []; - - // WITH clause - const withClause = ctx.withClause(); - if (withClause) { - parts.push(this.visitWithClauseString(withClause)); - } - - // SELECT - parts.push("SELECT"); - - // DISTINCT - if (ctx.DISTINCT()) { - parts.push("DISTINCT"); - } - - // TOP clause - const topClause = ctx.topClause(); - if (topClause) { - parts.push(this.visitTopClauseString(topClause)); - } - - // Column list - const columnExprList = ctx.columnExprList(); - if (columnExprList) { - parts.push(this.visitColumnExprListString(columnExprList)); - } - - // FROM clause - const fromClause = ctx.fromClause(); - if (fromClause) { - parts.push(this.visitFromClauseString(fromClause)); - } - - // Array JOIN - const arrayJoinClause = ctx.arrayJoinClause(); - if (arrayJoinClause) { - parts.push(this.visitArrayJoinClauseString(arrayJoinClause)); - } - - // PREWHERE - const prewhereClause = ctx.prewhereClause(); - if (prewhereClause) { - parts.push(this.visitPrewhereClauseString(prewhereClause)); - } - - // Base query is everything up to WHERE - config.baseQuery = parts.join(" "); - - // WHERE - extract to whereClauses array - const whereClause = ctx.whereClause(); - if (whereClause) { - const whereExpr = this.visitWhereClauseString(whereClause); - // Remove "WHERE " prefix - const whereCondition = whereExpr.replace(/^WHERE\s+/i, ""); - config.whereClauses.push({ - clause: whereCondition, - }); - } - - // GROUP BY - const groupByClause = ctx.groupByClause(); - if (groupByClause) { - const groupByText = this.visitGroupByClauseString(groupByClause); - // Remove "GROUP BY " prefix - config.groupBy = groupByText.replace(/^GROUP\s+BY\s+/i, ""); - } - - // HAVING - add as WHERE clause (ClickHouse doesn't distinguish) - const havingClause = ctx.havingClause(); - if (havingClause) { - const havingText = this.visitHavingClauseString(havingClause); - // Remove "HAVING " prefix - const havingCondition = havingText.replace(/^HAVING\s+/i, ""); - config.whereClauses.push({ - clause: havingCondition, - }); - } - - // WINDOW - const windowClause = ctx.windowClause(); - if (windowClause) { - const windowText = this.visitWindowClauseString(windowClause); - // Append to base query - config.baseQuery += " " + windowText; - } - - // ORDER BY - const orderByClause = ctx.orderByClause(); - if (orderByClause) { - const orderByText = this.visitOrderByClauseString(orderByClause); - // Remove "ORDER BY " prefix - config.orderBy = orderByText.replace(/^ORDER\s+BY\s+/i, ""); - } - - // LIMIT BY - const limitByClause = ctx.limitByClause(); - if (limitByClause) { - const limitByText = this.visitLimitByClauseString(limitByClause); - // Append to base query (LIMIT BY is not supported by query builder) - config.baseQuery += " " + limitByText; - } - - // LIMIT / OFFSET - const limitAndOffsetClause = ctx.limitAndOffsetClause(); - if (limitAndOffsetClause) { - const limitText = this.visitLimitAndOffsetClauseString(limitAndOffsetClause); - // Try to extract limit number - const limitMatch = limitText.match(/LIMIT\s+(\d+)/i); - if (limitMatch) { - config.limit = parseInt(limitMatch[1], 10); - } else { - // If we can't parse it, append to base query - config.baseQuery += " " + limitText; - } - } - - const offsetOnlyClause = ctx.offsetOnlyClause(); - if (offsetOnlyClause) { - const offsetText = this.visitOffsetOnlyClauseString(offsetOnlyClause); - // Append to base query (OFFSET is not directly supported by query builder) - config.baseQuery += " " + offsetText; - } - - // SETTINGS - const settingsClause = ctx.settingsClause(); - if (settingsClause) { - const settingsText = this.visitSettingsClauseString(settingsClause); - // Append to base query - config.baseQuery += " " + settingsText; - } - - return config; - } - - // Helper methods that return strings for building SQL fragments - // These are private and used internally, not part of the visitor interface - private visitColumnExprListString(ctx: ColumnExprListContext): string { - const exprs: string[] = []; - const columnExprs = ctx.columnExpr(); - for (const expr of columnExprs) { - exprs.push(this.visitColumnExprString(expr)); - } - return exprs.join(", "); - } - - private visitColumnExprString(ctx: ColumnExprContext): string { - return this.getTextFromContext(ctx); - } - - private visitFromClauseString(ctx: FromClauseContext): string { - const joinExpr = ctx.joinExpr(); - return `FROM ${this.visitJoinExprString(joinExpr)}`; - } - - private visitJoinExprString(ctx: JoinExprContext): string { - return this.getTextFromContext(ctx); - } - - private visitWhereClauseString(ctx: WhereClauseContext): string { - const columnExpr = ctx.columnExpr(); - return `WHERE ${this.visitColumnExprString(columnExpr)}`; - } - - private visitGroupByClauseString(ctx: GroupByClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitHavingClauseString(ctx: HavingClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitOrderByClauseString(ctx: OrderByClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitLimitByClauseString(ctx: LimitByClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitLimitAndOffsetClauseString(ctx: LimitAndOffsetClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitOffsetOnlyClauseString(ctx: OffsetOnlyClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitSettingsClauseString(ctx: SettingsClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitWithClauseString(ctx: WithClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitTopClauseString(ctx: TopClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitArrayJoinClauseString(ctx: ArrayJoinClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitPrewhereClauseString(ctx: PrewhereClauseContext): string { - return this.getTextFromContext(ctx); - } - - private visitWindowClauseString(ctx: WindowClauseContext): string { - return this.getTextFromContext(ctx); - } - - // Interface methods that return QueryConfig (optional, so we can skip some) - visitColumnExprList(ctx: ColumnExprListContext): QueryConfig { - return { - baseQuery: this.visitColumnExprListString(ctx), - whereClauses: [], - }; - } - - visitColumnExpr(ctx: ColumnExprContext): QueryConfig { - return { - baseQuery: this.visitColumnExprString(ctx), - whereClauses: [], - }; - } - - visitFromClause(ctx: FromClauseContext): QueryConfig { - return { - baseQuery: this.visitFromClauseString(ctx), - whereClauses: [], - }; - } - - visitJoinExpr(ctx: JoinExprContext): QueryConfig { - return { - baseQuery: this.visitJoinExprString(ctx), - whereClauses: [], - }; - } - - visitWhereClause(ctx: WhereClauseContext): QueryConfig { - return { - baseQuery: this.visitWhereClauseString(ctx), - whereClauses: [], - }; - } - - visitGroupByClause(ctx: GroupByClauseContext): QueryConfig { - return { - baseQuery: this.visitGroupByClauseString(ctx), - whereClauses: [], - }; - } - - visitHavingClause(ctx: HavingClauseContext): QueryConfig { - return { - baseQuery: this.visitHavingClauseString(ctx), - whereClauses: [], - }; - } - - visitOrderByClause(ctx: OrderByClauseContext): QueryConfig { - return { - baseQuery: this.visitOrderByClauseString(ctx), - whereClauses: [], - }; - } - - visitLimitByClause(ctx: LimitByClauseContext): QueryConfig { - return { - baseQuery: this.visitLimitByClauseString(ctx), - whereClauses: [], - }; - } - - visitLimitAndOffsetClause(ctx: LimitAndOffsetClauseContext): QueryConfig { - return { - baseQuery: this.visitLimitAndOffsetClauseString(ctx), - whereClauses: [], - }; - } - - visitOffsetOnlyClause(ctx: OffsetOnlyClauseContext): QueryConfig { - return { - baseQuery: this.visitOffsetOnlyClauseString(ctx), - whereClauses: [], - }; - } - - visitSettingsClause(ctx: SettingsClauseContext): QueryConfig { - return { - baseQuery: this.visitSettingsClauseString(ctx), - whereClauses: [], - }; - } - - visitWithClause(ctx: WithClauseContext): QueryConfig { - return { - baseQuery: this.visitWithClauseString(ctx), - whereClauses: [], - }; - } - - visitTopClause(ctx: TopClauseContext): QueryConfig { - return { - baseQuery: this.visitTopClauseString(ctx), - whereClauses: [], - }; - } - - visitArrayJoinClause(ctx: ArrayJoinClauseContext): QueryConfig { - return { - baseQuery: this.visitArrayJoinClauseString(ctx), - whereClauses: [], - }; - } - - visitPrewhereClause(ctx: PrewhereClauseContext): QueryConfig { - return { - baseQuery: this.visitPrewhereClauseString(ctx), - whereClauses: [], - }; - } - - visitWindowClause(ctx: WindowClauseContext): QueryConfig { - return { - baseQuery: this.visitWindowClauseString(ctx), - whereClauses: [], - }; - } - - /** - * Convert a QueryConfig back to SQL string - * Used for UNION operations and subqueries - */ - private configToSql(config: QueryConfig): string { - let query = config.baseQuery; - if (config.whereClauses.length > 0) { - const clauses = config.whereClauses.map((w) => w.clause); - query += " WHERE " + clauses.join(" AND "); - } - if (config.groupBy) { - query += ` GROUP BY ${config.groupBy}`; - } - if (config.orderBy) { - query += ` ORDER BY ${config.orderBy}`; - } - if (config.limit !== undefined) { - query += ` LIMIT ${config.limit}`; - } - return query; - } - - /** - * Extract the original text from a parse tree context - * This uses the text property which contains the original input text - */ - private getTextFromContext(ctx: ParserRuleContext): string { - return (ctx as any).text || ""; - } - - // Required by ParseTreeVisitor interface - visit(tree: ParseTree): QueryConfig { - // For generic parse trees, return empty config - return { - baseQuery: (tree as any).text || "", - whereClauses: [], - }; - } - - visitChildren(node: ParserRuleContext): QueryConfig { - // Visit all children and combine their configs - if (!node.children || node.children.length === 0) { - return this.visit(node); - } - // For now, just return the text representation - return { - baseQuery: (node as any).text || "", - whereClauses: [], - }; - } - - // Visit terminal nodes - visitTerminal(node: TerminalNode): QueryConfig { - return { - baseQuery: node.text, - whereClauses: [], - }; - } - - visitErrorNode(node: ErrorNode): QueryConfig { - return { - baseQuery: "", - whereClauses: [], - }; - } -} diff --git a/internal-packages/tsql/src/query/QueryConfig.ts b/internal-packages/tsql/src/query/QueryConfig.ts deleted file mode 100644 index 1ba2fab75..000000000 --- a/internal-packages/tsql/src/query/QueryConfig.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { QueryParams } from "@internal/clickhouse/client/queryBuilder.js"; - -/** - * Configuration object that represents a parsed TSQL query - * This can be used to build a ClickhouseQueryBuilder - * The structure matches ClickhouseQueryBuilder's API - */ -export interface QueryConfig { - /** - * The base SELECT query without WHERE, GROUP BY, ORDER BY, LIMIT clauses - * Example: "SELECT id, name FROM users" - */ - baseQuery: string; - - /** - * WHERE clause conditions - * Each entry represents a call to queryBuilder.where(clause, params) - * The clauses will be joined with AND by the query builder - */ - whereClauses: Array<{ - clause: string; - params?: QueryParams; - }>; - - /** - * GROUP BY clause string - * Will be passed to queryBuilder.groupBy() - */ - groupBy?: string; - - /** - * ORDER BY clause string - * Will be passed to queryBuilder.orderBy() - */ - orderBy?: string; - - /** - * LIMIT value - * Will be passed to queryBuilder.limit() - */ - limit?: number; -} diff --git a/internal-packages/tsql/src/query/ast.ts b/internal-packages/tsql/src/query/ast.ts new file mode 100644 index 000000000..a78b76edf --- /dev/null +++ b/internal-packages/tsql/src/query/ast.ts @@ -0,0 +1,601 @@ +// TypeScript translation of posthog/hogql/ast.py +// Keep this file in sync with the Python version + +import type { HogQLContext } from "./context"; +import type { + DatabaseField, + ExpressionField, + FieldOrTable, + FieldTraverser, + LazyJoin, + LazyTable, + StringArrayDatabaseField, + StringJSONDatabaseField, + Table, + UnknownDatabaseField, + VirtualTable, +} from "./models"; +import type { ConstantDataType, HogQLQuerySettings } from "./constants"; + +// Base types +export interface AST { + start?: number; + end?: number; + accept?(visitor: any): any; +} + +export interface Type extends AST { + get_child?(name: string, context: HogQLContext): Type; + has_child?(name: string, context: HogQLContext): boolean; + resolve_constant_type?(context: HogQLContext): ConstantType; + resolve_column_constant_type?(name: string, context: HogQLContext): ConstantType; +} + +export interface Expr extends AST { + type?: Type; +} + +export interface ConstantType extends Type { + data_type: ConstantDataType; + nullable?: boolean; + print_type?(): string; +} + +export interface UnknownType extends ConstantType { + data_type: "unknown"; +} + +export interface CTE extends Expr { + name: string; + expr: Expr; + cte_type: "column" | "subquery"; +} + +// Type system +export type TableOrSelectType = + | BaseTableType + | SelectSetQueryType + | SelectQueryType + | SelectQueryAliasType; + +export interface FieldAliasType extends Type { + alias: string; + type: Type; +} + +export interface BaseTableType extends Type { + resolve_database_table?(context: HogQLContext): Table; +} + +export interface TableType extends BaseTableType { + table: Table; +} + +export interface LazyJoinType extends BaseTableType { + table_type: TableOrSelectType; + field: string; + lazy_join: LazyJoin; +} + +export interface LazyTableType extends BaseTableType { + table: LazyTable; +} + +export interface TableAliasType extends BaseTableType { + alias: string; + table_type: TableType | LazyTableType; +} + +export interface VirtualTableType extends BaseTableType { + table_type: TableOrSelectType; + field: string; + virtual_table: VirtualTable; +} + +export interface SelectQueryType extends Type { + aliases: Record; + columns: Record; + tables: Record; + ctes: Record; + anonymous_tables: (SelectQueryType | SelectSetQueryType)[]; + parent?: SelectQueryType | SelectSetQueryType; + is_lambda_type?: boolean; +} + +export interface SelectSetQueryType extends Type { + types: (SelectQueryType | SelectSetQueryType)[]; +} + +export interface SelectViewType extends BaseTableType { + view_name: string; + alias: string; + select_query_type: SelectQueryType | SelectSetQueryType; +} + +export interface SelectQueryAliasType extends Type { + alias: string; + select_query_type: SelectQueryType | SelectSetQueryType; +} + +export interface IntegerType extends ConstantType { + data_type: "int"; +} + +export interface DecimalType extends ConstantType { + data_type: "unknown"; +} + +export interface FloatType extends ConstantType { + data_type: "float"; +} + +export interface StringType extends ConstantType { + data_type: "str"; +} + +export interface StringJSONType extends StringType {} + +export interface StringArrayType extends StringType {} + +export interface BooleanType extends ConstantType { + data_type: "bool"; +} + +export interface DateType extends ConstantType { + data_type: "date"; +} + +export interface DateTimeType extends ConstantType { + data_type: "datetime"; +} + +export interface IntervalType extends ConstantType { + data_type: "unknown"; +} + +export interface UUIDType extends ConstantType { + data_type: "uuid"; +} + +export interface ArrayType extends ConstantType { + data_type: "array"; + item_type: ConstantType; +} + +export interface TupleType extends ConstantType { + data_type: "tuple"; + item_types: ConstantType[]; + repeat?: boolean; +} + +export interface CallType extends Type { + name: string; + arg_types: ConstantType[]; + param_types?: ConstantType[]; + return_type: ConstantType; +} + +export interface AsteriskType extends Type { + table_type: TableOrSelectType; +} + +export interface FieldTraverserType extends Type { + chain: (string | number)[]; + table_type: TableOrSelectType; +} + +export interface ExpressionFieldType extends Type { + name: string; + expr: Expr; + table_type: TableOrSelectType; + isolate_scope?: boolean; +} + +export interface FieldType extends Type { + name: string; + table_type: TableOrSelectType; +} + +export interface UnresolvedFieldType extends Type { + name: string; +} + +export interface PropertyType extends Type { + chain: (string | number)[]; + field_type: FieldType; + joined_subquery?: SelectQueryAliasType; + joined_subquery_field_name?: string; +} + +export interface LambdaArgumentType extends Type { + name: string; +} + +// Enums +export enum ArithmeticOperationOp { + Add = "+", + Sub = "-", + Mult = "*", + Div = "/", + Mod = "%", +} + +export enum CompareOperationOp { + Eq = "==", + NotEq = "!=", + Gt = ">", + GtEq = ">=", + Lt = "<", + LtEq = "<=", + Like = "like", + ILike = "ilike", + NotLike = "not like", + NotILike = "not ilike", + In = "in", + GlobalIn = "global in", + NotIn = "not in", + GlobalNotIn = "global not in", + InCohort = "in cohort", + NotInCohort = "not in cohort", + Regex = "=~", + IRegex = "=~*", + NotRegex = "!~", + NotIRegex = "!~*", +} + +export const NEGATED_COMPARE_OPS: CompareOperationOp[] = [ + CompareOperationOp.NotEq, + CompareOperationOp.NotLike, + CompareOperationOp.NotILike, + CompareOperationOp.NotIn, + CompareOperationOp.GlobalNotIn, + CompareOperationOp.NotInCohort, + CompareOperationOp.NotRegex, + CompareOperationOp.NotIRegex, +]; + +export type SetOperator = + | "UNION ALL" + | "UNION DISTINCT" + | "INTERSECT" + | "INTERSECT DISTINCT" + | "EXCEPT"; + +// Declaration and Statement types +export interface Declaration extends AST {} + +export interface VariableAssignment extends Declaration { + left: Expr; + right: Expr; +} + +export interface VariableDeclaration extends Declaration { + name: string; + expr?: Expr; +} + +export interface Statement extends Declaration {} + +export interface ExprStatement extends Statement { + expr?: Expr; +} + +export interface ReturnStatement extends Statement { + expr?: Expr; +} + +export interface ThrowStatement extends Statement { + expr: Expr; +} + +export interface TryCatchStatement extends Statement { + try_stmt: Statement; + catches: [string | null, string | null, Statement][]; + finally_stmt?: Statement; +} + +export interface IfStatement extends Statement { + expr: Expr; + then: Statement; + else_?: Statement; +} + +export interface WhileStatement extends Statement { + expr: Expr; + body: Statement; +} + +export interface ForStatement extends Statement { + initializer?: VariableDeclaration | VariableAssignment | Expr; + condition?: Expr; + increment?: Expr; + body: Statement; +} + +export interface ForInStatement extends Statement { + keyVar?: string; + valueVar: string; + expr: Expr; + body: Statement; +} + +export interface Function extends Statement { + name: string; + params: string[]; + body: Statement; +} + +export interface Block extends Statement { + declarations: Declaration[]; +} + +export interface Program extends AST { + declarations: Declaration[]; +} + +// Expression types +export interface Alias extends Expr { + alias: string; + expr: Expr; + hidden?: boolean; + from_asterisk?: boolean; +} + +export interface ArithmeticOperation extends Expr { + left: Expr; + right: Expr; + op: ArithmeticOperationOp; +} + +export interface And extends Expr { + type?: ConstantType; + exprs: Expr[]; +} + +export interface Or extends Expr { + exprs: Expr[]; + type?: ConstantType; +} + +export interface CompareOperation extends Expr { + left: Expr; + right: Expr; + op: CompareOperationOp; + type?: ConstantType; +} + +export interface Not extends Expr { + expr: Expr; + type?: ConstantType; +} + +export interface BetweenExpr extends Expr { + expr: Expr; + low: Expr; + high: Expr; + negated?: boolean; + type?: ConstantType; +} + +export interface OrderExpr extends Expr { + expr: Expr; + order?: "ASC" | "DESC"; +} + +export interface ArrayAccess extends Expr { + array: Expr; + property: Expr; + nullish?: boolean; +} + +export interface Array extends Expr { + exprs: Expr[]; +} + +export interface Dict extends Expr { + items: [Expr, Expr][]; +} + +export interface TupleAccess extends Expr { + tuple: Expr; + index: number; + nullish?: boolean; +} + +export interface Tuple extends Expr { + exprs: Expr[]; +} + +export interface Lambda extends Expr { + args: string[]; + expr: Expr | Block; +} + +export interface Constant extends Expr { + value: any; +} + +export interface Field extends Expr { + chain: (string | number)[]; + from_asterisk?: boolean; +} + +export interface Placeholder extends Expr { + expr: Expr; + // Computed properties + chain?: (string | number)[] | null; + field?: string | null; +} + +export interface Call extends Expr { + name: string; + args: Expr[]; + params?: Expr[]; + distinct?: boolean; +} + +export interface ExprCall extends Expr { + expr: Expr; + args: Expr[]; +} + +export interface JoinConstraint extends Expr { + expr: Expr; + constraint_type: "ON" | "USING"; +} + +export interface JoinExpr extends Expr { + type?: TableOrSelectType; + join_type?: string; + table?: SelectQuery | SelectSetQuery | Placeholder | HogQLXTag | Field; + table_args?: Expr[]; + alias?: string; + table_final?: boolean; + constraint?: JoinConstraint; + next_join?: JoinExpr; + sample?: SampleExpr; +} + +export interface WindowFrameExpr extends Expr { + frame_type?: "CURRENT ROW" | "PRECEDING" | "FOLLOWING"; + frame_value?: number; +} + +export interface WindowExpr extends Expr { + partition_by?: Expr[]; + order_by?: OrderExpr[]; + frame_method?: "ROWS" | "RANGE"; + frame_start?: WindowFrameExpr; + frame_end?: WindowFrameExpr; +} + +export interface WindowFunction extends Expr { + name: string; + args?: Expr[]; + exprs?: Expr[]; + over_expr?: WindowExpr; + over_identifier?: string; +} + +export interface LimitByExpr extends Expr { + n: Expr; + exprs: Expr[]; + offset_value?: Expr; +} + +export interface SelectQuery extends Expr { + type?: SelectQueryType; + ctes?: Record; + select: Expr[]; + distinct?: boolean; + select_from?: JoinExpr; + array_join_op?: string; + array_join_list?: Expr[]; + window_exprs?: Record; + where?: Expr; + prewhere?: Expr; + having?: Expr; + group_by?: Expr[]; + order_by?: OrderExpr[]; + limit?: Expr; + limit_by?: LimitByExpr; + limit_with_ties?: boolean; + offset?: Expr; + settings?: HogQLQuerySettings; + view_name?: string; +} + +export interface SelectSetNode extends AST { + select_query: SelectQuery | SelectSetQuery; + set_operator: SetOperator; +} + +export interface SelectSetQuery extends Expr { + type?: SelectSetQueryType; + initial_select_query: SelectQuery | SelectSetQuery; + subsequent_select_queries: SelectSetNode[]; + // Equivalent to select_queries() method + select_queries?(): (SelectQuery | SelectSetQuery)[]; +} + +// Add static method equivalent for SelectSetQuery.create_from_queries() +export namespace SelectSetQuery { + export function createFromQueries( + queries: (SelectQuery | SelectSetQuery)[], + set_operator: SetOperator + ): SelectQuery | SelectSetQuery { + return createSelectSetQueryFromQueries(queries, set_operator); + } +} + +export interface RatioExpr extends Expr { + left: Constant; + right?: Constant; +} + +export interface SampleExpr extends Expr { + sample_value: RatioExpr; + offset_value?: RatioExpr; +} + +export interface HogQLXAttribute extends AST { + name: string; + value: any; +} + +export interface HogQLXTag extends Expr { + kind: string; + attributes: HogQLXAttribute[]; + // Equivalent to to_dict() method + to_dict?(): Record; +} + +// Helper function to create empty SelectQuery (equivalent to SelectQuery.empty()) +export function createEmptySelectQuery(columns?: Record): SelectQuery { + if (!columns) { + columns = { _: { name: "_" } as UnknownDatabaseField }; + } + + return { + select: Object.entries(columns).map(([column, field]) => ({ + alias: column, + expr: { value: (field as DatabaseField).default_value?.() ?? null } as Constant, + })) as Alias[], + where: { value: false } as Constant, + } as SelectQuery; +} + +// Add static method equivalent for SelectQuery.empty() +export namespace SelectQuery { + export function empty(columns?: Record): SelectQuery { + return createEmptySelectQuery(columns); + } +} + +// Helper function for SelectSetQuery.select_queries() +export function selectQueries(query: SelectSetQuery): (SelectQuery | SelectSetQuery)[] { + return [ + query.initial_select_query, + ...query.subsequent_select_queries.map((node) => node.select_query), + ]; +} + +// Helper function to create SelectSetQuery from multiple queries +export function createSelectSetQueryFromQueries( + queries: (SelectQuery | SelectSetQuery)[], + set_operator: SetOperator +): SelectQuery | SelectSetQuery { + if (queries.length === 0) { + throw new Error("Cannot create a SelectSetQuery from an empty list of queries"); + } else if (queries.length === 1) { + return queries[0]; + } + + return { + initial_select_query: queries[0], + subsequent_select_queries: queries.slice(1).map((query) => ({ + select_query: query, + set_operator, + })) as SelectSetNode[], + } as SelectSetQuery; +} diff --git a/internal-packages/tsql/src/query/constants.ts b/internal-packages/tsql/src/query/constants.ts new file mode 100644 index 000000000..c4014ba70 --- /dev/null +++ b/internal-packages/tsql/src/query/constants.ts @@ -0,0 +1,72 @@ +// TypeScript translation of posthog/hogql/constants.py +// Keep this file in sync with the Python version + +export type ConstantDataType = + | "int" + | "float" + | "str" + | "bool" + | "array" + | "tuple" + | "date" + | "datetime" + | "uuid" + | "unknown"; + +export type ConstantSupportedPrimitive = number | string | boolean | Date | null; +export type ConstantSupportedData = + | ConstantSupportedPrimitive + | ConstantSupportedPrimitive[] + | [ConstantSupportedPrimitive, ...ConstantSupportedPrimitive[]]; + +export const KEYWORDS = ["true", "false", "null"] as const; +export const RESERVED_KEYWORDS = [...KEYWORDS, "team_id"] as const; + +export const DEFAULT_RETURNED_ROWS = 100; +export const MAX_SELECT_RETURNED_ROWS = 50000; +export const MAX_SELECT_RETENTION_LIMIT = 100000; +export const MAX_SELECT_HEATMAPS_LIMIT = 1000000; +export const MAX_SELECT_COHORT_CALCULATION_LIMIT = 1000000000; +export const MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY = 22 * 1024 * 1024 * 1024; +export const CSV_EXPORT_LIMIT = 300000; +export const CSV_EXPORT_BREAKDOWN_LIMIT_INITIAL = 512; +export const CSV_EXPORT_BREAKDOWN_LIMIT_LOW = 64; +export const BREAKDOWN_VALUES_LIMIT = 25; +export const BREAKDOWN_VALUES_LIMIT_FOR_COUNTRIES = 300; + +export enum LimitContext { + QUERY = "query", + QUERY_ASYNC = "query_async", + EXPORT = "export", + COHORT_CALCULATION = "cohort_calculation", + HEATMAPS = "heatmaps", + SAVED_QUERY = "saved_query", + RETENTION = "retention", +} + +// Settings applied at the SELECT level +export interface HogQLQuerySettings { + optimize_aggregation_in_order?: boolean; + date_time_output_format?: string; + date_time_input_format?: string; + join_algorithm?: string; +} + +// Settings applied on top of all HogQL queries +export interface HogQLGlobalSettings extends HogQLQuerySettings { + readonly?: number; + max_execution_time?: number; + max_memory_usage?: number; + max_threads?: number; + allow_experimental_object_type?: boolean; + format_csv_allow_double_quotes?: boolean; + max_ast_elements?: number; + max_expanded_ast_elements?: number; + max_bytes_before_external_group_by?: number; + allow_experimental_analyzer?: boolean; + transform_null_in?: boolean; + optimize_min_equality_disjunction_chain_length?: number; + allow_experimental_join_condition?: boolean; + preferred_block_size_bytes?: number; + use_hive_partitioning?: number; +} diff --git a/internal-packages/tsql/src/query/context.ts b/internal-packages/tsql/src/query/context.ts new file mode 100644 index 000000000..e964bf299 --- /dev/null +++ b/internal-packages/tsql/src/query/context.ts @@ -0,0 +1,57 @@ +// TypeScript translation of posthog/hogql/context.py +// Keep this file in sync with the Python version + +import type { LimitContext } from "./constants"; +import type { Database } from "./database"; +import type { PropertySwapper } from "./property_types"; +import type { HogQLTimings } from "./timings"; + +export interface HogQLNotice { + start?: number; + end?: number; + message: string; + fix?: string; +} + +export interface HogQLQueryModifiers { + optimizeJoinedFilters?: boolean; + debug?: boolean; + timings?: boolean; + useMaterializedViews?: boolean; + formatCsvAllowDoubleQuotes?: boolean; + convertToProjectTimezone?: boolean; + usePreaggregatedTableTransforms?: boolean; + optimizeProjections?: boolean; +} + +export interface HogQLFieldAccess { + input: string[]; + type?: "run"; + field?: string; + sql: string; +} + +export interface Team { + id: number; + project_id: number; +} + +export interface HogQLContext { + team_id?: number; + team?: Team; + database?: Database; + values: Record; + within_non_hogql_query?: boolean; + enable_select_queries?: boolean; + limit_top_select?: boolean; + limit_context?: LimitContext; + output_format?: string | null; + globals?: Record; + warnings: HogQLNotice[]; + notices: HogQLNotice[]; + errors: HogQLNotice[]; + timings: HogQLTimings; + modifiers: HogQLQueryModifiers; + debug?: boolean; + property_swapper?: PropertySwapper; +} diff --git a/internal-packages/tsql/src/query/database.ts b/internal-packages/tsql/src/query/database.ts new file mode 100644 index 000000000..7edd86ecf --- /dev/null +++ b/internal-packages/tsql/src/query/database.ts @@ -0,0 +1,648 @@ +// TypeScript translation of posthog/hogql/database/database.py +// Keep this file in sync with the Python version +// +// NOTE: This implementation requires database/ORM access for: +// - serialize() method (needs DataWarehouseTable, DataWarehouseSavedQuery queries) +// - create_for() method (needs Team, DataWarehouseJoin, DataWarehouseSavedQuery queries) +// Adapt these methods to your database/ORM setup + +import type { + Table, + TableNode, + FieldOrTable, + DatabaseField, + ExpressionField, + FieldTraverser, + LazyJoin, + VirtualTable, + FunctionCallTable, + SavedQuery, +} from './models'; +import type { HogQLContext, Team } from '../context'; +import type { HogQLQueryModifiers } from '../context'; +import type { HogQLTimings } from '../timings'; +import type { Expr, ConstantType } from '../ast'; +// Type imports for type checking +import type { + StringType, + BooleanType, + DateType, + DateTimeType, + UUIDType, + ArrayType, + TupleType, + IntegerType, + FloatType, + DecimalType, +} from '../ast'; +import { QueryError, ResolutionError } from '../errors'; +import { parseExpr } from '../parser'; +import { HogQLTimings as HogQLTimingsClass } from '../timings'; + +// Type definitions for schema serialization (adapt to your schema types) +export interface DatabaseSchemaTable { + // Base schema table type +} + +export interface DatabaseSchemaPostHogTable extends DatabaseSchemaTable { + fields: Record; + id: string; + name: string; +} + +export interface DatabaseSchemaSystemTable extends DatabaseSchemaTable { + fields: Record; + id: string; + name: string; +} + +export interface DatabaseSchemaDataWarehouseTable extends DatabaseSchemaTable { + fields: Record; + id: string; + name: string; + format?: string; + url_pattern?: string; + schema?: DatabaseSchemaSchema; + source?: DatabaseSchemaSource; + row_count?: number; +} + +export interface DatabaseSchemaViewTable extends DatabaseSchemaTable { + fields: Record; + id: string; + name: string; + query: { query: string }; + row_count?: number; +} + +export interface DatabaseSchemaManagedViewTable extends DatabaseSchemaTable { + fields: Record; + id: string; + name: string; + kind: string; + source_id?: string; + query: { query: string }; +} + +export interface DatabaseSchemaEndpointTable extends DatabaseSchemaTable { + fields: Record; + id: string; + name: string; + query: { query: string }; + row_count?: number; + status?: string; +} + +export interface DatabaseSchemaField { + name: string; + hogql_value: string; + type: DatabaseSerializedFieldType; + schema_valid: boolean; + fields?: string[]; + table?: string; + chain?: Array; + id?: string; +} + +export interface DatabaseSchemaSchema { + id: string; + name: string; + should_sync: boolean; + incremental: boolean; + status: string; + last_synced_at: string; +} + +export interface DatabaseSchemaSource { + id: string; + status: string; + source_type: string; + prefix: string; + last_synced_at?: string | null; +} + +export enum DatabaseSerializedFieldType { + STRING = 'string', + INTEGER = 'integer', + FLOAT = 'float', + DECIMAL = 'decimal', + BOOLEAN = 'boolean', + DATE = 'date', + DATETIME = 'datetime', + UUID = 'uuid', + ARRAY = 'array', + JSON = 'json', + TUPLE = 'tuple', + UNKNOWN = 'unknown', + EXPRESSION = 'expression', + VIEW = 'view', + LAZY_TABLE = 'lazy_table', + VIRTUAL_TABLE = 'virtual_table', + FIELD_TRAVERSER = 'field_traverser', +} + +export interface SerializedField { + key: string; + name: string; + type: DatabaseSerializedFieldType; + schema_valid: boolean; + fields?: string[]; + table?: string; + chain?: Array; +} + +import { TableNodeImpl } from './models'; + +export class Database { + // Users can query from the tables below + tables: TableNode; + + private _warehouseTableNames: string[] = []; + private _warehouseSelfManagedTableNames: string[] = []; + private _viewTableNames: string[] = []; + + private _timezone?: string | null; + private _weekStartDay?: string | null; // WeekStartDay enum + + private _serializationErrors: Record = {}; + + constructor(timezone?: string | null, weekStartDay?: string | null) { + // Initialize with root TableNode + this.tables = new TableNodeImpl('root'); + this._timezone = timezone || null; + this._weekStartDay = weekStartDay || null; + + // NOTE: In Python, tables are initialized with all PostHog tables. + // You'll need to initialize these based on your table definitions. + // For now, this is a minimal structure. + } + + getTimezone(): string { + return this._timezone || 'UTC'; + } + + getWeekStartDay(): string { + return this._weekStartDay || 'sunday'; // Adapt to your WeekStartDay enum + } + + getSerializationErrors(): Record { + /** Return any errors encountered during serialization. */ + return { ...this._serializationErrors }; + } + + hasTable(tableName: string | string[]): boolean { + const path = typeof tableName === 'string' ? tableName.split('.') : tableName; + return this.tables.has_child ? this.tables.has_child(path) : false; + } + + getTableNode(tableName: string | string[]): TableNode { + let path: string[]; + if (typeof tableName === 'string') { + path = tableName.split('.'); + } else { + path = tableName; + } + + // Handle edge case where tableName is a list with a single string containing dots + if (path.length === 1 && path[0].includes('.')) { + path = path[0].split('.'); + } + + if (!this.tables.get_child) { + throw new ResolutionError(`TableNode.get_child not implemented`); + } + return this.tables.get_child(path); + } + + getTable(tableName: string | string[]): Table { + try { + const node = this.getTableNode(tableName); + if (!node.get) { + throw new ResolutionError('TableNode.get not implemented'); + } + const table = node.get(); + if (!table || typeof table !== 'object' || !('fields' in table)) { + throw new ResolutionError('Table is not set'); + } + return table as Table; + } catch (e) { + const name = Array.isArray(tableName) ? tableName.join('.') : tableName; + if (e instanceof ResolutionError) { + throw new QueryError(`Unknown table \`${name}\`.`); + } + throw e; + } + } + + getAllTableNames(): string[] { + const warehouseTableNames = this._warehouseTableNames.filter((x) => x.includes('.')); + + return [ + ...this.getPosthogTableNames(), + ...warehouseTableNames, + ...this._warehouseSelfManagedTableNames, + ...this._viewTableNames, + ]; + } + + // These are the tables exposed via SQL editor autocomplete and data management + getPosthogTableNames(): string[] { + return ['events', 'groups', 'persons', 'sessions', ...this.getSystemTableNames()]; + } + + getSystemTableNames(): string[] { + // NOTE: Adapt this based on your SystemTables implementation + const systemNode = this.tables.children['system']; + if (systemNode && systemNode.resolve_all_table_names) { + return ['query_log', ...systemNode.resolve_all_table_names()]; + } + return ['query_log']; + } + + getWarehouseTableNames(): string[] { + return [...this._warehouseTableNames, ...this._warehouseSelfManagedTableNames]; + } + + getViewNames(): string[] { + return this._viewTableNames; + } + + private _addWarehouseTables(node: TableNode): void { + if (this.tables.merge_with) { + this.tables.merge_with(node); + } + if (node.resolve_all_table_names) { + const names = node.resolve_all_table_names(); + this._warehouseTableNames.push(...names.sort()); + } + } + + private _addWarehouseSelfManagedTables(node: TableNode): void { + if (this.tables.merge_with) { + this.tables.merge_with(node); + } + if (node.resolve_all_table_names) { + const names = node.resolve_all_table_names(); + this._warehouseSelfManagedTableNames.push(...names.sort()); + } + } + + private _addViews(node: TableNode): void { + if (this.tables.merge_with) { + this.tables.merge_with(node); + } + if (node.resolve_all_table_names) { + const names = node.resolve_all_table_names(); + this._viewTableNames.push(...names.sort()); + } + } + + serialize( + context: HogQLContext, + includeOnly?: Set + ): Record { + // NOTE: This method requires database queries to fetch: + // - DataWarehouseTable objects + // - DataWarehouseSavedQuery objects + // - External data sources and schemas + // + // Adapt this to your database/ORM setup + + const tables: Record = {}; + + if (!context.team_id) { + throw new ResolutionError('Must provide team_id to serialize database'); + } + + // PostHog tables + const posthogTableNames = this.getPosthogTableNames(); + for (const tableName of posthogTableNames) { + if (includeOnly && !includeOnly.has(tableName)) { + continue; + } + + let fieldInput: Record = {}; + const table = this.getTable(tableName); + if ('get_asterisk' in table && typeof table.get_asterisk === 'function') { + fieldInput = table.get_asterisk() || {}; + } else if ('fields' in table) { + fieldInput = table.fields; + } + + const fields = serializeFields(fieldInput, context, tableName.split('.'), undefined, 'posthog'); + const fieldsDict: Record = {}; + for (const field of fields) { + fieldsDict[field.name] = field; + } + tables[tableName] = { + fields: fieldsDict, + id: tableName, + name: tableName, + } as DatabaseSchemaPostHogTable; + } + + // System tables + const systemTables = this.getSystemTableNames(); + for (const tableKey of systemTables) { + if (includeOnly && !includeOnly.has(tableKey)) { + continue; + } + + let systemFieldInput: Record = {}; + const table = this.getTable(tableKey); + if ('get_asterisk' in table && typeof table.get_asterisk === 'function') { + systemFieldInput = table.get_asterisk() || {}; + } else if ('fields' in table) { + systemFieldInput = table.fields; + } + + const fields = serializeFields(systemFieldInput, context, tableKey.split('.'), undefined, 'posthog'); + const fieldsDict: Record = {}; + for (const field of fields) { + fieldsDict[field.name] = field; + } + tables[tableKey] = { + fields: fieldsDict, + id: tableKey, + name: tableKey, + } as DatabaseSchemaSystemTable; + } + + // NOTE: Data Warehouse Tables and Views processing requires database queries + // Implement based on your database/ORM setup: + // - Fetch DataWarehouseTable objects + // - Fetch DataWarehouseSavedQuery objects + // - Process and serialize them + + return tables; + } + + static createFor( + teamId?: number, + options?: { + team?: Team; + modifiers?: HogQLQueryModifiers; + timings?: HogQLTimings; + } + ): Database { + // NOTE: This method requires extensive database/ORM access: + // - Team model queries + // - DataWarehouseTable queries + // - DataWarehouseSavedQuery queries + // - DataWarehouseJoin queries + // - GroupTypeMapping queries + // - Feature flag checks + // + // This is a skeleton structure - adapt to your setup + + const timings = options?.timings || new HogQLTimingsClass(); + const { team, modifiers } = options || {}; + + // Validate team/teamId + if (!teamId && !team) { + throw new Error('Either team_id or team must be provided'); + } + + if (team && teamId && team.id !== teamId) { + throw new Error('team_id and team must be the same'); + } + + // NOTE: Fetch team from database if not provided + // const fetchedTeam = team || await Team.findById(teamId); + + // Create database instance + const database = timings.measure('database', () => { + // NOTE: Get timezone and week_start_day from team + // const timezone = fetchedTeam.timezone; + // const weekStartDay = fetchedTeam.week_start_day; + return new Database(undefined, undefined); + }); + + // NOTE: Apply modifiers, setup tables, etc. + // This requires extensive database access and table setup logic + // See Python implementation for full details + + return database; + } +} + +// Helper functions + +const HOGQL_CHARACTERS_TO_BE_WRAPPED = ['@', '-', '!', '$', '+']; + +function constantTypeToSerializedFieldType(constantType: ConstantType): DatabaseSerializedFieldType | null { + // Type checking for ConstantType subtypes + // NOTE: In TypeScript, we need to check properties rather than instanceof + // since these are interfaces, not classes + + if ('data_type' in constantType) { + const dataType = constantType.data_type; + if (dataType === 'str') { + return DatabaseSerializedFieldType.STRING; + } + if (dataType === 'bool') { + return DatabaseSerializedFieldType.BOOLEAN; + } + if (dataType === 'date') { + return DatabaseSerializedFieldType.DATE; + } + if (dataType === 'datetime') { + return DatabaseSerializedFieldType.DATETIME; + } + if (dataType === 'uuid') { + return DatabaseSerializedFieldType.STRING; + } + if (dataType === 'array') { + return DatabaseSerializedFieldType.ARRAY; + } + if (dataType === 'tuple') { + return DatabaseSerializedFieldType.JSON; + } + if (dataType === 'int') { + return DatabaseSerializedFieldType.INTEGER; + } + if (dataType === 'float') { + return DatabaseSerializedFieldType.FLOAT; + } + } + + // Fallback: check print_type if available + if ('print_type' in constantType && typeof constantType.print_type === 'function') { + const printed = constantType.print_type(); + if (printed === 'String' || printed === 'JSON' || printed === 'Array') { + return printed === 'String' + ? DatabaseSerializedFieldType.STRING + : printed === 'JSON' + ? DatabaseSerializedFieldType.JSON + : DatabaseSerializedFieldType.ARRAY; + } + if (printed === 'Boolean') return DatabaseSerializedFieldType.BOOLEAN; + if (printed === 'Date') return DatabaseSerializedFieldType.DATE; + if (printed === 'DateTime') return DatabaseSerializedFieldType.DATETIME; + if (printed === 'UUID') return DatabaseSerializedFieldType.STRING; + if (printed === 'Integer') return DatabaseSerializedFieldType.INTEGER; + if (printed === 'Float') return DatabaseSerializedFieldType.FLOAT; + if (printed === 'Decimal') return DatabaseSerializedFieldType.DECIMAL; + } + + return null; +} + +export function serializeFields( + fieldInput: Record, + context: HogQLContext, + tableChain: string[], + dbColumns?: Record, // DataWarehouseTableColumns + tableType: 'posthog' | 'external' = 'posthog' +): DatabaseSchemaField[] { + // NOTE: This requires resolve_types_from_table from resolver + // Import as needed: import { resolveTypesFromTable } from '../resolver'; + + const fieldOutput: DatabaseSchemaField[] = []; + + for (const [fieldKey, field] of Object.entries(fieldInput)) { + let schemaValid = true; + + if (dbColumns) { + const column = dbColumns[fieldKey]; + if (typeof column === 'string') { + schemaValid = true; + } else if (column && typeof column === 'object') { + schemaValid = column.valid !== false; + } + } + + let hogqlValue: string; + if (HOGQL_CHARACTERS_TO_BE_WRAPPED.some((char) => fieldKey.includes(char))) { + hogqlValue = `\`${fieldKey}\``; + } else { + hogqlValue = fieldKey; + } + + if ('hidden' in field && field.hidden) { + continue; + } + + if (fieldKey === 'team_id' && tableType === 'posthog') { + // Skip team_id for posthog tables + continue; + } else if ('name' in field && 'get_constant_type' in field) { + // DatabaseField + const dbField = field as DatabaseField; + let fieldType: DatabaseSerializedFieldType; + + // Determine field type based on DatabaseField subclass + // NOTE: You'll need to check instanceof or use type guards + // For now, using a simplified approach + if (dbField.get_constant_type) { + const constantType = dbField.get_constant_type(); + fieldType = constantTypeToSerializedFieldType(constantType) || DatabaseSerializedFieldType.UNKNOWN; + } else { + fieldType = DatabaseSerializedFieldType.UNKNOWN; + } + + fieldOutput.push({ + name: fieldKey, + hogql_value: hogqlValue, + type: fieldType, + schema_valid: schemaValid, + }); + } else if ('expr' in field) { + // ExpressionField + const exprField = field as ExpressionField; + // NOTE: Requires resolve_types_from_table + // const resolvedExpr = resolveTypesFromTable(exprField.expr, tableChain, context, 'hogql'); + // const constantType = resolvedExpr.type?.resolve_constant_type(context); + // const fieldType = constantTypeToSerializedFieldType(constantType) || DatabaseSerializedFieldType.EXPRESSION; + + fieldOutput.push({ + name: fieldKey, + hogql_value: hogqlValue, + type: DatabaseSerializedFieldType.EXPRESSION, + schema_valid: schemaValid, + }); + } else if ('resolve_table' in field) { + // LazyJoin + const lazyJoin = field as LazyJoin; + if (lazyJoin.resolve_table) { + const resolvedTable = lazyJoin.resolve_table(context); + const type = + 'id' in resolvedTable && resolvedTable.id + ? DatabaseSerializedFieldType.VIEW + : DatabaseSerializedFieldType.LAZY_TABLE; + + fieldOutput.push({ + name: fieldKey, + hogql_value: hogqlValue, + type, + schema_valid: schemaValid, + table: resolvedTable.to_printed_hogql ? resolvedTable.to_printed_hogql() : fieldKey, + fields: 'fields' in resolvedTable ? Object.keys(resolvedTable.fields) : [], + id: 'id' in resolvedTable && resolvedTable.id ? String(resolvedTable.id) : fieldKey, + }); + } + } else if ('fields' in field && !('resolve_table' in field)) { + // VirtualTable + const virtualTable = field as VirtualTable; + fieldOutput.push({ + name: fieldKey, + hogql_value: hogqlValue, + type: DatabaseSerializedFieldType.VIRTUAL_TABLE, + schema_valid: schemaValid, + table: virtualTable.to_printed_hogql ? virtualTable.to_printed_hogql() : fieldKey, + fields: Object.keys(virtualTable.fields), + }); + } else if ('chain' in field) { + // FieldTraverser + const traverser = field as FieldTraverser; + fieldOutput.push({ + name: fieldKey, + hogql_value: hogqlValue, + type: DatabaseSerializedFieldType.FIELD_TRAVERSER, + schema_valid: schemaValid, + chain: traverser.chain, + }); + } + } + + return fieldOutput; +} + +// Helper functions for database setup (simplified versions) +function usePersonPropertiesFromEvents(database: Database): void { + const table = database.getTable('events'); + table.fields['person'] = { chain: ['poe'] } as FieldTraverser; +} + +function usePersonIdFromPersonOverrides(database: Database): void { + const table = database.getTable('events'); + table.fields['event_person_id'] = { name: 'person_id' } as DatabaseField; + // NOTE: Setup LazyJoin and ExpressionField for override logic + // This requires complex setup - see Python implementation +} + +function useErrorTrackingIssueIdFromErrorTrackingIssueOverrides(database: Database): void { + const table = database.getTable('events'); + // NOTE: Setup ExpressionField and LazyJoin for error tracking + // See Python implementation for details +} + +function setupGroupKeyFields(database: Database, team: Team): void { + // NOTE: Requires GroupTypeMapping queries from database + // See Python implementation for full logic + const table = database.getTable('events'); + // Setup group key fields based on GroupTypeMapping +} + +function useVirtualFields( + database: Database, + modifiers: HogQLQueryModifiers, + timings: HogQLTimings +): void { + // NOTE: Requires channel type creation functions + // See Python implementation for full virtual field setup + const eventsTable = database.getTable('events'); + const personsTable = database.getTable('persons'); + const groupsTable = database.getTable('groups'); + // Setup virtual fields like initial_referring_domain_type, initial_channel_type, revenue fields +} diff --git a/internal-packages/tsql/src/query/errors.ts b/internal-packages/tsql/src/query/errors.ts new file mode 100644 index 000000000..235db7891 --- /dev/null +++ b/internal-packages/tsql/src/query/errors.ts @@ -0,0 +1,62 @@ +// TypeScript translation of posthog/hogql/errors.py +// Keep this file in sync with the Python version + +import type { Expr } from "./ast"; + +export class BaseHogQLError extends Error { + message: string; + start?: number; + end?: number; + + constructor( + message: string, + options?: { + start?: number; + end?: number; + node?: Expr; + } + ) { + super(message); + this.message = message; + + if (options?.node && options.node.start !== undefined && options.node.end !== undefined) { + this.start = options.node.start; + this.end = options.node.end; + } else { + this.start = options?.start; + this.end = options?.end; + } + } +} + +export class ExposedHogQLError extends BaseHogQLError { + /** An exception that can be exposed to the user. */ +} + +export class InternalHogQLError extends BaseHogQLError { + /** An internal exception in the HogQL engine. */ +} + +export class SyntaxError extends ExposedHogQLError { + /** The input does not conform to HogQL syntax. */ +} + +export class QueryError extends ExposedHogQLError { + /** The query is invalid, though correct syntactically. */ +} + +export class NotImplementedError extends InternalHogQLError { + /** This feature isn't implemented in HogQL (yet). */ +} + +export class ParsingError extends InternalHogQLError { + /** Parsing failed. */ +} + +export class ImpossibleASTError extends InternalHogQLError { + /** Parsing or resolution resulted in an impossible AST. */ +} + +export class ResolutionError extends InternalHogQLError { + /** Resolution of a table/field/expression failed. */ +} diff --git a/internal-packages/tsql/src/query/models.ts b/internal-packages/tsql/src/query/models.ts new file mode 100644 index 000000000..7b8ebff94 --- /dev/null +++ b/internal-packages/tsql/src/query/models.ts @@ -0,0 +1,254 @@ +// TypeScript translation of posthog/hogql/database/models.py +// Keep this file in sync with the Python version + +import type { Expr, ConstantType } from "./ast"; +import type { HogQLContext } from "./context"; + +export interface FieldOrTable { + hidden?: boolean; +} + +export interface DatabaseField extends FieldOrTable { + name: string; + array?: boolean; + nullable?: boolean; + is_nullable?(): boolean; + get_constant_type?(): ConstantType; + default_value?(): any; +} + +export interface IntegerDatabaseField extends DatabaseField {} +export interface FloatDatabaseField extends DatabaseField {} +export interface DecimalDatabaseField extends DatabaseField {} +export interface StringDatabaseField extends DatabaseField {} +export interface UnknownDatabaseField extends DatabaseField {} +export interface StringJSONDatabaseField extends DatabaseField {} +export interface StringArrayDatabaseField extends DatabaseField {} +export interface FloatArrayDatabaseField extends DatabaseField {} +export interface DateDatabaseField extends DatabaseField {} +export interface DateTimeDatabaseField extends DatabaseField {} +export interface BooleanDatabaseField extends DatabaseField {} +export interface UUIDDatabaseField extends DatabaseField {} + +export interface ExpressionField extends DatabaseField { + expr: Expr; + isolate_scope?: boolean; +} + +export interface FieldTraverser extends FieldOrTable { + chain: Array; +} + +export interface Table extends FieldOrTable { + fields: Record; + has_field?(name: string | number): boolean; + get_field?(name: string | number): FieldOrTable; + to_printed_clickhouse?(context: HogQLContext): string; + to_printed_hogql?(): string; + avoid_asterisk_fields?(): string[]; + get_asterisk?(): Record; +} + +export interface LazyJoin extends FieldOrTable { + join_function?(from_table: Table, to_table: Table, requesting_table: Table): Expr; + resolve_table?(context: HogQLContext): Table; +} + +export interface LazyTable extends Table {} + +export interface VirtualTable extends Table {} + +export interface SavedQuery extends Table { + query: Expr; +} + +export interface FunctionCallTable extends Table { + call_function?(context: HogQLContext): Expr; +} + +export interface TableNode { + name: "root" | string; + table?: FieldOrTable | null; + children: Record; + get?(): FieldOrTable; + has_child?(path: string[]): boolean; + get_child?(path: string[]): TableNode; + add_child?( + child: TableNode, + options?: { + table_conflict_mode?: "override" | "ignore"; + children_conflict_mode?: "override" | "merge" | "ignore"; + } + ): void; + merge_with?( + other: TableNode, + options?: { + table_conflict_mode?: "override" | "ignore"; + children_conflict_mode?: "override" | "merge" | "ignore"; + } + ): void; + resolve_all_table_names?(): string[]; +} + +// Basic TableNode implementation class +export class TableNodeImpl implements TableNode { + name: "root" | string; + table?: FieldOrTable | null; + children: Record; + + constructor(name: "root" | string = "root", table?: FieldOrTable | null) { + this.name = name; + this.table = table || null; + this.children = {}; + } + + get(): FieldOrTable { + if (this.table === null || this.table === undefined) { + throw new Error(`Table is not set at \`${this.name}\``); + } + return this.table; + } + + has_child(path: string[]): boolean { + if (path.length === 0) { + return this.table !== null && this.table !== undefined; + } + + const [first, ...restOfPath] = path; + if (!(first in this.children)) { + return false; + } + + return this.children[first].has_child ? this.children[first].has_child!(restOfPath) : false; + } + + get_child(path: string[]): TableNode { + if (path.length === 0) { + return this; + } + + const [first, ...restOfPath] = path; + if (!(first in this.children)) { + throw new Error(`Unknown child \`${first}\` at \`${this.name}\`.`); + } + + return this.children[first].get_child + ? this.children[first].get_child!(restOfPath) + : this.children[first]; + } + + add_child( + child: TableNode, + options?: { + table_conflict_mode?: "override" | "ignore"; + children_conflict_mode?: "override" | "merge" | "ignore"; + } + ): void { + const tableConflictMode = options?.table_conflict_mode || "ignore"; + const childrenConflictMode = options?.children_conflict_mode || "merge"; + + if (child.name in this.children) { + if (childrenConflictMode === "override") { + this.children[child.name] = child; + } else if (childrenConflictMode === "merge") { + const existing = this.children[child.name]; + if (existing.merge_with) { + existing.merge_with(child, { + table_conflict_mode: tableConflictMode, + children_conflict_mode: childrenConflictMode, + }); + } + } + // ignore mode: do nothing + return; + } + + this.children[child.name] = child; + } + + merge_with( + other: TableNode, + options?: { + table_conflict_mode?: "override" | "ignore"; + children_conflict_mode?: "override" | "merge" | "ignore"; + } + ): void { + const tableConflictMode = options?.table_conflict_mode || "ignore"; + const childrenConflictMode = options?.children_conflict_mode || "merge"; + + if (other.table !== null && other.table !== undefined) { + if (this.table === null || this.table === undefined) { + this.table = other.table; + } else { + // Conflict - check conflict mode + if (tableConflictMode === "override") { + this.table = other.table; + } + // ignore mode: do nothing + } + } + + for (const child of Object.values(other.children)) { + this.add_child(child, { + table_conflict_mode: tableConflictMode, + children_conflict_mode: childrenConflictMode, + }); + } + } + + resolve_all_table_names(): string[] { + const names: string[] = []; + + if (this.table !== null && this.table !== undefined) { + names.push(this.name); + } + + for (const child of Object.values(this.children)) { + const childNames = child.resolve_all_table_names ? child.resolve_all_table_names() : []; + + // The root node should NOT include itself in the names + if (this.name === "root") { + names.push(...childNames); + } else { + names.push(...childNames.map((x) => `${this.name}.${x}`)); + } + } + + return names; + } + + static createNestedForChain(chain: string[], table: Table): TableNode { + if (chain.length === 0) { + throw new Error("Chain must have at least one element"); + } + + const start = new TableNodeImpl(chain[0]); + let current: TableNode = start; + + for (let i = 1; i < chain.length; i++) { + const child = new TableNodeImpl(chain[i]); + if (current.add_child) { + current.add_child(child); + } else { + current.children[child.name] = child; + } + current = child; + } + + current.table = table; + return start; + } +} + +export interface LazyTableToAdd { + lazy_table: LazyTable; + fields_accessed: Record>; +} + +export interface LazyJoinToAdd { + from_table: string; + to_table: string; + lazy_join: LazyJoin; + lazy_join_type: any; // LazyJoinType from ast.ts + fields_accessed: Record>; +} diff --git a/internal-packages/tsql/src/query/parse_string.ts b/internal-packages/tsql/src/query/parse_string.ts new file mode 100644 index 000000000..be38f752d --- /dev/null +++ b/internal-packages/tsql/src/query/parse_string.ts @@ -0,0 +1,65 @@ +// TypeScript translation of posthog/hogql/parse_string.py +// Keep this file in sync with the Python version + +import { SyntaxError } from './errors'; + +function replaceCommonEscapeCharacters(text: string): string { + // copied from clickhouse_driver/util/escape.py + // Note: \a (bell) and \v (vertical tab) are not directly supported in JavaScript strings + // but we handle them as escape sequences that get replaced + text = text.replace(/\\b/g, '\b'); + text = text.replace(/\\f/g, '\f'); + text = text.replace(/\\r/g, '\r'); + text = text.replace(/\\n/g, '\n'); + text = text.replace(/\\t/g, '\t'); + text = text.replace(/\\0/g, ''); // NUL characters are ignored + text = text.replace(/\\a/g, '\x07'); // Bell character (ASCII 7) + text = text.replace(/\\v/g, '\x0B'); // Vertical tab (ASCII 11) + text = text.replace(/\\\\/g, '\\'); + return text; +} + +export function parseStringLiteralText(text: string): string { + /** Converts a string received from antlr via ctx.getText() into a JavaScript string */ + let result: string; + + if (text.startsWith("'") && text.endsWith("'")) { + result = text.slice(1, -1); + result = result.replace(/''/g, "'"); + result = result.replace(/\\'/g, "'"); + } else if (text.startsWith('"') && text.endsWith('"')) { + result = text.slice(1, -1); + result = result.replace(/""/g, '"'); + result = result.replace(/\\"/g, '"'); + } else if (text.startsWith('`') && text.endsWith('`')) { + result = text.slice(1, -1); + result = result.replace(/``/g, '`'); + result = result.replace(/\\`/g, '`'); + } else if (text.startsWith('{') && text.endsWith('}')) { + result = text.slice(1, -1); + result = result.replace(/{{/g, '{'); + result = result.replace(/\\{/g, '{'); + } else { + throw new SyntaxError(`Invalid string literal, must start and end with the same quote type: ${text}`); + } + + return replaceCommonEscapeCharacters(result); +} + +export function parseStringLiteralCtx(ctx: { getText(): string }): string { + /** Converts a STRING_LITERAL received from antlr via ctx.getText() into a JavaScript string */ + const text = ctx.getText(); + return parseStringLiteralText(text); +} + +export function parseStringTextCtx(ctx: { getText(): string }, escapeQuotes: boolean = true): string { + /** Converts a STRING_TEXT received from antlr via ctx.getText() into a JavaScript string */ + let text = ctx.getText(); + if (escapeQuotes) { + text = text.replace(/''/g, "'"); + text = text.replace(/\\'/g, "'"); + } + text = text.replace(/\\{/g, '{'); + return replaceCommonEscapeCharacters(text); +} + diff --git a/internal-packages/tsql/src/query/parser.test.ts b/internal-packages/tsql/src/query/parser.test.ts new file mode 100644 index 000000000..5cd50b44d --- /dev/null +++ b/internal-packages/tsql/src/query/parser.test.ts @@ -0,0 +1,359 @@ +import { describe, it, expect } from "vitest"; +import { CharStreams, CommonTokenStream } from "antlr4ts"; +import { TSQLLexer } from "../grammar/TSQLLexer.js"; +import { TSQLParser } from "../grammar/TSQLParser.js"; +import { TSQLParseTreeConverter } from "./parser.js"; +import type { + SelectQuery, + SelectSetQuery, + Field, + Constant, + Call, + CompareOperation, + ArithmeticOperation, + Alias, + JoinExpr, +} from "./ast.js"; +import { ArithmeticOperationOp, CompareOperationOp } from "./ast.js"; +import { SyntaxError } from "./errors.js"; + +/** + * Helper function to parse TSQL input and convert to AST + */ +function parseAndConvert(input: string) { + const inputStream = CharStreams.fromString(input); + const lexer = new TSQLLexer(inputStream); + const tokenStream = new CommonTokenStream(lexer); + const parser = new TSQLParser(tokenStream); + const parseTree = parser.select(); + const converter = new TSQLParseTreeConverter(); + return converter.visit(parseTree); +} + +describe("TSQLParseTreeConverter", () => { + describe("SELECT statements", () => { + it("should convert a simple SELECT statement", () => { + const ast = parseAndConvert("SELECT * FROM users"); + + expect(ast).toBeDefined(); + expect("select" in ast).toBe(true); + const selectQuery = ast as SelectQuery; + expect(selectQuery.select).toBeDefined(); + expect(selectQuery.select_from).toBeDefined(); + expect(selectQuery.select.length).toBe(1); + expect((selectQuery.select[0] as Field).chain).toEqual(["*"]); + }); + + it("should convert SELECT with multiple columns", () => { + const ast = parseAndConvert("SELECT id, name, email FROM users"); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.select.length).toBe(3); + expect((selectQuery.select[0] as Field).chain).toEqual(["id"]); + expect((selectQuery.select[1] as Field).chain).toEqual(["name"]); + expect((selectQuery.select[2] as Field).chain).toEqual(["email"]); + }); + + it("should convert SELECT with DISTINCT", () => { + const ast = parseAndConvert("SELECT DISTINCT id FROM users"); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.distinct).toBe(true); + }); + + it("should convert SELECT with WHERE clause", () => { + const ast = parseAndConvert("SELECT * FROM users WHERE id = 1"); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.where).toBeDefined(); + const whereExpr = selectQuery.where as CompareOperation; + expect(whereExpr.op).toBe(CompareOperationOp.Eq); + expect((whereExpr.left as Field).chain).toEqual(["id"]); + expect((whereExpr.right as Constant).value).toBe(1); + }); + + it("should convert SELECT with ORDER BY", () => { + const ast = parseAndConvert("SELECT * FROM users ORDER BY id DESC"); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.order_by).toBeDefined(); + expect(selectQuery.order_by!.length).toBe(1); + expect(selectQuery.order_by![0].order).toBe("DESC"); + expect((selectQuery.order_by![0].expr as Field).chain).toEqual(["id"]); + }); + + it("should convert SELECT with LIMIT", () => { + const ast = parseAndConvert("SELECT * FROM users LIMIT 10"); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.limit).toBeDefined(); + expect((selectQuery.limit as Constant).value).toBe(10); + }); + + it("should convert SELECT with LIMIT and OFFSET", () => { + const ast = parseAndConvert("SELECT * FROM users LIMIT 10 OFFSET 5"); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.limit).toBeDefined(); + expect((selectQuery.limit as Constant).value).toBe(10); + expect(selectQuery.offset).toBeDefined(); + expect((selectQuery.offset as Constant).value).toBe(5); + }); + + it("should convert SELECT with GROUP BY", () => { + const ast = parseAndConvert("SELECT category, COUNT(*) FROM products GROUP BY category"); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.group_by).toBeDefined(); + expect(selectQuery.group_by!.length).toBe(1); + expect((selectQuery.group_by![0] as Field).chain).toEqual(["category"]); + }); + + it("should convert SELECT with HAVING", () => { + const ast = parseAndConvert( + "SELECT category FROM products GROUP BY category HAVING COUNT(*) > 10" + ); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.having).toBeDefined(); + const havingExpr = selectQuery.having as CompareOperation; + expect(havingExpr.op).toBe(CompareOperationOp.Gt); + }); + }); + + describe("expressions", () => { + it("should convert numeric constants", () => { + const ast = parseAndConvert("SELECT 42 FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as Constant; + expect(expr.value).toBe(42); + }); + + it("should convert string constants", () => { + const ast = parseAndConvert("SELECT 'hello' FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as Constant; + expect(expr.value).toBe("hello"); + }); + + it("should convert boolean constants", () => { + const ast = parseAndConvert("SELECT true FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as Constant; + expect(expr.value).toBe(true); + }); + + it("should convert arithmetic addition", () => { + const ast = parseAndConvert("SELECT 1 + 2 FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as ArithmeticOperation; + expect(expr.op).toBe(ArithmeticOperationOp.Add); + expect((expr.left as Constant).value).toBe(1); + expect((expr.right as Constant).value).toBe(2); + }); + + it("should convert arithmetic subtraction", () => { + const ast = parseAndConvert("SELECT 5 - 3 FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as ArithmeticOperation; + expect(expr.op).toBe(ArithmeticOperationOp.Sub); + }); + + it("should convert arithmetic multiplication", () => { + const ast = parseAndConvert("SELECT 2 * 3 FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as ArithmeticOperation; + expect(expr.op).toBe(ArithmeticOperationOp.Mult); + }); + + it("should convert arithmetic division", () => { + const ast = parseAndConvert("SELECT 10 / 2 FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as ArithmeticOperation; + expect(expr.op).toBe(ArithmeticOperationOp.Div); + }); + + it("should convert comparison equals", () => { + const ast = parseAndConvert("SELECT * FROM users WHERE id = 1"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.where as CompareOperation; + expect(expr.op).toBe(CompareOperationOp.Eq); + }); + + it("should convert comparison not equals", () => { + const ast = parseAndConvert("SELECT * FROM users WHERE id != 1"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.where as CompareOperation; + expect(expr.op).toBe(CompareOperationOp.NotEq); + }); + + it("should convert comparison less than", () => { + const ast = parseAndConvert("SELECT * FROM users WHERE id < 10"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.where as CompareOperation; + expect(expr.op).toBe(CompareOperationOp.Lt); + }); + + it("should convert comparison greater than", () => { + const ast = parseAndConvert("SELECT * FROM users WHERE id > 5"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.where as CompareOperation; + expect(expr.op).toBe(CompareOperationOp.Gt); + }); + + it("should convert LIKE comparison", () => { + const ast = parseAndConvert("SELECT * FROM users WHERE name LIKE '%john%'"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.where as CompareOperation; + expect(expr.op).toBe(CompareOperationOp.Like); + }); + + it("should convert IN comparison", () => { + const ast = parseAndConvert("SELECT * FROM users WHERE id IN (1, 2, 3)"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.where as CompareOperation; + expect(expr.op).toBe(CompareOperationOp.In); + }); + + it("should convert function calls", () => { + const ast = parseAndConvert("SELECT COUNT(*) FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as Call; + expect(expr.name).toBe("count"); + }); + + it("should convert nested field access", () => { + const ast = parseAndConvert("SELECT user.profile.name FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as Field; + expect(expr.chain).toEqual(["user", "profile", "name"]); + }); + + it("should convert aliased expressions", () => { + const ast = parseAndConvert("SELECT id AS user_id FROM users"); + const selectQuery = ast as SelectQuery; + const expr = selectQuery.select[0] as Alias; + expect(expr.alias).toBe("user_id"); + expect((expr.expr as Field).chain).toEqual(["id"]); + }); + }); + + describe("JOINs", () => { + it("should convert INNER JOIN", () => { + const ast = parseAndConvert( + "SELECT * FROM users INNER JOIN orders ON users.id = orders.user_id" + ); + const selectQuery = ast as SelectQuery; + expect(selectQuery.select_from).toBeDefined(); + const joinExpr = selectQuery.select_from as JoinExpr; + expect(joinExpr.next_join).toBeDefined(); + expect(joinExpr.next_join!.join_type).toBe("INNER JOIN"); + expect(joinExpr.next_join!.constraint).toBeDefined(); + expect(joinExpr.next_join!.constraint!.constraint_type).toBe("ON"); + }); + + it("should convert LEFT JOIN", () => { + const ast = parseAndConvert( + "SELECT * FROM users LEFT JOIN orders ON users.id = orders.user_id" + ); + const selectQuery = ast as SelectQuery; + const joinExpr = selectQuery.select_from as JoinExpr; + expect(joinExpr.next_join!.join_type).toContain("LEFT"); + }); + + it("should convert CROSS JOIN", () => { + const ast = parseAndConvert("SELECT * FROM users CROSS JOIN orders"); + const selectQuery = ast as SelectQuery; + const joinExpr = selectQuery.select_from as JoinExpr; + expect(joinExpr.next_join!.join_type).toBe("CROSS JOIN"); + }); + }); + + describe("UNION queries", () => { + it("should convert UNION query", () => { + const ast = parseAndConvert("SELECT id FROM users UNION SELECT id FROM customers"); + expect("initial_select_query" in ast).toBe(true); + const setQuery = ast as SelectSetQuery; + expect(setQuery.initial_select_query).toBeDefined(); + expect(setQuery.subsequent_select_queries).toBeDefined(); + expect(setQuery.subsequent_select_queries.length).toBe(1); + expect(setQuery.subsequent_select_queries[0].set_operator).toBe("UNION"); + }); + + it("should convert UNION ALL query", () => { + const ast = parseAndConvert("SELECT id FROM users UNION ALL SELECT id FROM customers"); + const setQuery = ast as SelectSetQuery; + expect(setQuery.subsequent_select_queries[0].set_operator).toBe("UNION ALL"); + }); + }); + + describe("WITH clauses (CTEs)", () => { + it("should convert SELECT with WITH clause", () => { + const ast = parseAndConvert( + "WITH recent_users AS (SELECT * FROM users WHERE created_at > '2024-01-01') SELECT * FROM recent_users" + ); + const selectQuery = ast as SelectQuery; + expect(selectQuery.ctes).toBeDefined(); + expect(selectQuery.ctes!["recent_users"]).toBeDefined(); + expect(selectQuery.ctes!["recent_users"].cte_type).toBe("subquery"); + }); + }); + + describe("error handling", () => { + it("should preserve position information in errors", () => { + const input = "SELECT * FROM users WHERE invalid syntax"; + const inputStream = CharStreams.fromString(input); + const lexer = new TSQLLexer(inputStream); + const tokenStream = new CommonTokenStream(lexer); + const parser = new TSQLParser(tokenStream); + + // This might not parse correctly, but if it does and we visit an error node, + // it should throw with position info + try { + const parseTree = parser.select(); + const converter = new TSQLParseTreeConverter(); + converter.visit(parseTree); + } catch (error) { + if (error instanceof SyntaxError) { + // Error should have position information if available + expect(error).toBeInstanceOf(SyntaxError); + } + } + }); + }); + + describe("complex queries", () => { + it("should convert a complex query with multiple clauses", () => { + const ast = parseAndConvert( + "SELECT category, COUNT(*) as count " + + "FROM products " + + "WHERE price > 100 " + + "GROUP BY category " + + "HAVING COUNT(*) > 5 " + + "ORDER BY count DESC " + + "LIMIT 10" + ); + + const selectQuery = ast as SelectQuery; + expect(selectQuery.select.length).toBe(2); + expect(selectQuery.where).toBeDefined(); + expect(selectQuery.group_by).toBeDefined(); + expect(selectQuery.having).toBeDefined(); + expect(selectQuery.order_by).toBeDefined(); + expect(selectQuery.limit).toBeDefined(); + }); + + it("should convert query with multiple JOINs", () => { + const ast = parseAndConvert( + "SELECT * FROM users " + + "INNER JOIN orders ON users.id = orders.user_id " + + "LEFT JOIN products ON orders.product_id = products.id" + ); + + const selectQuery = ast as SelectQuery; + const joinExpr = selectQuery.select_from as JoinExpr; + expect(joinExpr.next_join).toBeDefined(); + expect(joinExpr.next_join!.next_join).toBeDefined(); + }); + }); +}); diff --git a/internal-packages/tsql/src/query/parser.ts b/internal-packages/tsql/src/query/parser.ts new file mode 100644 index 000000000..4ead761c4 --- /dev/null +++ b/internal-packages/tsql/src/query/parser.ts @@ -0,0 +1,1381 @@ +import { ParserRuleContext } from "antlr4ts/ParserRuleContext"; +import { ParseTree } from "antlr4ts/tree/ParseTree"; +import { TerminalNode } from "antlr4ts/tree/TerminalNode"; +import { ErrorNode } from "antlr4ts/tree/ErrorNode"; +import { Token } from "antlr4ts/Token"; +import { TSQLParserVisitor } from "../grammar/TSQLParserVisitor.js"; +import { + Alias, + And, + ArithmeticOperation, + ArithmeticOperationOp, + Array as ArrayExpression, + ArrayAccess, + BetweenExpr, + Block, + Call, + CompareOperation, + CompareOperationOp, + Constant, + CTE, + Dict, + Expr, + ExprCall, + ExprStatement, + Field, + ForInStatement, + ForStatement, + Function, + IfStatement, + JoinConstraint, + JoinExpr, + Lambda, + LimitByExpr, + Not, + OrderExpr, + Or, + Placeholder, + Program, + RatioExpr, + ReturnStatement, + SampleExpr, + SelectQuery, + SelectSetNode, + SelectSetQuery, + SetOperator, + Statement, + ThrowStatement, + TryCatchStatement, + Tuple, + TupleAccess, + VariableAssignment, + VariableDeclaration, + WhileStatement, + WindowExpr, + WindowFrameExpr, + WindowFunction, + HogQLXAttribute, + HogQLXTag, + Declaration, +} from "./ast"; +import { RESERVED_KEYWORDS } from "./constants"; +import { SyntaxError, BaseHogQLError, NotImplementedError } from "./errors"; +import type { HogQLTimings } from "./timings"; +import { parseStringLiteralCtx, parseStringLiteralText, parseStringTextCtx } from "./parse_string"; + +/** + * Token with position information. + * antlr4ts Token interface has startIndex/stopIndex, but runtime may also expose start/stop. + * This type represents the union of both possibilities. + */ +type TokenWithPosition = Token & { + start?: number; + stop?: number; +}; + +/** + * Extract start position from a token, handling both start and startIndex properties. + */ +function getTokenStart(token: Token | undefined): number | undefined { + if (!token) return undefined; + const tokenWithPos = token as TokenWithPosition; + // Try start first (runtime property), then fall back to startIndex (type-safe property) + return tokenWithPos.start ?? tokenWithPos.startIndex; +} + +/** + * Extract stop position from a token, handling both stop and stopIndex properties. + */ +function getTokenStop(token: Token | undefined): number | undefined { + if (!token) return undefined; + const tokenWithPos = token as TokenWithPosition; + // Try stop first (runtime property), then fall back to stopIndex (type-safe property) + return tokenWithPos.stop ?? tokenWithPos.stopIndex; +} + +/** + * Visitor that converts TSQL AST to a QueryConfig + * The QueryConfig can then be used to build a ClickhouseQueryBuilder + */ +export class TSQLParseTreeConverter implements TSQLParserVisitor { + start?: number; + + constructor(start?: number) { + this.start = start; + } + + visit(ctx: ParserRuleContext): any { + const start = getTokenStart(ctx.start); + const stop = getTokenStop(ctx.stop); + const end = stop !== undefined ? stop + 1 : undefined; + try { + const node = this.visitChildren(ctx); + if (node && typeof node === "object" && "start" in node && this.start !== undefined) { + node.start = start; + node.end = end; + } + return node; + } catch (e: any) { + if (e instanceof BaseHogQLError) { + if ( + start !== undefined && + end !== undefined && + (e.start === undefined || e.end === undefined) + ) { + e.start = start; + e.end = end; + } + } + throw e; + } + } + + /** + * Visit a parse tree node, dispatching to the appropriate visitor method. + * Uses type guards to safely handle ParseTree, ParserRuleContext, TerminalNode, and ErrorNode. + */ + private visitParseTree(node: ParseTree): any { + // ErrorNode extends TerminalNode, so check ErrorNode first + if (this.isErrorNode(node)) { + return this.visitErrorNode(node); + } + if (this.isTerminalNode(node)) { + return this.visitTerminal(node); + } + if (this.isParserRuleContext(node)) { + return this.visit(node); + } + // Fallback: use accept method for double dispatch + return node.accept(this); + } + + /** + * Type guard to check if a ParseTree is an ErrorNode. + */ + private isErrorNode(node: ParseTree): node is ErrorNode { + return "symbol" in node && node.symbol !== undefined; + } + + /** + * Type guard to check if a ParseTree is a TerminalNode. + */ + private isTerminalNode(node: ParseTree): node is TerminalNode { + return "symbol" in node && !("ruleIndex" in node); + } + + /** + * Type guard to check if a ParseTree is a ParserRuleContext. + */ + private isParserRuleContext(node: ParseTree): node is ParserRuleContext { + return "ruleIndex" in node && "start" in node; + } + + visitChildren(ctx: ParserRuleContext): any { + if (!ctx.children || ctx.children.length === 0) { + return null; + } + + const results: any[] = []; + for (const child of ctx.children) { + results.push(this.visitParseTree(child)); + } + + // Return single result if only one child, otherwise return array + return results.length === 1 ? results[0] : results; + } + + visitTerminal(node: TerminalNode): any { + // Terminal nodes are leaf nodes (tokens) in the parse tree + // Typically not needed for AST conversion, but required by interface + return null; + } + + visitErrorNode(node: ErrorNode): any { + // Error nodes represent syntax errors in the parse tree + // Throw a syntax error with position information + const symbol = node.symbol; + // ErrorNode has a symbol property with text + const text = symbol?.text || ""; + const start = getTokenStart(symbol); + const end = symbol ? (getTokenStop(symbol) ?? -1) + 1 : undefined; + throw new SyntaxError(`Syntax error: ${text}`, { + start, + end, + }); + } + + // Program and declarations + visitProgram(ctx: any): Program { + const declarations: Declaration[] = []; + // Implement based on your parser context structure + throw new NotImplementedError("visitProgram not implemented"); + } + + visitDeclaration(ctx: any): Declaration { + return this.visitChildren(ctx); + } + + visitExpression(ctx: any): Expr { + return this.visitChildren(ctx); + } + + visitVarDecl(ctx: any): VariableDeclaration { + return { + name: this.visitIdentifier(ctx.identifier()), + expr: ctx.expression() ? this.visit(ctx.expression()) : undefined, + }; + } + + visitVarAssignment(ctx: any): VariableAssignment { + return { + left: this.visit(ctx.expression(0)), + right: this.visit(ctx.expression(1)), + }; + } + + visitStatement(ctx: any): Statement { + return this.visitChildren(ctx); + } + + visitExprStmt(ctx: any): ExprStatement { + return { + expr: this.visit(ctx.expression()), + }; + } + + visitReturnStmt(ctx: any): ReturnStatement { + return { + expr: ctx.expression() ? this.visit(ctx.expression()) : undefined, + }; + } + + visitThrowStmt(ctx: any): ThrowStatement { + return { + expr: ctx.expression() ? this.visit(ctx.expression()) : undefined, + }; + } + + visitCatchBlock(ctx: any): [string | null, string | null, Statement] { + return [ + ctx.catchVar ? this.visit(ctx.catchVar) : null, + ctx.catchType ? this.visit(ctx.catchType) : null, + this.visit(ctx.catchStmt), + ]; + } + + visitTryCatchStmt(ctx: any): TryCatchStatement { + return { + try_stmt: this.visit(ctx.tryStmt), + catches: ctx.catchBlock().map((c: any) => this.visit(c)), + finally_stmt: ctx.finallyStmt ? this.visit(ctx.finallyStmt) : undefined, + }; + } + + visitIfStmt(ctx: any): IfStatement { + return { + expr: this.visit(ctx.expression()), + then: this.visit(ctx.statement(0)), + else_: ctx.statement(1) ? this.visit(ctx.statement(1)) : undefined, + }; + } + + visitWhileStmt(ctx: any): WhileStatement { + return { + expr: this.visit(ctx.expression()), + body: ctx.statement() ? this.visit(ctx.statement()) : undefined, + }; + } + + visitForInStmt(ctx: any): ForInStatement { + const firstIdentifier = this.visitIdentifier(ctx.identifier(0)); + const secondIdentifier = ctx.identifier(1) ? this.visitIdentifier(ctx.identifier(1)) : null; + return { + valueVar: secondIdentifier ?? firstIdentifier, + keyVar: secondIdentifier ? firstIdentifier : undefined, + expr: this.visit(ctx.expression()), + body: this.visit(ctx.statement()), + }; + } + + visitForStmt(ctx: any): ForStatement { + const initializer = + ctx.initializerVarDeclr || ctx.initializerVarAssignment || ctx.initializerExpression; + const increment = + ctx.incrementVarDeclr || ctx.incrementVarAssignment || ctx.incrementExpression; + + return { + initializer: initializer ? this.visit(initializer) : undefined, + condition: ctx.condition ? this.visit(ctx.condition) : undefined, + increment: increment ? this.visit(increment) : undefined, + body: this.visit(ctx.statement()), + }; + } + + visitFuncStmt(ctx: any): Function { + return { + name: this.visitIdentifier(ctx.identifier()), + params: ctx.identifierList() ? this.visit(ctx.identifierList()) : [], + body: this.visit(ctx.block()), + }; + } + + visitKvPairList(ctx: any): [Expr, Expr][] { + return ctx.kvPair().map((kv: any) => this.visit(kv)); + } + + visitKvPair(ctx: any): [Expr, Expr] { + const exprs = ctx.expression(); + return [this.visit(exprs[0]), this.visit(exprs[1])]; + } + + visitIdentifierList(ctx: any): string[] { + return ctx.identifier().map((ident: any) => this.visitIdentifier(ident)); + } + + visitEmptyStmt(ctx: any): ExprStatement { + return { expr: undefined }; + } + + visitBlock(ctx: any): Block { + const declarations: Declaration[] = []; + // Implement based on your parser structure + throw new NotImplementedError("visitBlock not implemented"); + } + + // SELECT statements + visitSelect(ctx: any): SelectQuery | SelectSetQuery | HogQLXTag { + return this.visit(ctx.selectSetStmt() || ctx.selectStmt() || ctx.hogqlxTagElement()); + } + + visitSelectSetStmt(ctx: any): SelectQuery | SelectSetQuery { + const selectQueries: SelectSetNode[] = []; + const initialQuery = this.visit(ctx.selectStmtWithParens()); + + for (const subsequent of ctx.subsequentSelectSetClause()) { + let unionType: SetOperator; + if (subsequent.UNION() && subsequent.ALL()) { + unionType = "UNION ALL"; + } else if (subsequent.UNION() && subsequent.DISTINCT()) { + unionType = "UNION DISTINCT"; + } else if (subsequent.INTERSECT() && subsequent.DISTINCT()) { + unionType = "INTERSECT DISTINCT"; + } else if (subsequent.INTERSECT()) { + unionType = "INTERSECT"; + } else if (subsequent.EXCEPT()) { + unionType = "EXCEPT"; + } else { + throw new SyntaxError( + "Set operator must be one of UNION ALL, UNION DISTINCT, INTERSECT, INTERSECT DISTINCT, and EXCEPT" + ); + } + const selectQuery = this.visit(subsequent.selectStmtWithParens()); + selectQueries.push({ + select_query: selectQuery, + set_operator: unionType, + }); + } + + if (selectQueries.length === 0) { + return initialQuery; + } + return { + initial_select_query: initialQuery, + subsequent_select_queries: selectQueries, + }; + } + + visitSelectStmtWithParens(ctx: any): SelectQuery | SelectSetQuery | Placeholder { + return this.visit(ctx.selectStmt() || ctx.selectSetStmt() || ctx.placeholder()); + } + + visitSelectStmt(ctx: any): SelectQuery { + const selectQuery: SelectQuery = { + ctes: ctx.withClause() ? this.visit(ctx.withClause()) : undefined, + select: ctx.columnExprList() ? this.visit(ctx.columnExprList()) : [], + distinct: ctx.DISTINCT() ? true : undefined, + select_from: ctx.fromClause() ? this.visit(ctx.fromClause()) : undefined, + where: ctx.whereClause() ? this.visit(ctx.whereClause()) : undefined, + prewhere: ctx.prewhereClause() ? this.visit(ctx.prewhereClause()) : undefined, + having: ctx.havingClause() ? this.visit(ctx.havingClause()) : undefined, + group_by: ctx.groupByClause() ? this.visit(ctx.groupByClause()) : undefined, + order_by: ctx.orderByClause() ? this.visit(ctx.orderByClause()) : undefined, + limit_by: ctx.limitByClause() ? this.visit(ctx.limitByClause()) : undefined, + }; + + if (ctx.windowClause()) { + selectQuery.window_exprs = {}; + const windowClause = ctx.windowClause(); + for (let index = 0; index < windowClause.windowExpr().length; index++) { + const name = this.visit(windowClause.identifier()[index]); + selectQuery.window_exprs![name] = this.visit(windowClause.windowExpr()[index]); + } + } + + if (ctx.limitAndOffsetClause()) { + const limitAndOffsetClause = ctx.limitAndOffsetClause(); + selectQuery.limit = this.visit(limitAndOffsetClause.columnExpr(0)); + if (limitAndOffsetClause.columnExpr(1)) { + selectQuery.offset = this.visit(limitAndOffsetClause.columnExpr(1)); + } + if (limitAndOffsetClause.WITH() && limitAndOffsetClause.TIES()) { + selectQuery.limit_with_ties = true; + } + } else if (ctx.offsetOnlyClause()) { + selectQuery.offset = this.visit(ctx.offsetOnlyClause().columnExpr()); + } + + if (ctx.arrayJoinClause()) { + const arrayJoinClause = ctx.arrayJoinClause(); + if (!selectQuery.select_from) { + throw new SyntaxError("Using ARRAY JOIN without a FROM clause is not permitted"); + } + if (arrayJoinClause.LEFT()) { + selectQuery.array_join_op = "LEFT ARRAY JOIN"; + } else if (arrayJoinClause.INNER()) { + selectQuery.array_join_op = "INNER ARRAY JOIN"; + } else { + selectQuery.array_join_op = "ARRAY JOIN"; + } + selectQuery.array_join_list = this.visit(arrayJoinClause.columnExprList()); + if (selectQuery.array_join_list) { + for (const expr of selectQuery.array_join_list) { + if (!("alias" in expr)) { + throw new SyntaxError("ARRAY JOIN arrays must have an alias", { + start: expr.start, + end: expr.end, + }); + } + } + } + } + + if (ctx.topClause()) { + throw new NotImplementedError("Unsupported: SelectStmt.topClause()"); + } + if (ctx.settingsClause()) { + throw new NotImplementedError("Unsupported: SelectStmt.settingsClause()"); + } + + return selectQuery; + } + + visitWithClause(ctx: any): Record { + return this.visit(ctx.withExprList()); + } + + visitFromClause(ctx: any): JoinExpr { + return this.visit(ctx.joinExpr()); + } + + visitPrewhereClause(ctx: any): Expr { + return this.visit(ctx.columnExpr()); + } + + visitWhereClause(ctx: any): Expr { + return this.visit(ctx.columnExpr()); + } + + visitGroupByClause(ctx: any): Expr[] { + return this.visit(ctx.columnExprList()); + } + + visitHavingClause(ctx: any): Expr { + return this.visit(ctx.columnExpr()); + } + + visitOrderByClause(ctx: any): OrderExpr[] { + return this.visit(ctx.orderExprList()); + } + + visitLimitByClause(ctx: any): LimitByExpr { + const limitExpr = this.visit(ctx.limitExpr()); + + // If limitExpr is a tuple (n, offset), split it + if (Array.isArray(limitExpr) && limitExpr.length === 2) { + const [n, offsetValue] = limitExpr; + return { + n, + offset_value: offsetValue, + exprs: this.visit(ctx.columnExprList()), + }; + } + + // If no offset, just use limitExpr as n + return { + n: limitExpr, + offset_value: undefined, + exprs: this.visit(ctx.columnExprList()), + }; + } + + visitLimitExpr(ctx: any): Expr | [Expr, Expr] { + const n = this.visit(ctx.columnExpr(0)); + + // Check if we have an offset (second expression) + if (ctx.columnExpr(1)) { + const offsetValue = this.visit(ctx.columnExpr(1)); + // For "LIMIT a, b" syntax: a is offset, b is limit + if (ctx.COMMA()) { + return [offsetValue, n]; // Return tuple as (offset, limit) + } + // For "LIMIT a OFFSET b" syntax: a is limit, b is offset + return [n, offsetValue]; + } + + return n; + } + + // JOIN expressions + visitJoinExprOp(ctx: any): JoinExpr { + const join1: JoinExpr = this.visit(ctx.joinExpr(0)); + const join2: JoinExpr = this.visit(ctx.joinExpr(1)); + + if (ctx.joinOp()) { + join2.join_type = `${this.visit(ctx.joinOp())} JOIN`; + } else { + join2.join_type = "JOIN"; + } + join2.constraint = this.visit(ctx.joinConstraintClause()); + + let lastJoin = join1; + while (lastJoin.next_join) { + lastJoin = lastJoin.next_join; + } + lastJoin.next_join = join2; + + return join1; + } + + visitJoinExprTable(ctx: any): JoinExpr { + const sample = ctx.sampleClause() ? this.visit(ctx.sampleClause()) : undefined; + const table = this.visit(ctx.tableExpr()); + const tableFinal = ctx.FINAL() ? true : undefined; + if ("table" in table) { + // visitTableExprAlias returns a JoinExpr to pass the alias + // visitTableExprFunction returns a JoinExpr to pass the args + table.table_final = tableFinal; + table.sample = sample; + return table; + } + return { + table, + table_final: tableFinal, + sample, + }; + } + + visitJoinExprParens(ctx: any): JoinExpr { + return this.visit(ctx.joinExpr()); + } + + visitJoinExprCrossOp(ctx: any): JoinExpr { + const join1: JoinExpr = this.visit(ctx.joinExpr(0)); + const join2: JoinExpr = this.visit(ctx.joinExpr(1)); + join2.join_type = "CROSS JOIN"; + let lastJoin = join1; + while (lastJoin.next_join) { + lastJoin = lastJoin.next_join; + } + lastJoin.next_join = join2; + return join1; + } + + visitJoinOpInner(ctx: any): string { + const tokens: string[] = []; + if (ctx.ALL()) tokens.push("ALL"); + if (ctx.ANY()) tokens.push("ANY"); + if (ctx.ASOF()) tokens.push("ASOF"); + tokens.push("INNER"); + return tokens.join(" "); + } + + visitJoinOpLeftRight(ctx: any): string { + const tokens: string[] = []; + if (ctx.LEFT()) tokens.push("LEFT"); + if (ctx.RIGHT()) tokens.push("RIGHT"); + if (ctx.OUTER()) tokens.push("OUTER"); + if (ctx.SEMI()) tokens.push("SEMI"); + if (ctx.ALL()) tokens.push("ALL"); + if (ctx.ANTI()) tokens.push("ANTI"); + if (ctx.ANY()) tokens.push("ANY"); + if (ctx.ASOF()) tokens.push("ASOF"); + return tokens.join(" "); + } + + visitJoinOpFull(ctx: any): string { + const tokens: string[] = []; + if (ctx.FULL()) tokens.push("FULL"); + if (ctx.OUTER()) tokens.push("OUTER"); + if (ctx.ALL()) tokens.push("ALL"); + if (ctx.ANY()) tokens.push("ANY"); + return tokens.join(" "); + } + + visitJoinConstraintClause(ctx: any): JoinConstraint { + const columnExprList = this.visit(ctx.columnExprList()); + if (columnExprList.length !== 1) { + throw new NotImplementedError("Unsupported: JOIN ... ON with multiple expressions"); + } + return { + expr: columnExprList[0], + constraint_type: ctx.USING() ? "USING" : "ON", + }; + } + + visitSampleClause(ctx: any): SampleExpr { + const ratioExpressions = ctx.ratioExpr(); + const sampleRatioExpr = this.visit(ratioExpressions[0]); + const offsetRatioExpr = + ratioExpressions.length > 1 && ctx.OFFSET() ? this.visit(ratioExpressions[1]) : undefined; + + return { + sample_value: sampleRatioExpr, + offset_value: offsetRatioExpr, + }; + } + + visitOrderExprList(ctx: any): OrderExpr[] { + return ctx.orderExpr().map((expr: any) => this.visit(expr)); + } + + visitOrderExpr(ctx: any): OrderExpr { + const order = ctx.DESC() || ctx.DESCENDING() ? "DESC" : "ASC"; + return { + expr: this.visit(ctx.columnExpr()), + order: order as "ASC" | "DESC", + }; + } + + visitRatioExpr(ctx: any): RatioExpr { + if (ctx.placeholder()) { + return this.visit(ctx.placeholder()); + } + + const numberLiterals = ctx.numberLiteral(); + const left = numberLiterals[0]; + const right = ctx.SLASH() && numberLiterals.length > 1 ? numberLiterals[1] : null; + + return { + left: this.visitNumberLiteral(left), + right: right ? this.visitNumberLiteral(right) : undefined, + }; + } + + visitWindowExpr(ctx: any): WindowExpr { + const frame = ctx.winFrameClause(); + const visitedFrame = frame ? this.visit(frame) : undefined; + return { + partition_by: ctx.winPartitionByClause() ? this.visit(ctx.winPartitionByClause()) : undefined, + order_by: ctx.winOrderByClause() ? this.visit(ctx.winOrderByClause()) : undefined, + frame_method: frame && frame.RANGE() ? "RANGE" : frame && frame.ROWS() ? "ROWS" : undefined, + frame_start: Array.isArray(visitedFrame) ? visitedFrame[0] : visitedFrame, + frame_end: Array.isArray(visitedFrame) ? visitedFrame[1] : undefined, + }; + } + + visitWinPartitionByClause(ctx: any): Expr[] { + return this.visit(ctx.columnExprList()); + } + + visitWinOrderByClause(ctx: any): OrderExpr[] { + return this.visit(ctx.orderExprList()); + } + + visitWinFrameClause(ctx: any): WindowFrameExpr | [WindowFrameExpr, WindowFrameExpr] { + return this.visit(ctx.winFrameExtend()); + } + + visitFrameStart(ctx: any): WindowFrameExpr { + return this.visit(ctx.winFrameBound()); + } + + visitFrameBetween(ctx: any): [WindowFrameExpr, WindowFrameExpr] { + return [this.visit(ctx.winFrameBound(0)), this.visit(ctx.winFrameBound(1))]; + } + + visitWinFrameBound(ctx: any): WindowFrameExpr { + if (ctx.PRECEDING()) { + return { + frame_type: "PRECEDING", + frame_value: ctx.numberLiteral() + ? (this.visit(ctx.numberLiteral()) as Constant).value + : undefined, + }; + } + if (ctx.FOLLOWING()) { + return { + frame_type: "FOLLOWING", + frame_value: ctx.numberLiteral() + ? (this.visit(ctx.numberLiteral()) as Constant).value + : undefined, + }; + } + return { frame_type: "CURRENT ROW" }; + } + + // Column expressions + visitColumnExprList(ctx: any): Expr[] { + return ctx.columnExpr().map((c: any) => this.visit(c)); + } + + visitColumnExprTernaryOp(ctx: any): Call { + return { + name: "if", + args: [ + this.visit(ctx.columnExpr(0)), + this.visit(ctx.columnExpr(1)), + this.visit(ctx.columnExpr(2)), + ], + }; + } + + visitColumnExprAlias(ctx: any): Alias { + let alias: string; + if (ctx.identifier()) { + alias = this.visitIdentifier(ctx.identifier()); + } else if (ctx.STRING_LITERAL()) { + alias = parseStringLiteralCtx(ctx.STRING_LITERAL()); + } else { + throw new SyntaxError("Must specify an alias"); + } + const expr = this.visit(ctx.columnExpr()); + + if (RESERVED_KEYWORDS.includes(alias.toLowerCase() as any)) { + throw new SyntaxError( + `"${alias}" cannot be an alias or identifier, as it's a reserved keyword` + ); + } + + return { expr, alias }; + } + + visitColumnExprNegate(ctx: any): ArithmeticOperation { + return { + op: ArithmeticOperationOp.Sub, + left: { value: 0 } as Constant, + right: this.visit(ctx.columnExpr()), + }; + } + + visitColumnExprDict(ctx: any): Dict { + return { + items: ctx.kvPairList() ? this.visit(ctx.kvPairList()) : [], + }; + } + + visitColumnExprSubquery(ctx: any): SelectQuery | SelectSetQuery { + return this.visit(ctx.selectSetStmt()); + } + + visitColumnExprLiteral(ctx: any): Expr { + return this.visitChildren(ctx); + } + + visitColumnExprArray(ctx: any): ArrayExpression { + return { + exprs: ctx.columnExprList() ? this.visit(ctx.columnExprList()) : [], + }; + } + + visitColumnExprPrecedence1(ctx: any): ArithmeticOperation { + let op: ArithmeticOperationOp; + if (ctx.SLASH()) { + op = ArithmeticOperationOp.Div; + } else if (ctx.ASTERISK()) { + op = ArithmeticOperationOp.Mult; + } else if (ctx.PERCENT()) { + op = ArithmeticOperationOp.Mod; + } else { + throw new NotImplementedError(`Unsupported ColumnExprPrecedence1: ${ctx.getText()}`); + } + const left = this.visit(ctx.left); + const right = this.visit(ctx.right); + return { left, right, op }; + } + + visitColumnExprPrecedence2(ctx: any): ArithmeticOperation | Call { + const left = this.visit(ctx.left); + const right = this.visit(ctx.right); + + if (ctx.PLUS()) { + return { left, right, op: ArithmeticOperationOp.Add }; + } else if (ctx.DASH()) { + return { left, right, op: ArithmeticOperationOp.Sub }; + } else if (ctx.CONCAT()) { + const args: Expr[] = []; + if ("name" in left && left.name === "concat" && "args" in left) { + args.push(...left.args); + } else { + args.push(left); + } + + if ("name" in right && right.name === "concat" && "args" in right) { + args.push(...right.args); + } else { + args.push(right); + } + + return { name: "concat", args }; + } else { + throw new NotImplementedError(`Unsupported ColumnExprPrecedence2: ${ctx.getText()}`); + } + } + + visitColumnExprPrecedence3(ctx: any): CompareOperation { + const left = this.visit(ctx.left); + const right = this.visit(ctx.right); + + let op: CompareOperationOp; + if (ctx.EQ_SINGLE() || ctx.EQ_DOUBLE()) { + op = CompareOperationOp.Eq; + } else if (ctx.NOT_EQ()) { + op = CompareOperationOp.NotEq; + } else if (ctx.LT()) { + op = CompareOperationOp.Lt; + } else if (ctx.LT_EQ()) { + op = CompareOperationOp.LtEq; + } else if (ctx.GT()) { + op = CompareOperationOp.Gt; + } else if (ctx.GT_EQ()) { + op = CompareOperationOp.GtEq; + } else if (ctx.LIKE()) { + op = ctx.NOT() ? CompareOperationOp.NotLike : CompareOperationOp.Like; + } else if (ctx.ILIKE()) { + op = ctx.NOT() ? CompareOperationOp.NotILike : CompareOperationOp.ILike; + } else if (ctx.REGEX_SINGLE() || ctx.REGEX_DOUBLE()) { + op = CompareOperationOp.Regex; + } else if (ctx.NOT_REGEX()) { + op = CompareOperationOp.NotRegex; + } else if (ctx.IREGEX_SINGLE() || ctx.IREGEX_DOUBLE()) { + op = CompareOperationOp.IRegex; + } else if (ctx.NOT_IREGEX()) { + op = CompareOperationOp.NotIRegex; + } else if (ctx.IN()) { + if (ctx.COHORT()) { + op = ctx.NOT() ? CompareOperationOp.NotInCohort : CompareOperationOp.InCohort; + } else { + op = ctx.NOT() ? CompareOperationOp.NotIn : CompareOperationOp.In; + } + } else { + throw new NotImplementedError(`Unsupported ColumnExprPrecedence3: ${ctx.getText()}`); + } + + return { left, right, op }; + } + + visitColumnExprInterval(ctx: any): Call { + let name: string; + const interval = ctx.interval(); + if (interval.SECOND()) { + name = "toIntervalSecond"; + } else if (interval.MINUTE()) { + name = "toIntervalMinute"; + } else if (interval.HOUR()) { + name = "toIntervalHour"; + } else if (interval.DAY()) { + name = "toIntervalDay"; + } else if (interval.WEEK()) { + name = "toIntervalWeek"; + } else if (interval.MONTH()) { + name = "toIntervalMonth"; + } else if (interval.QUARTER()) { + name = "toIntervalQuarter"; + } else if (interval.YEAR()) { + name = "toIntervalYear"; + } else { + throw new NotImplementedError(`Unsupported interval type: ${interval.getText()}`); + } + + return { name, args: [this.visit(ctx.columnExpr())] }; + } + + visitColumnExprIsNull(ctx: any): CompareOperation { + return { + left: this.visit(ctx.columnExpr()), + right: { value: null } as Constant, + op: ctx.NOT() ? CompareOperationOp.NotEq : CompareOperationOp.Eq, + }; + } + + visitColumnExprTuple(ctx: any): Tuple { + return { + exprs: ctx.columnExprList() ? this.visit(ctx.columnExprList()) : [], + }; + } + + visitColumnExprArrayAccess(ctx: any): ArrayAccess { + const object: Expr = this.visit(ctx.columnExpr(0)); + const property: Expr = this.visit(ctx.columnExpr(1)); + return { array: object, property }; + } + + visitColumnExprNullArrayAccess(ctx: any): ArrayAccess { + const object: Expr = this.visit(ctx.columnExpr(0)); + const property: Expr = this.visit(ctx.columnExpr(1)); + return { array: object, property, nullish: true }; + } + + visitColumnExprPropertyAccess(ctx: any): ArrayAccess { + const object = this.visit(ctx.columnExpr()); + const property = { value: this.visitIdentifier(ctx.identifier()) } as Constant; + return { array: object, property }; + } + + visitColumnExprNullPropertyAccess(ctx: any): ArrayAccess { + const object = this.visit(ctx.columnExpr()); + const property = { value: this.visitIdentifier(ctx.identifier()) } as Constant; + return { array: object, property, nullish: true }; + } + + visitColumnExprBetween(ctx: any): BetweenExpr { + return { + expr: this.visit(ctx.columnExpr(0)), + low: this.visit(ctx.columnExpr(1)), + high: this.visit(ctx.columnExpr(2)), + negated: !!ctx.NOT(), + }; + } + + visitColumnExprParens(ctx: any): Expr { + return this.visit(ctx.columnExpr()); + } + + visitColumnExprAnd(ctx: any): And { + let left = this.visit(ctx.columnExpr(0)); + const leftArray = "exprs" in left ? left.exprs : [left]; + + let right = this.visit(ctx.columnExpr(1)); + const rightArray = "exprs" in right ? right.exprs : [right]; + + return { exprs: [...leftArray, ...rightArray] }; + } + + visitColumnExprOr(ctx: any): Or { + let left = this.visit(ctx.columnExpr(0)); + const leftArray = "exprs" in left ? left.exprs : [left]; + + let right = this.visit(ctx.columnExpr(1)); + const rightArray = "exprs" in right ? right.exprs : [right]; + + return { exprs: [...leftArray, ...rightArray] }; + } + + visitColumnExprTupleAccess(ctx: any): TupleAccess { + const tuple = this.visit(ctx.columnExpr()); + const index = parseInt(ctx.DECIMAL_LITERAL().getText()); + return { tuple, index }; + } + + visitColumnExprNullTupleAccess(ctx: any): TupleAccess { + const tuple = this.visit(ctx.columnExpr()); + const index = parseInt(ctx.DECIMAL_LITERAL().getText()); + return { tuple, index, nullish: true }; + } + + visitColumnExprCase(ctx: any): Call { + const columns = ctx.columnExpr().map((column: any) => this.visit(column)); + if (ctx.caseExpr) { + const args: Expr[] = [ + columns[0], + { exprs: [] } as ArrayExpression, + { exprs: [] } as ArrayExpression, + , + columns[columns.length - 1], + ]; + for (let index = 1; index < columns.length - 1; index++) { + const arrayIndex = ((index - 1) % 2) + 1; + (args[arrayIndex] as ArrayExpression).exprs.push(columns[index]); + } + return { name: "transform", args }; + } else if (columns.length === 3) { + return { name: "if", args: columns }; + } else { + return { name: "multiIf", args: columns }; + } + } + + visitColumnExprNot(ctx: any): Not { + return { expr: this.visit(ctx.columnExpr()) }; + } + + visitColumnExprWinFunctionTarget(ctx: any): WindowFunction { + return { + name: this.visitIdentifier(ctx.identifier(0)), + exprs: ctx.columnExprs ? this.visit(ctx.columnExprs) : [], + args: ctx.columnArgList ? this.visit(ctx.columnArgList) : [], + over_identifier: this.visitIdentifier(ctx.identifier(1)), + }; + } + + visitColumnExprWinFunction(ctx: any): WindowFunction { + return { + name: this.visitIdentifier(ctx.identifier()), + exprs: ctx.columnExprs ? this.visit(ctx.columnExprs) : [], + args: ctx.columnArgList ? this.visit(ctx.columnArgList) : [], + over_expr: ctx.windowExpr() ? this.visit(ctx.windowExpr()) : undefined, + }; + } + + visitColumnExprIdentifier(ctx: any): Expr { + return this.visit(ctx.columnIdentifier()); + } + + visitColumnExprFunction(ctx: any): Call { + const name = this.visitIdentifier(ctx.identifier()); + + let parameters: Expr[] | undefined = ctx.columnExprs ? this.visit(ctx.columnExprs) : undefined; + // two sets of parameters fn()(), return an empty list for the first even if no parameters + if (ctx.LPAREN && ctx.LPAREN().length > 1 && parameters === undefined) { + parameters = []; + } + + const args: Expr[] = ctx.columnArgList ? this.visit(ctx.columnArgList) : []; + const distinct = ctx.DISTINCT() ? true : false; + return { name, params: parameters, args, distinct }; + } + + visitColumnExprAsterisk(ctx: any): Field { + if (ctx.tableIdentifier()) { + const table = this.visit(ctx.tableIdentifier()); + return { chain: [...table, "*"] }; + } + return { chain: ["*"] }; + } + + visitColumnLambdaExpr(ctx: any): Lambda { + return { + args: ctx.identifier().map((identifier: any) => this.visitIdentifier(identifier)), + expr: ctx.columnExpr() ? this.visit(ctx.columnExpr()) : this.visit(ctx.block()), + }; + } + + visitWithExprList(ctx: any): Record { + const ctes: Record = {}; + for (const expr of ctx.withExpr()) { + const cte = this.visit(expr); + ctes[cte.name] = cte; + } + return ctes; + } + + visitWithExprSubquery(ctx: any): CTE { + const subquery = this.visit(ctx.selectSetStmt()); + const name = this.visitIdentifier(ctx.identifier()); + return { name, expr: subquery, cte_type: "subquery" }; + } + + visitWithExprColumn(ctx: any): CTE { + const expr = this.visit(ctx.columnExpr()); + const name = this.visitIdentifier(ctx.identifier()); + return { name, expr, cte_type: "column" }; + } + + visitColumnIdentifier(ctx: any): Expr { + if (ctx.placeholder()) { + return this.visit(ctx.placeholder()); + } + + const table = ctx.tableIdentifier() ? this.visit(ctx.tableIdentifier()) : []; + const nested = ctx.nestedIdentifier() ? this.visit(ctx.nestedIdentifier()) : []; + + if (table.length === 0 && nested.length > 0) { + const text = ctx.getText().toLowerCase(); + if (text === "true") { + return { value: true } as Constant; + } + if (text === "false") { + return { value: false } as Constant; + } + return { chain: nested } as Field; + } + + return { chain: [...table, ...nested] } as Field; + } + + visitNestedIdentifier(ctx: any): string[] { + return ctx.identifier().map((identifier: any) => this.visitIdentifier(identifier)); + } + + visitTableExprIdentifier(ctx: any): Field { + const chain = this.visit(ctx.tableIdentifier()); + return { chain }; + } + + visitTableExprSubquery(ctx: any): SelectQuery | SelectSetQuery { + return this.visit(ctx.selectSetStmt()); + } + + visitTableExprPlaceholder(ctx: any): Placeholder { + return this.visit(ctx.placeholder()); + } + + visitTableExprAlias(ctx: any): JoinExpr { + const alias: string = this.visit(ctx.alias() || ctx.identifier()); + if (RESERVED_KEYWORDS.includes(alias.toLowerCase() as any)) { + throw new SyntaxError( + `"${alias}" cannot be an alias or identifier, as it's a reserved keyword` + ); + } + const table = this.visit(ctx.tableExpr()); + if ("table" in table) { + table.alias = alias; + return table; + } + return { table, alias }; + } + + visitTableExprFunction(ctx: any): JoinExpr { + return this.visit(ctx.tableFunctionExpr()); + } + + visitTableExprTag(ctx: any): HogQLXTag { + return this.visit(ctx.hogqlxTagElement()); + } + + visitTableFunctionExpr(ctx: any): JoinExpr { + const name = this.visitIdentifier(ctx.identifier()); + const args = ctx.tableArgList() ? this.visit(ctx.tableArgList()) : []; + return { table: { chain: [name] } as Field, table_args: args }; + } + + visitTableIdentifier(ctx: any): string[] { + const nested = ctx.nestedIdentifier() ? this.visit(ctx.nestedIdentifier()) : []; + + if (ctx.databaseIdentifier()) { + return [this.visit(ctx.databaseIdentifier()), ...nested]; + } + + return nested; + } + + visitTableArgList(ctx: any): Expr[] { + return ctx.columnExpr().map((arg: any) => this.visit(arg)); + } + + visitDatabaseIdentifier(ctx: any): string { + return this.visitIdentifier(ctx.identifier()); + } + + visitNumberLiteral(ctx: any): Constant { + const text = ctx.getText().toLowerCase(); + if ( + text.includes(".") || + text.includes("e") || + text === "-inf" || + text === "inf" || + text === "nan" + ) { + return { value: parseFloat(text) }; + } + return { value: parseInt(text) }; + } + + visitLiteral(ctx: any): Constant { + if (ctx.NULL_SQL()) { + return { value: null }; + } + if (ctx.STRING_LITERAL()) { + const text = parseStringLiteralCtx(ctx); + return { value: text }; + } + return this.visitChildren(ctx); + } + + visitAlias(ctx: any): string { + let text = ctx.getText(); + if ( + text.length >= 2 && + ((text.startsWith("`") && text.endsWith("`")) || (text.startsWith('"') && text.endsWith('"'))) + ) { + text = parseStringLiteralText(text); + } + return text; + } + + visitIdentifier(ctx: any): string { + let text = ctx.getText(); + if ( + text.length >= 2 && + ((text.startsWith("`") && text.endsWith("`")) || (text.startsWith('"') && text.endsWith('"'))) + ) { + text = parseStringLiteralText(text); + } + return text; + } + + visitColumnExprNullish(ctx: any): Call { + return { + name: "ifNull", + args: [this.visit(ctx.columnExpr(0)), this.visit(ctx.columnExpr(1))], + }; + } + + visitColumnExprCall(ctx: any): ExprCall { + return { + expr: this.visit(ctx.columnExpr()), + args: ctx.columnExprList() ? this.visit(ctx.columnExprList()) : [], + }; + } + + visitColumnExprCallSelect(ctx: any): Call | ExprCall { + const expr = this.visit(ctx.columnExpr()); + if ("chain" in expr && expr.chain.length === 1) { + return { + name: String(expr.chain[0]), + args: [this.visit(ctx.selectSetStmt())], + }; + } + return { + expr, + args: [this.visit(ctx.selectSetStmt())], + }; + } + + visitHogqlxChildElement(ctx: any): Expr | HogQLXTag { + if (ctx.hogqlxTagElement()) { + return this.visit(ctx.hogqlxTagElement()); + } + if (ctx.hogqlxText()) { + return this.visit(ctx.hogqlxText()); + } + return this.visit(ctx.columnExpr()); + } + + visitHogqlxText(ctx: any): Constant { + return { value: ctx.HOGQLX_TEXT_TEXT().getText() }; + } + + visitHogqlxTagElementClosed(ctx: any): HogQLXTag { + const kind = this.visitIdentifier(ctx.identifier()); + const attributes = ctx.hogqlxTagAttribute() + ? ctx.hogqlxTagAttribute().map((a: any) => this.visit(a)) + : []; + return { kind, attributes }; + } + + visitHogqlxTagElementNested(ctx: any): HogQLXTag { + const opening = this.visitIdentifier(ctx.identifier(0)); + const closing = this.visitIdentifier(ctx.identifier(1)); + if (opening !== closing) { + throw new SyntaxError( + `Opening and closing HogQLX tags must match. Got ${opening} and ${closing}` + ); + } + + const attributes = ctx.hogqlxTagAttribute() + ? ctx.hogqlxTagAttribute().map((a: any) => this.visit(a)) + : []; + + // ── collect child nodes, discarding pure-indentation whitespace ── + const keptChildren: Expr[] = []; + for (const element of ctx.hogqlxChildElement()) { + const child = this.visit(element); + + if ("value" in child && typeof child.value === "string") { + const v = child.value; + const onlyWs = /^\s*$/.test(v); + const hasNl = v.includes("\n") || v.includes("\r"); + if (onlyWs && hasNl) { + continue; // drop indentation text node + } + } + + keptChildren.push(child); + } + + if (keptChildren.length > 0) { + if (attributes.some((a: HogQLXAttribute) => a.name === "children")) { + throw new SyntaxError( + "Can't have a HogQLX tag with both children and a 'children' attribute" + ); + } + attributes.push({ name: "children", value: keptChildren }); + } + + return { kind: opening, attributes }; + } + + visitHogqlxTagAttribute(ctx: any): HogQLXAttribute { + const name = this.visitIdentifier(ctx.identifier()); + if (ctx.columnExpr()) { + return { name, value: this.visit(ctx.columnExpr()) }; + } else if (ctx.string()) { + return { name, value: this.visit(ctx.string()) }; + } else { + return { name, value: { value: true } as Constant }; + } + } + + visitPlaceholder(ctx: any): Placeholder { + return { expr: this.visit(ctx.columnExpr()) }; + } + + visitColumnExprTemplateString(ctx: any): Expr { + return this.visit(ctx.templateString()); + } + + visitString(ctx: any): Constant | Expr { + if (ctx.STRING_LITERAL()) { + return { value: parseStringLiteralCtx(ctx.STRING_LITERAL()) }; + } + return this.visit(ctx.templateString()); + } + + visitTemplateString(ctx: any): Constant | Call { + const pieces: Expr[] = []; + for (const chunk of ctx.stringContents()) { + pieces.push(this.visit(chunk)); + } + + if (pieces.length === 0) { + return { value: "" }; + } else if (pieces.length === 1) { + const first = pieces[0]; + // If it's already a Constant or Call, return as-is, otherwise wrap in Call + if ("value" in first || "name" in first) { + return first as Constant | Call; + } + return { name: "concat", args: [first] }; + } + + return { name: "concat", args: pieces }; + } + + visitFullTemplateString(ctx: any): Constant | Call { + const pieces: Expr[] = []; + for (const chunk of ctx.stringContentsFull()) { + pieces.push(this.visit(chunk)); + } + + if (pieces.length === 0) { + return { value: "" }; + } else if (pieces.length === 1) { + const first = pieces[0]; + // If it's already a Constant or Call, return as-is, otherwise wrap in Call + if ("value" in first || "name" in first) { + return first as Constant | Call; + } + return { name: "concat", args: [first] }; + } + + return { name: "concat", args: pieces }; + } + + visitStringContents(ctx: any): Constant | Expr { + if (ctx.STRING_TEXT()) { + return { value: parseStringTextCtx(ctx.STRING_TEXT(), true) }; + } else if (ctx.columnExpr()) { + return this.visit(ctx.columnExpr()); + } + return { value: "" }; + } + + visitStringContentsFull(ctx: any): Constant | Expr { + if (ctx.FULL_STRING_TEXT()) { + return { value: parseStringTextCtx(ctx.FULL_STRING_TEXT(), false) }; + } else if (ctx.columnExpr()) { + return this.visit(ctx.columnExpr()); + } + return { value: "" }; + } +} diff --git a/internal-packages/tsql/src/query/property_types.ts b/internal-packages/tsql/src/query/property_types.ts new file mode 100644 index 000000000..9fefa579e --- /dev/null +++ b/internal-packages/tsql/src/query/property_types.ts @@ -0,0 +1,716 @@ +// TypeScript translation of posthog/hogql/transforms/property_types.py +// Keep this file in sync with the Python version + +import type { + AST, + Expr, + Field, + PropertyType, + FieldType, + BaseTableType, + VirtualTableType, + LazyJoinType, + LazyTableType, + Call, + Constant, + CallType, + DateTimeType, +} from "./ast"; +import type { HogQLContext } from "./context"; +import type { BooleanDatabaseField, DateTimeDatabaseField, Table } from "./models"; + +// Helper function to escape HogQL identifiers +function escapeHogQLIdentifier(identifier: string | number): string { + if (typeof identifier === "number") { + return String(identifier); + } + if (identifier.includes("%")) { + throw new Error( + `The HogQL identifier "${identifier}" is not permitted as it contains the "%" character` + ); + } + // HogQL allows dollars in the identifier + if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(identifier)) { + return identifier; + } + // Escape backticks and other special characters + const backquoteEscapeChars: Record = { + "\b": "\\b", + "\f": "\\f", + "\r": "\\r", + "\n": "\\n", + "\t": "\\t", + "\0": "\\0", + a: "\\a", + "\v": "\\v", + "\\": "\\\\", + "`": "\\`", + }; + return `\`${identifier + .split("") + .map((c) => backquoteEscapeChars[c] || c) + .join("")}\``; +} + +// Visitor dispatcher - converts node type to visitor method name +// Matches Python's camel_case_pattern.sub("_", class_name).lower() logic +function getVisitorMethodName(node: AST): string { + // Get the constructor name or use a type guard to determine the type + const nodeType = (node as any).constructor?.name || detectNodeType(node); + + if (!nodeType) { + return "visit_unknown"; + } + + // Convert CamelCase to snake_case (e.g., "PropertyType" -> "property_type") + const snakeCase = nodeType + .replace(/([A-Z])/g, "_$1") + .toLowerCase() + .replace(/^_/, ""); + + // Handle special cases (matching Python replacements) + const replacements: Record = { + hog_qlxtag: "hogqlx_tag", + hog_qlxattribute: "hogqlx_attribute", + uuidtype: "uuid_type", + string_jsontype: "string_json_type", + }; + + return replacements[snakeCase] || snakeCase; +} + +// Type detection helper (since we can't use instanceof with interfaces) +function detectNodeType(node: AST): string { + // Use property presence to detect node types + if ("chain" in node && "type" in node && !("name" in node)) { + return "Field"; + } + if ("chain" in node && "field_type" in node) { + return "PropertyType"; + } + if ("name" in node && "args" in node && !("expr" in node)) { + return "Call"; + } + if ("value" in node && !("name" in node) && !("args" in node)) { + return "Constant"; + } + // Add more type detection as needed + return ""; +} + +// Base visitor class - matches Python Visitor pattern +abstract class Visitor { + visit(node: AST | null | undefined): T { + if (node === null || node === undefined) { + return node as T; + } + + // Try using accept method if available (double dispatch) + if (node.accept) { + return node.accept(this) as T; + } + + // Fallback: use dispatcher + const methodName = getVisitorMethodName(node); + const method = (this as any)[methodName]; + + if (method && typeof method === "function") { + return method.call(this, node) as T; + } + + // Try visit_unknown as fallback + if ((this as any).visit_unknown) { + return (this as any).visit_unknown(node) as T; + } + + throw new Error(`${this.constructor.name} has no method ${methodName} or visit_unknown`); + } +} + +// TraversingVisitor - matches Python TraversingVisitor +class TraversingVisitor extends Visitor { + visitPropertyType(node: PropertyType): void { + this.visit(node.field_type); + } + + visitField(node: Field): void { + if (node.type) { + this.visit(node.type as any); + } + } + + visitCall(node: Call): void { + for (const arg of node.args) { + this.visit(arg); + } + if (node.params) { + for (const param of node.params) { + this.visit(param); + } + } + } + + visitConstant(node: Constant): void { + if (node.type) { + this.visit(node.type as any); + } + } + + // Default handler for unknown types - traverse common properties + visit_unknown(node: AST): void { + // Traverse children based on common AST node properties + if ("expr" in node) { + this.visit((node as any).expr); + } + if ("exprs" in node) { + for (const expr of (node as any).exprs) { + this.visit(expr); + } + } + if ("left" in node && "right" in node) { + this.visit((node as any).left); + this.visit((node as any).right); + } + if ("args" in node) { + for (const arg of (node as any).args) { + this.visit(arg); + } + } + if ("type" in node) { + this.visit((node as any).type); + } + } +} + +// CloningVisitor - matches Python CloningVisitor +class CloningVisitor extends Visitor { + protected clearTypes: boolean; + protected clearLocations: boolean; + + constructor(clearTypes: boolean = true, clearLocations: boolean = false) { + super(); + this.clearTypes = clearTypes; + this.clearLocations = clearLocations; + } + + visitField(node: Field): Field { + return { + ...node, + type: this.clearTypes ? undefined : node.type ? this.visit(node.type as any) : node.type, + start: this.clearLocations ? undefined : node.start, + end: this.clearLocations ? undefined : node.end, + }; + } + + visitPropertyType(node: PropertyType): PropertyType { + return { + ...node, + field_type: this.visit(node.field_type) as FieldType, + start: this.clearLocations ? undefined : node.start, + end: this.clearLocations ? undefined : node.end, + }; + } + + visitCall(node: Call): Call { + return { + ...node, + args: node.args.map((arg) => this.visit(arg)), + params: node.params ? node.params.map((param) => this.visit(param)) : undefined, + start: this.clearLocations ? undefined : node.start, + end: this.clearLocations ? undefined : node.end, + type: this.clearTypes ? undefined : node.type, + }; + } + + visitConstant(node: Constant): Constant { + return { + ...node, + start: this.clearLocations ? undefined : node.start, + end: this.clearLocations ? undefined : node.end, + type: this.clearTypes ? undefined : node.type, + }; + } + + // Default handler for unknown types - shallow clone + visit_unknown(node: AST): any { + const cloned: any = { ...node }; + + // Clone common properties + if ("expr" in node) { + cloned.expr = this.visit((node as any).expr); + } + if ("exprs" in node) { + cloned.exprs = (node as any).exprs.map((e: any) => this.visit(e)); + } + if ("left" in node && "right" in node) { + cloned.left = this.visit((node as any).left); + cloned.right = this.visit((node as any).right); + } + if ("args" in node) { + cloned.args = (node as any).args.map((a: any) => this.visit(a)); + } + if ("type" in node) { + cloned.type = this.clearTypes ? undefined : this.visit((node as any).type); + } + + if (this.clearLocations) { + cloned.start = undefined; + cloned.end = undefined; + } + + return cloned; + } +} + +// PropertyFinder: Traverses AST to find all property references +class PropertyFinder extends TraversingVisitor { + context: HogQLContext; + personProperties: Set = new Set(); + eventProperties: Set = new Set(); + groupProperties: Map> = new Map(); + foundTimestamps: boolean = false; + + constructor(context: HogQLContext) { + super(); + this.context = context; + } + + visitPropertyType(node: PropertyType): void { + if (node.field_type.name === "properties" && node.chain.length === 1) { + const tableType = node.field_type.table_type; + if (this.isBaseTableType(tableType)) { + const table = tableType.resolve_database_table?.(this.context); + if (table) { + const tableName = table.to_printed_hogql?.() || ""; + const propertyName = String(node.chain[0]); + + if (tableName === "persons" || tableName === "raw_persons") { + this.personProperties.add(propertyName); + } else if (tableName === "groups") { + if (this.isLazyJoinType(tableType)) { + if (tableType.field.startsWith("group_")) { + const groupId = parseInt(tableType.field.split("_")[1], 10); + if (!this.groupProperties.has(groupId)) { + this.groupProperties.set(groupId, new Set()); + } + this.groupProperties.get(groupId)!.add(propertyName); + } + } else if (this.isLazyTableType(tableType)) { + const globalGroupId = this.context.globals?.group_id; + if (typeof globalGroupId === "number") { + if (!this.groupProperties.has(globalGroupId)) { + this.groupProperties.set(globalGroupId, new Set()); + } + this.groupProperties.get(globalGroupId)!.add(propertyName); + } + } + } else if (tableName === "events") { + if (this.isVirtualTableType(tableType) && tableType.field === "poe") { + this.personProperties.add(propertyName); + } else { + this.eventProperties.add(propertyName); + } + } + } + } + } + super.visitPropertyType(node); + } + + visitField(node: Field): void { + super.visitField(node); + if (this.isFieldType(node.type)) { + const dbField = (node.type as any).resolve_database_field?.(this.context); + if (this.isDateTimeDatabaseField(dbField)) { + this.foundTimestamps = true; + } + } + } + + private isBaseTableType(type: any): type is BaseTableType { + return type && typeof type.resolve_database_table === "function"; + } + + private isLazyJoinType(type: any): type is LazyJoinType { + return type && "lazy_join" in type && "field" in type; + } + + private isLazyTableType(type: any): type is LazyTableType { + return type && "table" in type && !("lazy_join" in type); + } + + private isVirtualTableType(type: any): type is VirtualTableType { + return type && "virtual_table" in type && "field" in type; + } + + private isFieldType(type: any): type is FieldType { + return type && typeof type.resolve_database_field === "function"; + } + + private isDateTimeDatabaseField(field: any): field is DateTimeDatabaseField { + return field && "name" in field; // Simplified check + } +} + +// PropertySwapper: Transforms property accesses with type conversions +export class PropertySwapper extends CloningVisitor { + timezone: string; + eventProperties: Map; + personProperties: Map; + groupProperties: Map; + context: HogQLContext; + setTimeZones: boolean; + + constructor( + timezone: string, + eventProperties: Map | Record, + personProperties: Map | Record, + groupProperties: Map | Record, + context: HogQLContext, + setTimeZones: boolean + ) { + super(false); // Don't clear types + this.timezone = timezone; + this.eventProperties = + eventProperties instanceof Map ? eventProperties : new Map(Object.entries(eventProperties)); + this.personProperties = + personProperties instanceof Map + ? personProperties + : new Map(Object.entries(personProperties)); + this.groupProperties = + groupProperties instanceof Map ? groupProperties : new Map(Object.entries(groupProperties)); + this.context = context; + this.setTimeZones = setTimeZones; + } + + visitField(node: Field): any { + if (this.isFieldType(node.type)) { + if (this.setTimeZones) { + const dbField = (node.type as any).resolve_database_field?.(this.context); + if (this.isDateTimeDatabaseField(dbField)) { + return this.createToTimeZoneCall(node); + } + } + + if (this.isLazyJoinType(node.type.table_type)) { + const lazyJoinType = node.type.table_type; + const resolvedTable = lazyJoinType.lazy_join.resolve_table?.(this.context); + // Check if it's an S3Table-like table (has fields property) + if (resolvedTable && "fields" in resolvedTable) { + const field = node.chain[node.chain.length - 1]; + const fieldType = resolvedTable.fields[String(field)]; + let propType = "String"; + + if (this.isDateTimeDatabaseField(fieldType)) { + propType = "DateTime"; + } else if (this.isBooleanDatabaseField(fieldType)) { + propType = "Boolean"; + } + + return this.fieldTypeToPropertyCall(node, propType); + } + } + } + + const type = node.type; + if ( + this.isPropertyType(type) && + type.field_type.name === "properties" && + type.chain.length === 1 + ) { + const propertyName = String(type.chain[0]); + const tableType = type.field_type.table_type; + + if (this.isVirtualTableType(tableType) && tableType.field === "poe") { + if (this.personProperties.has(propertyName)) { + return this.convertStringPropertyToType(node, "person", propertyName); + } + } else if (this.isBaseTableType(tableType)) { + const table = tableType.resolve_database_table?.(this.context); + if (table) { + const tableName = table.to_printed_hogql?.() || ""; + + if (tableName === "persons" || tableName === "raw_persons") { + if (this.personProperties.has(propertyName)) { + return this.convertStringPropertyToType(node, "person", propertyName); + } + } else if (tableName === "groups") { + if (this.isLazyJoinType(tableType)) { + if (tableType.field.startsWith("group_")) { + const groupId = parseInt(tableType.field.split("_")[1], 10); + const groupKey = `${groupId}_${propertyName}`; + if (this.groupProperties.has(groupKey)) { + return this.convertStringPropertyToType(node, "group", groupKey); + } + } + } else if (this.isLazyTableType(tableType)) { + const globalGroupId = this.context.globals?.group_id; + if (typeof globalGroupId === "number") { + const groupKey = `${globalGroupId}_${propertyName}`; + if (this.groupProperties.has(groupKey)) { + return this.convertStringPropertyToType(node, "group", groupKey); + } + } + } + } else if (tableName === "events") { + if (this.eventProperties.has(propertyName)) { + return this.convertStringPropertyToType(node, "event", propertyName); + } + } + } + } + } + + if ( + this.isPropertyType(type) && + type.field_type.name === "person_properties" && + type.chain.length === 1 + ) { + const propertyName = String(type.chain[0]); + const tableType = type.field_type.table_type; + + if (this.isBaseTableType(tableType)) { + const table = tableType.resolve_database_table?.(this.context); + if (table) { + const tableName = table.to_printed_hogql?.() || ""; + if (tableName === "events") { + if (this.personProperties.has(propertyName)) { + return this.convertStringPropertyToType(node, "person", propertyName); + } + } + } + } + } + + return super.visitField(node); + } + + private convertStringPropertyToType( + node: Field, + propertyType: "event" | "person" | "group", + propertyName: string + ): Expr { + let posthogFieldType: string | undefined; + if (propertyType === "person") { + posthogFieldType = this.personProperties.get(propertyName); + } else if (propertyType === "group") { + posthogFieldType = this.groupProperties.get(propertyName); + } else { + posthogFieldType = this.eventProperties.get(propertyName); + } + + const fieldType = posthogFieldType === "Numeric" ? "Float" : posthogFieldType || "String"; + this.addPropertyNotice(node, propertyType, fieldType); + + return this.fieldTypeToPropertyCall(node, fieldType); + } + + private fieldTypeToPropertyCall(node: Field, fieldType: string): Expr { + if (fieldType === "DateTime") { + return this.createToDateTimeCall(node); + } + if (fieldType === "Float") { + return this.createToFloatCall(node); + } + if (fieldType === "Boolean") { + return this.createToBoolCall(node); + } + return node; + } + + private createToTimeZoneCall(node: Field): Call { + return { + name: "toTimeZone", + args: [node, this.createConstant(this.timezone)], + type: { + name: "toTimeZone", + arg_types: [{ data_type: "datetime" } as DateTimeType], + return_type: { data_type: "datetime" } as DateTimeType, + } as CallType, + start: node.start, + end: node.end, + } as Call; + } + + private createToDateTimeCall(node: Field): Call { + return { + name: "toDateTime", + args: [node], + start: node.start, + end: node.end, + }; + } + + private createToFloatCall(node: Field): Call { + return { + name: "toFloat", + args: [node], + start: node.start, + end: node.end, + }; + } + + private createToBoolCall(node: Field): Call { + return { + name: "toBool", + args: [ + { + name: "transform", + args: [ + { + name: "toString", + args: [node], + start: node.start, + end: node.end, + } as Call, + this.createConstant(["true", "false"]), + this.createConstant([1, 0]), + this.createConstant(null), + ], + start: node.start, + end: node.end, + } as Call, + ], + start: node.start, + end: node.end, + } as Call; + } + + private createConstant(value: any): Constant { + return { + value, + }; + } + + private addPropertyNotice( + node: Field, + propertyType: "event" | "person" | "group", + fieldType: string + ): void { + let propertyName = String(node.chain[node.chain.length - 1]); + let materializedColumn: any = null; // MaterializedColumn type not yet defined + + if (propertyType === "person") { + // if (this.context.modifiers.personsOnEventsMode !== "disabled") { + // materializedColumn = getMaterializedColumnForProperty('events', propertyName, 'person_properties'); + // } else { + // materializedColumn = getMaterializedColumnForProperty('person', propertyName, 'properties'); + // } + } else if (propertyType === "group") { + const nameParts = propertyName.split("_"); + nameParts.shift(); + propertyName = nameParts.join("_"); + // materializedColumn = getMaterializedColumnForProperty('groups', propertyName, 'properties'); + } else { + // materializedColumn = getMaterializedColumnForProperty('events', propertyName, 'properties'); + } + + let message = `${ + propertyType.charAt(0).toUpperCase() + propertyType.slice(1) + } property '${propertyName}' is of type '${fieldType}'.`; + if (this.context.debug) { + if (materializedColumn !== null) { + message += " This property is materialized ⚡️."; + } else { + message += " This property is not materialized 🐢."; + } + } + + this.addNotice(node, message); + } + + private addNotice(node: Field, message: string): void { + if (node.start === undefined || node.end === undefined) { + return; // Don't add notices for nodes without location + } + // Only highlight the last part of the chain + const lastPart = node.chain[node.chain.length - 1]; + const identifierLength = escapeHogQLIdentifier(lastPart).length; + this.context.notices.push({ + start: Math.max(node.start, node.end - identifierLength), + end: node.end, + message, + }); + } + + private isFieldType(type: any): type is FieldType { + return type && typeof type.resolve_database_field === "function"; + } + + private isPropertyType(type: any): type is PropertyType { + return type && "field_type" in type && "chain" in type; + } + + private isBaseTableType(type: any): type is BaseTableType { + return type && typeof type.resolve_database_table === "function"; + } + + private isLazyJoinType(type: any): type is LazyJoinType { + return type && "lazy_join" in type && "field" in type; + } + + private isLazyTableType(type: any): type is LazyTableType { + return type && "table" in type && !("lazy_join" in type); + } + + private isVirtualTableType(type: any): type is VirtualTableType { + return type && "virtual_table" in type && "field" in type; + } + + private isDateTimeDatabaseField(field: any): field is DateTimeDatabaseField { + return field && "name" in field; // Simplified check + } + + private isBooleanDatabaseField(field: any): field is BooleanDatabaseField { + return field && "name" in field; // Simplified check + } +} + +// Main function to build property swapper +export function buildPropertySwapper(node: AST, context: HogQLContext): void { + if (!context || !context.team_id) { + return; + } + + // NOTE: In TypeScript, you'll need to fetch the team from your database/ORM + // This is a placeholder - replace with your actual team fetching logic + // if (!context.team) { + // context.team = await Team.findById(context.team_id); + // } + + if (!context.team) { + return; + } + + // Find all properties + const propertyFinder = new PropertyFinder(context); + propertyFinder.visit(node); + + // NOTE: In TypeScript, you'll need to query PropertyDefinition from your database + // This is a placeholder - replace with your actual property definition fetching logic + // const eventPropertyValues = await PropertyDefinition.find({ + // project_id: context.team.project_id, + // name: { $in: Array.from(propertyFinder.eventProperties) }, + // type: { $in: [null, 'event'] }, + // }).select('name property_type'); + // const eventProperties = new Map( + // eventPropertyValues.filter((p: any) => p.property_type).map((p: any) => [p.name, p.property_type]) + // ); + + const eventProperties = new Map(); + const personProperties = new Map(); + const groupProperties = new Map(); + + // TODO: Implement actual property definition fetching from database + // For now, these are empty maps + + const timezone = (context.database as any)?._timezone || "UTC"; + context.property_swapper = new PropertySwapper( + timezone, + eventProperties, + personProperties, + groupProperties, + context, + true + ); +} diff --git a/internal-packages/tsql/src/query/TQuery.ts b/internal-packages/tsql/src/query/query.ts similarity index 100% rename from internal-packages/tsql/src/query/TQuery.ts rename to internal-packages/tsql/src/query/query.ts diff --git a/internal-packages/tsql/src/query/timings.ts b/internal-packages/tsql/src/query/timings.ts new file mode 100644 index 000000000..17838abfa --- /dev/null +++ b/internal-packages/tsql/src/query/timings.ts @@ -0,0 +1,108 @@ +// TypeScript translation of posthog/hogql/timings.py +// Keep this file in sync with the Python version + +/** + * Get performance counter in milliseconds (Node.js equivalent of perf_counter) + * Uses performance.now() which is available in: + * - Node.js 18+ (global) + * - Browser (global) + * - Node.js <18 via perf_hooks module + */ +function getPerformanceNow(): number { + // Check for global performance (Node.js 18+ or browser) + if (typeof globalThis !== 'undefined' && 'performance' in globalThis) { + const perf = (globalThis as any).performance; + if (perf && typeof perf.now === 'function') { + return perf.now(); + } + } + + // Fallback to Date.now() if performance API is not available + // Note: This is less precise but works everywhere + return Date.now(); +} + +export interface QueryTiming { + key: string; // Key identifying the timing measurement + time: number; // Time in seconds +} + +const TIMING_DECIMAL_PLACES = 3; // round to milliseconds + +// Not thread safe. +// See trends_query_runner for an example of how to use for multithreaded queries +export class HogQLTimings { + // Completed time in seconds for different parts of the HogQL query + timings: Record = {}; + + // Used for housekeeping + private _timingPointer: string; + private _timingStarts: Record = {}; + + constructor(_timingPointer: string = '.') { + this._timingPointer = _timingPointer; + this._timingStarts[this._timingPointer] = this.perfCounter(); + } + + cloneForSubquery(seriesIndex: number): HogQLTimings { + return new HogQLTimings(`${this._timingPointer}/series_${seriesIndex}`); + } + + clearTimings(): void { + this.timings = {}; + } + + /** + * Measure execution time of a function. + * Usage: timings.measure('operation', () => { ... }); + */ + measure(key: string, fn: () => T): T { + const lastKey = this._timingPointer; + const fullKey = `${this._timingPointer}/${key}`; + this._timingPointer = fullKey; + this._timingStarts[fullKey] = this.perfCounter(); + + try { + return fn(); + } finally { + const duration = (this.perfCounter() - this._timingStarts[fullKey]) / 1000; // Convert to seconds + this.timings[fullKey] = (this.timings[fullKey] || 0.0) + duration; + delete this._timingStarts[fullKey]; + this._timingPointer = lastKey; + } + } + + /** + * Get performance counter in milliseconds (Node.js equivalent of perf_counter) + */ + private perfCounter(): number { + return getPerformanceNow(); + } + + toDict(): Record { + const timings = { ...this.timings }; + // Process in reverse order to handle nested timings correctly + const keys = Object.keys(this._timingStarts).reverse(); + for (const key of keys) { + const start = this._timingStarts[key]; + const elapsed = (this.perfCounter() - start) / 1000; // Convert to seconds + timings[key] = this.round((timings[key] || 0.0) + elapsed); + } + return timings; + } + + toList(backOutStack: boolean = true): QueryTiming[] { + const timingDict = backOutStack ? this.toDict() : this.timings; + return Object.entries(timingDict).map(([key, time]) => ({ + key: key, + time: this.round(time), + })); + } + + /** + * Round to specified decimal places (milliseconds precision) + */ + private round(value: number): number { + return Math.round(value * Math.pow(10, TIMING_DECIMAL_PLACES)) / Math.pow(10, TIMING_DECIMAL_PLACES); + } +}