diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts new file mode 100644 index 000000000..c4bf7e171 --- /dev/null +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -0,0 +1,181 @@ +/** + * TSQL Query Execution for ClickHouse + * + * This module provides a safe interface for executing TSQL queries against ClickHouse + * with automatic tenant isolation and SQL injection protection. + */ + +import type { ClickHouseSettings } from "@clickhouse/client"; +import { z } from "zod"; +import type { ClickhouseReader } from "./types.js"; +import { QueryError } from "./errors.js"; + +/** + * Options for executing a TSQL query + */ +export interface ExecuteTSQLOptions { + /** The name of the operation (for logging/tracing) */ + name: string; + /** The TSQL query string to execute */ + query: string; + /** The Zod schema for validating output rows */ + schema: TOut; + /** The organization ID for tenant isolation */ + organizationId: string; + /** The project ID for tenant isolation */ + projectId: string; + /** The environment ID for tenant isolation */ + environmentId: string; + /** Schema registry defining allowed tables and columns */ + tableSchema: TSQLTableSchema[]; + /** Optional ClickHouse query settings */ + settings?: ClickHouseSettings; + /** Maximum number of rows to return (default: 10000) */ + maxRows?: number; + /** Timezone for date/time operations (default: UTC) */ + timezone?: string; +} + +/** + * Schema definition for a table accessible via TSQL + */ +export interface TSQLTableSchema { + /** The name of the table as used in TSQL queries */ + name: string; + /** The fully qualified ClickHouse table name */ + clickhouseName: string; + /** Column definitions */ + columns: Record; + /** Tenant isolation column configuration */ + tenantColumns: { + organizationId: string; + projectId: string; + environmentId: string; + }; +} + +/** + * Schema definition for a column + */ +export interface TSQLColumnSchema { + /** The column name */ + name: string; + /** The ClickHouse column name (if different from name) */ + clickhouseName?: string; + /** Whether the column can be selected */ + selectable?: boolean; + /** Whether the column can be used in WHERE */ + filterable?: boolean; + /** Whether the column can be used in ORDER BY */ + sortable?: boolean; + /** Whether the column can be used in GROUP BY */ + groupable?: boolean; +} + +/** + * Result type for TSQL query execution + */ +export type TSQLQueryResult = [QueryError, null] | [null, T[]]; + +/** + * Execute a TSQL query against ClickHouse + * + * This function: + * 1. Parses the TSQL query into an AST + * 2. Validates tables and columns against the schema + * 3. Injects tenant isolation WHERE clauses + * 4. Generates parameterized ClickHouse SQL + * 5. Executes the query and returns validated results + * + * @example + * ```typescript + * const [error, rows] = await executeTSQL(reader, { + * name: "get_task_runs", + * query: "SELECT id, status FROM task_runs WHERE status = 'completed' ORDER BY created_at DESC LIMIT 100", + * schema: z.object({ id: z.string(), status: z.string() }), + * organizationId: "org_123", + * projectId: "proj_456", + * environmentId: "env_789", + * tableSchema: [taskRunsSchema], + * }); + * ``` + */ +export async function executeTSQL( + reader: ClickhouseReader, + options: ExecuteTSQLOptions +): Promise>> { + // Lazy import to avoid circular dependencies and keep the TSQL package optional + const { + parseTSQLSelect, + createPrinterContext, + createSchemaRegistry, + printToClickHouse, + } = await import("@internal/tsql"); + + try { + // 1. Parse the TSQL query + const ast = parseTSQLSelect(options.query); + + // 2. Create schema registry from table schemas + const schemaRegistry = createSchemaRegistry(options.tableSchema); + + // 3. Create printer context with tenant IDs + const context = createPrinterContext({ + organizationId: options.organizationId, + projectId: options.projectId, + environmentId: options.environmentId, + schema: schemaRegistry, + settings: { + maxRows: options.maxRows ?? 10000, + timezone: options.timezone ?? "UTC", + }, + }); + + // 4. Print the AST to ClickHouse SQL + const { sql, params } = printToClickHouse(ast, context); + + // 5. Execute the query + const queryFn = reader.query({ + name: options.name, + query: sql, + params: z.record(z.any()), + schema: options.schema, + settings: options.settings, + }); + + return await queryFn(params); + } catch (error) { + if (error instanceof Error) { + return [new QueryError(error.message, { query: options.query }), null]; + } + return [new QueryError("Unknown error executing TSQL query", { query: options.query }), null]; + } +} + +/** + * Create a reusable TSQL query executor bound to specific table schemas + * + * @example + * ```typescript + * const tsqlExecutor = createTSQLExecutor(reader, [taskRunsSchema, taskEventsSchema]); + * + * const [error, rows] = await tsqlExecutor.execute({ + * name: "get_task_runs", + * query: "SELECT * FROM task_runs LIMIT 10", + * schema: taskRunRowSchema, + * organizationId: "org_123", + * projectId: "proj_456", + * environmentId: "env_789", + * }); + * ``` + */ +export function createTSQLExecutor(reader: ClickhouseReader, tableSchema: TSQLTableSchema[]) { + return { + execute: ( + options: Omit, "tableSchema"> + ): Promise>> => { + return executeTSQL(reader, { ...options, tableSchema }); + }, + }; +} + diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 4aceeb92d..f36fecc2a 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -31,6 +31,16 @@ export type * from "./taskRuns.js"; export type * from "./taskEvents.js"; export type * from "./client/queryBuilder.js"; +// TSQL query execution +export { + executeTSQL, + createTSQLExecutor, + type ExecuteTSQLOptions, + type TSQLTableSchema, + type TSQLColumnSchema, + type TSQLQueryResult, +} from "./client/tsql.js"; + export type ClickhouseCommonConfig = { keepAlive?: { enabled?: boolean; diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index 6580918fb..937527644 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -1 +1,166 @@ -import { Logger } from "@trigger.dev/core/logger"; +// TSQL - Type-Safe SQL Query Language +// A TypeScript port of PostHog's HogQL for ClickHouse queries + +import { CharStreams, CommonTokenStream } from "antlr4ts"; +import type { ANTLRErrorListener, RecognitionException, Recognizer } from "antlr4ts"; +import type { Token } from "antlr4ts/Token"; +import { TSQLLexer } from "./grammar/TSQLLexer.js"; +import { TSQLParser } from "./grammar/TSQLParser.js"; +import { TSQLParseTreeConverter } from "./query/parser.js"; +import type { SelectQuery, SelectSetQuery, Expression } from "./query/ast.js"; +import { SyntaxError } from "./query/errors.js"; + +/** + * Simple error listener that captures syntax errors + */ +class TSQLErrorListener implements ANTLRErrorListener { + public error: string | null = null; + + syntaxError( + _recognizer: Recognizer, + _offendingSymbol: Token | undefined, + line: number, + charPositionInLine: number, + msg: string, + _e: RecognitionException | undefined + ): void { + this.error = `Syntax error at line ${line}:${charPositionInLine}: ${msg}`; + } +} + +// Re-export AST types +export * from "./query/ast.js"; + +// Re-export errors +export * from "./query/errors.js"; + +// Re-export escape utilities +export { + escapeClickHouseIdentifier, + escapeTSQLIdentifier, + escapeClickHouseString, + escapeTSQLString, + getClickHouseType, +} from "./query/escape.js"; + +// Re-export function definitions +export { + TSQL_CLICKHOUSE_FUNCTIONS, + TSQL_AGGREGATIONS, + TSQL_COMPARISON_MAPPING, + findTSQLAggregation, + findTSQLFunction, + getAllExposedFunctionNames, + type TSQLFunctionMeta, +} from "./query/functions.js"; + +// Re-export schema types and functions +export { + type TableSchema, + type ColumnSchema, + type TenantColumnConfig, + type SchemaRegistry, + type ClickHouseType, + createSchemaRegistry, + findTable, + findColumn, + validateTable, + validateSelectColumn, + validateFilterColumn, + validateSortColumn, + validateGroupColumn, + column, +} from "./query/schema.js"; + +// Re-export printer context +export { + PrinterContext, + createPrinterContext, + type PrinterContextOptions, + type QuerySettings, + type QueryNotice, + DEFAULT_QUERY_SETTINGS, +} from "./query/printer_context.js"; + +// Re-export printer +export { ClickHousePrinter, printToClickHouse, type PrintResult } from "./query/printer.js"; + +// Re-export parser converter for advanced usage +export { TSQLParseTreeConverter } from "./query/parser.js"; + +/** + * Parse a TSQL SELECT query string into an AST + * + * @param query - The TSQL query string to parse + * @returns The parsed AST (SelectQuery or SelectSetQuery) + * @throws SyntaxError if the query is invalid + * + * @example + * ```typescript + * const ast = parseTSQLSelect("SELECT * FROM users WHERE id = 1"); + * ``` + */ +export function parseTSQLSelect(query: string): SelectQuery | SelectSetQuery { + const inputStream = CharStreams.fromString(query); + const lexer = new TSQLLexer(inputStream); + const tokenStream = new CommonTokenStream(lexer); + const parser = new TSQLParser(tokenStream); + + // Remove default error listeners and add custom one + parser.removeErrorListeners(); + const errorListener = new TSQLErrorListener(); + parser.addErrorListener(errorListener); + + const parseTree = parser.select(); + + if (errorListener.error) { + throw new SyntaxError(errorListener.error); + } + + const converter = new TSQLParseTreeConverter(); + const ast = converter.visit(parseTree); + + // Validate the result is a select query + if (typeof ast === "string" || !("expression_type" in ast)) { + throw new SyntaxError("Failed to parse SELECT query"); + } + + if (ast.expression_type !== "select_query" && ast.expression_type !== "select_set_query") { + throw new SyntaxError(`Expected SELECT query, got ${ast.expression_type}`); + } + + return ast as SelectQuery | SelectSetQuery; +} + +/** + * Parse a TSQL expression string into an AST + * + * @param expr - The TSQL expression string to parse + * @returns The parsed expression AST + * @throws SyntaxError if the expression is invalid + * + * @example + * ```typescript + * const ast = parseTSQLExpr("id = 1 AND name = 'test'"); + * ``` + */ +export function parseTSQLExpr(expr: string): Expression { + const inputStream = CharStreams.fromString(expr); + const lexer = new TSQLLexer(inputStream); + const tokenStream = new CommonTokenStream(lexer); + const parser = new TSQLParser(tokenStream); + + // Remove default error listeners and add custom one + parser.removeErrorListeners(); + const errorListener = new TSQLErrorListener(); + parser.addErrorListener(errorListener); + + const parseTree = parser.columnExpr(0); + + if (errorListener.error) { + throw new SyntaxError(errorListener.error); + } + + const converter = new TSQLParseTreeConverter(); + return converter.visit(parseTree) as Expression; +} diff --git a/internal-packages/tsql/src/query/escape.test.ts b/internal-packages/tsql/src/query/escape.test.ts new file mode 100644 index 000000000..ffce1f29b --- /dev/null +++ b/internal-packages/tsql/src/query/escape.test.ts @@ -0,0 +1,223 @@ +import { describe, it, expect } from "vitest"; +import { + escapeClickHouseIdentifier, + escapeTSQLIdentifier, + escapeClickHouseString, + escapeTSQLString, + getClickHouseType, + SQLValueEscaper, + safeIdentifier, +} from "./escape.js"; +import { QueryError } from "./errors.js"; + +describe("escapeClickHouseIdentifier", () => { + it("should pass through simple identifiers", () => { + expect(escapeClickHouseIdentifier("id")).toBe("id"); + expect(escapeClickHouseIdentifier("user_name")).toBe("user_name"); + expect(escapeClickHouseIdentifier("Column1")).toBe("Column1"); + expect(escapeClickHouseIdentifier("_private")).toBe("_private"); + }); + + it("should escape identifiers with special characters", () => { + expect(escapeClickHouseIdentifier("my column")).toBe("`my column`"); + expect(escapeClickHouseIdentifier("table-name")).toBe("`table-name`"); + expect(escapeClickHouseIdentifier("column.with.dots")).toBe("`column.with.dots`"); + }); + + it("should escape identifiers starting with numbers", () => { + expect(escapeClickHouseIdentifier("1column")).toBe("`1column`"); + expect(escapeClickHouseIdentifier("123")).toBe("`123`"); + }); + + it("should escape backticks in identifiers", () => { + expect(escapeClickHouseIdentifier("column`name")).toBe("`column\\`name`"); + }); + + it("should escape control characters", () => { + expect(escapeClickHouseIdentifier("col\nname")).toBe("`col\\nname`"); + expect(escapeClickHouseIdentifier("col\tname")).toBe("`col\\tname`"); + }); + + it("should throw for identifiers containing %", () => { + expect(() => escapeClickHouseIdentifier("column%name")).toThrow(QueryError); + }); +}); + +describe("escapeTSQLIdentifier", () => { + it("should pass through simple identifiers", () => { + expect(escapeTSQLIdentifier("id")).toBe("id"); + expect(escapeTSQLIdentifier("user_name")).toBe("user_name"); + }); + + it("should allow dollar signs in identifiers", () => { + expect(escapeTSQLIdentifier("$property")).toBe("$property"); + expect(escapeTSQLIdentifier("property$value")).toBe("property$value"); + }); + + it("should handle numeric identifiers", () => { + expect(escapeTSQLIdentifier(0)).toBe("0"); + expect(escapeTSQLIdentifier(123)).toBe("123"); + }); + + it("should throw for identifiers containing %", () => { + expect(() => escapeTSQLIdentifier("column%name")).toThrow(QueryError); + }); +}); + +describe("SQLValueEscaper", () => { + describe("ClickHouse dialect", () => { + const escaper = new SQLValueEscaper({ dialect: "clickhouse" }); + + it("should escape null", () => { + expect(escaper.visit(null)).toBe("NULL"); + expect(escaper.visit(undefined)).toBe("NULL"); + }); + + it("should escape booleans as numbers", () => { + expect(escaper.visit(true)).toBe("1"); + expect(escaper.visit(false)).toBe("0"); + }); + + it("should escape integers", () => { + expect(escaper.visit(0)).toBe("0"); + expect(escaper.visit(42)).toBe("42"); + expect(escaper.visit(-100)).toBe("-100"); + }); + + it("should escape floats", () => { + expect(escaper.visit(3.14)).toBe("3.14"); + expect(escaper.visit(-0.5)).toBe("-0.5"); + }); + + it("should escape special floats", () => { + expect(escaper.visit(NaN)).toBe("NaN"); + expect(escaper.visit(Infinity)).toBe("Inf"); + expect(escaper.visit(-Infinity)).toBe("-Inf"); + }); + + it("should escape strings with quotes", () => { + expect(escaper.visit("hello")).toBe("'hello'"); + expect(escaper.visit("hello'world")).toBe("'hello\\'world'"); + }); + + it("should escape strings with control characters", () => { + expect(escaper.visit("line1\nline2")).toBe("'line1\\nline2'"); + expect(escaper.visit("col1\tcol2")).toBe("'col1\\tcol2'"); + }); + + it("should escape arrays", () => { + expect(escaper.visit([1, 2, 3])).toBe("[1, 2, 3]"); + expect(escaper.visit(["a", "b"])).toBe("['a', 'b']"); + expect(escaper.visit(["hello", "world"])).toBe("['hello', 'world']"); + }); + + it("should escape nested arrays", () => { + expect( + escaper.visit([ + [1, 2], + [3, 4], + ]) + ).toBe("[[1, 2], [3, 4]]"); + }); + + it("should escape dates with toDateTime64", () => { + const date = new Date("2024-01-15T10:30:00.500Z"); + const result = escaper.visit(date); + expect(result).toContain("toDateTime64"); + expect(result).toContain("2024-01-15"); + }); + }); + + describe("TSQL dialect", () => { + const escaper = new SQLValueEscaper({ dialect: "tsql" }); + + it("should escape booleans as keywords", () => { + expect(escaper.visit(true)).toBe("true"); + expect(escaper.visit(false)).toBe("false"); + }); + + it("should escape dates with toDateTime", () => { + const date = new Date("2024-01-15T10:30:00.500Z"); + const result = escaper.visit(date); + expect(result).toContain("toDateTime"); + expect(result).toContain("2024-01-15"); + }); + }); +}); + +describe("escapeClickHouseString", () => { + it("should escape string values", () => { + expect(escapeClickHouseString("test")).toBe("'test'"); + }); + + it("should handle null", () => { + expect(escapeClickHouseString(null)).toBe("NULL"); + }); + + it("should handle numbers", () => { + expect(escapeClickHouseString(42)).toBe("42"); + }); +}); + +describe("escapeTSQLString", () => { + it("should escape string values", () => { + expect(escapeTSQLString("test")).toBe("'test'"); + }); + + it("should handle booleans differently from ClickHouse", () => { + expect(escapeTSQLString(true)).toBe("true"); + expect(escapeTSQLString(false)).toBe("false"); + }); +}); + +describe("getClickHouseType", () => { + it("should return String for strings", () => { + expect(getClickHouseType("hello")).toBe("String"); + }); + + it("should return UInt8 for booleans", () => { + expect(getClickHouseType(true)).toBe("UInt8"); + expect(getClickHouseType(false)).toBe("UInt8"); + }); + + it("should return Int32 for small integers", () => { + expect(getClickHouseType(42)).toBe("Int32"); + expect(getClickHouseType(-100)).toBe("Int32"); + }); + + it("should return Int64 for large integers", () => { + expect(getClickHouseType(3000000000)).toBe("Int64"); + expect(getClickHouseType(-3000000000)).toBe("Int64"); + }); + + it("should return Float64 for floats", () => { + expect(getClickHouseType(3.14)).toBe("Float64"); + }); + + it("should return DateTime64(6) for dates", () => { + expect(getClickHouseType(new Date())).toBe("DateTime64(6)"); + }); + + it("should return Array type for arrays", () => { + expect(getClickHouseType(["a", "b"])).toBe("Array(String)"); + expect(getClickHouseType([1, 2])).toBe("Array(Int32)"); + expect(getClickHouseType([])).toBe("Array(String)"); + }); + + it("should return Nullable(String) for null", () => { + expect(getClickHouseType(null)).toBe("Nullable(String)"); + expect(getClickHouseType(undefined)).toBe("Nullable(String)"); + }); +}); + +describe("safeIdentifier", () => { + it("should return identifier unchanged if no %", () => { + expect(safeIdentifier("column")).toBe("column"); + expect(safeIdentifier("table_name")).toBe("table_name"); + }); + + it("should remove % characters", () => { + expect(safeIdentifier("column%name")).toBe("columnname"); + expect(safeIdentifier("%%test%%")).toBe("test"); + }); +}); diff --git a/internal-packages/tsql/src/query/escape.ts b/internal-packages/tsql/src/query/escape.ts new file mode 100644 index 000000000..af442b50b --- /dev/null +++ b/internal-packages/tsql/src/query/escape.ts @@ -0,0 +1,264 @@ +// TypeScript port of posthog/hogql/escape_sql.py +// Keep this file in sync with the Python version + +import { QueryError } from "./errors"; + +/** + * Character escape maps for ClickHouse string escaping + * Copied from clickhouse_driver.util.escape + * + * Note: In JavaScript, \a and \v are not recognized escape sequences like in Python. + * We use the actual ASCII codes: \x07 for bell (Python's \a) and \x0B for vertical tab. + */ +const escapeCharsMap: Record = { + "\b": "\\b", + "\f": "\\f", + "\r": "\\r", + "\n": "\\n", + "\t": "\\t", + "\0": "\\0", + "\x07": "\\a", // Bell character (ASCII 7) - Python's \a + "\x0B": "\\v", // Vertical tab (ASCII 11) - use explicit code since JS \v may not work in all contexts + "\\": "\\\\", +}; + +const singlequoteEscapeCharsMap: Record = { + ...escapeCharsMap, + "'": "\\'", +}; + +const backquoteEscapeCharsMap: Record = { + ...escapeCharsMap, + "`": "\\`", +}; + +/** + * Sanitize an identifier by removing % characters + */ +export function safeIdentifier(identifier: string): string { + if (identifier.includes("%")) { + return identifier.replace(/%/g, ""); + } + return identifier; +} + +/** + * Escape a string value for use as a parameter in ClickHouse + * Copied from clickhouse_driver.util.escape_param + */ +export function escapeParamClickhouse(value: string): string { + const escaped = value + .split("") + .map((c) => singlequoteEscapeCharsMap[c] || c) + .join(""); + return `'${escaped}'`; +} + +/** + * Escape an identifier for use in TSQL/HogQL queries + * Adapted from clickhouse_driver.util.escape with support for $ in identifiers + */ +export function escapeTSQLIdentifier(identifier: string | number): string { + if (typeof identifier === "number") { + // In TSQL we allow integers as identifiers to access array elements + return String(identifier); + } + + if (identifier.includes("%")) { + throw new QueryError( + `The TSQL identifier "${identifier}" is not permitted as it contains the "%" character` + ); + } + + // TSQL allows dollars in the identifier (same regex as frontend escapePropertyAsTSQLIdentifier) + if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(identifier)) { + return identifier; + } + + const escaped = identifier + .split("") + .map((c) => backquoteEscapeCharsMap[c] || c) + .join(""); + return `\`${escaped}\``; +} + +/** + * Escape an identifier for use in ClickHouse queries + * Copied from clickhouse_driver.util.escape, adapted from single quotes to backquotes + */ +export function escapeClickHouseIdentifier(identifier: string): string { + if (identifier.includes("%")) { + throw new QueryError( + `The identifier "${identifier}" is not permitted as it contains the "%" character` + ); + } + + if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) { + return identifier; + } + + const escaped = identifier + .split("") + .map((c) => backquoteEscapeCharsMap[c] || c) + .join(""); + return `\`${escaped}\``; +} + +/** + * Type for values that can be escaped as SQL strings + */ +export type EscapableValue = + | null + | undefined + | string + | number + | boolean + | Date + | EscapableValue[] + | [EscapableValue, ...EscapableValue[]]; + +/** + * SQL Value Escaper class that handles different types of values + * Port of SQLValueEscaper from escape_sql.py + */ +export class SQLValueEscaper { + private timezone: string; + private dialect: "tsql" | "clickhouse"; + + constructor(options: { timezone?: string; dialect?: "tsql" | "clickhouse" } = {}) { + this.timezone = options.timezone || "UTC"; + this.dialect = options.dialect || "clickhouse"; + } + + visit(value: EscapableValue): string { + if (value === null || value === undefined) { + return this.visitNull(); + } + if (typeof value === "string") { + return this.visitString(value); + } + if (typeof value === "boolean") { + return this.visitBoolean(value); + } + if (typeof value === "number") { + if (Number.isInteger(value)) { + return this.visitInt(value); + } + return this.visitFloat(value); + } + if (value instanceof Date) { + return this.visitDateTime(value); + } + if (Array.isArray(value)) { + return this.visitArray(value); + } + + throw new QueryError(`SQLValueEscaper cannot handle value of type ${typeof value}`); + } + + private visitNull(): string { + return "NULL"; + } + + private visitString(value: string): string { + return escapeParamClickhouse(value); + } + + private visitBoolean(value: boolean): string { + if (this.dialect === "clickhouse") { + return value ? "1" : "0"; + } + return value ? "true" : "false"; + } + + private visitInt(value: number): string { + return String(value); + } + + private visitFloat(value: number): string { + if (Number.isNaN(value)) { + return "NaN"; + } + if (!Number.isFinite(value)) { + return value < 0 ? "-Inf" : "Inf"; + } + return String(value); + } + + private visitDateTime(value: Date): string { + // Format: YYYY-MM-DD HH:MM:SS.ffffff + const pad = (n: number, len: number = 2) => String(n).padStart(len, "0"); + + const year = value.getUTCFullYear(); + const month = pad(value.getUTCMonth() + 1); + const day = pad(value.getUTCDate()); + const hours = pad(value.getUTCHours()); + const minutes = pad(value.getUTCMinutes()); + const seconds = pad(value.getUTCSeconds()); + const ms = pad(value.getUTCMilliseconds(), 3); + + const datetimeString = `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${ms}000`; + + if (this.dialect === "tsql") { + return `toDateTime(${this.visitString(datetimeString)})`; + } + return `toDateTime64(${this.visitString(datetimeString)}, 6, ${this.visitString(this.timezone)})`; + } + + private visitArray(value: EscapableValue[]): string { + return `[${value.map((x) => this.visit(x)).join(", ")}]`; + } +} + +/** + * Escape a value for use in a TSQL/HogQL query string + */ +export function escapeTSQLString(value: EscapableValue, timezone?: string): string { + return new SQLValueEscaper({ timezone, dialect: "tsql" }).visit(value); +} + +/** + * Escape a value for use in a ClickHouse query string + */ +export function escapeClickHouseString(value: EscapableValue, timezone?: string): string { + return new SQLValueEscaper({ timezone, dialect: "clickhouse" }).visit(value); +} + +/** + * Get the ClickHouse type string for a value + * Used when creating parameterized query placeholders like {param: Type} + */ +export function getClickHouseType(value: unknown): string { + if (value === null || value === undefined) { + return "Nullable(String)"; + } + if (typeof value === "string") { + return "String"; + } + if (typeof value === "boolean") { + return "UInt8"; + } + if (typeof value === "number") { + if (Number.isInteger(value)) { + // Use Int64 for large integers, Int32 for smaller ones + if (value > 2147483647 || value < -2147483648) { + return "Int64"; + } + return "Int32"; + } + return "Float64"; + } + if (value instanceof Date) { + return "DateTime64(6)"; + } + if (Array.isArray(value)) { + if (value.length === 0) { + return "Array(String)"; + } + const itemType = getClickHouseType(value[0]); + return `Array(${itemType})`; + } + // Default to String for unknown types + return "String"; +} + diff --git a/internal-packages/tsql/src/query/functions.ts b/internal-packages/tsql/src/query/functions.ts new file mode 100644 index 000000000..d150b0636 --- /dev/null +++ b/internal-packages/tsql/src/query/functions.ts @@ -0,0 +1,648 @@ +// TypeScript port of posthog/hogql/functions/mapping.py and aggregations.py +// Keep this file in sync with the Python version + +import { CompareOperationOp } from "./ast"; + +/** + * Metadata for a TSQL function + */ +export interface TSQLFunctionMeta { + /** The ClickHouse function name to use */ + clickhouseName: string; + /** Minimum number of arguments */ + minArgs: number; + /** Maximum number of arguments (undefined means unlimited) */ + maxArgs?: number; + /** Minimum number of parameters (for parametric functions) */ + minParams?: number; + /** Maximum number of parameters */ + maxParams?: number; + /** Whether this is an aggregate function */ + aggregate?: boolean; + /** Whether function is case-sensitive */ + caseSensitive?: boolean; + /** Whether function is timezone-aware (will append timezone as last arg) */ + tzAware?: boolean; + /** Whether the function uses placeholder arguments like {} */ + usingPlaceholderArguments?: boolean; +} + +/** + * Comparison function mappings from function names to CompareOperationOp + */ +export const TSQL_COMPARISON_MAPPING: Record = { + equals: CompareOperationOp.Eq, + notEquals: CompareOperationOp.NotEq, + less: CompareOperationOp.Lt, + greater: CompareOperationOp.Gt, + lessOrEquals: CompareOperationOp.LtEq, + greaterOrEquals: CompareOperationOp.GtEq, + like: CompareOperationOp.Like, + ilike: CompareOperationOp.ILike, + notLike: CompareOperationOp.NotLike, + notILike: CompareOperationOp.NotILike, + in: CompareOperationOp.In, + notIn: CompareOperationOp.NotIn, +}; + +/** + * ClickHouse functions available in TSQL + * Port of HOGQL_CLICKHOUSE_FUNCTIONS from mapping.py + */ +export const TSQL_CLICKHOUSE_FUNCTIONS: Record = { + // Comparison + equals: { clickhouseName: "equals", minArgs: 2, maxArgs: 2 }, + notEquals: { clickhouseName: "notEquals", minArgs: 2, maxArgs: 2 }, + less: { clickhouseName: "less", minArgs: 2, maxArgs: 2 }, + greater: { clickhouseName: "greater", minArgs: 2, maxArgs: 2 }, + lessOrEquals: { clickhouseName: "lessOrEquals", minArgs: 2, maxArgs: 2 }, + greaterOrEquals: { clickhouseName: "greaterOrEquals", minArgs: 2, maxArgs: 2 }, + + // Logical + and: { clickhouseName: "and", minArgs: 2 }, + or: { clickhouseName: "or", minArgs: 2 }, + xor: { clickhouseName: "xor", minArgs: 2 }, + not: { clickhouseName: "not", minArgs: 1, maxArgs: 1, caseSensitive: false }, + + // Conditional + if: { clickhouseName: "if", minArgs: 3, maxArgs: 3, caseSensitive: false }, + multiIf: { clickhouseName: "multiIf", minArgs: 3 }, + + // In + in: { clickhouseName: "in", minArgs: 2, maxArgs: 2 }, + notIn: { clickhouseName: "notIn", minArgs: 2, maxArgs: 2 }, + + // Arithmetic + plus: { clickhouseName: "plus", minArgs: 2, maxArgs: 2 }, + minus: { clickhouseName: "minus", minArgs: 2, maxArgs: 2 }, + multiply: { clickhouseName: "multiply", minArgs: 2, maxArgs: 2 }, + divide: { clickhouseName: "divide", minArgs: 2, maxArgs: 2 }, + intDiv: { clickhouseName: "intDiv", minArgs: 2, maxArgs: 2 }, + intDivOrZero: { clickhouseName: "intDivOrZero", minArgs: 2, maxArgs: 2 }, + modulo: { clickhouseName: "modulo", minArgs: 2, maxArgs: 2 }, + moduloOrZero: { clickhouseName: "moduloOrZero", minArgs: 2, maxArgs: 2 }, + positiveModulo: { clickhouseName: "positiveModulo", minArgs: 2, maxArgs: 2 }, + negate: { clickhouseName: "negate", minArgs: 1, maxArgs: 1 }, + abs: { clickhouseName: "abs", minArgs: 1, maxArgs: 1 }, + gcd: { clickhouseName: "gcd", minArgs: 2, maxArgs: 2 }, + lcm: { clickhouseName: "lcm", minArgs: 2, maxArgs: 2 }, + + // Mathematical + exp: { clickhouseName: "exp", minArgs: 1, maxArgs: 1 }, + log: { clickhouseName: "log", minArgs: 1, maxArgs: 1 }, + ln: { clickhouseName: "log", minArgs: 1, maxArgs: 1 }, + exp2: { clickhouseName: "exp2", minArgs: 1, maxArgs: 1 }, + log2: { clickhouseName: "log2", minArgs: 1, maxArgs: 1 }, + exp10: { clickhouseName: "exp10", minArgs: 1, maxArgs: 1 }, + log10: { clickhouseName: "log10", minArgs: 1, maxArgs: 1 }, + sqrt: { clickhouseName: "sqrt", minArgs: 1, maxArgs: 1 }, + cbrt: { clickhouseName: "cbrt", minArgs: 1, maxArgs: 1 }, + erf: { clickhouseName: "erf", minArgs: 1, maxArgs: 1 }, + erfc: { clickhouseName: "erfc", minArgs: 1, maxArgs: 1 }, + lgamma: { clickhouseName: "lgamma", minArgs: 1, maxArgs: 1 }, + tgamma: { clickhouseName: "tgamma", minArgs: 1, maxArgs: 1 }, + sin: { clickhouseName: "sin", minArgs: 1, maxArgs: 1 }, + cos: { clickhouseName: "cos", minArgs: 1, maxArgs: 1 }, + tan: { clickhouseName: "tan", minArgs: 1, maxArgs: 1 }, + asin: { clickhouseName: "asin", minArgs: 1, maxArgs: 1 }, + acos: { clickhouseName: "acos", minArgs: 1, maxArgs: 1 }, + atan: { clickhouseName: "atan", minArgs: 1, maxArgs: 1 }, + pow: { clickhouseName: "pow", minArgs: 2, maxArgs: 2 }, + power: { clickhouseName: "power", minArgs: 2, maxArgs: 2 }, + round: { clickhouseName: "round", minArgs: 1, maxArgs: 2 }, + floor: { clickhouseName: "floor", minArgs: 1, maxArgs: 2 }, + ceil: { clickhouseName: "ceil", minArgs: 1, maxArgs: 2 }, + ceiling: { clickhouseName: "ceiling", minArgs: 1, maxArgs: 2 }, + trunc: { clickhouseName: "trunc", minArgs: 1, maxArgs: 2 }, + truncate: { clickhouseName: "truncate", minArgs: 1, maxArgs: 2 }, + sign: { clickhouseName: "sign", minArgs: 1, maxArgs: 1 }, + + // String functions + empty: { clickhouseName: "empty", minArgs: 1, maxArgs: 1 }, + notEmpty: { clickhouseName: "notEmpty", minArgs: 1, maxArgs: 1 }, + length: { clickhouseName: "length", minArgs: 1, maxArgs: 1 }, + lengthUTF8: { clickhouseName: "lengthUTF8", minArgs: 1, maxArgs: 1 }, + char_length: { clickhouseName: "char_length", minArgs: 1, maxArgs: 1 }, + character_length: { clickhouseName: "character_length", minArgs: 1, maxArgs: 1 }, + lower: { clickhouseName: "lower", minArgs: 1, maxArgs: 1 }, + upper: { clickhouseName: "upper", minArgs: 1, maxArgs: 1 }, + lowerUTF8: { clickhouseName: "lowerUTF8", minArgs: 1, maxArgs: 1 }, + upperUTF8: { clickhouseName: "upperUTF8", minArgs: 1, maxArgs: 1 }, + reverse: { clickhouseName: "reverse", minArgs: 1, maxArgs: 1 }, + reverseUTF8: { clickhouseName: "reverseUTF8", minArgs: 1, maxArgs: 1 }, + concat: { clickhouseName: "concat", minArgs: 1 }, + concatAssumeInjective: { clickhouseName: "concatAssumeInjective", minArgs: 1 }, + substring: { clickhouseName: "substring", minArgs: 2, maxArgs: 3 }, + substr: { clickhouseName: "substring", minArgs: 2, maxArgs: 3 }, + mid: { clickhouseName: "substring", minArgs: 2, maxArgs: 3 }, + substringUTF8: { clickhouseName: "substringUTF8", minArgs: 2, maxArgs: 3 }, + appendTrailingCharIfAbsent: { clickhouseName: "appendTrailingCharIfAbsent", minArgs: 2, maxArgs: 2 }, + convertCharset: { clickhouseName: "convertCharset", minArgs: 3, maxArgs: 3 }, + base58Encode: { clickhouseName: "base58Encode", minArgs: 1, maxArgs: 1 }, + base58Decode: { clickhouseName: "base58Decode", minArgs: 1, maxArgs: 1 }, + base64Encode: { clickhouseName: "base64Encode", minArgs: 1, maxArgs: 1 }, + base64Decode: { clickhouseName: "base64Decode", minArgs: 1, maxArgs: 1 }, + tryBase64Decode: { clickhouseName: "tryBase64Decode", minArgs: 1, maxArgs: 1 }, + endsWith: { clickhouseName: "endsWith", minArgs: 2, maxArgs: 2 }, + startsWith: { clickhouseName: "startsWith", minArgs: 2, maxArgs: 2 }, + trim: { clickhouseName: "trim", minArgs: 1, maxArgs: 2 }, + trimLeft: { clickhouseName: "trimLeft", minArgs: 1, maxArgs: 2 }, + trimRight: { clickhouseName: "trimRight", minArgs: 1, maxArgs: 2 }, + ltrim: { clickhouseName: "trimLeft", minArgs: 1, maxArgs: 1 }, + rtrim: { clickhouseName: "trimRight", minArgs: 1, maxArgs: 1 }, + leftPad: { clickhouseName: "leftPad", minArgs: 2, maxArgs: 3 }, + rightPad: { clickhouseName: "rightPad", minArgs: 2, maxArgs: 3 }, + leftPadUTF8: { clickhouseName: "leftPadUTF8", minArgs: 2, maxArgs: 3 }, + rightPadUTF8: { clickhouseName: "rightPadUTF8", minArgs: 2, maxArgs: 3 }, + left: { clickhouseName: "left", minArgs: 2, maxArgs: 2 }, + right: { clickhouseName: "right", minArgs: 2, maxArgs: 2 }, + repeat: { clickhouseName: "repeat", minArgs: 2, maxArgs: 2 }, + space: { clickhouseName: "space", minArgs: 1, maxArgs: 1 }, + replace: { clickhouseName: "replace", minArgs: 3, maxArgs: 3 }, + replaceOne: { clickhouseName: "replaceOne", minArgs: 3, maxArgs: 3 }, + replaceAll: { clickhouseName: "replaceAll", minArgs: 3, maxArgs: 3 }, + replaceRegexpOne: { clickhouseName: "replaceRegexpOne", minArgs: 3, maxArgs: 3 }, + replaceRegexpAll: { clickhouseName: "replaceRegexpAll", minArgs: 3, maxArgs: 3 }, + position: { clickhouseName: "position", minArgs: 2, maxArgs: 2 }, + positionCaseInsensitive: { clickhouseName: "positionCaseInsensitive", minArgs: 2, maxArgs: 2 }, + positionUTF8: { clickhouseName: "positionUTF8", minArgs: 2, maxArgs: 2 }, + positionCaseInsensitiveUTF8: { clickhouseName: "positionCaseInsensitiveUTF8", minArgs: 2, maxArgs: 2 }, + locate: { clickhouseName: "locate", minArgs: 2, maxArgs: 2 }, + match: { clickhouseName: "match", minArgs: 2, maxArgs: 2 }, + multiMatchAny: { clickhouseName: "multiMatchAny", minArgs: 2, maxArgs: 2 }, + multiMatchAnyIndex: { clickhouseName: "multiMatchAnyIndex", minArgs: 2, maxArgs: 2 }, + multiMatchAllIndices: { clickhouseName: "multiMatchAllIndices", minArgs: 2, maxArgs: 2 }, + multiSearchFirstPosition: { clickhouseName: "multiSearchFirstPosition", minArgs: 2, maxArgs: 2 }, + multiSearchFirstIndex: { clickhouseName: "multiSearchFirstIndex", minArgs: 2, maxArgs: 2 }, + multiSearchAny: { clickhouseName: "multiSearchAny", minArgs: 2, maxArgs: 2 }, + extract: { clickhouseName: "extract", minArgs: 2, maxArgs: 2 }, + extractAll: { clickhouseName: "extractAll", minArgs: 2, maxArgs: 2 }, + extractAllGroupsHorizontal: { clickhouseName: "extractAllGroupsHorizontal", minArgs: 2, maxArgs: 2 }, + extractAllGroupsVertical: { clickhouseName: "extractAllGroupsVertical", minArgs: 2, maxArgs: 2 }, + like: { clickhouseName: "like", minArgs: 2, maxArgs: 2 }, + ilike: { clickhouseName: "ilike", minArgs: 2, maxArgs: 2 }, + notLike: { clickhouseName: "notLike", minArgs: 2, maxArgs: 2 }, + notILike: { clickhouseName: "notILike", minArgs: 2, maxArgs: 2 }, + splitByChar: { clickhouseName: "splitByChar", minArgs: 2, maxArgs: 3 }, + splitByString: { clickhouseName: "splitByString", minArgs: 2, maxArgs: 3 }, + splitByRegexp: { clickhouseName: "splitByRegexp", minArgs: 2, maxArgs: 3 }, + arrayStringConcat: { clickhouseName: "arrayStringConcat", minArgs: 1, maxArgs: 2 }, + format: { clickhouseName: "format", minArgs: 1 }, + coalesce: { clickhouseName: "coalesce", minArgs: 1 }, + ifNull: { clickhouseName: "ifNull", minArgs: 2, maxArgs: 2 }, + nullIf: { clickhouseName: "nullIf", minArgs: 2, maxArgs: 2 }, + assumeNotNull: { clickhouseName: "assumeNotNull", minArgs: 1, maxArgs: 1 }, + toNullable: { clickhouseName: "toNullable", minArgs: 1, maxArgs: 1 }, + isNull: { clickhouseName: "isNull", minArgs: 1, maxArgs: 1 }, + isNotNull: { clickhouseName: "isNotNull", minArgs: 1, maxArgs: 1 }, + + // Type conversions + toString: { clickhouseName: "toString", minArgs: 1, maxArgs: 1 }, + toFixedString: { clickhouseName: "toFixedString", minArgs: 2, maxArgs: 2 }, + toUInt8: { clickhouseName: "toUInt8", minArgs: 1, maxArgs: 1 }, + toUInt16: { clickhouseName: "toUInt16", minArgs: 1, maxArgs: 1 }, + toUInt32: { clickhouseName: "toUInt32", minArgs: 1, maxArgs: 1 }, + toUInt64: { clickhouseName: "toUInt64", minArgs: 1, maxArgs: 1 }, + toInt8: { clickhouseName: "toInt8", minArgs: 1, maxArgs: 1 }, + toInt16: { clickhouseName: "toInt16", minArgs: 1, maxArgs: 1 }, + toInt32: { clickhouseName: "toInt32", minArgs: 1, maxArgs: 1 }, + toInt64: { clickhouseName: "toInt64", minArgs: 1, maxArgs: 1 }, + toInt128: { clickhouseName: "toInt128", minArgs: 1, maxArgs: 1 }, + toInt256: { clickhouseName: "toInt256", minArgs: 1, maxArgs: 1 }, + toUInt128: { clickhouseName: "toUInt128", minArgs: 1, maxArgs: 1 }, + toUInt256: { clickhouseName: "toUInt256", minArgs: 1, maxArgs: 1 }, + toFloat32: { clickhouseName: "toFloat32", minArgs: 1, maxArgs: 1 }, + toFloat64: { clickhouseName: "toFloat64", minArgs: 1, maxArgs: 1 }, + toDecimal32: { clickhouseName: "toDecimal32", minArgs: 2, maxArgs: 2 }, + toDecimal64: { clickhouseName: "toDecimal64", minArgs: 2, maxArgs: 2 }, + toDecimal128: { clickhouseName: "toDecimal128", minArgs: 2, maxArgs: 2 }, + toDecimal256: { clickhouseName: "toDecimal256", minArgs: 2, maxArgs: 2 }, + toDate: { clickhouseName: "toDate", minArgs: 1, maxArgs: 2 }, + toDateOrNull: { clickhouseName: "toDateOrNull", minArgs: 1, maxArgs: 2 }, + toDateOrZero: { clickhouseName: "toDateOrZero", minArgs: 1, maxArgs: 2 }, + toDate32: { clickhouseName: "toDate32", minArgs: 1, maxArgs: 2 }, + toDate32OrNull: { clickhouseName: "toDate32OrNull", minArgs: 1, maxArgs: 2 }, + toDate32OrZero: { clickhouseName: "toDate32OrZero", minArgs: 1, maxArgs: 2 }, + toDateTime: { clickhouseName: "toDateTime", minArgs: 1, maxArgs: 2 }, + toDateTimeOrNull: { clickhouseName: "toDateTimeOrNull", minArgs: 1, maxArgs: 2 }, + toDateTimeOrZero: { clickhouseName: "toDateTimeOrZero", minArgs: 1, maxArgs: 2 }, + toDateTime64: { clickhouseName: "toDateTime64", minArgs: 1, maxArgs: 3 }, + toDateTime64OrNull: { clickhouseName: "toDateTime64OrNull", minArgs: 1, maxArgs: 3 }, + toDateTime64OrZero: { clickhouseName: "toDateTime64OrZero", minArgs: 1, maxArgs: 3 }, + toUUID: { clickhouseName: "toUUID", minArgs: 1, maxArgs: 1 }, + toUUIDOrNull: { clickhouseName: "toUUIDOrNull", minArgs: 1, maxArgs: 1 }, + toUUIDOrZero: { clickhouseName: "toUUIDOrZero", minArgs: 1, maxArgs: 1 }, + toTypeName: { clickhouseName: "toTypeName", minArgs: 1, maxArgs: 1 }, + + // Date/time functions + now: { clickhouseName: "now", minArgs: 0, maxArgs: 1, tzAware: true }, + now64: { clickhouseName: "now64", minArgs: 0, maxArgs: 2, tzAware: true }, + today: { clickhouseName: "today", minArgs: 0, maxArgs: 0 }, + yesterday: { clickhouseName: "yesterday", minArgs: 0, maxArgs: 0 }, + toYear: { clickhouseName: "toYear", minArgs: 1, maxArgs: 1 }, + toQuarter: { clickhouseName: "toQuarter", minArgs: 1, maxArgs: 1 }, + toMonth: { clickhouseName: "toMonth", minArgs: 1, maxArgs: 1 }, + toDayOfYear: { clickhouseName: "toDayOfYear", minArgs: 1, maxArgs: 1 }, + toDayOfMonth: { clickhouseName: "toDayOfMonth", minArgs: 1, maxArgs: 1 }, + toDayOfWeek: { clickhouseName: "toDayOfWeek", minArgs: 1, maxArgs: 3 }, + toHour: { clickhouseName: "toHour", minArgs: 1, maxArgs: 1 }, + toMinute: { clickhouseName: "toMinute", minArgs: 1, maxArgs: 1 }, + toSecond: { clickhouseName: "toSecond", minArgs: 1, maxArgs: 1 }, + toUnixTimestamp: { clickhouseName: "toUnixTimestamp", minArgs: 1, maxArgs: 2 }, + toStartOfYear: { clickhouseName: "toStartOfYear", minArgs: 1, maxArgs: 1 }, + toStartOfQuarter: { clickhouseName: "toStartOfQuarter", minArgs: 1, maxArgs: 1 }, + toStartOfMonth: { clickhouseName: "toStartOfMonth", minArgs: 1, maxArgs: 1 }, + toMonday: { clickhouseName: "toMonday", minArgs: 1, maxArgs: 1 }, + toStartOfWeek: { clickhouseName: "toStartOfWeek", minArgs: 1, maxArgs: 2 }, + toStartOfDay: { clickhouseName: "toStartOfDay", minArgs: 1, maxArgs: 1 }, + toStartOfHour: { clickhouseName: "toStartOfHour", minArgs: 1, maxArgs: 1 }, + toStartOfMinute: { clickhouseName: "toStartOfMinute", minArgs: 1, maxArgs: 1 }, + toStartOfSecond: { clickhouseName: "toStartOfSecond", minArgs: 1, maxArgs: 1 }, + toStartOfFiveMinutes: { clickhouseName: "toStartOfFiveMinutes", minArgs: 1, maxArgs: 1 }, + toStartOfTenMinutes: { clickhouseName: "toStartOfTenMinutes", minArgs: 1, maxArgs: 1 }, + toStartOfFifteenMinutes: { clickhouseName: "toStartOfFifteenMinutes", minArgs: 1, maxArgs: 1 }, + toStartOfInterval: { clickhouseName: "toStartOfInterval", minArgs: 2, maxArgs: 4 }, + toTime: { clickhouseName: "toTime", minArgs: 1, maxArgs: 2 }, + toISOYear: { clickhouseName: "toISOYear", minArgs: 1, maxArgs: 1 }, + toISOWeek: { clickhouseName: "toISOWeek", minArgs: 1, maxArgs: 1 }, + toWeek: { clickhouseName: "toWeek", minArgs: 1, maxArgs: 3 }, + toYearWeek: { clickhouseName: "toYearWeek", minArgs: 1, maxArgs: 3 }, + date_add: { clickhouseName: "date_add", minArgs: 3, maxArgs: 3 }, + date_diff: { clickhouseName: "date_diff", minArgs: 3, maxArgs: 4 }, + date_sub: { clickhouseName: "date_sub", minArgs: 3, maxArgs: 3 }, + date_trunc: { clickhouseName: "date_trunc", minArgs: 2, maxArgs: 3 }, + dateDiff: { clickhouseName: "dateDiff", minArgs: 3, maxArgs: 4 }, + dateAdd: { clickhouseName: "dateAdd", minArgs: 3, maxArgs: 3 }, + dateSub: { clickhouseName: "dateSub", minArgs: 3, maxArgs: 3 }, + dateTrunc: { clickhouseName: "dateTrunc", minArgs: 2, maxArgs: 3 }, + addSeconds: { clickhouseName: "addSeconds", minArgs: 2, maxArgs: 2 }, + addMinutes: { clickhouseName: "addMinutes", minArgs: 2, maxArgs: 2 }, + addHours: { clickhouseName: "addHours", minArgs: 2, maxArgs: 2 }, + addDays: { clickhouseName: "addDays", minArgs: 2, maxArgs: 2 }, + addWeeks: { clickhouseName: "addWeeks", minArgs: 2, maxArgs: 2 }, + addMonths: { clickhouseName: "addMonths", minArgs: 2, maxArgs: 2 }, + addQuarters: { clickhouseName: "addQuarters", minArgs: 2, maxArgs: 2 }, + addYears: { clickhouseName: "addYears", minArgs: 2, maxArgs: 2 }, + subtractSeconds: { clickhouseName: "subtractSeconds", minArgs: 2, maxArgs: 2 }, + subtractMinutes: { clickhouseName: "subtractMinutes", minArgs: 2, maxArgs: 2 }, + subtractHours: { clickhouseName: "subtractHours", minArgs: 2, maxArgs: 2 }, + subtractDays: { clickhouseName: "subtractDays", minArgs: 2, maxArgs: 2 }, + subtractWeeks: { clickhouseName: "subtractWeeks", minArgs: 2, maxArgs: 2 }, + subtractMonths: { clickhouseName: "subtractMonths", minArgs: 2, maxArgs: 2 }, + subtractQuarters: { clickhouseName: "subtractQuarters", minArgs: 2, maxArgs: 2 }, + subtractYears: { clickhouseName: "subtractYears", minArgs: 2, maxArgs: 2 }, + toTimeZone: { clickhouseName: "toTimeZone", minArgs: 2, maxArgs: 2 }, + formatDateTime: { clickhouseName: "formatDateTime", minArgs: 2, maxArgs: 3 }, + parseDateTime: { clickhouseName: "parseDateTime", minArgs: 2, maxArgs: 3 }, + parseDateTimeBestEffort: { clickhouseName: "parseDateTimeBestEffort", minArgs: 1, maxArgs: 2, tzAware: true }, + parseDateTimeBestEffortOrNull: { clickhouseName: "parseDateTimeBestEffortOrNull", minArgs: 1, maxArgs: 2, tzAware: true }, + parseDateTimeBestEffortOrZero: { clickhouseName: "parseDateTimeBestEffortOrZero", minArgs: 1, maxArgs: 2, tzAware: true }, + parseDateTime64BestEffort: { clickhouseName: "parseDateTime64BestEffort", minArgs: 1, maxArgs: 3, tzAware: true }, + parseDateTime64BestEffortOrNull: { clickhouseName: "parseDateTime64BestEffortOrNull", minArgs: 1, maxArgs: 3, tzAware: true }, + parseDateTime64BestEffortOrZero: { clickhouseName: "parseDateTime64BestEffortOrZero", minArgs: 1, maxArgs: 3, tzAware: true }, + + // Interval functions + toIntervalSecond: { clickhouseName: "toIntervalSecond", minArgs: 1, maxArgs: 1 }, + toIntervalMinute: { clickhouseName: "toIntervalMinute", minArgs: 1, maxArgs: 1 }, + toIntervalHour: { clickhouseName: "toIntervalHour", minArgs: 1, maxArgs: 1 }, + toIntervalDay: { clickhouseName: "toIntervalDay", minArgs: 1, maxArgs: 1 }, + toIntervalWeek: { clickhouseName: "toIntervalWeek", minArgs: 1, maxArgs: 1 }, + toIntervalMonth: { clickhouseName: "toIntervalMonth", minArgs: 1, maxArgs: 1 }, + toIntervalQuarter: { clickhouseName: "toIntervalQuarter", minArgs: 1, maxArgs: 1 }, + toIntervalYear: { clickhouseName: "toIntervalYear", minArgs: 1, maxArgs: 1 }, + + // Array functions + array: { clickhouseName: "array", minArgs: 0 }, + range: { clickhouseName: "range", minArgs: 1, maxArgs: 3 }, + arrayElement: { clickhouseName: "arrayElement", minArgs: 2, maxArgs: 2 }, + has: { clickhouseName: "has", minArgs: 2, maxArgs: 2 }, + hasAll: { clickhouseName: "hasAll", minArgs: 2, maxArgs: 2 }, + hasAny: { clickhouseName: "hasAny", minArgs: 2, maxArgs: 2 }, + hasSubstr: { clickhouseName: "hasSubstr", minArgs: 2, maxArgs: 2 }, + indexOf: { clickhouseName: "indexOf", minArgs: 2, maxArgs: 2 }, + arrayCount: { clickhouseName: "arrayCount", minArgs: 1, maxArgs: 2 }, + countEqual: { clickhouseName: "countEqual", minArgs: 2, maxArgs: 2 }, + arrayEnumerate: { clickhouseName: "arrayEnumerate", minArgs: 1, maxArgs: 1 }, + arrayEnumerateDense: { clickhouseName: "arrayEnumerateDense", minArgs: 1 }, + arrayEnumerateUniq: { clickhouseName: "arrayEnumerateUniq", minArgs: 1 }, + arrayEnumerateUniqRanked: { clickhouseName: "arrayEnumerateUniqRanked", minArgs: 1 }, + arrayPopBack: { clickhouseName: "arrayPopBack", minArgs: 1, maxArgs: 1 }, + arrayPopFront: { clickhouseName: "arrayPopFront", minArgs: 1, maxArgs: 1 }, + arrayPushBack: { clickhouseName: "arrayPushBack", minArgs: 2, maxArgs: 2 }, + arrayPushFront: { clickhouseName: "arrayPushFront", minArgs: 2, maxArgs: 2 }, + arrayResize: { clickhouseName: "arrayResize", minArgs: 2, maxArgs: 3 }, + arraySlice: { clickhouseName: "arraySlice", minArgs: 2, maxArgs: 3 }, + arraySort: { clickhouseName: "arraySort", minArgs: 1, maxArgs: 2 }, + arrayPartialSort: { clickhouseName: "arrayPartialSort", minArgs: 2, maxArgs: 3 }, + arrayReverseSort: { clickhouseName: "arrayReverseSort", minArgs: 1, maxArgs: 2 }, + arrayPartialReverseSort: { clickhouseName: "arrayPartialReverseSort", minArgs: 2, maxArgs: 3 }, + arrayShuffle: { clickhouseName: "arrayShuffle", minArgs: 1, maxArgs: 2 }, + arrayUniq: { clickhouseName: "arrayUniq", minArgs: 1 }, + arrayJoin: { clickhouseName: "arrayJoin", minArgs: 1, maxArgs: 1 }, + arrayDifference: { clickhouseName: "arrayDifference", minArgs: 1, maxArgs: 1 }, + arrayDistinct: { clickhouseName: "arrayDistinct", minArgs: 1, maxArgs: 1 }, + arrayIntersect: { clickhouseName: "arrayIntersect", minArgs: 1 }, + arrayReduce: { clickhouseName: "arrayReduce", minArgs: 2 }, + arrayReverse: { clickhouseName: "arrayReverse", minArgs: 1, maxArgs: 1 }, + arrayFlatten: { clickhouseName: "arrayFlatten", minArgs: 1, maxArgs: 1 }, + arrayCompact: { clickhouseName: "arrayCompact", minArgs: 1, maxArgs: 1 }, + arrayZip: { clickhouseName: "arrayZip", minArgs: 1 }, + arrayMap: { clickhouseName: "arrayMap", minArgs: 2, maxArgs: 2 }, + arrayFilter: { clickhouseName: "arrayFilter", minArgs: 2, maxArgs: 2 }, + arrayFill: { clickhouseName: "arrayFill", minArgs: 2, maxArgs: 2 }, + arrayReverseFill: { clickhouseName: "arrayReverseFill", minArgs: 2, maxArgs: 2 }, + arraySplit: { clickhouseName: "arraySplit", minArgs: 2, maxArgs: 2 }, + arrayReverseSplit: { clickhouseName: "arrayReverseSplit", minArgs: 2, maxArgs: 2 }, + arrayExists: { clickhouseName: "arrayExists", minArgs: 1, maxArgs: 2 }, + arrayAll: { clickhouseName: "arrayAll", minArgs: 1, maxArgs: 2 }, + arrayFirst: { clickhouseName: "arrayFirst", minArgs: 1, maxArgs: 2 }, + arrayLast: { clickhouseName: "arrayLast", minArgs: 1, maxArgs: 2 }, + arrayFirstIndex: { clickhouseName: "arrayFirstIndex", minArgs: 1, maxArgs: 2 }, + arrayLastIndex: { clickhouseName: "arrayLastIndex", minArgs: 1, maxArgs: 2 }, + arrayMin: { clickhouseName: "arrayMin", minArgs: 1, maxArgs: 2 }, + arrayMax: { clickhouseName: "arrayMax", minArgs: 1, maxArgs: 2 }, + arraySum: { clickhouseName: "arraySum", minArgs: 1, maxArgs: 2 }, + arrayAvg: { clickhouseName: "arrayAvg", minArgs: 1, maxArgs: 2 }, + arrayCumSum: { clickhouseName: "arrayCumSum", minArgs: 1, maxArgs: 2 }, + arrayCumSumNonNegative: { clickhouseName: "arrayCumSumNonNegative", minArgs: 1, maxArgs: 2 }, + arrayProduct: { clickhouseName: "arrayProduct", minArgs: 1, maxArgs: 1 }, + + // JSON functions + JSONHas: { clickhouseName: "JSONHas", minArgs: 1 }, + JSONLength: { clickhouseName: "JSONLength", minArgs: 1 }, + JSONType: { clickhouseName: "JSONType", minArgs: 1 }, + JSONExtractUInt: { clickhouseName: "JSONExtractUInt", minArgs: 1 }, + JSONExtractInt: { clickhouseName: "JSONExtractInt", minArgs: 1 }, + JSONExtractFloat: { clickhouseName: "JSONExtractFloat", minArgs: 1 }, + JSONExtractBool: { clickhouseName: "JSONExtractBool", minArgs: 1 }, + JSONExtractString: { clickhouseName: "JSONExtractString", minArgs: 1 }, + JSONExtract: { clickhouseName: "JSONExtract", minArgs: 2 }, + JSONExtractRaw: { clickhouseName: "JSONExtractRaw", minArgs: 1 }, + JSONExtractArrayRaw: { clickhouseName: "JSONExtractArrayRaw", minArgs: 1 }, + JSONExtractKeysAndValues: { clickhouseName: "JSONExtractKeysAndValues", minArgs: 2, maxArgs: 2 }, + JSONExtractKeys: { clickhouseName: "JSONExtractKeys", minArgs: 1 }, + toJSONString: { clickhouseName: "toJSONString", minArgs: 1, maxArgs: 1 }, + + // Tuple functions + tuple: { clickhouseName: "tuple", minArgs: 0 }, + tupleElement: { clickhouseName: "tupleElement", minArgs: 2, maxArgs: 3 }, + untuple: { clickhouseName: "untuple", minArgs: 1, maxArgs: 1 }, + + // Map functions + map: { clickhouseName: "map", minArgs: 0 }, + mapFromArrays: { clickhouseName: "mapFromArrays", minArgs: 2, maxArgs: 2 }, + mapContains: { clickhouseName: "mapContains", minArgs: 2, maxArgs: 2 }, + mapKeys: { clickhouseName: "mapKeys", minArgs: 1, maxArgs: 1 }, + mapValues: { clickhouseName: "mapValues", minArgs: 1, maxArgs: 1 }, + + // Hash functions + MD5: { clickhouseName: "MD5", minArgs: 1, maxArgs: 1 }, + SHA1: { clickhouseName: "SHA1", minArgs: 1, maxArgs: 1 }, + SHA224: { clickhouseName: "SHA224", minArgs: 1, maxArgs: 1 }, + SHA256: { clickhouseName: "SHA256", minArgs: 1, maxArgs: 1 }, + SHA384: { clickhouseName: "SHA384", minArgs: 1, maxArgs: 1 }, + SHA512: { clickhouseName: "SHA512", minArgs: 1, maxArgs: 1 }, + sipHash64: { clickhouseName: "sipHash64", minArgs: 1 }, + sipHash128: { clickhouseName: "sipHash128", minArgs: 1 }, + cityHash64: { clickhouseName: "cityHash64", minArgs: 1 }, + intHash32: { clickhouseName: "intHash32", minArgs: 1, maxArgs: 1 }, + intHash64: { clickhouseName: "intHash64", minArgs: 1, maxArgs: 1 }, + farmHash64: { clickhouseName: "farmHash64", minArgs: 1 }, + farmFingerprint64: { clickhouseName: "farmFingerprint64", minArgs: 1 }, + xxHash32: { clickhouseName: "xxHash32", minArgs: 1 }, + xxHash64: { clickhouseName: "xxHash64", minArgs: 1 }, + murmurHash2_32: { clickhouseName: "murmurHash2_32", minArgs: 1 }, + murmurHash2_64: { clickhouseName: "murmurHash2_64", minArgs: 1 }, + murmurHash3_32: { clickhouseName: "murmurHash3_32", minArgs: 1 }, + murmurHash3_64: { clickhouseName: "murmurHash3_64", minArgs: 1 }, + murmurHash3_128: { clickhouseName: "murmurHash3_128", minArgs: 1 }, + hex: { clickhouseName: "hex", minArgs: 1, maxArgs: 1 }, + unhex: { clickhouseName: "unhex", minArgs: 1, maxArgs: 1 }, + + // URL functions + protocol: { clickhouseName: "protocol", minArgs: 1, maxArgs: 1 }, + domain: { clickhouseName: "domain", minArgs: 1, maxArgs: 1 }, + domainWithoutWWW: { clickhouseName: "domainWithoutWWW", minArgs: 1, maxArgs: 1 }, + topLevelDomain: { clickhouseName: "topLevelDomain", minArgs: 1, maxArgs: 1 }, + firstSignificantSubdomain: { clickhouseName: "firstSignificantSubdomain", minArgs: 1, maxArgs: 1 }, + cutToFirstSignificantSubdomain: { clickhouseName: "cutToFirstSignificantSubdomain", minArgs: 1, maxArgs: 1 }, + cutToFirstSignificantSubdomainWithWWW: { clickhouseName: "cutToFirstSignificantSubdomainWithWWW", minArgs: 1, maxArgs: 1 }, + port: { clickhouseName: "port", minArgs: 1, maxArgs: 2 }, + path: { clickhouseName: "path", minArgs: 1, maxArgs: 1 }, + pathFull: { clickhouseName: "pathFull", minArgs: 1, maxArgs: 1 }, + queryString: { clickhouseName: "queryString", minArgs: 1, maxArgs: 1 }, + fragment: { clickhouseName: "fragment", minArgs: 1, maxArgs: 1 }, + extractURLParameter: { clickhouseName: "extractURLParameter", minArgs: 2, maxArgs: 2 }, + extractURLParameters: { clickhouseName: "extractURLParameters", minArgs: 1, maxArgs: 1 }, + encodeURLComponent: { clickhouseName: "encodeURLComponent", minArgs: 1, maxArgs: 1 }, + decodeURLComponent: { clickhouseName: "decodeURLComponent", minArgs: 1, maxArgs: 1 }, + + // UUID functions + generateUUIDv4: { clickhouseName: "generateUUIDv4", minArgs: 0, maxArgs: 0 }, + UUIDStringToNum: { clickhouseName: "UUIDStringToNum", minArgs: 1, maxArgs: 1 }, + UUIDNumToString: { clickhouseName: "UUIDNumToString", minArgs: 1, maxArgs: 1 }, + + // Other functions + isFinite: { clickhouseName: "isFinite", minArgs: 1, maxArgs: 1 }, + isInfinite: { clickhouseName: "isInfinite", minArgs: 1, maxArgs: 1 }, + ifNotFinite: { clickhouseName: "ifNotFinite", minArgs: 1, maxArgs: 1 }, + isNaN: { clickhouseName: "isNaN", minArgs: 1, maxArgs: 1 }, + bar: { clickhouseName: "bar", minArgs: 4, maxArgs: 4 }, + transform: { clickhouseName: "transform", minArgs: 3, maxArgs: 4 }, + formatReadableDecimalSize: { clickhouseName: "formatReadableDecimalSize", minArgs: 1, maxArgs: 1 }, + formatReadableSize: { clickhouseName: "formatReadableSize", minArgs: 1, maxArgs: 1 }, + formatReadableQuantity: { clickhouseName: "formatReadableQuantity", minArgs: 1, maxArgs: 1 }, + formatReadableTimeDelta: { clickhouseName: "formatReadableTimeDelta", minArgs: 1, maxArgs: 2 }, + least: { clickhouseName: "least", minArgs: 2, maxArgs: 2, caseSensitive: false }, + greatest: { clickhouseName: "greatest", minArgs: 2, maxArgs: 2, caseSensitive: false }, + min2: { clickhouseName: "min2", minArgs: 2, maxArgs: 2 }, + max2: { clickhouseName: "max2", minArgs: 2, maxArgs: 2 }, + runningDifference: { clickhouseName: "runningDifference", minArgs: 1, maxArgs: 1 }, + runningDifferenceStartingWithFirstValue: { clickhouseName: "runningDifferenceStartingWithFirstValue", minArgs: 1, maxArgs: 1 }, + neighbor: { clickhouseName: "neighbor", minArgs: 2, maxArgs: 3 }, + + // Window functions + rank: { clickhouseName: "rank", minArgs: 0, maxArgs: 0 }, + dense_rank: { clickhouseName: "dense_rank", minArgs: 0, maxArgs: 0 }, + row_number: { clickhouseName: "row_number", minArgs: 0, maxArgs: 0 }, + first_value: { clickhouseName: "first_value", minArgs: 1, maxArgs: 1 }, + last_value: { clickhouseName: "last_value", minArgs: 1, maxArgs: 1 }, + nth_value: { clickhouseName: "nth_value", minArgs: 2, maxArgs: 2 }, + lagInFrame: { clickhouseName: "lagInFrame", minArgs: 1, maxArgs: 3 }, + leadInFrame: { clickhouseName: "leadInFrame", minArgs: 1, maxArgs: 3 }, + lag: { clickhouseName: "lagInFrame", minArgs: 1, maxArgs: 3 }, + lead: { clickhouseName: "leadInFrame", minArgs: 1, maxArgs: 3 }, +}; + +/** + * Aggregate functions available in TSQL + * Port of HOGQL_AGGREGATIONS from aggregations.py + */ +export const TSQL_AGGREGATIONS: Record = { + // Standard aggregate functions + count: { clickhouseName: "count", minArgs: 0, maxArgs: 1, aggregate: true, caseSensitive: false }, + countIf: { clickhouseName: "countIf", minArgs: 1, maxArgs: 2, aggregate: true }, + countDistinct: { clickhouseName: "countDistinct", minArgs: 1, maxArgs: 1, aggregate: true }, + countDistinctIf: { clickhouseName: "countDistinctIf", minArgs: 1, maxArgs: 2, aggregate: true }, + min: { clickhouseName: "min", minArgs: 1, maxArgs: 1, aggregate: true, caseSensitive: false }, + minIf: { clickhouseName: "minIf", minArgs: 2, maxArgs: 2, aggregate: true }, + max: { clickhouseName: "max", minArgs: 1, maxArgs: 1, aggregate: true, caseSensitive: false }, + maxIf: { clickhouseName: "maxIf", minArgs: 2, maxArgs: 2, aggregate: true }, + sum: { clickhouseName: "sum", minArgs: 1, maxArgs: 1, aggregate: true, caseSensitive: false }, + sumIf: { clickhouseName: "sumIf", minArgs: 2, maxArgs: 2, aggregate: true }, + avg: { clickhouseName: "avg", minArgs: 1, maxArgs: 1, aggregate: true, caseSensitive: false }, + avgIf: { clickhouseName: "avgIf", minArgs: 2, maxArgs: 2, aggregate: true }, + any: { clickhouseName: "any", minArgs: 1, maxArgs: 1, aggregate: true }, + anyIf: { clickhouseName: "anyIf", minArgs: 2, maxArgs: 2, aggregate: true }, + anyLast: { clickhouseName: "anyLast", minArgs: 1, maxArgs: 1, aggregate: true }, + anyLastIf: { clickhouseName: "anyLastIf", minArgs: 2, maxArgs: 2, aggregate: true }, + anyHeavy: { clickhouseName: "anyHeavy", minArgs: 1, maxArgs: 1, aggregate: true }, + anyHeavyIf: { clickhouseName: "anyHeavyIf", minArgs: 2, maxArgs: 2, aggregate: true }, + argMin: { clickhouseName: "argMin", minArgs: 2, maxArgs: 2, aggregate: true }, + argMinIf: { clickhouseName: "argMinIf", minArgs: 3, maxArgs: 3, aggregate: true }, + argMax: { clickhouseName: "argMax", minArgs: 2, maxArgs: 2, aggregate: true }, + argMaxIf: { clickhouseName: "argMaxIf", minArgs: 3, maxArgs: 3, aggregate: true }, + stddevPop: { clickhouseName: "stddevPop", minArgs: 1, maxArgs: 1, aggregate: true }, + stddevSamp: { clickhouseName: "stddevSamp", minArgs: 1, maxArgs: 1, aggregate: true }, + varPop: { clickhouseName: "varPop", minArgs: 1, maxArgs: 1, aggregate: true }, + varSamp: { clickhouseName: "varSamp", minArgs: 1, maxArgs: 1, aggregate: true }, + covarPop: { clickhouseName: "covarPop", minArgs: 2, maxArgs: 2, aggregate: true }, + covarSamp: { clickhouseName: "covarSamp", minArgs: 2, maxArgs: 2, aggregate: true }, + corr: { clickhouseName: "corr", minArgs: 2, maxArgs: 2, aggregate: true }, + + // Array aggregations + groupArray: { clickhouseName: "groupArray", minArgs: 1, maxArgs: 1, aggregate: true }, + groupArrayIf: { clickhouseName: "groupArrayIf", minArgs: 2, maxArgs: 2, aggregate: true }, + groupUniqArray: { clickhouseName: "groupUniqArray", minArgs: 1, maxArgs: 1, aggregate: true }, + groupUniqArrayIf: { clickhouseName: "groupUniqArrayIf", minArgs: 2, maxArgs: 2, aggregate: true }, + groupArrayInsertAt: { clickhouseName: "groupArrayInsertAt", minArgs: 2, maxArgs: 2, aggregate: true }, + groupArrayMovingAvg: { clickhouseName: "groupArrayMovingAvg", minArgs: 1, maxArgs: 1, aggregate: true }, + groupArrayMovingSum: { clickhouseName: "groupArrayMovingSum", minArgs: 1, maxArgs: 1, aggregate: true }, + groupArraySample: { clickhouseName: "groupArraySample", minArgs: 1, maxArgs: 1, minParams: 1, maxParams: 2, aggregate: true }, + array_agg: { clickhouseName: "groupArray", minArgs: 1, maxArgs: 1, aggregate: true }, + + // Bitmap aggregations + groupBitmap: { clickhouseName: "groupBitmap", minArgs: 1, maxArgs: 1, aggregate: true }, + groupBitmapAnd: { clickhouseName: "groupBitmapAnd", minArgs: 1, maxArgs: 1, aggregate: true }, + groupBitmapOr: { clickhouseName: "groupBitmapOr", minArgs: 1, maxArgs: 1, aggregate: true }, + groupBitmapXor: { clickhouseName: "groupBitmapXor", minArgs: 1, maxArgs: 1, aggregate: true }, + + // Uniq functions + uniq: { clickhouseName: "uniq", minArgs: 1, aggregate: true }, + uniqIf: { clickhouseName: "uniqIf", minArgs: 2, aggregate: true }, + uniqExact: { clickhouseName: "uniqExact", minArgs: 1, aggregate: true }, + uniqExactIf: { clickhouseName: "uniqExactIf", minArgs: 2, aggregate: true }, + uniqHLL12: { clickhouseName: "uniqHLL12", minArgs: 1, aggregate: true }, + uniqTheta: { clickhouseName: "uniqTheta", minArgs: 1, aggregate: true }, + + // Quantile functions + median: { clickhouseName: "median", minArgs: 1, maxArgs: 1, aggregate: true }, + medianIf: { clickhouseName: "medianIf", minArgs: 2, maxArgs: 2, aggregate: true }, + medianExact: { clickhouseName: "medianExact", minArgs: 1, maxArgs: 1, aggregate: true }, + quantile: { clickhouseName: "quantile", minArgs: 1, maxArgs: 1, minParams: 1, maxParams: 1, aggregate: true }, + quantileIf: { clickhouseName: "quantileIf", minArgs: 2, maxArgs: 2, minParams: 1, maxParams: 1, aggregate: true }, + quantiles: { clickhouseName: "quantiles", minArgs: 1, aggregate: true }, + + // Statistical functions + simpleLinearRegression: { clickhouseName: "simpleLinearRegression", minArgs: 2, maxArgs: 2, aggregate: true }, + contingency: { clickhouseName: "contingency", minArgs: 2, maxArgs: 2, aggregate: true }, + cramersV: { clickhouseName: "cramersV", minArgs: 2, maxArgs: 2, aggregate: true }, + theilsU: { clickhouseName: "theilsU", minArgs: 2, maxArgs: 2, aggregate: true }, + + // Sum/Map variants + sumMap: { clickhouseName: "sumMap", minArgs: 1, maxArgs: 2, aggregate: true }, + minMap: { clickhouseName: "minMap", minArgs: 1, maxArgs: 2, aggregate: true }, + maxMap: { clickhouseName: "maxMap", minArgs: 1, maxArgs: 2, aggregate: true }, + + // TopK + topK: { clickhouseName: "topK", minArgs: 1, maxArgs: 1, minParams: 1, maxParams: 1, aggregate: true }, + + // Funnel + windowFunnel: { clickhouseName: "windowFunnel", minArgs: 1, maxArgs: 99, aggregate: true }, +}; + +/** + * Find a function in the TSQL functions map + * Supports case-insensitive lookup for non-case-sensitive functions + */ +function findFunction( + name: string, + functions: Record +): TSQLFunctionMeta | undefined { + const func = functions[name]; + if (func !== undefined) { + return func; + } + + const lowerFunc = functions[name.toLowerCase()]; + if (lowerFunc === undefined) { + return undefined; + } + + // If we haven't found a function with the case preserved, but we have found it in lowercase, + // then the function names are different case-wise only. + if (lowerFunc.caseSensitive) { + return undefined; + } + + return lowerFunc; +} + +/** + * Find a TSQL aggregation function by name + */ +export function findTSQLAggregation(name: string): TSQLFunctionMeta | undefined { + return findFunction(name, TSQL_AGGREGATIONS); +} + +/** + * Find a TSQL function by name + */ +export function findTSQLFunction(name: string): TSQLFunctionMeta | undefined { + return findFunction(name, TSQL_CLICKHOUSE_FUNCTIONS); +} + +/** + * Get all exposed function names (for autocomplete, suggestions, etc.) + */ +export function getAllExposedFunctionNames(): string[] { + const functionNames = Object.keys(TSQL_CLICKHOUSE_FUNCTIONS).filter((name) => !name.startsWith("_")); + const aggregationNames = Object.keys(TSQL_AGGREGATIONS).filter((name) => !name.startsWith("_")); + return [...functionNames, ...aggregationNames]; +} + +/** + * Validate function arguments + */ +export function validateFunctionArgs( + args: unknown[], + minArgs: number, + maxArgs: number | undefined, + functionName: string, + options: { + functionTerm?: string; + argumentTerm?: string; + } = {} +): void { + const { functionTerm = "function", argumentTerm = "argument" } = options; + + const tooFew = args.length < minArgs; + const tooMany = maxArgs !== undefined && args.length > maxArgs; + + if (minArgs === maxArgs && (tooFew || tooMany)) { + throw new Error( + `${functionTerm.charAt(0).toUpperCase() + functionTerm.slice(1)} '${functionName}' expects ${minArgs} ${argumentTerm}${minArgs !== 1 ? "s" : ""}, found ${args.length}` + ); + } + if (tooFew) { + throw new Error( + `${functionTerm.charAt(0).toUpperCase() + functionTerm.slice(1)} '${functionName}' expects at least ${minArgs} ${argumentTerm}${minArgs !== 1 ? "s" : ""}, found ${args.length}` + ); + } + if (tooMany) { + throw new Error( + `${functionTerm.charAt(0).toUpperCase() + functionTerm.slice(1)} '${functionName}' expects at most ${maxArgs} ${argumentTerm}${maxArgs !== 1 ? "s" : ""}, found ${args.length}` + ); + } +} + diff --git a/internal-packages/tsql/src/query/printer.test.ts b/internal-packages/tsql/src/query/printer.test.ts new file mode 100644 index 000000000..84f274d80 --- /dev/null +++ b/internal-packages/tsql/src/query/printer.test.ts @@ -0,0 +1,606 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { parseTSQLSelect, parseTSQLExpr } from "../index.js"; +import { ClickHousePrinter, printToClickHouse } from "./printer.js"; +import { createPrinterContext, PrinterContext } from "./printer_context.js"; +import { createSchemaRegistry, column, type TableSchema, type SchemaRegistry } from "./schema.js"; +import { QueryError, SyntaxError } from "./errors.js"; + +/** + * Test table schemas + */ +const taskRunsSchema: TableSchema = { + name: "task_runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + id: { name: "id", type: "String", ...column("String") }, + status: { name: "status", type: "String", ...column("String") }, + task_identifier: { name: "task_identifier", type: "String", ...column("String") }, + created_at: { name: "created_at", type: "DateTime64", ...column("DateTime64") }, + updated_at: { name: "updated_at", type: "DateTime64", ...column("DateTime64") }, + started_at: { name: "started_at", type: "Nullable(DateTime64)", ...column("Nullable(DateTime64)") }, + completed_at: { name: "completed_at", type: "Nullable(DateTime64)", ...column("Nullable(DateTime64)") }, + duration_ms: { name: "duration_ms", type: "Nullable(UInt64)", ...column("Nullable(UInt64)") }, + organization_id: { name: "organization_id", type: "String", ...column("String") }, + project_id: { name: "project_id", type: "String", ...column("String") }, + environment_id: { name: "environment_id", type: "String", ...column("String") }, + queue_name: { name: "queue_name", type: "String", ...column("String") }, + is_test: { name: "is_test", type: "UInt8", ...column("UInt8") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, +}; + +const taskEventsSchema: TableSchema = { + name: "task_events", + clickhouseName: "trigger_dev.task_events_v2", + columns: { + id: { name: "id", type: "String", ...column("String") }, + run_id: { name: "run_id", type: "String", ...column("String") }, + event_type: { name: "event_type", type: "String", ...column("String") }, + timestamp: { name: "timestamp", type: "DateTime64", ...column("DateTime64") }, + payload: { name: "payload", type: "String", ...column("String") }, + organization_id: { name: "organization_id", type: "String", ...column("String") }, + project_id: { name: "project_id", type: "String", ...column("String") }, + environment_id: { name: "environment_id", type: "String", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, +}; + +/** + * Helper to create a test context + */ +function createTestContext(overrides: Partial[0]> = {}): PrinterContext { + const schema = createSchemaRegistry([taskRunsSchema, taskEventsSchema]); + return createPrinterContext({ + organizationId: "org_test123", + projectId: "proj_test456", + environmentId: "env_test789", + schema, + ...overrides, + }); +} + +/** + * Helper to print a query and get SQL + params + */ +function printQuery(query: string, context?: PrinterContext) { + const ast = parseTSQLSelect(query); + const ctx = context ?? createTestContext(); + return printToClickHouse(ast, ctx); +} + +describe("ClickHousePrinter", () => { + describe("Basic SELECT statements", () => { + it("should print a simple SELECT *", () => { + const { sql, params } = printQuery("SELECT * FROM task_runs"); + + expect(sql).toContain("SELECT *"); + expect(sql).toContain("FROM trigger_dev.task_runs_v2"); + // Should include tenant guards + expect(sql).toContain("organization_id"); + expect(sql).toContain("project_id"); + expect(sql).toContain("environment_id"); + }); + + it("should print SELECT with specific columns", () => { + const { sql, params } = printQuery("SELECT id, status, created_at FROM task_runs"); + + expect(sql).toContain("SELECT id, status, created_at"); + expect(sql).toContain("FROM trigger_dev.task_runs_v2"); + }); + + it("should print SELECT DISTINCT", () => { + const { sql } = printQuery("SELECT DISTINCT status FROM task_runs"); + + expect(sql).toContain("SELECT DISTINCT status"); + }); + + it("should print SELECT with aliases", () => { + const { sql } = printQuery("SELECT id AS run_id, status AS run_status FROM task_runs"); + + expect(sql).toContain("id AS run_id"); + expect(sql).toContain("status AS run_status"); + }); + }); + + describe("WHERE clauses", () => { + it("should print WHERE with equality comparison", () => { + const { sql, params } = printQuery("SELECT * FROM task_runs WHERE status = 'completed'"); + + expect(sql).toContain("WHERE"); + expect(sql).toContain("equals("); + // Value should be parameterized + expect(Object.values(params)).toContain("completed"); + }); + + it("should print WHERE with multiple conditions", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE status = 'completed' AND is_test = 0"); + + expect(sql).toContain("and("); + expect(sql).toContain("equals("); + }); + + it("should print WHERE with OR conditions", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE status = 'completed' OR status = 'failed'"); + + expect(sql).toContain("or("); + }); + + it("should print WHERE with NOT", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE NOT status = 'pending'"); + + expect(sql).toContain("not("); + }); + + it("should print WHERE with BETWEEN", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE duration_ms BETWEEN 100 AND 1000"); + + expect(sql).toContain("BETWEEN"); + }); + + it("should print WHERE with IN", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE status IN ('completed', 'failed')"); + + expect(sql).toContain("in("); + }); + + it("should print WHERE with NOT IN", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE status NOT IN ('pending')"); + + expect(sql).toContain("notIn("); + }); + + it("should print WHERE with LIKE", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE task_identifier LIKE 'email%'"); + + expect(sql).toContain("like("); + }); + + it("should print WHERE with ILIKE", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE task_identifier ILIKE '%Email%'"); + + expect(sql).toContain("ilike("); + }); + + it("should handle NULL comparisons", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE started_at = NULL"); + + expect(sql).toContain("isNull("); + }); + + it("should handle IS NOT NULL comparisons", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE started_at != NULL"); + + expect(sql).toContain("isNotNull("); + }); + }); + + describe("ORDER BY clauses", () => { + it("should print ORDER BY ASC", () => { + const { sql } = printQuery("SELECT * FROM task_runs ORDER BY created_at ASC"); + + expect(sql).toContain("ORDER BY created_at ASC"); + }); + + it("should print ORDER BY DESC", () => { + const { sql } = printQuery("SELECT * FROM task_runs ORDER BY created_at DESC"); + + expect(sql).toContain("ORDER BY created_at DESC"); + }); + + it("should print ORDER BY with multiple columns", () => { + const { sql } = printQuery("SELECT * FROM task_runs ORDER BY status ASC, created_at DESC"); + + expect(sql).toContain("ORDER BY status ASC, created_at DESC"); + }); + }); + + describe("LIMIT and OFFSET", () => { + it("should print LIMIT", () => { + const { sql } = printQuery("SELECT * FROM task_runs LIMIT 10"); + + expect(sql).toContain("LIMIT 10"); + }); + + it("should print LIMIT with OFFSET", () => { + const { sql } = printQuery("SELECT * FROM task_runs LIMIT 10 OFFSET 20"); + + expect(sql).toContain("LIMIT 10"); + expect(sql).toContain("OFFSET 20"); + }); + + it("should cap LIMIT to maxRows setting", () => { + const context = createTestContext({ settings: { maxRows: 100 } }); + const { sql } = printQuery("SELECT * FROM task_runs LIMIT 1000", context); + + expect(sql).toContain("LIMIT 100"); + }); + + it("should add default LIMIT when none specified", () => { + const context = createTestContext({ settings: { maxRows: 10000 } }); + const { sql } = printQuery("SELECT * FROM task_runs", context); + + expect(sql).toContain("LIMIT 10000"); + }); + }); + + describe("GROUP BY clauses", () => { + it("should print GROUP BY", () => { + const { sql } = printQuery("SELECT status, count(*) FROM task_runs GROUP BY status"); + + expect(sql).toContain("GROUP BY status"); + }); + + it("should print GROUP BY with multiple columns", () => { + const { sql } = printQuery("SELECT status, queue_name, count(*) FROM task_runs GROUP BY status, queue_name"); + + expect(sql).toContain("GROUP BY status, queue_name"); + }); + + it("should print GROUP BY with HAVING", () => { + const { sql } = printQuery("SELECT status, count(*) as cnt FROM task_runs GROUP BY status HAVING cnt > 10"); + + expect(sql).toContain("GROUP BY status"); + expect(sql).toContain("HAVING"); + expect(sql).toContain("greater("); + }); + }); + + describe("Aggregate functions", () => { + it("should print COUNT", () => { + const { sql } = printQuery("SELECT count(*) FROM task_runs"); + + expect(sql).toContain("count(*)"); + }); + + it("should print COUNT DISTINCT", () => { + const { sql } = printQuery("SELECT count(DISTINCT status) FROM task_runs"); + + expect(sql).toContain("count(DISTINCT status)"); + }); + + it("should print SUM", () => { + const { sql } = printQuery("SELECT sum(duration_ms) FROM task_runs"); + + expect(sql).toContain("sum(duration_ms)"); + }); + + it("should print AVG", () => { + const { sql } = printQuery("SELECT avg(duration_ms) FROM task_runs"); + + expect(sql).toContain("avg(duration_ms)"); + }); + + it("should print MIN and MAX", () => { + const { sql } = printQuery("SELECT min(created_at), max(created_at) FROM task_runs"); + + expect(sql).toContain("min(created_at)"); + expect(sql).toContain("max(created_at)"); + }); + }); + + describe("Arithmetic operations", () => { + it("should print addition", () => { + const { sql } = printQuery("SELECT duration_ms + 100 FROM task_runs"); + + expect(sql).toContain("plus(duration_ms, 100)"); + }); + + it("should print subtraction", () => { + const { sql } = printQuery("SELECT duration_ms - 100 FROM task_runs"); + + expect(sql).toContain("minus(duration_ms, 100)"); + }); + + it("should print multiplication", () => { + const { sql } = printQuery("SELECT duration_ms * 2 FROM task_runs"); + + expect(sql).toContain("multiply(duration_ms, 2)"); + }); + + it("should print division", () => { + const { sql } = printQuery("SELECT duration_ms / 1000 FROM task_runs"); + + expect(sql).toContain("divide(duration_ms, 1000)"); + }); + + it("should print modulo", () => { + const { sql } = printQuery("SELECT duration_ms % 60 FROM task_runs"); + + expect(sql).toContain("modulo(duration_ms, 60)"); + }); + }); + + describe("Tenant isolation", () => { + it("should inject tenant guards for single table", () => { + const context = createTestContext({ + organizationId: "org_abc", + projectId: "proj_def", + environmentId: "env_ghi", + }); + const { sql, params } = printQuery("SELECT * FROM task_runs", context); + + // Should have WHERE clause with tenant columns + expect(sql).toContain("WHERE"); + expect(sql).toContain("organization_id"); + expect(sql).toContain("project_id"); + expect(sql).toContain("environment_id"); + + // Values should be parameterized + expect(Object.values(params)).toContain("org_abc"); + expect(Object.values(params)).toContain("proj_def"); + expect(Object.values(params)).toContain("env_ghi"); + }); + + it("should combine tenant guards with user WHERE clause", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE status = 'completed'"); + + // Should have both tenant guard AND user condition + expect(sql).toContain("and("); + expect(sql).toContain("organization_id"); + expect(sql).toContain("equals("); + }); + }); + + describe("SQL injection prevention", () => { + it("should parameterize string values", () => { + const { sql, params } = printQuery("SELECT * FROM task_runs WHERE status = 'DROP TABLE users'"); + + // The malicious string should be in params, not in SQL + expect(sql).not.toContain("DROP TABLE"); + expect(Object.values(params)).toContain("DROP TABLE users"); + }); + + it("should safely handle identifiers with special characters", () => { + // This should either escape or reject + expect(() => { + printQuery("SELECT * FROM task_runs WHERE `weird`column` = 'test'"); + }).toThrow(); + }); + + it("should reject identifiers with % character", () => { + expect(() => { + const context = createTestContext(); + // Create a schema with a table name containing % + const badSchema = createSchemaRegistry([ + { + ...taskRunsSchema, + name: "task%runs", + }, + ]); + const badContext = createPrinterContext({ + organizationId: "org_test", + projectId: "proj_test", + environmentId: "env_test", + schema: badSchema, + }); + printQuery("SELECT * FROM `task%runs`", badContext); + }).toThrow(); + }); + + it("should parameterize numeric values inline", () => { + const { sql, params } = printQuery("SELECT * FROM task_runs WHERE duration_ms > 1000"); + + // Numbers can be inlined safely + expect(sql).toContain("1000"); + }); + + it("should handle boolean values safely", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE is_test = 1"); + + expect(sql).toContain("equals(is_test, 1)"); + }); + }); + + describe("Subqueries", () => { + it("should print subquery in FROM clause", () => { + const { sql } = printQuery(` + SELECT status, cnt + FROM ( + SELECT status, count(*) as cnt + FROM task_runs + GROUP BY status + ) + `); + + expect(sql).toContain("SELECT status, cnt"); + expect(sql).toContain("FROM ("); + expect(sql).toContain("count(*)"); + }); + + it("should print subquery in WHERE clause", () => { + const { sql } = printQuery(` + SELECT * FROM task_runs + WHERE id IN (SELECT run_id FROM task_events WHERE event_type = 'completed') + `); + + expect(sql).toContain("in(id,"); + expect(sql).toContain("SELECT run_id FROM"); + }); + }); + + describe("UNION queries", () => { + it("should print UNION ALL", () => { + const { sql } = printQuery(` + SELECT id, status FROM task_runs WHERE status = 'completed' + UNION ALL + SELECT id, status FROM task_runs WHERE status = 'failed' + `); + + expect(sql).toContain("UNION ALL"); + }); + }); + + describe("Window functions", () => { + it("should print ROW_NUMBER", () => { + const { sql } = printQuery(` + SELECT id, status, row_number() OVER (PARTITION BY status ORDER BY created_at DESC) as rn + FROM task_runs + `); + + expect(sql).toContain("row_number()"); + expect(sql).toContain("OVER ("); + expect(sql).toContain("PARTITION BY status"); + expect(sql).toContain("ORDER BY created_at DESC"); + }); + }); + + describe("Functions", () => { + it("should print toDateTime", () => { + const { sql } = printQuery("SELECT toDateTime(created_at) FROM task_runs"); + + expect(sql).toContain("toDateTime(created_at)"); + }); + + it("should print now()", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE created_at > now()"); + + expect(sql).toContain("now()"); + }); + + it("should print string functions", () => { + const { sql } = printQuery("SELECT lower(status), upper(queue_name) FROM task_runs"); + + expect(sql).toContain("lower(status)"); + expect(sql).toContain("upper(queue_name)"); + }); + + it("should print conditional functions", () => { + const { sql } = printQuery("SELECT if(is_test = 1, 'test', 'prod') FROM task_runs"); + + expect(sql).toContain("if("); + }); + + it("should print coalesce", () => { + const { sql } = printQuery("SELECT coalesce(started_at, created_at) FROM task_runs"); + + expect(sql).toContain("coalesce(started_at, created_at)"); + }); + }); + + describe("Arrays and tuples", () => { + it("should print array literals", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE status IN ['completed', 'failed']"); + + expect(sql).toContain("["); + expect(sql).toContain("]"); + }); + + it("should print tuple", () => { + const { sql } = printQuery("SELECT tuple(id, status) FROM task_runs"); + + expect(sql).toContain("tuple(id, status)"); + }); + }); + + describe("Error handling", () => { + it("should throw QueryError for unknown tables", () => { + expect(() => { + printQuery("SELECT * FROM unknown_table"); + }).toThrow(QueryError); + }); + + it("should throw QueryError for unknown functions", () => { + expect(() => { + printQuery("SELECT unknown_function(id) FROM task_runs"); + }).toThrow(QueryError); + }); + + it("should throw QueryError for nested aggregations", () => { + expect(() => { + printQuery("SELECT sum(count(*)) FROM task_runs"); + }).toThrow(QueryError); + }); + + it("should throw SyntaxError for malformed queries", () => { + expect(() => { + parseTSQLSelect("SELECT * FORM task_runs"); // typo: FORM instead of FROM + }).toThrow(); + }); + }); + + describe("Pretty printing", () => { + it("should format SQL with newlines when pretty=true", () => { + const ast = parseTSQLSelect("SELECT id, status FROM task_runs WHERE status = 'completed' ORDER BY created_at"); + const context = createTestContext(); + const printer = new ClickHousePrinter(context, { pretty: true }); + const { sql } = printer.print(ast); + + expect(sql).toContain("\n"); + }); + + it("should produce single-line SQL when pretty=false", () => { + const ast = parseTSQLSelect("SELECT id, status FROM task_runs WHERE status = 'completed' ORDER BY created_at"); + const context = createTestContext(); + const printer = new ClickHousePrinter(context, { pretty: false }); + const { sql } = printer.print(ast); + + // Count newlines - there should be very few or none in the main query structure + const newlineCount = (sql.match(/\n/g) || []).length; + expect(newlineCount).toBeLessThan(3); + }); + }); + + describe("Parameter generation", () => { + it("should generate unique parameter names", () => { + const { params } = printQuery(` + SELECT * FROM task_runs + WHERE status = 'completed' AND queue_name = 'email' AND task_identifier = 'send' + `); + + // Should have multiple unique parameter keys + const keys = Object.keys(params); + expect(new Set(keys).size).toBe(keys.length); + }); + + it("should include correct types in placeholders", () => { + const { sql, params } = printQuery("SELECT * FROM task_runs WHERE status = 'test'"); + + // Should have String type in placeholder + expect(sql).toMatch(/\{tsql_val_\d+: String\}/); + }); + }); +}); + +describe("Edge cases", () => { + it("should handle empty string values", () => { + const { sql, params } = printQuery("SELECT * FROM task_runs WHERE status = ''"); + + expect(Object.values(params)).toContain(""); + }); + + it("should handle special characters in strings", () => { + const { sql, params } = printQuery("SELECT * FROM task_runs WHERE status = 'test\nvalue'"); + + // The string value should be parameterized + expect(Object.keys(params).length).toBeGreaterThan(0); + }); + + it("should handle large numbers", () => { + // Use a number that JavaScript can safely represent + const { sql } = printQuery("SELECT * FROM task_runs WHERE duration_ms > 1000000000000"); + + expect(sql).toContain("1000000000000"); + }); + + it("should handle negative numbers", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE duration_ms > -1000"); + + // Negative numbers might be expressed as subtraction or negate + expect(sql).toMatch(/-1000|minus|negate/); + }); + + it("should handle floating point numbers", () => { + const { sql } = printQuery("SELECT * FROM task_runs WHERE duration_ms > 1.5"); + + expect(sql).toContain("1.5"); + }); +}); + diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts new file mode 100644 index 000000000..67aad5552 --- /dev/null +++ b/internal-packages/tsql/src/query/printer.ts @@ -0,0 +1,970 @@ +// TypeScript port of posthog/hogql/printer.py +// ClickHouse SQL printer with tenant isolation and schema validation + +import { + And, + Alias, + ArithmeticOperation, + ArithmeticOperationOp, + Array as ASTArray, + ArrayAccess, + AST, + BetweenExpr, + Call, + CompareOperation, + CompareOperationOp, + Constant, + CTE, + Dict, + Expression, + Field, + JoinConstraint, + JoinExpr, + Lambda, + LimitByExpr, + Not, + Or, + OrderExpr, + Placeholder, + RatioExpr, + SampleExpr, + SelectQuery, + SelectSetQuery, + Tuple, + TupleAccess, + WindowExpr, + WindowFrameExpr, + WindowFunction, +} from "./ast"; +import { escapeClickHouseIdentifier, escapeTSQLIdentifier, escapeClickHouseString } from "./escape"; +import { ImpossibleASTError, NotImplementedError, QueryError } from "./errors"; +import { + TSQL_CLICKHOUSE_FUNCTIONS, + TSQL_AGGREGATIONS, + TSQL_COMPARISON_MAPPING, + findTSQLAggregation, + findTSQLFunction, + validateFunctionArgs, +} from "./functions"; +import { PrinterContext } from "./printer_context"; +import { findTable, validateTable, TableSchema } from "./schema"; + +/** + * Result of printing an AST to ClickHouse SQL + */ +export interface PrintResult { + /** The generated ClickHouse SQL query */ + sql: string; + /** Parameter values for parameterized query execution */ + params: Record; +} + +/** + * Response from visiting a JoinExpr node + */ +interface JoinExprResponse { + /** The printed SQL for the JOIN */ + printedSql: string; + /** Additional WHERE clause to add (e.g., tenant isolation guards) */ + where: Expression | null; +} + +/** + * ClickHouse SQL Printer + * + * Converts a TSQL AST to a parameterized ClickHouse SQL query with: + * - Automatic tenant isolation (organization_id, project_id, environment_id) + * - Schema-based table/column validation + * - SQL injection protection via parameterized queries + */ +export class ClickHousePrinter { + /** Stack of AST nodes being visited (for context) */ + private stack: AST[] = []; + /** Indent level for pretty printing */ + private indentLevel = -1; + /** Tab size for pretty printing */ + private tabSize = 4; + /** Whether to pretty print output */ + private pretty: boolean; + + constructor( + private context: PrinterContext, + options: { pretty?: boolean } = {} + ) { + this.pretty = options.pretty ?? false; + } + + /** + * Print an AST node to ClickHouse SQL + */ + print(node: SelectQuery | SelectSetQuery): PrintResult { + const sql = this.visit(node); + return { + sql, + params: this.context.getParams(), + }; + } + + /** + * Get current indentation string + */ + private indent(extra = 0): string { + return " ".repeat(this.tabSize * (this.indentLevel + extra)); + } + + /** + * Visit an AST node and return its SQL representation + */ + private visit(node: AST | null | undefined): string { + if (node === null || node === undefined) { + return ""; + } + + this.stack.push(node); + this.indentLevel++; + + let response: string; + + // Type-based dispatch + const nodeType = (node as Expression).expression_type; + + switch (nodeType) { + case "select_set_query": + response = this.visitSelectSetQuery(node as SelectSetQuery); + break; + case "select_query": + response = this.visitSelectQuery(node as SelectQuery); + break; + case "cte": + response = this.visitCTE(node as CTE); + break; + case "alias": + response = this.visitAlias(node as Alias); + break; + case "arithmetic_operation": + response = this.visitArithmeticOperation(node as ArithmeticOperation); + break; + case "and": + response = this.visitAnd(node as And); + break; + case "or": + response = this.visitOr(node as Or); + break; + case "compare_operation": + response = this.visitCompareOperation(node as CompareOperation); + break; + case "not": + response = this.visitNot(node as Not); + break; + case "between_expr": + response = this.visitBetweenExpr(node as BetweenExpr); + break; + case "order_expr": + response = this.visitOrderExpr(node as OrderExpr); + break; + case "array_access": + response = this.visitArrayAccess(node as ArrayAccess); + break; + case "array": + response = this.visitArray(node as ASTArray); + break; + case "dict": + response = this.visitDict(node as Dict); + break; + case "tuple_access": + response = this.visitTupleAccess(node as TupleAccess); + break; + case "tuple": + response = this.visitTuple(node as Tuple); + break; + case "lambda": + response = this.visitLambda(node as Lambda); + break; + case "constant": + response = this.visitConstant(node as Constant); + break; + case "field": + response = this.visitField(node as Field); + break; + case "placeholder": + response = this.visitPlaceholder(node as Placeholder); + break; + case "call": + response = this.visitCall(node as Call); + break; + case "join_expr": + // JoinExpr is handled specially since it returns more than just SQL + throw new ImpossibleASTError("JoinExpr should be handled via visitJoinExpr"); + case "join_constraint": + response = this.visitJoinConstraint(node as JoinConstraint); + break; + case "window_frame_expr": + response = this.visitWindowFrameExpr(node as WindowFrameExpr); + break; + case "window_expr": + response = this.visitWindowExpr(node as WindowExpr); + break; + case "window_function": + response = this.visitWindowFunction(node as WindowFunction); + break; + case "limit_by_expr": + response = this.visitLimitByExpr(node as LimitByExpr); + break; + case "ratio_expr": + response = this.visitRatioExpr(node as RatioExpr); + break; + case "sample_expr": + response = this.visitSampleExpr(node as SampleExpr); + break; + default: + throw new NotImplementedError(`Unknown expression type: ${nodeType}`); + } + + this.indentLevel--; + this.stack.pop(); + + return response; + } + + // ============================================================ + // SELECT Query Visitors + // ============================================================ + + private visitSelectSetQuery(node: SelectSetQuery): string { + this.indentLevel--; + let ret = this.visit(node.initial_select_query); + if (this.pretty) { + ret = ret.trim(); + } + + for (const expr of node.subsequent_select_queries) { + let query = this.visit(expr.select_query); + if (this.pretty) { + query = query.trim(); + } + if (expr.set_operator !== undefined) { + if (this.pretty) { + ret += `\n${this.indent(1)}${expr.set_operator}\n${this.indent(1)}`; + } else { + ret += ` ${expr.set_operator} `; + } + } + ret += query; + } + + this.indentLevel++; + + // Wrap in parentheses if not top level + if (this.stack.length > 1) { + return `(${ret.trim()})`; + } + return ret; + } + + private visitSelectQuery(node: SelectQuery): string { + // Determine if this is a top-level query + const partOfSelectUnion = this.stack.length >= 2 && this.isSelectSetQuery(this.stack[this.stack.length - 2]); + const isTopLevelQuery = this.stack.length <= 1 || (this.stack.length === 2 && partOfSelectUnion); + + // Build WHERE clause starting with any existing where + let where: Expression | undefined = node.where; + + // Process CTEs + const cteStrings: string[] = []; + if (node.ctes) { + for (const [name, cte] of Object.entries(node.ctes)) { + cteStrings.push(`${this.printIdentifier(name)} AS (${this.visit(cte.expr)})`); + } + } + + // Process joins and collect tenant guards + const joinedTables: string[] = []; + let nextJoin: JoinExpr | undefined = node.select_from; + + while (nextJoin) { + const visitedJoin = this.visitJoinExpr(nextJoin); + joinedTables.push(visitedJoin.printedSql); + + // Add tenant guard to WHERE clause + const extraWhere = visitedJoin.where; + if (extraWhere !== null) { + if (where === undefined) { + where = extraWhere; + } else if ((where as And).expression_type === "and") { + where = { expression_type: "and", exprs: [extraWhere, ...(where as And).exprs] } as And; + } else { + where = { expression_type: "and", exprs: [extraWhere, where] } as And; + } + } + + nextJoin = nextJoin.next_join; + } + + // Process SELECT columns + let columns: string[]; + if (node.select && node.select.length > 0) { + columns = node.select.map((col) => this.visit(col)); + } else { + columns = ["1"]; + } + + // Process WINDOW definitions + let windowClause: string | null = null; + if (node.window_exprs && Object.keys(node.window_exprs).length > 0) { + const windowDefs = Object.entries(node.window_exprs).map( + ([name, expr]) => `${this.printIdentifier(name)} AS (${this.visit(expr)})` + ); + windowClause = windowDefs.join(", "); + } + + // Process other clauses + const prewhere = node.prewhere ? this.visit(node.prewhere) : null; + const whereStr = where ? this.visit(where) : null; + const groupBy = node.group_by ? node.group_by.map((col) => this.visit(col)) : null; + const having = node.having ? this.visit(node.having) : null; + const orderBy = node.order_by ? node.order_by.map((col) => this.visit(col)) : null; + + // Process ARRAY JOIN + let arrayJoin = ""; + if (node.array_join_op) { + if (!["ARRAY JOIN", "LEFT ARRAY JOIN", "INNER ARRAY JOIN"].includes(node.array_join_op)) { + throw new ImpossibleASTError(`Invalid ARRAY JOIN operation: ${node.array_join_op}`); + } + arrayJoin = node.array_join_op; + if (!node.array_join_list || node.array_join_list.length === 0) { + throw new ImpossibleASTError("Invalid ARRAY JOIN without an array"); + } + arrayJoin += ` ${node.array_join_list.map((expr) => this.visit(expr)).join(", ")}`; + } + + // Format spacing + const space = this.pretty ? `\n${this.indent(1)}` : " "; + const comma = this.pretty ? `,\n${this.indent(1)}` : ", "; + + // Build SQL clauses + const clauses: (string | null)[] = [ + `SELECT${space}${node.distinct ? "DISTINCT " : ""}${columns.join(comma)}`, + joinedTables.length > 0 ? `FROM${space}${joinedTables.join(space)}` : null, + arrayJoin || null, + prewhere ? `PREWHERE${space}${prewhere}` : null, + whereStr ? `WHERE${space}${whereStr}` : null, + groupBy && groupBy.length > 0 ? `GROUP BY${space}${groupBy.join(comma)}` : null, + having ? `HAVING${space}${having}` : null, + windowClause ? `WINDOW${space}${windowClause}` : null, + orderBy && orderBy.length > 0 ? `ORDER BY${space}${orderBy.join(comma)}` : null, + ]; + + // Process LIMIT + let limit = node.limit; + if (isTopLevelQuery && this.context.maxRows) { + const maxLimit = this.context.maxRows; + if (limit !== undefined) { + // Cap the limit to maxRows + if ((limit as Constant).expression_type === "constant") { + const constLimit = limit as Constant; + if (typeof constLimit.value === "number") { + constLimit.value = Math.min(constLimit.value, maxLimit); + } + } + } else { + // Add default limit + limit = { expression_type: "constant", value: maxLimit } as Constant; + } + } + + // Add LIMIT BY + if (node.limit_by) { + const limitByExprs = node.limit_by.exprs.map((e) => this.visit(e)).join(", "); + const offsetPart = node.limit_by.offset_value ? ` OFFSET ${this.visit(node.limit_by.offset_value)}` : ""; + clauses.push(`LIMIT ${this.visit(node.limit_by.n)}${offsetPart} BY ${limitByExprs}`); + } + + // Add LIMIT/OFFSET + if (limit !== undefined) { + clauses.push(`LIMIT ${this.visit(limit)}`); + if (node.limit_with_ties) { + clauses.push("WITH TIES"); + } + if (node.offset !== undefined) { + clauses.push(`OFFSET ${this.visit(node.offset)}`); + } + } + + // Add CTEs + let response: string; + if (this.pretty) { + response = clauses + .filter((c) => c !== null) + .map((c) => `${this.indent()}${c}`) + .join("\n"); + } else { + response = clauses.filter((c) => c !== null).join(" "); + } + + // Add WITH clause for CTEs + if (cteStrings.length > 0) { + const ctePrefix = `WITH ${cteStrings.join(", ")}`; + response = `${ctePrefix} ${response}`; + } + + // Wrap subqueries in parentheses + if (!partOfSelectUnion && !isTopLevelQuery) { + response = this.pretty ? `(${response.trim()})` : `(${response})`; + } + + return response; + } + + // ============================================================ + // JOIN Expression Visitor + // ============================================================ + + private visitJoinExpr(node: JoinExpr): JoinExprResponse { + let extraWhere: Expression | null = null; + const joinStrings: string[] = []; + + // Add join type + if (node.join_type) { + joinStrings.push(node.join_type); + } + + // Handle table reference + if (node.table) { + const tableExpr = node.table; + + if ((tableExpr as Field).expression_type === "field") { + // Direct table reference + const field = tableExpr as Field; + const tableName = field.chain[0]; + if (typeof tableName !== "string") { + throw new QueryError("Table name must be a string"); + } + + // Look up table schema and get ClickHouse table name + const tableSchema = this.lookupTable(tableName); + joinStrings.push(tableSchema.clickhouseName); + + // Add tenant isolation guard + extraWhere = this.createTenantGuard(tableSchema, node.alias || tableName); + } else if ( + (tableExpr as SelectQuery).expression_type === "select_query" || + (tableExpr as SelectSetQuery).expression_type === "select_set_query" + ) { + // Subquery + joinStrings.push(this.visit(tableExpr)); + } else if ((tableExpr as Placeholder).expression_type === "placeholder") { + // Placeholder - visit inner expression + joinStrings.push(this.visit(tableExpr)); + } else { + throw new QueryError(`Unsupported table expression type: ${(tableExpr as Expression).expression_type}`); + } + } + + // Add alias + if (node.alias) { + joinStrings.push(`AS ${this.printIdentifier(node.alias)}`); + } + + // Add FINAL + if (node.table_final) { + joinStrings.push("FINAL"); + } + + // Add SAMPLE + if (node.sample) { + const sampleClause = this.visitSampleExpr(node.sample); + if (sampleClause) { + joinStrings.push(sampleClause); + } + } + + // Add constraint + if (node.constraint) { + joinStrings.push(`${node.constraint.constraint_type} ${this.visit(node.constraint)}`); + } + + return { + printedSql: joinStrings.join(" "), + where: extraWhere, + }; + } + + // ============================================================ + // Tenant Isolation + // ============================================================ + + /** + * Create a WHERE clause expression for tenant isolation + */ + private createTenantGuard(tableSchema: TableSchema, tableAlias: string): And { + const { tenantColumns } = tableSchema; + + // Create equality comparisons for each tenant column + const orgGuard: CompareOperation = { + expression_type: "compare_operation", + op: CompareOperationOp.Eq, + left: { expression_type: "field", chain: [tableAlias, tenantColumns.organizationId] } as Field, + right: { expression_type: "constant", value: this.context.organizationId } as Constant, + }; + + const projectGuard: CompareOperation = { + expression_type: "compare_operation", + op: CompareOperationOp.Eq, + left: { expression_type: "field", chain: [tableAlias, tenantColumns.projectId] } as Field, + right: { expression_type: "constant", value: this.context.projectId } as Constant, + }; + + const envGuard: CompareOperation = { + expression_type: "compare_operation", + op: CompareOperationOp.Eq, + left: { expression_type: "field", chain: [tableAlias, tenantColumns.environmentId] } as Field, + right: { expression_type: "constant", value: this.context.environmentId } as Constant, + }; + + return { + expression_type: "and", + exprs: [orgGuard, projectGuard, envGuard], + }; + } + + // ============================================================ + // Expression Visitors + // ============================================================ + + private visitCTE(node: CTE): string { + return this.visit(node.expr); + } + + private visitAlias(node: Alias): string { + const expr = this.visit(node.expr); + if (node.hidden) { + return expr; + } + return `${expr} AS ${this.printIdentifier(node.alias)}`; + } + + private visitArithmeticOperation(node: ArithmeticOperation): string { + const left = this.visit(node.left); + const right = this.visit(node.right); + + switch (node.op) { + case ArithmeticOperationOp.Add: + return `plus(${left}, ${right})`; + case ArithmeticOperationOp.Sub: + return `minus(${left}, ${right})`; + case ArithmeticOperationOp.Mult: + return `multiply(${left}, ${right})`; + case ArithmeticOperationOp.Div: + return `divide(${left}, ${right})`; + case ArithmeticOperationOp.Mod: + return `modulo(${left}, ${right})`; + default: + throw new ImpossibleASTError(`Unknown ArithmeticOperationOp: ${node.op}`); + } + } + + private visitAnd(node: And): string { + if (node.exprs.length === 1) { + return this.visit(node.exprs[0]); + } + + // Optimization: filter out constant true values, short-circuit on false + const exprs: string[] = []; + for (const expr of node.exprs) { + const printed = this.visit(expr); + if (printed === "0") { + // Short-circuit: and(..., 0, ...) => 0 + return "0"; + } + if (printed !== "1") { + // Skip constant true values + exprs.push(printed); + } + } + + if (exprs.length === 0) { + return "1"; + } + if (exprs.length === 1) { + return exprs[0]; + } + return `and(${exprs.join(", ")})`; + } + + private visitOr(node: Or): string { + if (node.exprs.length === 1) { + return this.visit(node.exprs[0]); + } + + // Optimization: filter out constant false values, short-circuit on true + const exprs: string[] = []; + for (const expr of node.exprs) { + const printed = this.visit(expr); + if (printed === "1") { + // Short-circuit: or(..., 1, ...) => 1 + return "1"; + } + if (printed !== "0") { + // Skip constant false values + exprs.push(printed); + } + } + + if (exprs.length === 0) { + return "0"; + } + if (exprs.length === 1) { + return exprs[0]; + } + return `or(${exprs.join(", ")})`; + } + + private visitNot(node: Not): string { + return `not(${this.visit(node.expr)})`; + } + + private visitCompareOperation(node: CompareOperation): string { + const left = this.visit(node.left); + const right = this.visit(node.right); + + switch (node.op) { + case CompareOperationOp.Eq: + // Handle NULL comparison + if ((node.right as Constant).expression_type === "constant" && (node.right as Constant).value === null) { + return `isNull(${left})`; + } + if ((node.left as Constant).expression_type === "constant" && (node.left as Constant).value === null) { + return `isNull(${right})`; + } + return `equals(${left}, ${right})`; + + case CompareOperationOp.NotEq: + // Handle NULL comparison + if ((node.right as Constant).expression_type === "constant" && (node.right as Constant).value === null) { + return `isNotNull(${left})`; + } + if ((node.left as Constant).expression_type === "constant" && (node.left as Constant).value === null) { + return `isNotNull(${right})`; + } + return `notEquals(${left}, ${right})`; + + case CompareOperationOp.Lt: + return `less(${left}, ${right})`; + case CompareOperationOp.LtEq: + return `lessOrEquals(${left}, ${right})`; + case CompareOperationOp.Gt: + return `greater(${left}, ${right})`; + case CompareOperationOp.GtEq: + return `greaterOrEquals(${left}, ${right})`; + case CompareOperationOp.Like: + return `like(${left}, ${right})`; + case CompareOperationOp.ILike: + return `ilike(${left}, ${right})`; + case CompareOperationOp.NotLike: + return `notLike(${left}, ${right})`; + case CompareOperationOp.NotILike: + return `notILike(${left}, ${right})`; + case CompareOperationOp.In: + return `in(${left}, ${right})`; + case CompareOperationOp.NotIn: + return `notIn(${left}, ${right})`; + case CompareOperationOp.GlobalIn: + return `globalIn(${left}, ${right})`; + case CompareOperationOp.GlobalNotIn: + return `globalNotIn(${left}, ${right})`; + case CompareOperationOp.Regex: + return `match(${left}, ${right})`; + case CompareOperationOp.NotRegex: + return `not(match(${left}, ${right}))`; + case CompareOperationOp.IRegex: + return `match(${left}, concat('(?i)', ${right}))`; + case CompareOperationOp.NotIRegex: + return `not(match(${left}, concat('(?i)', ${right})))`; + default: + throw new ImpossibleASTError(`Unknown CompareOperationOp: ${node.op}`); + } + } + + private visitBetweenExpr(node: BetweenExpr): string { + const expr = this.visit(node.expr); + const low = this.visit(node.low); + const high = this.visit(node.high); + const notKw = node.negated ? " NOT" : ""; + return `${expr}${notKw} BETWEEN ${low} AND ${high}`; + } + + private visitOrderExpr(node: OrderExpr): string { + const expr = this.visit(node.expr); + return `${expr} ${node.order || "ASC"}`; + } + + private visitArrayAccess(node: ArrayAccess): string { + const array = this.visit(node.array); + const property = this.visit(node.property); + return `${array}[${property}]`; + } + + private visitArray(node: ASTArray): string { + const elements = node.exprs.map((e) => this.visit(e)); + return `[${elements.join(", ")}]`; + } + + private visitDict(node: Dict): string { + // Convert dict to tuple format for ClickHouse + let str = "tuple('__hx_tag', '__hx_obj'"; + for (const [key, value] of node.items) { + str += `, ${this.visit(key)}, ${this.visit(value)}`; + } + return str + ")"; + } + + private visitTupleAccess(node: TupleAccess): string { + const tuple = this.visit(node.tuple); + const index = node.index; + const isSimple = + (node.tuple as Field).expression_type === "field" || + (node.tuple as Tuple).expression_type === "tuple" || + (node.tuple as Call).expression_type === "call"; + return isSimple ? `${tuple}.${index}` : `(${tuple}).${index}`; + } + + private visitTuple(node: Tuple): string { + const elements = node.exprs.map((e) => this.visit(e)); + return `tuple(${elements.join(", ")})`; + } + + private visitLambda(node: Lambda): string { + const identifiers = node.args.map((arg) => this.printIdentifier(arg)); + if (identifiers.length === 0) { + throw new QueryError("Lambdas require at least one argument"); + } + const args = identifiers.length === 1 ? identifiers[0] : `(${identifiers.join(", ")})`; + return `${args} -> ${this.visit(node.expr as Expression)}`; + } + + private visitConstant(node: Constant): string { + const value = node.value; + + // Inline simple constants + if (value === null) { + return "NULL"; + } + if (typeof value === "boolean") { + return value ? "1" : "0"; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + if (Number.isNaN(value)) return "nan"; + return value > 0 ? "inf" : "-inf"; + } + return String(value); + } + + // Use parameterized query for strings and other types + return this.context.addValue(value); + } + + private visitField(node: Field): string { + if (node.chain.length === 0) { + throw new ImpossibleASTError("Field chain is empty"); + } + + // Handle asterisk + if (node.chain.length === 1 && node.chain[0] === "*") { + return "*"; + } + + // Print each chain element + return node.chain.map((part) => this.printIdentifierOrIndex(part)).join("."); + } + + private visitPlaceholder(node: Placeholder): string { + return this.visit(node.expr); + } + + private visitCall(node: Call): string { + const name = node.name; + + // Check if this is a comparison function + if (name in TSQL_COMPARISON_MAPPING) { + const op = TSQL_COMPARISON_MAPPING[name]; + if (node.args.length !== 2) { + throw new QueryError(`Comparison '${name}' requires exactly two arguments`); + } + return this.visitCompareOperation({ + expression_type: "compare_operation", + left: node.args[0], + right: node.args[1], + op, + }); + } + + // Check for aggregation function + const aggMeta = findTSQLAggregation(name); + if (aggMeta) { + validateFunctionArgs(node.args, aggMeta.minArgs, aggMeta.maxArgs, name, { + functionTerm: "aggregation", + }); + + // Check for nested aggregations + for (const stackNode of this.stack.slice().reverse()) { + if ((stackNode as SelectQuery).expression_type === "select_query") { + break; + } + if ((stackNode as Call).expression_type === "call" && stackNode !== node) { + const stackCall = stackNode as Call; + if (findTSQLAggregation(stackCall.name)) { + throw new QueryError( + `Aggregation '${name}' cannot be nested inside another aggregation '${stackCall.name}'` + ); + } + } + } + + const args = node.args.map((arg) => this.visit(arg)); + const params = node.params ? node.params.map((p) => this.visit(p)) : null; + const paramsPart = params ? `(${params.join(", ")})` : ""; + const distinctPart = node.distinct ? "DISTINCT " : ""; + return `${aggMeta.clickhouseName}${paramsPart}(${distinctPart}${args.join(", ")})`; + } + + // Check for regular function + const funcMeta = findTSQLFunction(name); + if (funcMeta) { + validateFunctionArgs(node.args, funcMeta.minArgs, funcMeta.maxArgs, name); + + const args = node.args.map((arg) => this.visit(arg)); + const params = node.params ? node.params.map((p) => this.visit(p)) : null; + const paramsPart = params ? `(${params.join(", ")})` : ""; + return `${funcMeta.clickhouseName}${paramsPart}(${args.join(", ")})`; + } + + // Unknown function - throw error + throw new QueryError(`Unknown function: ${name}`); + } + + private visitJoinConstraint(node: JoinConstraint): string { + return this.visit(node.expr); + } + + private visitWindowFrameExpr(node: WindowFrameExpr): string { + if (node.frame_type === "CURRENT ROW") { + return "CURRENT ROW"; + } + if (node.frame_value !== undefined) { + return `${node.frame_value} ${node.frame_type}`; + } + return `UNBOUNDED ${node.frame_type}`; + } + + private visitWindowExpr(node: WindowExpr): string { + const parts: string[] = []; + + if (node.partition_by && node.partition_by.length > 0) { + parts.push(`PARTITION BY ${node.partition_by.map((e) => this.visit(e)).join(", ")}`); + } + + if (node.order_by && node.order_by.length > 0) { + parts.push(`ORDER BY ${node.order_by.map((e) => this.visit(e)).join(", ")}`); + } + + if (node.frame_method && node.frame_start) { + let frameStr = `${node.frame_method} `; + if (node.frame_end) { + frameStr += `BETWEEN ${this.visit(node.frame_start)} AND ${this.visit(node.frame_end)}`; + } else { + frameStr += this.visit(node.frame_start); + } + parts.push(frameStr); + } + + return parts.join(" "); + } + + private visitWindowFunction(node: WindowFunction): string { + const args = node.args ? node.args.map((a) => this.visit(a)) : []; + const funcCall = `${node.name}(${args.join(", ")})`; + + if (node.over_identifier) { + return `${funcCall} OVER ${this.printIdentifier(node.over_identifier)}`; + } + if (node.over_expr) { + return `${funcCall} OVER (${this.visit(node.over_expr)})`; + } + return funcCall; + } + + private visitLimitByExpr(node: LimitByExpr): string { + const exprs = node.exprs.map((e) => this.visit(e)).join(", "); + const offsetPart = node.offset_value ? ` OFFSET ${this.visit(node.offset_value)}` : ""; + return `LIMIT ${this.visit(node.n)}${offsetPart} BY ${exprs}`; + } + + private visitRatioExpr(node: RatioExpr): string { + const left = this.visit(node.left); + if (node.right) { + return `${left}/${this.visit(node.right)}`; + } + return left; + } + + private visitSampleExpr(node: SampleExpr): string { + const sample = this.visitRatioExpr(node.sample_value); + if (node.offset_value) { + return `SAMPLE ${sample} OFFSET ${this.visitRatioExpr(node.offset_value)}`; + } + return `SAMPLE ${sample}`; + } + + // ============================================================ + // Helper Methods + // ============================================================ + + /** + * Print an identifier safely escaped for ClickHouse + */ + private printIdentifier(name: string): string { + return escapeClickHouseIdentifier(name); + } + + /** + * Print an identifier or array index + */ + private printIdentifierOrIndex(part: string | number): string { + if (typeof part === "number") { + return String(part); + } + return escapeClickHouseIdentifier(part); + } + + /** + * Check if a node is a SelectSetQuery + */ + private isSelectSetQuery(node: AST): boolean { + return (node as SelectSetQuery).expression_type === "select_set_query"; + } + + /** + * Look up a table in the schema registry + */ + private lookupTable(tableName: string): TableSchema { + return validateTable(this.context.schema, tableName); + } +} + +// ============================================================ +// Public API +// ============================================================ + +/** + * Print a TSQL AST to ClickHouse SQL + */ +export function printToClickHouse( + node: SelectQuery | SelectSetQuery, + context: PrinterContext, + options: { pretty?: boolean } = {} +): PrintResult { + const printer = new ClickHousePrinter(context, options); + return printer.print(node); +} + diff --git a/internal-packages/tsql/src/query/printer_context.ts b/internal-packages/tsql/src/query/printer_context.ts new file mode 100644 index 000000000..02aec6dd3 --- /dev/null +++ b/internal-packages/tsql/src/query/printer_context.ts @@ -0,0 +1,199 @@ +// TypeScript port of posthog/hogql/context.py +// Adapted for ClickHouse client's {param: Type} syntax + +import { getClickHouseType } from "./escape"; +import { SchemaRegistry } from "./schema"; + +/** + * Settings that control query execution behavior + */ +export interface QuerySettings { + /** Maximum number of rows to return */ + maxRows?: number; + /** Timezone for date/time operations */ + timezone?: string; + /** Whether to allow full table scans */ + allowFullTableScans?: boolean; + /** Query timeout in seconds */ + timeoutSeconds?: number; +} + +/** + * Default query settings + */ +export const DEFAULT_QUERY_SETTINGS: Required = { + maxRows: 10000, + timezone: "UTC", + allowFullTableScans: false, + timeoutSeconds: 60, +}; + +/** + * A warning or notice collected during query printing + */ +export interface QueryNotice { + code: string; + message: string; + start?: number; + end?: number; +} + +/** + * Context for the TSQL to ClickHouse printer + * + * Holds: + * - Tenant IDs for automatic WHERE clause injection + * - Schema registry for table/column validation + * - Parameter accumulator for SQL injection safety + * - Query settings and execution options + */ +export class PrinterContext { + /** Accumulated parameter values for parameterized query */ + private values: Record = {}; + + /** Counter for generating unique parameter names */ + private paramCounter = 0; + + /** Warnings collected during printing */ + readonly warnings: QueryNotice[] = []; + + /** Errors collected during printing */ + readonly errors: QueryNotice[] = []; + + constructor( + /** The organization ID for tenant isolation */ + public readonly organizationId: string, + /** The project ID for tenant isolation */ + public readonly projectId: string, + /** The environment ID for tenant isolation */ + public readonly environmentId: string, + /** Schema registry containing allowed tables and columns */ + public readonly schema: SchemaRegistry, + /** Query execution settings */ + public readonly settings: QuerySettings = {} + ) { + // Initialize with default settings + this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings }; + } + + /** + * Get the timezone setting + */ + get timezone(): string { + return this.settings.timezone ?? DEFAULT_QUERY_SETTINGS.timezone; + } + + /** + * Get the max rows setting + */ + get maxRows(): number { + return this.settings.maxRows ?? DEFAULT_QUERY_SETTINGS.maxRows; + } + + /** + * Add a value to the parameter map and return a ClickHouse placeholder + * + * @param value The value to parameterize + * @returns A placeholder string like "{tsql_val_0: String}" + */ + addValue(value: unknown): string { + const key = `tsql_val_${this.paramCounter++}`; + this.values[key] = value; + const chType = getClickHouseType(value); + return `{${key}: ${chType}}`; + } + + /** + * Add a value with a specific key (for named parameters) + * + * @param key The parameter name + * @param value The value + * @returns A placeholder string like "{key: Type}" + */ + addNamedValue(key: string, value: unknown): string { + this.values[key] = value; + const chType = getClickHouseType(value); + return `{${key}: ${chType}}`; + } + + /** + * Get all accumulated parameter values + */ + getParams(): Record { + return { ...this.values }; + } + + /** + * Add a warning notice + */ + addWarning(code: string, message: string, start?: number, end?: number): void { + this.warnings.push({ code, message, start, end }); + } + + /** + * Add an error notice + */ + addError(code: string, message: string, start?: number, end?: number): void { + this.errors.push({ code, message, start, end }); + } + + /** + * Check if any errors were collected + */ + hasErrors(): boolean { + return this.errors.length > 0; + } + + /** + * Create a child context that shares the same parameter accumulator + * Useful for handling subqueries while keeping parameters unified + */ + createChildContext(): PrinterContext { + const child = new PrinterContext( + this.organizationId, + this.projectId, + this.environmentId, + this.schema, + this.settings + ); + // Share the same values map so parameters are unified + child.values = this.values; + // Share the same counter reference via closure + const parentCounter = this.paramCounter; + const parentThis = this; + Object.defineProperty(child, "paramCounter", { + get() { + return parentThis.paramCounter; + }, + set(v: number) { + parentThis.paramCounter = v; + }, + }); + return child; + } +} + +/** + * Options for creating a printer context + */ +export interface PrinterContextOptions { + organizationId: string; + projectId: string; + environmentId: string; + schema: SchemaRegistry; + settings?: QuerySettings; +} + +/** + * Create a new PrinterContext + */ +export function createPrinterContext(options: PrinterContextOptions): PrinterContext { + return new PrinterContext( + options.organizationId, + options.projectId, + options.environmentId, + options.schema, + options.settings + ); +} + diff --git a/internal-packages/tsql/src/query/query.ts b/internal-packages/tsql/src/query/query.ts deleted file mode 100644 index 020ea3a2f..000000000 --- a/internal-packages/tsql/src/query/query.ts +++ /dev/null @@ -1,94 +0,0 @@ -// import { CharStreams, CommonTokenStream } from "antlr4ts"; -// import { TSQLLexer } from "../grammar/TSQLLexer.js"; -// import { TSQLParser } from "../grammar/TSQLParser.js"; -// import { ClickHouseQueryVisitor } from "./parser.js"; -// import type { ClickHouse } from "@internal/clickhouse"; -// import { ClickhouseQueryBuilder } from "@internal/clickhouse/client/queryBuilder.js"; -// import { z } from "zod"; - -// export interface TQueryOptions { -// organizationId: string; -// projectId: string; -// environmentId: string; -// } - -// export class TQuery { -// private readonly organizationId: string; -// private readonly projectId: string; -// private readonly environmentId: string; -// private readonly clickhouseReader: ClickHouse; - -// constructor(clickhouseReader: ClickHouse, options: TQueryOptions) { -// this.clickhouseReader = clickhouseReader; -// this.organizationId = options.organizationId; -// this.projectId = options.projectId; -// this.environmentId = options.environmentId; -// } - -// /** -// * Execute a TSQL query and return the results -// * @param input TSQL query string -// * @param schema Zod schema for the output rows -// * @returns Promise with query results -// */ -// async query>( -// input: string, -// schema: TOutput -// ): Promise<[Error | null, z.output[] | null]> { -// // Parse the TSQL input -// const inputStream = CharStreams.fromString(input); -// const lexer = new TSQLLexer(inputStream); -// const tokenStream = new CommonTokenStream(lexer as any); -// const parser = new TSQLParser(tokenStream); - -// // Parse as a SELECT statement -// const tree = parser.select(); - -// // Convert AST to QueryConfig -// const visitor = new ClickHouseQueryVisitor(); -// const queryConfig = visitor.visit(tree); - -// // Use ClickhouseQueryBuilder to build the query -// const queryBuilder = new ClickhouseQueryBuilder( -// "tsql-query", -// queryConfig.baseQuery, -// this.clickhouseReader.reader, -// schema -// ); - -// // Add existing WHERE clauses from the TSQL query -// for (const whereClause of queryConfig.whereClauses) { -// queryBuilder.where(whereClause.clause, whereClause.params); -// } - -// // Add scoping WHERE clauses -// queryBuilder -// .where("organization_id = {organizationId: String}", { -// organizationId: this.organizationId, -// }) -// .where("project_id = {projectId: String}", { -// projectId: this.projectId, -// }) -// .where("environment_id = {environmentId: String}", { -// environmentId: this.environmentId, -// }); - -// // Add GROUP BY if present -// if (queryConfig.groupBy) { -// queryBuilder.groupBy(queryConfig.groupBy); -// } - -// // Add ORDER BY if present -// if (queryConfig.orderBy) { -// queryBuilder.orderBy(queryConfig.orderBy); -// } - -// // Add LIMIT if present -// if (queryConfig.limit !== undefined) { -// queryBuilder.limit(queryConfig.limit); -// } - -// // Execute the query -// return await queryBuilder.execute(); -// } -// } diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts new file mode 100644 index 000000000..84c4ad798 --- /dev/null +++ b/internal-packages/tsql/src/query/schema.ts @@ -0,0 +1,290 @@ +// Schema definitions for TSQL query validation +// Defines allowed tables, columns, and tenant isolation configuration + +import { QueryError } from "./errors"; + +/** + * ClickHouse data types supported by TSQL + */ +export type ClickHouseType = + | "String" + | "UInt8" + | "UInt16" + | "UInt32" + | "UInt64" + | "Int8" + | "Int16" + | "Int32" + | "Int64" + | "Float32" + | "Float64" + | "Date" + | "Date32" + | "DateTime" + | "DateTime64" + | "UUID" + | "Bool" + | "JSON" + | "Nullable(String)" + | "Nullable(UInt8)" + | "Nullable(UInt16)" + | "Nullable(UInt32)" + | "Nullable(UInt64)" + | "Nullable(Int8)" + | "Nullable(Int16)" + | "Nullable(Int32)" + | "Nullable(Int64)" + | "Nullable(Float32)" + | "Nullable(Float64)" + | "Nullable(Date)" + | "Nullable(Date32)" + | "Nullable(DateTime)" + | "Nullable(DateTime64)" + | "Nullable(UUID)" + | "Nullable(Bool)" + | "LowCardinality(String)" + | `Array(${string})` + | `Map(${string}, ${string})`; + +/** + * Schema definition for a single column + */ +export interface ColumnSchema { + /** The name of the column as exposed to TSQL queries */ + name: string; + /** The actual ClickHouse column name (if different from `name`) */ + clickhouseName?: string; + /** The ClickHouse data type */ + type: ClickHouseType; + /** Whether this column can be selected */ + selectable?: boolean; + /** Whether this column can be used in WHERE clauses */ + filterable?: boolean; + /** Whether this column can be used in ORDER BY clauses */ + sortable?: boolean; + /** Whether this column can be used in GROUP BY clauses */ + groupable?: boolean; + /** Description of the column for documentation/autocomplete */ + description?: string; +} + +/** + * Configuration for tenant isolation columns + * These columns are automatically added to WHERE clauses + */ +export interface TenantColumnConfig { + /** The column name for organization ID filtering */ + organizationId: string; + /** The column name for project ID filtering */ + projectId: string; + /** The column name for environment ID filtering */ + environmentId: string; +} + +/** + * Schema definition for a table + */ +export interface TableSchema { + /** The name of the table as exposed to TSQL queries */ + name: string; + /** The fully qualified ClickHouse table name (e.g., "trigger_dev.task_runs_v2") */ + clickhouseName: string; + /** Column definitions for this table */ + columns: Record; + /** Tenant isolation column configuration */ + tenantColumns: TenantColumnConfig; + /** Description of the table for documentation/autocomplete */ + description?: string; + /** Whether this table can be joined to other tables */ + joinable?: boolean; +} + +/** + * Schema registry containing all allowed tables + */ +export interface SchemaRegistry { + /** Map of table names to their schemas */ + tables: Record; + /** Default tenant column names (used when a table doesn't specify its own) */ + defaultTenantColumns: TenantColumnConfig; +} + +/** + * Create a basic column schema with common defaults + */ +export function column( + type: ClickHouseType, + options: Partial> = {} +): Omit { + return { + type, + selectable: true, + filterable: true, + sortable: true, + groupable: true, + ...options, + }; +} + +/** + * Create a schema registry from a list of table schemas + */ +export function createSchemaRegistry( + tables: TableSchema[], + defaultTenantColumns?: TenantColumnConfig +): SchemaRegistry { + const tableMap: Record = {}; + for (const table of tables) { + tableMap[table.name] = table; + } + return { + tables: tableMap, + defaultTenantColumns: defaultTenantColumns ?? { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; +} + +/** + * Look up a table schema by name + */ +export function findTable(schema: SchemaRegistry, tableName: string): TableSchema | undefined { + return schema.tables[tableName]; +} + +/** + * Look up a column schema by table and column name + */ +export function findColumn( + schema: SchemaRegistry, + tableName: string, + columnName: string +): ColumnSchema | undefined { + const table = findTable(schema, tableName); + if (!table) return undefined; + return table.columns[columnName]; +} + +/** + * Validate that a table exists in the schema + * @throws QueryError if the table is not found + */ +export function validateTable(schema: SchemaRegistry, tableName: string): TableSchema { + const table = findTable(schema, tableName); + if (!table) { + const availableTables = Object.keys(schema.tables).join(", "); + throw new QueryError( + `Table "${tableName}" is not accessible. Available tables: ${availableTables || "(none)"}` + ); + } + return table; +} + +/** + * Validate that a column exists in a table and can be selected + * @throws QueryError if the column is not found or not selectable + */ +export function validateSelectColumn( + schema: SchemaRegistry, + tableName: string, + columnName: string +): ColumnSchema { + const table = validateTable(schema, tableName); + const col = table.columns[columnName]; + if (!col) { + const availableColumns = Object.keys(table.columns).join(", "); + throw new QueryError( + `Column "${columnName}" does not exist on table "${tableName}". Available columns: ${availableColumns}` + ); + } + if (col.selectable === false) { + throw new QueryError(`Column "${columnName}" on table "${tableName}" is not selectable`); + } + return col; +} + +/** + * Validate that a column can be used in a WHERE clause + * @throws QueryError if the column is not filterable + */ +export function validateFilterColumn( + schema: SchemaRegistry, + tableName: string, + columnName: string +): ColumnSchema { + const table = validateTable(schema, tableName); + const col = table.columns[columnName]; + if (!col) { + throw new QueryError(`Column "${columnName}" does not exist on table "${tableName}"`); + } + if (col.filterable === false) { + throw new QueryError(`Column "${columnName}" on table "${tableName}" cannot be used in WHERE`); + } + return col; +} + +/** + * Validate that a column can be used in ORDER BY + * @throws QueryError if the column is not sortable + */ +export function validateSortColumn( + schema: SchemaRegistry, + tableName: string, + columnName: string +): ColumnSchema { + const table = validateTable(schema, tableName); + const col = table.columns[columnName]; + if (!col) { + throw new QueryError(`Column "${columnName}" does not exist on table "${tableName}"`); + } + if (col.sortable === false) { + throw new QueryError(`Column "${columnName}" on table "${tableName}" cannot be used in ORDER BY`); + } + return col; +} + +/** + * Validate that a column can be used in GROUP BY + * @throws QueryError if the column is not groupable + */ +export function validateGroupColumn( + schema: SchemaRegistry, + tableName: string, + columnName: string +): ColumnSchema { + const table = validateTable(schema, tableName); + const col = table.columns[columnName]; + if (!col) { + throw new QueryError(`Column "${columnName}" does not exist on table "${tableName}"`); + } + if (col.groupable === false) { + throw new QueryError(`Column "${columnName}" on table "${tableName}" cannot be used in GROUP BY`); + } + return col; +} + +/** + * Get the actual ClickHouse column name (handles aliasing) + */ +export function getClickHouseColumnName(col: ColumnSchema): string { + return col.clickhouseName ?? col.name; +} + +/** + * Get all column names available for autocomplete + */ +export function getTableColumnNames(schema: SchemaRegistry, tableName: string): string[] { + const table = findTable(schema, tableName); + if (!table) return []; + return Object.keys(table.columns); +} + +/** + * Get all table names available for autocomplete + */ +export function getAllTableNames(schema: SchemaRegistry): string[] { + return Object.keys(schema.tables); +} +