From cd9dec6506e95c8ea14ee396dc5bc380fea02a1d Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 17 Dec 2025 14:02:22 +0000 Subject: [PATCH] Added virtual column support --- .../routes/storybook.tsql-editor/route.tsx | 42 ++- internal-packages/clickhouse/src/tsql.test.ts | 267 ++++++++++++++++++ internal-packages/tsql/src/index.ts | 3 + .../tsql/src/query/printer.test.ts | 212 ++++++++++++++ internal-packages/tsql/src/query/printer.ts | 123 +++++++- .../tsql/src/query/schema.test.ts | 93 ++++++ internal-packages/tsql/src/query/schema.ts | 41 +++ 7 files changed, 778 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/routes/storybook.tsql-editor/route.tsx b/apps/webapp/app/routes/storybook.tsql-editor/route.tsx index 8c89e45e5..a461478b6 100644 --- a/apps/webapp/app/routes/storybook.tsql-editor/route.tsx +++ b/apps/webapp/app/routes/storybook.tsql-editor/route.tsx @@ -40,6 +40,22 @@ const runsSchema: TableSchema = { name: "duration_ms", ...column("Nullable(UInt64)", { description: "Run duration in milliseconds" }), }, + // Virtual column: computed from started_at and completed_at + execution_duration: { + name: "execution_duration", + ...column("Nullable(Int64)", { + description: "Computed execution time in milliseconds (virtual column)", + }), + expression: "dateDiff('millisecond', started_at, completed_at)", + }, + // Virtual column: duration in seconds for convenience + duration_seconds: { + name: "duration_seconds", + ...column("Float64", { + description: "Duration in seconds (virtual column)", + }), + expression: "duration_ms / 1000.0", + }, organization_id: { name: "organization_id", ...column("String") }, project_id: { name: "project_id", ...column("String") }, environment_id: { name: "environment_id", ...column("String") }, @@ -88,6 +104,18 @@ const exampleQueries = [ name: "Enum IN clause", query: "SELECT * FROM runs WHERE status IN ('PENDING', 'QUEUED', 'EXECUTING') LIMIT 50", }, + { + name: "Virtual columns", + query: `SELECT + id, + status, + execution_duration, + duration_seconds +FROM runs +WHERE execution_duration > 5000 +ORDER BY execution_duration DESC +LIMIT 20`, + }, { name: "Aggregation", query: "SELECT status, count(*) as count FROM runs GROUP BY status ORDER BY count DESC", @@ -268,8 +296,15 @@ export default function Story() { {Object.entries(table.columns).map(([name, col]) => (
- {name} + + {name} + {col.type} + {col.expression && ( + + virtual + + )} {col.description && ( - {col.description} )} @@ -279,6 +314,11 @@ export default function Story() { Allowed: {col.allowedValues.join(", ")}
)} + {col.expression && ( +
+ Expression: {col.expression} +
+ )}
))} diff --git a/internal-packages/clickhouse/src/tsql.test.ts b/internal-packages/clickhouse/src/tsql.test.ts index b463828c2..a43e9e806 100644 --- a/internal-packages/clickhouse/src/tsql.test.ts +++ b/internal-packages/clickhouse/src/tsql.test.ts @@ -49,6 +49,8 @@ const defaultTaskRun = { batch_id: "", created_at: Date.now(), updated_at: Date.now(), + started_at: null as number | null, + completed_at: null as number | null, tags: [] as string[], output: null, error: null, @@ -755,3 +757,268 @@ describe("TSQL Optional Tenant Filter Tests", () => { } ); }); + +describe("TSQL Virtual Column Tests", () => { + /** + * Schema with virtual (computed) columns + */ + const virtualColumnSchema: TableSchema = { + name: "task_runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + run_id: { name: "run_id", ...column("String") }, + friendly_id: { name: "friendly_id", ...column("String") }, + status: { name: "status", ...column("String") }, + task_identifier: { name: "task_identifier", ...column("String") }, + queue: { name: "queue", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + environment_type: { name: "environment_type", ...column("String") }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + created_at: { name: "created_at", ...column("DateTime") }, + updated_at: { name: "updated_at", ...column("DateTime") }, + started_at: { name: "started_at", ...column("Nullable(DateTime64)") }, + completed_at: { name: "completed_at", ...column("Nullable(DateTime64)") }, + usage_duration_ms: { name: "usage_duration_ms", ...column("UInt32") }, + is_test: { name: "is_test", ...column("Bool") }, + tags: { name: "tags", ...column("Array(String)") }, + // Virtual column: execution_duration computes milliseconds between started_at and completed_at + execution_duration: { + name: "execution_duration", + ...column("Nullable(Int64)"), + expression: "dateDiff('millisecond', started_at, completed_at)", + description: "Time between started_at and completed_at in milliseconds", + }, + // Virtual column: usage_duration_seconds converts ms to seconds + usage_duration_seconds: { + name: "usage_duration_seconds", + ...column("Float64"), + expression: "usage_duration_ms / 1000.0", + description: "Usage duration in seconds", + }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + clickhouseTest( + "should select virtual column and compute correct value", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const insert = insertTaskRuns(client, { async_insert: 0 }); + + const now = Date.now(); + const startedAt = now - 5000; // 5 seconds ago + const completedAt = now; + + await insert([ + createTaskRun({ + run_id: "run_virtual_1", + started_at: startedAt, + completed_at: completedAt, + usage_duration_ms: 3500, + }), + ]); + + const [error, rows] = await executeTSQL(client, { + name: "test-virtual-column-select", + query: "SELECT run_id, execution_duration, usage_duration_seconds FROM task_runs", + schema: z.object({ + run_id: z.string(), + execution_duration: z.number().nullable(), + usage_duration_seconds: z.number(), + }), + organizationId: "org_tenant1", + projectId: "proj_tenant1", + environmentId: "env_tenant1", + tableSchema: [virtualColumnSchema], + }); + + expect(error).toBeNull(); + expect(rows).toHaveLength(1); + // execution_duration should be approximately 5000ms (difference between started_at and completed_at) + expect(rows?.[0].execution_duration).toBeCloseTo(5000, -2); // within 100ms tolerance + // usage_duration_seconds should be 3.5 (3500ms / 1000) + expect(rows?.[0].usage_duration_seconds).toBeCloseTo(3.5, 1); + } + ); + + clickhouseTest( + "should filter by virtual column in WHERE clause", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const insert = insertTaskRuns(client, { async_insert: 0 }); + + const now = Date.now(); + + await insert([ + createTaskRun({ + run_id: "run_short", + started_at: now - 1000, // 1 second duration + completed_at: now, + }), + createTaskRun({ + run_id: "run_long", + started_at: now - 10000, // 10 second duration + completed_at: now, + }), + createTaskRun({ + run_id: "run_very_long", + started_at: now - 60000, // 60 second duration + completed_at: now, + }), + ]); + + // Query runs with execution_duration > 5000ms (5 seconds) + const [error, rows] = await executeTSQL(client, { + name: "test-virtual-column-where", + query: "SELECT run_id FROM task_runs WHERE execution_duration > 5000", + schema: z.object({ run_id: z.string() }), + organizationId: "org_tenant1", + projectId: "proj_tenant1", + environmentId: "env_tenant1", + tableSchema: [virtualColumnSchema], + }); + + expect(error).toBeNull(); + expect(rows).toHaveLength(2); + expect(rows?.map((r) => r.run_id).sort()).toEqual(["run_long", "run_very_long"]); + } + ); + + clickhouseTest("should order by virtual column", async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const insert = insertTaskRuns(client, { async_insert: 0 }); + + const now = Date.now(); + + await insert([ + createTaskRun({ + run_id: "run_order_a", + usage_duration_ms: 1000, + }), + createTaskRun({ + run_id: "run_order_b", + usage_duration_ms: 3000, + }), + createTaskRun({ + run_id: "run_order_c", + usage_duration_ms: 2000, + }), + ]); + + // Order by usage_duration_seconds descending (virtual column) + const [error, rows] = await executeTSQL(client, { + name: "test-virtual-column-order", + query: + "SELECT run_id, usage_duration_seconds FROM task_runs ORDER BY usage_duration_seconds DESC", + schema: z.object({ + run_id: z.string(), + usage_duration_seconds: z.number(), + }), + organizationId: "org_tenant1", + projectId: "proj_tenant1", + environmentId: "env_tenant1", + tableSchema: [virtualColumnSchema], + }); + + expect(error).toBeNull(); + expect(rows).toHaveLength(3); + // Should be ordered by usage_duration_seconds DESC: b (3), c (2), a (1) + expect(rows?.[0].run_id).toBe("run_order_b"); + expect(rows?.[0].usage_duration_seconds).toBeCloseTo(3.0, 1); + expect(rows?.[1].run_id).toBe("run_order_c"); + expect(rows?.[1].usage_duration_seconds).toBeCloseTo(2.0, 1); + expect(rows?.[2].run_id).toBe("run_order_a"); + expect(rows?.[2].usage_duration_seconds).toBeCloseTo(1.0, 1); + }); + + clickhouseTest( + "should use virtual column with explicit alias", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const insert = insertTaskRuns(client, { async_insert: 0 }); + + await insert([ + createTaskRun({ + run_id: "run_alias_test", + usage_duration_ms: 5000, + }), + ]); + + // Use virtual column with custom alias + const [error, rows] = await executeTSQL(client, { + name: "test-virtual-column-alias", + query: "SELECT run_id, usage_duration_seconds AS dur_sec FROM task_runs", + schema: z.object({ + run_id: z.string(), + dur_sec: z.number(), + }), + organizationId: "org_tenant1", + projectId: "proj_tenant1", + environmentId: "env_tenant1", + tableSchema: [virtualColumnSchema], + }); + + expect(error).toBeNull(); + expect(rows).toHaveLength(1); + expect(rows?.[0].dur_sec).toBeCloseTo(5.0, 1); + } + ); + + clickhouseTest( + "should handle null values in virtual column expression", + async ({ clickhouseContainer }) => { + const client = new ClickhouseClient({ + name: "test", + url: clickhouseContainer.getConnectionUrl(), + }); + + const insert = insertTaskRuns(client, { async_insert: 0 }); + + await insert([ + createTaskRun({ + run_id: "run_null_times", + // started_at and completed_at will be null + }), + ]); + + const [error, rows] = await executeTSQL(client, { + name: "test-virtual-column-null", + query: "SELECT run_id, execution_duration FROM task_runs", + schema: z.object({ + run_id: z.string(), + execution_duration: z.number().nullable(), + }), + organizationId: "org_tenant1", + projectId: "proj_tenant1", + environmentId: "env_tenant1", + tableSchema: [virtualColumnSchema], + }); + + expect(error).toBeNull(); + expect(rows).toHaveLength(1); + // execution_duration should be null when started_at or completed_at is null + expect(rows?.[0].execution_duration).toBeNull(); + } + ); +}); diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index c669ca6a0..ca36228d0 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -78,6 +78,9 @@ export { getInternalValue, getAllowedUserValues, isValidUserValue, + // Virtual column utilities + isVirtualColumn, + getVirtualColumnExpression, } from "./query/schema.js"; // Re-export printer context diff --git a/internal-packages/tsql/src/query/printer.test.ts b/internal-packages/tsql/src/query/printer.test.ts index 9c6cf20e0..6e5bd034a 100644 --- a/internal-packages/tsql/src/query/printer.test.ts +++ b/internal-packages/tsql/src/query/printer.test.ts @@ -823,3 +823,215 @@ describe("Edge cases", () => { expect(sql).toContain("1.5"); }); }); + +describe("Virtual columns", () => { + /** + * Schema with virtual (computed) columns + */ + const virtualColumnSchema: TableSchema = { + name: "runs", + clickhouseName: "trigger_dev.task_runs_v2", + columns: { + run_id: { name: "run_id", ...column("String") }, + status: { name: "status", ...column("String") }, + started_at: { name: "started_at", ...column("Nullable(DateTime64)") }, + completed_at: { name: "completed_at", ...column("Nullable(DateTime64)") }, + usage_duration_ms: { name: "usage_duration_ms", ...column("UInt32") }, + // Virtual column: execution_duration computes the time between started_at and completed_at + execution_duration: { + name: "execution_duration", + ...column("Nullable(Int64)"), + expression: "dateDiff('millisecond', started_at, completed_at)", + description: "Time between started_at and completed_at in milliseconds", + }, + // Virtual column: is_long_running checks if execution took more than 60 seconds + is_long_running: { + name: "is_long_running", + ...column("UInt8"), + expression: + "if(completed_at IS NOT NULL AND started_at IS NOT NULL, dateDiff('second', started_at, completed_at) > 60, 0)", + }, + // Virtual column with simple arithmetic + usage_duration_seconds: { + name: "usage_duration_seconds", + ...column("Float64"), + expression: "usage_duration_ms / 1000.0", + }, + organization_id: { name: "organization_id", ...column("String") }, + project_id: { name: "project_id", ...column("String") }, + environment_id: { name: "environment_id", ...column("String") }, + }, + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + }; + + function createVirtualColumnContext() { + const schema = createSchemaRegistry([virtualColumnSchema]); + return createPrinterContext({ + organizationId: "org_test", + projectId: "proj_test", + environmentId: "env_test", + schema, + }); + } + + describe("SELECT clause", () => { + it("should expand bare virtual column to expression with alias", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT execution_duration FROM runs", ctx); + + // Virtual column should be expanded to its expression with AS alias + expect(sql).toContain("(dateDiff('millisecond', started_at, completed_at))"); + expect(sql).toContain("AS execution_duration"); + }); + + it("should expand virtual column with explicit alias", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT execution_duration AS dur FROM runs", ctx); + + // Virtual column should use the user-provided alias + expect(sql).toContain("(dateDiff('millisecond', started_at, completed_at))"); + expect(sql).toContain("AS dur"); + expect(sql).not.toContain("AS execution_duration"); + }); + + it("should expand qualified virtual column reference", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT runs.execution_duration FROM runs", ctx); + + // Qualified reference should also expand + expect(sql).toContain("(dateDiff('millisecond', started_at, completed_at))"); + expect(sql).toContain("AS execution_duration"); + }); + + it("should handle multiple virtual columns in SELECT", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery( + "SELECT run_id, execution_duration, is_long_running FROM runs", + ctx + ); + + expect(sql).toContain("run_id"); + expect(sql).toContain("(dateDiff('millisecond', started_at, completed_at))"); + expect(sql).toContain("AS execution_duration"); + expect(sql).toContain( + "(if(completed_at IS NOT NULL AND started_at IS NOT NULL, dateDiff('second', started_at, completed_at) > 60, 0))" + ); + expect(sql).toContain("AS is_long_running"); + }); + + it("should mix regular and virtual columns correctly", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT run_id, status, execution_duration, started_at FROM runs", ctx); + + // Regular columns should be normal + expect(sql).toMatch(/\brun_id\b/); + expect(sql).toMatch(/\bstatus\b/); + expect(sql).toMatch(/\bstarted_at\b/); + // Virtual column should be expanded + expect(sql).toContain("(dateDiff('millisecond', started_at, completed_at)) AS execution_duration"); + }); + }); + + describe("WHERE clause", () => { + it("should expand virtual column in WHERE equality", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT * FROM runs WHERE execution_duration > 1000", ctx); + + // Virtual column in WHERE should expand without AS + expect(sql).toContain("greater((dateDiff('millisecond', started_at, completed_at)), 1000)"); + }); + + it("should expand virtual column in WHERE with comparison operators", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT * FROM runs WHERE execution_duration >= 5000", ctx); + + expect(sql).toContain("greaterOrEquals((dateDiff('millisecond', started_at, completed_at)), 5000)"); + }); + + it("should expand virtual column in WHERE with multiple conditions", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery( + "SELECT * FROM runs WHERE execution_duration > 1000 AND is_long_running = 1", + ctx + ); + + expect(sql).toContain("(dateDiff('millisecond', started_at, completed_at))"); + expect(sql).toContain( + "(if(completed_at IS NOT NULL AND started_at IS NOT NULL, dateDiff('second', started_at, completed_at) > 60, 0))" + ); + }); + + it("should expand qualified virtual column in WHERE", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT * FROM runs WHERE runs.execution_duration > 1000", ctx); + + expect(sql).toContain("greater((dateDiff('millisecond', started_at, completed_at)), 1000)"); + }); + }); + + describe("ORDER BY clause", () => { + it("should expand virtual column in ORDER BY", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT * FROM runs ORDER BY execution_duration DESC", ctx); + + expect(sql).toContain("ORDER BY (dateDiff('millisecond', started_at, completed_at)) DESC"); + }); + + it("should expand virtual column in ORDER BY with multiple columns", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT * FROM runs ORDER BY status ASC, execution_duration DESC", ctx); + + expect(sql).toContain("status ASC"); + expect(sql).toContain("(dateDiff('millisecond', started_at, completed_at)) DESC"); + }); + }); + + describe("GROUP BY clause", () => { + it("should expand virtual column in GROUP BY", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery( + "SELECT is_long_running, count(*) FROM runs GROUP BY is_long_running", + ctx + ); + + // Both SELECT and GROUP BY should have the expansion + expect(sql).toContain("GROUP BY (if(completed_at IS NOT NULL AND started_at IS NOT NULL, dateDiff('second', started_at, completed_at) > 60, 0))"); + }); + }); + + describe("Complex expressions", () => { + it("should handle arithmetic virtual columns", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT usage_duration_seconds FROM runs", ctx); + + expect(sql).toContain("(usage_duration_ms / 1000.0)"); + expect(sql).toContain("AS usage_duration_seconds"); + }); + + it("should handle virtual column in arithmetic expression", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT execution_duration / 1000 AS dur_seconds FROM runs", ctx); + + // The virtual column expression should be wrapped in the arithmetic + expect(sql).toContain("divide((dateDiff('millisecond', started_at, completed_at)), 1000)"); + expect(sql).toContain("AS dur_seconds"); + }); + }); + + describe("Regular columns unchanged", () => { + it("should not affect regular columns without expression", () => { + const ctx = createVirtualColumnContext(); + const { sql } = printQuery("SELECT run_id, status, started_at FROM runs", ctx); + + // Regular columns should appear as-is + expect(sql).toContain("run_id, status, started_at"); + // No extra parentheses or AS for regular columns in basic SELECT + expect(sql).not.toMatch(/\(run_id\)/); + expect(sql).not.toMatch(/\(status\)/); + }); + }); +}); diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index b58426583..4151274f6 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -47,7 +47,14 @@ import { validateFunctionArgs, } from "./functions"; import { PrinterContext } from "./printer_context"; -import { findTable, validateTable, TableSchema, ColumnSchema, getInternalValue } from "./schema"; +import { + findTable, + validateTable, + TableSchema, + ColumnSchema, + getInternalValue, + isVirtualColumn, +} from "./schema"; /** * Result of printing an AST to ClickHouse SQL @@ -316,7 +323,7 @@ export class ClickHousePrinter { // Process SELECT columns let columns: string[]; if (node.select && node.select.length > 0) { - columns = node.select.map((col) => this.visit(col)); + columns = node.select.map((col) => this.visitSelectColumn(col)); } else { columns = ["1"]; } @@ -430,6 +437,73 @@ export class ClickHousePrinter { return response; } + /** + * Visit a SELECT column expression, handling virtual columns specially + * + * For bare Field expressions that reference virtual columns, we need to add + * an AS alias to preserve the column name in the result set. + * + * Examples: + * - `SELECT execution_duration` → `SELECT (expr) AS execution_duration` + * - `SELECT execution_duration AS dur` → `SELECT (expr) AS dur` (Alias handles it) + * - `SELECT run_id` → `SELECT run_id` (not a virtual column) + */ + private visitSelectColumn(col: Expression): string { + // Check if this is a bare Field (not wrapped in Alias) + if ((col as Field).expression_type === "field") { + const field = col as Field; + const virtualColumnName = this.getVirtualColumnNameForField(field.chain); + + if (virtualColumnName !== null) { + // Visit the field (which will return the expression) + const visited = this.visit(col); + // Add the alias to preserve the column name + return `${visited} AS ${this.printIdentifier(virtualColumnName)}`; + } + } + + // For non-virtual columns or expressions already wrapped in Alias, visit normally + return this.visit(col); + } + + /** + * Get the virtual column name if a field chain references a virtual column + * @returns The column name (as exposed in TSQL), or null if not a virtual column + */ + private getVirtualColumnNameForField(chain: Array): string | null { + if (chain.length === 0) return null; + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return null; + + // Case 1: Qualified reference like table.column + if (chain.length >= 2) { + const tableAlias = firstPart; + const tableSchema = this.tableContexts.get(tableAlias); + if (!tableSchema) return null; + + const columnName = chain[1]; + if (typeof columnName !== "string") return null; + + const columnSchema = tableSchema.columns[columnName]; + if (columnSchema && isVirtualColumn(columnSchema)) { + return columnName; + } + return null; + } + + // Case 2: Unqualified reference like just "column" + const columnName = firstPart; + for (const tableSchema of this.tableContexts.values()) { + const columnSchema = tableSchema.columns[columnName]; + if (columnSchema && isVirtualColumn(columnSchema)) { + return columnName; + } + } + + return null; + } + // ============================================================ // JOIN Expression Visitor // ============================================================ @@ -950,6 +1024,13 @@ export class ClickHousePrinter { } } + // Check if this field is a virtual column + const virtualExpression = this.getVirtualColumnExpressionForField(node.chain); + if (virtualExpression !== null) { + // Return the expression wrapped in parentheses + return `(${virtualExpression})`; + } + // Try to resolve column names through table context const resolvedChain = this.resolveFieldChain(node.chain); @@ -957,6 +1038,44 @@ export class ClickHousePrinter { return resolvedChain.map((part) => this.printIdentifierOrIndex(part)).join("."); } + /** + * Check if a field chain references a virtual column and return its expression + * @returns The virtual column expression, or null if not a virtual column + */ + private getVirtualColumnExpressionForField(chain: Array): string | null { + if (chain.length === 0) return null; + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return null; + + // Case 1: Qualified reference like table.column + if (chain.length >= 2) { + const tableAlias = firstPart; + const tableSchema = this.tableContexts.get(tableAlias); + if (!tableSchema) return null; + + const columnName = chain[1]; + if (typeof columnName !== "string") return null; + + const columnSchema = tableSchema.columns[columnName]; + if (columnSchema && isVirtualColumn(columnSchema)) { + return columnSchema.expression!; + } + return null; + } + + // Case 2: Unqualified reference like just "column" + const columnName = firstPart; + for (const tableSchema of this.tableContexts.values()) { + const columnSchema = tableSchema.columns[columnName]; + if (columnSchema && isVirtualColumn(columnSchema)) { + return columnSchema.expression!; + } + } + + return null; + } + /** * Resolve field chain to use ClickHouse column names where applicable * Handles both qualified (table.column) and unqualified (column) references diff --git a/internal-packages/tsql/src/query/schema.test.ts b/internal-packages/tsql/src/query/schema.test.ts index 9dac7fc9b..485e24227 100644 --- a/internal-packages/tsql/src/query/schema.test.ts +++ b/internal-packages/tsql/src/query/schema.test.ts @@ -5,6 +5,8 @@ import { getInternalValue, getAllowedUserValues, isValidUserValue, + isVirtualColumn, + getVirtualColumnExpression, type ColumnSchema, } from "./schema.js"; @@ -166,3 +168,94 @@ describe("Value mapping helper functions", () => { }); }); +describe("Virtual column helper functions", () => { + const virtualColumn: ColumnSchema = { + name: "execution_duration", + ...column("Nullable(Int64)"), + expression: "dateDiff('millisecond', started_at, completed_at)", + description: "Time between started_at and completed_at in milliseconds", + }; + + const regularColumn: ColumnSchema = { + name: "status", + ...column("String"), + }; + + const columnWithEmptyExpression: ColumnSchema = { + name: "bad_column", + ...column("String"), + expression: "", + }; + + describe("isVirtualColumn", () => { + it("should return true for columns with expression defined", () => { + expect(isVirtualColumn(virtualColumn)).toBe(true); + }); + + it("should return false for regular columns without expression", () => { + expect(isVirtualColumn(regularColumn)).toBe(false); + }); + + it("should return false for columns with empty expression", () => { + expect(isVirtualColumn(columnWithEmptyExpression)).toBe(false); + }); + + it("should return false for columns with undefined expression", () => { + const col: ColumnSchema = { + name: "test", + ...column("String"), + expression: undefined, + }; + expect(isVirtualColumn(col)).toBe(false); + }); + }); + + describe("getVirtualColumnExpression", () => { + it("should return the expression for virtual columns", () => { + expect(getVirtualColumnExpression(virtualColumn)).toBe( + "dateDiff('millisecond', started_at, completed_at)" + ); + }); + + it("should return undefined for regular columns", () => { + expect(getVirtualColumnExpression(regularColumn)).toBeUndefined(); + }); + + it("should return undefined for columns with empty expression", () => { + expect(getVirtualColumnExpression(columnWithEmptyExpression)).toBeUndefined(); + }); + }); + + describe("virtual column schema definition", () => { + it("should allow defining virtual columns with all standard column options", () => { + const virtualWithOptions: ColumnSchema = { + name: "computed_value", + type: "Float64", + expression: "usage_duration_ms / 1000.0", + selectable: true, + filterable: true, + sortable: true, + groupable: false, // Might not want to group by computed values + description: "Usage duration in seconds", + }; + + expect(isVirtualColumn(virtualWithOptions)).toBe(true); + expect(virtualWithOptions.groupable).toBe(false); + expect(virtualWithOptions.selectable).toBe(true); + }); + + it("should support complex expressions with ClickHouse functions", () => { + const complexVirtual: ColumnSchema = { + name: "is_long_running", + ...column("UInt8"), + expression: + "if(completed_at IS NOT NULL AND started_at IS NOT NULL, dateDiff('second', started_at, completed_at) > 60, 0)", + }; + + expect(isVirtualColumn(complexVirtual)).toBe(true); + expect(getVirtualColumnExpression(complexVirtual)).toContain("dateDiff"); + expect(getVirtualColumnExpression(complexVirtual)).toContain("if("); + }); + }); +}); + diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts index 5a064211c..3e1b4c9f8 100644 --- a/internal-packages/tsql/src/query/schema.ts +++ b/internal-packages/tsql/src/query/schema.ts @@ -77,6 +77,24 @@ export interface ColumnSchema { * and results will display user-friendly names instead of internal values. */ valueMap?: Record; + /** + * For virtual (computed) columns: the raw ClickHouse SQL expression. + * Use actual ClickHouse column names in the expression. + * + * When set, this column becomes a virtual column that doesn't exist in the + * underlying table but is computed from the expression at query time. + * + * @example + * ```typescript + * { + * name: "execution_duration", + * type: "Nullable(Int64)", + * expression: "dateDiff('millisecond', started_at, completed_at)", + * description: "Time between started_at and completed_at in milliseconds" + * } + * ``` + */ + expression?: string; } /** @@ -283,6 +301,29 @@ export function getClickHouseColumnName(col: ColumnSchema): string { return col.clickhouseName ?? col.name; } +/** + * Check if a column is a virtual (computed) column + * + * Virtual columns have an expression property that defines how they are computed + * from other columns. They don't exist in the underlying table. + * + * @param col - The column schema to check + * @returns true if the column is virtual, false otherwise + */ +export function isVirtualColumn(col: ColumnSchema): boolean { + return col.expression !== undefined && col.expression.length > 0; +} + +/** + * Get the expression for a virtual column + * + * @param col - The column schema + * @returns The expression string, or undefined if not a virtual column + */ +export function getVirtualColumnExpression(col: ColumnSchema): string | undefined { + return isVirtualColumn(col) ? col.expression : undefined; +} + /** * Get the user-friendly display value for an internal value (case-insensitive) * Used for transforming query results back to user-friendly format