From 957defe32d8d4f41317bebcbd0dc87d3555e777b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 16 Dec 2025 23:28:41 +0000 Subject: [PATCH] Lints now based on possible tables, columns and values --- .../components/code/tsql/tsqlCompletion.ts | 127 ++++- .../app/components/code/tsql/tsqlLinter.ts | 30 +- .../routes/storybook.tsql-editor/route.tsx | 77 ++- internal-packages/tsql/src/index.ts | 8 + internal-packages/tsql/src/query/schema.ts | 2 + internal-packages/tsql/src/query/validator.ts | 465 ++++++++++++++++++ 6 files changed, 691 insertions(+), 18 deletions(-) create mode 100644 internal-packages/tsql/src/query/validator.ts diff --git a/apps/webapp/app/components/code/tsql/tsqlCompletion.ts b/apps/webapp/app/components/code/tsql/tsqlCompletion.ts index caf0b4260..8c90546cb 100644 --- a/apps/webapp/app/components/code/tsql/tsqlCompletion.ts +++ b/apps/webapp/app/components/code/tsql/tsqlCompletion.ts @@ -1,5 +1,5 @@ import type { CompletionContext, CompletionResult, Completion } from "@codemirror/autocomplete"; -import type { TableSchema } from "@internal/tsql"; +import type { TableSchema, ColumnSchema } from "@internal/tsql"; import { TSQL_CLICKHOUSE_FUNCTIONS, TSQL_AGGREGATIONS, @@ -185,15 +185,74 @@ type CompletionContextType = | "table" // After FROM or JOIN | "column" // After SELECT, WHERE, ORDER BY, GROUP BY, etc. | "alias" // After table_name. + | "value" // After comparison operator (=, !=, IN, etc.) | "general"; // Anywhere else +/** + * Result of context detection + */ +interface ContextResult { + type: CompletionContextType; + tablePrefix?: string; + /** Column being compared (for value context) */ + columnName?: string; + /** Table alias for the column (for value context) */ + columnTableAlias?: string; +} + +/** + * Extract column name from text before a comparison operator + * Handles: "column =", "table.column =", "column IN", etc. + */ +function extractColumnBeforeOperator(textBefore: string): { columnName: string; tableAlias?: string } | null { + // Match patterns like: column =, column !=, column IN, table.column =, etc. + // We need to capture the column (and optional table prefix) before the operator + const patterns = [ + // column = or column != or column <> (with optional whitespace) + /(\w+)\.(\w+)\s*(?:=|!=|<>)\s*$/i, + /(\w+)\s*(?:=|!=|<>)\s*$/i, + // column IN ( or column NOT IN ( + /(\w+)\.(\w+)\s+(?:NOT\s+)?IN\s*\(\s*$/i, + /(\w+)\s+(?:NOT\s+)?IN\s*\(\s*$/i, + // After a comma in IN clause - need to find the column before IN + /(\w+)\.(\w+)\s+(?:NOT\s+)?IN\s*\([^)]*,\s*$/i, + /(\w+)\s+(?:NOT\s+)?IN\s*\([^)]*,\s*$/i, + ]; + + for (const pattern of patterns) { + const match = textBefore.match(pattern); + if (match) { + if (match.length === 3) { + // table.column pattern + return { tableAlias: match[1], columnName: match[2] }; + } else { + // just column pattern + return { columnName: match[1] }; + } + } + } + + return null; +} + function determineContext( doc: string, pos: number -): { type: CompletionContextType; tablePrefix?: string } { +): ContextResult { // Get text before cursor const textBefore = doc.slice(0, pos); + // Check if we're in a value context (after comparison operator) + // This should be checked before other contexts + const columnInfo = extractColumnBeforeOperator(textBefore); + if (columnInfo) { + return { + type: "value", + columnName: columnInfo.columnName, + columnTableAlias: columnInfo.tableAlias, + }; + } + // Check if we're completing after a dot (table.column) const dotMatch = textBefore.match(/(\w+)\.\s*$/); if (dotMatch) { @@ -234,6 +293,48 @@ function determineContext( return { type: "general" }; } +/** + * Find a column schema by name in the tables map + */ +function findColumnSchema( + columnName: string, + tableAlias: string | undefined, + tables: Map +): ColumnSchema | null { + if (tableAlias) { + // Look in specific table + const tableSchema = tables.get(tableAlias.toLowerCase()); + if (tableSchema) { + return tableSchema.columns[columnName] || null; + } + } else { + // Look in all tables + for (const tableSchema of tables.values()) { + const col = tableSchema.columns[columnName]; + if (col) { + return col; + } + } + } + return null; +} + +/** + * Create completions for enum values + */ +function createEnumValueCompletions(columnSchema: ColumnSchema): Completion[] { + if (!columnSchema.allowedValues || columnSchema.allowedValues.length === 0) { + return []; + } + + return columnSchema.allowedValues.map((value) => ({ + label: `'${value}'`, + type: "enum", + detail: columnSchema.description || "allowed value", + boost: 3, // Highest priority for enum values in value context + })); +} + /** * Create a TSQL-aware autocompletion source * @@ -249,8 +350,8 @@ export function createTSQLCompletion( const tableCompletions = createTableCompletions(schema); return (context: CompletionContext): CompletionResult | null => { - // Get the word being typed - const word = context.matchBefore(/[\w.]+/); + // Get the word being typed - include single quotes for value completion + const word = context.matchBefore(/[\w.']+/); // Don't show completions if no word is being typed and not explicitly triggered if (!word && !context.explicit) { @@ -281,6 +382,22 @@ export function createTSQLCompletion( } break; + case "value": + // After comparison operator, show enum values if available + if (queryContext.columnName) { + const tables = extractTablesFromQuery(doc, schema); + const columnSchema = findColumnSchema( + queryContext.columnName, + queryContext.columnTableAlias, + tables + ); + + if (columnSchema) { + options = createEnumValueCompletions(columnSchema); + } + } + break; + case "column": // After SELECT, WHERE, etc., show columns, functions, and some keywords { @@ -328,7 +445,7 @@ export function createTSQLCompletion( return { from, options, - validFor: /^[\w.]*$/, + validFor: /^[\w.']*$/, }; }; } diff --git a/apps/webapp/app/components/code/tsql/tsqlLinter.ts b/apps/webapp/app/components/code/tsql/tsqlLinter.ts index 2d0bc1561..72794e3e9 100644 --- a/apps/webapp/app/components/code/tsql/tsqlLinter.ts +++ b/apps/webapp/app/components/code/tsql/tsqlLinter.ts @@ -1,7 +1,7 @@ import type { EditorView } from "@codemirror/view"; import type { Diagnostic } from "@codemirror/lint"; import type { TableSchema } from "@internal/tsql"; -import { parseTSQLSelect, SyntaxError, QueryError } from "@internal/tsql"; +import { parseTSQLSelect, SyntaxError, QueryError, validateQuery } from "@internal/tsql"; /** * Configuration for the TSQL linter @@ -78,6 +78,8 @@ function findTokenEnd(doc: string, start: number): number { export function createTSQLLinter( config: TSQLLinterConfig = {} ): (view: EditorView) => Diagnostic[] { + const { schema = [] } = config; + return (view: EditorView): Diagnostic[] => { const content = view.state.doc.toString().trim(); @@ -90,10 +92,30 @@ export function createTSQLLinter( try { // Try to parse the query - parseTSQLSelect(content); + const ast = parseTSQLSelect(content); - // If parsing succeeds, we could do additional schema validation here - // For now, we just validate syntax + // If parsing succeeds and we have a schema, run schema validation + if (schema.length > 0) { + const validationResult = validateQuery(ast, schema); + + for (const issue of validationResult.issues) { + // Map validation severity to CodeMirror diagnostic severity + const severity: "error" | "warning" | "info" = + issue.severity === "error" + ? "error" + : issue.severity === "warning" + ? "warning" + : "info"; + + diagnostics.push({ + from: 0, + to: content.length, + severity, + message: issue.message, + source: "tsql", + }); + } + } } catch (error) { if (error instanceof SyntaxError) { const position = parseErrorPosition(error.message); diff --git a/apps/webapp/app/routes/storybook.tsql-editor/route.tsx b/apps/webapp/app/routes/storybook.tsql-editor/route.tsx index 6dacb7907..8c89e45e5 100644 --- a/apps/webapp/app/routes/storybook.tsql-editor/route.tsx +++ b/apps/webapp/app/routes/storybook.tsql-editor/route.tsx @@ -2,6 +2,9 @@ import { useState } from "react"; import { TSQLEditor } from "~/components/code/TSQLEditor"; import { column, type TableSchema } from "@internal/tsql"; +const RUN_STATUSES = ["PENDING", "QUEUED", "EXECUTING", "COMPLETED", "FAILED", "CANCELED"] as const; +const LOG_LEVELS = ["DEBUG", "INFO", "WARN", "ERROR"] as const; + const runsSchema: TableSchema = { name: "runs", clickhouseName: "trigger_dev.task_runs_v2", @@ -16,7 +19,10 @@ const runsSchema: TableSchema = { task_id: { name: "task_id", ...column("String", { description: "Task identifier" }) }, status: { name: "status", - ...column("String", { description: "Run status (PENDING, EXECUTING, COMPLETED, FAILED)" }), + ...column("String", { + description: "Run status", + allowedValues: [...RUN_STATUSES], + }), }, created_at: { name: "created_at", @@ -52,7 +58,13 @@ const logsSchema: TableSchema = { columns: { id: { name: "id", ...column("String", { description: "Event identifier" }) }, run_id: { name: "run_id", ...column("String", { description: "Associated run ID" }) }, - level: { name: "level", ...column("String", { description: "Log level (INFO, WARN, ERROR)" }) }, + level: { + name: "level", + ...column("String", { + description: "Log level", + allowedValues: [...LOG_LEVELS], + }), + }, message: { name: "message", ...column("String", { description: "Log message content" }) }, timestamp: { name: "timestamp", ...column("DateTime64", { description: "Event timestamp" }) }, organization_id: { name: "organization_id", ...column("String") }, @@ -72,6 +84,10 @@ const exampleQueries = [ name: "With WHERE clause", query: "SELECT id, task_id, status, created_at FROM runs WHERE status = 'COMPLETED' LIMIT 100", }, + { + name: "Enum IN clause", + query: "SELECT * FROM runs WHERE status IN ('PENDING', 'QUEUED', 'EXECUTING') LIMIT 50", + }, { name: "Aggregation", query: "SELECT status, count(*) as count FROM runs GROUP BY status ORDER BY count DESC", @@ -134,8 +150,8 @@ export default function Story() {

Editor with Schema

- Try typing to see autocomplete suggestions. Available tables: runs,{" "} - logs + Try typing to see autocomplete suggestions. Type status = to see enum value + suggestions. Available tables: runs, logs

+ {/* Invalid enum value example */} +
+

With Invalid Enum Value

+

+ The linter validates enum values against the schema. Try changing{" "} + 'INVALID_STATUS' to a valid status like 'COMPLETED'. +

+
+ +
+
+ + {/* Unknown column example */} +
+

With Unknown Column

+

+ The linter warns about unknown column names. Try changing unknown_col to a + valid column like status. +

+
+ +
+
+ {/* Available tables reference */}

Available Schema

@@ -214,11 +266,18 @@ export default function Story() {

{table.description}

{Object.entries(table.columns).map(([name, col]) => ( -
- {name} - {col.type} - {col.description && ( - - {col.description} +
+
+ {name} + {col.type} + {col.description && ( + - {col.description} + )} +
+ {col.allowedValues && col.allowedValues.length > 0 && ( +
+ Allowed: {col.allowedValues.join(", ")} +
)}
))} diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index effbaa024..e7b8d05bd 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -91,6 +91,14 @@ export { ClickHousePrinter, printToClickHouse, type PrintResult } from "./query/ // Re-export parser converter for advanced usage export { TSQLParseTreeConverter } from "./query/parser.js"; +// Re-export validator +export { + validateQuery, + type ValidationResult, + type ValidationIssue, + type ValidationSeverity, +} from "./query/validator.js"; + /** * Parse a TSQL SELECT query string into an AST * diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts index 84c4ad798..01485a9be 100644 --- a/internal-packages/tsql/src/query/schema.ts +++ b/internal-packages/tsql/src/query/schema.ts @@ -66,6 +66,8 @@ export interface ColumnSchema { groupable?: boolean; /** Description of the column for documentation/autocomplete */ description?: string; + /** Allowed values for this column (for enum-like columns) */ + allowedValues?: string[]; } /** diff --git a/internal-packages/tsql/src/query/validator.ts b/internal-packages/tsql/src/query/validator.ts new file mode 100644 index 000000000..9f9959485 --- /dev/null +++ b/internal-packages/tsql/src/query/validator.ts @@ -0,0 +1,465 @@ +// Schema validation for TSQL queries +// Validates column names and enum values against the schema + +import type { + SelectQuery, + SelectSetQuery, + Expression, + Field, + CompareOperation, + Constant, + And, + Or, + Not, + Alias, + OrderExpr, + Call, + JoinExpr, + BetweenExpr, + Array as ASTArray, +} from "./ast.js"; +import type { TableSchema, ColumnSchema } from "./schema.js"; +import { CompareOperationOp } from "./ast.js"; + +/** + * Severity of a validation issue + */ +export type ValidationSeverity = "error" | "warning" | "info"; + +/** + * A validation issue found in the query + */ +export interface ValidationIssue { + /** The error/warning message */ + message: string; + /** Severity of the issue */ + severity: ValidationSeverity; + /** The type of issue */ + type: "unknown_column" | "unknown_table" | "invalid_enum_value"; + /** Optional: the column name that caused the issue */ + columnName?: string; + /** Optional: the table name that caused the issue */ + tableName?: string; + /** Optional: the invalid value */ + invalidValue?: string; + /** Optional: list of allowed values */ + allowedValues?: string[]; +} + +/** + * Result of validating a query + */ +export interface ValidationResult { + /** Whether the query is valid */ + valid: boolean; + /** List of issues found */ + issues: ValidationIssue[]; +} + +/** + * Context for tracking tables and columns during validation + */ +interface ValidationContext { + /** Map of table aliases/names to their schemas */ + tables: Map; + /** The schema array for lookups */ + schema: TableSchema[]; + /** Accumulated issues */ + issues: ValidationIssue[]; +} + +/** + * Validate a parsed TSQL query against a schema + * + * @param ast - The parsed query AST + * @param schema - Array of table schemas to validate against + * @returns Validation result with any issues found + */ +export function validateQuery( + ast: SelectQuery | SelectSetQuery, + schema: TableSchema[] +): ValidationResult { + const context: ValidationContext = { + tables: new Map(), + schema, + issues: [], + }; + + if (ast.expression_type === "select_set_query") { + validateSelectSetQuery(ast, context); + } else { + validateSelectQuery(ast, context); + } + + return { + valid: context.issues.filter((i) => i.severity === "error").length === 0, + issues: context.issues, + }; +} + +/** + * Validate a SELECT SET query (UNION, INTERSECT, etc.) + */ +function validateSelectSetQuery(node: SelectSetQuery, context: ValidationContext): void { + if (node.initial_select_query.expression_type === "select_set_query") { + validateSelectSetQuery(node.initial_select_query, context); + } else { + validateSelectQuery(node.initial_select_query, context); + } + + for (const subsequent of node.subsequent_select_queries) { + if (subsequent.select_query.expression_type === "select_set_query") { + validateSelectSetQuery(subsequent.select_query as SelectSetQuery, context); + } else { + validateSelectQuery(subsequent.select_query as SelectQuery, context); + } + } +} + +/** + * Validate a SELECT query + */ +function validateSelectQuery(node: SelectQuery, context: ValidationContext): void { + // First, extract tables from FROM clause to build context + if (node.select_from) { + extractTablesFromJoin(node.select_from, context); + } + + // Validate SELECT columns + if (node.select) { + for (const expr of node.select) { + validateExpression(expr, context); + } + } + + // Validate WHERE clause + if (node.where) { + validateExpression(node.where, context); + } + + // Validate GROUP BY + if (node.group_by) { + for (const expr of node.group_by) { + validateExpression(expr, context); + } + } + + // Validate HAVING + if (node.having) { + validateExpression(node.having, context); + } + + // Validate ORDER BY + if (node.order_by) { + for (const expr of node.order_by) { + validateExpression(expr, context); + } + } +} + +/** + * Extract table schemas from JOIN expressions + */ +function extractTablesFromJoin(node: JoinExpr, context: ValidationContext): void { + if (node.table) { + const tableExpr = node.table; + + if ((tableExpr as Field).expression_type === "field") { + const field = tableExpr as Field; + const tableName = field.chain[0]; + + if (typeof tableName === "string") { + // Find the table schema + const tableSchema = context.schema.find( + (t) => t.name.toLowerCase() === tableName.toLowerCase() + ); + + if (tableSchema) { + // Register with alias if provided, otherwise use table name + const key = node.alias || tableName; + context.tables.set(key.toLowerCase(), tableSchema); + } else { + // Unknown table + context.issues.push({ + message: `Unknown table "${tableName}". Available tables: ${ + context.schema.map((t) => t.name).join(", ") || "(none)" + }`, + severity: "warning", + type: "unknown_table", + tableName, + }); + } + } + } else if ( + (tableExpr as SelectQuery).expression_type === "select_query" || + (tableExpr as SelectSetQuery).expression_type === "select_set_query" + ) { + // Subquery - validate it recursively + if ((tableExpr as SelectSetQuery).expression_type === "select_set_query") { + validateSelectSetQuery(tableExpr as SelectSetQuery, context); + } else { + validateSelectQuery(tableExpr as SelectQuery, context); + } + } + } + + // Process next join in chain + if (node.next_join) { + extractTablesFromJoin(node.next_join, context); + } +} + +/** + * Validate an expression and its children + */ +function validateExpression(expr: Expression, context: ValidationContext): void { + if (!expr || typeof expr !== "object") return; + + const exprType = expr.expression_type; + + switch (exprType) { + case "field": + validateField(expr as Field, context); + break; + + case "compare_operation": + validateCompareOperation(expr as CompareOperation, context); + break; + + case "and": + for (const e of (expr as And).exprs) { + validateExpression(e, context); + } + break; + + case "or": + for (const e of (expr as Or).exprs) { + validateExpression(e, context); + } + break; + + case "not": + validateExpression((expr as Not).expr, context); + break; + + case "alias": + validateExpression((expr as Alias).expr, context); + break; + + case "order_expr": + validateExpression((expr as OrderExpr).expr, context); + break; + + case "call": + for (const arg of (expr as Call).args) { + validateExpression(arg, context); + } + break; + + case "between_expr": + validateExpression((expr as BetweenExpr).expr, context); + validateExpression((expr as BetweenExpr).low, context); + validateExpression((expr as BetweenExpr).high, context); + break; + + case "array": + for (const e of (expr as ASTArray).exprs) { + validateExpression(e, context); + } + break; + + // Other expression types that we don't need to deeply validate + case "constant": + case "select_query": + case "select_set_query": + // Skip - constants don't need validation, subqueries are handled separately + break; + } +} + +/** + * Validate a field reference + */ +function validateField(field: Field, context: ValidationContext): void { + const chain = field.chain; + if (chain.length === 0) return; + + // Handle asterisk + if (chain[0] === "*") return; + if (chain.length === 2 && chain[1] === "*") return; + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return; + + // Case 1: Qualified reference like table.column + if (chain.length >= 2) { + const tableAlias = firstPart.toLowerCase(); + const columnName = chain[1]; + + if (typeof columnName !== "string") return; + + const tableSchema = context.tables.get(tableAlias); + if (tableSchema) { + // Check if column exists + if (!tableSchema.columns[columnName]) { + const availableColumns = Object.keys(tableSchema.columns).join(", "); + context.issues.push({ + message: `Unknown column "${columnName}" on table "${tableAlias}". Available columns: ${availableColumns}`, + severity: "warning", + type: "unknown_column", + columnName, + tableName: tableAlias, + }); + } + } + return; + } + + // Case 2: Unqualified reference - try to find in any table + const columnName = firstPart; + let found = false; + + for (const tableSchema of context.tables.values()) { + if (tableSchema.columns[columnName]) { + found = true; + break; + } + } + + if (!found && context.tables.size > 0) { + // Only report if we have tables to check against + const allColumns = new Set(); + for (const tableSchema of context.tables.values()) { + for (const col of Object.keys(tableSchema.columns)) { + allColumns.add(col); + } + } + context.issues.push({ + message: `Unknown column "${columnName}". Available columns: ${Array.from(allColumns).join( + ", " + )}`, + severity: "warning", + type: "unknown_column", + columnName, + }); + } +} + +/** + * Validate a comparison operation, including enum value checks + */ +function validateCompareOperation(op: CompareOperation, context: ValidationContext): void { + // Validate both sides recursively + validateExpression(op.left, context); + validateExpression(op.right, context); + + // Check for enum value validation + // We look for patterns like: column = 'value' or column IN ('value1', 'value2') + const columnInfo = extractColumnFromExpression(op.left, context); + if (!columnInfo) return; + + const { columnSchema, columnName, tableName } = columnInfo; + + // Only validate if the column has allowedValues + if (!columnSchema.allowedValues || columnSchema.allowedValues.length === 0) return; + + // Check the comparison type + switch (op.op) { + case CompareOperationOp.Eq: + case CompareOperationOp.NotEq: + // Single value comparison + validateEnumValue(op.right, columnSchema, columnName, tableName, context); + break; + + case CompareOperationOp.In: + case CompareOperationOp.NotIn: + case CompareOperationOp.GlobalIn: + case CompareOperationOp.GlobalNotIn: + // Array of values + if ((op.right as ASTArray).expression_type === "array") { + for (const elem of (op.right as ASTArray).exprs) { + validateEnumValue(elem, columnSchema, columnName, tableName, context); + } + } + break; + } +} + +/** + * Extract column information from an expression if it's a simple column reference + */ +function extractColumnFromExpression( + expr: Expression, + context: ValidationContext +): { columnSchema: ColumnSchema; columnName: string; tableName?: string } | null { + if ((expr as Field).expression_type !== "field") return null; + + const field = expr as Field; + const chain = field.chain; + + if (chain.length === 0) return null; + + const firstPart = chain[0]; + if (typeof firstPart !== "string") return null; + + // Qualified reference: table.column + if (chain.length >= 2) { + const tableAlias = firstPart.toLowerCase(); + const columnName = chain[1]; + + if (typeof columnName !== "string") return null; + + const tableSchema = context.tables.get(tableAlias); + if (!tableSchema) return null; + + const columnSchema = tableSchema.columns[columnName]; + if (!columnSchema) return null; + + return { columnSchema, columnName, tableName: tableAlias }; + } + + // Unqualified reference + const columnName = firstPart; + for (const [tableName, tableSchema] of context.tables.entries()) { + const columnSchema = tableSchema.columns[columnName]; + if (columnSchema) { + return { columnSchema, columnName, tableName }; + } + } + + return null; +} + +/** + * Validate that a value matches the allowed enum values for a column + */ +function validateEnumValue( + expr: Expression, + columnSchema: ColumnSchema, + columnName: string, + tableName: string | undefined, + context: ValidationContext +): void { + if ((expr as Constant).expression_type !== "constant") return; + + const constant = expr as Constant; + if (typeof constant.value !== "string") return; + + const value = constant.value; + const allowedValues = columnSchema.allowedValues!; + + if (!allowedValues.includes(value)) { + const columnRef = tableName ? `${tableName}.${columnName}` : columnName; + context.issues.push({ + message: `Invalid value "${value}" for column "${columnRef}". Allowed values: ${allowedValues.join( + ", " + )}`, + severity: "error", + type: "invalid_enum_value", + columnName, + tableName, + invalidValue: value, + allowedValues, + }); + } +}