Lints now based on possible tables, columns and values

This commit is contained in:
Matt Aitken
2025-12-16 23:28:41 +00:00
parent 3594727488
commit 957defe32d
6 changed files with 691 additions and 18 deletions
@@ -1,5 +1,5 @@
import type { CompletionContext, CompletionResult, Completion } from "@codemirror/autocomplete";
import type { TableSchema } from "@internal/tsql";
import type { TableSchema, ColumnSchema } from "@internal/tsql";
import {
TSQL_CLICKHOUSE_FUNCTIONS,
TSQL_AGGREGATIONS,
@@ -185,15 +185,74 @@ type CompletionContextType =
| "table" // After FROM or JOIN
| "column" // After SELECT, WHERE, ORDER BY, GROUP BY, etc.
| "alias" // After table_name.
| "value" // After comparison operator (=, !=, IN, etc.)
| "general"; // Anywhere else
/**
* Result of context detection
*/
interface ContextResult {
type: CompletionContextType;
tablePrefix?: string;
/** Column being compared (for value context) */
columnName?: string;
/** Table alias for the column (for value context) */
columnTableAlias?: string;
}
/**
* Extract column name from text before a comparison operator
* Handles: "column =", "table.column =", "column IN", etc.
*/
function extractColumnBeforeOperator(textBefore: string): { columnName: string; tableAlias?: string } | null {
// Match patterns like: column =, column !=, column IN, table.column =, etc.
// We need to capture the column (and optional table prefix) before the operator
const patterns = [
// column = or column != or column <> (with optional whitespace)
/(\w+)\.(\w+)\s*(?:=|!=|<>)\s*$/i,
/(\w+)\s*(?:=|!=|<>)\s*$/i,
// column IN ( or column NOT IN (
/(\w+)\.(\w+)\s+(?:NOT\s+)?IN\s*\(\s*$/i,
/(\w+)\s+(?:NOT\s+)?IN\s*\(\s*$/i,
// After a comma in IN clause - need to find the column before IN
/(\w+)\.(\w+)\s+(?:NOT\s+)?IN\s*\([^)]*,\s*$/i,
/(\w+)\s+(?:NOT\s+)?IN\s*\([^)]*,\s*$/i,
];
for (const pattern of patterns) {
const match = textBefore.match(pattern);
if (match) {
if (match.length === 3) {
// table.column pattern
return { tableAlias: match[1], columnName: match[2] };
} else {
// just column pattern
return { columnName: match[1] };
}
}
}
return null;
}
function determineContext(
doc: string,
pos: number
): { type: CompletionContextType; tablePrefix?: string } {
): ContextResult {
// Get text before cursor
const textBefore = doc.slice(0, pos);
// Check if we're in a value context (after comparison operator)
// This should be checked before other contexts
const columnInfo = extractColumnBeforeOperator(textBefore);
if (columnInfo) {
return {
type: "value",
columnName: columnInfo.columnName,
columnTableAlias: columnInfo.tableAlias,
};
}
// Check if we're completing after a dot (table.column)
const dotMatch = textBefore.match(/(\w+)\.\s*$/);
if (dotMatch) {
@@ -234,6 +293,48 @@ function determineContext(
return { type: "general" };
}
/**
* Find a column schema by name in the tables map
*/
function findColumnSchema(
columnName: string,
tableAlias: string | undefined,
tables: Map<string, TableSchema>
): ColumnSchema | null {
if (tableAlias) {
// Look in specific table
const tableSchema = tables.get(tableAlias.toLowerCase());
if (tableSchema) {
return tableSchema.columns[columnName] || null;
}
} else {
// Look in all tables
for (const tableSchema of tables.values()) {
const col = tableSchema.columns[columnName];
if (col) {
return col;
}
}
}
return null;
}
/**
* Create completions for enum values
*/
function createEnumValueCompletions(columnSchema: ColumnSchema): Completion[] {
if (!columnSchema.allowedValues || columnSchema.allowedValues.length === 0) {
return [];
}
return columnSchema.allowedValues.map((value) => ({
label: `'${value}'`,
type: "enum",
detail: columnSchema.description || "allowed value",
boost: 3, // Highest priority for enum values in value context
}));
}
/**
* Create a TSQL-aware autocompletion source
*
@@ -249,8 +350,8 @@ export function createTSQLCompletion(
const tableCompletions = createTableCompletions(schema);
return (context: CompletionContext): CompletionResult | null => {
// Get the word being typed
const word = context.matchBefore(/[\w.]+/);
// Get the word being typed - include single quotes for value completion
const word = context.matchBefore(/[\w.']+/);
// Don't show completions if no word is being typed and not explicitly triggered
if (!word && !context.explicit) {
@@ -281,6 +382,22 @@ export function createTSQLCompletion(
}
break;
case "value":
// After comparison operator, show enum values if available
if (queryContext.columnName) {
const tables = extractTablesFromQuery(doc, schema);
const columnSchema = findColumnSchema(
queryContext.columnName,
queryContext.columnTableAlias,
tables
);
if (columnSchema) {
options = createEnumValueCompletions(columnSchema);
}
}
break;
case "column":
// After SELECT, WHERE, etc., show columns, functions, and some keywords
{
@@ -328,7 +445,7 @@ export function createTSQLCompletion(
return {
from,
options,
validFor: /^[\w.]*$/,
validFor: /^[\w.']*$/,
};
};
}
@@ -1,7 +1,7 @@
import type { EditorView } from "@codemirror/view";
import type { Diagnostic } from "@codemirror/lint";
import type { TableSchema } from "@internal/tsql";
import { parseTSQLSelect, SyntaxError, QueryError } from "@internal/tsql";
import { parseTSQLSelect, SyntaxError, QueryError, validateQuery } from "@internal/tsql";
/**
* Configuration for the TSQL linter
@@ -78,6 +78,8 @@ function findTokenEnd(doc: string, start: number): number {
export function createTSQLLinter(
config: TSQLLinterConfig = {}
): (view: EditorView) => Diagnostic[] {
const { schema = [] } = config;
return (view: EditorView): Diagnostic[] => {
const content = view.state.doc.toString().trim();
@@ -90,10 +92,30 @@ export function createTSQLLinter(
try {
// Try to parse the query
parseTSQLSelect(content);
const ast = parseTSQLSelect(content);
// If parsing succeeds, we could do additional schema validation here
// For now, we just validate syntax
// If parsing succeeds and we have a schema, run schema validation
if (schema.length > 0) {
const validationResult = validateQuery(ast, schema);
for (const issue of validationResult.issues) {
// Map validation severity to CodeMirror diagnostic severity
const severity: "error" | "warning" | "info" =
issue.severity === "error"
? "error"
: issue.severity === "warning"
? "warning"
: "info";
diagnostics.push({
from: 0,
to: content.length,
severity,
message: issue.message,
source: "tsql",
});
}
}
} catch (error) {
if (error instanceof SyntaxError) {
const position = parseErrorPosition(error.message);
@@ -2,6 +2,9 @@ import { useState } from "react";
import { TSQLEditor } from "~/components/code/TSQLEditor";
import { column, type TableSchema } from "@internal/tsql";
const RUN_STATUSES = ["PENDING", "QUEUED", "EXECUTING", "COMPLETED", "FAILED", "CANCELED"] as const;
const LOG_LEVELS = ["DEBUG", "INFO", "WARN", "ERROR"] as const;
const runsSchema: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
@@ -16,7 +19,10 @@ const runsSchema: TableSchema = {
task_id: { name: "task_id", ...column("String", { description: "Task identifier" }) },
status: {
name: "status",
...column("String", { description: "Run status (PENDING, EXECUTING, COMPLETED, FAILED)" }),
...column("String", {
description: "Run status",
allowedValues: [...RUN_STATUSES],
}),
},
created_at: {
name: "created_at",
@@ -52,7 +58,13 @@ const logsSchema: TableSchema = {
columns: {
id: { name: "id", ...column("String", { description: "Event identifier" }) },
run_id: { name: "run_id", ...column("String", { description: "Associated run ID" }) },
level: { name: "level", ...column("String", { description: "Log level (INFO, WARN, ERROR)" }) },
level: {
name: "level",
...column("String", {
description: "Log level",
allowedValues: [...LOG_LEVELS],
}),
},
message: { name: "message", ...column("String", { description: "Log message content" }) },
timestamp: { name: "timestamp", ...column("DateTime64", { description: "Event timestamp" }) },
organization_id: { name: "organization_id", ...column("String") },
@@ -72,6 +84,10 @@ const exampleQueries = [
name: "With WHERE clause",
query: "SELECT id, task_id, status, created_at FROM runs WHERE status = 'COMPLETED' LIMIT 100",
},
{
name: "Enum IN clause",
query: "SELECT * FROM runs WHERE status IN ('PENDING', 'QUEUED', 'EXECUTING') LIMIT 50",
},
{
name: "Aggregation",
query: "SELECT status, count(*) as count FROM runs GROUP BY status ORDER BY count DESC",
@@ -134,8 +150,8 @@ export default function Story() {
<div className="flex flex-col gap-2">
<h2 className="text-lg font-semibold text-text-bright">Editor with Schema</h2>
<p className="text-sm text-text-dimmed">
Try typing to see autocomplete suggestions. Available tables: <code>runs</code>,{" "}
<code>logs</code>
Try typing to see autocomplete suggestions. Type <code>status = </code> to see enum value
suggestions. Available tables: <code>runs</code>, <code>logs</code>
</p>
<div className="overflow-hidden rounded-lg border border-grid-dimmed">
<TSQLEditor
@@ -199,6 +215,42 @@ export default function Story() {
</div>
</div>
{/* Invalid enum value example */}
<div className="flex flex-col gap-2">
<h2 className="text-lg font-semibold text-text-bright">With Invalid Enum Value</h2>
<p className="text-sm text-text-dimmed">
The linter validates enum values against the schema. Try changing{" "}
<code>'INVALID_STATUS'</code> to a valid status like <code>'COMPLETED'</code>.
</p>
<div className="overflow-hidden rounded-lg border border-grid-dimmed">
<TSQLEditor
defaultValue="SELECT * FROM runs WHERE status = 'INVALID_STATUS' LIMIT 10"
schema={exampleSchema}
linterEnabled={true}
showCopyButton={true}
className="min-h-[100px]"
/>
</div>
</div>
{/* Unknown column example */}
<div className="flex flex-col gap-2">
<h2 className="text-lg font-semibold text-text-bright">With Unknown Column</h2>
<p className="text-sm text-text-dimmed">
The linter warns about unknown column names. Try changing <code>unknown_col</code> to a
valid column like <code>status</code>.
</p>
<div className="overflow-hidden rounded-lg border border-grid-dimmed">
<TSQLEditor
defaultValue="SELECT id, unknown_col FROM runs LIMIT 10"
schema={exampleSchema}
linterEnabled={true}
showCopyButton={true}
className="min-h-[100px]"
/>
</div>
</div>
{/* Available tables reference */}
<div className="flex flex-col gap-2">
<h2 className="text-lg font-semibold text-text-bright">Available Schema</h2>
@@ -214,11 +266,18 @@ export default function Story() {
<p className="mb-3 text-xs text-text-dimmed">{table.description}</p>
<div className="space-y-1">
{Object.entries(table.columns).map(([name, col]) => (
<div key={name} className="flex items-baseline gap-2 text-xs">
<code className="text-blue-400">{name}</code>
<span className="text-charcoal-400">{col.type}</span>
{col.description && (
<span className="text-text-dimmed">- {col.description}</span>
<div key={name} className="flex flex-col gap-0.5 text-xs">
<div className="flex items-baseline gap-2">
<code className="text-blue-400">{name}</code>
<span className="text-charcoal-400">{col.type}</span>
{col.description && (
<span className="text-text-dimmed">- {col.description}</span>
)}
</div>
{col.allowedValues && col.allowedValues.length > 0 && (
<div className="ml-4 text-green-400/70">
Allowed: {col.allowedValues.join(", ")}
</div>
)}
</div>
))}
+8
View File
@@ -91,6 +91,14 @@ export { ClickHousePrinter, printToClickHouse, type PrintResult } from "./query/
// Re-export parser converter for advanced usage
export { TSQLParseTreeConverter } from "./query/parser.js";
// Re-export validator
export {
validateQuery,
type ValidationResult,
type ValidationIssue,
type ValidationSeverity,
} from "./query/validator.js";
/**
* Parse a TSQL SELECT query string into an AST
*
@@ -66,6 +66,8 @@ export interface ColumnSchema {
groupable?: boolean;
/** Description of the column for documentation/autocomplete */
description?: string;
/** Allowed values for this column (for enum-like columns) */
allowedValues?: string[];
}
/**
@@ -0,0 +1,465 @@
// Schema validation for TSQL queries
// Validates column names and enum values against the schema
import type {
SelectQuery,
SelectSetQuery,
Expression,
Field,
CompareOperation,
Constant,
And,
Or,
Not,
Alias,
OrderExpr,
Call,
JoinExpr,
BetweenExpr,
Array as ASTArray,
} from "./ast.js";
import type { TableSchema, ColumnSchema } from "./schema.js";
import { CompareOperationOp } from "./ast.js";
/**
* Severity of a validation issue
*/
export type ValidationSeverity = "error" | "warning" | "info";
/**
* A validation issue found in the query
*/
export interface ValidationIssue {
/** The error/warning message */
message: string;
/** Severity of the issue */
severity: ValidationSeverity;
/** The type of issue */
type: "unknown_column" | "unknown_table" | "invalid_enum_value";
/** Optional: the column name that caused the issue */
columnName?: string;
/** Optional: the table name that caused the issue */
tableName?: string;
/** Optional: the invalid value */
invalidValue?: string;
/** Optional: list of allowed values */
allowedValues?: string[];
}
/**
* Result of validating a query
*/
export interface ValidationResult {
/** Whether the query is valid */
valid: boolean;
/** List of issues found */
issues: ValidationIssue[];
}
/**
* Context for tracking tables and columns during validation
*/
interface ValidationContext {
/** Map of table aliases/names to their schemas */
tables: Map<string, TableSchema>;
/** The schema array for lookups */
schema: TableSchema[];
/** Accumulated issues */
issues: ValidationIssue[];
}
/**
* Validate a parsed TSQL query against a schema
*
* @param ast - The parsed query AST
* @param schema - Array of table schemas to validate against
* @returns Validation result with any issues found
*/
export function validateQuery(
ast: SelectQuery | SelectSetQuery,
schema: TableSchema[]
): ValidationResult {
const context: ValidationContext = {
tables: new Map(),
schema,
issues: [],
};
if (ast.expression_type === "select_set_query") {
validateSelectSetQuery(ast, context);
} else {
validateSelectQuery(ast, context);
}
return {
valid: context.issues.filter((i) => i.severity === "error").length === 0,
issues: context.issues,
};
}
/**
* Validate a SELECT SET query (UNION, INTERSECT, etc.)
*/
function validateSelectSetQuery(node: SelectSetQuery, context: ValidationContext): void {
if (node.initial_select_query.expression_type === "select_set_query") {
validateSelectSetQuery(node.initial_select_query, context);
} else {
validateSelectQuery(node.initial_select_query, context);
}
for (const subsequent of node.subsequent_select_queries) {
if (subsequent.select_query.expression_type === "select_set_query") {
validateSelectSetQuery(subsequent.select_query as SelectSetQuery, context);
} else {
validateSelectQuery(subsequent.select_query as SelectQuery, context);
}
}
}
/**
* Validate a SELECT query
*/
function validateSelectQuery(node: SelectQuery, context: ValidationContext): void {
// First, extract tables from FROM clause to build context
if (node.select_from) {
extractTablesFromJoin(node.select_from, context);
}
// Validate SELECT columns
if (node.select) {
for (const expr of node.select) {
validateExpression(expr, context);
}
}
// Validate WHERE clause
if (node.where) {
validateExpression(node.where, context);
}
// Validate GROUP BY
if (node.group_by) {
for (const expr of node.group_by) {
validateExpression(expr, context);
}
}
// Validate HAVING
if (node.having) {
validateExpression(node.having, context);
}
// Validate ORDER BY
if (node.order_by) {
for (const expr of node.order_by) {
validateExpression(expr, context);
}
}
}
/**
* Extract table schemas from JOIN expressions
*/
function extractTablesFromJoin(node: JoinExpr, context: ValidationContext): void {
if (node.table) {
const tableExpr = node.table;
if ((tableExpr as Field).expression_type === "field") {
const field = tableExpr as Field;
const tableName = field.chain[0];
if (typeof tableName === "string") {
// Find the table schema
const tableSchema = context.schema.find(
(t) => t.name.toLowerCase() === tableName.toLowerCase()
);
if (tableSchema) {
// Register with alias if provided, otherwise use table name
const key = node.alias || tableName;
context.tables.set(key.toLowerCase(), tableSchema);
} else {
// Unknown table
context.issues.push({
message: `Unknown table "${tableName}". Available tables: ${
context.schema.map((t) => t.name).join(", ") || "(none)"
}`,
severity: "warning",
type: "unknown_table",
tableName,
});
}
}
} else if (
(tableExpr as SelectQuery).expression_type === "select_query" ||
(tableExpr as SelectSetQuery).expression_type === "select_set_query"
) {
// Subquery - validate it recursively
if ((tableExpr as SelectSetQuery).expression_type === "select_set_query") {
validateSelectSetQuery(tableExpr as SelectSetQuery, context);
} else {
validateSelectQuery(tableExpr as SelectQuery, context);
}
}
}
// Process next join in chain
if (node.next_join) {
extractTablesFromJoin(node.next_join, context);
}
}
/**
* Validate an expression and its children
*/
function validateExpression(expr: Expression, context: ValidationContext): void {
if (!expr || typeof expr !== "object") return;
const exprType = expr.expression_type;
switch (exprType) {
case "field":
validateField(expr as Field, context);
break;
case "compare_operation":
validateCompareOperation(expr as CompareOperation, context);
break;
case "and":
for (const e of (expr as And).exprs) {
validateExpression(e, context);
}
break;
case "or":
for (const e of (expr as Or).exprs) {
validateExpression(e, context);
}
break;
case "not":
validateExpression((expr as Not).expr, context);
break;
case "alias":
validateExpression((expr as Alias).expr, context);
break;
case "order_expr":
validateExpression((expr as OrderExpr).expr, context);
break;
case "call":
for (const arg of (expr as Call).args) {
validateExpression(arg, context);
}
break;
case "between_expr":
validateExpression((expr as BetweenExpr).expr, context);
validateExpression((expr as BetweenExpr).low, context);
validateExpression((expr as BetweenExpr).high, context);
break;
case "array":
for (const e of (expr as ASTArray).exprs) {
validateExpression(e, context);
}
break;
// Other expression types that we don't need to deeply validate
case "constant":
case "select_query":
case "select_set_query":
// Skip - constants don't need validation, subqueries are handled separately
break;
}
}
/**
* Validate a field reference
*/
function validateField(field: Field, context: ValidationContext): void {
const chain = field.chain;
if (chain.length === 0) return;
// Handle asterisk
if (chain[0] === "*") return;
if (chain.length === 2 && chain[1] === "*") return;
const firstPart = chain[0];
if (typeof firstPart !== "string") return;
// Case 1: Qualified reference like table.column
if (chain.length >= 2) {
const tableAlias = firstPart.toLowerCase();
const columnName = chain[1];
if (typeof columnName !== "string") return;
const tableSchema = context.tables.get(tableAlias);
if (tableSchema) {
// Check if column exists
if (!tableSchema.columns[columnName]) {
const availableColumns = Object.keys(tableSchema.columns).join(", ");
context.issues.push({
message: `Unknown column "${columnName}" on table "${tableAlias}". Available columns: ${availableColumns}`,
severity: "warning",
type: "unknown_column",
columnName,
tableName: tableAlias,
});
}
}
return;
}
// Case 2: Unqualified reference - try to find in any table
const columnName = firstPart;
let found = false;
for (const tableSchema of context.tables.values()) {
if (tableSchema.columns[columnName]) {
found = true;
break;
}
}
if (!found && context.tables.size > 0) {
// Only report if we have tables to check against
const allColumns = new Set<string>();
for (const tableSchema of context.tables.values()) {
for (const col of Object.keys(tableSchema.columns)) {
allColumns.add(col);
}
}
context.issues.push({
message: `Unknown column "${columnName}". Available columns: ${Array.from(allColumns).join(
", "
)}`,
severity: "warning",
type: "unknown_column",
columnName,
});
}
}
/**
* Validate a comparison operation, including enum value checks
*/
function validateCompareOperation(op: CompareOperation, context: ValidationContext): void {
// Validate both sides recursively
validateExpression(op.left, context);
validateExpression(op.right, context);
// Check for enum value validation
// We look for patterns like: column = 'value' or column IN ('value1', 'value2')
const columnInfo = extractColumnFromExpression(op.left, context);
if (!columnInfo) return;
const { columnSchema, columnName, tableName } = columnInfo;
// Only validate if the column has allowedValues
if (!columnSchema.allowedValues || columnSchema.allowedValues.length === 0) return;
// Check the comparison type
switch (op.op) {
case CompareOperationOp.Eq:
case CompareOperationOp.NotEq:
// Single value comparison
validateEnumValue(op.right, columnSchema, columnName, tableName, context);
break;
case CompareOperationOp.In:
case CompareOperationOp.NotIn:
case CompareOperationOp.GlobalIn:
case CompareOperationOp.GlobalNotIn:
// Array of values
if ((op.right as ASTArray).expression_type === "array") {
for (const elem of (op.right as ASTArray).exprs) {
validateEnumValue(elem, columnSchema, columnName, tableName, context);
}
}
break;
}
}
/**
* Extract column information from an expression if it's a simple column reference
*/
function extractColumnFromExpression(
expr: Expression,
context: ValidationContext
): { columnSchema: ColumnSchema; columnName: string; tableName?: string } | null {
if ((expr as Field).expression_type !== "field") return null;
const field = expr as Field;
const chain = field.chain;
if (chain.length === 0) return null;
const firstPart = chain[0];
if (typeof firstPart !== "string") return null;
// Qualified reference: table.column
if (chain.length >= 2) {
const tableAlias = firstPart.toLowerCase();
const columnName = chain[1];
if (typeof columnName !== "string") return null;
const tableSchema = context.tables.get(tableAlias);
if (!tableSchema) return null;
const columnSchema = tableSchema.columns[columnName];
if (!columnSchema) return null;
return { columnSchema, columnName, tableName: tableAlias };
}
// Unqualified reference
const columnName = firstPart;
for (const [tableName, tableSchema] of context.tables.entries()) {
const columnSchema = tableSchema.columns[columnName];
if (columnSchema) {
return { columnSchema, columnName, tableName };
}
}
return null;
}
/**
* Validate that a value matches the allowed enum values for a column
*/
function validateEnumValue(
expr: Expression,
columnSchema: ColumnSchema,
columnName: string,
tableName: string | undefined,
context: ValidationContext
): void {
if ((expr as Constant).expression_type !== "constant") return;
const constant = expr as Constant;
if (typeof constant.value !== "string") return;
const value = constant.value;
const allowedValues = columnSchema.allowedValues!;
if (!allowedValues.includes(value)) {
const columnRef = tableName ? `${tableName}.${columnName}` : columnName;
context.issues.push({
message: `Invalid value "${value}" for column "${columnRef}". Allowed values: ${allowedValues.join(
", "
)}`,
severity: "error",
type: "invalid_enum_value",
columnName,
tableName,
invalidValue: value,
allowedValues,
});
}
}