From 49df40cb115c621dfe7bfd2e4efffab91aa44236 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 9 Jan 2026 11:39:36 +0000 Subject: [PATCH] TRQL and the Query page (#2843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TRQL (pronounced Treacle like the delicious British dark sweet syrup) is the TRiggerQueryLanguage. It allows users to safely write queries on their data. The queries are safely turned into ClickHouse queries which are tenant-safe and not SQL injectable. https://github.com/user-attachments/assets/bbfca473-b3fc-4150-8fe6-79e8840a2d29 This started out as a translation of HogQL by PostHog from Python to TypeScript. Features - Tenant safe queries. - Many underlying ClickHouse features including functions and aggregations. - Virtual columns, which are exposed to users as real columns but are actually expressions. - Transformations of data types and where clauses. - Simple JSON path querying. - Limits on execution time. - Reporting of query statistics. ## Query page There’s a new Query page (currently behind a feature flag) where you can write TRQL queries and execute them against your environment, project or organization. Features - Executing TRQL queries - Syntax highlighting and errors - Autocomplete - AI generation/editing of queries - Help and examples - Table with auto-inferred data types from the table schema - Table cell renderers for our special types like Run ids, environments, machines, tasks, queues, etc. - Copy/export as CSV/JSON - Line and bar graphs with grouping and stacking - History of queries --- apps/webapp/app/components/AlphaBadge.tsx | 32 + .../app/components/code/AIQueryInput.tsx | 401 + .../app/components/code/ChartConfigPanel.tsx | 521 + apps/webapp/app/components/code/CodeBlock.tsx | 4 + .../app/components/code/QueryResultsChart.tsx | 989 ++ .../webapp/app/components/code/TSQLEditor.tsx | 271 + .../app/components/code/TSQLResultsTable.tsx | 459 + apps/webapp/app/components/code/tsql/index.ts | 6 + .../code/tsql/tsqlCompletion.test.ts | 172 + .../components/code/tsql/tsqlCompletion.ts | 464 + .../components/code/tsql/tsqlLinter.test.ts | 79 + .../app/components/code/tsql/tsqlLinter.ts | 213 + .../app/components/navigation/SideMenu.tsx | 17 + .../app/components/primitives/DateTime.tsx | 202 +- .../app/components/primitives/Table.tsx | 75 +- .../app/components/primitives/Tooltip.tsx | 11 +- .../app/components/runs/v3/QueueName.tsx | 41 + .../app/components/runs/v3/TaskRunStatus.tsx | 106 +- apps/webapp/app/env.server.ts | 1 + apps/webapp/app/hooks/useFeatureFlags.ts | 11 + .../OrganizationsPresenter.server.ts | 10 + .../presenters/v3/QueryPresenter.server.ts | 44 + .../route.tsx | 2169 +++ .../route.tsx | 30 +- ...tParam.env.$envParam.query.ai-generate.tsx | 204 + apps/webapp/app/routes/runs.$runParam.ts | 68 + .../app/routes/storybook.table/route.tsx | 28 + .../routes/storybook.tsql-editor/route.tsx | 331 + apps/webapp/app/routes/storybook/route.tsx | 12 +- .../app/services/queryService.server.ts | 116 + .../services/runsReplicationService.server.ts | 1 + apps/webapp/app/tailwind.css | 91 + apps/webapp/app/utils/dataExport.ts | 79 + apps/webapp/app/utils/numberFormatter.ts | 11 + apps/webapp/app/utils/pathBuilder.ts | 14 +- apps/webapp/app/v3/featureFlags.server.ts | 3 + apps/webapp/app/v3/querySchemas.ts | 425 + .../app/v3/services/aiQueryService.server.ts | 487 + apps/webapp/evals/aiQuery.eval.ts | 374 + apps/webapp/package.json | 24 +- apps/webapp/remix.config.js | 2 + .../code/tsql/tsqlCompletion.test.ts | 174 + .../components/code/tsql/tsqlLinter.test.ts | 79 + internal-packages/clickhouse/package.json | 1 + .../012_add_task_runs_v2_max_duration.sql | 10 + .../clickhouse/src/client/client.ts | 180 + .../clickhouse/src/client/noop.ts | 35 +- .../clickhouse/src/client/tsql.ts | 200 + .../clickhouse/src/client/types.ts | 64 + internal-packages/clickhouse/src/index.ts | 13 + internal-packages/clickhouse/src/taskRuns.ts | 1 + internal-packages/clickhouse/src/tsql.test.ts | 1548 ++ .../clickhouse/tsconfig.build.json | 5 +- .../migration.sql | 40 + .../database/prisma/schema.prisma | 54 + internal-packages/tsql/.gitignore | 3 + internal-packages/tsql/NOTICE.md | 22 + internal-packages/tsql/README.md | 19 + internal-packages/tsql/package.json | 23 + .../tsql/src/grammar/TSQLLexer.common.g4 | 288 + .../tsql/src/grammar/TSQLLexer.interp | 570 + .../tsql/src/grammar/TSQLLexer.tokens | 208 + .../tsql/src/grammar/TSQLLexer.ts | 1730 +++ .../tsql/src/grammar/TSQLLexer.typescript.g4 | 81 + .../tsql/src/grammar/TSQLParser.g4 | 315 + .../tsql/src/grammar/TSQLParser.interp | 437 + .../tsql/src/grammar/TSQLParser.tokens | 208 + .../tsql/src/grammar/TSQLParser.ts | 11732 ++++++++++++++++ .../tsql/src/grammar/TSQLParserVisitor.ts | 1312 ++ .../tsql/src/grammar/parser.test.ts | 166 + internal-packages/tsql/src/index.ts | 277 + internal-packages/tsql/src/query/ast.ts | 672 + internal-packages/tsql/src/query/constants.ts | 71 + internal-packages/tsql/src/query/context.ts | 56 + internal-packages/tsql/src/query/database.ts | 585 + internal-packages/tsql/src/query/errors.ts | 61 + .../tsql/src/query/escape.test.ts | 241 + internal-packages/tsql/src/query/escape.ts | 265 + internal-packages/tsql/src/query/functions.ts | 648 + internal-packages/tsql/src/query/models.ts | 253 + .../tsql/src/query/parse_string.ts | 65 + .../tsql/src/query/parser.test.ts | 629 + internal-packages/tsql/src/query/parser.ts | 1845 +++ .../tsql/src/query/printer.test.ts | 2280 +++ internal-packages/tsql/src/query/printer.ts | 2290 +++ .../tsql/src/query/printer_context.ts | 215 + .../tsql/src/query/property_types.ts | 720 + .../tsql/src/query/results.test.ts | 234 + internal-packages/tsql/src/query/results.ts | 201 + .../tsql/src/query/schema.test.ts | 456 + internal-packages/tsql/src/query/schema.ts | 684 + .../tsql/src/query/security.test.ts | 601 + internal-packages/tsql/src/query/timings.ts | 107 + .../tsql/src/query/validator.test.ts | 124 + internal-packages/tsql/src/query/validator.ts | 553 + internal-packages/tsql/tsconfig.json | 25 + internal-packages/tsql/vitest.config.ts | 19 + package.json | 3 +- patches/antlr4ts@0.5.0-alpha.4.patch | 470 + pnpm-lock.yaml | 851 +- 100 files changed, 43032 insertions(+), 585 deletions(-) create mode 100644 apps/webapp/app/components/AlphaBadge.tsx create mode 100644 apps/webapp/app/components/code/AIQueryInput.tsx create mode 100644 apps/webapp/app/components/code/ChartConfigPanel.tsx create mode 100644 apps/webapp/app/components/code/QueryResultsChart.tsx create mode 100644 apps/webapp/app/components/code/TSQLEditor.tsx create mode 100644 apps/webapp/app/components/code/TSQLResultsTable.tsx create mode 100644 apps/webapp/app/components/code/tsql/index.ts create mode 100644 apps/webapp/app/components/code/tsql/tsqlCompletion.test.ts create mode 100644 apps/webapp/app/components/code/tsql/tsqlCompletion.ts create mode 100644 apps/webapp/app/components/code/tsql/tsqlLinter.test.ts create mode 100644 apps/webapp/app/components/code/tsql/tsqlLinter.ts create mode 100644 apps/webapp/app/components/runs/v3/QueueName.tsx create mode 100644 apps/webapp/app/hooks/useFeatureFlags.ts create mode 100644 apps/webapp/app/presenters/v3/QueryPresenter.server.ts create mode 100644 apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx create mode 100644 apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-generate.tsx create mode 100644 apps/webapp/app/routes/runs.$runParam.ts create mode 100644 apps/webapp/app/routes/storybook.tsql-editor/route.tsx create mode 100644 apps/webapp/app/services/queryService.server.ts create mode 100644 apps/webapp/app/utils/dataExport.ts create mode 100644 apps/webapp/app/v3/querySchemas.ts create mode 100644 apps/webapp/app/v3/services/aiQueryService.server.ts create mode 100644 apps/webapp/evals/aiQuery.eval.ts create mode 100644 apps/webapp/test/components/code/tsql/tsqlCompletion.test.ts create mode 100644 apps/webapp/test/components/code/tsql/tsqlLinter.test.ts create mode 100644 internal-packages/clickhouse/schema/012_add_task_runs_v2_max_duration.sql create mode 100644 internal-packages/clickhouse/src/client/tsql.ts create mode 100644 internal-packages/clickhouse/src/tsql.test.ts create mode 100644 internal-packages/database/prisma/migrations/20251220201230_add_customer_query/migration.sql create mode 100644 internal-packages/tsql/.gitignore create mode 100644 internal-packages/tsql/NOTICE.md create mode 100644 internal-packages/tsql/README.md create mode 100644 internal-packages/tsql/package.json create mode 100644 internal-packages/tsql/src/grammar/TSQLLexer.common.g4 create mode 100644 internal-packages/tsql/src/grammar/TSQLLexer.interp create mode 100644 internal-packages/tsql/src/grammar/TSQLLexer.tokens create mode 100644 internal-packages/tsql/src/grammar/TSQLLexer.ts create mode 100644 internal-packages/tsql/src/grammar/TSQLLexer.typescript.g4 create mode 100644 internal-packages/tsql/src/grammar/TSQLParser.g4 create mode 100644 internal-packages/tsql/src/grammar/TSQLParser.interp create mode 100644 internal-packages/tsql/src/grammar/TSQLParser.tokens create mode 100644 internal-packages/tsql/src/grammar/TSQLParser.ts create mode 100644 internal-packages/tsql/src/grammar/TSQLParserVisitor.ts create mode 100644 internal-packages/tsql/src/grammar/parser.test.ts create mode 100644 internal-packages/tsql/src/index.ts create mode 100644 internal-packages/tsql/src/query/ast.ts create mode 100644 internal-packages/tsql/src/query/constants.ts create mode 100644 internal-packages/tsql/src/query/context.ts create mode 100644 internal-packages/tsql/src/query/database.ts create mode 100644 internal-packages/tsql/src/query/errors.ts create mode 100644 internal-packages/tsql/src/query/escape.test.ts create mode 100644 internal-packages/tsql/src/query/escape.ts create mode 100644 internal-packages/tsql/src/query/functions.ts create mode 100644 internal-packages/tsql/src/query/models.ts create mode 100644 internal-packages/tsql/src/query/parse_string.ts create mode 100644 internal-packages/tsql/src/query/parser.test.ts create mode 100644 internal-packages/tsql/src/query/parser.ts create mode 100644 internal-packages/tsql/src/query/printer.test.ts create mode 100644 internal-packages/tsql/src/query/printer.ts create mode 100644 internal-packages/tsql/src/query/printer_context.ts create mode 100644 internal-packages/tsql/src/query/property_types.ts create mode 100644 internal-packages/tsql/src/query/results.test.ts create mode 100644 internal-packages/tsql/src/query/results.ts create mode 100644 internal-packages/tsql/src/query/schema.test.ts create mode 100644 internal-packages/tsql/src/query/schema.ts create mode 100644 internal-packages/tsql/src/query/security.test.ts create mode 100644 internal-packages/tsql/src/query/timings.ts create mode 100644 internal-packages/tsql/src/query/validator.test.ts create mode 100644 internal-packages/tsql/src/query/validator.ts create mode 100644 internal-packages/tsql/tsconfig.json create mode 100644 internal-packages/tsql/vitest.config.ts create mode 100644 patches/antlr4ts@0.5.0-alpha.4.patch diff --git a/apps/webapp/app/components/AlphaBadge.tsx b/apps/webapp/app/components/AlphaBadge.tsx new file mode 100644 index 000000000..58da1a994 --- /dev/null +++ b/apps/webapp/app/components/AlphaBadge.tsx @@ -0,0 +1,32 @@ +import { cn } from "~/utils/cn"; +import { Badge } from "./primitives/Badge"; +import { SimpleTooltip } from "./primitives/Tooltip"; + +export function AlphaBadge({ + inline = false, + className, +}: { + inline?: boolean; + className?: string; +}) { + return ( + + Alpha + + } + content="This feature is in Alpha." + disableHoverableContent + /> + ); +} + +export function AlphaTitle({ children }: { children: React.ReactNode }) { + return ( + <> + {children} + + + ); +} diff --git a/apps/webapp/app/components/code/AIQueryInput.tsx b/apps/webapp/app/components/code/AIQueryInput.tsx new file mode 100644 index 000000000..e3e3ed9c7 --- /dev/null +++ b/apps/webapp/app/components/code/AIQueryInput.tsx @@ -0,0 +1,401 @@ +import { PencilSquareIcon, PlusIcon, SparklesIcon } from "@heroicons/react/20/solid"; +import { AnimatePresence, motion } from "framer-motion"; +import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react"; +import { AISparkleIcon } from "~/assets/icons/AISparkleIcon"; + +// Lazy load streamdown components to avoid SSR issues +const StreamdownRenderer = lazy(() => + import("streamdown").then((mod) => ({ + default: ({ children, isAnimating }: { children: string; isAnimating: boolean }) => ( + + {children} + + ), + })) +); +import { Button } from "~/components/primitives/Buttons"; +import { Spinner } from "~/components/primitives/Spinner"; +import { useEnvironment } from "~/hooks/useEnvironment"; +import { useOrganization } from "~/hooks/useOrganizations"; +import { useProject } from "~/hooks/useProject"; +import { cn } from "~/utils/cn"; + +type StreamEventType = + | { type: "thinking"; content: string } + | { type: "tool_call"; tool: string; args: unknown } + | { type: "result"; success: true; query: string } + | { type: "result"; success: false; error: string }; + +export type AIQueryMode = "new" | "edit"; + +interface AIQueryInputProps { + onQueryGenerated: (query: string) => void; + /** Set this to a prompt to auto-populate and immediately submit */ + autoSubmitPrompt?: string; + /** Get the current query in the editor (used for edit mode) */ + getCurrentQuery?: () => string; +} + +export function AIQueryInput({ + onQueryGenerated, + autoSubmitPrompt, + getCurrentQuery, +}: AIQueryInputProps) { + const [prompt, setPrompt] = useState(""); + const [mode, setMode] = useState("new"); + const [isLoading, setIsLoading] = useState(false); + const [thinking, setThinking] = useState(""); + const [error, setError] = useState(null); + const [showThinking, setShowThinking] = useState(false); + const [lastResult, setLastResult] = useState<"success" | "error" | null>(null); + const textareaRef = useRef(null); + const abortControllerRef = useRef(null); + const lastAutoSubmitRef = useRef(null); + + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + + const resourcePath = `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/query/ai-generate`; + + // Can only use edit mode if there's a current query + const canEdit = Boolean(getCurrentQuery?.()?.trim()); + + // If mode is edit but there's no current query, switch to new + useEffect(() => { + if (mode === "edit" && !canEdit) { + setMode("new"); + } + }, [mode, canEdit]); + + const submitQuery = useCallback( + async (queryPrompt: string, submitMode: AIQueryMode = mode) => { + if (!queryPrompt.trim() || isLoading) return; + const currentQuery = getCurrentQuery?.(); + if (submitMode === "edit" && !currentQuery?.trim()) return; + + setIsLoading(true); + setThinking(""); + setError(null); + setShowThinking(true); + setLastResult(null); + + // Abort any existing request + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + } + abortControllerRef.current = new AbortController(); + + try { + const formData = new FormData(); + formData.append("prompt", queryPrompt); + formData.append("mode", submitMode); + if (submitMode === "edit" && currentQuery) { + formData.append("currentQuery", currentQuery); + } + + const response = await fetch(resourcePath, { + method: "POST", + body: formData, + signal: abortControllerRef.current.signal, + }); + + if (!response.ok) { + const errorData = (await response.json()) as { error?: string }; + setError(errorData.error || "Failed to generate query"); + setIsLoading(false); + setLastResult("error"); + return; + } + + const reader = response.body?.getReader(); + if (!reader) { + setError("No response stream"); + setIsLoading(false); + setLastResult("error"); + return; + } + + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + // Process complete events from buffer + const lines = buffer.split("\n\n"); + buffer = lines.pop() || ""; // Keep incomplete line in buffer + + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const event = JSON.parse(line.slice(6)) as StreamEventType; + processStreamEvent(event); + } catch { + // Ignore parse errors + } + } + } + } + + // Process any remaining data + if (buffer.startsWith("data: ")) { + try { + const event = JSON.parse(buffer.slice(6)) as StreamEventType; + processStreamEvent(event); + } catch { + // Ignore parse errors + } + } + } catch (err) { + if (err instanceof Error && err.name === "AbortError") { + // Request was aborted, ignore + return; + } + setError(err instanceof Error ? err.message : "An error occurred"); + setLastResult("error"); + } finally { + setIsLoading(false); + } + }, + [isLoading, resourcePath, mode, getCurrentQuery] + ); + + const processStreamEvent = useCallback( + (event: StreamEventType) => { + switch (event.type) { + case "thinking": + setThinking((prev) => prev + event.content); + break; + case "tool_call": + setThinking((prev) => prev + `\nValidating query...\n`); + break; + case "result": + if (event.success) { + onQueryGenerated(event.query); + setPrompt(""); + setLastResult("success"); + // Keep thinking visible to show what happened + } else { + setError(event.error); + setLastResult("error"); + } + break; + } + }, + [onQueryGenerated] + ); + + const handleSubmit = useCallback( + (e?: React.FormEvent) => { + e?.preventDefault(); + submitQuery(prompt); + }, + [prompt, submitQuery] + ); + + // Auto-submit when autoSubmitPrompt changes + useEffect(() => { + if ( + autoSubmitPrompt && + autoSubmitPrompt.trim() && + autoSubmitPrompt !== lastAutoSubmitRef.current && + !isLoading + ) { + lastAutoSubmitRef.current = autoSubmitPrompt; + setPrompt(autoSubmitPrompt); + submitQuery(autoSubmitPrompt); + } + }, [autoSubmitPrompt, isLoading, submitQuery]); + + // Cleanup on unmount + useEffect(() => { + return () => { + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + } + }; + }, []); + + // Auto-hide error after delay + useEffect(() => { + if (error) { + const timer = setTimeout(() => setError(null), 15000); + return () => clearTimeout(timer); + } + }, [error]); + + return ( +
+ {/* Gradient border wrapper like the schedules AI input */} +
+
+
+