github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Query: time limits, performance improvements, styling (#2953)

Browse Source 
Summary
- Query: add time limits, performance improvements, and styling updates

Changes
- Add ClickHouse output_text and error_text columns with indexes
- Automatically use _text columns for JSON based on query pattern;
support JSON column data prefixes
- Add idempotency key and scope columns
- Add enforcedWhereClause for tenant and time restrictions, instead of
the old tenant stuff.
- Implement basic time filter limiting and set default time period based
on plan; show message when results are clipped
- UX: resizable code area (including vertical splits), collapsible
sidebar, fix table/chart vertical sizing, max height for chart legend in
fullscreen
- Styling and UI tweaks: improved chart legend styling, more chart
colours, thinner line chart stroke, pricing callout color, improved
layout for callouts
- Features: generate and save AI titles
<!-- devin-review-badge-begin -->

---

<a
href="https://app.devin.ai/review/triggerdotdev/trigger.dev/pull/2953">
  <picture>
<source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-open-in-devin-review-dark.svg?v=1">
<img
src="https://static.devin.ai/assets/gh-open-in-devin-review-light.svg?v=1"
alt="Open with Devin">
  </picture>
</a>
<!-- devin-review-badge-end -->
This commit is contained in:
Matt Aitken
2026-01-29 13:02:47 +00:00
committed by GitHub
parent c0b86efbd3
commit f53db6fd16
36 changed files with 2987 additions and 798 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/webapp/app/components/code/QueryResultsChart.tsx
+30 -2
View File
@@ -5,9 +5,10 @@ import { Chart } from "~/components/primitives/charts/ChartCompound";
import { Paragraph } from "../primitives/Paragraph";
import type { AggregationType, ChartConfiguration } from "./ChartConfigPanel";
// Color palette for chart series
// Color palette for chart series - 30 distinct colors for large datasets
const CHART_COLORS = [
"#7655fd", // Primary purple
// Primary colors
"#7655fd", // Purple
"#22c55e", // Green
"#f59e0b", // Amber
"#ef4444", // Red
@@ -17,6 +18,28 @@ const CHART_COLORS = [
"#14b8a6", // Teal
"#f97316", // Orange
"#6366f1", // Indigo
// Extended palette
"#84cc16", // Lime
"#0ea5e9", // Sky
"#f43f5e", // Rose
"#a855f7", // Fuchsia
"#eab308", // Yellow
"#10b981", // Emerald
"#3b82f6", // Blue
"#d946ef", // Magenta
"#78716c", // Stone
"#facc15", // Gold
// Additional distinct colors
"#2dd4bf", // Turquoise
"#fb923c", // Light orange
"#a3e635", // Yellow-green
"#38bdf8", // Light blue
"#c084fc", // Light purple
"#4ade80", // Light green
"#fbbf24", // Light amber
"#f472b6", // Light pink
"#67e8f9", // Light cyan
"#818cf8", // Light indigo
];
function getSeriesColor(index: number): string {
@@ -30,6 +53,8 @@ interface QueryResultsChartProps {
fullLegend?: boolean;
/** Callback when "View all" legend button is clicked */
onViewAllLegendItems?: () => void;
/** When true, constrains legend to max 50% height with scrolling */
legendScrollable?: boolean;
}
interface TransformedData {
@@ -702,6 +727,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
config,
fullLegend = false,
onViewAllLegendItems,
legendScrollable = false,
}: QueryResultsChartProps) {
const {
xAxisColumn,
@@ -872,6 +898,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
minHeight="300px"
fillContainer
onViewAllLegendItems={onViewAllLegendItems}
legendScrollable={legendScrollable}
>
<Chart.Bar
xAxisProps={xAxisPropsForBar}
@@ -896,6 +923,7 @@ export const QueryResultsChart = memo(function QueryResultsChart({
minHeight="300px"
fillContainer
onViewAllLegendItems={onViewAllLegendItems}
legendScrollable={legendScrollable}
>
<Chart.Line
xAxisProps={xAxisPropsForLine}
apps/webapp/app/components/code/TSQLResultsTable.tsx
+4 -1
View File
@@ -672,7 +672,9 @@ function EnvironmentCellValue({ value }: { value: string }) {
}
function JSONCellValue({ value }: { value: unknown }) {
const jsonString = JSON.stringify(value);
// If the value is already a string (e.g., from a textColumn optimization),
// use it directly without double-stringifying
const jsonString = typeof value === "string" ? value : JSON.stringify(value);
const isTruncated = jsonString.length > MAX_STRING_DISPLAY_LENGTH;
if (isTruncated) {
@@ -1137,6 +1139,7 @@ export const TSQLResultsTable = memo(function TSQLResultsTable({
height: `${rowVirtualizer.getTotalSize()}px`,
position: "relative",
}}
className="bg-background-dimmed divide-y divide-charcoal-700"
>
{rowVirtualizer.getVirtualItems().map((virtualRow) => {
const row = tableRows[virtualRow.index];
apps/webapp/app/components/primitives/AnimatedNumber.tsx
+58 -3
View File
@@ -1,9 +1,64 @@
import { animate, motion, useMotionValue, useTransform } from "framer-motion";
import { useEffect } from "react";
import { useEffect, useMemo } from "react";
export function AnimatedNumber({ value, duration = 0.5 }: { value: number; duration?: number }) {
/**
* Determines the number of decimal places to display based on the value.
* - For integers or large numbers (>=100), no decimals
* - For numbers >= 10, 1 decimal place
* - For numbers >= 1, 2 decimal places
* - For smaller numbers, up to 4 decimal places
*/
function getDecimalPlaces(value: number): number {
if (Number.isInteger(value)) return 0;
const absValue = Math.abs(value);
if (absValue >= 100) return 0;
if (absValue >= 10) return 1;
if (absValue >= 1) return 2;
if (absValue >= 0.1) return 3;
return 4;
}
/**
* Sanitizes a decimal places value to ensure it's valid for toLocaleString.
* - Coerces to a finite number (handles NaN, Infinity, -Infinity)
* - Rounds to an integer
* - Clamps to the valid 0-20 range for toLocaleString options
*/
function sanitizeDecimals(decimals: number): number {
if (!Number.isFinite(decimals)) {
return 0;
}
return Math.min(20, Math.max(0, Math.round(decimals)));
}
export function AnimatedNumber({
value,
duration = 0.5,
decimalPlaces,
}: {
value: number;
duration?: number;
/** Number of decimal places to display. If not provided, auto-detects based on value. */
decimalPlaces?: number;
}) {
const motionValue = useMotionValue(value);
let display = useTransform(motionValue, (current) => Math.round(current).toLocaleString());
// Determine decimal places - use provided value or auto-detect, then sanitize
const safeDecimals = useMemo(() => {
const rawDecimals = decimalPlaces !== undefined ? decimalPlaces : getDecimalPlaces(value);
return sanitizeDecimals(rawDecimals);
}, [decimalPlaces, value]);
const display = useTransform(motionValue, (current) => {
if (safeDecimals === 0) {
return Math.round(current).toLocaleString();
}
return current.toLocaleString(undefined, {
minimumFractionDigits: safeDecimals,
maximumFractionDigits: safeDecimals,
});
});
useEffect(() => {
animate(motionValue, value, {
apps/webapp/app/components/primitives/Callout.tsx
+5 -4
View File
@@ -1,4 +1,5 @@
import {
CreditCardIcon,
ExclamationCircleIcon,
ExclamationTriangleIcon,
InformationCircleIcon,
@@ -60,10 +61,10 @@ export const variantClasses = {
linkClassName: "transition hover:bg-blue-400/20",
},
pricing: {
className: "border-charcoal-700 bg-charcoal-800",
icon: <ChartBarIcon className="h-5 w-5 shrink-0 text-text-dimmed" />,
textColor: "text-text-bright",
linkClassName: "transition hover:bg-charcoal-750",
className: "border-indigo-400/20 bg-indigo-800/30",
icon: <CreditCardIcon className="h-5 w-5 shrink-0 text-indigo-400" />,
textColor: "text-indigo-300",
linkClassName: "transition hover:bg-indigo-400/20",
},
} as const;
apps/webapp/app/components/primitives/Resizable.tsx
+28 -7
View File
@@ -26,19 +26,40 @@ const ResizableHandle = ({
}) => (
<PanelResizer
className={cn(
"group relative flex w-0.75 items-center justify-center focus-custom after:absolute after:inset-y-0 after:left-1/2 after:w-1 after:-translate-x-1/2 data-[panel-group-direction=vertical]:h-px data-[panel-group-direction=vertical]:w-full data-[panel-group-direction=vertical]:after:left-0 data-[panel-group-direction=vertical]:after:h-1 data-[panel-group-direction=vertical]:after:w-full data-[panel-group-direction=vertical]:after:-translate-y-1/2 data-[panel-group-direction=vertical]:after:translate-x-0 hover:w-0.75 [&[data-panel-group-direction=vertical]>div]:rotate-90",
// Base styles
"group relative flex items-center justify-center focus-custom",
// Horizontal orientation (default)
"w-0.75 after:absolute after:inset-y-0 after:left-1/2 after:w-1 after:-translate-x-1/2",
// Vertical orientation
"data-[handle-orientation=vertical]:h-0.75 data-[handle-orientation=vertical]:w-full",
"data-[handle-orientation=vertical]:after:inset-x-0 data-[handle-orientation=vertical]:after:inset-y-auto",
"data-[handle-orientation=vertical]:after:top-1/2 data-[handle-orientation=vertical]:after:left-0",
"data-[handle-orientation=vertical]:after:h-1 data-[handle-orientation=vertical]:after:w-full",
"data-[handle-orientation=vertical]:after:-translate-y-1/2 data-[handle-orientation=vertical]:after:translate-x-0",
className
)}
size="3px"
{...props}
>
<div className="absolute left-[0.0625rem] top-0 h-full w-px bg-grid-bright transition group-hover:left-0 group-hover:w-0.75 group-hover:bg-lavender-500" />
{/* Horizontal orientation line indicator */}
<div className="absolute left-[0.0625rem] top-0 h-full w-px bg-grid-bright transition group-hover:left-0 group-hover:w-0.75 group-hover:bg-lavender-500 group-data-[handle-orientation=vertical]:hidden" />
{/* Vertical orientation line indicator */}
<div className="absolute left-0 top-[0.0625rem] hidden h-px w-full bg-grid-bright transition group-hover:top-0 group-hover:h-0.75 group-hover:bg-lavender-500 group-data-[handle-orientation=vertical]:block" />
{withHandle && (
<div className="z-10 flex h-5 w-3 flex-col items-center justify-center gap-[0.1875rem] bg-background-dimmed group-hover:hidden">
{Array.from({ length: 3 }).map((_, index) => (
<div key={index} className="h-[0.1875rem] w-0.75 rounded-full bg-charcoal-600" />
))}
</div>
<>
{/* Horizontal orientation dots (vertical arrangement) */}
<div className="z-10 flex h-5 w-0.75 flex-col items-center justify-center gap-[0.1875rem] bg-background-dimmed group-hover:hidden group-data-[handle-orientation=vertical]:hidden">
{Array.from({ length: 3 }).map((_, index) => (
<div key={index} className="h-[0.1875rem] w-0.75 rounded-full bg-charcoal-600" />
))}
</div>
{/* Vertical orientation dots (horizontal arrangement) */}
<div className="z-10 hidden h-0.75 w-5 flex-row items-center justify-center gap-[0.1875rem] bg-background-dimmed group-hover:hidden group-data-[handle-orientation=vertical]:flex">
{Array.from({ length: 3 }).map((_, index) => (
<div key={index} className="h-0.75 w-[0.1875rem] rounded-full bg-charcoal-600" />
))}
</div>
</>
)}
</PanelResizer>
);
apps/webapp/app/components/primitives/charts/Card.tsx
+2 -2
View File
@@ -6,7 +6,7 @@ export const Card = ({ children, className }: { children: ReactNode; className?:
return (
<div
className={cn(
"flex flex-col rounded-lg border border-grid-bright bg-background-bright pb-2 pt-4",
"flex flex-col rounded-lg border border-grid-bright bg-background-bright pb-1.5 pt-3",
className
)}
>
@@ -17,7 +17,7 @@ export const Card = ({ children, className }: { children: ReactNode; className?:
const CardHeader = ({ children }: { children: ReactNode }) => {
return (
<Header3 className="mb-4 flex items-center justify-between gap-2 px-4">{children}</Header3>
<Header3 className="mb-3 flex items-center justify-between gap-2 px-3">{children}</Header3>
);
};
apps/webapp/app/components/primitives/charts/ChartLegendCompound.tsx
+67 -52
View File
@@ -17,6 +17,8 @@ export type ChartLegendCompoundProps = {
totalLabel?: string;
/** Callback when "View all" button is clicked */
onViewAllLegendItems?: () => void;
/** When true, constrains legend to max 50% height with scrolling */
scrollable?: boolean;
};
/**
@@ -37,6 +39,7 @@ export function ChartLegendCompound({
className,
totalLabel = "Total",
onViewAllLegendItems,
scrollable = false,
}: ChartLegendCompoundProps) {
const { config, dataKey, dataKeys, highlight, labelFormatter } = useChartContext();
const totals = useSeriesTotal();
@@ -128,11 +131,17 @@ export function ChartLegendCompound({
const isHovering = (highlight.activePayload?.length ?? 0) > 0;
return (
<div className={cn("flex flex-col pt-4 text-sm", className)}>
<div
className={cn(
"flex flex-col pt-4 text-sm",
scrollable && "max-h-[50%] min-h-0",
className
)}
>
{/* Total row */}
<div
className={cn(
"flex w-full items-center justify-between gap-2 rounded px-2 py-1 transition",
"flex w-full shrink-0 items-center justify-between gap-2 rounded px-2 py-1 transition",
isHovering ? "text-text-bright" : "text-text-dimmed"
)}
>
@@ -143,62 +152,68 @@ export function ChartLegendCompound({
</div>
{/* Separator */}
<div className="mx-2 my-1 border-t border-charcoal-750" />
<div className="mx-2 my-1 shrink-0 border-t border-charcoal-750" />
{legendItems.visible.map((item) => {
const total = currentData[item.dataKey] ?? 0;
const isActive = highlight.activeBarKey === item.dataKey;
{/* Legend items - scrollable when scrollable prop is true */}
<div className={cn("flex flex-col", scrollable && "min-h-0 flex-1 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600")}>
{legendItems.visible.map((item) => {
const total = currentData[item.dataKey] ?? 0;
const isActive = highlight.activeBarKey === item.dataKey;
return (
<div
key={item.dataKey}
className={cn(
"relative flex w-full cursor-pointer items-center justify-between gap-2 rounded px-2 py-1 transition",
total === 0 && "opacity-50"
)}
onMouseEnter={() => highlight.setHoveredLegendItem(item.dataKey)}
onMouseLeave={() => highlight.reset()}
>
{/* Active highlight background */}
{isActive && item.color && (
<div
className="absolute inset-0 rounded opacity-10"
style={{ backgroundColor: item.color }}
/>
)}
<div className="relative flex w-full items-center justify-between gap-2">
<div className="flex items-center gap-1.5">
{item.color && (
<div
className="h-3 w-1 shrink-0 rounded-[2px]"
style={{ backgroundColor: item.color }}
/>
)}
<span className={isActive ? "text-text-bright" : "text-text-dimmed"}>
{item.label}
return (
<div
key={item.dataKey}
className={cn(
"relative flex w-full cursor-pointer items-center justify-between gap-2 rounded px-2 py-1 transition",
total === 0 && "opacity-50"
)}
onMouseEnter={() => highlight.setHoveredLegendItem(item.dataKey)}
onMouseLeave={() => highlight.reset()}
>
{/* Active highlight background */}
{isActive && item.color && (
<div
className="absolute inset-0 rounded opacity-10"
style={{ backgroundColor: item.color }}
/>
)}
<div className="relative flex w-full items-center justify-between gap-3">
<div className="flex items-center gap-1.5">
{item.color && (
<div
className="w-1 shrink-0 self-stretch rounded-[2px]"
style={{ backgroundColor: item.color }}
/>
)}
<span className={isActive ? "text-text-bright" : "text-text-dimmed"}>
{item.label}
</span>
</div>
<span
className={cn(
"self-start tabular-nums",
isActive ? "text-text-bright" : "text-text-dimmed"
)}
>
<AnimatedNumber value={total} duration={0.25} />
</span>
</div>
<span
className={cn("tabular-nums", isActive ? "text-text-bright" : "text-text-dimmed")}
>
<AnimatedNumber value={total} duration={0.25} />
</span>
</div>
</div>
);
})}
);
})}
{/* View more row - replaced by hovered hidden item when applicable */}
{legendItems.remaining > 0 &&
(legendItems.hoveredHiddenItem ? (
<HoveredHiddenItemRow
item={legendItems.hoveredHiddenItem}
value={currentData[legendItems.hoveredHiddenItem.dataKey] ?? 0}
remainingCount={legendItems.remaining - 1}
/>
) : (
<ViewAllDataRow remainingCount={legendItems.remaining} onViewAll={onViewAllLegendItems} />
))}
{/* View more row - replaced by hovered hidden item when applicable */}
{legendItems.remaining > 0 &&
(legendItems.hoveredHiddenItem ? (
<HoveredHiddenItemRow
item={legendItems.hoveredHiddenItem}
value={currentData[legendItems.hoveredHiddenItem.dataKey] ?? 0}
remainingCount={legendItems.remaining - 1}
/>
) : (
<ViewAllDataRow remainingCount={legendItems.remaining} onViewAll={onViewAllLegendItems} />
))}
</div>
</div>
);
}
apps/webapp/app/components/primitives/charts/ChartLine.tsx
+2 -2
View File
@@ -172,7 +172,7 @@ export function ChartLineRenderer({
stroke={config[key]?.color}
fill={config[key]?.color}
fillOpacity={0.6}
strokeWidth={2}
strokeWidth={1}
stackId="stack"
isAnimationActive={false}
/>
@@ -220,7 +220,7 @@ export function ChartLineRenderer({
dataKey={key}
type={lineType}
stroke={config[key]?.color}
strokeWidth={2}
strokeWidth={1}
dot={false}
activeDot={{ r: 4 }}
isAnimationActive={false}
apps/webapp/app/components/primitives/charts/ChartRoot.tsx
+7
View File
@@ -31,6 +31,8 @@ export type ChartRootProps = {
legendTotalLabel?: string;
/** Callback when "View all" legend button is clicked */
onViewAllLegendItems?: () => void;
/** When true, constrains legend to max 50% height with scrolling */
legendScrollable?: boolean;
/** When true, chart fills its parent container height and distributes space between chart and legend */
fillContainer?: boolean;
children: React.ComponentProps<typeof RechartsPrimitive.ResponsiveContainer>["children"];
@@ -72,6 +74,7 @@ export function ChartRoot({
maxLegendItems = 5,
legendTotalLabel,
onViewAllLegendItems,
legendScrollable = false,
fillContainer = false,
children,
}: ChartRootProps) {
@@ -94,6 +97,7 @@ export function ChartRoot({
maxLegendItems={maxLegendItems}
legendTotalLabel={legendTotalLabel}
onViewAllLegendItems={onViewAllLegendItems}
legendScrollable={legendScrollable}
fillContainer={fillContainer}
>
{children}
@@ -109,6 +113,7 @@ type ChartRootInnerProps = {
maxLegendItems?: number;
legendTotalLabel?: string;
onViewAllLegendItems?: () => void;
legendScrollable?: boolean;
fillContainer?: boolean;
children: React.ComponentProps<typeof RechartsPrimitive.ResponsiveContainer>["children"];
};
@@ -120,6 +125,7 @@ function ChartRootInner({
maxLegendItems = 5,
legendTotalLabel,
onViewAllLegendItems,
legendScrollable = false,
fillContainer = false,
children,
}: ChartRootInnerProps) {
@@ -160,6 +166,7 @@ function ChartRootInner({
maxItems={maxLegendItems}
totalLabel={legendTotalLabel}
onViewAllLegendItems={onViewAllLegendItems}
scrollable={legendScrollable}
/>
)}
</div>
apps/webapp/app/components/runs/v3/SharedFilters.tsx
+76 -80
View File
@@ -4,36 +4,31 @@ import {
endOfDay,
endOfMonth,
endOfWeek,
isSaturday,
isSunday,
previousSaturday,
startOfDay,
startOfMonth,
startOfWeek,
startOfYear,
subDays,
subMonths,
subWeeks,
subWeeks
} from "date-fns";
import parse from "parse-duration";
import { startTransition, useCallback, useEffect, useState, type ReactNode } from "react";
import simplur from "simplur";
import { AppliedFilter } from "~/components/primitives/AppliedFilter";
import { Callout } from "~/components/primitives/Callout";
import { DateTime } from "~/components/primitives/DateTime";
import { DateTimePicker } from "~/components/primitives/DateTimePicker";
import { Label } from "~/components/primitives/Label";
import { Paragraph } from "~/components/primitives/Paragraph";
import { RadioButtonCircle } from "~/components/primitives/RadioButton";
import { ComboboxProvider, SelectPopover, SelectProvider } from "~/components/primitives/Select";
import { useOptionalOrganization } from "~/hooks/useOrganizations";
import { useSearchParams } from "~/hooks/useSearchParam";
import { type ShortcutDefinition } from "~/hooks/useShortcutKeys";
import { cn } from "~/utils/cn";
import { Button } from "../../primitives/Buttons";
import { organizationBillingPath } from "~/utils/pathBuilder";
import { Button, LinkButton } from "../../primitives/Buttons";
import { filterIcon } from "./RunFilters";
export type DisplayableEnvironment = Pick<RuntimeEnvironment, "type" | "id"> & {
userName?: string;
};
export function FilterMenuProvider({
children,
onClose,
@@ -95,6 +90,10 @@ const timePeriods = [
label: "3 days",
value: "3d",
},
{
label: "5 days",
value: "5d",
},
{
label: "7 days",
value: "7d",
@@ -106,11 +105,7 @@ const timePeriods = [
{
label: "30 days",
value: "30d",
},
{
label: "90 days",
value: "90d",
},
}
];
const timeUnits = [
@@ -128,6 +123,22 @@ function parsePeriodString(period: string): { value: number; unit: string } | nu
return null;
}
const MS_PER_DAY = 1000 * 60 * 60 * 24;
// Convert a period string to days using parse-duration
function periodToDays(period: string): number {
const ms = parse(period);
if (!ms) return 0;
return ms / MS_PER_DAY;
}
// Calculate the number of days a date range spans from now
function dateRangeToDays(from?: Date): number {
if (!from) return 0;
const now = new Date();
return Math.ceil((now.getTime() - from.getTime()) / MS_PER_DAY);
}
const DEFAULT_PERIOD = "7d";
const defaultPeriodMs = parse(DEFAULT_PERIOD);
if (!defaultPeriodMs) {
@@ -292,6 +303,8 @@ export interface TimeFilterProps {
applyShortcut?: ShortcutDefinition | undefined;
/** Callback when the user applies a time filter selection, receives the applied values */
onValueChange?: (values: TimeFilterApplyValues) => void;
/** When set an upgrade message will be shown if you select a period further back than this number of days */
maxPeriodDays?: number;
}
export function TimeFilter({
@@ -303,6 +316,7 @@ export function TimeFilter({
hideLabel = false,
applyShortcut,
onValueChange,
maxPeriodDays,
}: TimeFilterProps = {}) {
const { value } = useSearchParams();
const periodValue = period ?? value("period");
@@ -339,6 +353,7 @@ export function TimeFilter({
labelName={labelName}
applyShortcut={applyShortcut}
onValueChange={onValueChange}
maxPeriodDays={maxPeriodDays}
/>
)}
</FilterMenuProvider>
@@ -356,6 +371,8 @@ function getInitialCustomDuration(period?: string): { value: string; unit: strin
return { value: "", unit: "m" };
}
type SectionType = "duration" | "dateRange";
export function TimeDropdown({
trigger,
period,
@@ -366,6 +383,7 @@ export function TimeDropdown({
applyShortcut,
onApply,
onValueChange,
maxPeriodDays,
}: {
trigger: ReactNode;
period?: string;
@@ -377,14 +395,16 @@ export function TimeDropdown({
onApply?: (values: TimeFilterApplyValues) => void;
/** When provided, the component operates in controlled mode and skips URL navigation */
onValueChange?: (values: TimeFilterApplyValues) => void;
/** When set an upgrade message will be shown if you select a period further back than this number of days */
maxPeriodDays?: number;
}) {
const organization = useOptionalOrganization();
const [open, setOpen] = useState<boolean | undefined>();
const { replace } = useSearchParams();
const [fromValue, setFromValue] = useState(from);
const [toValue, setToValue] = useState(to);
// Section selection state: "duration" or "dateRange"
type SectionType = "duration" | "dateRange";
const initialSection: SectionType = from || to ? "dateRange" : "duration";
const [activeSection, setActiveSection] = useState<SectionType>(initialSection);
const [validationError, setValidationError] = useState<string | null>(null);
@@ -418,9 +438,28 @@ export function TimeDropdown({
return !isNaN(value) && value > 0;
})();
// Calculate if the current selection exceeds maxPeriodDays
const exceedsMaxPeriod = (() => {
if (!maxPeriodDays) return false;
if (activeSection === "duration") {
const periodToCheck = selectedPeriod === "custom" ? `${customValue}${customUnit}` : selectedPeriod;
if (!periodToCheck) return false;
return periodToDays(periodToCheck) > maxPeriodDays;
} else {
// For date range, check if fromValue is further back than maxPeriodDays
return dateRangeToDays(fromValue) > maxPeriodDays;
}
})();
const applySelection = useCallback(() => {
setValidationError(null);
if (exceedsMaxPeriod) {
setValidationError(`Your plan allows a maximum of ${maxPeriodDays} days. Upgrade for longer retention.`);
return;
}
if (activeSection === "duration") {
// Validate custom duration
if (selectedPeriod === "custom" && !isCustomDurationValid) {
@@ -498,6 +537,8 @@ export function TimeDropdown({
replace,
onApply,
onValueChange,
exceedsMaxPeriod,
maxPeriodDays
]);
return (
@@ -683,7 +724,7 @@ export function TimeDropdown({
/>
</div>
{/* Quick select date ranges */}
<div className="mt-2 grid grid-cols-3 gap-2" onClick={(e) => e.stopPropagation()}>
<div className="mt-2 grid grid-cols-2 gap-2" onClick={(e) => e.stopPropagation()}>
<QuickDateButton
label="Yesterday"
isActive={selectedQuickDate === "yesterday"}
@@ -702,45 +743,26 @@ export function TimeDropdown({
onClick={() => {
const today = new Date();
setFromValue(startOfDay(today));
setToValue(today);
setToValue(endOfDay(today));
setActiveSection("dateRange");
setValidationError(null);
setSelectedQuickDate("today");
}}
/>
</div>
<div className="mt-2 grid grid-cols-3 gap-2" onClick={(e) => e.stopPropagation()}>
<QuickDateButton
label="This week"
isActive={selectedQuickDate === "thisWeek"}
onClick={() => {
const now = new Date();
setFromValue(startOfWeek(now, { weekStartsOn: 1 }));
setToValue(now);
setToValue(endOfWeek(now, { weekStartsOn: 1 }));
setActiveSection("dateRange");
setValidationError(null);
setSelectedQuickDate("thisWeek");
}}
/>
<QuickDateButton
label="Last weekend"
isActive={selectedQuickDate === "lastWeekend"}
onClick={() => {
const now = new Date();
let saturday: Date;
if (isSaturday(now)) {
saturday = subDays(now, 7);
} else if (isSunday(now)) {
saturday = subDays(now, 8);
} else {
saturday = previousSaturday(now);
}
const sunday = endOfDay(subDays(saturday, -1));
setFromValue(startOfDay(saturday));
setToValue(sunday);
setActiveSection("dateRange");
setValidationError(null);
setSelectedQuickDate("lastWeekend");
}}
/>
<QuickDateButton
label="Last week"
isActive={selectedQuickDate === "lastWeek"}
@@ -753,56 +775,18 @@ export function TimeDropdown({
setSelectedQuickDate("lastWeek");
}}
/>
<QuickDateButton
label="Last weekdays"
isActive={selectedQuickDate === "lastWeekdays"}
onClick={() => {
const lastWeek = subWeeks(new Date(), 1);
const monday = startOfWeek(lastWeek, { weekStartsOn: 1 });
const friday = endOfDay(subDays(monday, -4)); // Monday + 4 days = Friday
setFromValue(startOfDay(monday));
setToValue(friday);
setActiveSection("dateRange");
setValidationError(null);
setSelectedQuickDate("lastWeekdays");
}}
/>
<QuickDateButton
label="Last month"
isActive={selectedQuickDate === "lastMonth"}
onClick={() => {
const lastMonth = subMonths(new Date(), 1);
setFromValue(startOfMonth(lastMonth));
setToValue(endOfMonth(lastMonth));
setActiveSection("dateRange");
setValidationError(null);
setSelectedQuickDate("lastMonth");
}}
/>
<QuickDateButton
label="This month"
isActive={selectedQuickDate === "thisMonth"}
onClick={() => {
const now = new Date();
setFromValue(startOfMonth(now));
setToValue(now);
setToValue(endOfMonth(now));
setActiveSection("dateRange");
setValidationError(null);
setSelectedQuickDate("thisMonth");
}}
/>
<QuickDateButton
label="Year to date"
isActive={selectedQuickDate === "yearToDate"}
onClick={() => {
const now = new Date();
setFromValue(startOfYear(now));
setToValue(now);
setActiveSection("dateRange");
setValidationError(null);
setSelectedQuickDate("yearToDate");
}}
/>
</div>
{validationError && activeSection === "dateRange" && (
<Paragraph variant="extra-small" className="mt-2 text-error">
@@ -812,6 +796,17 @@ export function TimeDropdown({
</div>
</div>
{/* Upgrade callout when exceeding maxPeriodDays */}
{exceedsMaxPeriod && organization && (
<Callout
variant="pricing"
cta={<LinkButton variant="primary/small" to={organizationBillingPath({ slug: organization.slug })}>Upgrade</LinkButton>}
className="items-center"
>
{simplur`Your plan allows a maximum of ${maxPeriodDays} day[|s].`}
</Callout>
)}
{/* Action buttons */}
<div className="flex justify-between gap-1 border-t border-grid-bright px-0 pt-3">
<Button
@@ -839,6 +834,7 @@ export function TimeDropdown({
applySelection();
}}
type="button"
disabled={exceedsMaxPeriod}
>
Apply
</Button>
apps/webapp/app/env.server.ts
+1 -1
View File
@@ -521,7 +521,6 @@ const EnvironmentSchema = z
PROD_USAGE_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(),
CENTS_PER_RUN: z.coerce.number().default(0),
CENTS_PER_QUERY_BYTE_SECOND: z.coerce.number().default(0),
EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"),
RESOURCE_MONITOR_ENABLED: z.string().default("0"),
@@ -1197,6 +1196,7 @@ const EnvironmentSchema = z
QUERY_CLICKHOUSE_MAX_AST_ELEMENTS: z.coerce.number().int().default(4_000_000),
QUERY_CLICKHOUSE_MAX_EXPANDED_AST_ELEMENTS: z.coerce.number().int().default(4_000_000),
QUERY_CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY: z.coerce.number().int().default(0),
QUERY_CLICKHOUSE_MAX_RETURNED_ROWS: z.coerce.number().int().default(10_000),
// Query page concurrency limits
QUERY_DEFAULT_ORG_CONCURRENCY_LIMIT: z.coerce.number().int().default(3),
apps/webapp/app/presenters/v3/QueryPresenter.server.ts
+4
View File
@@ -8,6 +8,8 @@ export type QueryHistoryItem = {
scope: QueryScope;
createdAt: Date;
userName: string | null;
/** AI-generated title summarizing the query */
title: string | null;
/** Time filter settings */
filterPeriod: string | null;
filterFrom: Date | null;
@@ -24,6 +26,7 @@ export class QueryPresenter extends BasePresenter {
id: true,
query: true,
scope: true,
title: true,
createdAt: true,
filterPeriod: true,
filterFrom: true,
@@ -43,6 +46,7 @@ export class QueryPresenter extends BasePresenter {
scope: q.scope.toLowerCase() as QueryScope,
createdAt: q.createdAt,
userName: q.user?.displayName ?? q.user?.name ?? null,
title: q.title,
filterPeriod: q.filterPeriod,
filterFrom: q.filterFrom,
filterTo: q.filterTo,
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx
+1 -2
View File
@@ -48,7 +48,7 @@ LIMIT 20`,
total_cost,
usage_duration,
machine,
created_at
triggered_at
FROM runs
WHERE triggered_at > now() - INTERVAL 7 DAY
ORDER BY total_cost DESC
@@ -79,4 +79,3 @@ export function ExamplesContent({
</div>
);
}
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHelpSidebar.tsx
+59 -30
View File
@@ -36,53 +36,82 @@ export function QueryHelpSidebar({
onValueChange={onTabChange}
className="flex min-h-0 flex-col overflow-hidden pt-1"
>
<ClientTabsList variant="underline" className="mx-3 shrink-0">
<ClientTabsTrigger value="ai" variant="underline" layoutId="query-help-tabs">
<div className="flex items-center gap-0.5">
<AISparkleIcon className="size-4" /> AI
</div>
</ClientTabsTrigger>
<ClientTabsTrigger value="guide" variant="underline" layoutId="query-help-tabs">
Writing TRQL
</ClientTabsTrigger>
<ClientTabsTrigger value="schema" variant="underline" layoutId="query-help-tabs">
Table schema
</ClientTabsTrigger>
<ClientTabsTrigger value="examples" variant="underline" layoutId="query-help-tabs">
Examples
</ClientTabsTrigger>
</ClientTabsList>
<div className="h-fit overflow-x-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
<ClientTabsList variant="underline" className="mx-3 shrink-0">
<ClientTabsTrigger
value="ai"
variant="underline"
layoutId="query-help-tabs"
className="shrink-0"
>
<div className="flex items-center gap-0.5">
<AISparkleIcon className="size-4" /> AI
</div>
</ClientTabsTrigger>
<ClientTabsTrigger
value="guide"
variant="underline"
layoutId="query-help-tabs"
className="shrink-0"
>
Writing TRQL
</ClientTabsTrigger>
<ClientTabsTrigger
value="schema"
variant="underline"
layoutId="query-help-tabs"
className="shrink-0"
>
Table schema
</ClientTabsTrigger>
<ClientTabsTrigger
value="examples"
variant="underline"
layoutId="query-help-tabs"
className="shrink-0"
>
Examples
</ClientTabsTrigger>
</ClientTabsList>
</div>
<ClientTabsContent
value="ai"
className="min-h-0 flex-1 overflow-y-auto p-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
className="min-h-0 flex-1 overflow-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
>
<AITabContent
onQueryGenerated={onQueryGenerated}
onTimeFilterChange={onTimeFilterChange}
getCurrentQuery={getCurrentQuery}
aiFixRequest={aiFixRequest}
/>
<div className="min-w-64 p-3">
<AITabContent
onQueryGenerated={onQueryGenerated}
onTimeFilterChange={onTimeFilterChange}
getCurrentQuery={getCurrentQuery}
aiFixRequest={aiFixRequest}
/>
</div>
</ClientTabsContent>
<ClientTabsContent
value="guide"
className="min-h-0 flex-1 overflow-y-auto p-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
className="min-h-0 flex-1 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
>
<TRQLGuideContent onTryExample={onTryExample} />
<div className="min-w-64 p-3">
<TRQLGuideContent onTryExample={onTryExample} />
</div>
</ClientTabsContent>
<ClientTabsContent
value="schema"
className="min-h-0 flex-1 overflow-y-auto p-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
className="min-h-0 flex-1 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
>
<TableSchemaContent />
<div className="min-w-64 p-3">
<TableSchemaContent />
</div>
</ClientTabsContent>
<ClientTabsContent
value="examples"
className="min-h-0 flex-1 overflow-y-auto p-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
className="min-h-0 flex-1 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
>
<ExamplesContent onTryExample={onTryExample} />
<div className="min-w-64 p-3">
<ExamplesContent onTryExample={onTryExample} />
</div>
</ClientTabsContent>
</ClientTabs>
</div>
);
}
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/QueryHistoryPopover.tsx
+23 -11
View File
@@ -36,9 +36,14 @@ const SQL_KEYWORDS = [
];
function highlightSQL(query: string): React.ReactNode[] {
// Normalize whitespace for display (let CSS line-clamp handle truncation)
const normalized = query.replace(/\s+/g, " ").slice(0, 200);
const suffix = "";
// Normalize: collapse multiple spaces/tabs to single space, but preserve newlines
// Then trim each line and limit total length
const normalized = query
.split("\n")
.map((line) => line.replace(/[ \t]+/g, " ").trim())
.filter((line) => line.length > 0)
.join("\n")
.slice(0, 500);
// Create a regex pattern that matches keywords as whole words (case insensitive)
const keywordPattern = new RegExp(
@@ -69,10 +74,6 @@ function highlightSQL(query: string): React.ReactNode[] {
parts.push(normalized.slice(lastIndex));
}
if (suffix) {
parts.push(suffix);
}
return parts;
}
@@ -118,10 +119,21 @@ export function QueryHistoryPopover({
}}
className="flex w-full items-center gap-2 rounded-sm px-2 py-2 outline-none transition-colors focus-custom hover:bg-charcoal-900"
>
<div className="flex flex-1 flex-col items-start overflow-hidden">
<p className="line-clamp-2 w-full break-words text-left font-mono text-xs text-[#9b99ff]">
{highlightSQL(item.query)}
</p>
<div className="flex flex-1 flex-col items-start gap-0.5 overflow-hidden">
{item.title ? (
<>
<p className="w-full truncate text-left text-sm font-medium text-text-bright">
{item.title}
</p>
<p className="line-clamp-4 w-full whitespace-pre-wrap text-left font-mono text-xs text-text-dimmed">
{highlightSQL(item.query)}
</p>
</>
) : (
<p className="line-clamp-4 w-full whitespace-pre-wrap text-left font-mono text-xs text-[#9b99ff]">
{highlightSQL(item.query)}
</p>
)}
<div className="flex items-center gap-1.5 text-xs text-text-dimmed">
<span className="capitalize">{item.scope}</span>
{valueLabel && <span>· {valueLabel}</span>}
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/route.tsx
+419 -123
View File
@@ -1,13 +1,23 @@
import { ArrowDownTrayIcon, ArrowsPointingInIcon, ArrowsPointingOutIcon, ArrowTrendingUpIcon, ClipboardIcon } from "@heroicons/react/20/solid";
import type { OutputColumnMetadata, WhereClauseFallback } from "@internal/clickhouse";
import {
ArrowDownTrayIcon,
ArrowsPointingOutIcon,
ArrowTrendingUpIcon,
ClipboardIcon,
TableCellsIcon,
} from "@heroicons/react/20/solid";
import type { OutputColumnMetadata } from "@internal/clickhouse";
import { type WhereClauseCondition } from "@internal/tsql";
import { useFetcher } from "@remix-run/react";
import {
redirect,
type ActionFunctionArgs,
type LoaderFunctionArgs,
} from "@remix-run/server-runtime";
import parse from "parse-duration";
import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from "react";
import { flushSync } from "react-dom";
import { typedjson, useTypedFetcher, useTypedLoaderData } from "remix-typedjson";
import simplur from "simplur";
import { z } from "zod";
import { AISparkleIcon } from "~/assets/icons/AISparkleIcon";
import { AlphaTitle } from "~/components/AlphaBadge";
@@ -21,9 +31,7 @@ import { autoFormatSQL, TSQLEditor } from "~/components/code/TSQLEditor";
import { TSQLResultsTable } from "~/components/code/TSQLResultsTable";
import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
import { PageBody, PageContainer } from "~/components/layout/AppLayout";
import { TimeFilter, timeFilters } from "~/components/runs/v3/SharedFilters";
import { useSearchParams } from "~/hooks/useSearchParam";
import { Button } from "~/components/primitives/Buttons";
import { Button, LinkButton } from "~/components/primitives/Buttons";
import { Callout } from "~/components/primitives/Callout";
import { Card } from "~/components/primitives/charts/Card";
import {
@@ -32,6 +40,7 @@ import {
ClientTabsList,
ClientTabsTrigger,
} from "~/components/primitives/ClientTabs";
import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog";
import { Header3 } from "~/components/primitives/Headers";
import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
import { Paragraph } from "~/components/primitives/Paragraph";
@@ -49,27 +58,30 @@ import {
import { Select, SelectItem } from "~/components/primitives/Select";
import { Spinner } from "~/components/primitives/Spinner";
import { Switch } from "~/components/primitives/Switch";
import { SimpleTooltip } from "~/components/primitives/Tooltip";
import { TimeFilter, timeFilters } from "~/components/runs/v3/SharedFilters";
import { prisma } from "~/db.server";
import { env } from "~/env.server";
import { useEnvironment } from "~/hooks/useEnvironment";
import { useOrganization } from "~/hooks/useOrganizations";
import { useProject } from "~/hooks/useProject";
import { useSearchParams } from "~/hooks/useSearchParam";
import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { QueryPresenter, type QueryHistoryItem } from "~/presenters/v3/QueryPresenter.server";
import type { action as titleAction } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title";
import { getLimit } from "~/services/platform.v3.server";
import { executeQuery, type QueryScope } from "~/services/queryService.server";
import { requireUser } from "~/services/session.server";
import { downloadFile, rowsToCSV, rowsToJSON } from "~/utils/dataExport";
import { EnvironmentParamSchema } from "~/utils/pathBuilder";
import { EnvironmentParamSchema, organizationBillingPath } from "~/utils/pathBuilder";
import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server";
import { querySchemas } from "~/v3/querySchemas";
import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
import { QueryHelpSidebar } from "./QueryHelpSidebar";
import { QueryHistoryPopover } from "./QueryHistoryPopover";
import type { AITimeFilter } from "./types";
import { formatQueryStats } from "./utils";
import { requireUser } from "~/services/session.server";
import parse from "parse-duration";
import { SimpleTooltip } from "~/components/primitives/Tooltip";
import { Dialog, DialogContent, DialogHeader, DialogPortal, DialogTrigger } from "~/components/primitives/Dialog";
import { DialogOverlay } from "@radix-ui/react-dialog";
import { formatDurationNanoseconds } from "@trigger.dev/core/v3";
/** Convert a Date or ISO string to ISO string format */
function toISOString(value: Date | string): string {
@@ -159,12 +171,21 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
return typedjson({
defaultQuery,
defaultPeriod: await getDefaultPeriod(project.organizationId),
history,
isAdmin,
maxRows: env.QUERY_CLICKHOUSE_MAX_RETURNED_ROWS,
});
};
const DEFAULT_PERIOD = "7d";
async function getDefaultPeriod(organizationId: string): Promise<string> {
const idealDefaultPeriodDays = 7;
const maxQueryPeriod = await getLimit(organizationId, "queryPeriodDays", 30);
if (maxQueryPeriod < idealDefaultPeriodDays) {
return `${maxQueryPeriod}d`;
}
return `${idealDefaultPeriodDays}d`;
}
const ActionSchema = z.object({
query: z.string().min(1, "Query is required"),
@@ -193,8 +214,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
columns: null,
stats: null,
hiddenColumns: null,
reachedMaxRows: null,
explainOutput: null,
generatedSql: null,
periodClipped: null,
},
{ status: 403 }
);
@@ -209,8 +232,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
columns: null,
stats: null,
hiddenColumns: null,
reachedMaxRows: null,
explainOutput: null,
generatedSql: null,
periodClipped: null,
},
{ status: 404 }
);
@@ -225,8 +250,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
columns: null,
stats: null,
hiddenColumns: null,
reachedMaxRows: null,
explainOutput: null,
generatedSql: null,
periodClipped: null,
},
{ status: 404 }
);
@@ -250,8 +277,10 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
columns: null,
stats: null,
hiddenColumns: null,
reachedMaxRows: null,
explainOutput: null,
generatedSql: null,
periodClipped: null,
},
{ status: 400 }
);
@@ -263,31 +292,54 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
const explain = explainParam === "true" && isAdmin;
// Build time filter fallback for triggered_at column
const defaultPeriod = await getDefaultPeriod(project.organizationId);
const timeFilter = timeFilters({
period: period ?? undefined,
from: from ?? undefined,
to: to ?? undefined,
defaultPeriod: DEFAULT_PERIOD,
defaultPeriod,
});
let triggeredAtFallback: WhereClauseFallback;
if (timeFilter.from && timeFilter.to) {
// Both from and to specified - use BETWEEN
triggeredAtFallback = { op: "between", low: timeFilter.from, high: timeFilter.to };
} else if (timeFilter.from) {
// Only from specified
triggeredAtFallback = { op: "gte", value: timeFilter.from };
} else if (timeFilter.to) {
// Only to specified
triggeredAtFallback = { op: "lte", value: timeFilter.to };
} else {
// Calculate the effective "from" date the user is requesting (for period clipping check)
// This is null only when the user specifies just a "to" date (rare case)
let requestedFromDate: Date | null = null;
if (timeFilter.from) {
requestedFromDate = new Date(timeFilter.from);
} else if (!timeFilter.to) {
// Period specified (or default) - calculate from now
const periodMs = parse(timeFilter.period ?? DEFAULT_PERIOD) ?? 7 * 24 * 60 * 60 * 1000;
triggeredAtFallback = { op: "gte", value: new Date(Date.now() - periodMs) };
const periodMs = parse(timeFilter.period ?? defaultPeriod) ?? 7 * 24 * 60 * 60 * 1000;
requestedFromDate = new Date(Date.now() - periodMs);
}
// Build the fallback WHERE condition based on what the user specified
let triggeredAtFallback: WhereClauseCondition;
if (timeFilter.from && timeFilter.to) {
triggeredAtFallback = { op: "between", low: timeFilter.from, high: timeFilter.to };
} else if (timeFilter.from) {
triggeredAtFallback = { op: "gte", value: timeFilter.from };
} else if (timeFilter.to) {
triggeredAtFallback = { op: "lte", value: timeFilter.to };
} else {
triggeredAtFallback = { op: "gte", value: requestedFromDate! };
}
const maxQueryPeriod = await getLimit(project.organizationId, "queryPeriodDays", 30);
const maxQueryPeriodDate = new Date(Date.now() - maxQueryPeriod * 24 * 60 * 60 * 1000);
// Check if the requested time period exceeds the plan limit
const periodClipped = requestedFromDate !== null && requestedFromDate < maxQueryPeriodDate;
// Force tenant isolation and time period limits
const enforcedWhereClause = {
organization_id: { op: "eq", value: project.organizationId },
project_id:
scope === "project" || scope === "environment" ? { op: "eq", value: project.id } : undefined,
environment_id: scope === "environment" ? { op: "eq", value: environment.id } : undefined,
triggered_at: { op: "gte", value: maxQueryPeriodDate },
} satisfies Record<string, WhereClauseCondition | undefined>;
try {
const [error, result] = await executeQuery({
const [error, result, queryId] = await executeQuery({
name: "query-page",
query,
schema: z.record(z.any()),
@@ -298,6 +350,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
projectId: project.id,
environmentId: environment.id,
explain,
enforcedWhereClause,
whereClauseFallback: {
triggered_at: triggeredAtFallback,
},
@@ -323,8 +376,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
columns: null,
stats: null,
hiddenColumns: null,
reachedMaxRows: null,
explainOutput: null,
generatedSql: null,
queryId: null,
periodClipped: null,
},
{ status: 400 }
);
@@ -336,8 +392,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
columns: result.columns,
stats: result.stats,
hiddenColumns: result.hiddenColumns ?? null,
reachedMaxRows: result.reachedMaxRows,
explainOutput: result.explainOutput ?? null,
generatedSql: result.generatedSql ?? null,
queryId,
periodClipped: periodClipped ? maxQueryPeriod : null,
});
} catch (err) {
const errorMessage = err instanceof Error ? err.message : "Unknown error executing query";
@@ -348,8 +407,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
columns: null,
stats: null,
hiddenColumns: null,
reachedMaxRows: null,
explainOutput: null,
generatedSql: null,
queryId: null,
periodClipped: null,
},
{ status: 500 }
);
@@ -368,18 +430,45 @@ interface QueryEditorFormHandle {
const QueryEditorForm = forwardRef<
QueryEditorFormHandle,
{
defaultPeriod: string;
defaultQuery: string;
defaultScope: QueryScope;
defaultTimeFilter?: { period?: string; from?: string; to?: string };
history: QueryHistoryItem[];
fetcher: ReturnType<typeof useTypedFetcher<typeof action>>;
isAdmin: boolean;
onQuerySubmit?: () => void;
onHistorySelected?: (item: QueryHistoryItem) => void;
}
>(function QueryEditorForm({ defaultQuery, defaultScope, defaultTimeFilter, history, fetcher, isAdmin }, ref) {
>(function QueryEditorForm(
{
defaultPeriod,
defaultQuery,
defaultScope,
defaultTimeFilter,
history,
fetcher,
isAdmin,
onQuerySubmit,
onHistorySelected,
},
ref
) {
const isLoading = fetcher.state === "submitting" || fetcher.state === "loading";
const [query, setQuery] = useState(defaultQuery);
const [scope, setScope] = useState<QueryScope>(defaultScope);
const formRef = useRef<HTMLFormElement>(null);
const prevFetcherState = useRef(fetcher.state);
const plan = useCurrentPlan();
const maxPeriodDays = plan?.v3Subscription?.plan?.limits?.queryPeriodDays?.number;
// Notify parent when query is submitted (for title generation)
useEffect(() => {
if (prevFetcherState.current !== "submitting" && fetcher.state === "submitting") {
onQuerySubmit?.();
}
prevFetcherState.current = fetcher.state;
}, [fetcher.state, onQuerySubmit]);
// Get time filter values - initialize from props (which may come from history)
const [period, setPeriod] = useState<string | undefined>(defaultTimeFilter?.period);
@@ -406,18 +495,23 @@ const QueryEditorForm = forwardRef<
[query]
);
const handleHistorySelected = useCallback((item: QueryHistoryItem) => {
setQuery(item.query);
setScope(item.scope);
// Apply time filter from history item
// Note: filterFrom/filterTo might be Date objects or ISO strings depending on serialization
setPeriod(item.filterPeriod ?? undefined);
setFrom(item.filterFrom ? toISOString(item.filterFrom) : undefined);
setTo(item.filterTo ? toISOString(item.filterTo) : undefined);
}, []);
const handleHistorySelected = useCallback(
(item: QueryHistoryItem) => {
setQuery(item.query);
setScope(item.scope);
// Apply time filter from history item
// Note: filterFrom/filterTo might be Date objects or ISO strings depending on serialization
setPeriod(item.filterPeriod ?? undefined);
setFrom(item.filterFrom ? toISOString(item.filterFrom) : undefined);
setTo(item.filterTo ? toISOString(item.filterTo) : undefined);
// Notify parent about history selection (for title)
onHistorySelected?.(item);
},
[onHistorySelected]
);
return (
<div className="flex flex-col gap-2 bg-charcoal-900 pb-2">
<div className="flex h-full flex-col gap-2 bg-charcoal-900 pb-2">
<TSQLEditor
defaultValue={query}
onChange={setQuery}
@@ -425,10 +519,13 @@ const QueryEditorForm = forwardRef<
linterEnabled={true}
showCopyButton={true}
showClearButton={true}
minHeight="200px"
className="min-h-[200px]"
className="min-h-0 flex-1"
/>
<fetcher.Form ref={formRef} method="post" className="flex items-center justify-between gap-2 px-2">
<fetcher.Form
ref={formRef}
method="post"
className="flex items-center justify-between gap-2 px-2"
>
<input type="hidden" name="query" value={query} />
<input type="hidden" name="scope" value={scope} />
{/* Pass time filter values to action */}
@@ -468,14 +565,16 @@ const QueryEditorForm = forwardRef<
</Select>
{queryHasTriggeredAt ? (
<SimpleTooltip
button={<Button variant="tertiary/small" disabled={true} type="button">
Set in query
</Button>}
button={
<Button variant="tertiary/small" disabled={true} type="button">
Set in query
</Button>
}
content="Your query includes a WHERE clause with triggered_at so this filter is disabled."
/>
) : (
<TimeFilter
defaultPeriod={DEFAULT_PERIOD}
defaultPeriod={defaultPeriod}
labelName="Triggered"
hideLabel
period={period}
@@ -492,6 +591,7 @@ const QueryEditorForm = forwardRef<
fetcher.submit(formRef.current);
}
}}
maxPeriodDays={maxPeriodDays}
/>
)}
<Button
@@ -510,22 +610,28 @@ const QueryEditorForm = forwardRef<
});
export default function Page() {
const { defaultQuery, history, isAdmin } = useTypedLoaderData<typeof loader>();
const { defaultPeriod, defaultQuery, history, isAdmin, maxRows } =
useTypedLoaderData<typeof loader>();
const fetcher = useTypedFetcher<typeof action>();
const results = fetcher.data;
const { replace: replaceSearchParams } = useSearchParams();
const organization = useOrganization();
const project = useProject();
const environment = useEnvironment();
// Use most recent history item if available, otherwise fall back to defaults
const initialQuery = history.length > 0 ? history[0].query : defaultQuery;
const initialScope: QueryScope = history.length > 0 ? history[0].scope : "environment";
const initialTimeFilter = history.length > 0
? {
period: history[0].filterPeriod ?? undefined,
// Note: filterFrom/filterTo might be Date objects or ISO strings depending on serialization
from: history[0].filterFrom ? toISOString(history[0].filterFrom) : undefined,
to: history[0].filterTo ? toISOString(history[0].filterTo) : undefined,
}
: undefined;
const initialTimeFilter =
history.length > 0
? {
period: history[0].filterPeriod ?? undefined,
// Note: filterFrom/filterTo might be Date objects or ISO strings depending on serialization
from: history[0].filterFrom ? toISOString(history[0].filterFrom) : undefined,
to: history[0].filterTo ? toISOString(history[0].filterTo) : undefined,
}
: undefined;
const editorRef = useRef<QueryEditorFormHandle>(null);
const [prettyFormatting, setPrettyFormatting] = useState(true);
@@ -534,6 +640,53 @@ export default function Page() {
const [sidebarTab, setSidebarTab] = useState<string>("ai");
const [aiFixRequest, setAiFixRequest] = useState<{ prompt: string; key: number } | null>(null);
// Title generation state
const titleFetcher = useFetcher<typeof titleAction>();
const isTitleLoading = titleFetcher.state !== "idle";
const generatedTitle = titleFetcher.data?.title;
const [historyTitle, setHistoryTitle] = useState<string | null>(
history.length > 0 ? history[0].title ?? null : null
);
// Effective title: history title takes precedence, then generated
const queryTitle = historyTitle ?? generatedTitle ?? null;
// Track whether we should generate a title for the current results
const [shouldGenerateTitle, setShouldGenerateTitle] = useState(false);
// Trigger title generation when query succeeds (only for new queries, not history)
useEffect(() => {
if (
results?.rows &&
!results.error &&
results.queryId &&
shouldGenerateTitle &&
!historyTitle &&
titleFetcher.state === "idle"
) {
const currentQuery = editorRef.current?.getQuery();
if (currentQuery) {
titleFetcher.submit(
{ query: currentQuery, queryId: results.queryId },
{
method: "POST",
action: `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/query/ai-title`,
encType: "application/json",
}
);
setShouldGenerateTitle(false);
}
}
}, [
results,
shouldGenerateTitle,
historyTitle,
titleFetcher,
organization.slug,
project.slug,
environment.slug,
]);
const handleTryFixError = useCallback((errorMessage: string) => {
setSidebarTab("ai");
setAiFixRequest((prev) => ({
@@ -576,6 +729,18 @@ export default function Page() {
setChartConfig(config);
}, []);
// Handle query submission - prepare for title generation
const handleQuerySubmit = useCallback(() => {
setHistoryTitle(null); // Clear history title when running a new query
setShouldGenerateTitle(true); // Enable title generation for new results
}, []);
// Handle history selection - use existing title if available
const handleHistorySelected = useCallback((item: QueryHistoryItem) => {
setHistoryTitle(item.title ?? null);
setShouldGenerateTitle(false); // Don't generate title for history items
}, []);
return (
<PageContainer>
<NavBar>
@@ -584,23 +749,38 @@ export default function Page() {
<PageBody scrollable={false}>
<ResizablePanelGroup orientation="horizontal" className="h-full max-h-full bg-charcoal-800">
<ResizablePanel id="query-main" className="h-full">
<div className="grid h-full grid-rows-[auto_1fr] overflow-hidden">
<ResizablePanelGroup orientation="vertical" className="h-full overflow-hidden">
{/* Query editor - isolated component to prevent re-renders */}
<QueryEditorForm
ref={editorRef}
defaultQuery={initialQuery}
defaultScope={initialScope}
defaultTimeFilter={initialTimeFilter}
history={history}
fetcher={fetcher}
isAdmin={isAdmin}
/>
<ResizablePanel
id="query-editor"
min="100px"
default="300px"
className="overflow-hidden"
>
<QueryEditorForm
ref={editorRef}
defaultPeriod={defaultPeriod}
defaultQuery={initialQuery}
defaultScope={initialScope}
defaultTimeFilter={initialTimeFilter}
history={history}
fetcher={fetcher}
isAdmin={isAdmin}
onQuerySubmit={handleQuerySubmit}
onHistorySelected={handleHistorySelected}
/>
</ResizablePanel>
<ResizableHandle id="query-editor-handle" />
{/* Results */}
<div className="grid max-h-full grid-rows-[1fr] overflow-hidden border-t border-grid-dimmed bg-charcoal-800">
<ResizablePanel
id="query-results"
min="200px"
className="overflow-hidden bg-charcoal-800"
>
<ClientTabs
value={resultsView}
onValueChange={(v) => setResultsView(v as "table" | "graph")}
className="grid min-h-0 grid-rows-[auto_1fr] overflow-hidden"
className="grid h-full max-h-full min-h-0 grid-rows-[auto_1fr] overflow-hidden"
>
<ClientTabsList
variant="underline"
@@ -620,12 +800,22 @@ export default function Page() {
{results?.rows ? (
<div className="flex flex-1 items-center justify-end gap-2 overflow-hidden border-b border-grid-dimmed pl-3">
<div className="flex items-center gap-2 overflow-hidden truncate">
<span className="text-xs text-text-dimmed">
{results?.rows?.length ? `${results.rows.length} Results` : "Results"}
</span>
{results.reachedMaxRows ? (
<SimpleTooltip
buttonClassName="text-warning text-xs"
button={`${results.rows.length.toLocaleString()} Results`}
content={`Results are limited to ${maxRows.toLocaleString()} rows maximum.`}
/>
) : (
<span className="text-xs text-text-dimmed">
{results.rows.length > 0
? `${results.rows.length.toLocaleString()} Results`
: "Results"}
</span>
)}
{results?.stats && (
<span className="text-xs text-text-dimmed">
{formatQueryStats(results.stats)}
{formatDurationNanoseconds(parseInt(results.stats.elapsed_ns, 10))}
</span>
)}
</div>
@@ -687,19 +877,32 @@ export default function Page() {
</div>
</div>
) : results?.rows && results?.columns ? (
<div className="flex h-full flex-col overflow-hidden">
{results.hiddenColumns && results.hiddenColumns.length > 0 && (
<Callout variant="warning" className="m-2 shrink-0 text-sm">
<code>SELECT *</code> doesn't return all columns because it's slow. The
following columns are not shown:{" "}
<span className="font-mono text-xs">
{results.hiddenColumns.join(", ")}
</span>
. Specify them explicitly to include them.
</Callout>
)}
<div className="h-full bg-charcoal-900 p-2">
<Card className="h-full overflow-hidden p-0">
<div
className={`grid h-full max-h-full overflow-hidden bg-charcoal-900 ${
hasQueryResultsCallouts(results.hiddenColumns, results.periodClipped)
? "grid-rows-[auto_1fr]"
: "grid-rows-[1fr]"
}`}
>
<QueryResultsCallouts
hiddenColumns={results.hiddenColumns}
periodClipped={results.periodClipped}
organizationSlug={organization.slug}
/>
<div className="overflow-hidden p-2">
<Card className="h-full overflow-hidden px-0 pb-0">
<Card.Header>
<div className="flex items-center gap-1.5">
<TableCellsIcon className="size-5 text-indigo-500" />
{isTitleLoading ? (
<span className="flex items-center gap-2 text-text-dimmed">
<Spinner className="size-3" /> Generating title...
</span>
) : (
queryTitle ?? "Results"
)}
</div>
</Card.Header>
<Card.Content className="min-h-0 flex-1 overflow-hidden p-0">
<TSQLResultsTable
rows={results.rows}
@@ -718,15 +921,30 @@ export default function Page() {
</ClientTabsContent>
<ClientTabsContent
value="graph"
className="m-0 grid min-h-0 grid-rows-[1fr] overflow-hidden"
className={`m-0 grid h-full max-h-full min-h-0 overflow-hidden bg-charcoal-900 ${
results?.rows &&
results.rows.length > 0 &&
hasQueryResultsCallouts(results.hiddenColumns, results.periodClipped)
? "grid-rows-[auto_1fr]"
: "grid-rows-[1fr]"
}`}
>
{results?.rows && results?.columns && results.rows.length > 0 ? (
<ResultsChart
rows={results.rows}
columns={results.columns}
chartConfig={chartConfig}
onChartConfigChange={handleChartConfigChange}
/>
<>
<QueryResultsCallouts
hiddenColumns={results.hiddenColumns}
periodClipped={results.periodClipped}
organizationSlug={organization.slug}
/>
<ResultsChart
rows={results.rows}
columns={results.columns}
chartConfig={chartConfig}
onChartConfigChange={handleChartConfigChange}
queryTitle={queryTitle}
isTitleLoading={isTitleLoading}
/>
</>
) : (
<Paragraph variant="small" className="p-4 text-text-dimmed">
Run a query to visualize results.
@@ -734,13 +952,15 @@ export default function Page() {
)}
</ClientTabsContent>
</ClientTabs>
</div>
</div>
</ResizablePanel>
</ResizablePanelGroup>
</ResizablePanel>
<ResizableHandle id="query-handle" />
<ResizablePanel
id="query-help"
min="200px"
collapsible
collapsedSize="20px"
default="400px"
max="500px"
className="w-full"
@@ -844,52 +1064,128 @@ function ScopeItem({ scope }: { scope: QueryScope }) {
}
}
function QueryResultsCallouts({
hiddenColumns,
periodClipped,
organizationSlug,
}: {
hiddenColumns: string[] | null | undefined;
periodClipped: number | null | undefined;
organizationSlug: string;
}) {
const hasCallouts = (hiddenColumns && hiddenColumns.length > 0) || periodClipped;
if (!hasCallouts) {
return null;
}
return (
<div className="flex flex-col gap-2 px-2 pt-2">
{hiddenColumns && hiddenColumns.length > 0 && (
<Callout variant="warning" className="shrink-0 text-sm">
<code>SELECT *</code> doesn't return all columns because it's slow. The following columns
are not shown: <span className="font-mono text-xs">{hiddenColumns.join(", ")}</span>.
Specify them explicitly to include them.
</Callout>
)}
{periodClipped && (
<Callout
variant="pricing"
cta={
<LinkButton
variant="primary/small"
to={organizationBillingPath({ slug: organizationSlug })}
>
Upgrade
</LinkButton>
}
className="items-center"
>
{simplur`Results are limited to the last ${periodClipped} day[|s] based on your plan.`}
</Callout>
)}
</div>
);
}
function hasQueryResultsCallouts(
hiddenColumns: string[] | null | undefined,
periodClipped: number | null | undefined
): boolean {
return (hiddenColumns && hiddenColumns.length > 0) || !!periodClipped;
}
function ResultsChart({
rows,
columns,
chartConfig,
onChartConfigChange,
queryTitle,
isTitleLoading,
}: {
rows: Record<string, unknown>[];
columns: OutputColumnMetadata[];
chartConfig: ChartConfiguration;
onChartConfigChange: (config: ChartConfiguration) => void;
queryTitle: string | null;
isTitleLoading: boolean;
}) {
const [isOpen, setIsOpen] = useState(false);
return (
<><ResizablePanelGroup className="h-full overflow-hidden">
<ResizablePanel id="chart-results">
<div className="h-full bg-charcoal-900 p-2 overflow-hidden">
<Card className="h-full">
<Card.Header>
<div className="flex items-center gap-1.5">
<ArrowTrendingUpIcon className="size-5 text-indigo-500" />
Chart
</div>
<Card.Accessory>
<Button variant="minimal/small" LeadingIcon={ArrowsPointingOutIcon} onClick={() => setIsOpen(true)} />
</Card.Accessory>
</Card.Header>
<Card.Content className="h-full flex-1 min-h-0">
<QueryResultsChart rows={rows} columns={columns} config={chartConfig} onViewAllLegendItems={() => setIsOpen(true)} />
</Card.Content>
</Card>
const titleContent = isTitleLoading ? (
<span className="flex items-center gap-2 text-text-dimmed">
<Spinner className="size-3" /> Generating title...
</span>
) : (
queryTitle ?? "Chart"
);
</div>
</ResizablePanel>
<ResizableHandle id="chart-split" />
<ResizablePanel id="chart-config" min="50px" default="200px">
<ChartConfigPanel columns={columns} config={chartConfig} onChange={onChartConfigChange} />
</ResizablePanel>
</ResizablePanelGroup>
return (
<>
<ResizablePanelGroup className="overflow-hidden">
<ResizablePanel id="chart-results">
<div className="h-full overflow-hidden bg-charcoal-900 p-2">
<Card className="h-full">
<Card.Header>
<div className="flex items-center gap-1.5">
<ArrowTrendingUpIcon className="size-5 text-indigo-500" />
{titleContent}
</div>
<Card.Accessory>
<Button
variant="minimal/small"
LeadingIcon={ArrowsPointingOutIcon}
onClick={() => setIsOpen(true)}
/>
</Card.Accessory>
</Card.Header>
<Card.Content className="h-full min-h-0 flex-1">
<QueryResultsChart
rows={rows}
columns={columns}
config={chartConfig}
onViewAllLegendItems={() => setIsOpen(true)}
/>
</Card.Content>
</Card>
</div>
</ResizablePanel>
<ResizableHandle id="chart-split" />
<ResizablePanel id="chart-config" min="50px" default="200px">
<ChartConfigPanel columns={columns} config={chartConfig} onChange={onChartConfigChange} />
</ResizablePanel>
</ResizablePanelGroup>
<Dialog open={isOpen} onOpenChange={setIsOpen}>
<DialogContent fullscreen>
<DialogHeader>
Chart
</DialogHeader>
<div className="h-full min-h-0 flex-1 overflow-hidden w-full pt-4">
<QueryResultsChart rows={rows} columns={columns} config={chartConfig} fullLegend={true} />
<DialogHeader>{queryTitle ?? "Chart"}</DialogHeader>
<div className="h-full min-h-0 w-full flex-1 overflow-hidden pt-4">
<QueryResultsChart
rows={rows}
columns={columns}
config={chartConfig}
fullLegend={true}
legendScrollable={true}
/>
</div>
</DialogContent>
</Dialog>
apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/utils.ts
-32
View File
@@ -1,32 +0,0 @@
export function formatQueryStats(stats: {
read_rows: string;
read_bytes: string;
elapsed_ns: string;
byte_seconds: string;
}): string {
const readRows = parseInt(stats.read_rows, 10);
const readBytes = parseInt(stats.read_bytes, 10);
const elapsedNs = parseInt(stats.elapsed_ns, 10);
const byteSeconds = parseFloat(stats.byte_seconds);
const elapsedMs = elapsedNs / 1_000_000;
const formattedTime =
elapsedMs < 1000 ? `${elapsedMs.toFixed(1)}ms` : `${(elapsedMs / 1000).toFixed(2)}s`;
const formattedBytes = formatBytes(readBytes);
return `${readRows.toLocaleString()} rows read · ${formattedBytes} · ${formattedTime} · ${formatBytes(
byteSeconds
)}s`;
}
export function formatBytes(bytes: number): string {
if (bytes === 0) return "0 B";
if (bytes < 0) return "-" + formatBytes(-bytes);
const k = 1024;
const sizes = ["B", "KB", "MB", "GB"];
const i = Math.max(
0,
Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length - 1)
);
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
}
apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query.ai-title.tsx
+81
View File
@@ -0,0 +1,81 @@
import { openai } from "@ai-sdk/openai";
import { json, type ActionFunctionArgs } from "@remix-run/server-runtime";
import { tryCatch } from "@trigger.dev/core/utils";
import { z } from "zod";
import { prisma } from "~/db.server";
import { env } from "~/env.server";
import { findProjectBySlug } from "~/models/project.server";
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
import { requireUserId } from "~/services/session.server";
import { EnvironmentParamSchema } from "~/utils/pathBuilder";
import { AIQueryTitleService } from "~/v3/services/aiQueryTitleService.server";
const RequestSchema = z.object({
query: z.string().min(1, "Query is required"),
queryId: z.string().optional(),
});
export async function action({ request, params }: ActionFunctionArgs) {
const userId = await requireUserId(request);
const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params);
// Parse the request body
const [error, data] = await tryCatch(request.json());
if (error) {
return json({ success: false as const, error: error.message, title: null }, { status: 400 });
}
const submission = RequestSchema.safeParse(data);
if (!submission.success) {
return json(
{ success: false as const, error: "Invalid request data", title: null },
{ status: 400 }
);
}
const project = await findProjectBySlug(organizationSlug, projectParam, userId);
if (!project) {
return json(
{ success: false as const, error: "Project not found", title: null },
{ status: 404 }
);
}
const environment = await findEnvironmentBySlug(project.id, envParam, userId);
if (!environment) {
return json(
{ success: false as const, error: "Environment not found", title: null },
{ status: 404 }
);
}
if (!env.OPENAI_API_KEY) {
return json(
{ success: false as const, error: "OpenAI API key is not configured", title: null },
{ status: 400 }
);
}
const { query, queryId } = submission.data;
const service = new AIQueryTitleService(openai(env.AI_RUN_FILTER_MODEL ?? "gpt-4o-mini"));
const result = await service.generateTitle(query);
if (!result.success) {
return json({ success: false as const, error: result.error, title: null }, { status: 500 });
}
// Strip leading/trailing quotes that AI sometimes adds
const title = result.title.replace(/^["']|["']$/g, "");
// If a queryId was provided, update the CustomerQuery record with the title
if (queryId) {
await prisma.customerQuery.update({
where: { id: queryId, organizationId: project.organizationId },
data: { title },
});
}
return json({ success: true as const, title, error: null });
}
apps/webapp/app/services/queryService.server.ts
+50 -51
View File
@@ -7,7 +7,7 @@ import {
type TSQLQueryResult,
} from "@internal/clickhouse";
import type { CustomerQuerySource } from "@trigger.dev/database";
import type { TableSchema } from "@internal/tsql";
import type { TableSchema, WhereClauseCondition } from "@internal/tsql";
import { type z } from "zod";
import { prisma } from "~/db.server";
import { env } from "~/env.server";
@@ -56,17 +56,14 @@ function getDefaultClickhouseSettings(): ClickHouseSettings {
export type ExecuteQueryOptions<TOut extends z.ZodSchema> = Omit<
ExecuteTSQLOptions<TOut>,
"tableSchema" | "organizationId" | "projectId" | "environmentId" | "fieldMappings"
"tableSchema" | "fieldMappings"
> & {
organizationId: string;
projectId?: string;
environmentId?: string;
tableSchema: TableSchema[];
/** The scope of the query - determines tenant isolation */
scope: QueryScope;
/** Organization ID (required) */
organizationId: string;
/** Project ID (required for project/environment scope) */
projectId: string;
/** Environment ID (required for environment scope) */
environmentId: string;
/** History options for saving query to billing/audit */
history?: {
/** Where the query originated from */
@@ -89,18 +86,27 @@ export type ExecuteQueryOptions<TOut extends z.ZodSchema> = Omit<
customOrgConcurrencyLimit?: number;
};
/**
* Extended result type that includes the optional queryId when saved to history
*/
export type ExecuteQueryResult<T> =
| [error: Error, result: null, queryId: null]
| [error: null, result: T, queryId: string | null];
/**
* Execute a TSQL query against ClickHouse with tenant isolation
* Handles building tenant options, field mappings, and optionally saves to history
* Returns [error, result, queryId] where queryId is the CustomerQuery ID if saved to history
*/
export async function executeQuery<TOut extends z.ZodSchema>(
options: ExecuteQueryOptions<TOut>
): Promise<TSQLQueryResult<z.output<TOut>>> {
): Promise<ExecuteQueryResult<Exclude<TSQLQueryResult<z.output<TOut>>[1], null>>> {
const {
scope,
organizationId,
projectId,
environmentId,
enforcedWhereClause,
history,
customOrgConcurrencyLimit,
whereClauseFallback,
@@ -112,39 +118,22 @@ export async function executeQuery<TOut extends z.ZodSchema>(
const orgLimit = customOrgConcurrencyLimit ?? DEFAULT_ORG_CONCURRENCY_LIMIT;
// Acquire concurrency slot
const acquireResult = await queryConcurrencyLimiter.acquire({
key: organizationId,
requestId,
keyLimit: orgLimit,
globalLimit: GLOBAL_CONCURRENCY_LIMIT,
});
const acquireResult = await queryConcurrencyLimiter.acquire({
key: organizationId,
requestId,
keyLimit: orgLimit,
globalLimit: GLOBAL_CONCURRENCY_LIMIT,
});
if (!acquireResult.success) {
const errorMessage =
acquireResult.reason === "key_limit"
? `You've exceeded your query concurrency of ${orgLimit} for this organization. Please try again later.`
: "We're experiencing a lot of queries at the moment. Please try again later.";
return [new QueryError(errorMessage, { query: options.query }), null];
}
if (!acquireResult.success) {
const errorMessage =
acquireResult.reason === "key_limit"
? `You've exceeded your query concurrency of ${orgLimit} for this organization. Please try again later.`
: "We're experiencing a lot of queries at the moment. Please try again later.";
return [new QueryError(errorMessage, { query: options.query }), null, null];
}
try {
// Build tenant IDs based on scope
const tenantOptions: {
organizationId: string;
projectId?: string;
environmentId?: string;
} = {
organizationId,
};
if (scope === "project" || scope === "environment") {
tenantOptions.projectId = projectId;
}
if (scope === "environment") {
tenantOptions.environmentId = environmentId;
}
// Build field mappings for project_ref → project_id and environment_id → slug translation
const projects = await prisma.project.findMany({
where: { organizationId },
@@ -163,18 +152,29 @@ export async function executeQuery<TOut extends z.ZodSchema>(
const result = await executeTSQL(clickhouseClient.reader, {
...baseOptions,
...tenantOptions,
enforcedWhereClause,
fieldMappings,
whereClauseFallback,
clickhouseSettings: {
...getDefaultClickhouseSettings(),
...baseOptions.clickhouseSettings, // Allow caller overrides if needed
},
querySettings: {
maxRows: env.QUERY_CLICKHOUSE_MAX_RETURNED_ROWS,
...baseOptions.querySettings, // Allow caller overrides if needed
},
});
// If query failed, return early with no queryId
if (result[0] !== null) {
return [result[0], null, null];
}
let queryId: string | null = null;
// If query succeeded and history options provided, save to history
// Skip history for EXPLAIN queries (admin debugging) and when explicitly skipped (e.g., impersonating)
if (result[0] === null && history && !history.skip && !baseOptions.explain) {
if (history && !history.skip && !baseOptions.explain) {
// Check if this query is the same as the last one saved (avoid duplicate history entries)
const lastQuery = await prisma.customerQuery.findFirst({
where: {
@@ -183,7 +183,7 @@ export async function executeQuery<TOut extends z.ZodSchema>(
userId: history.userId ?? null,
},
orderBy: { createdAt: "desc" },
select: { query: true, scope: true, filterPeriod: true, filterFrom: true, filterTo: true },
select: { id: true, query: true, scope: true, filterPeriod: true, filterFrom: true, filterTo: true },
});
const timeFilter = history.timeFilter;
@@ -195,17 +195,15 @@ export async function executeQuery<TOut extends z.ZodSchema>(
lastQuery.filterFrom?.getTime() === (timeFilter?.from?.getTime() ?? undefined) &&
lastQuery.filterTo?.getTime() === (timeFilter?.to?.getTime() ?? undefined);
if (!isDuplicate) {
const stats = result[1].stats;
const byteSeconds = parseFloat(stats.byte_seconds) || 0;
const costInCents = byteSeconds * env.CENTS_PER_QUERY_BYTE_SECOND;
await prisma.customerQuery.create({
if (isDuplicate && lastQuery) {
// Return the existing query's ID for duplicate queries
queryId = lastQuery.id;
} else {
const created = await prisma.customerQuery.create({
data: {
query: options.query,
scope: scopeToEnum[scope],
stats: { ...stats },
costInCents,
stats: { ...result[1].stats },
source: history.source,
organizationId,
projectId: scope === "project" || scope === "environment" ? projectId : null,
@@ -216,10 +214,11 @@ export async function executeQuery<TOut extends z.ZodSchema>(
filterTo: history.timeFilter?.to ?? null,
},
});
queryId = created.id;
}
}
return result;
return [null, result[1], queryId];
} finally {
// Always release the concurrency slot
await queryConcurrencyLimiter.release({
apps/webapp/app/v3/querySchemas.ts
+12 -2
View File
@@ -167,10 +167,14 @@ export const runsSchema: TableSchema = {
expression: "if(depth > 0, true, false)",
},
// Useless until we show the user-provided key
idempotency_key: {
name: "idempotency_key",
...column("String", { description: "Idempotency key", example: "user-123-action-456" }),
clickhouseName: "idempotency_key_user",
...column("String", { description: "Idempotency key (available from 4.3.3)", example: "user-123-action-456" }),
},
idempotency_key_scope: {
name: "idempotency_key_scope",
...column("String", { description: "The idempotency key scope determines whether a task should be considered unique within a parent run, a specific attempt, or globally. An empty value means there's no idempotency key set (available from 4.3.3).", example: "run", allowedValues: ["global", "run", "attempt"], }),
},
region: {
name: "region",
@@ -325,6 +329,8 @@ export const runsSchema: TableSchema = {
// Output & error (JSON columns)
// For JSON columns, NULL checks are transformed to check for empty object '{}'
// So `error IS NULL` becomes `error = '{}'` and `error IS NOT NULL` becomes `error != '{}'`
// textColumn uses the pre-materialized text columns for better performance
// dataPrefix handles the internal {"data": ...} wrapper transparently
output: {
name: "output",
...column("JSON", {
@@ -332,6 +338,8 @@ export const runsSchema: TableSchema = {
example: '{"result": "success"}',
}),
nullValue: "'{}'", // Transform NULL checks to compare against empty object
textColumn: "output_text", // Use output_text for full JSON value queries
dataPrefix: "data", // Internal data is wrapped in {"data": ...}
},
error: {
name: "error",
@@ -341,6 +349,8 @@ export const runsSchema: TableSchema = {
example: '{"message": "Task failed"}',
}),
nullValue: "'{}'", // Transform NULL checks to compare against empty object
textColumn: "error_text", // Use error_text for full JSON value queries
dataPrefix: "data", // Internal data is wrapped in {"data": ...}
},
// Tags & versions
apps/webapp/app/v3/services/aiQueryTitleService.server.ts
+71
View File
@@ -0,0 +1,71 @@
import { openai } from "@ai-sdk/openai";
import { generateText, type LanguageModelV1 } from "ai";
import { env } from "~/env.server";
/**
* Result type for title generation
*/
export type AIQueryTitleResult =
| { success: true; title: string }
| { success: false; error: string };
/**
* Service for generating concise titles for SQL queries using AI
*/
export class AIQueryTitleService {
constructor(private readonly model: LanguageModelV1 = openai("gpt-4o-mini")) {}
/**
* Generate a concise title for a SQL query
*/
async generateTitle(query: string): Promise<AIQueryTitleResult> {
if (!env.OPENAI_API_KEY) {
return { success: false, error: "OpenAI API key is not configured" };
}
try {
const result = await generateText({
model: this.model,
system: `You are a helpful assistant that generates concise titles for SQL queries.
Your task is to create a short, descriptive title (5-10 words) that summarizes what the query does.
Guidelines:
- Focus on the main purpose/intent of the query
- Use plain language, not technical SQL terms
- Start with an action verb when appropriate (e.g., "Count", "List", "Show", "Find")
- Be specific about what data is being retrieved
- Do not include quotes around the title
- Do not include punctuation at the end
Examples:
- "Failed runs by hour over 7 days"
- "Top 50 most expensive task runs"
- "Run counts grouped by status"
- "Average execution time by task"
- "Recent runs with errors"`,
prompt: `Generate a concise title for this SQL query:\n\n${query}`,
maxTokens: 50,
experimental_telemetry: {
isEnabled: true,
metadata: {
feature: "ai-query-title",
},
},
});
const title = result.text.trim();
if (!title) {
return { success: false, error: "No title generated" };
}
return { success: true, title };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Failed to generate title",
};
}
}
}
apps/webapp/package.json
+1 -1
View File
@@ -121,7 +121,7 @@
"@trigger.dev/core": "workspace:*",
"@trigger.dev/database": "workspace:*",
"@trigger.dev/otlp-importer": "workspace:*",
"@trigger.dev/platform": "1.0.21",
"@trigger.dev/platform": "1.0.22",
"@trigger.dev/redis-worker": "workspace:*",
"@trigger.dev/sdk": "workspace:*",
"@types/pg": "8.6.6",
internal-packages/clickhouse/schema/014_update_output_error_text_to_extract_data.sql
+45
View File
@@ -0,0 +1,45 @@
-- +goose Up
-- Update the materialized columns to extract the 'data' field if it exists
-- This avoids the {"data": ...} wrapper in the text representation
-- Note: Direct JSON path access (output.data) returns null for nested objects,
-- so we use JSONExtractRaw on the stringified JSON instead
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN output_text String MATERIALIZED if (
toJSONString (output) = '{}',
'',
if (
length (JSONExtractRaw (toJSONString (output), 'data')) > 0,
JSONExtractRaw (toJSONString (output), 'data'),
toJSONString (output)
)
);
-- For error: extract error.data if it exists
ALTER TABLE trigger_dev.task_runs_v2
ADD COLUMN error_text String MATERIALIZED if (
toJSONString (error) = '{}',
'',
if (
length (JSONExtractRaw (toJSONString (error), 'data')) > 0,
JSONExtractRaw (toJSONString (error), 'data'),
toJSONString (error)
)
);
-- Add the indexes
ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_output_text output_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4;
ALTER TABLE trigger_dev.task_runs_v2 ADD INDEX idx_error_text error_text TYPE ngrambf_v1 (3, 131072, 3, 0) GRANULARITY 4;
-- +goose Down
ALTER TABLE trigger_dev.task_runs_v2
DROP INDEX IF EXISTS idx_output_text;
ALTER TABLE trigger_dev.task_runs_v2
DROP INDEX IF EXISTS idx_error_text;
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN IF EXISTS output_text;
ALTER TABLE trigger_dev.task_runs_v2
DROP COLUMN IF EXISTS error_text;
internal-packages/clickhouse/src/client/tsql.ts
+61 -25
View File
@@ -2,7 +2,7 @@
* TSQL Query Execution for ClickHouse
*
* This module provides a safe interface for executing TSQL queries against ClickHouse
* with automatic tenant isolation and SQL injection protection.
* with enforced WHERE clause conditions (tenant isolation + plan limits) and SQL injection protection.
*/
import type { ClickHouseSettings } from "@clickhouse/client";
@@ -14,7 +14,7 @@ import {
type TableSchema,
type QuerySettings,
type FieldMappings,
type WhereClauseFallback,
type WhereClauseCondition
} from "@internal/tsql";
import type { ClickhouseReader, QueryStats } from "./types.js";
import { QueryError } from "./errors.js";
@@ -25,7 +25,7 @@ const logger = new Logger("tsql", "info");
export type { QueryStats };
export type { TableSchema, QuerySettings, FieldMappings, WhereClauseFallback };
export type { TableSchema, QuerySettings, FieldMappings, WhereClauseCondition };
/**
* Options for executing a TSQL query
@@ -37,14 +37,26 @@ export interface ExecuteTSQLOptions<TOut extends z.ZodSchema> {
query: string;
/** The Zod schema for validating output rows */
schema: TOut;
/** The organization ID for tenant isolation (required) */
organizationId: string;
/** The project ID for tenant isolation (optional - omit to query across all projects) */
projectId?: string;
/** The environment ID for tenant isolation (optional - omit to query across all environments) */
environmentId?: string;
/** Schema registry defining allowed tables and columns */
tableSchema: TableSchema[];
/**
* REQUIRED: Conditions always applied at the table level.
* Must include tenant columns (e.g., organization_id) for multi-tenant tables.
* Applied to every table reference including subqueries, CTEs, and JOINs.
*
* @example
* ```typescript
* {
* // Tenant isolation
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* environment_id: { op: "eq", value: "env_789" },
* // Plan-based time limit
* triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
* }
* ```
*/
enforcedWhereClause: Record<string, WhereClauseCondition | undefined>;
/** Optional ClickHouse query settings */
clickhouseSettings?: ClickHouseSettings;
/** Optional TSQL query settings (maxRows, timezone, etc.) */
@@ -78,6 +90,7 @@ export interface ExecuteTSQLOptions<TOut extends z.ZodSchema> {
/**
* Fallback WHERE conditions to apply when the user hasn't filtered on a column.
* Key is the column name, value is the fallback condition.
* These are applied at the AST level (top-level query only).
*
* @example
* ```typescript
@@ -87,7 +100,7 @@ export interface ExecuteTSQLOptions<TOut extends z.ZodSchema> {
* }
* ```
*/
whereClauseFallback?: Record<string, WhereClauseFallback>;
whereClauseFallback?: Record<string, WhereClauseCondition>;
}
/**
@@ -102,6 +115,11 @@ export interface TSQLQuerySuccess<T> {
* Only populated when SELECT * is transformed to core columns only.
*/
hiddenColumns?: string[];
/**
* Whether the result count equals the maxRows limit.
* When true, the results may be truncated and more rows may exist.
*/
reachedMaxRows: boolean;
/**
* The raw EXPLAIN output from ClickHouse.
* Only populated when `explain: true` is passed.
@@ -123,7 +141,7 @@ export type TSQLQueryResult<T> = [QueryError, null] | [null, TSQLQuerySuccess<T>
* Execute a TSQL query against ClickHouse
*
* This function:
* 1. Compiles the TSQL query to ClickHouse SQL (parse, validate, inject tenant guards)
* 1. Compiles the TSQL query to ClickHouse SQL (parse, validate, inject enforced WHERE clauses)
* 2. Executes the query and returns validated results
*
* @example
@@ -132,10 +150,12 @@ export type TSQLQueryResult<T> = [QueryError, null] | [null, TSQLQuerySuccess<T>
* name: "get_task_runs",
* query: "SELECT id, status FROM task_runs WHERE status = 'completed' ORDER BY created_at DESC LIMIT 100",
* schema: z.object({ id: z.string(), status: z.string() }),
* organizationId: "org_123",
* projectId: "proj_456",
* environmentId: "env_789",
* tableSchema: [taskRunsSchema],
* enforcedWhereClause: {
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* environment_id: { op: "eq", value: "env_789" },
* },
* });
* ```
*/
@@ -145,18 +165,22 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
): Promise<TSQLQueryResult<z.output<TOut>>> {
const shouldTransformValues = options.transformValues ?? true;
const isExplain = options.explain ?? false;
const maxRows = options.querySettings?.maxRows;
let generatedSql: string | undefined;
let generatedParams: Record<string, unknown> | undefined;
try {
// 1. Compile the TSQL query to ClickHouse SQL
// Pass maxRows + 1 to fetch one extra row for overflow detection
const compiledSettings = maxRows !== undefined
? { ...options.querySettings, maxRows: maxRows + 1 }
: options.querySettings;
const { sql, params, columns, hiddenColumns } = compileTSQL(options.query, {
organizationId: options.organizationId,
projectId: options.projectId,
environmentId: options.environmentId,
tableSchema: options.tableSchema,
settings: options.querySettings,
enforcedWhereClause: options.enforcedWhereClause,
settings: compiledSettings,
fieldMappings: options.fieldMappings,
whereClauseFallback: options.whereClauseFallback,
});
@@ -231,26 +255,36 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
columns: [],
stats,
hiddenColumns,
reachedMaxRows: false,
explainOutput: combinedOutput,
generatedSql,
},
];
}
// Determine if we exceeded maxRows (we fetched maxRows + 1 to detect overflow)
const reachedMaxRows = maxRows !== undefined && rows !== undefined && rows.length > maxRows;
// Remove the overflow row if we got one (pop is O(1), slice would be O(n))
const finalRows = rows ?? [];
if (reachedMaxRows) {
finalRows.pop();
}
// Build the result, including hiddenColumns if present
const baseResult = { columns, stats, hiddenColumns };
const baseResult = { columns, stats, hiddenColumns, reachedMaxRows };
// 3. Transform result values if enabled
if (shouldTransformValues && rows) {
if (shouldTransformValues && finalRows.length > 0) {
const transformedRows = transformResults(
rows as Record<string, unknown>[],
finalRows as Record<string, unknown>[],
options.tableSchema,
{ fieldMappings: options.fieldMappings }
);
return [null, { rows: transformedRows as z.output<TOut>[], ...baseResult }];
}
return [null, { rows: rows ?? [], ...baseResult }];
return [null, { rows: finalRows as z.output<TOut>[], ...baseResult }];
} catch (error) {
const errorMessage = error instanceof Error ? error.message : "Unknown error";
@@ -284,9 +318,11 @@ export async function executeTSQL<TOut extends z.ZodSchema>(
* name: "get_task_runs",
* query: "SELECT * FROM task_runs LIMIT 10",
* schema: taskRunRowSchema,
* organizationId: "org_123",
* projectId: "proj_456",
* environmentId: "env_789",
* enforcedWhereClause: {
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* environment_id: { op: "eq", value: "env_789" },
* },
* });
* ```
*/
internal-packages/clickhouse/src/index.ts
+1 -1
View File
@@ -54,7 +54,7 @@ export {
type TSQLQuerySuccess,
type QueryStats,
type FieldMappings,
type WhereClauseFallback,
type WhereClauseCondition,
} from "./client/tsql.js";
export type { OutputColumnMetadata } from "@internal/tsql";
internal-packages/clickhouse/src/tsql.test.ts
+158 -92
View File
@@ -106,9 +106,11 @@ describe("TSQL Integration Tests", () => {
name: "test-simple-select",
query: "SELECT run_id, status FROM task_runs",
schema: z.object({ run_id: z.string(), status: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -145,9 +147,11 @@ describe("TSQL Integration Tests", () => {
name: "test-where-clause",
query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED_SUCCESSFULLY'",
schema: z.object({ run_id: z.string(), status: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -197,9 +201,11 @@ describe("TSQL Integration Tests", () => {
name: "test-tenant-isolation-1",
query: "SELECT run_id FROM task_runs",
schema: z.object({ run_id: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -212,9 +218,11 @@ describe("TSQL Integration Tests", () => {
name: "test-tenant-isolation-2",
query: "SELECT run_id FROM task_runs",
schema: z.object({ run_id: z.string() }),
organizationId: "org_tenant2",
projectId: "proj_tenant2",
environmentId: "env_tenant2",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant2" },
project_id: { op: "eq", value: "proj_tenant2" },
environment_id: { op: "eq", value: "env_tenant2" },
},
tableSchema: [taskRunsSchema],
});
@@ -254,9 +262,11 @@ describe("TSQL Integration Tests", () => {
name: "test-cross-tenant-attack",
query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED' OR 1=1",
schema: z.object({ run_id: z.string(), status: z.string() }),
organizationId: "org_attacker",
projectId: "proj_attacker",
environmentId: "env_attacker",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_attacker" },
project_id: { op: "eq", value: "proj_attacker" },
environment_id: { op: "eq", value: "env_attacker" },
},
tableSchema: [taskRunsSchema],
});
@@ -288,9 +298,11 @@ describe("TSQL Integration Tests", () => {
query:
"SELECT status, count(*) as cnt FROM task_runs GROUP BY status ORDER BY cnt DESC, status ASC",
schema: z.object({ status: z.string(), cnt: z.coerce.number() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -325,9 +337,11 @@ describe("TSQL Integration Tests", () => {
name: "test-order-limit",
query: "SELECT run_id FROM task_runs ORDER BY created_at DESC LIMIT 2",
schema: z.object({ run_id: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -347,9 +361,11 @@ describe("TSQL Integration Tests", () => {
name: "test-unknown-table",
query: "SELECT * FROM unknown_table",
schema: z.object({ id: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -378,9 +394,11 @@ describe("TSQL Integration Tests", () => {
name: "test-executor",
query: "SELECT run_id, status FROM task_runs WHERE status = 'PENDING'",
schema: z.object({ run_id: z.string(), status: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
});
expect(error).toBeNull();
@@ -406,9 +424,11 @@ describe("TSQL Integration Tests", () => {
name: "test-injection",
query: "SELECT run_id, status FROM task_runs WHERE status = 'DROP TABLE task_runs'",
schema: z.object({ run_id: z.string(), status: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -438,9 +458,11 @@ describe("TSQL Integration Tests", () => {
query:
"SELECT run_id, status FROM task_runs WHERE status IN ('COMPLETED_SUCCESSFULLY', 'FAILED')",
schema: z.object({ run_id: z.string(), status: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -467,9 +489,11 @@ describe("TSQL Integration Tests", () => {
name: "test-like-query",
query: "SELECT run_id, task_identifier FROM task_runs WHERE task_identifier LIKE 'email%'",
schema: z.object({ run_id: z.string(), task_identifier: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [taskRunsSchema],
});
@@ -530,8 +554,10 @@ describe("TSQL Optional Tenant Filter Tests", () => {
name: "test-cross-project-query",
query: "SELECT run_id FROM task_runs",
schema: z.object({ run_id: z.string() }),
organizationId: "org_multi",
// projectId and environmentId omitted - query across all
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_multi" },
// project_id and environment_id omitted - query across all
},
tableSchema: [taskRunsSchema],
});
@@ -590,9 +616,11 @@ describe("TSQL Optional Tenant Filter Tests", () => {
name: "test-cross-env-query",
query: "SELECT run_id FROM task_runs",
schema: z.object({ run_id: z.string() }),
organizationId: "org_envtest",
projectId: "proj_envtest",
// environmentId omitted - query across all environments
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_envtest" },
project_id: { op: "eq", value: "proj_envtest" },
// environment_id omitted - query across all environments
},
tableSchema: [taskRunsSchema],
});
@@ -649,8 +677,10 @@ describe("TSQL Optional Tenant Filter Tests", () => {
name: "test-org-isolation-1",
query: "SELECT run_id FROM task_runs",
schema: z.object({ run_id: z.string() }),
organizationId: "org_isolation_1",
// projectId and environmentId omitted
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_isolation_1" },
// project_id and environment_id omitted
},
tableSchema: [taskRunsSchema],
});
@@ -663,8 +693,10 @@ describe("TSQL Optional Tenant Filter Tests", () => {
name: "test-org-isolation-2",
query: "SELECT run_id FROM task_runs",
schema: z.object({ run_id: z.string() }),
organizationId: "org_isolation_2",
// projectId and environmentId omitted
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_isolation_2" },
// project_id and environment_id omitted
},
tableSchema: [taskRunsSchema],
});
@@ -706,8 +738,10 @@ describe("TSQL Optional Tenant Filter Tests", () => {
name: "test-or-bypass-attempt",
query: "SELECT run_id, status FROM task_runs WHERE status = 'COMPLETED' OR 1=1",
schema: z.object({ run_id: z.string(), status: z.string() }),
organizationId: "org_attacker",
// No project/env filter - but org filter should still protect
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_attacker" },
// No project/env filter - but org filter should still protect
},
tableSchema: [taskRunsSchema],
});
@@ -751,8 +785,10 @@ describe("TSQL Optional Tenant Filter Tests", () => {
name: "test-executor-optional",
query: "SELECT run_id FROM task_runs",
schema: z.object({ run_id: z.string() }),
organizationId: "org_executor_test",
// projectId and environmentId omitted
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_executor_test" },
// project_id and environment_id omitted
},
});
expect(error).toBeNull();
@@ -839,9 +875,11 @@ describe("TSQL Virtual Column Tests", () => {
execution_duration: z.number().nullable(),
usage_duration_seconds: z.number(),
}),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [virtualColumnSchema],
});
@@ -889,9 +927,11 @@ describe("TSQL Virtual Column Tests", () => {
name: "test-virtual-column-where",
query: "SELECT run_id FROM task_runs WHERE execution_duration > 5000",
schema: z.object({ run_id: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [virtualColumnSchema],
});
@@ -935,9 +975,11 @@ describe("TSQL Virtual Column Tests", () => {
run_id: z.string(),
usage_duration_seconds: z.number(),
}),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [virtualColumnSchema],
});
@@ -977,9 +1019,11 @@ describe("TSQL Virtual Column Tests", () => {
run_id: z.string(),
dur_sec: z.number(),
}),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [virtualColumnSchema],
});
@@ -1013,9 +1057,11 @@ describe("TSQL Virtual Column Tests", () => {
run_id: z.string(),
execution_duration: z.number().nullable(),
}),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [virtualColumnSchema],
});
@@ -1110,9 +1156,11 @@ describe("TSQL Virtual Column Tests", () => {
name: "test-expression-division-where",
query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost > 1.0",
schema: z.object({ run_id: z.string(), invocation_cost: z.number() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [costExpressionSchema],
});
@@ -1153,9 +1201,11 @@ describe("TSQL Virtual Column Tests", () => {
name: "test-expression-gte-where",
query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost >= 1.0",
schema: z.object({ run_id: z.string(), invocation_cost: z.number() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [costExpressionSchema],
});
@@ -1192,9 +1242,11 @@ describe("TSQL Virtual Column Tests", () => {
name: "test-expression-lt-where",
query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost < 1.0",
schema: z.object({ run_id: z.string(), invocation_cost: z.number() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [costExpressionSchema],
});
@@ -1236,9 +1288,11 @@ describe("TSQL Virtual Column Tests", () => {
query:
"SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost BETWEEN 1.0 AND 2.0",
schema: z.object({ run_id: z.string(), invocation_cost: z.number() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [costExpressionSchema],
});
@@ -1282,9 +1336,11 @@ describe("TSQL Virtual Column Tests", () => {
query:
"SELECT run_id FROM task_runs WHERE status = 'COMPLETED_SUCCESSFULLY' AND invocation_cost > 2.0",
schema: z.object({ run_id: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [costExpressionSchema],
});
@@ -1328,9 +1384,11 @@ describe("TSQL Virtual Column Tests", () => {
name: "test-expression-large-integer-where",
query: "SELECT run_id, invocation_cost FROM task_runs WHERE invocation_cost > 100",
schema: z.object({ run_id: z.string(), invocation_cost: z.number() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [costExpressionSchema],
});
@@ -1393,9 +1451,11 @@ describe("Field Mapping Tests", () => {
name: "test-field-mapping-select",
query: "SELECT run_id, project_ref FROM task_runs",
schema: z.object({ run_id: z.string(), project_ref: z.string().nullable() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [fieldMappingSchema],
fieldMappings: {
project: {
@@ -1434,9 +1494,11 @@ describe("Field Mapping Tests", () => {
name: "test-field-mapping-unmapped",
query: "SELECT run_id, project_ref FROM task_runs WHERE run_id = 'run_fm_unmapped'",
schema: z.object({ run_id: z.string(), project_ref: z.string().nullable() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [fieldMappingSchema],
fieldMappings: {
project: {
@@ -1481,9 +1543,11 @@ describe("Field Mapping Tests", () => {
name: "test-field-mapping-where",
query: "SELECT run_id FROM task_runs WHERE project_ref = 'my-project-ref'",
schema: z.object({ run_id: z.string() }),
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
tableSchema: [fieldMappingSchema],
fieldMappings: {
project: {
@@ -1530,7 +1594,9 @@ describe("Field Mapping Tests", () => {
query:
"SELECT run_id FROM task_runs WHERE project_ref IN ('my-project-ref', 'other-project')",
schema: z.object({ run_id: z.string() }),
organizationId: "org_tenant1",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
},
tableSchema: [fieldMappingSchema],
fieldMappings: {
project: {
internal-packages/database/prisma/migrations/20260124203524_customer_query_add_title_remove_cost/migration.sql
+7
View File
@@ -0,0 +1,7 @@
-- AlterTable
ALTER TABLE "CustomerQuery"
ADD COLUMN IF NOT EXISTS "title" TEXT;
-- AlterTable
ALTER TABLE "CustomerQuery"
DROP COLUMN IF EXISTS "costInCents";
internal-packages/database/prisma/schema.prisma
+2 -2
View File
@@ -2452,8 +2452,8 @@ model CustomerQuery {
/// Query execution statistics from ClickHouse
stats Json
/// Cost of the query in cents (for Stripe metering)
costInCents Float @default(0)
/// AI-generated title summarizing the query
title String?
/// Where the query originated from
source CustomerQuerySource @default(DASHBOARD)
internal-packages/tsql/src/index.test.ts
+363 -10
View File
@@ -5,12 +5,12 @@ import {
isColumnReferencedInExpression,
createFallbackExpression,
injectFallbackConditions,
type WhereClauseFallback,
type WhereClauseCondition,
} from "./index.js";
import { column, type TableSchema } from "./query/schema.js";
/**
* Test table schema for whereClauseFallback tests
* Test table schema for enforcedWhereClause tests
*/
const taskRunsSchema: TableSchema = {
name: "task_runs",
@@ -21,6 +21,7 @@ const taskRunsSchema: TableSchema = {
created_at: { name: "created_at", ...column("DateTime64") },
updated_at: { name: "updated_at", ...column("DateTime64") },
time: { name: "time", ...column("DateTime64") },
triggered_at: { name: "triggered_at", ...column("DateTime64") },
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
@@ -32,6 +33,46 @@ const taskRunsSchema: TableSchema = {
},
};
/**
* Test table schema with tenant columns (lookup table with tenant isolation)
*/
const lookupTableSchema: TableSchema = {
name: "lookup_table",
clickhouseName: "trigger_dev.lookup_table",
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
columns: {
id: { name: "id", ...column("String") },
name: { name: "name", ...column("String") },
},
};
/**
* Test table schema WITHOUT tenant columns (e.g., global reference data)
*/
// @ts-expect-error - tenant columns are required but not set
const nonTenantTableSchema: TableSchema = {
name: "reference_data",
clickhouseName: "trigger_dev.reference_data",
// No tenantColumns - this is a global table
columns: {
id: { name: "id", ...column("String") },
value: { name: "value", ...column("String") },
},
};
/**
* Base options with tenant isolation for tests
*/
const baseEnforcedWhereClause: Record<string, WhereClauseCondition> = {
organization_id: { op: "eq", value: "org_test123" },
project_id: { op: "eq", value: "proj_test456" },
environment_id: { op: "eq", value: "env_test789" },
};
describe("isColumnReferencedInExpression", () => {
it("should detect column in simple WHERE clause", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-01-01'");
@@ -126,7 +167,7 @@ describe("createFallbackExpression", () => {
describe("injectFallbackConditions", () => {
it("should inject fallback when column is not in WHERE", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE status = 'completed'");
const fallbacks: Record<string, WhereClauseFallback> = {
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" },
};
@@ -140,7 +181,7 @@ describe("injectFallbackConditions", () => {
it("should NOT inject fallback when column is already in WHERE", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-06-01'");
const fallbacks: Record<string, WhereClauseFallback> = {
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" },
};
@@ -154,7 +195,7 @@ describe("injectFallbackConditions", () => {
it("should inject fallback when query has no WHERE clause", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs LIMIT 10");
const fallbacks: Record<string, WhereClauseFallback> = {
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" },
};
@@ -167,7 +208,7 @@ describe("injectFallbackConditions", () => {
it("should inject multiple fallbacks", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs LIMIT 10");
const fallbacks: Record<string, WhereClauseFallback> = {
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" },
status: { op: "eq", value: "completed" },
};
@@ -182,7 +223,7 @@ describe("injectFallbackConditions", () => {
it("should only inject fallbacks for unreferenced columns", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-06-01'");
const fallbacks: Record<string, WhereClauseFallback> = {
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" }, // Should NOT be injected
status: { op: "eq", value: "completed" }, // Should be injected
};
@@ -197,10 +238,8 @@ describe("injectFallbackConditions", () => {
describe("compileTSQL with whereClauseFallback", () => {
const baseOptions = {
organizationId: "org_test123",
projectId: "proj_test456",
environmentId: "env_test789",
tableSchema: [taskRunsSchema],
enforcedWhereClause: baseEnforcedWhereClause,
};
describe("simple comparison fallbacks", () => {
@@ -474,3 +513,317 @@ describe("compileTSQL with whereClauseFallback", () => {
});
});
describe("compileTSQL with enforcedWhereClause", () => {
describe("validation tests", () => {
it("should throw error when required tenant column is missing", () => {
expect(() =>
compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {}, // Missing organization_id
})
).toThrow("Table 'task_runs' requires 'organization_id' in enforcedWhereClause");
});
it("should throw error when organization_id is missing but other tenant columns are present", () => {
expect(() =>
compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
project_id: { op: "eq", value: "proj_123" },
environment_id: { op: "eq", value: "env_456" },
},
})
).toThrow("Table 'task_runs' requires 'organization_id' in enforcedWhereClause");
});
it("should work with non-tenant table and empty enforcedWhereClause", () => {
const { sql } = compileTSQL("SELECT id FROM reference_data", {
tableSchema: [nonTenantTableSchema],
enforcedWhereClause: {},
});
expect(sql).toContain("SELECT");
expect(sql).toContain("FROM");
});
it("should work with only organization_id (project and env are optional)", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
},
});
expect(sql).toContain("organization_id");
expect(sql).not.toContain("project_id");
expect(sql).not.toContain("environment_id");
});
});
describe("basic functionality", () => {
it("should apply single enforced condition", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
},
});
expect(sql).toContain("equals(");
expect(sql).toContain("organization_id");
});
it("should apply multiple enforced conditions", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
project_id: { op: "eq", value: "proj_456" },
environment_id: { op: "eq", value: "env_789" },
},
});
expect(sql).toContain("organization_id");
expect(sql).toContain("project_id");
expect(sql).toContain("environment_id");
});
it("should apply enforced condition even when user filters on same field", () => {
const { sql } = compileTSQL(
"SELECT id FROM task_runs WHERE triggered_at > '2025-01-01'",
{
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
}
);
// Should have BOTH the user's condition AND the enforced condition
// User's condition: greater(triggered_at, '2025-01-01')
// Enforced condition: greaterOrEquals(triggered_at, '2024-01-01')
const triggeredAtMatches = sql.match(/triggered_at/g) || [];
expect(triggeredAtMatches.length).toBeGreaterThanOrEqual(2);
});
it("should apply different comparison operators", () => {
const { sql: sqlGt } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
time: { op: "gt", value: "2024-01-01" },
},
});
expect(sqlGt).toContain("greater(");
const { sql: sqlLt } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
time: { op: "lt", value: "2024-12-31" },
},
});
expect(sqlLt).toContain("less(");
const { sql: sqlNeq } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
status: { op: "neq", value: "deleted" },
},
});
expect(sqlNeq).toContain("notEquals(");
});
it("should apply BETWEEN condition", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
time: { op: "between", low: "2024-01-01", high: "2024-12-31" },
},
});
expect(sql).toContain("time BETWEEN");
});
it("should handle Date values in enforced conditions", () => {
const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
const { sql, params } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: sevenDaysAgo },
},
});
expect(sql).toContain("triggered_at");
expect(sql).toContain("toDateTime64");
});
});
describe("enforcedWhereClause + whereClauseFallback interaction", () => {
it("should apply both enforced and fallback conditions when user doesn't filter", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
whereClauseFallback: {
status: { op: "eq", value: "completed" },
},
});
// Should have both enforced (triggered_at) and fallback (status)
expect(sql).toContain("triggered_at");
expect(sql).toContain("status");
});
it("should apply enforced but not fallback when user filters on fallback column", () => {
const { sql, params } = compileTSQL(
"SELECT id FROM task_runs WHERE status = 'failed'",
{
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
whereClauseFallback: {
status: { op: "eq", value: "completed" },
},
}
);
// Enforced triggered_at should be applied
expect(sql).toContain("triggered_at");
// User's status = 'failed' should be there (as a parameter)
expect(Object.values(params)).toContain("failed");
// The fallback 'completed' should NOT be applied since user filtered on status
expect(Object.values(params)).not.toContain("completed");
});
it("should apply both enforced and fallback on same field (enforced always, fallback only if not filtered)", () => {
// User doesn't filter on triggered_at, so BOTH enforced AND fallback apply
const { sql } = compileTSQL("SELECT id FROM task_runs WHERE status = 'completed'", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-06-01" }, // Enforced: last 6 months
},
whereClauseFallback: {
triggered_at: { op: "gte", value: "2024-01-01" }, // Fallback: last year
},
});
// Both should be applied (enforced at printer level, fallback at AST level)
const triggeredAtMatches = sql.match(/triggered_at/g) || [];
expect(triggeredAtMatches.length).toBeGreaterThanOrEqual(2);
});
it("should skip fallback but keep enforced when user filters on same field", () => {
const { sql } = compileTSQL(
"SELECT id FROM task_runs WHERE triggered_at > '2025-01-01'",
{
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-06-01" }, // Enforced: always applied
},
whereClauseFallback: {
triggered_at: { op: "gte", value: "2024-01-01" }, // Fallback: skipped since user filtered
},
}
);
// User's condition + enforced should be present
// Fallback should NOT be applied since user filtered on triggered_at
// Count distinct triggered_at conditions
const triggeredAtMatches = sql.match(/triggered_at/g) || [];
// Should be 2: user's condition + enforced condition (NOT 3, no fallback)
expect(triggeredAtMatches.length).toBe(2);
});
});
describe("security tests", () => {
it("should apply enforced conditions to UNION queries", () => {
const { sql } = compileTSQL(
"SELECT id FROM task_runs WHERE status = 'completed' UNION ALL SELECT id FROM task_runs WHERE status = 'failed'",
{
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
}
);
// Both parts of the UNION should have the enforced conditions
const orgMatches = sql.match(/organization_id/g) || [];
expect(orgMatches.length).toBe(2);
const triggeredAtMatches = sql.match(/triggered_at/g) || [];
expect(triggeredAtMatches.length).toBe(2);
});
it("should NOT be bypassable via OR clause", () => {
const { sql } = compileTSQL(
"SELECT id FROM task_runs WHERE status = 'completed' OR 1=1",
{
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
}
);
// The enforced conditions should be ANDed with the entire user WHERE clause
// So the structure should be: (enforced AND enforced AND ...) AND (user_where)
expect(sql).toContain("organization_id");
expect(sql).toContain("triggered_at");
// The 1=1 should be within the user's OR clause, not affecting enforced conditions
});
it("should skip enforced conditions for columns that don't exist in table", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
nonexistent_column: { op: "eq", value: "test" },
},
});
// Should not contain nonexistent_column
expect(sql).not.toContain("nonexistent_column");
// Should still have organization_id
expect(sql).toContain("organization_id");
});
});
describe("edge cases", () => {
it("should handle empty enforced conditions for non-tenant table", () => {
const { sql } = compileTSQL("SELECT id FROM reference_data", {
tableSchema: [nonTenantTableSchema],
enforcedWhereClause: {},
});
expect(sql).toContain("SELECT");
expect(sql).not.toContain("WHERE"); // No WHERE clause needed
});
it("should properly format numeric values", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
},
});
// org_123 should be parameterized, not inlined
expect(sql).toContain("tsql_val_");
});
});
});
internal-packages/tsql/src/index.ts
+54 -51
View File
@@ -23,7 +23,13 @@ import { CompareOperationOp } from "./query/ast.js";
import { SyntaxError as TSQLSyntaxError } from "./query/errors.js";
import { TSQLParseTreeConverter } from "./query/parser.js";
import { printToClickHouse, type PrintResult } from "./query/printer.js";
import { createPrinterContext, type QuerySettings } from "./query/printer_context.js";
import {
createPrinterContext,
type BetweenCondition,
type QuerySettings,
type SimpleComparisonCondition,
type WhereClauseCondition,
} from "./query/printer_context.js";
import { createSchemaRegistry, type FieldMappings, type TableSchema } from "./query/schema.js";
/**
@@ -113,9 +119,12 @@ export {
createPrinterContext,
DEFAULT_QUERY_SETTINGS,
PrinterContext,
type BetweenCondition,
type PrinterContextOptions,
type QueryNotice,
type QuerySettings,
type SimpleComparisonCondition,
type WhereClauseCondition,
} from "./query/printer_context.js";
// Re-export printer
@@ -304,7 +313,7 @@ function createValueExpression(value: Date | string | number): Expression {
/**
* Map fallback operator to CompareOperationOp
*/
function mapFallbackOpToCompareOp(op: SimpleComparisonFallback["op"]): CompareOperationOp {
function mapFallbackOpToCompareOp(op: SimpleComparisonCondition["op"]): CompareOperationOp {
switch (op) {
case "eq":
return CompareOperationOp.Eq;
@@ -330,7 +339,7 @@ function mapFallbackOpToCompareOp(op: SimpleComparisonFallback["op"]): CompareOp
*/
export function createFallbackExpression(
column: string,
fallback: WhereClauseFallback
fallback: WhereClauseCondition
): Expression {
const fieldExpr: Field = {
expression_type: "field",
@@ -367,7 +376,7 @@ export function createFallbackExpression(
*/
export function injectFallbackConditions(
ast: SelectQuery | SelectSetQuery,
fallbacks: Record<string, WhereClauseFallback>
fallbacks: Record<string, WhereClauseCondition>
): SelectQuery | SelectSetQuery {
// Handle SelectSetQuery (UNION, etc.) - apply to each query in the set
if (ast.expression_type === "select_set_query") {
@@ -434,46 +443,31 @@ export function injectFallbackConditions(
};
}
/**
* A simple comparison fallback condition (e.g., column > value)
*/
export interface SimpleComparisonFallback {
/** The comparison operator */
op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte";
/** The value to compare against */
value: Date | string | number;
}
/**
* A between fallback condition (e.g., column BETWEEN low AND high)
*/
export interface BetweenFallback {
/** The between operator */
op: "between";
/** The low bound of the range */
low: Date | string | number;
/** The high bound of the range */
high: Date | string | number;
}
/**
* A WHERE clause fallback condition.
* Used to apply default filters when the user hasn't specified one for a column.
*/
export type WhereClauseFallback = SimpleComparisonFallback | BetweenFallback;
/**
* Options for compiling a TSQL query to ClickHouse SQL
*/
export interface CompileTSQLOptions {
/** The organization ID for tenant isolation (required) */
organizationId: string;
/** The project ID for tenant isolation (optional - omit to query across all projects) */
projectId?: string;
/** The environment ID for tenant isolation (optional - omit to query across all environments) */
environmentId?: string;
/** Schema definitions for allowed tables and columns */
tableSchema: TableSchema[];
/**
* REQUIRED: Conditions always applied at the table level.
* Must include tenant columns (e.g., organization_id) for multi-tenant tables.
* Applied to every table reference including subqueries, CTEs, and JOINs.
*
* @example
* ```typescript
* {
* // Tenant isolation
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* environment_id: { op: "eq", value: "env_789" },
* // Plan-based time limit
* triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
* }
* ```
*/
enforcedWhereClause: Record<string, WhereClauseCondition | undefined>;
/** Optional query settings */
settings?: Partial<QuerySettings>;
/**
@@ -491,6 +485,7 @@ export interface CompileTSQLOptions {
/**
* Fallback WHERE conditions to apply when the user hasn't filtered on a column.
* Key is the column name, value is the fallback condition.
* These are applied at the AST level (top-level query only).
*
* @example
* ```typescript
@@ -505,7 +500,7 @@ export interface CompileTSQLOptions {
* }
* ```
*/
whereClauseFallback?: Record<string, WhereClauseFallback>;
whereClauseFallback?: Record<string, WhereClauseCondition>;
}
/**
@@ -514,24 +509,28 @@ export interface CompileTSQLOptions {
* This function:
* 1. Parses the TSQL query into an AST
* 2. Validates tables and columns against the schema
* 3. Injects tenant isolation WHERE clauses
* 4. Generates parameterized ClickHouse SQL
* 3. Injects enforced WHERE clauses (tenant isolation + plan limits) at printer level
* 4. Optionally injects fallback WHERE conditions at AST level
* 5. Generates parameterized ClickHouse SQL
*
* @param query - The TSQL query string to compile
* @param options - Compilation options including tenant IDs and schema
* @param options - Compilation options including enforcedWhereClause and schema
* @returns The compiled SQL and parameters
* @throws TSQLSyntaxError if the query is invalid
* @throws QueryError if tables/columns are not allowed
* @throws QueryError if tables/columns are not allowed or required tenant columns are missing
*
* @example
* ```typescript
* const { sql, params } = compileTSQL(
* "SELECT * FROM task_runs WHERE status = 'completed' LIMIT 100",
* {
* organizationId: "org_123",
* projectId: "proj_456",
* environmentId: "env_789",
* tableSchema: [taskRunsSchema],
* enforcedWhereClause: {
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* environment_id: { op: "eq", value: "env_789" },
* triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) },
* },
* }
* );
* ```
@@ -540,7 +539,7 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe
// 1. Parse the TSQL query
let ast = parseTSQLSelect(query);
// 2. Inject fallback WHERE conditions if provided
// 2. Inject fallback WHERE conditions if provided (applied at AST level - top-level query only)
if (options.whereClauseFallback && Object.keys(options.whereClauseFallback).length > 0) {
ast = injectFallbackConditions(ast, options.whereClauseFallback);
}
@@ -548,16 +547,20 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe
// 3. Create schema registry from table schemas
const schemaRegistry = createSchemaRegistry(options.tableSchema);
// 4. Create printer context with tenant IDs and field mappings
// 4. Strip undefined values from enforcedWhereClause
const enforcedWhereClause = Object.fromEntries(
Object.entries(options.enforcedWhereClause).filter(([_, value]) => value !== undefined)
) as Record<string, WhereClauseCondition>;
// 5. Create printer context with enforced WHERE clause and field mappings
const context = createPrinterContext({
organizationId: options.organizationId,
projectId: options.projectId,
environmentId: options.environmentId,
schema: schemaRegistry,
settings: options.settings,
fieldMappings: options.fieldMappings,
enforcedWhereClause,
});
// 5. Print the AST to ClickHouse SQL
// 6. Print the AST to ClickHouse SQL (enforced conditions applied at printer level)
return printToClickHouse(ast, context);
}
internal-packages/tsql/src/query/printer.test.ts
+618 -89
View File
@@ -85,10 +85,12 @@ function createTestContext(
): PrinterContext {
const schema = createSchemaRegistry([taskRunsSchema, taskEventsSchema]);
return createPrinterContext({
organizationId: "org_test123",
projectId: "proj_test456",
environmentId: "env_test789",
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test123" },
project_id: { op: "eq", value: "proj_test456" },
environment_id: { op: "eq", value: "env_test789" },
},
...overrides,
});
}
@@ -153,10 +155,12 @@ describe("ClickHousePrinter", () => {
it("should expand SELECT * with column name mapping", () => {
const schema = createSchemaRegistry([runsSchema]);
const ctx = createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
const { sql, columns } = printQuery("SELECT * FROM runs", ctx);
@@ -216,10 +220,12 @@ describe("ClickHousePrinter", () => {
const schema = createSchemaRegistry([schemaWithVirtual]);
const ctx = createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
const { sql, columns } = printQuery("SELECT * FROM runs", ctx);
@@ -241,15 +247,17 @@ describe("ClickHousePrinter", () => {
describe("Table and column name mapping", () => {
function createMappedContext() {
const schema = createSchemaRegistry([runsSchema]);
return createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
});
}
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
it("should map user-friendly table name to ClickHouse name", () => {
it("should map user-friendly table name to ClickHouse name", () => {
const ctx = createMappedContext();
const { sql } = printQuery("SELECT * FROM runs", ctx);
@@ -472,15 +480,17 @@ describe("ClickHousePrinter", () => {
function createJsonContext() {
const schema = createSchemaRegistry([jsonSchema]);
return createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
});
}
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
it("should transform IS NULL to equals empty object for JSON columns with nullValue", () => {
it("should transform IS NULL to equals empty object for JSON columns with nullValue", () => {
const ctx = createJsonContext();
const { sql } = printQuery("SELECT * FROM runs WHERE error IS NULL", ctx);
@@ -597,6 +607,501 @@ describe("ClickHousePrinter", () => {
expect(sql).toContain("GROUP BY status");
expect(sql).not.toContain(".:String");
});
it("should NOT add .:String type hint for JSON subfield in WHERE comparison", () => {
const ctx = createJsonContext();
const { sql } = printQuery(
"SELECT id FROM runs WHERE error.data.name = 'test'",
ctx
);
// WHERE clause should NOT have .:String type hint (it breaks the query)
expect(sql).toContain("equals(error.data.name,");
expect(sql).not.toContain("error.data.name.:String");
});
it("should NOT add .:String for JSON subfield in WHERE with LIKE", () => {
const ctx = createJsonContext();
const { sql } = printQuery(
"SELECT id FROM runs WHERE error.message LIKE '%error%'",
ctx
);
// WHERE clause should NOT have .:String type hint
expect(sql).toContain("like(error.message,");
expect(sql).not.toContain("error.message.:String");
});
it("should NOT add .:String in SELECT or WHERE when no GROUP BY", () => {
const ctx = createJsonContext();
const { sql } = printQuery(
"SELECT error.data.name FROM runs WHERE error.data.name = 'test'",
ctx
);
// SELECT should NOT have .:String (no GROUP BY, so no need for type hint)
expect(sql).toContain("error.data.name AS error_data_name");
expect(sql).not.toContain(".:String");
// WHERE should NOT have .:String
expect(sql).toContain("equals(error.data.name,");
});
it("should add .:String in GROUP BY but not in WHERE for same query", () => {
const ctx = createJsonContext();
const { sql } = printQuery(
"SELECT error.data.name, count() AS cnt FROM runs WHERE error.data.name = 'test' GROUP BY error.data.name",
ctx
);
// SELECT should have .:String
expect(sql).toContain("error.data.name.:String AS error_data_name");
// GROUP BY should have .:String
expect(sql).toContain("GROUP BY error.data.name.:String");
// WHERE should NOT have .:String
expect(sql).toContain("equals(error.data.name,");
expect(sql).not.toMatch(/equals\(error\.data\.name\.:String/);
});
});
describe("textColumn optimization for JSON columns", () => {
// Create a schema with JSON columns that have textColumn set
const textColumnSchema: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
columns: {
id: { name: "id", ...column("String") },
output: {
name: "output",
...column("JSON"),
nullValue: "'{}'",
textColumn: "output_text",
},
error: {
name: "error",
...column("JSON"),
nullValue: "'{}'",
textColumn: "error_text",
},
status: { name: "status", ...column("String") },
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
function createTextColumnContext() {
const schema = createSchemaRegistry([textColumnSchema]);
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
describe("SELECT clause", () => {
it("should use text column when selecting bare JSON column", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT output FROM runs", ctx);
// Should use the text column with an alias to preserve the column name
expect(sql).toContain("output_text AS output");
});
it("should use text column for multiple JSON columns", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT output, error FROM runs", ctx);
expect(sql).toContain("output_text AS output");
expect(sql).toContain("error_text AS error");
});
it("should use JSON column for subfield access without .:String when no GROUP BY", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT output.data.name FROM runs", ctx);
// Should use the original JSON column without .:String (no GROUP BY)
expect(sql).toContain("output.data.name AS output_data_name");
expect(sql).not.toContain("output_text");
expect(sql).not.toContain(".:String");
});
});
describe("SELECT * expansion", () => {
it("should use text columns when expanding SELECT *", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT * FROM runs", ctx);
// Should use text columns for JSON columns
expect(sql).toContain("output_text AS output");
expect(sql).toContain("error_text AS error");
});
});
describe("WHERE clause", () => {
it("should use text column for exact equality comparison", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT id FROM runs WHERE output = '{}'", ctx);
expect(sql).toContain("equals(output_text,");
expect(sql).not.toMatch(/equals\(output,/);
});
it("should use text column for inequality comparison", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT id FROM runs WHERE output != '{}'", ctx);
expect(sql).toContain("notEquals(output_text,");
});
it("should use text column for LIKE comparison", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT id FROM runs WHERE output LIKE '%error%'", ctx);
expect(sql).toContain("like(output_text,");
expect(sql).not.toMatch(/like\(output,/);
});
it("should use text column for ILIKE comparison", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT id FROM runs WHERE error ILIKE '%failed%'", ctx);
expect(sql).toContain("ilike(error_text,");
});
it("should use text column for NOT LIKE comparison", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT id FROM runs WHERE output NOT LIKE '%test%'", ctx);
expect(sql).toContain("notLike(output_text,");
});
it("should use JSON column for subfield comparison without .:String", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery(
"SELECT id FROM runs WHERE output.data.name = 'test'",
ctx
);
// Should use the original JSON column, not the text column
// And should NOT have .:String in WHERE (breaks the query)
expect(sql).toContain("equals(output.data.name,");
expect(sql).not.toContain("output_text");
expect(sql).not.toContain("output.data.name.:String");
});
it("should still use nullValue transformation for IS NULL", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT id FROM runs WHERE output IS NULL", ctx);
// NULL check should use the text column with nullValue
expect(sql).toContain("equals(output_text, '{}')");
});
it("should still use nullValue transformation for IS NOT NULL", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT id FROM runs WHERE error IS NOT NULL", ctx);
expect(sql).toContain("notEquals(error_text, '{}')");
});
});
describe("edge cases", () => {
it("should work with columns without textColumn defined", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT status FROM runs WHERE status = 'completed'", ctx);
// Regular column should work as before
expect(sql).toContain("status");
expect(sql).not.toContain("status_text");
});
it("should use text column for aliased JSON columns in SELECT", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT output AS result FROM runs", ctx);
// Should use text column with user's alias
expect(sql).toContain("output_text AS result");
});
it("should use text column for table-qualified JSON columns in SELECT", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery("SELECT runs.output FROM runs", ctx);
// Should use text column
expect(sql).toContain("output_text AS output");
});
it("should use text column in both SELECT and WHERE for same query", () => {
const ctx = createTextColumnContext();
const { sql } = printQuery(
"SELECT output FROM runs WHERE output LIKE '%test%'",
ctx
);
// SELECT should use text column
expect(sql).toContain("output_text AS output");
// WHERE should use text column
expect(sql).toContain("like(output_text,");
});
});
describe("JOINs with textColumn", () => {
// Create a second schema with the same JSON column names to test JOIN ambiguity
const runsSchemaWithTextColumn: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
columns: {
id: { name: "id", ...column("String") },
output: {
name: "output",
...column("JSON"),
nullValue: "'{}'",
textColumn: "output_text",
},
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
const eventsSchemaWithTextColumn: TableSchema = {
name: "events",
clickhouseName: "trigger_dev.task_events_v2",
columns: {
id: { name: "id", ...column("String") },
run_id: { name: "run_id", ...column("String") },
output: {
name: "output",
...column("JSON"),
nullValue: "'{}'",
textColumn: "output_text",
},
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
function createJoinTextColumnContext() {
const schema = createSchemaRegistry([runsSchemaWithTextColumn, eventsSchemaWithTextColumn]);
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
it("should qualify text column with table alias in JOIN WHERE clause to avoid ambiguity", () => {
const ctx = createJoinTextColumnContext();
const { sql } = printQuery(
`SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE r.output = '{}'`,
ctx
);
// The text column should be table-qualified to avoid ambiguity
// since both tables have an output_text column
expect(sql).toContain("equals(r.output_text,");
// Should NOT have unqualified output_text in the comparison
expect(sql).not.toMatch(/equals\(output_text,/);
});
it("should qualify text column with table alias for LIKE in JOIN", () => {
const ctx = createJoinTextColumnContext();
const { sql } = printQuery(
`SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE e.output LIKE '%error%'`,
ctx
);
// Should use table-qualified text column
expect(sql).toContain("like(e.output_text,");
expect(sql).not.toMatch(/like\(output_text,/);
});
it("should handle multiple qualified text column comparisons in JOIN", () => {
const ctx = createJoinTextColumnContext();
const { sql } = printQuery(
`SELECT r.id FROM runs r JOIN events e ON r.id = e.run_id WHERE r.output = '{}' AND e.output != '{}'`,
ctx
);
// Both comparisons should be table-qualified
expect(sql).toContain("equals(r.output_text,");
expect(sql).toContain("notEquals(e.output_text,");
});
});
});
describe("dataPrefix for JSON columns", () => {
// Create a schema with JSON columns that have dataPrefix set
const dataPrefixSchema: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
columns: {
id: { name: "id", ...column("String") },
output: {
name: "output",
...column("JSON"),
nullValue: "'{}'",
dataPrefix: "data",
},
error: {
name: "error",
...column("JSON"),
nullValue: "'{}'",
dataPrefix: "data",
},
status: { name: "status", ...column("String") },
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
function createDataPrefixContext() {
const schema = createSchemaRegistry([dataPrefixSchema]);
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
describe("SELECT clause", () => {
it("should inject dataPrefix into JSON subfield path", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery("SELECT output.message FROM runs", ctx);
// Should transform output.message to output.data.message
expect(sql).toContain("output.data.message");
});
it("should generate clean alias without dataPrefix", () => {
const ctx = createDataPrefixContext();
const { sql, columns } = printQuery("SELECT output.message FROM runs", ctx);
// Alias should be output_message, not output_data_message
expect(sql).toContain("AS output_message");
expect(sql).not.toContain("AS output_data_message");
expect(columns).toContainEqual(
expect.objectContaining({ name: "output_message" })
);
});
it("should handle nested paths with dataPrefix", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery("SELECT output.user.name FROM runs", ctx);
// Should transform output.user.name to output.data.user.name
expect(sql).toContain("output.data.user.name");
// Alias should be output_user_name
expect(sql).toContain("AS output_user_name");
});
it("should work with multiple JSON columns with dataPrefix", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery("SELECT output.msg, error.code FROM runs", ctx);
expect(sql).toContain("output.data.msg");
expect(sql).toContain("error.data.code");
expect(sql).toContain("AS output_msg");
expect(sql).toContain("AS error_code");
});
it("should not affect bare JSON column selection", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery("SELECT output FROM runs", ctx);
// Bare column should not have dataPrefix injected
expect(sql).not.toContain("output.data");
expect(sql).toMatch(/SELECT\s+output[\s,]/);
});
});
describe("WHERE clause", () => {
it("should inject dataPrefix into WHERE comparison", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery(
"SELECT id FROM runs WHERE output.status = 'success'",
ctx
);
// Should transform output.status to output.data.status
expect(sql).toContain("output.data.status");
});
it("should inject dataPrefix into LIKE comparison", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery(
"SELECT id FROM runs WHERE error.message LIKE '%failed%'",
ctx
);
expect(sql).toContain("error.data.message");
});
});
describe("GROUP BY clause", () => {
it("should inject dataPrefix into GROUP BY", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery(
"SELECT output.type, count() AS cnt FROM runs GROUP BY output.type",
ctx
);
// Should inject dataPrefix in both SELECT and GROUP BY
expect(sql).toContain("output.data.type");
expect(sql).toContain("GROUP BY output.data.type");
});
});
describe("edge cases", () => {
it("should not affect columns without dataPrefix", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery("SELECT status FROM runs", ctx);
// Regular column should not be affected
expect(sql).toContain("status");
expect(sql).not.toContain("status.data");
});
it("should work with explicit alias on JSON subfield", () => {
const ctx = createDataPrefixContext();
const { sql } = printQuery("SELECT output.message AS msg FROM runs", ctx);
// Should inject dataPrefix but use user's alias
expect(sql).toContain("output.data.message");
expect(sql).toContain("AS msg");
});
});
});
describe("ORDER BY clauses", () => {
@@ -742,9 +1247,11 @@ describe("ClickHousePrinter", () => {
describe("Tenant isolation", () => {
it("should inject tenant guards for single table", () => {
const context = createTestContext({
organizationId: "org_abc",
projectId: "proj_def",
environmentId: "env_ghi",
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_abc" },
project_id: { op: "eq", value: "proj_def" },
environment_id: { op: "eq", value: "env_ghi" },
},
});
const { sql, params } = printQuery("SELECT * FROM task_runs", context);
@@ -1057,15 +1564,17 @@ describe("Value mapping (valueMap)", () => {
function createValueMapContext() {
const schema = createSchemaRegistry([statusMappedSchema]);
return createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
});
}
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
it("should transform user-friendly value to internal value in equality comparison", () => {
it("should transform user-friendly value to internal value in equality comparison", () => {
const ctx = createValueMapContext();
const { sql, params } = printQuery("SELECT * FROM runs WHERE status = 'Completed'", ctx);
@@ -1173,15 +1682,17 @@ describe("WHERE transform (whereTransform)", () => {
function createPrefixedContext() {
const schema = createSchemaRegistry([prefixedIdSchema]);
return createPrinterContext({
organizationId: "org_test123",
projectId: "proj_test456",
environmentId: "env_test789",
schema,
});
}
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test123" },
project_id: { op: "eq", value: "proj_test456" },
environment_id: { op: "eq", value: "env_test789" },
},
});
}
it("should strip prefix from value in equality comparison", () => {
it("should strip prefix from value in equality comparison", () => {
const ctx = createPrefixedContext();
const { params } = printQuery("SELECT * FROM runs WHERE batch_id = 'batch_abc123'", ctx);
@@ -1397,16 +1908,18 @@ describe("Virtual columns", () => {
function createVirtualColumnContext() {
const schema = createSchemaRegistry([virtualColumnSchema]);
return createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
});
}
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
describe("SELECT clause", () => {
it("should expand bare virtual column to expression with alias", () => {
describe("SELECT clause", () => {
it("should expand bare virtual column to expression with alias", () => {
const ctx = createVirtualColumnContext();
const { sql } = printQuery("SELECT execution_duration FROM runs", ctx);
@@ -1638,16 +2151,18 @@ describe("Expression columns with division (cost/invocation_cost pattern)", () =
function createCostExpressionContext() {
const schema = createSchemaRegistry([costExpressionSchema]);
return createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
});
}
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
describe("WHERE clause with division expression columns", () => {
it("should expand invocation_cost > 100 to (base_cost_in_cents / 100.0) > 100", () => {
describe("WHERE clause with division expression columns", () => {
it("should expand invocation_cost > 100 to (base_cost_in_cents / 100.0) > 100", () => {
const ctx = createCostExpressionContext();
const { sql } = printQuery("SELECT * FROM runs WHERE invocation_cost > 100", ctx);
@@ -1782,16 +2297,18 @@ describe("Column metadata", () => {
function createMetadataTestContext() {
const schema = createSchemaRegistry([schemaWithRenderTypes]);
return createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
});
}
return createPrinterContext({
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
describe("Basic column metadata", () => {
it("should return column metadata for simple field references", () => {
describe("Basic column metadata", () => {
it("should return column metadata for simple field references", () => {
const ctx = createMetadataTestContext();
const { columns } = printQuery("SELECT run_id, created_at FROM runs", ctx);
@@ -2193,10 +2710,12 @@ describe("Unknown column blocking", () => {
// Using the internal name directly should be blocked
const schema = createSchemaRegistry([runsSchema]);
const ctx = createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
// 'created_at' is not in runsSchema - only 'created' which maps to 'created_at'
@@ -2210,10 +2729,12 @@ describe("Unknown column blocking", () => {
// When user types 'created_at', we should suggest 'created'
const schema = createSchemaRegistry([runsSchema]);
const ctx = createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
expect(() => {
@@ -2346,9 +2867,6 @@ describe("Field Mapping Value Transformation", () => {
function createFieldMappingContext(): PrinterContext {
const schemaRegistry = createSchemaRegistry([fieldMappingSchema]);
return new PrinterContext(
"org_123",
"proj_456",
"env_789",
schemaRegistry,
{},
{
@@ -2356,6 +2874,11 @@ describe("Field Mapping Value Transformation", () => {
proj_tenant1: "my-project-ref",
proj_other: "other-project",
},
},
{
organization_id: { op: "eq", value: "org_123" },
project_id: { op: "eq", value: "proj_456" },
environment_id: { op: "eq", value: "env_789" },
}
);
}
@@ -2474,20 +2997,24 @@ describe("Internal-only column blocking", () => {
function createHiddenTenantContext(): PrinterContext {
const schema = createSchemaRegistry([hiddenTenantSchema]);
return createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
function createHiddenFilterContext(): PrinterContext {
const schema = createSchemaRegistry([hiddenFilterSchema]);
return createPrinterContext({
organizationId: "org_test",
projectId: "proj_test",
environmentId: "env_test",
schema,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test" },
project_id: { op: "eq", value: "proj_test" },
environment_id: { op: "eq", value: "env_test" },
},
});
}
@@ -2655,10 +3182,12 @@ describe("Required Filters", () => {
function createRequiredFiltersContext(): PrinterContext {
const schemaRegistry = createSchemaRegistry([schemaWithRequiredFilters]);
return createPrinterContext({
organizationId: "org_test123",
projectId: "proj_test456",
environmentId: "env_test789",
schema: schemaRegistry,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_test123" },
project_id: { op: "eq", value: "proj_test456" },
environment_id: { op: "eq", value: "env_test789" },
},
});
}
internal-packages/tsql/src/query/printer.ts
+444 -76
View File
@@ -46,7 +46,7 @@ import {
findTSQLFunction,
validateFunctionArgs,
} from "./functions";
import { PrinterContext } from "./printer_context";
import { PrinterContext, WhereClauseCondition } from "./printer_context";
import {
findTable,
validateTable,
@@ -114,6 +114,8 @@ export class ClickHousePrinter {
private outputColumns: OutputColumnMetadata[] = [];
/** Whether we're currently processing GROUP BY expressions */
private inGroupByContext = false;
/** Whether the current query has a GROUP BY clause (used for JSON subfield type hints) */
private queryHasGroupBy = false;
/** Columns hidden when SELECT * is expanded to core columns only */
private hiddenColumns: string[] = [];
/**
@@ -392,6 +394,11 @@ export class ClickHousePrinter {
}
}
// Track if query has GROUP BY for JSON subfield type hint decisions
// (ClickHouse requires .:String for Dynamic types in GROUP BY, and SELECT must match)
const savedQueryHasGroupBy = this.queryHasGroupBy;
this.queryHasGroupBy = !!node.group_by;
// Process SELECT columns and collect metadata
// Using flatMap because asterisk expansion can return multiple columns
// Set inProjectionContext to block internal-only columns in user projections
@@ -543,6 +550,7 @@ export class ClickHousePrinter {
// Restore saved contexts (for nested queries)
this.selectAliases = savedAliases;
this.queryHasGroupBy = savedQueryHasGroupBy;
this.tableContexts = savedTableContexts;
this.allowedInternalColumns = savedInternalColumns;
this.internalOnlyColumns = savedInternalOnlyColumns;
@@ -627,37 +635,47 @@ export class ClickHousePrinter {
let sqlResult: string;
if ((col as Field).expression_type === "field") {
const field = col as Field;
const virtualColumnName = this.getVirtualColumnNameForField(field.chain);
if (virtualColumnName !== null) {
// Visit the field (which will return the expression)
const visited = this.visit(col);
// Add the alias to preserve the column name
sqlResult = `${visited} AS ${this.printIdentifier(virtualColumnName)}`;
// Check if this is a bare JSON field that should use a text column
const textColumn = this.getTextColumnForField(field.chain);
if (textColumn !== null && outputName) {
// Use the text column instead of the JSON column, with alias to preserve name
sqlResult = `${this.printIdentifier(textColumn)} AS ${this.printIdentifier(outputName)}`;
} else {
// Visit the field to get the ClickHouse SQL
const visited = this.visit(col);
const virtualColumnName = this.getVirtualColumnNameForField(field.chain);
// Check if this is a JSON subfield access (will have .:String type hint)
// If so, add an alias to preserve the nice column name (dots → underscores)
const isJsonSubfield = this.isJsonSubfieldAccess(field.chain);
if (isJsonSubfield) {
// Build the alias using underscores (e.g., "error_data_name")
const aliasName = field.chain.filter((p): p is string => typeof p === "string").join("_");
sqlResult = `${visited} AS ${this.printIdentifier(aliasName)}`;
// Override output name for metadata
effectiveOutputName = aliasName;
}
// Check if the column has a different clickhouseName - if so, add an alias
// to ensure results come back with the user-facing name
else if (
outputName &&
sourceColumn?.clickhouseName &&
sourceColumn.clickhouseName !== outputName
) {
sqlResult = `${visited} AS ${this.printIdentifier(outputName)}`;
if (virtualColumnName !== null) {
// Visit the field (which will return the expression)
const visited = this.visit(col);
// Add the alias to preserve the column name
sqlResult = `${visited} AS ${this.printIdentifier(virtualColumnName)}`;
} else {
sqlResult = visited;
// Visit the field to get the ClickHouse SQL
const visited = this.visit(col);
// Check if this is a JSON subfield access (will have .:String type hint)
// If so, add an alias to preserve the nice column name (dots → underscores)
const isJsonSubfield = this.isJsonSubfieldAccess(field.chain);
if (isJsonSubfield) {
// Build the alias using underscores, excluding any dataPrefix
// e.g., output.message -> "output_message" (not "output_data_message")
const dataPrefix = this.getDataPrefixForField(field.chain);
const aliasName = this.buildAliasWithoutDataPrefix(field.chain, dataPrefix);
sqlResult = `${visited} AS ${this.printIdentifier(aliasName)}`;
// Override output name for metadata
effectiveOutputName = aliasName;
}
// Check if the column has a different clickhouseName - if so, add an alias
// to ensure results come back with the user-facing name
else if (
outputName &&
sourceColumn?.clickhouseName &&
sourceColumn.clickhouseName !== outputName
) {
sqlResult = `${visited} AS ${this.printIdentifier(outputName)}`;
} else {
sqlResult = visited;
}
}
}
} else if (
@@ -675,8 +693,23 @@ export class ClickHousePrinter {
} else {
sqlResult = visited;
}
} else if ((col as Alias).expression_type === "alias") {
// Handle Alias expressions - check if inner expression is a bare JSON field with textColumn
const alias = col as Alias;
if ((alias.expr as Field).expression_type === "field") {
const innerField = alias.expr as Field;
const textColumn = this.getTextColumnForField(innerField.chain);
if (textColumn !== null) {
// Use the text column with the user's explicit alias
sqlResult = `${this.printIdentifier(textColumn)} AS ${this.printIdentifier(alias.alias)}`;
} else {
sqlResult = this.visit(col);
}
} else {
sqlResult = this.visit(col);
}
} else {
// For Alias expressions or other types, visit normally
// For other types, visit normally
sqlResult = this.visit(col);
}
@@ -817,6 +850,11 @@ export class ClickHousePrinter {
if (isVirtualColumn(columnSchema)) {
// Virtual column: use the expression with an alias
sqlResult = `(${columnSchema.expression}) AS ${this.printIdentifier(columnName)}`;
} else if (columnSchema.textColumn) {
// JSON column with text column optimization: use the text column with alias
sqlResult = `${this.printIdentifier(columnSchema.textColumn)} AS ${this.printIdentifier(
columnName
)}`;
} else {
// Regular column: use the actual ClickHouse column name
const clickhouseName = columnSchema.clickhouseName ?? columnName;
@@ -1438,6 +1476,9 @@ export class ClickHousePrinter {
// Look up table schema and get ClickHouse table name
const tableSchema = this.lookupTable(tableName);
// Validate that required tenant columns are present in enforcedWhereClause
this.validateRequiredTenantColumns(tableSchema);
// Always add the TSQL table name as an alias if no explicit alias is provided
// This ensures table-qualified column references work in WHERE clauses
// (needed to avoid alias conflicts when columns have expressions)
@@ -1486,8 +1527,8 @@ export class ClickHousePrinter {
}
}
// Add tenant isolation guard
extraWhere = this.createTenantGuard(tableSchema, effectiveAlias);
// Add enforced WHERE clause guard (tenant isolation + plan limits)
extraWhere = this.createEnforcedGuard(tableSchema, effectiveAlias);
} else if (
(tableExpr as SelectQuery).expression_type === "select_query" ||
(tableExpr as SelectSetQuery).expression_type === "select_set_query"
@@ -1534,77 +1575,202 @@ export class ClickHousePrinter {
}
// ============================================================
// Tenant Isolation
// Enforced WHERE Clause
// ============================================================
/**
* Create a WHERE clause expression for tenant isolation and required filters
* Note: We use just the column name without table prefix since ClickHouse
* requires the actual table name (task_runs_v2), not the TSQL alias (task_runs)
* Validate that required tenant columns are present in enforcedWhereClause.
*
* Organization ID is always required. Project ID and Environment ID are optional -
* if not provided, the query will return results across all projects/environments.
* If a table defines `tenantColumns.organizationId`, the `enforcedWhereClause`
* MUST include that column to ensure tenant isolation. This prevents accidental
* data leaks when the caller forgets to include tenant isolation conditions.
*
* Required filters from the table schema are also always included.
* @throws QueryError if a required tenant column is missing
*/
private createTenantGuard(tableSchema: TableSchema, _tableAlias: string): And | CompareOperation {
const { tenantColumns, requiredFilters } = tableSchema;
private validateRequiredTenantColumns(tableSchema: TableSchema): void {
const { tenantColumns } = tableSchema;
if (!tenantColumns) return;
// Organization guard is always required
const orgGuard: CompareOperation = {
expression_type: "compare_operation",
op: CompareOperationOp.Eq,
left: { expression_type: "field", chain: [tenantColumns.organizationId] } as Field,
right: { expression_type: "constant", value: this.context.organizationId } as Constant,
// Organization ID is always required if the table defines it
if (tenantColumns.organizationId) {
const orgColumn = tenantColumns.organizationId;
if (!this.context.enforcedWhereClause[orgColumn]) {
throw new QueryError(
`Table '${tableSchema.name}' requires '${orgColumn}' in enforcedWhereClause for tenant isolation`
);
}
}
// Note: projectId and environmentId are optional - no validation needed
}
/**
* Format a Date as a ClickHouse-compatible DateTime64 string.
* ClickHouse expects format: 'YYYY-MM-DD HH:MM:SS.mmm' (in UTC)
*/
private formatDateForClickHouse(date: Date): string {
const year = date.getUTCFullYear();
const month = String(date.getUTCMonth() + 1).padStart(2, "0");
const day = String(date.getUTCDate()).padStart(2, "0");
const hours = String(date.getUTCHours()).padStart(2, "0");
const minutes = String(date.getUTCMinutes()).padStart(2, "0");
const seconds = String(date.getUTCSeconds()).padStart(2, "0");
const ms = String(date.getUTCMilliseconds()).padStart(3, "0");
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${ms}`;
}
/**
* Create an AST expression for a value.
* Date values are wrapped in toDateTime64() for ClickHouse compatibility.
*/
private createValueExpression(value: Date | string | number): Expression {
if (value instanceof Date) {
// Wrap Date in toDateTime64(formatted_string, 3) for ClickHouse DateTime64(3) columns
return {
expression_type: "call",
name: "toDateTime64",
args: [
{ expression_type: "constant", value: this.formatDateForClickHouse(value) } as Constant,
{ expression_type: "constant", value: 3 } as Constant,
],
} as Call;
}
return { expression_type: "constant", value } as Constant;
}
/**
* Map condition operator to CompareOperationOp
*/
private mapConditionOpToCompareOp(
op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte"
): CompareOperationOp {
switch (op) {
case "eq":
return CompareOperationOp.Eq;
case "neq":
return CompareOperationOp.NotEq;
case "gt":
return CompareOperationOp.Gt;
case "gte":
return CompareOperationOp.GtEq;
case "lt":
return CompareOperationOp.Lt;
case "lte":
return CompareOperationOp.LtEq;
}
}
/**
* Create an AST expression from a WhereClauseCondition
*
* @param column - The column name
* @param condition - The condition to apply
* @param tableAlias - Optional table alias to qualify the column reference.
* When provided, constructs the field chain as [tableAlias, column]
* so resolveFieldChain will resolve to the correct table in multi-join queries.
* @returns The AST expression for the condition
*/
private createConditionExpression(
column: string,
condition: WhereClauseCondition,
tableAlias?: string
): Expression {
// When tableAlias is provided, qualify the field chain to ensure it binds
// to the correct table in multi-join queries
const fieldExpr: Field = {
expression_type: "field",
chain: tableAlias ? [tableAlias, column] : [column],
};
// Collect all guards - org is always included
const guards: CompareOperation[] = [orgGuard];
// Only add project guard if projectId is provided
if (this.context.projectId !== undefined) {
const projectGuard: CompareOperation = {
expression_type: "compare_operation",
op: CompareOperationOp.Eq,
left: { expression_type: "field", chain: [tenantColumns.projectId] } as Field,
right: { expression_type: "constant", value: this.context.projectId } as Constant,
if (condition.op === "between") {
const betweenExpr: BetweenExpr = {
expression_type: "between_expr",
expr: fieldExpr,
low: this.createValueExpression(condition.low),
high: this.createValueExpression(condition.high),
};
guards.push(projectGuard);
return betweenExpr;
}
// Only add environment guard if environmentId is provided
if (this.context.environmentId !== undefined) {
const envGuard: CompareOperation = {
expression_type: "compare_operation",
op: CompareOperationOp.Eq,
left: { expression_type: "field", chain: [tenantColumns.environmentId] } as Field,
right: { expression_type: "constant", value: this.context.environmentId } as Constant,
};
guards.push(envGuard);
// Simple comparison
const compareExpr: CompareOperation = {
expression_type: "compare_operation",
left: fieldExpr,
right: this.createValueExpression(condition.value),
op: this.mapConditionOpToCompareOp(condition.op),
};
return compareExpr;
}
/**
* Create a WHERE clause expression for enforced conditions and required filters.
*
* This method applies:
* 1. All conditions from enforcedWhereClause (tenant isolation + plan limits)
* 2. Required filters from the table schema (e.g., engine = 'V2')
*
* Conditions are applied if the column exists in either:
* - The exposed columns (tableSchema.columns)
* - The tenant columns (tableSchema.tenantColumns)
*
* This ensures the same enforcedWhereClause can be used across different tables.
*
* All guard expressions are qualified with the table alias to ensure they bind
* to the correct table in multi-join queries, preventing potential security
* issues where an unqualified column reference could bind to the wrong table.
*/
private createEnforcedGuard(tableSchema: TableSchema, tableAlias: string): Expression | null {
const { requiredFilters, tenantColumns } = tableSchema;
const guards: Expression[] = [];
// Build a set of valid columns for this table (exposed + tenant columns)
const validColumns = new Set<string>(Object.keys(tableSchema.columns));
if (tenantColumns) {
if (tenantColumns.organizationId) validColumns.add(tenantColumns.organizationId);
if (tenantColumns.projectId) validColumns.add(tenantColumns.projectId);
if (tenantColumns.environmentId) validColumns.add(tenantColumns.environmentId);
}
// Add required filters from the table schema
// Apply all enforced conditions for columns that exist in this table
// Pass tableAlias to ensure guards are qualified and bind to the correct table
for (const [column, condition] of Object.entries(this.context.enforcedWhereClause)) {
// Skip undefined/null conditions (allows conditional inclusion like project_id?: condition)
if (condition === undefined || condition === null) {
continue;
}
// Only apply if column exists in this table's schema or is a tenant column
if (validColumns.has(column)) {
guards.push(this.createConditionExpression(column, condition, tableAlias));
}
}
// Add required filters from the table schema (e.g., engine = 'V2')
// Also qualified with table alias to ensure correct binding in multi-join queries
if (requiredFilters && requiredFilters.length > 0) {
for (const filter of requiredFilters) {
const filterGuard: CompareOperation = {
expression_type: "compare_operation",
op: CompareOperationOp.Eq,
left: { expression_type: "field", chain: [filter.column] } as Field,
left: { expression_type: "field", chain: [tableAlias, filter.column] } as Field,
right: { expression_type: "constant", value: filter.value } as Constant,
};
guards.push(filterGuard);
}
}
// If only org guard, return it directly (no need for AND wrapper)
// Return null if no guards (empty enforcedWhereClause and no requiredFilters)
if (guards.length === 0) {
return null;
}
// If only one guard, return it directly (no need for AND wrapper)
if (guards.length === 1) {
return orgGuard;
return guards[0];
}
return {
expression_type: "and",
exprs: guards,
};
} as And;
}
// ============================================================
@@ -1711,7 +1877,39 @@ export class ClickHousePrinter {
// Transform the right side if it contains user-friendly values
const transformedRight = this.transformValueMapExpression(node.right, columnSchema);
const left = this.visit(node.left);
// Check if we should use a text column for bare JSON field comparisons
// This applies to: Eq, NotEq, Like, ILike, NotLike, NotILike
const textColumnOps = [
CompareOperationOp.Eq,
CompareOperationOp.NotEq,
CompareOperationOp.Like,
CompareOperationOp.ILike,
CompareOperationOp.NotLike,
CompareOperationOp.NotILike,
];
const useTextColumn = textColumnOps.includes(node.op);
const leftTextColumn = useTextColumn ? this.getTextColumnForExpression(node.left) : null;
// Build the left side, qualifying the text column with table alias if present
let left: string;
if (leftTextColumn) {
// Check if the field is qualified with a table alias (e.g., r.output)
// and prepend that alias to the text column to avoid ambiguity in JOINs
const fieldNode = node.left as Field;
if (fieldNode.expression_type === "field" && fieldNode.chain.length >= 2) {
const firstPart = fieldNode.chain[0];
if (typeof firstPart === "string" && this.tableContexts.has(firstPart)) {
// The field is qualified with a table alias, prepend it to the text column
left = this.printIdentifier(firstPart) + "." + this.printIdentifier(leftTextColumn);
} else {
left = this.printIdentifier(leftTextColumn);
}
} else {
left = this.printIdentifier(leftTextColumn);
}
} else {
left = this.visit(node.left);
}
const right = this.visit(transformedRight);
switch (node.op) {
@@ -2074,19 +2272,31 @@ export class ClickHousePrinter {
return `(${virtualExpression})`;
}
// Inject dataPrefix for JSON columns if needed (e.g., output.message -> output.data.message)
const chainWithPrefix = this.injectDataPrefix(node.chain);
// Try to resolve column names through table context
const resolvedChain = this.resolveFieldChain(node.chain);
const resolvedChain = this.resolveFieldChain(chainWithPrefix);
// Print each chain element
let result = resolvedChain.map((part) => this.printIdentifierOrIndex(part)).join(".");
// For JSON column subfield access (e.g., error.data.name), add .:String type hint
// This is required because ClickHouse's Dynamic/Variant types are not allowed in
// GROUP BY without type casting, and SELECT/GROUP BY expressions must match
// This is ONLY required when the query has GROUP BY, because:
// 1. ClickHouse's Dynamic/Variant types are not allowed in GROUP BY without type casting
// 2. SELECT/GROUP BY expressions must match
// For queries without GROUP BY, the .:String type hint actually breaks the query
// (returns NULL instead of the actual value)
// We also skip this in WHERE comparisons where it breaks the query
if (resolvedChain.length > 1) {
// Check if the root column (first part) is a JSON column
const rootColumnSchema = this.resolveFieldToColumnSchema([node.chain[0]]);
if (rootColumnSchema?.type === "JSON") {
// Add .:String ONLY for GROUP BY queries, and NOT in WHERE comparisons
if (
rootColumnSchema?.type === "JSON" &&
this.queryHasGroupBy &&
!this.isInWhereComparisonContext()
) {
// Add .:String type hint for JSON subfield access
result = `${result}.:String`;
}
@@ -2114,6 +2324,20 @@ export class ClickHousePrinter {
return false;
}
/**
* Check if we're inside a WHERE/HAVING comparison operation.
* Unlike isInComparisonContext(), this does NOT include GROUP BY context.
* Used to skip .:String type hints in WHERE clauses where they break queries.
*/
private isInWhereComparisonContext(): boolean {
for (const node of this.stack) {
if ((node as CompareOperation).expression_type === "compare_operation") {
return true;
}
}
return false;
}
/**
* Resolve field chain with table alias prefix to avoid alias conflicts.
* This is used in WHERE clauses when a column has whereTransform to ensure
@@ -2155,6 +2379,125 @@ export class ClickHousePrinter {
return rootColumnSchema?.type === "JSON";
}
/**
* Check if a field should use a text column instead of the JSON column.
* Returns the text column name if the field is a bare JSON field with textColumn defined,
* or null if the original column should be used.
*
* A "bare" JSON field means selecting the entire column (e.g., SELECT output)
* rather than accessing a subfield (e.g., SELECT output.data.name).
*/
private getTextColumnForField(chain: Array<string | number>): string | null {
if (chain.length === 0) return null;
const firstPart = chain[0];
if (typeof firstPart !== "string") return null;
let columnSchema: ColumnSchema | null = null;
if (chain.length === 1) {
// Unqualified: just column name
columnSchema = this.resolveFieldToColumnSchema(chain);
} else if (chain.length === 2) {
// Could be table.column (qualified) - check if first part is a table alias
const tableSchema = this.tableContexts.get(firstPart);
if (tableSchema) {
const columnName = chain[1];
if (typeof columnName === "string") {
columnSchema = tableSchema.columns[columnName] || null;
}
}
// If not a table alias, it's JSON path access (e.g., output.data) - return null
}
// chain.length > 2 means JSON path access - return null
return columnSchema?.textColumn ?? null;
}
/**
* Get the text column for an expression if it's a bare JSON field.
* Returns null if the expression is not a field or doesn't have a textColumn.
*/
private getTextColumnForExpression(expr: Expression): string | null {
if ((expr as Field).expression_type !== "field") return null;
return this.getTextColumnForField((expr as Field).chain);
}
/**
* Get the dataPrefix for a field chain if the root column has one defined.
* Returns null if the column doesn't have a dataPrefix or if this isn't a subfield access.
*/
private getDataPrefixForField(chain: Array<string | number>): string | null {
if (chain.length < 2) return null; // Need at least column.subfield
const firstPart = chain[0];
if (typeof firstPart !== "string") return null;
// Check if first part is a table alias (table.column.subfield)
const tableSchema = this.tableContexts.get(firstPart);
if (tableSchema) {
// Qualified: table.column.subfield - need at least 3 parts
if (chain.length < 3) return null;
const columnName = chain[1];
if (typeof columnName !== "string") return null;
const columnSchema = tableSchema.columns[columnName];
return columnSchema?.dataPrefix ?? null;
}
// Unqualified: column.subfield
const columnSchema = this.resolveFieldToColumnSchema([firstPart]);
return columnSchema?.dataPrefix ?? null;
}
/**
* Inject dataPrefix into a field chain if the root column has one defined.
* e.g., [output, message] -> [output, data, message] when dataPrefix is "data"
* Returns the original chain if no dataPrefix applies.
*/
private injectDataPrefix(chain: Array<string | number>): Array<string | number> {
const dataPrefix = this.getDataPrefixForField(chain);
if (!dataPrefix) return chain;
const firstPart = chain[0];
if (typeof firstPart !== "string") return chain;
// Check if first part is a table alias
const tableSchema = this.tableContexts.get(firstPart);
if (tableSchema) {
// Qualified: table.column.subfield -> table.column.dataPrefix.subfield
// [table, column, subfield] -> [table, column, dataPrefix, subfield]
return [chain[0], chain[1], dataPrefix, ...chain.slice(2)];
}
// Unqualified: column.subfield -> column.dataPrefix.subfield
// [column, subfield] -> [column, dataPrefix, subfield]
return [chain[0], dataPrefix, ...chain.slice(1)];
}
/**
* Build an alias name for a field chain, excluding the dataPrefix if present.
* e.g., [output, message] with dataPrefix "data" -> "output_message"
* This gives users clean column names without the internal data wrapper.
*/
private buildAliasWithoutDataPrefix(
chain: Array<string | number>,
dataPrefix: string | null
): string {
// Filter to just string parts and join with underscores
const parts = chain.filter((p): p is string => typeof p === "string");
if (dataPrefix) {
// Remove the dataPrefix from the parts (it's an implementation detail)
const prefixIndex = parts.indexOf(dataPrefix);
if (prefixIndex > 0) {
// Only remove if it's not the first element (column name)
parts.splice(prefixIndex, 1);
}
}
return parts.join("_");
}
/**
* Resolve a field chain to its column schema (if it references a known column)
*/
@@ -2380,6 +2723,31 @@ export class ClickHousePrinter {
return columnSchema.clickhouseName || columnSchema.name;
}
// Check if this is a tenant column that's not exposed in the schema's columns
// These are internal columns used for tenant isolation guards
const { tenantColumns, requiredFilters } = tableSchema;
if (tenantColumns) {
if (
columnName === tenantColumns.organizationId ||
columnName === tenantColumns.projectId ||
columnName === tenantColumns.environmentId
) {
// Tenant columns are already ClickHouse column names, return as-is
return columnName;
}
}
// Check if this is a required filter column (e.g., engine = 'V2')
// These are internal columns used for enforced filters
if (requiredFilters) {
for (const filter of requiredFilters) {
if (columnName === filter.column) {
// Required filter columns are already ClickHouse column names, return as-is
return columnName;
}
}
}
// Column not in schema - this is a security issue, block access
// Check if the user typed a ClickHouse column name instead of the TSQL name
for (const [tsqlName, colSchema] of Object.entries(tableSchema.columns)) {
internal-packages/tsql/src/query/printer_context.ts
+64 -22
View File
@@ -18,6 +18,34 @@ export interface QuerySettings {
timeoutSeconds?: number;
}
/**
* A simple comparison condition (e.g., column > value)
*/
export interface SimpleComparisonCondition {
/** The comparison operator */
op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte";
/** The value to compare against */
value: Date | string | number;
}
/**
* A between condition (e.g., column BETWEEN low AND high)
*/
export interface BetweenCondition {
/** The between operator */
op: "between";
/** The low bound of the range */
low: Date | string | number;
/** The high bound of the range */
high: Date | string | number;
}
/**
* A WHERE clause condition that can be either a simple comparison or a BETWEEN.
* Used for both enforcedWhereClause (always applied) and whereClauseFallback (default when user doesn't filter).
*/
export type WhereClauseCondition = SimpleComparisonCondition | BetweenCondition;
/**
* Default query settings
*/
@@ -42,7 +70,7 @@ export interface QueryNotice {
* Context for the TSQL to ClickHouse printer
*
* Holds:
* - Tenant IDs for automatic WHERE clause injection
* - Enforced WHERE conditions for tenant isolation and plan limits
* - Schema registry for table/column validation
* - Parameter accumulator for SQL injection safety
* - Query settings and execution options
@@ -64,23 +92,30 @@ export class PrinterContext {
/** Runtime field mappings for dynamic value translation */
readonly fieldMappings: FieldMappings;
/**
* Enforced WHERE conditions that are ALWAYS applied at the table level.
* Used for tenant isolation (org_id, project_id, env_id) and plan-based limits.
* Applied to every table reference including subqueries, CTEs, and JOINs.
*/
readonly enforcedWhereClause: Record<string, WhereClauseCondition>;
constructor(
/** The organization ID for tenant isolation (required) */
public readonly organizationId: string,
/** The project ID for tenant isolation (optional - omit to query across all projects) */
public readonly projectId: string | undefined,
/** The environment ID for tenant isolation (optional - omit to query across all environments) */
public readonly environmentId: string | undefined,
/** Schema registry containing allowed tables and columns */
public readonly schema: SchemaRegistry,
/** Query execution settings */
public readonly settings: QuerySettings = {},
/** Runtime field mappings for dynamic value translation */
fieldMappings: FieldMappings = {}
fieldMappings: FieldMappings = {},
/**
* Enforced WHERE conditions that are ALWAYS applied at the table level.
* Must include tenant columns (e.g., organization_id) for multi-tenant tables.
*/
enforcedWhereClause: Record<string, WhereClauseCondition> = {}
) {
// Initialize with default settings
this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings };
this.fieldMappings = fieldMappings;
this.enforcedWhereClause = enforcedWhereClause;
}
/**
@@ -157,12 +192,10 @@ export class PrinterContext {
*/
createChildContext(): PrinterContext {
const child = new PrinterContext(
this.organizationId,
this.projectId,
this.environmentId,
this.schema,
this.settings,
this.fieldMappings
this.fieldMappings,
this.enforcedWhereClause
);
// Share the same values map so parameters are unified
child.values = this.values;
@@ -184,19 +217,30 @@ export class PrinterContext {
* Options for creating a printer context
*/
export interface PrinterContextOptions {
/** The organization ID for tenant isolation (required) */
organizationId: string;
/** The project ID for tenant isolation (optional - omit to query across all projects) */
projectId?: string;
/** The environment ID for tenant isolation (optional - omit to query across all environments) */
environmentId?: string;
/** Schema registry containing allowed tables and columns */
schema: SchemaRegistry;
/** Query execution settings */
settings?: QuerySettings;
/**
* Runtime field mappings for dynamic value translation.
* Maps internal ClickHouse values to external user-facing values.
*/
fieldMappings?: FieldMappings;
/**
* REQUIRED: Conditions always applied at the table level.
* Must include tenant columns (e.g., organization_id) for multi-tenant tables.
* Applied to every table reference including subqueries, CTEs, and JOINs.
*
* @example
* ```typescript
* {
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
* }
* ```
*/
enforcedWhereClause: Record<string, WhereClauseCondition>;
}
/**
@@ -204,12 +248,10 @@ export interface PrinterContextOptions {
*/
export function createPrinterContext(options: PrinterContextOptions): PrinterContext {
return new PrinterContext(
options.organizationId,
options.projectId,
options.environmentId,
options.schema,
options.settings,
options.fieldMappings
options.fieldMappings,
options.enforcedWhereClause
);
}
internal-packages/tsql/src/query/schema.ts
+36
View File
@@ -214,6 +214,42 @@ export interface ColumnSchema {
* ```
*/
nullValue?: string;
/**
* Alternative text column to use when selecting or comparing the full JSON value.
*
* For JSON columns, this allows using a pre-materialized string column
* which is more efficient than reading from the JSON column directly.
*
* @example
* ```typescript
* {
* name: "output",
* type: "JSON",
* textColumn: "output_text",
* }
* ```
*/
textColumn?: string;
/**
* Prefix path for JSON column data access.
*
* When set, user paths like `output.message` are automatically transformed
* to `output.data.message` in the actual query, and result aliases exclude
* the prefix (e.g., `output_message` instead of `output_data_message`).
*
* This is useful when JSON data is stored wrapped in a container object
* (e.g., `{"data": actualData}`) to handle arrays and primitives.
*
* @example
* ```typescript
* {
* name: "output",
* type: "JSON",
* dataPrefix: "data", // output.message → output.data.message
* }
* ```
*/
dataPrefix?: string;
}
/**
internal-packages/tsql/src/query/security.test.ts
+128 -19
View File
@@ -53,10 +53,12 @@ const taskEventsSchema: TableSchema = {
};
const defaultOptions: CompileTSQLOptions = {
organizationId: "org_tenant1",
projectId: "proj_tenant1",
environmentId: "env_tenant1",
tableSchema: [taskRunsSchema, taskEventsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
};
function compile(query: string, options: Partial<CompileTSQLOptions> = {}) {
@@ -412,8 +414,10 @@ describe("Optional Tenant Filters", () => {
describe("Organization ID is always required", () => {
it("should always inject organization guard even with optional project/env", () => {
const { sql, params } = compile("SELECT * FROM task_runs", {
projectId: undefined,
environmentId: undefined,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
});
const whereClause = getWhereClause(sql);
@@ -431,8 +435,11 @@ describe("Optional Tenant Filters", () => {
describe("Project ID is optional", () => {
it("should inject org and project guards when project is provided", () => {
const { sql, params } = compile("SELECT * FROM task_runs", {
projectId: "proj_tenant1",
environmentId: undefined,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
// environment_id omitted
},
});
const whereClause = getWhereClause(sql);
@@ -449,8 +456,10 @@ describe("Optional Tenant Filters", () => {
it("should allow querying across all projects when projectId is omitted", () => {
const { sql, params } = compile("SELECT * FROM task_runs", {
projectId: undefined,
environmentId: undefined,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
});
const whereClause = getWhereClause(sql);
@@ -479,8 +488,11 @@ describe("Optional Tenant Filters", () => {
it("should allow querying across all environments when environmentId is omitted", () => {
const { sql, params } = compile("SELECT * FROM task_runs", {
projectId: "proj_tenant1",
environmentId: undefined,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
// environment_id omitted
},
});
const whereClause = getWhereClause(sql);
@@ -501,8 +513,10 @@ describe("Optional Tenant Filters", () => {
const { sql, params } = compile(
"SELECT * FROM task_runs WHERE organization_id = 'org_other'",
{
projectId: undefined,
environmentId: undefined,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
}
);
@@ -518,8 +532,10 @@ describe("Optional Tenant Filters", () => {
JOIN task_events e ON r.id = e.run_id
`,
{
projectId: undefined,
environmentId: undefined,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
}
);
@@ -540,8 +556,10 @@ describe("Optional Tenant Filters", () => {
SELECT id, status FROM task_runs WHERE status = 'failed'
`,
{
projectId: undefined,
environmentId: undefined,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
}
);
@@ -557,8 +575,10 @@ describe("Optional Tenant Filters", () => {
WHERE id IN (SELECT run_id FROM task_events)
`,
{
projectId: undefined,
environmentId: undefined,
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
}
);
@@ -570,6 +590,95 @@ describe("Optional Tenant Filters", () => {
});
});
describe("Multi-join Tenant Guard Qualification", () => {
/**
* Security Test: Verifies that tenant guards are properly table-qualified in multi-join queries.
*
* The bug: createEnforcedGuard was building unqualified guard expressions like:
* organization_id = 'org_tenant1'
*
* In a multi-table join where both tables have the same column (organization_id),
* an unqualified reference could potentially bind to the wrong table during resolution,
* or be ambiguous. The guards should be qualified like:
* r.organization_id = 'org_tenant1' AND e.organization_id = 'org_tenant1'
*
* This ensures each table's guard binds to the correct table, not just any matching column.
*/
it("should qualify tenant guards with table alias in JOIN queries", () => {
const { sql } = compile(`
SELECT r.id, e.event_type
FROM task_runs r
JOIN task_events e ON r.id = e.run_id
`);
// The guards should be table-qualified to prevent binding to the wrong table
// Look for pattern like: r.organization_id and e.organization_id (with table alias prefix)
// The exact format in ClickHouse SQL is just "alias.column" after resolution
// Count qualified organization_id references (should have table prefixes)
// In the WHERE clause, we should see both r.organization_id and e.organization_id
const whereClause = sql.substring(sql.indexOf("WHERE"));
// Both tables should have their own qualified tenant guards
// The pattern should be: table_alias.organization_id for each table
expect(whereClause).toMatch(/\br\b[^,]*organization_id/);
expect(whereClause).toMatch(/\be\b[^,]*organization_id/);
});
it("should qualify tenant guards with table alias in LEFT JOIN queries", () => {
const { sql } = compile(`
SELECT r.id, e.event_type
FROM task_runs r
LEFT JOIN task_events e ON r.id = e.run_id
`);
const whereClause = sql.substring(sql.indexOf("WHERE"));
// Both tables should have qualified guards
expect(whereClause).toMatch(/\br\b[^,]*organization_id/);
expect(whereClause).toMatch(/\be\b[^,]*organization_id/);
});
it("should qualify tenant guards in multi-way JOIN queries", () => {
const { sql } = compile(`
SELECT r.id, e1.event_type, e2.event_type
FROM task_runs r
JOIN task_events e1 ON r.id = e1.run_id
JOIN task_events e2 ON r.id = e2.run_id
`);
const whereClause = sql.substring(sql.indexOf("WHERE"));
// All three table aliases should have qualified guards
expect(whereClause).toMatch(/\br\b[^,]*organization_id/);
expect(whereClause).toMatch(/\be1\b[^,]*organization_id/);
expect(whereClause).toMatch(/\be2\b[^,]*organization_id/);
});
it("should ensure guards cannot bind to wrong table by verifying separate qualifications", () => {
const { sql, params } = compile(`
SELECT r.id, e.event_type
FROM task_runs r
JOIN task_events e ON r.id = e.run_id
WHERE r.status = 'completed'
`);
// Count organization_id occurrences with different table prefixes
// This ensures each table gets its own guard, not shared/ambiguous references
const orgIdPattern = /(\w+)\.organization_id/g;
const matches = [...sql.matchAll(orgIdPattern)];
const tableAliases = matches.map(m => m[1]);
// Should have at least 2 different table aliases for organization_id
// (one for task_runs alias 'r' and one for task_events alias 'e')
expect(tableAliases).toContain("r");
expect(tableAliases).toContain("e");
// Both should use the same tenant value (parameterized)
expect(Object.values(params)).toContain("org_tenant1");
});
});
describe("Edge Cases", () => {
it("should handle empty string values", () => {
const { params } = compile("SELECT * FROM task_runs WHERE status = ''");
pnpm-lock.yaml
Generated
+5 -5
View File
@@ -497,8 +497,8 @@ importers:
specifier: workspace:*
version: link:../../internal-packages/otlp-importer
'@trigger.dev/platform':
specifier: 1.0.21
version: 1.0.21
specifier: 1.0.22
version: 1.0.22
'@trigger.dev/redis-worker':
specifier: workspace:*
version: link:../../packages/redis-worker
@@ -10301,8 +10301,8 @@ packages:
react: ^18.2.0
react-dom: 18.2.0
'@trigger.dev/platform@1.0.21':
resolution: {integrity: sha512-D1p+Y5pj21Un8hhN7oS/X7c+mhHKL58w1nwI9XYxbKUK1cNIIVhEMNZ0IyYmYuLelSARUXYePlKSl0v4hlusZg==}
'@trigger.dev/platform@1.0.22':
resolution: {integrity: sha512-tvPf40wqEDcQCZsHt/9A+WoQ08z+uObSWQ+oahqCgp3dSgKOUH8NdzZ/2ISSRiCkN2jURixNiUyDJmgsZipExg==}
'@types/acorn@4.0.6':
resolution: {integrity: sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==}
@@ -30255,7 +30255,7 @@ snapshots:
react: 18.2.0
react-dom: 18.2.0(react@18.2.0)
'@trigger.dev/platform@1.0.21':
'@trigger.dev/platform@1.0.22':
dependencies:
zod: 3.23.8