diff --git a/src/lib.rs b/src/lib.rs index d0c13b3f..3b4d869f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,7 +85,7 @@ pub mod webserver; use crate::app_config::AppConfig; use crate::filesystem::FileSystem; -use crate::webserver::database::ParsedSqlFile; +use crate::webserver::database::SqlFile; use crate::webserver::oidc::OidcState; use file_cache::FileCache; use std::path::{Path, PathBuf}; @@ -106,7 +106,7 @@ pub const DEFAULT_404_FILE: &str = "default_404.sql"; pub struct AppState { pub db: Database, all_templates: AllTemplates, - sql_file_cache: FileCache, + sql_file_cache: FileCache, file_system: FileSystem, config: AppConfig, pub oidc_state: Option>, @@ -124,11 +124,11 @@ impl AppState { let file_system = FileSystem::init(&config.web_root, &db).await; sql_file_cache.add_static( PathBuf::from("index.sql"), - ParsedSqlFile::new(&db, include_str!("index.sql"), Path::new("index.sql")), + SqlFile::new(&db, include_str!("index.sql"), Path::new("index.sql")), ); sql_file_cache.add_static( PathBuf::from(DEFAULT_404_FILE), - ParsedSqlFile::new( + SqlFile::new( &db, include_str!("default_404.sql"), Path::new(DEFAULT_404_FILE), diff --git a/src/webserver/database/error_highlighting.rs b/src/webserver/database/error_highlighting.rs index c2b7595f..7798727d 100644 --- a/src/webserver/database/error_highlighting.rs +++ b/src/webserver/database/error_highlighting.rs @@ -3,7 +3,7 @@ use std::{ path::{Path, PathBuf}, }; -use super::sql::{SourceSpan, StmtWithParams}; +use super::sql::SourceSpan; #[derive(Debug)] struct NiceDatabaseError { @@ -114,14 +114,15 @@ pub fn display_db_error( #[must_use] pub fn display_stmt_db_error( source_file: &Path, - stmt: &StmtWithParams, + query: &str, + query_position: SourceSpan, db_err: sqlx::error::Error, ) -> anyhow::Error { anyhow::Error::new(NiceDatabaseError { source_file: source_file.to_path_buf(), db_err, - query: stmt.query.clone(), - query_position: Some(stmt.query_position), + query: query.to_owned(), + query_position: Some(query_position), }) } diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs index 1d66990c..379b9568 100644 --- a/src/webserver/database/execute_queries.rs +++ b/src/webserver/database/execute_queries.rs @@ -10,17 +10,16 @@ use tracing::Instrument; use super::csv_import::run_csv_import; use super::error_highlighting::{display_stmt_db_error, display_stmt_error}; use super::sql::{ - DelayedFunctionCall, ParsedSqlFile, ParsedStatement, SimpleSelectValue, StmtWithParams, + DatabaseQuery, FileStatement, OutputColumn, Query, QueryBody, SingleRowQuery, SourceSpan, + SqlFile, }; -use super::sqlpage_functions::functions::SqlPageFunctionName; +use super::sqlpage_expr::{NoInputs, RowExpr, RowInputs}; use crate::dynamic_component::parse_dynamic_rows; use crate::utils::add_value_to_map; use crate::webserver::ErrorWithStatus; use crate::webserver::http_request_info::ExecutionContext; -use crate::webserver::request_variables::SetVariablesMap; use crate::webserver::single_or_vec::SingleOrVec; -use super::syntax_tree::{StmtParam, extract_req_param}; use super::{Database, DbItem, error_highlighting::display_db_error}; use sqlx::any::{AnyArguments, AnyQueryResult, AnyRow, AnyStatement, AnyTypeInfo}; use sqlx::pool::PoolConnection; @@ -30,6 +29,11 @@ use sqlx::{ pub type DbConn = Option>; +struct QueryResult { + item: DbItem, + inputs: RowInputs, +} + fn source_line_number(line: usize) -> i64 { i64::try_from(line).unwrap_or(i64::MAX) } @@ -138,14 +142,14 @@ fn create_db_query_span( fn create_query_metrics<'a>( request: &'a ExecutionContext, source_file: &Path, - statement: &StmtWithParams, - query: &StatementWithParams<'_>, + source_span: SourceSpan, + query: &BoundQuery<'_>, ) -> (tracing::Span, DbQueryMetricsContext<'a>) { let db_system_name = request.app_state.db.info.database_type.otel_name(); let (query_span, operation_name) = create_db_query_span( query.sql, source_file, - statement.query_position.start.line, + source_span.start.line, db_system_name, ); let query_metrics = DbQueryMetricsContext::new( @@ -174,7 +178,7 @@ impl Database { #[allow(clippy::too_many_lines)] // Keeps the single-connection statement dispatcher together. pub fn stream_query_results_with_conn<'a>( - sql_file: &'a ParsedSqlFile, + sql_file: &'a SqlFile, request: &'a ExecutionContext, db_connection: &'a mut DbConn, ) -> impl Stream + 'a { @@ -182,24 +186,30 @@ pub fn stream_query_results_with_conn<'a>( async_stream::try_stream! { for res in &sql_file.statements { match res { - ParsedStatement::CsvImport(csv_import) => { + FileStatement::CsvImport(csv_import) => { let connection = take_connection(&request.app_state.db, db_connection, request).await?; log::debug!("Executing CSV import: {csv_import:?}"); run_csv_import(connection, csv_import, request).await.with_context(|| format!("Failed to import the CSV file {:?} into the table {:?}", csv_import.uploaded_file, csv_import.table_name))?; }, - ParsedStatement::StmtWithParams(stmt) => { - let query = bind_parameters(stmt, request, db_connection) + FileStatement::Query(statement) => match &statement.body { + QueryBody::SingleRow(query) => { + let row = execute_single_row(query, request, db_connection) .await - .map_err(|e| with_stmt_position(source_file, stmt.query_position, e))?; + .map_err(|error| with_stmt_position(source_file, statement.source_span, error))?; + for item in parse_dynamic_rows(DbItem::Row(row)) { + yield item; + } + } + QueryBody::Database(stmt) => { + let query = bind_query(stmt, request, db_connection) + .await + .map_err(|error| with_stmt_position(source_file, statement.source_span, error))?; request.server_timing.record("bind_params"); log::trace!("Executing query {:?}", query.sql); - let (query_span, mut query_metrics) = create_query_metrics(request, source_file, stmt, &query); + let (query_span, mut query_metrics) = create_query_metrics(request, source_file, statement.source_span, &query); let mut error = None; let mut returned_rows: i64 = 0; - let defer_functions_until_stream_closed = stmt - .delayed_functions - .iter() - .any(|f| f.function == SqlPageFunctionName::run_sql); + let buffer_rows = stmt.must_buffer_rows(); let mut deferred_query_results = Vec::new(); { let connection = take_connection(&request.app_state.db, db_connection, request).await?; @@ -210,37 +220,38 @@ pub fn stream_query_results_with_conn<'a>( query_metrics.add_duration(start_next.elapsed()); let Some(elem) = next_elem else { break; }; - let mut query_result = parse_single_sql_result(source_file, stmt, elem); - if let DbItem::Error(e) = query_result { + let mut query_result = parse_single_sql_result(source_file, stmt, statement.source_span, elem); + if let DbItem::Error(e) = query_result.item { error = Some(e); break; } - if matches!(query_result, DbItem::Row(_)) { + if matches!(query_result.item, DbItem::Row(_)) { returned_rows += 1; } - apply_json_columns(&mut query_result, &stmt.json_columns); - if defer_functions_until_stream_closed { + apply_json_columns(&mut query_result.item, &stmt.json_columns); + if buffer_rows { deferred_query_results.push(query_result); } else { - if let Err(err) = apply_delayed_functions(request, &stmt.delayed_functions, &mut query_result) + let mut computed_connection = None; + if let Err(err) = evaluate_computed_columns(request, &stmt.computed_columns, &mut query_result, &mut computed_connection) .instrument(query_span.clone()) .await { error = Some(err); break; } - for db_item in parse_dynamic_rows(query_result) { + for db_item in parse_dynamic_rows(query_result.item) { yield db_item; } } } drop(stream); } - if error.is_none() && defer_functions_until_stream_closed { + if error.is_none() && buffer_rows { for mut query_result in deferred_query_results { - if let Err(err) = apply_delayed_functions_on_connection( + if let Err(err) = evaluate_computed_columns( request, - &stmt.delayed_functions, + &stmt.computed_columns, &mut query_result, db_connection, ) @@ -250,7 +261,7 @@ pub fn stream_query_results_with_conn<'a>( error = Some(err); break; } - for db_item in parse_dynamic_rows(query_result) { + for db_item in parse_dynamic_rows(query_result.item) { yield db_item; } } @@ -262,28 +273,15 @@ pub fn stream_query_results_with_conn<'a>( } else { query_metrics.record_success(returned_rows); } + } }, - ParsedStatement::SetVariable { variable, value} => { - execute_set_variable_query(db_connection, request, variable, value, source_file).await + FileStatement::SetVariable { target, value} => { + execute_set_variable_query(db_connection, request, target, value, source_file).await .with_context(|| - format!("Failed to set the {variable} variable to {value:?}") + format!("Failed to set the {} variable to {value:?}", target.0) )?; }, - ParsedStatement::StaticSimpleSet { variable, value} => { - execute_set_simple_static(db_connection, request, variable, value, source_file).await - .with_context(|| - format!("Failed to set the {variable} variable to {value:?}") - )?; - }, - ParsedStatement::StaticSimpleSelect { values, query_position } => { - let row = exec_static_simple_select(values, request, db_connection) - .await - .map_err(|e| with_stmt_position(source_file, *query_position, e))?; - for i in parse_dynamic_rows(DbItem::Row(row)) { - yield i; - } - } - ParsedStatement::Error(e) => yield DbItem::Error(clone_anyhow_err(source_file, e)), + FileStatement::Error(e) => yield DbItem::Error(clone_anyhow_err(source_file, e)), } } } @@ -325,21 +323,20 @@ pub fn stop_at_first_error( .take_until(error_rx) } -/// Executes the sqlpage pseudo-functions contained in a static simple select -async fn exec_static_simple_select( - columns: &[(String, SimpleSelectValue)], +async fn execute_single_row( + query: &SingleRowQuery, req: &ExecutionContext, db_connection: &mut DbConn, ) -> anyhow::Result { - let mut map = serde_json::Map::with_capacity(columns.len()); - for (name, value) in columns { - let value = match value { - SimpleSelectValue::Static(s) => s.clone(), - SimpleSelectValue::Dynamic(p) => { - extract_req_param_as_json(p, req, db_connection).await? - } - }; - map = add_value_to_map(map, (name.clone(), value)); + let mut map = serde_json::Map::with_capacity(query.columns.len()); + let mut inputs = NoInputs; + for column in &query.columns { + let value = column + .value + .evaluate(req, db_connection, &mut inputs) + .await? + .into_json(); + map = add_value_to_map(map, (column.name.clone(), value)); } Ok(serde_json::Value::Object(map)) } @@ -354,24 +351,10 @@ async fn try_rollback_transaction(db_connection: &mut AnyConnection) { } } -/// Extracts the value of a parameter from the request. -/// Returns `Ok(None)` when NULL should be used as the parameter value. -async fn extract_req_param_as_json( - param: &StmtParam, - request: &ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result { - if let Some(val) = extract_req_param(param, request, db_connection).await? { - Ok(serde_json::Value::String(val.into_owned())) - } else { - Ok(serde_json::Value::Null) - } -} - /// This function is used to create a pinned boxed stream of query results. /// This allows recursive calls. pub fn stream_query_results_boxed<'a>( - sql_file: &'a ParsedSqlFile, + sql_file: &'a SqlFile, request: &'a ExecutionContext, db_connection: &'a mut DbConn, ) -> Pin + 'a>> { @@ -385,16 +368,17 @@ pub fn stream_query_results_boxed<'a>( async fn execute_set_variable_query<'a>( db_connection: &'a mut DbConn, request: &'a ExecutionContext, - variable: &StmtParam, - statement: &StmtWithParams, + variable: &super::sql::VariableName, + statement: &Query, source_file: &Path, ) -> anyhow::Result<()> { let value = execute_scalar_query(db_connection, request, statement, source_file).await?; - let (mut vars, name) = vars_and_name(request, variable)?; - - log::debug!("Setting variable {name} to {value:?}"); - vars.insert(name.to_owned(), value.map(SingleOrVec::Single)); + log::debug!("Setting variable {} to {value:?}", variable.0); + request + .set_variables + .borrow_mut() + .insert(variable.0.clone(), value.map(SingleOrVec::Single)); Ok(()) } @@ -402,55 +386,76 @@ async fn execute_set_variable_query<'a>( async fn execute_scalar_query<'a>( db_connection: &'a mut DbConn, request: &'a ExecutionContext, - statement: &StmtWithParams, + statement: &Query, source_file: &Path, ) -> anyhow::Result> { - let query = bind_parameters(statement, request, db_connection).await?; - let connection = take_connection(&request.app_state.db, db_connection, request).await?; + let QueryBody::Database(database_query) = &statement.body else { + let QueryBody::SingleRow(single_row) = &statement.body else { + unreachable!() + }; + let row = execute_single_row(single_row, request, db_connection).await?; + return scalar_value_from_row(DbItem::Row(row)); + }; + let query = bind_query(database_query, request, db_connection).await?; log::debug!("Executing scalar query: {:?}", query.sql); let (query_span, mut query_metrics) = - create_query_metrics(request, source_file, statement, &query); + create_query_metrics(request, source_file, statement.source_span, &query); - let mut stream = connection.fetch_many(query); let mut scalar_row = None; let mut returned_rows: i64 = 0; let mut error = None; - loop { - let start_next = std::time::Instant::now(); - let next_elem = stream.next().instrument(query_span.clone()).await; - query_metrics.add_duration(start_next.elapsed()); - let Some(elem) = next_elem else { break }; + { + let connection = take_connection(&request.app_state.db, db_connection, request).await?; + let mut stream = connection.fetch_many(query); + loop { + let start_next = std::time::Instant::now(); + let next_elem = stream.next().instrument(query_span.clone()).await; + query_metrics.add_duration(start_next.elapsed()); + let Some(elem) = next_elem else { break }; - match parse_single_sql_result(source_file, statement, elem) { - row @ DbItem::Row(_) => { - returned_rows += 1; - if scalar_row.is_some() { - error = Some(anyhow!( - "SET scalar query returned more than one row. A SET subquery must return zero or one row." - )); + let result = + parse_single_sql_result(source_file, database_query, statement.source_span, elem); + match result.item { + row @ DbItem::Row(_) => { + returned_rows += 1; + if scalar_row.is_some() { + error = Some(anyhow!( + "SET scalar query returned more than one row. A SET subquery must return zero or one row." + )); + break; + } + scalar_row = Some(QueryResult { + item: row, + inputs: result.inputs, + }); + } + DbItem::FinishedQuery => {} + DbItem::Error(err) => { + error = Some(err); break; } - scalar_row = Some(row); - } - DbItem::FinishedQuery => {} - DbItem::Error(err) => { - error = Some(err); - break; } } + drop(stream); } - drop(stream); let value = if let Some(error) = error { + let connection = take_connection(&request.app_state.db, db_connection, request).await?; return Err( record_error_and_rollback(connection, &query_metrics, returned_rows, error).await, ); } else if let Some(mut row) = scalar_row { - apply_json_columns(&mut row, &statement.json_columns); - if let Err(error) = apply_delayed_functions(request, &statement.delayed_functions, &mut row) - .instrument(query_span.clone()) - .await + apply_json_columns(&mut row.item, &database_query.json_columns); + if let Err(error) = evaluate_computed_columns( + request, + &database_query.computed_columns, + &mut row, + db_connection, + ) + .instrument(query_span.clone()) + .await { + let connection = take_connection(&request.app_state.db, db_connection, request).await?; return Err(record_error_and_rollback( connection, &query_metrics, @@ -459,7 +464,7 @@ async fn execute_scalar_query<'a>( ) .await); } - scalar_value_from_row(row)? + scalar_value_from_row(row.item)? } else { None }; @@ -505,53 +510,6 @@ fn json_to_scalar_string(value: Value) -> Option { } } -async fn execute_set_simple_static<'a>( - db_connection: &'a mut DbConn, - request: &'a ExecutionContext, - variable: &StmtParam, - value: &SimpleSelectValue, - _source_file: &Path, -) -> anyhow::Result<()> { - let value_str = match value { - SimpleSelectValue::Static(json_value) => match json_value { - serde_json::Value::Null => None, - serde_json::Value::String(s) => Some(s.clone()), - other => Some(other.to_string()), - }, - SimpleSelectValue::Dynamic(stmt_param) => { - extract_req_param(stmt_param, request, db_connection) - .await? - .map(std::borrow::Cow::into_owned) - } - }; - - let (mut vars, name) = vars_and_name(request, variable)?; - - log::debug!("Setting variable {name} to static value {value_str:?}"); - vars.insert(name.to_owned(), value_str.map(SingleOrVec::Single)); - Ok(()) -} - -fn vars_and_name<'a, 'b>( - request: &'a ExecutionContext, - variable: &'b StmtParam, -) -> anyhow::Result<(std::cell::RefMut<'a, SetVariablesMap>, &'b str)> { - match variable { - StmtParam::PostOrGet(name) | StmtParam::Get(name) => { - if request.post_variables.contains_key(name) { - log::warn!( - "Deprecation warning! Setting the value of ${name}, but there is already a form field named :{name}. This will stop working soon. Please rename the variable, or use :{name} directly if you intended to overwrite the posted form field value." - ); - } - Ok((request.set_variables.borrow_mut(), name)) - } - StmtParam::Post(name) => Ok((request.set_variables.borrow_mut(), name)), - _ => Err(anyhow!( - "Only GET and POST variables can be set, not {variable:?}" - )), - } -} - async fn take_connection<'a>( db: &'a Database, conn: &'a mut DbConn, @@ -625,23 +583,42 @@ async fn set_trace_context(connection: &mut AnyConnection, db: &Database) { #[inline] fn parse_single_sql_result( source_file: &Path, - stmt: &StmtWithParams, + query: &DatabaseQuery, + source_span: SourceSpan, res: sqlx::Result>, -) -> DbItem { +) -> QueryResult { match res { Ok(Either::Right(r)) => { if log::log_enabled!(log::Level::Trace) { debug_row(&r); } - DbItem::Row(super::sql_to_json::row_to_json(&r)) + match super::sql_to_json::row_to_json_with_inputs(&r, query.row_input_json.len()) { + Ok((row, mut inputs)) => { + decode_json_values(&mut inputs, &query.row_input_json); + QueryResult { + item: DbItem::Row(row), + inputs: RowInputs::new(inputs), + } + } + Err(error) => QueryResult { + item: DbItem::Error(error), + inputs: RowInputs::new(Vec::new()), + }, + } } Ok(Either::Left(res)) => { log::debug!("Finished query with result: {res:?}"); - DbItem::FinishedQuery + QueryResult { + item: DbItem::FinishedQuery, + inputs: RowInputs::new(Vec::new()), + } } Err(err) => { - let nice_err = display_stmt_db_error(source_file, stmt, err); - DbItem::Error(nice_err) + let nice_err = display_stmt_db_error(source_file, &query.sql, source_span, err); + QueryResult { + item: DbItem::Error(nice_err), + inputs: RowInputs::new(Vec::new()), + } } } } @@ -667,16 +644,6 @@ fn debug_row(r: &AnyRow) { } fn clone_anyhow_err(source_file: &Path, err: &anyhow::Error) -> anyhow::Error { - if let Some(func_err) = err.downcast_ref::() { - let line = func_err.line; - let loc = if line > 0 { - format!(":{line}") - } else { - String::new() - }; - return anyhow::anyhow!("{}{loc} {}", source_file.display(), func_err); - } - let mut e = anyhow!( "{} contains a syntax error preventing SQLPage from parsing and preparing its SQL statements.", source_file.display() @@ -687,18 +654,22 @@ fn clone_anyhow_err(source_file: &Path, err: &anyhow::Error) -> anyhow::Error { e } -async fn bind_parameters<'a>( - stmt: &'a StmtWithParams, +async fn bind_query<'a>( + query: &'a DatabaseQuery, request: &'a ExecutionContext, db_connection: &mut DbConn, -) -> anyhow::Result> { - let sql = stmt.query.as_str(); +) -> anyhow::Result> { + let sql = query.sql.as_str(); log::debug!("Preparing statement: {sql}"); let mut arguments = AnyArguments::default(); - let mut param_values = Vec::with_capacity(stmt.params.len()); - for (param_idx, param) in stmt.params.iter().enumerate() { - log::trace!("\tevaluating parameter {}: {}", param_idx + 1, param); - let argument = extract_req_param(param, request, db_connection).await?; + let mut param_values = Vec::with_capacity(query.bindings.len()); + let mut inputs = NoInputs; + for (param_idx, binding) in query.bindings.iter().enumerate() { + log::trace!("\tevaluating binding {}: {:?}", param_idx + 1, binding); + let argument = binding + .evaluate(request, db_connection, &mut inputs) + .await? + .into_function_argument(); log::debug!( "\tparameter {}: {}", param_idx + 1, @@ -711,8 +682,8 @@ async fn bind_parameters<'a>( Some(Cow::Borrowed(v)) => arguments.add(v), } } - let has_arguments = !stmt.params.is_empty(); - Ok(StatementWithParams { + let has_arguments = !query.bindings.is_empty(); + Ok(BoundQuery { sql, arguments, has_arguments, @@ -720,79 +691,26 @@ async fn bind_parameters<'a>( }) } -async fn apply_delayed_functions( +async fn evaluate_computed_columns( request: &ExecutionContext, - delayed_functions: &[DelayedFunctionCall], - item: &mut DbItem, + columns: &[OutputColumn], + result: &mut QueryResult, + db_connection: &mut DbConn, ) -> anyhow::Result<()> { - // We need to open new connections for each delayed function call, because we are still fetching the results of the current query in the main connection. - let mut db_conn = None; - if let DbItem::Row(serde_json::Value::Object(results)) = item { - for f in delayed_functions { - log::trace!("Applying delayed function {} to {:?}", f.function, results); - apply_single_delayed_function(request, &mut db_conn, f, results).await?; - log::trace!( - "Delayed function applied {}. Result: {:?}", - f.function, - results - ); + if let DbItem::Row(serde_json::Value::Object(results)) = &mut result.item { + for column in columns { + let value = column + .value + .evaluate(request, db_connection, &mut result.inputs) + .await? + .into_json(); + let old_results = std::mem::take(results); + *results = add_value_to_map(old_results, (column.name.clone(), value)); } } Ok(()) } -async fn apply_delayed_functions_on_connection( - request: &ExecutionContext, - delayed_functions: &[DelayedFunctionCall], - item: &mut DbItem, - db_connection: &mut DbConn, -) -> anyhow::Result<()> { - if let DbItem::Row(serde_json::Value::Object(results)) = item { - for f in delayed_functions { - log::trace!("Applying delayed function {} to {:?}", f.function, results); - apply_single_delayed_function(request, db_connection, f, results).await?; - log::trace!( - "Delayed function applied {}. Result: {:?}", - f.function, - results - ); - } - } - Ok(()) -} - -async fn apply_single_delayed_function( - request: &ExecutionContext, - db_connection: &mut DbConn, - f: &DelayedFunctionCall, - row: &mut serde_json::Map, -) -> anyhow::Result<()> { - let mut params = Vec::new(); - for arg in &f.argument_col_names { - let Some(arg_value) = row.remove(arg) else { - anyhow::bail!( - "The column {arg} is missing in the result set, but it is required by the {} function.", - f.function - ); - }; - params.push(json_to_fn_param(arg_value)); - } - let result_str = f.function.evaluate(request, db_connection, params).await?; - let result_json = result_str - .map(Cow::into_owned) - .map_or(serde_json::Value::Null, serde_json::Value::String); - row.insert(f.target_col_name.clone(), result_json); - Ok(()) -} - -fn json_to_fn_param(json: serde_json::Value) -> Option> { - match json { - serde_json::Value::String(s) => Some(Cow::Owned(s)), - serde_json::Value::Null => None, - _ => Some(Cow::Owned(json.to_string())), - } -} - fn apply_json_columns(item: &mut DbItem, json_columns: &[String]) { if let DbItem::Row(Value::Object(row)) = item { for column in json_columns { @@ -823,14 +741,26 @@ fn apply_json_columns(item: &mut DbItem, json_columns: &[String]) { } } -pub struct StatementWithParams<'a> { +fn decode_json_values(values: &mut [Value], json_flags: &[bool]) { + debug_assert_eq!(values.len(), json_flags.len()); + for (value, decode_as_json) in values.iter_mut().zip(json_flags) { + if *decode_as_json + && let Value::String(json) = value + && let Ok(parsed) = serde_json::from_str(json) + { + *value = parsed; + } + } +} + +pub struct BoundQuery<'a> { sql: &'a str, arguments: AnyArguments<'a>, has_arguments: bool, param_values: Vec>, } -impl<'q> sqlx::Execute<'q, Any> for StatementWithParams<'q> { +impl<'q> sqlx::Execute<'q, Any> for BoundQuery<'q> { fn sql(&self) -> &'q str { self.sql } diff --git a/src/webserver/database/mod.rs b/src/webserver/database/mod.rs index b73e0cb1..1b4acf1f 100644 --- a/src/webserver/database/mod.rs +++ b/src/webserver/database/mod.rs @@ -4,14 +4,13 @@ mod csv_import; pub mod execute_queries; pub mod migrations; mod sql; +mod sqlpage_expr; mod sqlpage_functions; -mod syntax_tree; mod error_highlighting; mod sql_to_json; -pub use sql::ParsedSqlFile; -use sql::{DB_PLACEHOLDERS, DbPlaceHolder}; +pub use sql::SqlFile; use sqlx::any::AnyKind; // SupportedDatabase is defined in this module @@ -131,12 +130,10 @@ impl std::fmt::Display for Database { #[inline] #[must_use] pub fn make_placeholder(dbms: AnyKind, arg_number: usize) -> String { - if let Some((_, placeholder)) = DB_PLACEHOLDERS.iter().find(|(kind, _)| *kind == dbms) { - match *placeholder { - DbPlaceHolder::PrefixedNumber { prefix } => format!("{prefix}{arg_number}"), - DbPlaceHolder::Positional { placeholder } => placeholder.to_string(), - } - } else { - unreachable!("missing dbms: {dbms:?} in DB_PLACEHOLDERS ({DB_PLACEHOLDERS:?})") + match dbms { + AnyKind::Sqlite => format!("?{arg_number}"), + AnyKind::Postgres => format!("${arg_number}"), + AnyKind::Mssql => format!("@p{arg_number}"), + AnyKind::MySql | AnyKind::Odbc => "?".to_owned(), } } diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs index 457c9b2e..92ffb91f 100644 --- a/src/webserver/database/sql.rs +++ b/src/webserver/database/sql.rs @@ -1,288 +1,175 @@ -use super::SupportedDatabase; -use super::csv_import::{CsvImport, extract_csv_copy_statement}; -use super::sqlpage_functions::functions::SqlPageFunctionName; -use super::syntax_tree::StmtParam; -use crate::file_cache::AsyncFromStrWithState; -use crate::webserver::database::DbInfo; -use crate::webserver::database::error_highlighting::quote_source_with_highlight; -use crate::{AppState, Database}; +use std::fmt::Write as _; +use std::path::Path; + use async_trait::async_trait; use sqlparser::ast::helpers::attached_token::AttachedToken; use sqlparser::ast::{ - CastKind, DataType, Expr, Ident, ObjectName, ObjectNamePart, SelectFlavor, SelectItem, Set, - SetExpr, Spanned, Statement, Value, ValueWithSpan, -}; -use sqlparser::dialect::{ - Dialect, DuckDbDialect, GenericDialect, MsSqlDialect, MySqlDialect, OracleDialect, - PostgreSqlDialect, SQLiteDialect, SnowflakeDialect, + Expr, Ident, ObjectName, ObjectNamePart, SelectFlavor, SelectItem, Set, SetExpr, Spanned, + Statement, Value, }; +use sqlparser::dialect::Dialect; use sqlparser::parser::{Parser, ParserError}; use sqlparser::tokenizer::Token::{self, EOF, SemiColon}; use sqlparser::tokenizer::{Location, Span, TokenWithSpan, Tokenizer}; -use sqlx::any::AnyKind; -use std::fmt::Write; -use std::path::{Path, PathBuf}; -use std::str::FromStr; -mod delayed_functions; -mod parameter_extraction; -use self::delayed_functions::extract_delayed_functions_from_query; -pub(super) use self::parameter_extraction::{ - DB_PLACEHOLDERS, DbPlaceHolder, ParamExtractContext, SqlPageFunctionError, - function_args_to_stmt_params, -}; -use self::parameter_extraction::{ - ParameterExtractor, TEMP_PLACEHOLDER_PREFIX, extract_ident_param, validate_function_calls, +use super::csv_import::extract_csv_copy_statement; +use super::{Database, DbInfo, SupportedDatabase}; +use crate::AppState; +use crate::file_cache::AsyncFromStrWithState; +use crate::webserver::database::error_highlighting::quote_source_with_highlight; + +mod dialect; +mod rewrite; +mod statement; + +#[cfg(test)] +pub(super) use statement::SourceLocation; +pub use statement::SqlFile; +pub(super) use statement::{ + DatabaseQuery, FileStatement, OutputColumn, Query, QueryBody, SingleRowQuery, SourceSpan, + VariableName, }; -#[derive(Default)] -pub struct ParsedSqlFile { - pub(super) statements: Vec, - pub source_path: PathBuf, -} - -impl ParsedSqlFile { +impl SqlFile { #[must_use] - pub fn new(db: &Database, sql: &str, source_path: &Path) -> ParsedSqlFile { - let dialect = dialect_for_db(db.info.database_type); + pub fn new(db: &Database, sql: &str, source_path: &Path) -> Self { + let dialect = dialect::parser_dialect(db.info.database_type); log::debug!( "Parsing SQL file {} using dialect {:?}", source_path.display(), dialect ); - let parsed_statements = match parse_sql(&db.info, dialect.as_ref(), sql) { - Ok(parsed) => parsed, - Err(err) => return Self::from_err(err, source_path), + let statements = match parse_sql(&db.info, dialect.as_ref(), sql) { + Ok(statements) => statements.collect::>().into_boxed_slice(), + Err(error) => { + return Self::from_error(error, source_path); + } }; - let statements = parsed_statements.collect(); - ParsedSqlFile { + Self { statements, source_path: source_path.to_path_buf(), } } - fn from_err(e: impl Into, source_path: &Path) -> Self { + fn from_error(error: impl Into, source_path: &Path) -> Self { Self { - statements: vec![ParsedStatement::Error( - e.into() + statements: vec![FileStatement::Error( + error + .into() .context(format!("Error parsing file {}", source_path.display())), - )], + )] + .into_boxed_slice(), source_path: source_path.to_path_buf(), } } } -#[async_trait(? Send)] -impl AsyncFromStrWithState for ParsedSqlFile { +#[async_trait(?Send)] +impl AsyncFromStrWithState for SqlFile { async fn from_str_with_state( app_state: &AppState, source: &str, source_path: &Path, ) -> anyhow::Result { - Ok(ParsedSqlFile::new(&app_state.db, source, source_path)) + Ok(Self::new(&app_state.db, source, source_path)) } } -/// A single SQL statement that has been parsed from a SQL file. -#[derive(Debug, PartialEq)] -pub(super) struct StmtWithParams { - /// The SQL query with placeholders for parameters. - pub query: String, - /// The line and column of the first token in the query. - pub query_position: SourceSpan, - /// Parameters that should be bound to the query. - /// They can contain functions that will be called before the query is executed, - /// the result of which will be bound to the query. - pub params: Vec, - /// Functions that are called on the result set after the query has been executed, - /// and which can be passed the result of the query as an argument. - pub delayed_functions: Vec, - /// Columns that are JSON columns, and which should be converted to JSON objects after the query is executed. - /// Only relevant for databases that do not have a native JSON type, and which return JSON values as text. - pub json_columns: Vec, -} - -/// A location in the source code. -#[derive(Debug, PartialEq, Clone, Copy)] -pub(super) struct SourceSpan { - pub start: SourceLocation, - pub end: SourceLocation, -} - -/// A location in the source code. -#[derive(Debug, PartialEq, Clone, Copy)] -pub(super) struct SourceLocation { - pub line: usize, - pub column: usize, -} - -#[derive(Debug)] -pub(super) enum ParsedStatement { - StmtWithParams(StmtWithParams), - StaticSimpleSelect { - values: Vec<(String, SimpleSelectValue)>, - query_position: SourceSpan, - }, - SetVariable { - variable: StmtParam, - value: StmtWithParams, - }, - StaticSimpleSet { - variable: StmtParam, - value: SimpleSelectValue, - }, - CsvImport(CsvImport), - Error(anyhow::Error), -} - -#[derive(Debug, PartialEq, Clone)] -pub(super) enum SimpleSelectValue { - Static(serde_json::Value), - Dynamic(StmtParam), -} - fn parse_sql<'a>( - db_info: &'a DbInfo, + database: &'a DbInfo, dialect: &'a dyn Dialect, sql: &'a str, -) -> anyhow::Result + 'a> { - log::trace!("Parsing {} SQL: {sql}", db_info.dbms_name); - +) -> anyhow::Result + 'a> { + log::trace!("Parsing {} SQL: {sql}", database.dbms_name); let tokens = Tokenizer::new(dialect, sql) .tokenize_with_location() - .map_err(|err| { - let location = err.location; - anyhow::Error::new(err).context(format!("The SQLPage parser could not understand the SQL file. Tokenization failed. Please check for syntax errors:\n{}", quote_source_with_highlight(sql, location.line, location.column))) + .map_err(|error| { + let location = error.location; + anyhow::Error::new(error).context(format!( + "The SQLPage parser could not understand the SQL file. Tokenization failed. Please check for syntax errors:\n{}", + quote_source_with_highlight(sql, location.line, location.column) + )) })?; let mut parser = Parser::new(dialect).with_tokens_with_locations(tokens); let mut has_error = false; Ok(std::iter::from_fn(move || { if has_error { - // Return the first error and ignore the rest return None; } - let statement = parse_single_statement(&mut parser, db_info, sql); - log::debug!("Parsed statement: {statement:?}"); - if let Some(ParsedStatement::Error(_)) = &statement { + let statement = parse_single_statement(&mut parser, database, sql); + if matches!(statement, Some(FileStatement::Error(_))) { has_error = true; } statement })) } -fn transform_to_positional_placeholders(stmt: &mut StmtWithParams, kind: AnyKind) { - if let Some((_, DbPlaceHolder::Positional { placeholder })) = DB_PLACEHOLDERS - .iter() - .find(|(placeholder_kind, _)| *placeholder_kind == kind) - { - let mut new_params = Vec::new(); - let mut query = stmt.query.clone(); - while let Some(pos) = query.find(TEMP_PLACEHOLDER_PREFIX) { - let start_of_number = pos + TEMP_PLACEHOLDER_PREFIX.len(); - let end = query[start_of_number..] - .find(|c: char| !c.is_ascii_digit()) - .map_or(query.len(), |i| start_of_number + i); - let param_idx = query[start_of_number..end].parse::().unwrap_or(1) - 1; - query.replace_range(pos..end, placeholder); - new_params.push(stmt.params[param_idx].clone()); - } - stmt.query = query; - stmt.params = new_params; - } -} - fn parse_single_statement( parser: &mut Parser<'_>, - db_info: &DbInfo, + database: &DbInfo, source_sql: &str, -) -> Option { +) -> Option { if parser.peek_token() == EOF { return None; } - let mut stmt = match parser.parse_statement() { - Ok(stmt) => stmt, - Err(err) => return Some(syntax_error(err, parser, source_sql)), + let mut statement = match parser.parse_statement() { + Ok(statement) => statement, + Err(error) => return Some(syntax_error(error, parser, source_sql)), }; let mut semicolon = false; while parser.consume_token(&SemiColon) { semicolon = true; } - let dbms = db_info.database_type; - if let Some(parsed) = extract_set_variable(&mut stmt, db_info) { - return Some(parsed); + if let Some(statement) = extract_set_variable(&mut statement, database) { + return Some(statement); } - if let Some(csv_import) = extract_csv_copy_statement(&mut stmt) { - return Some(ParsedStatement::CsvImport(csv_import)); + if let Some(csv_import) = extract_csv_copy_statement(&mut statement) { + return Some(FileStatement::CsvImport(csv_import)); } - // SQLPage preprocessing is intentionally staged: - // - // 1. Whole projected `sqlpage.*` calls are moved out of the SQL statement before - // parameter extraction. Their argument expressions stay in the database query - // as ordinary projected columns, and the SQLPage function runs later for each - // returned row. This keeps side-effectful functions deterministic: no row means - // no call. - // 2. Remaining `sqlpage.*` calls are pre-query parameters. `ParameterExtractor` - // converts those calls to `StmtParam::FunctionCall` before generic request - // parameters inside the function arguments are rewritten to database - // placeholders. - // 3. Any `sqlpage.*` call still present after those two stages is invalid because - // it would have to run inside the database engine. - let delayed_functions = extract_delayed_functions_from_query(&mut stmt); - - let params = match ParameterExtractor::extract_parameters(&mut stmt, db_info.clone()) { - Ok(p) => p, - Err(err) => return Some(ParsedStatement::Error(err)), - }; - - if delayed_functions.is_empty() - && let Some(static_statement) = extract_static_simple_select(&stmt, ¶ms) - { - log::debug!( - "Optimised a static simple select to avoid a trivial database query: {stmt} optimized to {static_statement:?}" - ); - return Some(ParsedStatement::StaticSimpleSelect { - values: static_statement, - query_position: extract_query_start(&stmt), - }); - } - - if let Err(err) = validate_function_calls(&stmt) { - return Some(ParsedStatement::Error(err)); - } - let json_columns = extract_json_columns(&stmt, dbms); - let query = format!( - "{stmt}{semicolon}", - semicolon = if semicolon { ";" } else { "" } - ); - let mut stmt_with_params = StmtWithParams { - query, - query_position: extract_query_start(&stmt), - params, - delayed_functions, - json_columns, - }; - transform_to_positional_placeholders(&mut stmt_with_params, db_info.kind); - log::debug!("Final transformed statement: {}", stmt_with_params.query); - Some(ParsedStatement::StmtWithParams(stmt_with_params)) + Some( + match rewrite::rewrite_query(statement, database, semicolon) { + Ok(query) => FileStatement::Query(query), + Err(error) => FileStatement::Error(error), + }, + ) } -fn extract_query_start(stmt: &impl Spanned) -> SourceSpan { - let location = stmt.span(); - SourceSpan { - start: SourceLocation { - line: usize::try_from(location.start.line).unwrap_or(0), - column: usize::try_from(location.start.column).unwrap_or(0), - }, - end: SourceLocation { - line: usize::try_from(location.end.line).unwrap_or(0), - column: usize::try_from(location.end.column).unwrap_or(0), - }, +fn extract_set_variable(statement: &mut Statement, database: &DbInfo) -> Option { + let Statement::Set(Set::SingleAssignment { + variable: ObjectName(name), + values, + scope: None, + hivevar: false, + }) = statement + else { + return None; + }; + let ([ObjectNamePart::Identifier(identifier)], [value]) = + (name.as_mut_slice(), values.as_mut_slice()) + else { + return None; + }; + + let mut target = std::mem::take(&mut identifier.value); + if target.starts_with(['$', ':', '?']) { + target.remove(0); } + let expression = std::mem::replace(value, Expr::value(Value::Null)); + let value_statement = expression_to_query(expression); + Some( + match rewrite::rewrite_query(value_statement, database, false) { + Ok(value) => FileStatement::SetVariable { + target: VariableName(target), + value, + }, + Err(error) => FileStatement::Error(error), + }, + ) } -fn syntax_error(err: ParserError, parser: &Parser, sql: &str) -> ParsedStatement { +fn syntax_error(error: ParserError, parser: &Parser, sql: &str) -> FileStatement { let Span { start: Location { line: start_line, @@ -290,359 +177,138 @@ fn syntax_error(err: ParserError, parser: &Parser, sql: &str) -> ParsedStatement }, end: Location { line: end_line, .. }, } = parser.peek_token_no_skip().span; - - let mut msg = String::from( + let mut message = String::from( "Parsing failed: SQLPage couldn't understand the SQL file. Please check for syntax errors on ", ); if start_line == end_line { - write!(&mut msg, "line {start_line}:").unwrap(); + write!(&mut message, "line {start_line}:").unwrap(); } else { - write!(&mut msg, "lines {start_line} to {end_line}:").unwrap(); + write!(&mut message, "lines {start_line} to {end_line}:").unwrap(); } write!( - &mut msg, + &mut message, "\n{}", quote_source_with_highlight(sql, start_line, start_column) ) .unwrap(); - ParsedStatement::Error(anyhow::Error::from(err).context(msg)) -} - -fn dialect_for_db(dbms: SupportedDatabase) -> Box { - match dbms { - SupportedDatabase::Duckdb => Box::new(DuckDbDialect {}), - SupportedDatabase::Oracle => Box::new(OracleDialect {}), - SupportedDatabase::Postgres => Box::new(PostgreSqlDialect {}), - SupportedDatabase::Generic => Box::new(GenericDialect {}), - SupportedDatabase::Mssql => Box::new(MsSqlDialect {}), - SupportedDatabase::MySql => Box::new(MySqlDialect {}), - SupportedDatabase::Sqlite => Box::new(SQLiteDialect {}), - SupportedDatabase::Snowflake => Box::new(SnowflakeDialect {}), - } -} - -#[derive(Debug, PartialEq)] -pub struct DelayedFunctionCall { - pub function: SqlPageFunctionName, - pub argument_col_names: Vec, - pub target_col_name: String, -} - -fn extract_static_simple_select( - stmt: &Statement, - params: &[StmtParam], -) -> Option> { - let set_expr = match stmt { - Statement::Query(q) - if q.limit_clause.is_none() - && q.fetch.is_none() - && q.order_by.is_none() - && q.with.is_none() - && q.locks.is_empty() => - { - q.body.as_ref() - } - _ => return None, - }; - let select_items = match set_expr { - sqlparser::ast::SetExpr::Select(s) - if s.cluster_by.is_empty() - && s.distinct.is_none() - && s.distribute_by.is_empty() - && s.from.is_empty() - && s.group_by == sqlparser::ast::GroupByExpr::Expressions(vec![], vec![]) - && s.having.is_none() - && s.into.is_none() - && s.lateral_views.is_empty() - && s.named_window.is_empty() - && s.qualify.is_none() - && s.selection.is_none() - && s.sort_by.is_empty() - && s.top.is_none() => - { - &s.projection - } - _ => return None, - }; - let mut items = Vec::with_capacity(select_items.len()); - let mut params_iter = params.iter().cloned(); - for select_item in select_items { - let sqlparser::ast::SelectItem::ExprWithAlias { expr, alias } = select_item else { - return None; - }; - let value = expr_to_simple_select_val(&mut params_iter, expr)?; - let key = alias.value.clone(); - items.push((key, value)); - } - if let Some(p) = params_iter.next() { - log::error!("static select extraction failed because of extraneous parameter: {p:?}"); - return None; - } - Some(items) -} - -fn expr_to_simple_select_val( - params_iter: &mut impl Iterator, - expr: &Expr, -) -> Option { - use SimpleSelectValue::{Dynamic, Static}; - use serde_json::Value::{Bool, Null, Number, String}; - Some(match expr { - Expr::Value(ValueWithSpan { - value: Value::Boolean(b), - .. - }) => Static(Bool(*b)), - Expr::Value(ValueWithSpan { - value: Value::Number(n, _), - .. - }) => Static(Number(n.parse().ok()?)), - Expr::Value(ValueWithSpan { - value: Value::SingleQuotedString(s), - .. - }) => Static(String(s.clone())), - Expr::Value(ValueWithSpan { - value: Value::Null, .. - }) => Static(Null), - e if is_simple_select_placeholder(e) => { - if let Some(p) = params_iter.next() { - Dynamic(p) - } else { - log::error!("Parameter not extracted for placehorder: {expr:?}"); - return None; - } - } - other => { - log::trace!("Cancelling simple select optimization because of expr: {other:?}"); - return None; - } - }) -} - -fn is_simple_select_placeholder(e: &Expr) -> bool { - match e { - Expr::Value(ValueWithSpan { - value: Value::Placeholder(_), - .. - }) => true, - Expr::Cast { - expr, - data_type: DataType::Text | DataType::Varchar(_) | DataType::Char(_), - format: None, - kind: CastKind::Cast, - .. - } if is_simple_select_placeholder(expr) => true, - _ => false, - } -} - -fn extract_set_variable(stmt: &mut Statement, db_info: &DbInfo) -> Option { - if let Statement::Set(Set::SingleAssignment { - variable: ObjectName(name), - values, - scope: None, - hivevar: false, - }) = stmt - && let ([ObjectNamePart::Identifier(ident)], [value]) = - (name.as_mut_slice(), values.as_mut_slice()) - { - let variable = if let Some(variable) = extract_ident_param(ident) { - variable - } else { - StmtParam::PostOrGet(std::mem::take(&mut ident.value)) - }; - let owned_expr = std::mem::replace(value, Expr::value(Value::Null)); - let mut select_stmt = expr_to_value_query_statement(owned_expr); - // Keep SET expression preprocessing aligned with SELECT preprocessing: - // first isolate whole-expression SQLPage functions for delayed evaluation, - // then extract pre-query SQLPage calls and request parameters, then reject - // any SQLPage call that would have to execute inside the database. - let delayed_functions = extract_delayed_functions_from_query(&mut select_stmt); - let params = match ParameterExtractor::extract_parameters(&mut select_stmt, db_info.clone()) - { - Ok(p) => p, - Err(err) => return Some(ParsedStatement::Error(err)), - }; - if delayed_functions.is_empty() - && let Some(values) = extract_static_simple_select(&select_stmt, ¶ms) - && let [(_, value)] = values.as_slice() - { - return Some(ParsedStatement::StaticSimpleSet { - variable, - value: value.clone(), - }); - } - - if let Err(err) = validate_function_calls(&select_stmt) { - return Some(ParsedStatement::Error(err)); - } - let json_columns = extract_json_columns(&select_stmt, db_info.database_type); - let mut value = StmtWithParams { - query: select_stmt.to_string(), - query_position: extract_query_start(&select_stmt), - params, - delayed_functions, - json_columns, - }; - transform_to_positional_placeholders(&mut value, db_info.kind); - return Some(ParsedStatement::SetVariable { variable, value }); - } - None + FileStatement::Error(anyhow::Error::from(error).context(message)) } const SQLPAGE_FUNCTION_NAMESPACE: &str = "sqlpage"; -fn is_sqlpage_func(func_name_parts: &[ObjectNamePart]) -> bool { - if let [ - ObjectNamePart::Identifier(Ident { value, .. }), - ObjectNamePart::Identifier(Ident { .. }), - ] = func_name_parts - { - value == SQLPAGE_FUNCTION_NAMESPACE - } else { - false - } +pub(super) fn is_sqlpage_func(parts: &[ObjectNamePart]) -> bool { + matches!( + parts, + [ + ObjectNamePart::Identifier(Ident { + value, + quote_style: None, + .. + }), + ObjectNamePart::Identifier(Ident { quote_style: None, .. }) + ] if value.eq_ignore_ascii_case(SQLPAGE_FUNCTION_NAMESPACE) + ) } -fn extract_sqlpage_function_name( - func_name_parts: &[ObjectNamePart], -) -> Option { - if let [ - ObjectNamePart::Identifier(Ident { - value: namespace, .. - }), - ObjectNamePart::Identifier(Ident { value, .. }), - ] = func_name_parts - && namespace == SQLPAGE_FUNCTION_NAMESPACE - { - return SqlPageFunctionName::from_str(value).ok(); - } - None -} - -fn sqlpage_func_name(func_name_parts: &[ObjectNamePart]) -> &str { - if let [ - ObjectNamePart::Identifier(Ident { .. }), - ObjectNamePart::Identifier(Ident { value, .. }), - ] = func_name_parts - { - value - } else { - debug_assert!( - false, - "sqlpage function name should have been checked by is_sqlpage_func" - ); - "" - } -} - -fn extract_json_columns(stmt: &Statement, dbms: SupportedDatabase) -> Vec { - // Only extract JSON columns for databases without native JSON support - if matches!(dbms, SupportedDatabase::Postgres | SupportedDatabase::Mssql) { +pub(super) fn extract_json_columns( + statement: &Statement, + database: SupportedDatabase, +) -> Vec { + if matches!( + database, + SupportedDatabase::Postgres | SupportedDatabase::Mssql + ) { return Vec::new(); } - - let mut json_columns = Vec::new(); - - if let Statement::Query(query) = stmt - && let SetExpr::Select(select) = query.body.as_ref() - { - for item in &select.projection { - if let SelectItem::ExprWithAlias { expr, alias } = item - && is_json_function(expr) - { - json_columns.push(alias.value.clone()); - log::trace!("Found JSON column: {alias}"); + let Statement::Query(query) = statement else { + return Vec::new(); + }; + let SetExpr::Select(select) = query.body.as_ref() else { + return Vec::new(); + }; + select + .projection + .iter() + .filter_map(|item| match item { + SelectItem::ExprWithAlias { expr, alias } if is_json_expression(expr) => { + Some(alias.value.clone()) } - } - } - - json_columns + _ => None, + }) + .collect() } -fn is_json_function(expr: &Expr) -> bool { - match expr { +pub(super) fn is_json_expression(expression: &Expr) -> bool { + match expression { Expr::Function(function) => { - if let [ObjectNamePart::Identifier(Ident { value, .. })] = function.name.0.as_slice() { - [ - "json_object", - "json_array", - "json_build_object", - "json_build_array", - "to_json", - "to_jsonb", - "json_agg", - "jsonb_agg", - "json_arrayagg", - "json_objectagg", - "json_group_array", - "json_group_object", - "json", - "jsonb", - ] - .iter() - .any(|&func| value.eq_ignore_ascii_case(func)) - } else { - false - } - } - Expr::Cast { data_type, .. } => { - if matches!(data_type, DataType::JSON | DataType::JSONB) { - true - } else if let DataType::Custom(ObjectName(parts), _) = data_type { - if let [ObjectNamePart::Identifier(ident)] = parts.as_slice() { - ident.value.eq_ignore_ascii_case("json") - } else { - false - } - } else { - false - } + let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() else { + return false; + }; + [ + "json_object", + "json_array", + "json_build_object", + "json_build_array", + "to_json", + "to_jsonb", + "json_agg", + "jsonb_agg", + "json_arrayagg", + "json_objectagg", + "json_group_array", + "json_group_object", + "json", + "jsonb", + ] + .iter() + .any(|candidate| name.value.eq_ignore_ascii_case(candidate)) } + Expr::Cast { data_type, .. } => matches!( + data_type, + sqlparser::ast::DataType::JSON | sqlparser::ast::DataType::JSONB + ), _ => false, } } -fn expr_to_value_query_statement(expr: Expr) -> Statement { - if let Expr::Subquery(query) = expr { +fn expression_to_query(expression: Expr) -> Statement { + if let Expr::Subquery(query) = expression { return Statement::Query(query); } - Statement::Query(Box::new(sqlparser::ast::Query { with: None, - body: Box::new(sqlparser::ast::SetExpr::Select(Box::new( - sqlparser::ast::Select { - select_token: AttachedToken(TokenWithSpan::new( - Token::make_keyword("SELECT"), - expr.span(), - )), - distinct: None, - top: None, - projection: vec![SelectItem::ExprWithAlias { - expr, - alias: Ident::new("sqlpage_set_expr"), - }], - into: None, - from: vec![], - lateral_views: vec![], - selection: None, - group_by: sqlparser::ast::GroupByExpr::Expressions(vec![], vec![]), - cluster_by: vec![], - distribute_by: vec![], - sort_by: vec![], - having: None, - named_window: vec![], - qualify: None, - top_before_distinct: false, - prewhere: None, - window_before_qualify: false, - value_table_mode: None, - connect_by: Vec::new(), - optimizer_hints: vec![], - select_modifiers: None, - flavor: SelectFlavor::Standard, - exclude: None, - }, - ))), + body: Box::new(SetExpr::Select(Box::new(sqlparser::ast::Select { + select_token: AttachedToken(TokenWithSpan::new( + Token::make_keyword("SELECT"), + expression.span(), + )), + distinct: None, + top: None, + projection: vec![SelectItem::ExprWithAlias { + expr: expression, + alias: Ident::new("sqlpage_set_expr"), + }], + into: None, + from: vec![], + lateral_views: vec![], + selection: None, + group_by: sqlparser::ast::GroupByExpr::Expressions(vec![], vec![]), + cluster_by: vec![], + distribute_by: vec![], + sort_by: vec![], + having: None, + named_window: vec![], + qualify: None, + top_before_distinct: false, + prewhere: None, + window_before_qualify: false, + value_table_mode: None, + connect_by: Vec::new(), + optimizer_hints: vec![], + select_modifiers: None, + flavor: SelectFlavor::Standard, + exclude: None, + }))), order_by: None, limit_clause: None, fetch: None, @@ -655,74 +321,12 @@ fn expr_to_value_query_statement(expr: Expr) -> Statement { } #[cfg(test)] -mod test { - use super::super::sqlpage_functions::functions::SqlPageFunctionName; - use super::super::syntax_tree::SqlPageFunctionCall; - +mod tests { use super::*; + use sqlparser::dialect::{MySqlDialect, PostgreSqlDialect}; + use sqlx::any::AnyKind; - fn parse_stmt(sql: &str, dialect: &dyn Dialect) -> Statement { - let mut ast = Parser::parse_sql(dialect, sql).unwrap(); - assert_eq!(ast.len(), 1); - ast.pop().unwrap() - } - - fn parse_postgres_stmt(sql: &str) -> Statement { - parse_stmt(sql, &PostgreSqlDialect {}) - } - - #[test] - fn test_statement_rewrite() { - let mut ast = - parse_postgres_stmt("select $a from t where $x > $a OR $x = sqlpage.cookie('cookoo')"); - let db_info = create_test_db_info(SupportedDatabase::Postgres); - let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap(); - // $a -> $1 - // $x -> $2 - // sqlpage.cookie(...) -> $3 - assert_eq!( - ast.to_string(), - "SELECT CAST($1 AS TEXT) FROM t WHERE CAST($2 AS TEXT) > CAST($1 AS TEXT) OR CAST($2 AS TEXT) = CAST($3 AS TEXT)" - ); - assert_eq!( - parameters, - [ - StmtParam::PostOrGet("a".to_string()), - StmtParam::PostOrGet("x".to_string()), - StmtParam::FunctionCall(SqlPageFunctionCall { - function: SqlPageFunctionName::cookie, - arguments: vec![StmtParam::Literal("cookoo".to_string())] - }), - ] - ); - } - - #[test] - fn test_statement_rewrite_sqlite() { - let mut ast = parse_stmt("select $x, :y from t", &SQLiteDialect {}); - let db_info = create_test_db_info(SupportedDatabase::Sqlite); - let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap(); - assert_eq!( - ast.to_string(), - "SELECT CAST(?1 AS TEXT), CAST(?2 AS TEXT) FROM t" - ); - assert_eq!( - parameters, - [ - StmtParam::PostOrGet("x".to_string()), - StmtParam::Post("y".to_string()), - ] - ); - } - - const ALL_DIALECTS: &[(&dyn Dialect, SupportedDatabase)] = &[ - (&PostgreSqlDialect {}, SupportedDatabase::Postgres), - (&MsSqlDialect {}, SupportedDatabase::Mssql), - (&MySqlDialect {}, SupportedDatabase::MySql), - (&SQLiteDialect {}, SupportedDatabase::Sqlite), - ]; - - fn create_test_db_info(database_type: SupportedDatabase) -> DbInfo { + fn database(database_type: SupportedDatabase) -> DbInfo { let kind = match database_type { SupportedDatabase::Postgres => AnyKind::Postgres, SupportedDatabase::Mssql => AnyKind::Mssql, @@ -731,751 +335,158 @@ mod test { _ => AnyKind::Odbc, }; DbInfo { - dbms_name: database_type.display_name().to_string(), + dbms_name: database_type.display_name().to_owned(), database_type, kind, } } - #[test] - fn test_duckdb_odbc_dialect_selection() { - use std::any::Any; - - let dbms = SupportedDatabase::from_dbms_name("DuckDB"); - assert_eq!(dbms, SupportedDatabase::Duckdb); - let dialect = dialect_for_db(dbms); - assert_eq!(dialect.as_ref().type_id(), (DuckDbDialect {}).type_id()); - - let sql = "select {'a': 1, 'b': 2} as payload"; - let db_info = create_test_db_info(dbms); - let mut parsed = parse_sql(&db_info, dialect.as_ref(), sql).unwrap(); - let stmt = parsed.next().expect("expected one statement"); - assert!( - !matches!(stmt, ParsedStatement::Error(_)), - "duckdb dictionary literals should parse" - ); - - let pg_info = create_test_db_info(SupportedDatabase::Postgres); - let mut parsed = parse_sql(&pg_info, &PostgreSqlDialect {}, sql).unwrap(); - let stmt = parsed.next().expect("expected one statement"); - assert!( - matches!(stmt, ParsedStatement::Error(_)), - "postgres should reject duckdb dictionary literals" - ); - } - - #[test] - fn test_extract_delayed_functions_from_projection() { - let mut ast = parse_stmt( - "select sqlpage.fetch($x) as x, sqlpage.persist_uploaded_file('a', 'b') as y from t", + fn one(sql: &str) -> FileStatement { + parse_sql( + &database(SupportedDatabase::Postgres), &PostgreSqlDialect {}, - ); - let functions = extract_delayed_functions_from_query(&mut ast); - assert_eq!( - ast.to_string(), - "SELECT $x AS \"_sqlpage_f0_a0\", 'a' AS \"_sqlpage_f1_a0\", 'b' AS \"_sqlpage_f1_a1\" FROM t" - ); - assert_eq!( - functions, - vec![ - DelayedFunctionCall { - function: SqlPageFunctionName::fetch, - argument_col_names: vec!["_sqlpage_f0_a0".to_string()], - target_col_name: "x".to_string() - }, - DelayedFunctionCall { - function: SqlPageFunctionName::persist_uploaded_file, - argument_col_names: vec![ - "_sqlpage_f1_a0".to_string(), - "_sqlpage_f1_a1".to_string() - ], - target_col_name: "y".to_string() - } - ] - ); + sql, + ) + .unwrap() + .next() + .unwrap() } #[test] - fn test_projected_column_sqlpage_function_is_delayable() { - let mut ast = parse_postgres_stmt("select sqlpage.url_encode(url) as encoded from t"); - let functions = extract_delayed_functions_from_query(&mut ast); - assert_eq!(ast.to_string(), "SELECT url AS \"_sqlpage_f0_a0\" FROM t"); - assert_eq!( - functions, - vec![DelayedFunctionCall { - function: SqlPageFunctionName::url_encode, - argument_col_names: vec!["_sqlpage_f0_a0".to_string()], - target_col_name: "encoded".to_string() - }] - ); - } - - #[test] - fn test_constant_sqlpage_function_is_not_delayed() { - let mut ast = - parse_postgres_stmt("select sqlpage.url_encode(concat('/', $value)) as encoded"); - let functions = extract_delayed_functions_from_query(&mut ast); - - assert!(functions.is_empty()); - assert_eq!( - ast.to_string(), - "SELECT sqlpage.url_encode(concat('/', $value)) AS encoded" - ); - } - - #[test] - fn test_projected_sqlpage_function_with_nested_request_param_is_delayable() { - let sql = "select sqlpage.url_encode(json_object('recipient', :recipient))"; - for &(dialect, dbms) in ALL_DIALECTS { - let db_info = create_test_db_info(dbms); - let mut parsed = parse_sql(&db_info, dialect, sql).unwrap(); - let stmt = parsed.next().expect("one statement"); - let ParsedStatement::StmtWithParams(StmtWithParams { - query, - params, - delayed_functions, - .. - }) = stmt - else { - panic!("expected ParsedStatement::StmtWithParams for {dialect:?}: {stmt:?}"); - }; - assert_eq!( - params, - [StmtParam::Post("recipient".to_string())], - "Failed for dialect {dialect:?}" - ); - assert_eq!( - delayed_functions, - [DelayedFunctionCall { - function: SqlPageFunctionName::url_encode, - argument_col_names: vec!["_sqlpage_f0_a0".to_string()], - target_col_name: "sqlpage.url_encode(json_object('recipient', :recipient))" - .to_string() - }], - "Failed for dialect {dialect:?}" - ); - assert!( - !query.contains("sqlpage.url_encode"), - "Failed for dialect {dialect:?}: {query}" - ); - } - } - - #[test] - fn test_nested_sqlpage_functions_are_not_delayable() { - let mut ast = - parse_postgres_stmt("select coalesce(sqlpage.url_encode(url), '') as encoded from t"); - let functions = extract_delayed_functions_from_query(&mut ast); - assert_eq!(functions, []); - assert_eq!( - ast.to_string(), - "SELECT coalesce(sqlpage.url_encode(url), '') AS encoded FROM t" - ); - } - - #[test] - fn test_extract_delayed_functions_parameter_order() { - // The order of the function arguments should be preserved - // Otherwise the statement parameters will be bound to the wrong arguments - let sql = "select $a as a, sqlpage.exec('xxx', x = $b) as b, $c as c from t"; - let mut ast = parse_postgres_stmt(sql); - let delayed_functions = extract_delayed_functions_from_query(&mut ast); - assert_eq!( - ast.to_string(), - "SELECT $a AS a, 'xxx' AS \"_sqlpage_f0_a0\", x = $b AS \"_sqlpage_f0_a1\", $c AS c FROM t" - ); - assert_eq!( - delayed_functions, - &[DelayedFunctionCall { - function: SqlPageFunctionName::exec, - argument_col_names: vec![ - "_sqlpage_f0_a0".to_string(), - "_sqlpage_f0_a1".to_string() - ], - target_col_name: "b".to_string() - }] - ); - } - - #[test] - fn test_zero_argument_functions_are_delayable() { - let mut ast = parse_postgres_stmt("select sqlpage.version() as v from t"); - let delayed_functions = extract_delayed_functions_from_query(&mut ast); - assert_eq!(ast.to_string(), "SELECT NULL AS \"v\" FROM t"); - assert_eq!( - delayed_functions, - &[DelayedFunctionCall { - function: SqlPageFunctionName::version, - argument_col_names: vec![], - target_col_name: "v".to_string() - }] - ); - } - - #[test] - fn test_sqlpage_function_with_argument() { - for &(dialect, _kind) in ALL_DIALECTS { - let sql = "select sqlpage.fetch($x)"; - let mut ast = parse_stmt(sql, dialect); - let db_info = create_test_db_info(SupportedDatabase::Postgres); - let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap(); - assert_eq!( - parameters, - [StmtParam::FunctionCall(SqlPageFunctionCall { - function: SqlPageFunctionName::fetch, - arguments: vec![StmtParam::PostOrGet("x".to_string())] - })], - "Failed for dialect {dialect:?}" - ); - } - } - - #[test] - fn test_parse_sql_unsupported_expr_in_sqlpage_arg() { - let sql = "SELECT coalesce(sqlpage.link('x', json_build_object('k', c)), '') FROM (SELECT 1 AS c) t"; - let db_info = create_test_db_info(SupportedDatabase::Postgres); - let mut parsed = parse_sql(&db_info, &PostgreSqlDialect {}, sql).unwrap(); - let stmt = parsed.next().expect("one statement"); - let ParsedStatement::Error(err) = stmt else { - panic!("expected ParsedStatement::Error: {stmt:?}"); + fn database_only_parent_forces_binding() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select upper(sqlpage.url_encode('x'))") + else { + panic!("expected database query"); }; - let err_msg = format!("{err:#}"); - assert!( - err_msg.contains("Unsupported sqlpage function argument:"), - "{err_msg}" - ); - assert!(err_msg.contains("\"c\" is an sql expression, which cannot be passed as a nested sqlpage function argument."), "{err_msg}"); + assert_eq!(query.bindings.len(), 1); + assert!(query.computed_columns.is_empty()); + assert!(query.sql.contains("upper(CAST($1 AS TEXT))")); } #[test] - fn test_parse_sql_unemulated_function_in_sqlpage_arg() { - let sql = "SELECT coalesce(sqlpage.link('x', upper('a')), '') FROM (SELECT 1) t"; - let db_info = create_test_db_info(SupportedDatabase::Postgres); - let mut parsed = parse_sql(&db_info, &PostgreSqlDialect {}, sql).unwrap(); - let stmt = parsed.next().expect("one statement"); - let ParsedStatement::Error(err) = stmt else { - panic!("expected ParsedStatement::Error: {stmt:?}"); + fn emulated_parent_keeps_nested_call_per_row() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select coalesce(sqlpage.url_encode(value), '') as encoded from t") + else { + panic!("expected database query"); }; - let err_msg = format!("{err:#}"); - assert!( - err_msg.contains("Unsupported sqlpage function argument:"), - "{err_msg}" - ); - assert!( - err_msg.contains("\"upper\" is not a supported sqlpage function"), - "{err_msg}" - ); + assert!(query.bindings.is_empty()); + assert_eq!(query.row_input_json.len(), 1); + assert_eq!(query.computed_columns.len(), 1); + assert!(!query.sql.contains("sqlpage.")); } #[test] - fn test_request_param_in_nested_sqlpage_function_arg() { - let sql = "select coalesce(sqlpage.url_encode(json_object('recipient', :recipient)), '') as value"; - for &(dialect, dbms) in ALL_DIALECTS { - let db_info = create_test_db_info(dbms); - let mut parsed = parse_sql(&db_info, dialect, sql).unwrap(); - let stmt = parsed.next().expect("one statement"); - let ParsedStatement::StmtWithParams(StmtWithParams { - query, - params, - delayed_functions, - .. - }) = stmt - else { - panic!("expected ParsedStatement::StmtWithParams for {dialect:?}: {stmt:?}"); - }; - assert_eq!( - params, - [StmtParam::FunctionCall(SqlPageFunctionCall { - function: SqlPageFunctionName::url_encode, - arguments: vec![StmtParam::JsonObject(vec![ - StmtParam::Literal("recipient".to_string()), - StmtParam::Post("recipient".to_string()) - ])] - })], - "Failed for dialect {dialect:?}" - ); - assert!( - delayed_functions.is_empty(), - "Failed for dialect {dialect:?}" - ); - assert!( - !query.contains("sqlpage.url_encode"), - "Failed for dialect {dialect:?}: {query}" - ); - } + fn row_value_cannot_cross_database_only_parent() { + let FileStatement::Error(error) = one("select upper(sqlpage.url_encode(value)) from t") + else { + panic!("expected rewrite error"); + }; + assert!(format!("{error:#}").contains("required before the query")); } #[test] - fn test_set_variable_to_other_variable() { - let sql = "set x = $y"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - match parse_single_statement(&mut parser, &db_info, sql) { - Some(ParsedStatement::StaticSimpleSet { variable, value }) => { - assert_eq!( - variable, - StmtParam::PostOrGet("x".to_string()), - "{dialect:?}" - ); - assert_eq!( - value, - SimpleSelectValue::Dynamic(StmtParam::PostOrGet("y".to_string())) - ); - } - other => panic!("Failed for dialect {dialect:?}: {other:#?}"), - } - } - } - - #[test] - fn is_own_placeholder() { - assert!( - ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Postgres), - parameters: vec![], - extract_error: None, - } - .is_own_placeholder("$1") - ); - - assert!( - ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Postgres), - parameters: vec![StmtParam::Get("x".to_string())], - extract_error: None, - } - .is_own_placeholder("$2") - ); - - assert!( - !ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Postgres), - parameters: vec![], - extract_error: None, - } - .is_own_placeholder("$2") - ); - - assert!( - ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Sqlite), - parameters: vec![], - extract_error: None, - } - .is_own_placeholder("?1") - ); - - assert!( - !ParameterExtractor { - db_info: create_test_db_info(SupportedDatabase::Sqlite), - parameters: vec![], - extract_error: None, - } - .is_own_placeholder("$1") - ); - } - - #[test] - fn test_mssql_statement_rewrite() { - let mut ast = parse_stmt( - "select '' || $1 from [a schema].[a table]", - &MsSqlDialect {}, - ); - let db_info = create_test_db_info(SupportedDatabase::Mssql); - let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap(); - assert_eq!( - ast.to_string(), - "SELECT CONCAT('', CAST(@p1 AS VARCHAR(MAX))) FROM [a schema].[a table]" - ); - assert_eq!(parameters, [StmtParam::PostOrGet("1".to_string()),]); - } - - #[test] - fn test_static_extract() { - use SimpleSelectValue::Static; - - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt( - "select 'hello' as hello, 42 as answer, null as nothing, 'world' as hello" - ), - &[] - ), - Some(vec![ - ("hello".into(), Static("hello".into())), - ("answer".into(), Static(42.into())), - ("nothing".into(), Static(().into())), - ("hello".into(), Static("world".into())), - ]) - ); - } - - #[test] - fn test_simple_select_with_sqlpage_pseudofunction() { - let sql = "select 'text' as component, $x as contents, $y as title"; - let dialects: &[&dyn Dialect] = &[ - &PostgreSqlDialect {}, - &SQLiteDialect {}, + fn positional_bindings_follow_source_order() { + let database = database(SupportedDatabase::MySql); + let mut statements = parse_sql( + &database, &MySqlDialect {}, - &MsSqlDialect {}, - ]; - for &dialect in dialects { - use SimpleSelectValue::{Dynamic, Static}; - use StmtParam::PostOrGet; - use std::any::Any; - - let db_info = if dialect.type_id() == (PostgreSqlDialect {}).type_id() { - create_test_db_info(SupportedDatabase::Postgres) - } else if dialect.type_id() == (SQLiteDialect {}).type_id() { - create_test_db_info(SupportedDatabase::Sqlite) - } else if dialect.type_id() == (MySqlDialect {}).type_id() { - create_test_db_info(SupportedDatabase::MySql) - } else if dialect.type_id() == (MsSqlDialect {}).type_id() { - create_test_db_info(SupportedDatabase::Mssql) - } else { - create_test_db_info(SupportedDatabase::Generic) - }; - let parsed: Vec = parse_sql(&db_info, dialect, sql).unwrap().collect(); - match &parsed[..] { - [ParsedStatement::StaticSimpleSelect { values: q, .. }] => assert_eq!( - q, - &[ - ("component".into(), Static("text".into())), - ("contents".into(), Dynamic(PostOrGet("x".into()))), - ("title".into(), Dynamic(PostOrGet("y".into()))), - ] - ), - other => panic!("failed to extract simple select in {dialect:?}: {other:?}"), - } - } - } - - #[test] - fn test_simple_select_only_extraction() { - use SimpleSelectValue::{Dynamic, Static}; - use StmtParam::PostOrGet; - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'text' as component, $1 as contents"), - &[PostOrGet("cook".into())] - ), - Some(vec![ - ("component".into(), Static("text".into())), - ("contents".into(), Dynamic(PostOrGet("cook".into()))), - ]) - ); - } - - #[test] - fn test_extract_set_variable() { - let sql = "set x = CURRENT_TIMESTAMP"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - let stmt = parse_single_statement(&mut parser, &db_info, sql); - if let Some(ParsedStatement::SetVariable { - variable, - value: StmtWithParams { query, params, .. }, - }) = stmt - { - assert_eq!( - variable, - StmtParam::PostOrGet("x".to_string()), - "{dialect:?}" - ); - assert_eq!(query, "SELECT CURRENT_TIMESTAMP AS sqlpage_set_expr"); - assert!(params.is_empty()); - } else { - panic!("Failed for dialect {dialect:?}: {stmt:#?}"); - } - } - } - - #[test] - fn test_extract_set_variable_static() { - let sql = "set x = 'hello'"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - match parse_single_statement(&mut parser, &db_info, sql) { - Some(ParsedStatement::StaticSimpleSet { - variable: StmtParam::PostOrGet(var_name), - value: SimpleSelectValue::Static(value), - }) => { - assert_eq!(var_name, "x"); - assert_eq!(value, "hello"); - } - other => panic!("Failed for dialect {dialect:?}: {other:#?}"), - } - } - } - - #[test] - fn test_static_extract_doesnt_match() { - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer limit 0"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer order by 1"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer offset 1"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer where 1 = 0"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select 'hello' as hello, 42 as answer FROM t"), - &[] - ), - None - ); - assert_eq!( - extract_static_simple_select( - &parse_postgres_stmt("select x'CAFEBABE' as hello, 42 as answer"), - &[] - ), - None - ); - } - - #[test] - fn test_extract_json_columns() { - let sql = r" - WITH json_cte AS ( - SELECT json_build_object('a', x, 'b', y) AS cte_json - FROM generate_series(1, 3) x - JOIN generate_series(4, 6) y ON true - ) - SELECT - json_object('key', 'value') AS json_col1, - json_array(1, 2, 3) AS json_col2, - (SELECT json_build_object('nested', subq.val) - FROM (SELECT AVG(x) AS val FROM generate_series(1, 5) x) subq - ) AS json_col3, -- not supported because of the subquery - CASE - WHEN EXISTS (SELECT 1 FROM json_cte WHERE cte_json->>'a' = '2') - THEN to_json(ARRAY(SELECT cte_json FROM json_cte)) - ELSE json_build_array() - END AS json_col4, -- not supported because of the CASE - json_unknown_fn(regular_column) AS non_json_col, - CAST(json_col1 AS json) AS json_col6 - FROM some_table - CROSS JOIN json_cte - WHERE json_typeof(json_col1) = 'object' - "; - - let stmt = parse_postgres_stmt(sql); - let json_columns = extract_json_columns(&stmt, SupportedDatabase::Sqlite); - - assert_eq!( - json_columns, - vec![ - "json_col1".to_string(), - "json_col2".to_string(), - "json_col6".to_string() - ] - ); - } - - #[test] - fn test_set_variable_with_sqlpage_function() { - let sql = "set x = sqlpage.url_encode(UPPER('a'))"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - let stmt = parse_single_statement(&mut parser, &db_info, sql); - let Some(ParsedStatement::SetVariable { - variable, - value: - StmtWithParams { - query, - params, - delayed_functions, - json_columns, - .. - }, - }) = stmt - else { - panic!("for dialect {dialect:?}: {stmt:#?} instead of SetVariable"); - }; - assert_eq!( - variable, - StmtParam::PostOrGet("x".to_string()), - "{dialect:?}" - ); - assert_eq!( - delayed_functions, - [DelayedFunctionCall { - function: SqlPageFunctionName::url_encode, - argument_col_names: vec!["_sqlpage_f0_a0".to_string()], - target_col_name: "sqlpage_set_expr".to_string() - }] - ); - assert_eq!(query, "SELECT UPPER('a') AS \"_sqlpage_f0_a0\""); - assert_eq!(params, []); - assert_eq!(json_columns, Vec::::new()); - } - } - - #[test] - fn test_set_variable_subquery_uses_inner_query() { - let sql = "set x = (select sqlpage.fetch(url) from t where enabled)"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - let stmt = parse_single_statement(&mut parser, &db_info, sql); - let Some(ParsedStatement::SetVariable { - variable, - value: - StmtWithParams { - query, - params, - delayed_functions, - .. - }, - }) = stmt - else { - panic!("for dialect {dialect:?}: {stmt:#?} instead of SetVariable"); - }; - assert_eq!(variable, StmtParam::PostOrGet("x".to_string())); - assert_eq!( - delayed_functions, - [DelayedFunctionCall { - function: SqlPageFunctionName::fetch, - argument_col_names: vec!["_sqlpage_f0_a0".to_string()], - target_col_name: "sqlpage.fetch(url)".to_string() - }] - ); - assert_eq!( - query, - "SELECT url AS \"_sqlpage_f0_a0\" FROM t WHERE enabled" - ); - assert_eq!(params, []); - } - } - - #[test] - fn test_extract_json_columns_from_literal() { - let sql = r#" - SELECT - 'Pro Plan' as title, - JSON('{"icon":"database","color":"blue","description":"1GB Database"}') as item, - JSON('{"icon":"headset","color":"green","description":"Priority Support"}') as item - "#; - - let stmt = parse_stmt(sql, &SQLiteDialect {}); - let json_columns = extract_json_columns(&stmt, SupportedDatabase::Sqlite); - - assert!(json_columns.contains(&"item".to_string())); - assert!(!json_columns.contains(&"title".to_string())); - } - - #[test] - fn test_positional_placeholders() { - let sql = "select \ - @SQLPAGE_TEMP10 as a1, \ - @SQLPAGE_TEMP9 as a2, \ - @SQLPAGE_TEMP8 as a3, \ - @SQLPAGE_TEMP7 as a4, \ - @SQLPAGE_TEMP6 as a5, \ - @SQLPAGE_TEMP5 as a6, \ - @SQLPAGE_TEMP4 as a7, \ - @SQLPAGE_TEMP3 as a8, \ - @SQLPAGE_TEMP2 as a9, \ - @SQLPAGE_TEMP1 as a10 \ - @SQLPAGE_TEMP10 as a1bis \ - from t"; - let mut stmt = StmtWithParams { - query: sql.to_string(), - query_position: SourceSpan { - start: SourceLocation { line: 1, column: 1 }, - end: SourceLocation { line: 1, column: 1 }, - }, - params: vec![ - StmtParam::PostOrGet("x1".to_string()), - StmtParam::PostOrGet("x2".to_string()), - StmtParam::PostOrGet("x3".to_string()), - StmtParam::PostOrGet("x4".to_string()), - StmtParam::PostOrGet("x5".to_string()), - StmtParam::PostOrGet("x6".to_string()), - StmtParam::PostOrGet("x7".to_string()), - StmtParam::PostOrGet("x8".to_string()), - StmtParam::PostOrGet("x9".to_string()), - StmtParam::PostOrGet("x10".to_string()), - ], - delayed_functions: vec![], - json_columns: vec![], + "select $a, upper(sqlpage.url_encode($b))", + ) + .unwrap(); + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = statements.next().unwrap() + else { + panic!("expected database query"); }; - transform_to_positional_placeholders(&mut stmt, AnyKind::MySql); - assert_eq!( - stmt.query, - "select \ - ? as a1, \ - ? as a2, \ - ? as a3, \ - ? as a4, \ - ? as a5, \ - ? as a6, \ - ? as a7, \ - ? as a8, \ - ? as a9, \ - ? as a10 \ - ? as a1bis \ - from t" - ); - assert_eq!( - stmt.params, - vec![ - StmtParam::PostOrGet("x10".to_string()), - StmtParam::PostOrGet("x9".to_string()), - StmtParam::PostOrGet("x8".to_string()), - StmtParam::PostOrGet("x7".to_string()), - StmtParam::PostOrGet("x6".to_string()), - StmtParam::PostOrGet("x5".to_string()), - StmtParam::PostOrGet("x4".to_string()), - StmtParam::PostOrGet("x3".to_string()), - StmtParam::PostOrGet("x2".to_string()), - StmtParam::PostOrGet("x1".to_string()), - StmtParam::PostOrGet("x10".to_string()), - ] - ); + assert_eq!(query.bindings.len(), 2); + assert_eq!(query.sql.matches('?').count(), 2); } #[test] - fn test_set_variable_error_handling() { - let sql = "set x = db_function(sqlpage.fetch(other_db_function()))"; - for &(dialect, dbms) in ALL_DIALECTS { - let mut parser = Parser::new(dialect).try_with_sql(sql).unwrap(); - let db_info = create_test_db_info(dbms); - let stmt = parse_single_statement(&mut parser, &db_info, sql); - if let Some(ParsedStatement::Error(err)) = stmt { - assert!( - err.to_string() - .contains("Unsupported sqlpage function argument:"), - "Expected error for invalid function, got: {err}" - ); - } else { - panic!("Expected error for invalid function, got: {stmt:#?}"); - } - } + fn database_cannot_order_by_computed_column() { + let FileStatement::Error(error) = + one("select sqlpage.url_encode(value) as encoded from t order by encoded") + else { + panic!("expected rewrite error"); + }; + assert!(error.to_string().contains("ORDER BY")); + } + + #[test] + fn placeholder_like_literal_is_not_rewritten() { + let database = database(SupportedDatabase::MySql); + let statement = parse_sql( + &database, + &MySqlDialect {}, + "select '@SQLPAGE_TEMP1' as value from t where id = $id", + ) + .unwrap() + .next() + .unwrap(); + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = statement + else { + panic!("expected database query"); + }; + assert!(query.sql.contains("'@SQLPAGE_TEMP1'")); + assert_eq!(query.bindings.len(), 1); + } + + #[test] + fn effectful_bindings_are_not_deduplicated() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select 1 as value where sqlpage.random_string(1) <> sqlpage.random_string(1)") + else { + panic!("expected database query"); + }; + assert_eq!(query.bindings.len(), 2); + } + + #[test] + fn nested_run_sql_requires_buffering() { + let FileStatement::Query(Query { + body: QueryBody::Database(query), + .. + }) = one("select coalesce(sqlpage.run_sql(path), '') from files") + else { + panic!("expected database query"); + }; + assert!(query.must_buffer_rows()); + } + + #[test] + fn standalone_projection_has_no_database_query() { + let FileStatement::Query(Query { + body: QueryBody::SingleRow(query), + .. + }) = one("select sqlpage.url_encode('a b') as value") + else { + panic!("expected a single SQLPage-owned row"); + }; + assert_eq!(query.columns.len(), 1); + } + + #[test] + fn unquoted_sqlpage_names_are_case_insensitive() { + let FileStatement::Query(Query { + body: QueryBody::SingleRow(query), + .. + }) = one("select SQLPAGE.URL_ENCODE('a b') as value") + else { + panic!("expected a single SQLPage-owned row"); + }; + assert_eq!(query.columns.len(), 1); } } diff --git a/src/webserver/database/sql/delayed_functions.rs b/src/webserver/database/sql/delayed_functions.rs deleted file mode 100644 index 4e2e39fc..00000000 --- a/src/webserver/database/sql/delayed_functions.rs +++ /dev/null @@ -1,226 +0,0 @@ -use sqlparser::ast::{ - Expr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, - ObjectName, SelectItem, SetExpr, Statement, Value, -}; - -use super::{DelayedFunctionCall, SqlPageFunctionName, extract_sqlpage_function_name}; -use crate::webserver::database::sql::parameter_extraction::{ - ParamExtractContext, function_args_to_stmt_params, -}; - -/// The execution of standalone projected `SQLPage` functions is delayed until after -/// the query has been executed. For instance, `SELECT sqlpage.fetch(x) AS body FROM t` -/// is executed as `SELECT x AS _sqlpage_f0_a0 FROM t`; `sqlpage.fetch` is then -/// called with `_sqlpage_f0_a0` for each returned row. -pub(super) fn extract_delayed_functions_from_query( - stmt: &mut Statement, -) -> Vec { - match stmt { - Statement::Query(q) => { - let is_limited = q.limit_clause.is_some() || q.fetch.is_some(); - let SetExpr::Select(s) = q.body.as_mut() else { - return Vec::new(); - }; - - let is_constant_single_row = !is_limited && s.from.is_empty() && s.selection.is_none(); - extract_delayed_functions_from_projection(&mut s.projection, is_constant_single_row) - } - _ => Vec::new(), - } -} - -fn extract_delayed_functions_from_projection( - select_items: &mut Vec, - is_constant_single_row: bool, -) -> Vec { - let mut delayed_function_calls = Vec::new(); - let mut rewritten_projection = Vec::with_capacity(select_items.len()); - for item in std::mem::take(select_items) { - rewrite_select_item( - item, - is_constant_single_row, - &mut rewritten_projection, - &mut delayed_function_calls, - ); - } - *select_items = rewritten_projection; - delayed_function_calls -} - -fn rewrite_select_item( - item: SelectItem, - is_constant_single_row: bool, - rewritten_projection: &mut Vec, - delayed_function_calls: &mut Vec, -) { - match item { - SelectItem::ExprWithAlias { - expr: Expr::Function(function), - alias, - } => { - if let Some(func_name) = delayable_sqlpage_function(&function) - && (!is_constant_single_row || !preserves_null_concat_semantics(&function)) - { - let (replacement_items, delayed_call) = rewrite_function_projection( - function, - func_name, - alias.value, - delayed_function_calls.len(), - ); - rewritten_projection.extend(replacement_items); - delayed_function_calls.push(delayed_call); - } else { - rewritten_projection.push(SelectItem::ExprWithAlias { - expr: Expr::Function(function), - alias, - }); - } - } - SelectItem::UnnamedExpr(Expr::Function(function)) => { - if let Some(func_name) = delayable_sqlpage_function(&function) - && (!is_constant_single_row || !preserves_null_concat_semantics(&function)) - { - let target_col_name = function.to_string(); - let (replacement_items, delayed_call) = rewrite_function_projection( - function, - func_name, - target_col_name, - delayed_function_calls.len(), - ); - rewritten_projection.extend(replacement_items); - delayed_function_calls.push(delayed_call); - } else { - rewritten_projection.push(SelectItem::UnnamedExpr(Expr::Function(function))); - } - } - item => rewritten_projection.push(item), - } -} - -fn delayable_sqlpage_function(function: &Function) -> Option { - let Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - } = function - else { - return None; - }; - let func_name = extract_sqlpage_function_name(func_name_parts)?; - if !args.iter().all(function_arg_is_expr) { - log::error!("Unsupported argument to {func_name}: {args:?}"); - return None; - } - Some(func_name) -} - -fn preserves_null_concat_semantics(function: &Function) -> bool { - let FunctionArguments::List(FunctionArgumentList { args, .. }) = &function.args else { - return false; - }; - let mut args = args.clone(); - let has_concat = args.iter().any(function_arg_is_concat); - has_concat && function_args_to_stmt_params(&mut args, &ParamExtractContext::default()).is_ok() -} - -fn function_arg_is_concat(arg: &FunctionArg) -> bool { - let (FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) - | FunctionArg::Named { - arg: FunctionArgExpr::Expr(expr), - .. - }) = arg - else { - return false; - }; - - match expr { - Expr::BinaryOp { - op: sqlparser::ast::BinaryOperator::StringConcat, - .. - } => true, - Expr::Function(Function { - name: ObjectName(parts), - .. - }) => { - parts.len() == 1 - && parts[0] - .as_ident() - .is_some_and(|ident| ident.value.eq_ignore_ascii_case("concat")) - } - _ => false, - } -} - -fn rewrite_function_projection( - mut function: Function, - func_name: SqlPageFunctionName, - target_col_name: String, - func_idx: usize, -) -> (Vec, DelayedFunctionCall) { - let Function { - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - } = &mut function - else { - unreachable!("delayable_sqlpage_function checked the function shape") - }; - - let mut argument_col_names = Vec::with_capacity(args.len()); - let mut replacement_items = Vec::with_capacity(args.len().max(1)); - for (arg_idx, arg) in args.iter_mut().enumerate() { - let argument_col_name = format!("_sqlpage_f{func_idx}_a{arg_idx}"); - argument_col_names.push(argument_col_name.clone()); - replacement_items.push(SelectItem::ExprWithAlias { - expr: take_function_arg_expr(arg), - alias: Ident::with_quote('"', argument_col_name), - }); - } - - if replacement_items.is_empty() { - replacement_items.push(SelectItem::ExprWithAlias { - expr: Expr::value(Value::Null), - alias: Ident::with_quote('"', target_col_name.clone()), - }); - } - - ( - replacement_items, - DelayedFunctionCall { - function: func_name, - argument_col_names, - target_col_name, - }, - ) -} - -fn function_arg_is_expr(arg: &FunctionArg) -> bool { - matches!( - arg, - FunctionArg::Unnamed(FunctionArgExpr::Expr(_)) - | FunctionArg::Named { - arg: FunctionArgExpr::Expr(_), - .. - } - ) -} - -fn take_function_arg_expr(arg: &mut FunctionArg) -> Expr { - match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) - | FunctionArg::Named { - arg: FunctionArgExpr::Expr(expr), - .. - } => std::mem::replace(expr, Expr::value(Value::Null)), - _ => unreachable!("function_arg_is_expr was checked before taking arguments"), - } -} diff --git a/src/webserver/database/sql/dialect.rs b/src/webserver/database/sql/dialect.rs new file mode 100644 index 00000000..edc676f1 --- /dev/null +++ b/src/webserver/database/sql/dialect.rs @@ -0,0 +1,40 @@ +//! Database-specific parser dialects and bind-placeholder syntax. + +use sqlparser::dialect::{ + Dialect, DuckDbDialect, GenericDialect, MsSqlDialect, MySqlDialect, OracleDialect, + PostgreSqlDialect, SQLiteDialect, SnowflakeDialect, +}; +use sqlx::any::AnyKind; + +use crate::webserver::database::SupportedDatabase; + +/// Native bind-placeholder forms supported by `sqlx` database backends. +#[derive(Clone, Copy)] +pub(super) enum PlaceholderStyle { + Numbered { prefix: &'static str }, + Positional { token: &'static str }, +} + +/// Returns the `sqlparser` dialect matching the configured database. +pub(super) fn parser_dialect(database: SupportedDatabase) -> Box { + match database { + SupportedDatabase::Duckdb => Box::new(DuckDbDialect {}), + SupportedDatabase::Oracle => Box::new(OracleDialect {}), + SupportedDatabase::Postgres => Box::new(PostgreSqlDialect {}), + SupportedDatabase::Generic => Box::new(GenericDialect {}), + SupportedDatabase::Mssql => Box::new(MsSqlDialect {}), + SupportedDatabase::MySql => Box::new(MySqlDialect {}), + SupportedDatabase::Sqlite => Box::new(SQLiteDialect {}), + SupportedDatabase::Snowflake => Box::new(SnowflakeDialect {}), + } +} + +/// Returns the native placeholder syntax emitted into rewritten SQL. +pub(super) fn placeholder_style(kind: AnyKind) -> PlaceholderStyle { + match kind { + AnyKind::Sqlite => PlaceholderStyle::Numbered { prefix: "?" }, + AnyKind::Postgres => PlaceholderStyle::Numbered { prefix: "$" }, + AnyKind::Mssql => PlaceholderStyle::Numbered { prefix: "@p" }, + AnyKind::MySql | AnyKind::Odbc => PlaceholderStyle::Positional { token: "?" }, + } +} diff --git a/src/webserver/database/sql/parameter_extraction.rs b/src/webserver/database/sql/parameter_extraction.rs deleted file mode 100644 index 57bb08c4..00000000 --- a/src/webserver/database/sql/parameter_extraction.rs +++ /dev/null @@ -1,591 +0,0 @@ -use super::super::{DbInfo, SupportedDatabase}; -use super::{is_sqlpage_func, sqlpage_func_name}; -use crate::webserver::database::sqlpage_functions::func_call_to_param; -use crate::webserver::database::syntax_tree::StmtParam; -use sqlparser::ast::{ - BinaryOperator, CastKind, CharacterLength, DataType, Expr, Function, FunctionArg, - FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, - Spanned, Statement, Value, ValueWithSpan, Visit, VisitMut, Visitor, VisitorMut, -}; -use sqlx::any::AnyKind; -use std::ops::ControlFlow; - -pub(super) struct ParameterExtractor { - pub(super) db_info: DbInfo, - pub(super) parameters: Vec, - pub(super) extract_error: Option, -} - -#[derive(Debug)] -pub(crate) enum DbPlaceHolder { - PrefixedNumber { prefix: &'static str }, - Positional { placeholder: &'static str }, -} - -pub(crate) const DB_PLACEHOLDERS: [(AnyKind, DbPlaceHolder); 5] = [ - ( - AnyKind::Sqlite, - DbPlaceHolder::PrefixedNumber { prefix: "?" }, - ), - ( - AnyKind::Postgres, - DbPlaceHolder::PrefixedNumber { prefix: "$" }, - ), - ( - AnyKind::MySql, - DbPlaceHolder::Positional { placeholder: "?" }, - ), - ( - AnyKind::Mssql, - DbPlaceHolder::PrefixedNumber { prefix: "@p" }, - ), - ( - AnyKind::Odbc, - DbPlaceHolder::Positional { placeholder: "?" }, - ), -]; - -/// For positional parameters, we use a temporary placeholder during parameter extraction, -/// And then replace it with the actual placeholder during statement rewriting. -pub(crate) const TEMP_PLACEHOLDER_PREFIX: &str = "@SQLPAGE_TEMP"; - -fn get_placeholder_prefix(kind: AnyKind) -> &'static str { - if let Some((_, DbPlaceHolder::PrefixedNumber { prefix })) = DB_PLACEHOLDERS - .iter() - .find(|(placeholder_kind, _prefix)| *placeholder_kind == kind) - { - prefix - } else { - TEMP_PLACEHOLDER_PREFIX - } -} - -impl ParameterExtractor { - pub(super) fn extract_parameters( - sql_ast: &mut Statement, - db_info: DbInfo, - ) -> anyhow::Result> { - let mut this = Self { - db_info, - parameters: vec![], - extract_error: None, - }; - let _ = sql_ast.visit(&mut this); - if let Some(e) = this.extract_error { - return Err(e); - } - Ok(this.parameters) - } - - fn replace_with_placeholder(&mut self, value: &mut Expr, param: StmtParam) { - let placeholder = - if let Some(existing_idx) = self.parameters.iter().position(|p| *p == param) { - // Parameter already exists, use its index - self.make_placeholder_for_index(existing_idx + 1) - } else { - // New parameter, add it to the list - let placeholder = self.make_placeholder(); - log::trace!("Replacing {param} with {placeholder}"); - self.parameters.push(param); - placeholder - }; - *value = placeholder; - } - - fn make_placeholder_for_index(&self, index: usize) -> Expr { - let name = make_tmp_placeholder(self.db_info.kind, index); - let data_type = match self.db_info.database_type { - SupportedDatabase::MySql => DataType::Char(None), - SupportedDatabase::Mssql => DataType::Varchar(Some(CharacterLength::Max)), - SupportedDatabase::Postgres | SupportedDatabase::Sqlite => DataType::Text, - SupportedDatabase::Oracle => DataType::Varchar(Some(CharacterLength::IntegerLength { - length: 4000, - unit: None, - })), - _ => DataType::Varchar(None), - }; - let value = Expr::value(Value::Placeholder(name)); - Expr::Cast { - expr: Box::new(value), - data_type, - format: None, - kind: CastKind::Cast, - array: false, - } - } - - fn make_placeholder(&self) -> Expr { - self.make_placeholder_for_index(self.parameters.len() + 1) - } - - pub(super) fn is_own_placeholder(&self, param: &str) -> bool { - let prefix = get_placeholder_prefix(self.db_info.kind); - if let Some(param) = param.strip_prefix(prefix) - && let Ok(index) = param.parse::() - { - return index <= self.parameters.len() + 1; - } - false - } -} - -struct InvalidFunctionFinder; -impl Visitor for InvalidFunctionFinder { - type Break = (String, Vec); - fn pre_visit_expr(&mut self, value: &Expr) -> ControlFlow { - match value { - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - }) if is_sqlpage_func(func_name_parts) => { - let func_name = sqlpage_func_name(func_name_parts); - let arguments = args.clone(); - return ControlFlow::Break((func_name.to_string(), arguments)); - } - _ => (), - } - ControlFlow::Continue(()) - } -} - -pub(super) fn validate_function_calls(stmt: &Statement) -> anyhow::Result<()> { - let mut finder = InvalidFunctionFinder; - if let ControlFlow::Break((func_name, mut args)) = stmt.visit(&mut finder) { - let ctx = ParamExtractContext { - parent_func: Some(func_name.clone()), - }; - function_args_to_stmt_params(&mut args, &ctx)?; - - let args_str = FormatArguments(&args); - let error_msg = format!( - "Invalid SQLPage function call: sqlpage.{func_name}({args_str})\n\n\ - Arbitrary SQL expressions as function arguments are not supported.\n\n\ - SQLPage functions can either:\n\ - 1. Run BEFORE the query (to provide input values)\n\ - 2. Run AFTER the query (to process the results)\n\ - But they can't run DURING the query - the database doesn't know how to call them!\n\n\ - To fix this, you can either:\n\ - 1. Store the function argument in a variable first:\n\ - SET {func_name}_arg = ...;\n\ - SET {func_name}_result = sqlpage.{func_name}(${func_name}_arg);\n\ - SELECT * FROM example WHERE xxx = ${func_name}_result;\n\n\ - 2. Or move the function to the top level to process results:\n\ - SELECT sqlpage.{func_name}(...) FROM example;" - ); - Err(anyhow::anyhow!(error_msg)) - } else { - Ok(()) - } -} - -/** This is a helper struct to format a list of arguments for an error message. */ -struct FormatArguments<'a>(&'a [FunctionArg]); -impl std::fmt::Display for FormatArguments<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut args = self.0.iter(); - if let Some(arg) = args.next() { - write!(f, "{arg}")?; - } - for arg in args { - write!(f, ", {arg}")?; - } - Ok(()) - } -} - -#[derive(Clone, Default)] -pub(crate) struct ParamExtractContext { - pub parent_func: Option, -} - -impl ParamExtractContext { - fn with_parent(parent: &str) -> Self { - Self { - parent_func: Some(parent.to_string()), - } - } - - fn build_error(&self, e: &ExprToParamError, arguments: &[FunctionArg]) -> SqlPageFunctionError { - let line = e.line.unwrap_or(0); - let func_name = self.parent_func.as_deref().unwrap_or("unknown").to_string(); - let arguments_str = FormatArguments(arguments).to_string(); - - let reason = match &e.kind { - ExprToParamErrorKind::UnsupportedExpr { summary } => { - format!( - "\"{summary}\" is an sql expression, which cannot be passed as a nested sqlpage function argument." - ) - } - ExprToParamErrorKind::UnemulatedFunction { name } => { - format!( - "\"{name}\" is not a supported sqlpage function. Only a few basic sql functions like concat or json_object can be used inside sqlpage functions." - ) - } - ExprToParamErrorKind::NamedArgs => "Named function arguments are not supported.\n\ - Please use positional arguments only." - .to_string(), - }; - - SqlPageFunctionError { - line, - func_name, - arguments_str, - reason, - } - } -} - -#[derive(Debug)] -pub(crate) struct SqlPageFunctionError { - pub line: u64, - pub func_name: String, - pub arguments_str: String, - pub reason: String, -} - -impl std::fmt::Display for SqlPageFunctionError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Unsupported sqlpage function argument:\n\ -sqlpage.{func}({args_str})\n\n\ -{reason}\n\n\ -SQLPage functions can either:\n\ -1. Run BEFORE the query (to provide input values)\n\ -2. Run AFTER the query (to process the results)\n\ -But they can't run DURING the query - the database doesn't know how to call them!\n\n\ -To fix this, you can either:\n\ -1. Store the function argument in a variable first:\n\ -SET {func}_arg = ...;\n\ -SET {func}_result = sqlpage.{func}(${func}_arg);\n\ -SELECT * FROM example WHERE xxx = ${func}_result;\n\n\ -2. Or move the function to the top level to process results:\n\ -SELECT sqlpage.{func}(...) FROM example;", - func = self.func_name, - args_str = self.arguments_str, - reason = self.reason - ) - } -} -impl std::error::Error for SqlPageFunctionError {} - -#[derive(Debug)] -struct ExprToParamError { - line: Option, - kind: ExprToParamErrorKind, -} - -#[derive(Debug)] -enum ExprToParamErrorKind { - UnsupportedExpr { summary: String }, - UnemulatedFunction { name: String }, - NamedArgs, -} - -fn expr_summary(expr: &Expr) -> String { - match expr { - Expr::CompoundIdentifier(idents) => { - let s = idents - .iter() - .map(|i| i.value.as_str()) - .collect::>() - .join("."); - format!("column/table reference '{s}'") - } - _ => format!("{expr}"), - } -} - -fn function_arg_to_stmt_param( - arg: &mut FunctionArg, - ctx: &ParamExtractContext, -) -> Result { - let expr = function_arg_expr(arg).ok_or(ExprToParamError { - line: None, - kind: ExprToParamErrorKind::NamedArgs, - })?; - expr_to_stmt_param(expr, ctx) -} - -pub(crate) fn function_args_to_stmt_params( - arguments: &mut [FunctionArg], - ctx: &ParamExtractContext, -) -> anyhow::Result> { - let mut params = Vec::with_capacity(arguments.len()); - // We iterate manually so we can pass the entire `arguments` slice to into_error on failure - for arg in arguments.iter_mut() { - match function_arg_to_stmt_param(arg, ctx) { - Ok(p) => params.push(p), - Err(e) => { - let func_err = ctx.build_error(&e, arguments); - return Err(anyhow::Error::new(func_err)); - } - } - } - Ok(params) -} - -fn emulated_func_args_to_param( - func_name: &str, - args: &mut [FunctionArg], - line: u64, -) -> Result { - let inner = ParamExtractContext::with_parent(func_name); - if func_name.eq_ignore_ascii_case("concat") { - let mut concat_args = Vec::with_capacity(args.len()); - for a in args { - concat_args.push(function_arg_to_stmt_param(a, &inner)?); - } - Ok(StmtParam::Concat(concat_args)) - } else if func_name.eq_ignore_ascii_case("json_object") - || func_name.eq_ignore_ascii_case("jsonb_object") - || func_name.eq_ignore_ascii_case("json_build_object") - || func_name.eq_ignore_ascii_case("jsonb_build_object") - { - let mut json_obj_args = Vec::with_capacity(args.len()); - for a in args { - json_obj_args.push(function_arg_to_stmt_param(a, &inner)?); - } - Ok(StmtParam::JsonObject(json_obj_args)) - } else if func_name.eq_ignore_ascii_case("json_array") - || func_name.eq_ignore_ascii_case("jsonb_array") - || func_name.eq_ignore_ascii_case("json_build_array") - || func_name.eq_ignore_ascii_case("jsonb_build_array") - { - let mut json_obj_args = Vec::with_capacity(args.len()); - for a in args { - json_obj_args.push(function_arg_to_stmt_param(a, &inner)?); - } - Ok(StmtParam::JsonArray(json_obj_args)) - } else if func_name.eq_ignore_ascii_case("coalesce") { - let mut coalesce_args = Vec::with_capacity(args.len()); - for a in args { - coalesce_args.push(function_arg_to_stmt_param(a, &inner)?); - } - Ok(StmtParam::Coalesce(coalesce_args)) - } else { - Err(ExprToParamError { - line: Some(line), - kind: ExprToParamErrorKind::UnemulatedFunction { - name: func_name.to_string(), - }, - }) - } -} - -fn expr_to_stmt_param( - arg: &mut Expr, - ctx: &ParamExtractContext, -) -> Result { - let line = arg.span().start.line; - match arg { - Expr::Value(ValueWithSpan { - value: Value::Placeholder(placeholder), - .. - }) => Ok(map_param(std::mem::take(placeholder))), - Expr::Identifier(ident) => extract_ident_param(ident).ok_or_else(|| ExprToParamError { - line: Some(line), - kind: ExprToParamErrorKind::UnsupportedExpr { - summary: expr_summary(arg), - }, - }), - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - }) if is_sqlpage_func(func_name_parts) => Ok(func_call_to_param( - sqlpage_func_name(func_name_parts), - args.as_mut_slice(), - ctx, - )), - Expr::Value(ValueWithSpan { - value: Value::SingleQuotedString(param_value), - .. - }) => Ok(StmtParam::Literal(std::mem::take(param_value))), - Expr::Value(ValueWithSpan { - value: Value::Number(param_value, _is_long), - .. - }) => Ok(StmtParam::Literal(param_value.clone())), - Expr::Value(ValueWithSpan { - value: Value::Null, .. - }) => Ok(StmtParam::Null), - Expr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } => { - let left = expr_to_stmt_param(left, ctx)?; - let right = expr_to_stmt_param(right, ctx)?; - Ok(StmtParam::Concat(vec![left, right])) - } - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - .. - }) if func_name_parts.len() == 1 => { - let func_name = func_name_parts[0] - .as_ident() - .map(|ident| ident.value.as_str()) - .unwrap_or_default(); - emulated_func_args_to_param(func_name, args.as_mut_slice(), line) - } - _ => Err(ExprToParamError { - line: Some(line), - kind: ExprToParamErrorKind::UnsupportedExpr { - summary: expr_summary(arg), - }, - }), - } -} - -fn function_arg_expr(arg: &mut FunctionArg) -> Option<&mut Expr> { - match arg { - FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => Some(expr), - _ => None, - } -} - -#[inline] -#[must_use] -pub(super) fn make_tmp_placeholder(kind: AnyKind, arg_number: usize) -> String { - let prefix = if let Some((_, DbPlaceHolder::PrefixedNumber { prefix })) = - DB_PLACEHOLDERS.iter().find(|(db_typ, _)| *db_typ == kind) - { - prefix - } else { - TEMP_PLACEHOLDER_PREFIX - }; - format!("{prefix}{arg_number}") -} - -pub(super) fn extract_ident_param(Ident { value, .. }: &mut Ident) -> Option { - if value.starts_with('$') || value.starts_with(':') { - let name = std::mem::take(value); - Some(map_param(name)) - } else { - None - } -} - -fn map_param(mut name: String) -> StmtParam { - if name.is_empty() { - return StmtParam::PostOrGet(name); - } - let prefix = name.remove(0); - match prefix { - '$' => StmtParam::PostOrGet(name), - ':' => StmtParam::Post(name), - _ => StmtParam::Get(name), - } -} - -impl VisitorMut for ParameterExtractor { - type Break = (); - fn pre_visit_expr(&mut self, value: &mut Expr) -> ControlFlow { - match value { - Expr::Identifier(ident) => { - if let Some(param) = extract_ident_param(ident) { - self.replace_with_placeholder(value, param); - } - } - Expr::Value(ValueWithSpan { - value: Value::Placeholder(param), - .. - }) if !self.is_own_placeholder(param) => - // this check is to avoid recursively replacing placeholders in the form of '?', or '$1', '$2', which we emit ourselves - { - let name = std::mem::take(param); - self.replace_with_placeholder(value, map_param(name)); - } - Expr::Function(Function { - name: ObjectName(func_name_parts), - args: - FunctionArguments::List(FunctionArgumentList { - args, - duplicate_treatment: None, - .. - }), - filter: None, - null_treatment: None, - over: None, - .. - }) if is_sqlpage_func(func_name_parts) => { - let func_name = sqlpage_func_name(func_name_parts); - log::trace!("Handling builtin function: {func_name}"); - let arguments = std::mem::take(args); - let ctx = ParamExtractContext { - parent_func: Some(func_name.to_string()), - }; - let mut arguments_clone = arguments.clone(); - let param = func_call_to_param(func_name, &mut arguments_clone, &ctx); - if let StmtParam::Error(msg) = ¶m { - log::trace!("Skipping extraction of {func_name} due to: {msg}"); - *args = arguments; - return ControlFlow::Continue(()); - } - self.replace_with_placeholder(value, param); - } - // Replace 'str1' || 'str2' with CONCAT('str1', 'str2') for MSSQL - Expr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } if self.db_info.database_type == SupportedDatabase::Mssql => { - let left = std::mem::replace(left.as_mut(), Expr::value(Value::Null)); - let right = std::mem::replace(right.as_mut(), Expr::value(Value::Null)); - *value = Expr::Function(Function { - name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new("CONCAT"))]), - args: FunctionArguments::List(FunctionArgumentList { - args: vec![ - FunctionArg::Unnamed(FunctionArgExpr::Expr(left)), - FunctionArg::Unnamed(FunctionArgExpr::Expr(right)), - ], - duplicate_treatment: None, - clauses: Vec::new(), - }), - parameters: FunctionArguments::None, - over: None, - filter: None, - null_treatment: None, - within_group: Vec::new(), - uses_odbc_syntax: false, - }); - } - Expr::Cast { - kind: kind @ CastKind::DoubleColon, - .. - } if ![ - SupportedDatabase::Postgres, - SupportedDatabase::Duckdb, - SupportedDatabase::Snowflake, - SupportedDatabase::Generic, - ] - .contains(&self.db_info.database_type) => - { - log::warn!( - "Casting with '::' is not supported on your database. \ - For backwards compatibility with older SQLPage versions, we will transform it to CAST(... AS ...)." - ); - *kind = CastKind::Cast; - } - _ => (), - } - ControlFlow::<()>::Continue(()) - } -} diff --git a/src/webserver/database/sql/rewrite.rs b/src/webserver/database/sql/rewrite.rs new file mode 100644 index 00000000..95b713fd --- /dev/null +++ b/src/webserver/database/sql/rewrite.rs @@ -0,0 +1,899 @@ +//! Rewrites parsed SQL into database SQL and SQLPage-owned expressions. + +use std::ops::ControlFlow; +use std::str::FromStr as _; + +use anyhow::{Context as _, anyhow}; +use serde_json::Value as JsonValue; +use sqlparser::ast::{ + BinaryOperator, CastKind, CharacterLength, DataType, Expr as SqlExpr, Function, FunctionArg, + FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, + OrderByKind, SelectItem, SetExpr, Spanned as _, Statement as SqlStatement, Value, + ValueWithSpan, VisitMut, VisitorMut, +}; +use sqlparser::tokenizer::Span; + +use super::dialect::{PlaceholderStyle, placeholder_style}; +use super::statement::{ + DatabaseQuery, OutputColumn, Query, QueryBody, SingleRowQuery, SourceLocation, SourceSpan, +}; +use super::{extract_json_columns, is_json_expression, is_sqlpage_func}; +use crate::webserver::database::sqlpage_expr::{ + NoRowInput, RowExpr, RowInputId, SqlPageExpr, StandaloneExpr, VariableRef, VariableSource, +}; +use crate::webserver::database::sqlpage_functions::functions::SqlPageFunctionName; +use crate::webserver::database::{DbInfo, SupportedDatabase}; + +const SQLPAGE_INPUT_PREFIX: &str = "__sqlpage_input_"; + +#[derive(Debug)] +/// A binding retained with its source position until positional bindings can +/// be ordered as they appear in the rendered SQL. +struct PendingBinding { + value: StandaloneExpr, + span: Span, + sequence: usize, +} + +/// Mutable state used while rewriting one database query. +struct QueryRewriter<'a> { + database: &'a DbInfo, + bindings: Vec, + row_input_json: Vec, + private_projection: Vec, + error: Option, +} + +/// Result of partitioning one projected expression. +// Keeping the owned AST inline avoids one heap allocation for every ordinary +// projected expression. The enum is short-lived inside the rewriter. +#[allow(clippy::large_enum_variant)] +enum RewrittenProjection { + Database(SqlExpr), + PerRow(RowExpr), +} + +/// Defines how a SQL expression crossing into a SQLPage-owned expression is +/// represented at a particular evaluation site. +trait ExprEnvironment { + type Input; + + fn use_database_expr( + rewriter: &mut QueryRewriter<'_>, + expression: SqlExpr, + ) -> anyhow::Result>; +} + +/// Rejects database-owned inputs because no returned row is available. +struct StandaloneEnvironment; +/// Projects database-owned inputs into the current returned row. +struct RowEnvironment; + +impl ExprEnvironment for StandaloneEnvironment { + type Input = NoRowInput; + + fn use_database_expr( + _rewriter: &mut QueryRewriter<'_>, + expression: SqlExpr, + ) -> anyhow::Result { + if let SqlExpr::Function(function) = &expression + && let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() + { + return Err(anyhow!( + "{} is not a supported sqlpage function and cannot be evaluated before the query", + name.value + )); + } + Err(anyhow!( + "{expression} is a database expression, but its value is required before the query can run" + )) + } +} + +impl ExprEnvironment for RowEnvironment { + type Input = RowInputId; + + fn use_database_expr( + rewriter: &mut QueryRewriter<'_>, + expression: SqlExpr, + ) -> anyhow::Result { + let id = rewriter.add_row_input(expression)?; + Ok(SqlPageExpr::Input(id)) + } +} + +#[derive(Clone, Copy)] +/// SQL operations whose semantics are implemented by the shared `SQLPage` +/// expression evaluator. +enum EmulatedFunction { + Concat, + Coalesce, + JsonObject, + JsonArray, +} + +/// Rewrites one parsed statement into database SQL plus the `SQLPage` +/// expressions evaluated around it. +pub(super) fn rewrite_query( + mut statement: SqlStatement, + database: &DbInfo, + semicolon: bool, +) -> anyhow::Result { + let source_span = source_span(&statement); + let mut rewriter = QueryRewriter { + database, + bindings: Vec::new(), + row_input_json: Vec::new(), + private_projection: Vec::new(), + error: None, + }; + if let Some(single_row) = rewrite_single_row(&mut statement, &mut rewriter)? { + return Ok(Query { + body: QueryBody::SingleRow(single_row), + source_span, + }); + } + let computed_columns = rewrite_top_level_projection(&mut statement, &mut rewriter)?; + + let _ = statement.visit(&mut rewriter); + if let Some(error) = rewriter.error { + return Err(error); + } + + if let SqlStatement::Query(query) = &mut statement + && let SetExpr::Select(select) = query.body.as_mut() + && select.projection.is_empty() + { + select.projection.push(SelectItem::ExprWithAlias { + expr: SqlExpr::value(Value::Null), + alias: Ident::with_quote('"', format!("{SQLPAGE_INPUT_PREFIX}anchor")), + }); + rewriter.row_input_json.push(false); + } + + let json_columns = extract_json_columns(&statement, database.database_type) + .into_iter() + .filter(|name| !name.starts_with(SQLPAGE_INPUT_PREFIX)) + .collect(); + let bindings = rewriter.finish_bindings(); + let sql = format!( + "{statement}{semicolon}", + semicolon = if semicolon { ";" } else { "" } + ); + + Ok(Query { + body: QueryBody::Database(DatabaseQuery { + sql, + bindings, + row_input_json: rewriter.row_input_json.into_boxed_slice(), + computed_columns: computed_columns.into_boxed_slice(), + json_columns, + }), + source_span, + }) +} + +/// Removes SQLPage-owned projection expressions from the database projection +/// and appends the private database inputs they require. +fn rewrite_top_level_projection( + statement: &mut SqlStatement, + rewriter: &mut QueryRewriter<'_>, +) -> anyhow::Result>> { + let mut computed_columns = Vec::new(); + let SqlStatement::Query(query) = statement else { + return Ok(computed_columns); + }; + let SetExpr::Select(select) = query.body.as_mut() else { + return Ok(computed_columns); + }; + if select.distinct.is_some() && select.projection.iter().any(select_item_contains_sqlpage) { + anyhow::bail!( + "SQLPage-computed projections cannot be used with SELECT DISTINCT because DISTINCT must be evaluated by the database" + ); + } + + let mut database_projection = Vec::with_capacity(select.projection.len()); + for item in std::mem::take(&mut select.projection) { + match item { + SelectItem::ExprWithAlias { expr, alias } => { + match rewriter.rewrite_projection(expr)? { + RewrittenProjection::Database(expr) => { + database_projection.push(SelectItem::ExprWithAlias { expr, alias }); + } + RewrittenProjection::PerRow(value) => { + computed_columns.push(OutputColumn { + name: alias.value, + value, + }); + } + } + } + SelectItem::UnnamedExpr(expr) => { + let name = expr.to_string(); + match rewriter.rewrite_projection(expr)? { + RewrittenProjection::Database(expr) => { + database_projection.push(SelectItem::UnnamedExpr(expr)); + } + RewrittenProjection::PerRow(value) => { + computed_columns.push(OutputColumn { name, value }); + } + } + } + item => database_projection.push(item), + } + } + database_projection.append(&mut rewriter.private_projection); + select.projection = database_projection; + let computed_names = computed_columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(); + if let Some(order_by) = &query.order_by + && let OrderByKind::Expressions(expressions) = &order_by.kind + && expressions.iter().any(|ordering| { + computed_names.iter().any(|name| { + matches!( + &ordering.expr, + SqlExpr::Identifier(identifier) if identifier.value.eq_ignore_ascii_case(name) + ) + }) + }) + { + anyhow::bail!( + "ORDER BY cannot reference a SQLPage-computed column because ordering is performed by the database" + ); + } + Ok(computed_columns) +} + +/// Rewrites a guaranteed one-row query directly as standalone expressions, +/// avoiding both a database round trip and an intermediate row-expression tree. +fn rewrite_single_row( + statement: &mut SqlStatement, + rewriter: &mut QueryRewriter<'_>, +) -> anyhow::Result> { + if !has_single_row_shape(statement) { + return Ok(None); + } + let SqlStatement::Query(query) = statement else { + return Ok(None); + }; + let SetExpr::Select(select) = query.body.as_mut() else { + return Ok(None); + }; + for item in &select.projection { + let SelectItem::ExprWithAlias { expr, .. } = item else { + return Ok(None); + }; + if !can_build_standalone(expr)? { + return Ok(None); + } + } + + let mut columns = Vec::with_capacity(select.projection.len()); + for item in std::mem::take(&mut select.projection) { + let SelectItem::ExprWithAlias { expr, alias } = item else { + unreachable!("projection shape was checked") + }; + columns.push(OutputColumn { + name: alias.value, + value: build_sqlpage_expr::(rewriter, expr)?, + }); + } + Ok(Some(SingleRowQuery { + columns: columns.into_boxed_slice(), + })) +} + +fn has_single_row_shape(statement: &SqlStatement) -> bool { + let SqlStatement::Query(query) = statement else { + return false; + }; + if query.with.is_some() + || query.order_by.is_some() + || query.limit_clause.is_some() + || query.fetch.is_some() + || !query.locks.is_empty() + || query.for_clause.is_some() + || query.settings.is_some() + || query.format_clause.is_some() + || !query.pipe_operators.is_empty() + { + return false; + } + let SetExpr::Select(select) = query.body.as_ref() else { + return false; + }; + select.distinct.is_none() + && select.top.is_none() + && select.into.is_none() + && select.from.is_empty() + && select.lateral_views.is_empty() + && select.selection.is_none() + && select.group_by == sqlparser::ast::GroupByExpr::Expressions(vec![], vec![]) + && select.cluster_by.is_empty() + && select.distribute_by.is_empty() + && select.sort_by.is_empty() + && select.having.is_none() + && select.named_window.is_empty() + && select.qualify.is_none() + && select.prewhere.is_none() + && select.connect_by.is_empty() + && select.optimizer_hints.is_empty() + && select.select_modifiers.is_none() + && select.exclude.is_none() +} + +/// Checks standalone support without consuming or cloning the AST, allowing +/// callers to select a rewrite path before moving any nodes. +fn can_build_standalone(expression: &SqlExpr) -> anyhow::Result { + match expression { + SqlExpr::Value(ValueWithSpan { + value: + Value::Boolean(_) + | Value::Number(_, _) + | Value::SingleQuotedString(_) + | Value::Null + | Value::Placeholder(_), + .. + }) => Ok(true), + SqlExpr::Identifier(identifier) => Ok(variable_from_ident(identifier).is_some()), + SqlExpr::Function(function) => { + if recognize_sqlpage_function(function)?.is_none() + && emulated_function(function).is_none() + { + return Ok(false); + } + let FunctionArguments::List(arguments) = &function.args else { + return Ok(false); + }; + for argument in &arguments.args { + let FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) = argument else { + return Ok(false); + }; + if !can_build_standalone(expression)? { + return Ok(false); + } + } + Ok(true) + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => Ok(can_build_standalone(left)? && can_build_standalone(right)?), + _ => Ok(false), + } +} + +impl QueryRewriter<'_> { + /// Splits a projected expression at SQLPage-supported operations while + /// leaving opaque database operations in the SQL AST. + fn rewrite_projection(&mut self, expression: SqlExpr) -> anyhow::Result { + match expression { + SqlExpr::Function(function) => { + if recognize_sqlpage_function(&function)?.is_some() { + return build_sqlpage_expr::(self, SqlExpr::Function(function)) + .map(RewrittenProjection::PerRow); + } + if let Some(kind) = emulated_function(&function) { + return self.rewrite_emulated_projection(function, kind); + } + let mut expression = SqlExpr::Function(function); + self.rewrite_database_expression(&mut expression)?; + Ok(RewrittenProjection::Database(expression)) + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => { + let left = self.rewrite_projection(*left)?; + let right = self.rewrite_projection(*right)?; + match (left, right) { + (RewrittenProjection::Database(left), RewrittenProjection::Database(right)) => { + Ok(RewrittenProjection::Database(SqlExpr::BinaryOp { + left: Box::new(left), + op: BinaryOperator::StringConcat, + right: Box::new(right), + })) + } + (left, right) => Ok(RewrittenProjection::PerRow(SqlPageExpr::Concat( + vec![ + self.projection_into_row_expr(left)?, + self.projection_into_row_expr(right)?, + ] + .into_boxed_slice(), + ))), + } + } + mut expression => { + self.rewrite_database_expression(&mut expression)?; + Ok(RewrittenProjection::Database(expression)) + } + } + } + + fn rewrite_emulated_projection( + &mut self, + function: Function, + kind: EmulatedFunction, + ) -> anyhow::Result { + let (arguments, original) = take_expression_arguments(function)?; + let mut rewritten = Vec::with_capacity(arguments.len()); + let mut has_per_row = false; + for argument in arguments { + let argument = self.rewrite_projection(argument)?; + has_per_row |= matches!(argument, RewrittenProjection::PerRow(_)); + rewritten.push(argument); + } + if !has_per_row { + let arguments = rewritten + .into_iter() + .map(|argument| match argument { + RewrittenProjection::Database(expression) => expression, + RewrittenProjection::PerRow(_) => unreachable!(), + }) + .collect(); + return Ok(RewrittenProjection::Database(rebuild_function( + original, arguments, + ))); + } + + let arguments = rewritten + .into_iter() + .map(|argument| self.projection_into_row_expr(argument)) + .collect::>>()?; + Ok(RewrittenProjection::PerRow(build_emulated( + kind, arguments, + )?)) + } + + /// Converts a database-owned projection fragment into a typed row input, + /// while preserving an already per-row expression unchanged. + fn projection_into_row_expr( + &mut self, + projection: RewrittenProjection, + ) -> anyhow::Result { + match projection { + RewrittenProjection::Database(expression) => { + build_sqlpage_expr::(self, expression) + } + RewrittenProjection::PerRow(expression) => Ok(expression), + } + } + + fn rewrite_database_expression(&mut self, expression: &mut SqlExpr) -> anyhow::Result<()> { + let _ = expression.visit(self); + self.error.take().map_or(Ok(()), Err) + } + + fn add_binding(&mut self, value: StandaloneExpr, span: Span) -> SqlExpr { + let sequence = self.bindings.len(); + self.bindings.push(PendingBinding { + value, + span, + sequence, + }); + let placeholder = match placeholder_style(self.database.kind) { + PlaceholderStyle::Numbered { prefix } => format!("{prefix}{}", sequence + 1), + PlaceholderStyle::Positional { token } => token.to_owned(), + }; + cast_placeholder(placeholder, self.database.database_type) + } + + fn add_row_input(&mut self, mut expression: SqlExpr) -> anyhow::Result { + let decode_as_json = is_json_expression(&expression); + self.rewrite_database_expression(&mut expression)?; + let index = self.row_input_json.len(); + let name = format!("{SQLPAGE_INPUT_PREFIX}{index}"); + self.private_projection.push(SelectItem::ExprWithAlias { + expr: expression, + alias: Ident::with_quote('"', name), + }); + self.row_input_json.push(decode_as_json); + Ok(RowInputId::new(index)) + } + + /// Finalizes bindings in database placeholder order. Positional backends + /// require lexical ordering because every placeholder is spelled `?`. + fn finish_bindings(&mut self) -> Box<[StandaloneExpr]> { + if matches!( + placeholder_style(self.database.kind), + PlaceholderStyle::Positional { .. } + ) { + self.bindings.sort_by_key(|binding| { + ( + binding.span.start.line, + binding.span.start.column, + binding.sequence, + ) + }); + } + std::mem::take(&mut self.bindings) + .into_iter() + .map(|binding| binding.value) + .collect() + } +} + +impl VisitorMut for QueryRewriter<'_> { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &mut SqlExpr) -> ControlFlow { + if self.error.is_some() { + return ControlFlow::Break(()); + } + + let replacement = match expression { + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(_), + span, + }) if *span == Span::empty() => None, + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(_), + .. + }) + | SqlExpr::Identifier(_) => match variable_from_expr(expression) { + Some(variable) => { + let span = expression.span(); + Some(self.add_binding(SqlPageExpr::Variable(variable), span)) + } + None => None, + }, + SqlExpr::Function(function) => match recognize_sqlpage_function(function) { + Ok(Some(_)) => { + let span = expression.span(); + let owned = std::mem::replace(expression, SqlExpr::value(Value::Null)); + match build_sqlpage_expr::(self, owned) { + Ok(value) => Some(self.add_binding(value, span)), + Err(error) => { + self.error = Some(error.context( + "A SQLPage function used by the database could not be evaluated before the query", + )); + None + } + } + } + Ok(None) => None, + Err(error) => { + self.error = Some(error); + None + } + }, + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } if self.database.database_type == SupportedDatabase::Mssql => { + let left = std::mem::replace(left.as_mut(), SqlExpr::value(Value::Null)); + let right = std::mem::replace(right.as_mut(), SqlExpr::value(Value::Null)); + Some(make_function("CONCAT", vec![left, right])) + } + SqlExpr::Cast { + kind: kind @ CastKind::DoubleColon, + .. + } if ![ + SupportedDatabase::Postgres, + SupportedDatabase::Duckdb, + SupportedDatabase::Snowflake, + SupportedDatabase::Generic, + ] + .contains(&self.database.database_type) => + { + *kind = CastKind::Cast; + None + } + _ => None, + }; + + if let Some(replacement) = replacement { + *expression = replacement; + } + ControlFlow::Continue(()) + } +} + +/// Consumes an AST expression into the shared `SQLPage` expression type. The +/// environment determines whether opaque database fragments are illegal or +/// become private row inputs. +fn build_sqlpage_expr( + rewriter: &mut QueryRewriter<'_>, + expression: SqlExpr, +) -> anyhow::Result> { + match expression { + SqlExpr::Value(ValueWithSpan { value, .. }) => match value { + Value::Placeholder(name) => Ok(SqlPageExpr::Variable(variable_from_placeholder(name))), + Value::SingleQuotedString(text) => Ok(SqlPageExpr::Literal(JsonValue::String(text))), + Value::Number(number, _) => Ok(SqlPageExpr::Literal(JsonValue::Number( + number.parse().context("Invalid numeric SQL literal")?, + ))), + Value::Boolean(value) => Ok(SqlPageExpr::Literal(JsonValue::Bool(value))), + Value::Null => Ok(SqlPageExpr::Literal(JsonValue::Null)), + _ => { + Environment::use_database_expr(rewriter, SqlExpr::Value(ValueWithSpan::from(value))) + } + }, + SqlExpr::Identifier(identifier) => { + if let Some(variable) = variable_from_ident(&identifier) { + Ok(SqlPageExpr::Variable(variable)) + } else { + Environment::use_database_expr(rewriter, SqlExpr::Identifier(identifier)) + } + } + SqlExpr::Function(function) => { + if let Some(function_name) = recognize_sqlpage_function(&function)? { + let (arguments, _) = take_expression_arguments(function)?; + let arguments = arguments + .into_iter() + .map(|argument| build_sqlpage_expr::(rewriter, argument)) + .collect::>>()?; + Ok(SqlPageExpr::Call { + function: function_name, + arguments: arguments.into_boxed_slice(), + }) + } else if let Some(kind) = emulated_function(&function) { + let (arguments, _) = take_expression_arguments(function)?; + let arguments = arguments + .into_iter() + .map(|argument| build_sqlpage_expr::(rewriter, argument)) + .collect::>>()?; + build_emulated(kind, arguments) + } else { + Environment::use_database_expr(rewriter, SqlExpr::Function(function)) + } + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => Ok(SqlPageExpr::Concat( + vec![ + build_sqlpage_expr::(rewriter, *left)?, + build_sqlpage_expr::(rewriter, *right)?, + ] + .into_boxed_slice(), + )), + expression => Environment::use_database_expr(rewriter, expression), + } +} + +fn build_emulated( + kind: EmulatedFunction, + arguments: Vec>, +) -> anyhow::Result> { + Ok(match kind { + EmulatedFunction::Concat => SqlPageExpr::Concat(arguments.into_boxed_slice()), + EmulatedFunction::Coalesce => SqlPageExpr::Coalesce(arguments.into_boxed_slice()), + EmulatedFunction::JsonArray => SqlPageExpr::JsonArray(arguments.into_boxed_slice()), + EmulatedFunction::JsonObject => { + if !arguments.len().is_multiple_of(2) { + anyhow::bail!("JSON_OBJECT requires an even number of arguments"); + } + let mut arguments = arguments.into_iter(); + let mut entries = Vec::with_capacity(arguments.len() / 2); + while let Some(key) = arguments.next() { + let value = arguments.next().expect("argument count was checked"); + entries.push((key, value)); + } + SqlPageExpr::JsonObject(entries.into_boxed_slice()) + } + }) +} + +/// Recognizes and validates an unquoted `sqlpage.` call. A recognized +/// call is either rewritten or rejected and can never reach database SQL. +fn recognize_sqlpage_function(function: &Function) -> anyhow::Result> { + let ObjectName(parts) = &function.name; + if !is_sqlpage_func(parts) { + return Ok(None); + } + let [ + ObjectNamePart::Identifier(_), + ObjectNamePart::Identifier(name), + ] = parts.as_slice() + else { + unreachable!("is_sqlpage_func checked the name") + }; + if function.uses_odbc_syntax + || !matches!(function.parameters, FunctionArguments::None) + || function.filter.is_some() + || function.null_treatment.is_some() + || function.over.is_some() + || !function.within_group.is_empty() + { + anyhow::bail!( + "Modifiers are not supported on SQLPage function {}", + function.name + ); + } + let FunctionArguments::List(FunctionArgumentList { + duplicate_treatment: None, + clauses, + .. + }) = &function.args + else { + anyhow::bail!( + "Unsupported argument syntax for SQLPage function {}", + function.name + ); + }; + if !clauses.is_empty() { + anyhow::bail!( + "Argument clauses are not supported on SQLPage function {}", + function.name + ); + } + Ok(Some(SqlPageFunctionName::from_str(&name.value)?)) +} + +fn emulated_function(function: &Function) -> Option { + let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() else { + return None; + }; + if !matches!(function.parameters, FunctionArguments::None) + || function.filter.is_some() + || function.null_treatment.is_some() + || function.over.is_some() + || !function.within_group.is_empty() + { + return None; + } + match name.value.to_ascii_lowercase().as_str() { + "concat" => Some(EmulatedFunction::Concat), + "coalesce" => Some(EmulatedFunction::Coalesce), + "json_object" | "jsonb_object" | "json_build_object" | "jsonb_build_object" => { + Some(EmulatedFunction::JsonObject) + } + "json_array" | "jsonb_array" | "json_build_array" | "jsonb_build_array" => { + Some(EmulatedFunction::JsonArray) + } + _ => None, + } +} + +/// Moves expression arguments out of a function while retaining its emptied +/// AST shell so database-owned functions can be rebuilt without cloning. +fn take_expression_arguments(mut function: Function) -> anyhow::Result<(Vec, Function)> { + let FunctionArguments::List(arguments) = &mut function.args else { + anyhow::bail!("Unsupported arguments to {}", function.name); + }; + if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { + anyhow::bail!("Unsupported arguments to {}", function.name); + } + let arguments = std::mem::take(&mut arguments.args) + .into_iter() + .map(|argument| match argument { + FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) => Ok(expression), + _ => Err(anyhow!( + "Named and wildcard function arguments are not supported" + )), + }) + .collect::>>()?; + Ok((arguments, function)) +} + +fn rebuild_function(mut function: Function, expressions: Vec) -> SqlExpr { + let FunctionArguments::List(arguments) = &mut function.args else { + unreachable!() + }; + arguments.args = expressions + .into_iter() + .map(|expression| FunctionArg::Unnamed(FunctionArgExpr::Expr(expression))) + .collect(); + SqlExpr::Function(function) +} + +fn make_function(name: &str, expressions: Vec) -> SqlExpr { + SqlExpr::Function(Function { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new(name))]), + args: FunctionArguments::List(FunctionArgumentList { + args: expressions + .into_iter() + .map(|expression| FunctionArg::Unnamed(FunctionArgExpr::Expr(expression))) + .collect(), + duplicate_treatment: None, + clauses: Vec::new(), + }), + parameters: FunctionArguments::None, + over: None, + filter: None, + null_treatment: None, + within_group: Vec::new(), + uses_odbc_syntax: false, + }) +} + +fn variable_from_expr(expression: &SqlExpr) -> Option { + match expression { + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(name), + .. + }) => Some(variable_from_placeholder(name.clone())), + SqlExpr::Identifier(identifier) => variable_from_ident(identifier), + _ => None, + } +} + +fn variable_from_ident(identifier: &Ident) -> Option { + if identifier.quote_style.is_some() { + return None; + } + let prefix = identifier.value.chars().next()?; + matches!(prefix, '$' | ':' | '?').then(|| VariableRef { + name: identifier.value[prefix.len_utf8()..].to_owned(), + source: variable_source(prefix), + }) +} + +fn variable_from_placeholder(mut name: String) -> VariableRef { + let prefix = name.remove(0); + VariableRef { + name, + source: variable_source(prefix), + } +} + +fn variable_source(prefix: char) -> VariableSource { + match prefix { + '$' => VariableSource::SetOrUrl, + ':' => VariableSource::SetOrForm, + _ => VariableSource::Url, + } +} + +/// Wraps a generated placeholder in the backend-specific text cast expected +/// by `SQLPage`'s string-valued binding interface. +fn cast_placeholder(placeholder: String, database: SupportedDatabase) -> SqlExpr { + let data_type = match database { + SupportedDatabase::MySql => DataType::Char(None), + SupportedDatabase::Mssql => DataType::Varchar(Some(CharacterLength::Max)), + SupportedDatabase::Postgres | SupportedDatabase::Sqlite => DataType::Text, + SupportedDatabase::Oracle => DataType::Varchar(Some(CharacterLength::IntegerLength { + length: 4000, + unit: None, + })), + _ => DataType::Varchar(None), + }; + SqlExpr::Cast { + expr: Box::new(SqlExpr::value(Value::Placeholder(placeholder))), + data_type, + format: None, + kind: CastKind::Cast, + array: false, + } +} + +fn source_span(value: &impl sqlparser::ast::Spanned) -> SourceSpan { + let span = value.span(); + SourceSpan { + start: SourceLocation { + line: usize::try_from(span.start.line).unwrap_or(0), + column: usize::try_from(span.start.column).unwrap_or(0), + }, + end: SourceLocation { + line: usize::try_from(span.end.line).unwrap_or(0), + column: usize::try_from(span.end.column).unwrap_or(0), + }, + } +} + +fn select_item_contains_sqlpage(item: &SelectItem) -> bool { + struct Finder(bool); + impl sqlparser::ast::Visitor for Finder { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &SqlExpr) -> ControlFlow { + if let SqlExpr::Function(function) = expression + && is_sqlpage_func(&function.name.0) + { + self.0 = true; + return ControlFlow::Break(()); + } + ControlFlow::Continue(()) + } + } + let mut finder = Finder(false); + let _ = sqlparser::ast::Visit::visit(item, &mut finder); + finder.0 +} diff --git a/src/webserver/database/sql/statement.rs b/src/webserver/database/sql/statement.rs new file mode 100644 index 00000000..bf99dc01 --- /dev/null +++ b/src/webserver/database/sql/statement.rs @@ -0,0 +1,95 @@ +//! Immutable SQL-file statements consumed by the executor. + +use std::path::PathBuf; + +use super::super::csv_import::CsvImport; +use super::super::sqlpage_expr::{RowExpr, StandaloneExpr}; +use super::super::sqlpage_functions::functions::SqlPageFunctionName; + +/// A parsed and rewritten SQL file ready for repeated execution. +#[derive(Default)] +pub struct SqlFile { + pub(in crate::webserver::database) statements: Box<[FileStatement]>, + pub source_path: PathBuf, +} + +/// One statement in a SQL file. +#[derive(Debug)] +pub(in crate::webserver::database) enum FileStatement { + Query(Query), + SetVariable { target: VariableName, value: Query }, + CsvImport(CsvImport), + Error(anyhow::Error), +} + +/// A query and its original source location. +#[derive(Debug, PartialEq)] +pub(in crate::webserver::database) struct Query { + pub body: QueryBody, + pub source_span: SourceSpan, +} + +/// The legal ways `SQLPage` obtains rows. +/// +/// Keeping output expressions inside each variant prevents a synthetic row +/// from containing an expression that requires a database row input. +#[derive(Debug, PartialEq)] +pub(in crate::webserver::database) enum QueryBody { + Database(DatabaseQuery), + SingleRow(SingleRowQuery), +} + +/// A statement executed by the configured database. +#[derive(Debug, PartialEq)] +pub(in crate::webserver::database) struct DatabaseQuery { + pub sql: String, + /// Evaluated once, in placeholder order, before executing `sql`. + pub bindings: Box<[StandaloneExpr]>, + /// JSON decoding flags for private columns appended to the projection. + /// The slice length is the private-column count. + pub row_input_json: Box<[bool]>, + /// Evaluated once for every returned database row. + pub computed_columns: Box<[OutputColumn]>, + pub json_columns: Box<[String]>, +} + +impl DatabaseQuery { + /// Whether row evaluation needs the request's existing connection and + /// must therefore wait until the database stream is closed. + pub fn must_buffer_rows(&self) -> bool { + self.computed_columns + .iter() + .any(|column| column.value.contains_function(SqlPageFunctionName::run_sql)) + } +} + +/// Exactly one row generated without querying the database. +#[derive(Debug, PartialEq)] +pub(in crate::webserver::database) struct SingleRowQuery { + pub columns: Box<[OutputColumn]>, +} + +/// A named SQLPage-owned output expression. +#[derive(Debug, PartialEq, Eq)] +pub(in crate::webserver::database) struct OutputColumn { + pub name: String, + pub value: Expr, +} + +/// A validated variable name used as the target of a `SET` statement. +#[derive(Debug, PartialEq, Eq)] +pub(in crate::webserver::database) struct VariableName(pub String); + +/// A location in the SQL source. +#[derive(Debug, PartialEq, Clone, Copy)] +pub(in crate::webserver::database) struct SourceSpan { + pub start: SourceLocation, + pub end: SourceLocation, +} + +/// A line and column in the SQL source. +#[derive(Debug, PartialEq, Clone, Copy)] +pub(in crate::webserver::database) struct SourceLocation { + pub line: usize, + pub column: usize, +} diff --git a/src/webserver/database/sql_to_json.rs b/src/webserver/database/sql_to_json.rs index 2e9d7dad..8f77709b 100644 --- a/src/webserver/database/sql_to_json.rs +++ b/src/webserver/database/sql_to_json.rs @@ -9,6 +9,7 @@ use sqlx::postgres::types::PgRange; use sqlx::{Column, Row, TypeInfo, ValueRef}; use sqlx::{Decode, Type}; +#[cfg(test)] pub fn row_to_json(row: &AnyRow) -> Value { use Value::Object; @@ -22,6 +23,32 @@ pub fn row_to_json(row: &AnyRow) -> Value { Object(map) } +/// Decodes a row into user-visible columns and a private trailing input suffix. +/// +/// Every SQL value is decoded exactly once. Private values are addressed by +/// ordinal, so their generated SQL aliases cannot collide with user columns. +pub fn row_to_json_with_inputs( + row: &AnyRow, + input_count: usize, +) -> anyhow::Result<(Value, Vec)> { + let columns = row.columns(); + let public_count = columns + .len() + .checked_sub(input_count) + .ok_or_else(|| anyhow::anyhow!("The query returned fewer columns than SQLPage expected"))?; + let mut map = Map::new(); + let mut inputs = Vec::with_capacity(input_count); + for column in &columns[..public_count] { + let key = canonical_col_name(column); + let value = sql_to_json(row, column); + map = add_value_to_map(map, (key, value)); + } + for column in &columns[public_count..] { + inputs.push(sql_to_json(row, column)); + } + Ok((Value::Object(map), inputs)) +} + fn canonical_col_name(col: &AnyColumn) -> String { // Some databases fold all unquoted identifiers to uppercase but SQLPage uses lowercase property names if matches!(col.type_info().0, AnyTypeInfoKind::Odbc(_)) @@ -206,6 +233,19 @@ mod tests { Ok(()) } + #[actix_web::test] + async fn private_inputs_are_split_by_ordinal() -> anyhow::Result<()> { + let db_url = test_database_url(); + let mut connection = sqlx::AnyConnection::connect(&db_url).await?; + let row = sqlx::query("SELECT 1 AS __sqlpage_input_0, 2 AS __sqlpage_input_0") + .fetch_one(&mut connection) + .await?; + let (public, inputs) = row_to_json_with_inputs(&row, 1)?; + expect_json_object_equal(&public, &serde_json::json!({ "__sqlpage_input_0": 1 })); + assert_eq!(inputs, [serde_json::json!(2)]); + Ok(()) + } + #[actix_web::test] async fn test_postgres_types() -> anyhow::Result<()> { let Some(db_url) = db_specific_test("postgres") else { diff --git a/src/webserver/database/sqlpage_expr.rs b/src/webserver/database/sqlpage_expr.rs new file mode 100644 index 00000000..97a1782e --- /dev/null +++ b/src/webserver/database/sqlpage_expr.rs @@ -0,0 +1,319 @@ +//! Expressions evaluated by `SQLPage` instead of the database. +//! +//! The input type records whether an expression may read values from a +//! returned database row. The expression implementation and evaluator remain +//! shared between standalone and per-row expressions. + +use std::borrow::Cow; + +use anyhow::Context as _; +use serde_json::Value; + +use super::execute_queries::DbConn; +use super::sqlpage_functions::functions::SqlPageFunctionName; +use crate::webserver::http_request_info::ExecutionContext; +use crate::webserver::single_or_vec::SingleOrVec; + +/// An expression evaluated by `SQLPage`. +/// +/// Nodes are not shared automatically because function calls may be +/// effectful. `Input` identifies values supplied by the evaluation site. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum SqlPageExpr { + Literal(Value), + Variable(VariableRef), + Input(Input), + Call { + function: SqlPageFunctionName, + arguments: Box<[Self]>, + }, + Concat(Box<[Self]>), + Coalesce(Box<[Self]>), + JsonObject(Box<[(Self, Self)]>), + JsonArray(Box<[Self]>), +} + +/// Uninhabited input type for expressions that cannot read a database row. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum NoRowInput {} + +/// Identifies one private value projected by the database for `SQLPage`. +/// +/// The index is private and this type is intentionally not `Clone` or `Copy`. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct RowInputId(usize); + +impl RowInputId { + pub(super) fn new(index: usize) -> Self { + Self(index) + } + + pub(super) fn index(&self) -> usize { + self.0 + } +} + +/// An expression that can be evaluated without a returned database row. +pub(crate) type StandaloneExpr = SqlPageExpr; + +/// An expression evaluated once for each returned database row. +pub(crate) type RowExpr = SqlPageExpr; + +#[derive(Debug, PartialEq, Eq)] +/// A reference to request or previously assigned `SQLPage` state. +pub(crate) struct VariableRef { + pub name: String, + pub source: VariableSource, +} + +#[derive(Debug, PartialEq, Eq)] +/// Lookup precedence implied by `SQLPage`'s three variable syntaxes. +pub(crate) enum VariableSource { + /// Read only from URL parameters (`?name`). + Url, + /// Prefer a `SET` value, then read a URL parameter (`$name`). + SetOrUrl, + /// Prefer a `SET` value, then read a form field (`:name`). + SetOrForm, +} + +/// A possibly borrowed value produced by `SQLPage` expression evaluation. +pub(crate) enum SqlPageValue<'a> { + Null, + Text(Cow<'a, str>), + /// A number, boolean, array, or object. + Json(Cow<'a, Value>), +} + +impl<'a> SqlPageValue<'a> { + pub(crate) fn into_function_argument(self) -> Option> { + match self { + Self::Null => None, + Self::Text(text) => Some(text), + Self::Json(value) => Some(Cow::Owned(value.to_string())), + } + } + + pub(crate) fn into_json(self) -> Value { + match self { + Self::Null => Value::Null, + Self::Text(Cow::Borrowed(text)) => Value::String(text.to_owned()), + Self::Text(Cow::Owned(text)) => Value::String(text), + Self::Json(Cow::Borrowed(value)) => value.clone(), + Self::Json(Cow::Owned(value)) => value, + } + } +} + +/// Supplies external inputs to a `SQLPage` expression. +pub(crate) trait ExprInputs { + fn take(&mut self, input: &Input) -> anyhow::Result>; +} + +/// Input provider for standalone expressions. +pub(crate) struct NoInputs; + +impl ExprInputs for NoInputs { + fn take(&mut self, input: &NoRowInput) -> anyhow::Result> { + match *input {} + } +} + +/// Private values decoded from one returned database row. +pub(crate) struct RowInputs(Vec>); + +impl RowInputs { + pub(crate) fn new(values: Vec) -> Self { + Self(values.into_iter().map(Some).collect()) + } +} + +impl ExprInputs for RowInputs { + fn take(&mut self, input: &RowInputId) -> anyhow::Result> { + let value = self + .0 + .get_mut(input.index()) + .and_then(Option::take) + .with_context(|| { + format!( + "Row input {} is missing or was already consumed", + input.index() + ) + })?; + Ok(value_to_sqlpage_value(value)) + } +} + +fn value_to_sqlpage_value(value: Value) -> SqlPageValue<'static> { + match value { + Value::Null => SqlPageValue::Null, + Value::String(text) => SqlPageValue::Text(Cow::Owned(text)), + value => SqlPageValue::Json(Cow::Owned(value)), + } +} + +impl VariableRef { + fn evaluate<'a>(&self, request: &'a ExecutionContext) -> SqlPageValue<'a> { + let value = match self.source { + VariableSource::Url => request + .url_params + .get(&self.name) + .map(SingleOrVec::as_json_str), + VariableSource::SetOrForm => { + if let Some(value) = request.set_variables.borrow().get(&self.name) { + return value.as_ref().map_or(SqlPageValue::Null, |value| { + SqlPageValue::Text(Cow::Owned(value.as_json_str().into_owned())) + }); + } + request + .post_variables + .get(&self.name) + .map(SingleOrVec::as_json_str) + } + VariableSource::SetOrUrl => { + if let Some(value) = request.set_variables.borrow().get(&self.name) { + return value.as_ref().map_or(SqlPageValue::Null, |value| { + SqlPageValue::Text(Cow::Owned(value.as_json_str().into_owned())) + }); + } + let url_value = request.url_params.get(&self.name); + if request.post_variables.contains_key(&self.name) { + if url_value.is_some() { + log::warn!( + "Deprecation warning! There is both a URL parameter named '{}' and a form field named '{}'. SQLPage is using the URL parameter for ${}. Please use :{} to reference the form field explicitly.", + self.name, + self.name, + self.name, + self.name, + ); + } else { + log::warn!( + "Deprecation warning! ${} was used to reference a form field value (a POST variable). This now uses only URL parameters. Please use :{} instead.", + self.name, + self.name, + ); + } + } + url_value.map(SingleOrVec::as_json_str) + } + }; + value.map_or(SqlPageValue::Null, SqlPageValue::Text) + } +} + +impl SqlPageExpr { + /// Evaluates this expression from left to right. + pub(crate) async fn evaluate<'a>( + &'a self, + request: &'a ExecutionContext, + db_connection: &mut DbConn, + inputs: &mut impl ExprInputs, + ) -> anyhow::Result> { + match self { + Self::Literal(value) => Ok(match value { + Value::Null => SqlPageValue::Null, + Value::String(text) => SqlPageValue::Text(Cow::Borrowed(text)), + value => SqlPageValue::Json(Cow::Borrowed(value)), + }), + Self::Variable(variable) => Ok(variable.evaluate(request)), + Self::Input(input) => inputs.take(input).map(SqlPageValue::into_lifetime), + Self::Call { + function, + arguments, + } => { + let mut values = Vec::with_capacity(arguments.len()); + for argument in arguments { + values.push( + Box::pin(argument.evaluate(request, db_connection, inputs)) + .await? + .into_function_argument(), + ); + } + let result = function + .evaluate(request, db_connection, values) + .await + .with_context(|| format!("Error in function call {function}"))?; + Ok(result.map_or(SqlPageValue::Null, SqlPageValue::Text)) + } + Self::Concat(arguments) => { + let mut result = String::new(); + for argument in arguments { + let value = Box::pin(argument.evaluate(request, db_connection, inputs)).await?; + let Some(value) = value.into_function_argument() else { + return Ok(SqlPageValue::Null); + }; + result.push_str(&value); + } + Ok(SqlPageValue::Text(Cow::Owned(result))) + } + Self::Coalesce(arguments) => { + for argument in arguments { + let value = Box::pin(argument.evaluate(request, db_connection, inputs)).await?; + if !matches!(value, SqlPageValue::Null) { + return Ok(value); + } + } + Ok(SqlPageValue::Null) + } + Self::JsonObject(entries) => { + let mut object = serde_json::Map::with_capacity(entries.len()); + for (key, value) in entries { + let key = Box::pin(key.evaluate(request, db_connection, inputs)) + .await? + .into_function_argument() + .context("JSON object keys cannot be NULL")? + .into_owned(); + let value = Box::pin(value.evaluate(request, db_connection, inputs)) + .await? + .into_json(); + object.insert(key, value); + } + Ok(SqlPageValue::Json(Cow::Owned(Value::Object(object)))) + } + Self::JsonArray(elements) => { + let mut array = Vec::with_capacity(elements.len()); + for element in elements { + array.push( + Box::pin(element.evaluate(request, db_connection, inputs)) + .await? + .into_json(), + ); + } + Ok(SqlPageValue::Json(Cow::Owned(Value::Array(array)))) + } + } + } + + pub(crate) fn contains_function(&self, expected: SqlPageFunctionName) -> bool { + match self { + Self::Call { + function, + arguments, + } => { + *function == expected + || arguments + .iter() + .any(|argument| argument.contains_function(expected)) + } + Self::Concat(arguments) | Self::Coalesce(arguments) | Self::JsonArray(arguments) => { + arguments + .iter() + .any(|argument| argument.contains_function(expected)) + } + Self::JsonObject(entries) => entries.iter().any(|(key, value)| { + key.contains_function(expected) || value.contains_function(expected) + }), + Self::Literal(_) | Self::Variable(_) | Self::Input(_) => false, + } + } +} + +impl SqlPageValue<'static> { + fn into_lifetime<'a>(self) -> SqlPageValue<'a> { + match self { + Self::Null => SqlPageValue::Null, + Self::Text(text) => SqlPageValue::Text(text), + Self::Json(value) => SqlPageValue::Json(value), + } + } +} diff --git a/src/webserver/database/sqlpage_functions/functions.rs b/src/webserver/database/sqlpage_functions/functions.rs index e6709bd8..46508eb1 100644 --- a/src/webserver/database/sqlpage_functions/functions.rs +++ b/src/webserver/database/sqlpage_functions/functions.rs @@ -57,7 +57,7 @@ impl ::std::str::FromStr for SqlPageFunctionName { SqlPageFunctionName::ALL .iter() .copied() - .find(|function| function.name() == name) + .find(|function| function.name().eq_ignore_ascii_case(name)) .ok_or_else(|| { anyhow::anyhow!( "Unknown function {name:?}. Supported functions:\n{}", diff --git a/src/webserver/database/sqlpage_functions/mod.rs b/src/webserver/database/sqlpage_functions/mod.rs index aea396bb..d4b70ec2 100644 --- a/src/webserver/database/sqlpage_functions/mod.rs +++ b/src/webserver/database/sqlpage_functions/mod.rs @@ -2,20 +2,3 @@ mod function_traits; pub(super) mod functions; mod http_fetch_request; mod url_parameters; - -use sqlparser::ast::FunctionArg; - -use super::sql::ParamExtractContext; -use super::syntax_tree::SqlPageFunctionCall; -use super::syntax_tree::StmtParam; - -pub(super) fn func_call_to_param( - func_name: &str, - arguments: &mut [FunctionArg], - ctx: &ParamExtractContext, -) -> StmtParam { - SqlPageFunctionCall::from_func_call(func_name, arguments, ctx).map_or_else( - |e| StmtParam::Error(format!("{e:#}")), - StmtParam::FunctionCall, - ) -} diff --git a/src/webserver/database/syntax_tree.rs b/src/webserver/database/syntax_tree.rs deleted file mode 100644 index 31a88159..00000000 --- a/src/webserver/database/syntax_tree.rs +++ /dev/null @@ -1,321 +0,0 @@ -/// This module contains the syntax tree for sqlpage statement parameters. -/// In a query like `SELECT sqlpage.some_function($my_param)`, -/// The stored database statement will be just `SELECT $1`, -/// and the `StmtParam` will contain a the following tree: -/// -/// ```text -/// StmtParam::FunctionCall( -/// SqlPageFunctionCall { -/// function: SqlPageFunctionName::some_function, -/// arguments: vec![StmtParam::Get("$my_param")] -/// } -/// ) -/// ``` -use std::borrow::Cow; -use std::str::FromStr; - -use sqlparser::ast::FunctionArg; - -use crate::webserver::http_request_info::ExecutionContext; -use crate::webserver::single_or_vec::SingleOrVec; - -use super::{ - execute_queries::DbConn, sql::ParamExtractContext, sql::function_args_to_stmt_params, - sqlpage_functions::functions::SqlPageFunctionName, -}; -use anyhow::Context as _; - -/// Represents a parameter to a SQL statement. -/// Objects of this type are created during SQL parsing. -/// Every time a SQL statement is executed, the parameters are evaluated to produce the actual values that are passed to the database. -/// Parameter evaluation can involve asynchronous operations, and extracting values from the request. -#[derive(Debug, PartialEq, Eq, Clone)] -pub(crate) enum StmtParam { - Get(String), - Post(String), - PostOrGet(String), - Error(String), - Literal(String), - Null, - Concat(Vec), - Coalesce(Vec), - JsonObject(Vec), - JsonArray(Vec), - FunctionCall(SqlPageFunctionCall), -} - -impl std::fmt::Display for StmtParam { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - StmtParam::Get(name) => write!(f, "?{name}"), - StmtParam::Post(name) => write!(f, ":{name}"), - StmtParam::PostOrGet(name) => write!(f, "${name}"), - StmtParam::Literal(x) => write!(f, "'{}'", x.replace('\'', "''")), - StmtParam::Null => write!(f, "NULL"), - StmtParam::Concat(items) => { - write!(f, "CONCAT(")?; - for item in items { - write!(f, "{item}, ")?; - } - write!(f, ")") - } - StmtParam::Coalesce(items) => { - write!(f, "COALESCE(")?; - for item in items { - write!(f, "{item}, ")?; - } - write!(f, ")") - } - StmtParam::JsonObject(items) => { - write!(f, "JSON_OBJECT(")?; - for item in items { - write!(f, "{item}, ")?; - } - write!(f, ")") - } - StmtParam::JsonArray(items) => { - write!(f, "JSON_ARRAY(")?; - for item in items { - write!(f, "{item}, ")?; - } - write!(f, ")") - } - StmtParam::FunctionCall(call) => write!(f, "{call}"), - StmtParam::Error(x) => { - if let Some((i, _)) = x.char_indices().nth(21) { - write!(f, "## {}... ##", &x[..i]) - } else { - write!(f, "## {x} ##") - } - } - } - } -} - -/// Represents a call to a `sqlpage.` function. -/// Objects of this type are created during SQL parsing and used to evaluate the function at runtime. -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct SqlPageFunctionCall { - pub function: SqlPageFunctionName, - pub arguments: Vec, -} - -impl SqlPageFunctionCall { - pub fn from_func_call( - func_name: &str, - arguments: &mut [FunctionArg], - ctx: &ParamExtractContext, - ) -> anyhow::Result { - let function = SqlPageFunctionName::from_str(func_name)?; - let arguments = function_args_to_stmt_params(arguments, ctx)?; - Ok(Self { - function, - arguments, - }) - } - - pub async fn evaluate<'a>( - &self, - request: &'a ExecutionContext, - db_connection: &mut DbConn, - ) -> anyhow::Result>> { - let mut params = Vec::with_capacity(self.arguments.len()); - for param in &self.arguments { - params.push(Box::pin(extract_req_param(param, request, db_connection)).await?); - } - log::trace!("Starting function call to {self}"); - let result = self - .function - .evaluate(request, db_connection, params) - .await?; - log::trace!( - "Function call to {self} returned: {}", - result.as_deref().unwrap_or("NULL") - ); - Ok(result) - } -} - -impl std::fmt::Display for SqlPageFunctionCall { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}(", self.function)?; - // interleave the arguments with commas - let mut it = self.arguments.iter(); - if let Some(x) = it.next() { - write!(f, "{x}")?; - } - for x in it { - write!(f, ", {x}")?; - } - write!(f, ")") - } -} - -/// Extracts the value of a parameter from the request. -/// Returns `Ok(None)` when NULL should be used as the parameter value. -pub(super) async fn extract_req_param<'a>( - param: &StmtParam, - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - Ok(match param { - // sync functions - StmtParam::Get(x) => request.url_params.get(x).map(SingleOrVec::as_json_str), - StmtParam::Post(x) => { - if let Some(val) = request.set_variables.borrow().get(x) { - val.as_ref() - .map(|v| Cow::Owned(v.as_json_str().into_owned())) - } else { - request.post_variables.get(x).map(SingleOrVec::as_json_str) - } - } - StmtParam::PostOrGet(x) => { - if let Some(val) = request.set_variables.borrow().get(x) { - val.as_ref() - .map(|v| Cow::Owned(v.as_json_str().into_owned())) - } else { - let url_val = request.url_params.get(x); - if request.post_variables.contains_key(x) { - if url_val.is_some() { - log::warn!( - "Deprecation warning! There is both a URL parameter named '{x}' and a form field named '{x}'. \ - SQLPage is using the URL parameter for ${x}. Please use :{x} to reference the form field explicitly." - ); - } else { - log::warn!( - "Deprecation warning! ${x} was used to reference a form field value (a POST variable). \ - This now uses only URL parameters. Please use :{x} instead." - ); - } - } - url_val.map(SingleOrVec::as_json_str) - } - } - StmtParam::Error(x) => anyhow::bail!("{x}"), - StmtParam::Literal(x) => Some(Cow::Owned(x.clone())), - StmtParam::Null => None, - StmtParam::Concat(args) => concat_params(&args[..], request, db_connection).await?, - StmtParam::JsonObject(args) => { - json_object_params(&args[..], request, db_connection).await? - } - StmtParam::JsonArray(args) => json_array_params(&args[..], request, db_connection).await?, - StmtParam::Coalesce(args) => coalesce_params(&args[..], request, db_connection).await?, - StmtParam::FunctionCall(func) => { - func.evaluate(request, db_connection) - .await - .with_context(|| { - format!( - "Error in function call {func}.\nExpected {:#}", - func.function - ) - })? - } - }) -} - -async fn concat_params<'a>( - args: &[StmtParam], - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - let mut result = String::new(); - for arg in args { - let Some(arg) = Box::pin(extract_req_param(arg, request, db_connection)).await? else { - return Ok(None); - }; - result.push_str(&arg); - } - Ok(Some(Cow::Owned(result))) -} - -async fn coalesce_params<'a>( - args: &[StmtParam], - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - for arg in args { - if let Some(arg) = Box::pin(extract_req_param(arg, request, db_connection)).await? { - return Ok(Some(arg)); - } - } - Ok(None) -} - -async fn json_object_params<'a>( - args: &[StmtParam], - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - use serde::{Serializer, ser::SerializeMap}; - let mut result = Vec::new(); - let mut ser = serde_json::Serializer::new(&mut result); - let mut map_ser = ser.serialize_map(Some(args.len()))?; - let mut it = args.iter(); - while let Some(key) = it.next() { - let key = Box::pin(extract_req_param(key, request, db_connection)).await?; - map_ser.serialize_key(&key)?; - let val = it - .next() - .ok_or_else(|| anyhow::anyhow!("Odd number of arguments in JSON_OBJECT"))?; - - match val { - StmtParam::JsonObject(args) => { - let raw_json = Box::pin(json_object_params(args, request, db_connection)).await?; - let obj = cow_to_raw_json(raw_json.as_ref()); - map_ser.serialize_value(&obj)?; - } - StmtParam::JsonArray(args) => { - let raw_json = Box::pin(json_array_params(args, request, db_connection)).await?; - let obj = cow_to_raw_json(raw_json.as_ref()); - map_ser.serialize_value(&obj)?; - } - val => { - let evaluated = Box::pin(extract_req_param(val, request, db_connection)).await?; - map_ser.serialize_value(&evaluated)?; - } - } - } - map_ser.end()?; - Ok(Some(Cow::Owned(String::from_utf8(result)?))) -} - -async fn json_array_params<'a>( - args: &[StmtParam], - request: &'a ExecutionContext, - db_connection: &mut DbConn, -) -> anyhow::Result>> { - use serde::{Serializer, ser::SerializeSeq}; - let mut result = Vec::new(); - let mut ser = serde_json::Serializer::new(&mut result); - let mut seq_ser = ser.serialize_seq(Some(args.len()))?; - for element in args { - match element { - StmtParam::JsonObject(args) => { - let raw_json = json_object_params(args, request, db_connection).await?; - let obj = cow_to_raw_json(raw_json.as_ref()); - seq_ser.serialize_element(&obj)?; - } - StmtParam::JsonArray(args) => { - let raw_json = Box::pin(json_array_params(args, request, db_connection)).await?; - let obj = cow_to_raw_json(raw_json.as_ref()); - seq_ser.serialize_element(&obj)?; - } - element => { - let evaluated = - Box::pin(extract_req_param(element, request, db_connection)).await?; - seq_ser.serialize_element(&evaluated)?; - } - } - } - seq_ser.end()?; - Ok(Some(Cow::Owned(String::from_utf8(result)?))) -} - -fn cow_to_raw_json<'a>( - raw_json: Option<&'a impl AsRef>, -) -> Option<&'a serde_json::value::RawValue> { - raw_json - .map(AsRef::as_ref) - .map(serde_json::from_str::<&'a serde_json::value::RawValue>) - .map(Result::unwrap) -} diff --git a/src/webserver/http.rs b/src/webserver/http.rs index e37410a5..0f17bb95 100644 --- a/src/webserver/http.rs +++ b/src/webserver/http.rs @@ -9,7 +9,7 @@ use crate::webserver::database::execute_queries::stop_at_first_error; use crate::webserver::database::{DbItem, execute_queries::stream_query_results_with_conn}; use crate::webserver::http_request_info::extract_request_info; use crate::webserver::server_timing::ServerTiming; -use crate::{AppConfig, AppState, DEFAULT_404_FILE, ParsedSqlFile}; +use crate::{AppConfig, AppState, DEFAULT_404_FILE, SqlFile}; use actix_web::dev::{ServiceFactory, ServiceRequest, fn_service}; use actix_web::error::{ErrorBadRequest, ErrorInternalServerError}; use actix_web::http::header::Accept; @@ -209,7 +209,7 @@ enum ResponseWithWriter { async fn render_sql( srv_req: &mut ServiceRequest, - sql_file: Arc, + sql_file: Arc, server_timing: ServerTiming, ) -> actix_web::Result { let app_state = srv_req diff --git a/src/webserver/routing.rs b/src/webserver/routing.rs index 0bc92bf6..25c644a6 100644 --- a/src/webserver/routing.rs +++ b/src/webserver/routing.rs @@ -77,7 +77,7 @@ //! ``` use crate::filesystem::{FileAccess, FileSystem}; -use crate::webserver::database::ParsedSqlFile; +use crate::webserver::database::SqlFile; use crate::{AppState, file_cache::FileCache}; use RoutingAction::{CustomNotFound, Execute, NotFound, Redirect, Serve}; use awc::http::uri::PathAndQuery; @@ -109,14 +109,14 @@ pub trait RoutingConfig { } pub(crate) struct AppFileStore<'a> { - cache: &'a FileCache, + cache: &'a FileCache, filesystem: &'a FileSystem, app_state: &'a AppState, } impl<'a> AppFileStore<'a> { pub fn new( - cache: &'a FileCache, + cache: &'a FileCache, filesystem: &'a FileSystem, app_state: &'a AppState, ) -> Self { diff --git a/tests/sql_test_files/data/nested_sqlpage_functions.sql b/tests/sql_test_files/data/nested_sqlpage_functions.sql index 49eeb48e..796abb50 100644 --- a/tests/sql_test_files/data/nested_sqlpage_functions.sql +++ b/tests/sql_test_files/data/nested_sqlpage_functions.sql @@ -1 +1,9 @@ select 'It%20works%20%21' as expected, sqlpage.url_encode(sqlpage.read_file_as_text('tests/it_works.txt')) as actual; + +select 'It%20works%20%21' as expected, + sqlpage.url_encode(sqlpage.read_file_as_text(path)) as actual +from (select 'tests/it_works.txt' as path) paths; + +select '%2Fvalue' as expected, + coalesce(sqlpage.url_encode('/' || value), '') as actual +from (select 'value' as value) value_rows;