github-featured/sqlpage--sqlpage
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Preserve SQLite streaming in native drivers

Browse Source
This commit is contained in:
Ophir LOJKINE
2026-05-31 22:40:42 +02:00
parent 95c2674b85
commit c15388ed23
20 changed files with 1164 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+3 -47
View File
@@ -617,28 +617,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "aws-lc-rs"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
dependencies = [
"aws-lc-sys",
"zeroize",
]
[[package]]
name = "aws-lc-sys"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
dependencies = [
"cc",
"cmake",
"dunce",
"fs_extra",
]
[[package]]
name = "base16ct"
version = "0.2.0"
@@ -914,15 +892,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "cmake"
version = "0.1.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
dependencies = [
"cc",
]
[[package]]
name = "cmov"
version = "0.5.4"
@@ -1465,12 +1434,6 @@ version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "dyn-clone"
version = "1.0.20"
@@ -1702,12 +1665,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fs_extra"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "futures"
version = "0.3.32"
@@ -3525,8 +3482,8 @@ version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2"
dependencies = [
"aws-lc-rs",
"pem",
"ring",
"rustls-pki-types",
"time",
"yasna",
@@ -3734,7 +3691,6 @@ version = "0.23.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
dependencies = [
"aws-lc-rs",
"log",
"once_cell",
"ring",
@@ -3753,7 +3709,6 @@ dependencies = [
"async-io",
"async-trait",
"async-web-client",
"aws-lc-rs",
"base64 0.22.1",
"blocking",
"chrono",
@@ -3763,6 +3718,7 @@ dependencies = [
"log",
"pem",
"rcgen",
"ring",
"serde",
"serde_json",
"thiserror 2.0.18",
@@ -3828,7 +3784,6 @@ version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
@@ -4252,6 +4207,7 @@ dependencies = [
"awc",
"base64 0.22.1",
"bigdecimal 0.4.10",
"bytes",
"chrono",
"clap",
"config",
Cargo.toml
+5 -4
View File
@@ -58,16 +58,17 @@ actix-web-httpauth = "0.8.0"
rand = "0.10.0"
actix-multipart = "0.7.2"
base64 = "0.22"
bytes = "1"
hmac = "0.13"
sha2 = "0.11"
rustls-acme = "0.15"
rustls-acme = { version = "0.15", default-features = false, features = ["ring", "tls12", "webpki-roots"] }
dotenvy = "0.15.7"
csv-async = { version = "1.2.6", features = ["tokio"] }
rustls = { version = "0.23" } # keep in sync with actix-web, awc, and rustls-acme
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12", "logging"] } # keep in sync with actix-web, awc, and rustls-acme
rustls-native-certs = "0.8.1"
awc = { version = "3", features = ["rustls-0_23-webpki-roots"] }
clap = { version = "4.5.17", features = ["derive"] }
tokio-util = "0.7.12"
tokio-util = { version = "0.7.12", features = ["compat"] }
openidconnect = { version = "4.0.0", default-features = false, features = ["accept-rfc3339-timestamps"] }
encoding_rs = "0.8.35"
regex = "1"
@@ -96,7 +97,7 @@ tokio = { version = "1", features = ["rt", "time", "test-util"] }
[build-dependencies]
awc = { version = "3", features = ["rustls-0_23-webpki-roots"] }
rustls = "0.23"
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12", "logging"] }
actix-rt = "2.8"
libflate = "2"
futures-util = "0.3.21"
build.rs
+1 -1
View File
@@ -12,7 +12,7 @@ use std::time::Duration;
#[actix_rt::main]
async fn main() {
rustls::crypto::aws_lc_rs::default_provider()
rustls::crypto::ring::default_provider()
.install_default()
.unwrap();
src/filesystem.rs
+38 -38
View File
@@ -1,7 +1,7 @@
use crate::webserver::ErrorWithStatus;
use crate::webserver::database::SupportedDatabase;
use crate::webserver::{Database, StatusCodeResultExt, make_placeholder};
use crate::webserver::database::{DbParam, driver::DbValue};
use crate::webserver::{Database, StatusCodeResultExt, make_placeholder};
use crate::{AppState, TEMPLATES_DIR};
use anyhow::Context;
use chrono::{DateTime, Utc};
@@ -267,9 +267,9 @@ impl DbFsQueries {
log::debug!("Initializing database filesystem queries");
Self::check_table_available(db).await?;
Ok(Self {
was_modified: Self::make_was_modified_query(db).await?,
read_file: Self::make_read_file_query(db).await?,
exists: Self::make_exists_query(db).await?,
was_modified: Self::make_was_modified_query(db),
read_file: Self::make_read_file_query(db),
exists: Self::make_exists_query(db),
})
}
@@ -284,31 +284,31 @@ impl DbFsQueries {
Ok(())
}
async fn make_was_modified_query(db: &Database) -> anyhow::Result<String> {
fn make_was_modified_query(db: &Database) -> String {
let was_modified_query = format!(
"SELECT 1 from sqlpage_files WHERE last_modified >= {} AND path = {}",
make_placeholder(db.info.kind, 1),
make_placeholder(db.info.kind, 2)
);
log::debug!("Preparing the database filesystem was_modified_query: {was_modified_query}");
Ok(was_modified_query)
was_modified_query
}
async fn make_read_file_query(db: &Database) -> anyhow::Result<String> {
fn make_read_file_query(db: &Database) -> String {
let read_file_query = format!(
"SELECT contents from sqlpage_files WHERE path = {}",
make_placeholder(db.info.kind, 1),
);
log::debug!("Preparing the database filesystem read_file_query: {read_file_query}");
Ok(read_file_query)
read_file_query
}
async fn make_exists_query(db: &Database) -> anyhow::Result<String> {
fn make_exists_query(db: &Database) -> String {
let exists_query = format!(
"SELECT 1 from sqlpage_files WHERE path = {}",
make_placeholder(db.info.kind, 1),
);
Ok(exists_query)
exists_query
}
async fn file_modified_since_in_db(
@@ -351,26 +351,26 @@ impl DbFsQueries {
log::debug!("Reading file {} from the database", path.display());
let mut conn = app_state.db.connection.acquire().await?;
conn.fetch_optional(
&self.read_file,
&[DbParam::Text(path.display().to_string())],
)
.await
.map_err(anyhow::Error::from)
.and_then(|row| {
if let Some(row) = row {
match row.values.first() {
Some(DbValue::Bytes(bytes)) => Ok(bytes.clone()),
Some(DbValue::Text(text)) => Ok(text.as_bytes().to_vec()),
_ => Ok(Vec::new()),
}
} else {
Err(ErrorWithStatus {
status: actix_web::http::StatusCode::NOT_FOUND,
}
.into())
&self.read_file,
&[DbParam::Text(path.display().to_string())],
)
.await
.map_err(anyhow::Error::from)
.and_then(|row| {
if let Some(row) = row {
match row.values.first() {
Some(DbValue::Bytes(bytes)) => Ok(bytes.clone()),
Some(DbValue::Text(text)) => Ok(text.as_bytes().to_vec()),
_ => Ok(Vec::new()),
}
})
.with_context(|| format!("Unable to read {} from the database", path.display()))
} else {
Err(ErrorWithStatus {
status: actix_web::http::StatusCode::NOT_FOUND,
}
.into())
}
})
.with_context(|| format!("Unable to read {} from the database", path.display()))
}
async fn file_exists(&self, app_state: &AppState, path: &Path) -> anyhow::Result<bool> {
@@ -412,9 +412,9 @@ async fn test_sql_file_read_utf8() -> anyhow::Result<()> {
let create_table_sql = DbFsQueries::get_create_table_sql(state.db.info.database_type);
let db = &state.db;
let conn = &db.connection;
let mut conn = db.connection.acquire().await?;
conn.execute_command("DROP TABLE IF EXISTS sqlpage_files", &[]).await?;
conn.execute_command("DROP TABLE IF EXISTS sqlpage_files", &[])
.await?;
log::debug!("Creating table sqlpage_files: {create_table_sql}");
conn.execute_command(create_table_sql, &[]).await?;
@@ -425,13 +425,13 @@ async fn test_sql_file_read_utf8() -> anyhow::Result<()> {
make_placeholder(dbms, 2)
);
conn.execute_command(
&insert_sql,
&[
DbParam::Text("unit test file.txt".into()),
DbParam::Bytes("Héllö world! 😀".as_bytes().to_vec()),
],
)
.await?;
&insert_sql,
&[
DbParam::Text("unit test file.txt".into()),
DbParam::Bytes("Héllö world! 😀".as_bytes().to_vec()),
],
)
.await?;
let fs = FileSystem::init("/", db).await;
let actual = fs
src/lib.rs
+1 -1
View File
@@ -37,7 +37,7 @@
//! When processing a request, `SQLPage`:
//!
//! 1. Parses the SQL using sqlparser-rs. Once a SQL file is parsed, it is cached for later reuse.
//! 2. Executes queries through sqlx.
//! 2. Executes queries through native database drivers.
//! 3. Finds the requested component's handlebars template in the database or in the filesystem.
//! 4. Maps results to the component template, using handlebars-rs.
//! 5. Streams rendered HTML to the client.
src/telemetry_metrics.rs
+1 -1
View File
@@ -1,8 +1,8 @@
use crate::webserver::database::DbPool;
use opentelemetry::global;
use opentelemetry::metrics::{Histogram, ObservableGauge};
use opentelemetry_semantic_conventions::attribute as otel;
use opentelemetry_semantic_conventions::metric as otel_metric;
use crate::webserver::database::DbPool;
pub struct TelemetryMetrics {
pub http_request_duration: Histogram<f64>,
src/webserver/database/connect.rs
+6 -3
View File
@@ -23,8 +23,8 @@ impl Database {
}
log::debug!("Connecting to a {db_kind:?} database on {database_url}");
let on_connect_sql = read_connection_handler(config, ON_CONNECT_FILE);
let _on_reset_sql = read_connection_handler(config, ON_RESET_FILE);
if _on_reset_sql.is_some() {
let on_reset_sql = read_connection_handler(config, ON_RESET_FILE);
if on_reset_sql.is_some() {
log::warn!(
"{ON_RESET_FILE} is currently ignored by the native driver pool because connections are not reused yet"
);
@@ -111,7 +111,10 @@ fn read_connection_handler(config: &AppConfig, file_name: &str) -> Option<String
);
return None;
}
log::info!("Creating a custom SQL connection handler from {}", file.display());
log::info!(
"Creating a custom SQL connection handler from {}",
file.display()
);
match std::fs::read_to_string(&file) {
Ok(sql) => {
log::trace!("The custom SQL connection handler is:\n{sql}");
src/webserver/database/csv_import.rs
+11 -3
View File
@@ -324,7 +324,9 @@ async fn test_end_to_end() {
uploaded_file: "my_file.csv".into(),
}
);
let db = crate::webserver::Database::init(&test_config()).await.unwrap();
let db = crate::webserver::Database::init(&test_config())
.await
.unwrap();
let mut conn = db.connection.acquire().await.unwrap();
conn.execute_command("CREATE TABLE my_table (col1 TEXT, col2 TEXT)", &[])
.await
@@ -339,8 +341,14 @@ async fn test_end_to_end() {
.into_iter()
.filter_map(|item| match item {
super::driver::DbStatementResult::Row(row) => Some((
match &row.values[0] { super::driver::DbValue::Text(s) => s.clone(), other => format!("{other:?}") },
match &row.values[1] { super::driver::DbValue::Text(s) => s.clone(), other => format!("{other:?}") },
match &row.values[0] {
super::driver::DbValue::Text(s) => s.clone(),
other => format!("{other:?}"),
},
match &row.values[1] {
super::driver::DbValue::Text(s) => s.clone(),
other => format!("{other:?}"),
},
)),
super::driver::DbStatementResult::Finished => None,
})
src/webserver/database/driver.rs
+902 -15
View File
File diff suppressed because it is too large Load Diff
src/webserver/database/error_highlighting.rs
+7 -7
View File
@@ -3,8 +3,8 @@ use std::{
path::{Path, PathBuf},
};
use super::sql::{SourceSpan, StmtWithParams};
use super::DbError;
use super::sql::{SourceSpan, StmtWithParams};
#[derive(Debug)]
struct NiceDatabaseError {
@@ -44,7 +44,11 @@ impl std::fmt::Display for NiceDatabaseError {
self.source_file.display(),
self.db_err
)?;
if let DbError::Database { offset: Some(offset), .. } = &self.db_err {
if let DbError::Database {
offset: Some(offset),
..
} = &self.db_err
{
let mut offset = *offset;
for line in self.query.lines() {
if offset > line.len() {
@@ -94,11 +98,7 @@ impl std::error::Error for NicePositionedError {
/// Display a database error without any position information
#[must_use]
pub fn display_db_error(
source_file: &Path,
query: &str,
db_err: DbError,
) -> anyhow::Error {
pub fn display_db_error(source_file: &Path, query: &str, db_err: DbError) -> anyhow::Error {
anyhow::Error::new(NiceDatabaseError {
source_file: source_file.to_path_buf(),
db_err,
src/webserver/database/execute_queries.rs
+31 -24
View File
@@ -20,8 +20,8 @@ use crate::webserver::http_request_info::ExecutionContext;
use crate::webserver::request_variables::SetVariablesMap;
use crate::webserver::single_or_vec::SingleOrVec;
use super::driver::{DbParam, DbStatementResult};
use super::syntax_tree::{StmtParam, extract_req_param};
use super::driver::{DbStatementResult, DbParam};
use super::{Database, DbConnection, DbItem};
pub type DbConn = Option<DbConnection>;
@@ -129,6 +129,7 @@ fn create_db_query_span(
(span, operation_name)
}
#[allow(clippy::too_many_lines)]
pub fn stream_query_results_with_conn<'a>(
sql_file: &'a ParsedSqlFile,
request: &'a ExecutionContext,
@@ -164,17 +165,24 @@ pub fn stream_query_results_with_conn<'a>(
&request.app_state.telemetry_metrics,
);
record_query_params(&query_metrics.span, &query.param_values);
let results = connection
.execute(query.sql, &query.arguments)
.instrument(query_span.clone())
.await;
let mut error = None;
let mut returned_rows: i64 = 0;
let start_next = std::time::Instant::now();
match results {
Ok(results) => {
{
let mut results = connection.execute_stream(query.sql, &query.arguments);
loop {
let start_next = std::time::Instant::now();
let elem = results.next().instrument(query_span.clone()).await;
query_metrics.add_duration(start_next.elapsed());
for elem in results {
let Some(elem) = elem else {
break;
};
let elem = match elem {
Ok(elem) => elem,
Err(e) => {
error = Some(display_stmt_db_error(source_file, stmt, e));
break;
}
};
let mut query_result = parse_single_sql_result(source_file, stmt, elem);
if let DbItem::Error(e) = query_result {
error = Some(e);
@@ -196,11 +204,6 @@ pub fn stream_query_results_with_conn<'a>(
}
}
}
Err(e) => {
query_metrics.add_duration(start_next.elapsed());
error = Some(display_stmt_db_error(source_file, stmt, e));
}
}
if let Some(error) = error {
query_metrics.record_error(returned_rows, &error);
try_rollback_transaction(connection).await;
@@ -293,7 +296,7 @@ async fn exec_static_simple_select(
async fn try_rollback_transaction(db_connection: &mut DbConnection) {
log::debug!("Attempting to rollback transaction");
match db_connection.execute_command("ROLLBACK", &[]).await {
Ok(_) => log::debug!("Rolled back transaction"),
Ok(()) => log::debug!("Rolled back transaction"),
Err(e) => {
log::debug!("There was probably no transaction in progress when this happened: {e:?}");
}
@@ -531,15 +534,19 @@ fn debug_row(r: &super::driver::DbRow) {
use std::fmt::Write;
let mut row_str = String::new();
for (col, value) in r.columns.iter().zip(r.values.iter()) {
write!(
&mut row_str,
"[{:?} ({}): {:?}: {:?}]",
col.name,
if matches!(value, super::driver::DbValue::Null) { "NULL" } else { "NOT NULL" },
col,
value
)
.unwrap();
write!(
&mut row_str,
"[{:?} ({}): {:?}: {:?}]",
col.name,
if matches!(value, super::driver::DbValue::Null) {
"NULL"
} else {
"NOT NULL"
},
col,
value
)
.unwrap();
}
log::trace!("Received db row: {row_str}");
}
src/webserver/database/migrations.rs
+56 -19
View File
@@ -5,7 +5,7 @@ use anyhow::Context;
use sha2::{Digest, Sha384};
use super::error_highlighting::display_db_error;
use super::{Database, DbParam, make_placeholder};
use super::{Database, DbKind, DbParam, make_placeholder};
use crate::MIGRATIONS_DIR;
#[derive(Debug)]
@@ -45,7 +45,7 @@ pub async fn apply(config: &crate::app_config::AppConfig, db: &Database) -> anyh
}
let mut conn = db.connection.acquire().await?;
ensure_migrations_table(&mut conn).await?;
ensure_migrations_table(&mut conn, db.info.kind).await?;
for migration in migrations {
let applied = migration_row(&mut conn, db, migration.version).await?;
if let Some(applied_checksum) = applied {
@@ -58,12 +58,14 @@ pub async fn apply(config: &crate::app_config::AppConfig, db: &Database) -> anyh
}
let start = Instant::now();
if let Err(err) = conn.execute_command(&migration.sql, &[]).await {
return Err(display_db_error(&migration.path, &migration.sql, err).context(format!(
"Failed to apply {} migration {}",
db,
DisplayMigration(&migration)
)));
if let Err(err) = conn.execute_batch(&migration.sql).await {
return Err(
display_db_error(&migration.path, &migration.sql, err).context(format!(
"Failed to apply {} migration {}",
db,
DisplayMigration(&migration)
)),
);
}
let execution_time = i64::try_from(start.elapsed().as_millis()).unwrap_or(i64::MAX);
record_migration(&mut conn, db, &migration, execution_time).await?;
@@ -104,19 +106,54 @@ fn load_migrations(migrations_dir: &Path) -> anyhow::Result<Vec<Migration>> {
Ok(migrations)
}
async fn ensure_migrations_table(conn: &mut super::DbConnection) -> anyhow::Result<()> {
conn.execute_command(
"CREATE TABLE IF NOT EXISTS _sqlx_migrations (
async fn ensure_migrations_table(
conn: &mut super::DbConnection,
kind: DbKind,
) -> anyhow::Result<()> {
let sql = match kind {
DbKind::Sqlite => {
"CREATE TABLE IF NOT EXISTS _sqlx_migrations (
version BIGINT PRIMARY KEY,
description TEXT NOT NULL,
installed_on TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
success BOOLEAN NOT NULL,
checksum BLOB NOT NULL,
execution_time BIGINT NOT NULL
)",
&[],
)
.await?;
)"
}
DbKind::Postgres => {
"CREATE TABLE IF NOT EXISTS _sqlx_migrations (
version BIGINT PRIMARY KEY,
description TEXT NOT NULL,
installed_on TIMESTAMPTZ NOT NULL DEFAULT now(),
success BOOLEAN NOT NULL,
checksum BYTEA NOT NULL,
execution_time BIGINT NOT NULL
)"
}
DbKind::MySql | DbKind::Odbc => {
"CREATE TABLE IF NOT EXISTS _sqlx_migrations (
version BIGINT PRIMARY KEY,
description VARCHAR(255) NOT NULL,
installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
success BOOLEAN NOT NULL,
checksum BLOB NOT NULL,
execution_time BIGINT NOT NULL
)"
}
DbKind::Mssql => {
"IF OBJECT_ID(N'_sqlx_migrations', N'U') IS NULL
CREATE TABLE _sqlx_migrations (
version BIGINT PRIMARY KEY,
description NVARCHAR(255) NOT NULL,
installed_on DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME(),
success BIT NOT NULL,
checksum VARBINARY(MAX) NOT NULL,
execution_time BIGINT NOT NULL
)"
}
};
conn.execute_command(sql, &[]).await?;
Ok(())
}
@@ -130,7 +167,7 @@ async fn migration_row(
make_placeholder(db.info.kind, 1)
);
let row = conn
.fetch_optional(&sql, &[DbParam::Text(version.to_string())])
.fetch_optional(&sql, &[DbParam::Integer(version)])
.await?;
Ok(row.and_then(|row| match row.values.first() {
Some(super::driver::DbValue::Bytes(bytes)) => Some(bytes.clone()),
@@ -156,11 +193,11 @@ async fn record_migration(
conn.execute_command(
&sql,
&[
DbParam::Text(migration.version.to_string()),
DbParam::Integer(migration.version),
DbParam::Text(migration.description.clone()),
DbParam::Text("true".to_string()),
DbParam::Bool(true),
DbParam::Bytes(migration.checksum.clone()),
DbParam::Text(execution_time.to_string()),
DbParam::Integer(execution_time),
],
)
.await?;
src/webserver/database/mod.rs
+5 -5
View File
@@ -1,22 +1,22 @@
pub mod blob_to_data_url;
mod connect;
mod csv_import;
pub mod driver;
pub mod execute_queries;
pub mod migrations;
mod sql;
mod sqlpage_functions;
mod syntax_tree;
pub mod driver;
mod error_highlighting;
mod sql_to_json;
pub use driver::{DbConnection, DbError, DbKind, DbParam, DbPool};
pub use sql::ParsedSqlFile;
use sql::{DB_PLACEHOLDERS, DbPlaceHolder};
pub use driver::{DbConnection, DbError, DbKind, DbParam, DbPool};
// SupportedDatabase is defined in this module
/// Supported database types in `SQLPage`. Represents an actual DBMS, not a sqlx backend kind (like "Odbc")
/// Supported database types in `SQLPage`. Represents an actual DBMS, not a driver kind like ODBC.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SupportedDatabase {
Sqlite,
@@ -109,9 +109,9 @@ pub struct DbInfo {
}
impl Database {
pub async fn close(&self) -> anyhow::Result<()> {
pub fn close(&self) -> anyhow::Result<()> {
log::info!("Closing all database connections...");
self.connection.close().await;
self.connection.close();
Ok(())
}
}
src/webserver/database/sql.rs
+18 -6
View File
@@ -783,11 +783,11 @@ mod test {
fn create_test_db_info(database_type: SupportedDatabase) -> DbInfo {
let kind = match database_type {
SupportedDatabase::Postgres => super::DbKind::Postgres,
SupportedDatabase::Mssql => super::DbKind::Mssql,
SupportedDatabase::MySql => super::DbKind::MySql,
SupportedDatabase::Sqlite => super::DbKind::Sqlite,
_ => super::DbKind::Odbc,
SupportedDatabase::Postgres => crate::webserver::database::DbKind::Postgres,
SupportedDatabase::Mssql => crate::webserver::database::DbKind::Mssql,
SupportedDatabase::MySql => crate::webserver::database::DbKind::MySql,
SupportedDatabase::Sqlite => crate::webserver::database::DbKind::Sqlite,
_ => crate::webserver::database::DbKind::Odbc,
};
DbInfo {
dbms_name: database_type.display_name().to_string(),
@@ -1019,6 +1019,18 @@ mod test {
assert_eq!(parameters, [StmtParam::PostOrGet("1".to_string()),]);
}
#[test]
fn test_mysql_statement_rewrite() {
let mut ast = parse_stmt("select '' || $1 || 'x'", &MySqlDialect {});
let db_info = create_test_db_info(SupportedDatabase::MySql);
let parameters = ParameterExtractor::extract_parameters(&mut ast, db_info).unwrap();
assert_eq!(
ast.to_string(),
"SELECT CONCAT(CONCAT('', CAST(@SQLPAGE_TEMP1 AS CHAR)), 'x')"
);
assert_eq!(parameters, [StmtParam::PostOrGet("1".to_string()),]);
}
#[test]
fn test_static_extract() {
use SimpleSelectValue::Static;
@@ -1316,7 +1328,7 @@ mod test {
delayed_functions: vec![],
json_columns: vec![],
};
transform_to_positional_placeholders(&mut stmt, super::DbKind::MySql);
transform_to_positional_placeholders(&mut stmt, crate::webserver::database::DbKind::MySql);
assert_eq!(
stmt.query,
"select \
src/webserver/database/sql/parameter_extraction.rs
+8 -7
View File
@@ -1,8 +1,8 @@
use super::super::{DbInfo, SupportedDatabase};
use super::{is_sqlpage_func, sqlpage_func_name};
use crate::webserver::database::DbKind;
use crate::webserver::database::sqlpage_functions::func_call_to_param;
use crate::webserver::database::syntax_tree::StmtParam;
use crate::webserver::database::DbKind;
use sqlparser::ast::{
BinaryOperator, CastKind, CharacterLength, DataType, Expr, Function, FunctionArg,
FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart,
@@ -39,10 +39,7 @@ pub(crate) const DB_PLACEHOLDERS: [(DbKind, DbPlaceHolder); 5] = [
DbKind::Mssql,
DbPlaceHolder::PrefixedNumber { prefix: "@p" },
),
(
DbKind::Odbc,
DbPlaceHolder::Positional { placeholder: "?" },
),
(DbKind::Odbc, DbPlaceHolder::Positional { placeholder: "?" }),
];
/// For positional parameters, we use a temporary placeholder during parameter extraction,
@@ -541,12 +538,16 @@ impl VisitorMut for ParameterExtractor {
}
self.replace_with_placeholder(value, param);
}
// Replace 'str1' || 'str2' with CONCAT('str1', 'str2') for MSSQL
// Replace 'str1' || 'str2' with CONCAT('str1', 'str2') where pipes are not string concatenation.
Expr::BinaryOp {
left,
op: BinaryOperator::StringConcat,
right,
} if self.db_info.database_type == SupportedDatabase::Mssql => {
} if matches!(
self.db_info.database_type,
SupportedDatabase::Mssql | SupportedDatabase::MySql
) =>
{
let left = std::mem::replace(left.as_mut(), Expr::value(Value::Null));
let right = std::mem::replace(right.as_mut(), Expr::value(Value::Null));
*value = Expr::Function(Function {
src/webserver/database/sql_to_json.rs
+6 -6
View File
@@ -14,11 +14,7 @@ pub fn row_to_json(row: &DbRow) -> Value {
}
fn canonical_col_name(col: &DbColumn, kind: DbKind) -> String {
if matches!(kind, DbKind::Odbc)
&& col
.name
.chars()
.all(|c| c.is_ascii_uppercase() || c == '_')
if matches!(kind, DbKind::Odbc) && col.name.chars().all(|c| c.is_ascii_uppercase() || c == '_')
{
col.name.to_ascii_lowercase()
} else {
@@ -29,6 +25,7 @@ fn canonical_col_name(col: &DbColumn, kind: DbKind) -> String {
pub fn sql_value_to_json(value: &DbValue) -> Value {
match value {
DbValue::Null => Value::Null,
DbValue::Bool(b) => (*b).into(),
DbValue::Integer(i) => (*i).into(),
DbValue::Real(f) => (*f).into(),
DbValue::Text(s) => Value::String(s.clone()),
@@ -78,6 +75,9 @@ mod tests {
values: vec![DbValue::Text("hello".into())],
kind: DbKind::Odbc,
};
assert_eq!(row_to_json(&row), serde_json::json!({"title_text": "hello"}));
assert_eq!(
row_to_json(&row),
serde_json::json!({"title_text": "hello"})
);
}
}
src/webserver/database/sqlpage_functions/functions.rs
+16 -6
View File
@@ -7,7 +7,6 @@ use crate::webserver::{
sqlpage_functions::{http_fetch_request::HttpFetchRequest, url_parameters::URLParameters},
},
http_client::make_http_client,
request_variables::SetVariablesMap,
single_or_vec::SingleOrVec,
};
use anyhow::{Context, anyhow};
@@ -722,6 +721,17 @@ async fn run_sql<'a>(
log::debug!("run_sql: first argument is NULL, returning NULL");
return Ok(None);
};
if request
.included_sql_files
.iter()
.any(|path| path == sql_file_path.as_ref())
{
anyhow::bail!(
"Too many nested inclusions. run_sql cannot include a file that is already being executed in the same inclusion chain. \
Executing sqlpage.run_sql('{sql_file_path}') would create a loop. \
This is to prevent infinite loops and stack overflows."
);
}
let run_sql_span = tracing::info_span!(
"sqlpage.file",
otel.name = format!("SQL {sql_file_path}"),
@@ -738,14 +748,14 @@ async fn run_sql<'a>(
.instrument(run_sql_span.clone())
.await
.with_context(|| format!("run_sql: invalid path {sql_file_path:?}"))?;
let tmp_req = if let Some(variables) = variables {
let variables: SetVariablesMap = serde_json::from_str(&variables).with_context(|| {
let variables = if let Some(variables) = variables {
serde_json::from_str(&variables).with_context(|| {
format!("run_sql(\'{sql_file_path}\', \'{variables}\'): the second argument should be a JSON object with string keys and values")
})?;
request.fork_with_variables(variables)
})?
} else {
request.fork()
request.set_variables.borrow().clone()
};
let tmp_req = request.fork_for_run_sql(sql_file_path.as_ref(), variables);
let max_recursion_depth = app_state.config.max_recursion_depth;
if tmp_req.clone_depth > max_recursion_depth {
anyhow::bail!(
src/webserver/http.rs
+3 -3
View File
@@ -275,11 +275,11 @@ async fn render_sql(
let database_entries_stream =
stream_query_results_with_conn(&sql_file, &exec_ctx, &mut conn);
let database_entries_stream = stop_at_first_error(database_entries_stream);
let response_with_writer = build_response_header_and_stream(
let response_with_writer = Box::pin(build_response_header_and_stream(
Arc::clone(&app_state),
database_entries_stream,
request_context,
)
))
.await;
match response_with_writer {
Ok(ResponseWithWriter::RenderStream {
@@ -678,7 +678,7 @@ pub async fn run_server(config: &AppConfig, state: AppState) -> anyhow::Result<(
.with_context(|| "Unable to start the application")?;
// We are done, we can close the database connection
final_state.db.close().await?;
final_state.db.close()?;
Ok(())
}
src/webserver/http_request_info.rs
+16
View File
@@ -53,6 +53,7 @@ pub struct ExecutionContext {
pub request: Rc<RequestInfo>,
pub set_variables: RefCell<SetVariablesMap>,
pub clone_depth: u8,
pub included_sql_files: Rc<Vec<String>>,
}
impl ExecutionContext {
@@ -62,6 +63,7 @@ impl ExecutionContext {
request: Rc::new(request),
set_variables: RefCell::new(SetVariablesMap::new()),
clone_depth: 0,
included_sql_files: Rc::new(Vec::new()),
}
}
@@ -71,6 +73,7 @@ impl ExecutionContext {
request: Rc::clone(&self.request),
set_variables: RefCell::new(self.set_variables.borrow().clone()),
clone_depth: self.clone_depth + 1,
included_sql_files: Rc::clone(&self.included_sql_files),
}
}
@@ -80,6 +83,19 @@ impl ExecutionContext {
request: Rc::clone(&self.request),
set_variables: RefCell::new(variables),
clone_depth: self.clone_depth + 1,
included_sql_files: Rc::clone(&self.included_sql_files),
}
}
#[must_use]
pub fn fork_for_run_sql(&self, sql_file_path: &str, variables: SetVariablesMap) -> Self {
let mut included_sql_files = self.included_sql_files.as_ref().clone();
included_sql_files.push(sql_file_path.to_string());
Self {
request: Rc::clone(&self.request),
set_variables: RefCell::new(variables),
clone_depth: self.clone_depth + 1,
included_sql_files: Rc::new(included_sql_files),
}
}
tests/core/mod.rs
+30 -19
View File
@@ -1,9 +1,9 @@
use actix_web::{http::StatusCode, test};
use sqlpage::{
AppState,
webserver::database::DbParam,
webserver::{self, make_placeholder},
};
use sqlx::Executor as _;
use crate::common::{make_app_data_from_config, req_path, req_path_with_app_data, test_config};
@@ -58,19 +58,25 @@ async fn test_routing_with_db_fs() {
}
let drop_sql = "DROP TABLE IF EXISTS sqlpage_files";
state.db.connection.execute(drop_sql).await.unwrap();
let mut conn = state.db.connection.acquire().await.unwrap();
conn.execute_command(drop_sql, &[]).await.unwrap();
let create_table_sql =
sqlpage::filesystem::DbFsQueries::get_create_table_sql(state.db.info.database_type);
state.db.connection.execute(create_table_sql).await.unwrap();
conn.execute_command(create_table_sql, &[]).await.unwrap();
let insert_sql = format!(
"INSERT INTO sqlpage_files(path, contents) VALUES ('on_db.sql', {})",
make_placeholder(state.db.info.kind, 1)
);
sqlx::query(&insert_sql)
.bind("select ''text'' as component, ''Hi from db !'' AS contents;".as_bytes())
.execute(&state.db.connection)
.await
.unwrap();
conn.execute_command(
&insert_sql,
&[DbParam::Bytes(
"select ''text'' as component, ''Hi from db !'' AS contents;"
.as_bytes()
.to_vec(),
)],
)
.await
.unwrap();
let state = AppState::init(&config).await.unwrap();
let app_data = actix_web::web::Data::new(state);
@@ -101,23 +107,28 @@ async fn test_non_unicode_static_path_returns_bad_request_with_db_fs() {
let expected_db_path = "\u{FFFD}.txt";
let mut conn = state.db.connection.acquire().await.unwrap();
(&mut *conn)
.execute(sqlpage::filesystem::DbFsQueries::get_create_table_sql(
conn.execute_command(
sqlpage::filesystem::DbFsQueries::get_create_table_sql(
sqlpage::webserver::database::SupportedDatabase::Sqlite,
))
.await
.unwrap();
),
&[],
)
.await
.unwrap();
let insert_sql = format!(
"INSERT INTO sqlpage_files(path, contents) VALUES ({}, {})",
make_placeholder(state.db.info.kind, 1),
make_placeholder(state.db.info.kind, 2)
);
sqlx::query(&insert_sql)
.bind(expected_db_path)
.bind("file from db fs".as_bytes())
.execute(&mut *conn)
.await
.unwrap();
conn.execute_command(
&insert_sql,
&[
DbParam::Text(expected_db_path.into()),
DbParam::Bytes("file from db fs".as_bytes().to_vec()),
],
)
.await
.unwrap();
drop(conn);
let state = AppState::init(&config).await.unwrap();