Add structured XLSX extraction pipeline with table detection, OOXML metadata parsing, and semantic chunking
This commit is contained in:
Olow304
+3
-1
@@ -44,4 +44,6 @@ models/
|
||||
|
||||
issue_overview.md
|
||||
|
||||
feature_100x_memvid.md
|
||||
feature_100x_memvid.md
|
||||
|
||||
plan_model_robust.md
|
||||
@@ -0,0 +1,517 @@
|
||||
//! Row-aligned semantic chunking for XLSX spreadsheets.
|
||||
//!
|
||||
//! Produces structure-aware chunks that:
|
||||
//! - Never split a row across chunks
|
||||
//! - Prefix every chunk with sheet/table context and header row
|
||||
//! - Format rows as `Header: Value | Header: Value` for search accuracy
|
||||
//! - Skip empty cells for compact output
|
||||
#![allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
|
||||
|
||||
use crate::types::structure::{ChunkingResult, StructuredChunk};
|
||||
|
||||
use super::xlsx_table_detect::{CellValue, DetectedTable, SheetGrid};
|
||||
use super::xlsx_ooxml::{NumFmtKind, OoxmlMetadata, excel_serial_to_iso, format_currency, format_percentage};
|
||||
|
||||
/// Default target chunk size in characters.
|
||||
const DEFAULT_MAX_CHUNK_CHARS: usize = 1200;
|
||||
|
||||
/// Maximum number of chunks to produce from a single workbook.
|
||||
const MAX_SPREADSHEET_CHUNKS: usize = 500;
|
||||
|
||||
/// Options for XLSX semantic chunking.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct XlsxChunkingOptions {
|
||||
pub max_chars: usize,
|
||||
pub max_chunks: usize,
|
||||
}
|
||||
|
||||
impl Default for XlsxChunkingOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_chars: DEFAULT_MAX_CHUNK_CHARS,
|
||||
max_chunks: MAX_SPREADSHEET_CHUNKS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Format a cell value using OOXML metadata for type-aware rendering.
|
||||
#[must_use]
|
||||
pub fn format_cell_value(
|
||||
cell: &CellValue,
|
||||
fmt_kind: NumFmtKind,
|
||||
_metadata: &OoxmlMetadata,
|
||||
) -> String {
|
||||
match (cell, fmt_kind) {
|
||||
(CellValue::Empty, _) => String::new(),
|
||||
(CellValue::Text(s), _) => s.trim().to_string(),
|
||||
(CellValue::Number(v), NumFmtKind::Date | NumFmtKind::DateTime) => {
|
||||
excel_serial_to_iso(*v).unwrap_or_else(|| format!("{v}"))
|
||||
}
|
||||
(CellValue::Number(v), NumFmtKind::Percentage) => format_percentage(*v),
|
||||
(CellValue::Number(v), NumFmtKind::Currency) => format_currency(*v, "$"),
|
||||
(CellValue::Number(v), _) => {
|
||||
// Clean up float display — use integer format if no fractional part
|
||||
if (v.fract()).abs() < 1e-10 {
|
||||
format!("{}", *v as i64)
|
||||
} else {
|
||||
format!("{v}")
|
||||
}
|
||||
}
|
||||
(CellValue::Integer(v), NumFmtKind::Date | NumFmtKind::DateTime) => {
|
||||
excel_serial_to_iso(*v as f64).unwrap_or_else(|| format!("{v}"))
|
||||
}
|
||||
(CellValue::Integer(v), NumFmtKind::Percentage) => format_percentage(*v as f64),
|
||||
(CellValue::Integer(v), NumFmtKind::Currency) => format_currency(*v as f64, "$"),
|
||||
(CellValue::Integer(v), _) => format!("{v}"),
|
||||
(CellValue::Boolean(b), _) => if *b { "true" } else { "false" }.to_string(),
|
||||
(CellValue::DateTime(s), _) => s.clone(),
|
||||
(CellValue::Error(s), _) => s.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format a single row as `Header: Value | Header: Value`, skipping empty cells.
|
||||
fn format_row_with_headers(
|
||||
grid: &SheetGrid,
|
||||
row_idx: u32,
|
||||
headers: &[String],
|
||||
first_col: u32,
|
||||
last_col: u32,
|
||||
metadata: &OoxmlMetadata,
|
||||
) -> String {
|
||||
let mut parts = Vec::new();
|
||||
|
||||
for col in first_col..=last_col {
|
||||
let cell = grid.cell(row_idx, col);
|
||||
if cell.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fmt_kind = grid.num_fmt(row_idx, col);
|
||||
let formatted = format_cell_value(cell, fmt_kind, metadata);
|
||||
if formatted.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let col_offset = (col - first_col) as usize;
|
||||
let header = headers
|
||||
.get(col_offset)
|
||||
.filter(|h| !h.is_empty())
|
||||
.cloned();
|
||||
|
||||
if let Some(h) = header {
|
||||
parts.push(format!("{h}: {formatted}"));
|
||||
} else {
|
||||
parts.push(formatted);
|
||||
}
|
||||
}
|
||||
|
||||
parts.join(" | ")
|
||||
}
|
||||
|
||||
/// Build a context prefix for a chunk: `[Sheet: X] [Table: Y]`
|
||||
fn build_context_prefix(sheet_name: &str, table_name: &str) -> String {
|
||||
format!("[Sheet: {sheet_name}] [Table: {table_name}]")
|
||||
}
|
||||
|
||||
/// Build a header line: `Header1 | Header2 | Header3`
|
||||
fn build_header_line(headers: &[String]) -> String {
|
||||
let nonempty: Vec<&str> = headers.iter().map(String::as_str).filter(|h| !h.is_empty()).collect();
|
||||
if nonempty.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
nonempty.join(" | ")
|
||||
}
|
||||
}
|
||||
|
||||
/// Chunk a single detected table into structure-aware chunks.
|
||||
fn chunk_table(
|
||||
grid: &SheetGrid,
|
||||
table: &DetectedTable,
|
||||
metadata: &OoxmlMetadata,
|
||||
options: &XlsxChunkingOptions,
|
||||
chunk_index_start: usize,
|
||||
) -> Vec<StructuredChunk> {
|
||||
let context_prefix = build_context_prefix(&table.sheet_name, &table.name);
|
||||
let header_line = build_header_line(&table.headers);
|
||||
|
||||
// Build the fixed prefix that goes into every chunk
|
||||
let fixed_prefix = if header_line.is_empty() {
|
||||
format!("{context_prefix}\n")
|
||||
} else {
|
||||
format!("{context_prefix}\n{header_line}\n")
|
||||
};
|
||||
let prefix_len = fixed_prefix.len();
|
||||
|
||||
// Format all data rows
|
||||
let mut formatted_rows: Vec<String> = Vec::new();
|
||||
for row_idx in table.first_data_row..=table.last_data_row {
|
||||
let line = format_row_with_headers(
|
||||
grid,
|
||||
row_idx,
|
||||
&table.headers,
|
||||
table.first_col,
|
||||
table.last_col,
|
||||
metadata,
|
||||
);
|
||||
if !line.is_empty() {
|
||||
formatted_rows.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
if formatted_rows.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Bin-pack rows into chunks, respecting max_chars
|
||||
let mut chunks = Vec::new();
|
||||
let mut current_rows: Vec<String> = Vec::new();
|
||||
let mut current_len = prefix_len;
|
||||
|
||||
for row_text in &formatted_rows {
|
||||
let row_len = row_text.len() + 1; // +1 for newline
|
||||
|
||||
if !current_rows.is_empty() && current_len + row_len > options.max_chars {
|
||||
// Emit current chunk
|
||||
let text = format!("{fixed_prefix}{}", current_rows.join("\n"));
|
||||
chunks.push(text);
|
||||
current_rows.clear();
|
||||
current_len = prefix_len;
|
||||
}
|
||||
|
||||
current_rows.push(row_text.clone());
|
||||
current_len += row_len;
|
||||
}
|
||||
|
||||
// Emit final chunk
|
||||
if !current_rows.is_empty() {
|
||||
let text = format!("{fixed_prefix}{}", current_rows.join("\n"));
|
||||
chunks.push(text);
|
||||
}
|
||||
|
||||
// Convert to StructuredChunk
|
||||
let total_parts = chunks.len() as u32;
|
||||
let table_id = format!("{}:{}", table.sheet_name, table.name);
|
||||
|
||||
chunks
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, text)| {
|
||||
let char_count = text.len();
|
||||
let idx = chunk_index_start + i;
|
||||
|
||||
if total_parts == 1 {
|
||||
StructuredChunk::table(text, idx, &table_id, 0, char_count)
|
||||
} else {
|
||||
StructuredChunk::table_continuation(
|
||||
text,
|
||||
idx,
|
||||
&table_id,
|
||||
(i + 1) as u32,
|
||||
total_parts,
|
||||
&fixed_prefix,
|
||||
0,
|
||||
char_count,
|
||||
)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Chunk an entire workbook's detected tables into structured chunks.
|
||||
#[must_use]
|
||||
pub fn chunk_workbook(
|
||||
grids: &[SheetGrid],
|
||||
tables: &[DetectedTable],
|
||||
metadata: &OoxmlMetadata,
|
||||
options: &XlsxChunkingOptions,
|
||||
) -> ChunkingResult {
|
||||
let mut result = ChunkingResult::empty();
|
||||
let mut chunk_index = 0;
|
||||
|
||||
for table in tables {
|
||||
// Find the grid for this table's sheet
|
||||
let Some(grid) = grids.iter().find(|g| g.sheet_name == table.sheet_name) else {
|
||||
result.warn(format!(
|
||||
"No grid found for sheet '{}', skipping table '{}'",
|
||||
table.sheet_name, table.name
|
||||
));
|
||||
continue;
|
||||
};
|
||||
|
||||
let table_chunks = chunk_table(grid, table, metadata, options, chunk_index);
|
||||
|
||||
if table_chunks.len() > 1 {
|
||||
result.tables_split += 1;
|
||||
}
|
||||
result.tables_processed += 1;
|
||||
chunk_index += table_chunks.len();
|
||||
result.chunks.extend(table_chunks);
|
||||
|
||||
// Respect global chunk limit
|
||||
if result.chunks.len() >= options.max_chunks {
|
||||
result.warn(format!(
|
||||
"Hit max chunk limit ({}) — remaining tables skipped",
|
||||
options.max_chunks
|
||||
));
|
||||
result.chunks.truncate(options.max_chunks);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Generate backward-compatible flat text from grids (for `ReaderOutput.document.text`).
|
||||
#[must_use]
|
||||
pub fn generate_flat_text(
|
||||
grids: &[SheetGrid],
|
||||
tables: &[DetectedTable],
|
||||
metadata: &OoxmlMetadata,
|
||||
) -> String {
|
||||
let mut out = String::new();
|
||||
|
||||
for table in tables {
|
||||
let grid = match grids.iter().find(|g| g.sheet_name == table.sheet_name) {
|
||||
Some(g) => g,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
if !out.is_empty() {
|
||||
out.push('\n');
|
||||
}
|
||||
out.push_str(&format!("Sheet: {}\n", table.sheet_name));
|
||||
|
||||
// Header line
|
||||
if !table.headers.is_empty() {
|
||||
let header_line = table
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|h| !h.is_empty())
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" | ");
|
||||
if !header_line.is_empty() {
|
||||
out.push_str(&header_line);
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
// Data rows
|
||||
for row_idx in table.first_data_row..=table.last_data_row {
|
||||
let line = format_row_with_headers(
|
||||
grid,
|
||||
row_idx,
|
||||
&table.headers,
|
||||
table.first_col,
|
||||
table.last_col,
|
||||
metadata,
|
||||
);
|
||||
if !line.is_empty() {
|
||||
out.push_str(&line);
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out.trim().to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::reader::xlsx_table_detect::SheetGrid;
|
||||
use crate::types::structure::ChunkType;
|
||||
|
||||
fn make_grid(data: Vec<Vec<CellValue>>, sheet_name: &str) -> SheetGrid {
|
||||
let num_rows = data.len() as u32;
|
||||
let num_cols = data.iter().map(|r| r.len()).max().unwrap_or(0) as u32;
|
||||
SheetGrid {
|
||||
sheet_name: sheet_name.to_string(),
|
||||
rows: data,
|
||||
num_fmt_kinds: Vec::new(),
|
||||
num_rows,
|
||||
num_cols,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_cell_value_date() {
|
||||
let metadata = OoxmlMetadata::default();
|
||||
let cell = CellValue::Number(44927.0);
|
||||
let result = format_cell_value(&cell, NumFmtKind::Date, &metadata);
|
||||
assert_eq!(result, "2023-01-01");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_cell_value_percentage() {
|
||||
let metadata = OoxmlMetadata::default();
|
||||
let cell = CellValue::Number(0.153);
|
||||
let result = format_cell_value(&cell, NumFmtKind::Percentage, &metadata);
|
||||
assert_eq!(result, "15.3%");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_cell_value_currency() {
|
||||
let metadata = OoxmlMetadata::default();
|
||||
let cell = CellValue::Number(1234.56);
|
||||
let result = format_cell_value(&cell, NumFmtKind::Currency, &metadata);
|
||||
assert_eq!(result, "$1234.56");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_row_with_headers() {
|
||||
let grid = make_grid(
|
||||
vec![vec![
|
||||
CellValue::Text("Alice".into()),
|
||||
CellValue::Integer(30),
|
||||
CellValue::Text("Austin".into()),
|
||||
]],
|
||||
"Sheet1",
|
||||
);
|
||||
let metadata = OoxmlMetadata::default();
|
||||
let headers = vec![
|
||||
"Name".to_string(),
|
||||
"Age".to_string(),
|
||||
"City".to_string(),
|
||||
];
|
||||
|
||||
let result = format_row_with_headers(&grid, 0, &headers, 0, 2, &metadata);
|
||||
assert_eq!(result, "Name: Alice | Age: 30 | City: Austin");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_row_skips_empty() {
|
||||
let grid = make_grid(
|
||||
vec![vec![
|
||||
CellValue::Text("Alice".into()),
|
||||
CellValue::Empty,
|
||||
CellValue::Text("Austin".into()),
|
||||
]],
|
||||
"Sheet1",
|
||||
);
|
||||
let metadata = OoxmlMetadata::default();
|
||||
let headers = vec![
|
||||
"Name".to_string(),
|
||||
"Age".to_string(),
|
||||
"City".to_string(),
|
||||
];
|
||||
|
||||
let result = format_row_with_headers(&grid, 0, &headers, 0, 2, &metadata);
|
||||
assert_eq!(result, "Name: Alice | City: Austin");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunk_table_single_chunk() {
|
||||
let grid = make_grid(
|
||||
vec![
|
||||
vec![
|
||||
CellValue::Text("Name".into()),
|
||||
CellValue::Text("Value".into()),
|
||||
],
|
||||
vec![CellValue::Text("A".into()), CellValue::Integer(100)],
|
||||
vec![CellValue::Text("B".into()), CellValue::Integer(200)],
|
||||
],
|
||||
"Sheet1",
|
||||
);
|
||||
let metadata = OoxmlMetadata::default();
|
||||
let table = DetectedTable {
|
||||
name: "Revenue".to_string(),
|
||||
sheet_name: "Sheet1".to_string(),
|
||||
headers: vec!["Name".to_string(), "Value".to_string()],
|
||||
column_types: vec![],
|
||||
first_data_row: 1,
|
||||
last_data_row: 2,
|
||||
first_col: 0,
|
||||
last_col: 1,
|
||||
header_row: Some(0),
|
||||
confidence: 0.7,
|
||||
};
|
||||
|
||||
let options = XlsxChunkingOptions::default();
|
||||
let chunks = chunk_table(&grid, &table, &metadata, &options, 0);
|
||||
|
||||
assert_eq!(chunks.len(), 1);
|
||||
let text = &chunks[0].text;
|
||||
assert!(text.contains("[Sheet: Sheet1] [Table: Revenue]"));
|
||||
assert!(text.contains("Name | Value"));
|
||||
assert!(text.contains("Name: A | Value: 100"));
|
||||
assert!(text.contains("Name: B | Value: 200"));
|
||||
assert_eq!(chunks[0].chunk_type, ChunkType::Table);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chunk_table_splits_large() {
|
||||
let mut rows = vec![vec![
|
||||
CellValue::Text("Col1".into()),
|
||||
CellValue::Text("Col2".into()),
|
||||
]];
|
||||
// Add 50 data rows to exceed a small chunk limit
|
||||
for i in 0..50 {
|
||||
rows.push(vec![
|
||||
CellValue::Text(format!("Row{i} long text that takes up space in the chunk")),
|
||||
CellValue::Integer(i as i64 * 1000),
|
||||
]);
|
||||
}
|
||||
|
||||
let grid = make_grid(rows, "Sheet1");
|
||||
let metadata = OoxmlMetadata::default();
|
||||
let table = DetectedTable {
|
||||
name: "Data".to_string(),
|
||||
sheet_name: "Sheet1".to_string(),
|
||||
headers: vec!["Col1".to_string(), "Col2".to_string()],
|
||||
column_types: vec![],
|
||||
first_data_row: 1,
|
||||
last_data_row: 50,
|
||||
first_col: 0,
|
||||
last_col: 1,
|
||||
header_row: Some(0),
|
||||
confidence: 0.7,
|
||||
};
|
||||
|
||||
let options = XlsxChunkingOptions {
|
||||
max_chars: 300,
|
||||
max_chunks: 100,
|
||||
};
|
||||
let chunks = chunk_table(&grid, &table, &metadata, &options, 0);
|
||||
|
||||
assert!(chunks.len() > 1, "Should split into multiple chunks");
|
||||
// Every chunk should have the header context
|
||||
for chunk in &chunks {
|
||||
assert!(chunk.text.contains("[Sheet: Sheet1]"));
|
||||
assert!(chunk.text.contains("Col1 | Col2"));
|
||||
assert_eq!(chunk.chunk_type, ChunkType::TableContinuation);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_generate_flat_text() {
|
||||
let grid = make_grid(
|
||||
vec![
|
||||
vec![
|
||||
CellValue::Text("Name".into()),
|
||||
CellValue::Text("Score".into()),
|
||||
],
|
||||
vec![CellValue::Text("Alice".into()), CellValue::Integer(95)],
|
||||
],
|
||||
"Results",
|
||||
);
|
||||
let metadata = OoxmlMetadata::default();
|
||||
let table = DetectedTable {
|
||||
name: "Scores".to_string(),
|
||||
sheet_name: "Results".to_string(),
|
||||
headers: vec!["Name".to_string(), "Score".to_string()],
|
||||
column_types: vec![],
|
||||
first_data_row: 1,
|
||||
last_data_row: 1,
|
||||
first_col: 0,
|
||||
last_col: 1,
|
||||
header_row: Some(0),
|
||||
confidence: 0.7,
|
||||
};
|
||||
|
||||
let text = generate_flat_text(&[grid], &[table], &metadata);
|
||||
assert!(text.contains("Sheet: Results"));
|
||||
assert!(text.contains("Name | Score"));
|
||||
assert!(text.contains("Name: Alice | Score: 95"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,618 @@
|
||||
//! OOXML metadata parser for XLSX files.
|
||||
//!
|
||||
//! Extracts metadata that calamine cannot provide:
|
||||
//! - Number format classification (dates, currency, percentages)
|
||||
//! - Merged cell regions
|
||||
//! - Named table definitions
|
||||
//!
|
||||
//! Parses XML files from the XLSX zip:
|
||||
//! - `xl/styles.xml` → number formats + cell XF mappings
|
||||
//! - `xl/worksheets/sheetN.xml` → merged cell regions
|
||||
#![allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Cursor, Read};
|
||||
|
||||
use quick_xml::events::Event;
|
||||
use quick_xml::Reader as XmlReader;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use zip::ZipArchive;
|
||||
|
||||
use crate::Result;
|
||||
|
||||
/// Classification of Excel number formats.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NumFmtKind {
|
||||
#[default]
|
||||
General,
|
||||
Number,
|
||||
Date,
|
||||
Time,
|
||||
DateTime,
|
||||
Currency,
|
||||
Percentage,
|
||||
Scientific,
|
||||
Text,
|
||||
}
|
||||
|
||||
/// A merged cell region in a worksheet.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MergedRegion {
|
||||
pub top_row: u32,
|
||||
pub left_col: u32,
|
||||
pub bottom_row: u32,
|
||||
pub right_col: u32,
|
||||
}
|
||||
|
||||
/// A named table definition from OOXML.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TableDefinition {
|
||||
pub name: String,
|
||||
pub sheet_name: String,
|
||||
pub headers: Vec<String>,
|
||||
pub first_row: u32,
|
||||
pub last_row: u32,
|
||||
pub first_col: u32,
|
||||
pub last_col: u32,
|
||||
}
|
||||
|
||||
/// All OOXML metadata extracted from an XLSX file.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct OoxmlMetadata {
|
||||
/// Map from numFmtId → format kind
|
||||
pub num_fmts: HashMap<u32, NumFmtKind>,
|
||||
/// Cell XF entries: index → numFmtId (the cellXfs array from styles.xml)
|
||||
pub cell_xfs: Vec<u32>,
|
||||
/// Merged regions per sheet name
|
||||
pub merged_regions: HashMap<String, Vec<MergedRegion>>,
|
||||
/// Named table definitions
|
||||
pub table_defs: Vec<TableDefinition>,
|
||||
}
|
||||
|
||||
impl OoxmlMetadata {
|
||||
/// Get the number format kind for a cell XF index (the `s` attribute on `<c>` elements).
|
||||
#[must_use]
|
||||
pub fn num_fmt_for_xf(&self, xf_index: u32) -> NumFmtKind {
|
||||
self.cell_xfs
|
||||
.get(xf_index as usize)
|
||||
.and_then(|fmt_id| self.num_fmts.get(fmt_id))
|
||||
.copied()
|
||||
.unwrap_or(NumFmtKind::General)
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify a built-in Excel number format ID.
|
||||
///
|
||||
/// Excel reserves IDs 0-163 for built-in formats. The key date/time/currency ranges:
|
||||
/// - 0: General
|
||||
/// - 1-11: Number formats
|
||||
/// - 14-22: Date/Time formats
|
||||
/// - 37-44: Accounting/Currency
|
||||
/// - 45-48: Time/Duration
|
||||
/// - 49: Text (@)
|
||||
fn classify_builtin_fmt(id: u32) -> NumFmtKind {
|
||||
match id {
|
||||
0 => NumFmtKind::General,
|
||||
1..=4 | 37..=40 => NumFmtKind::Number,
|
||||
5..=8 | 41..=44 => NumFmtKind::Currency,
|
||||
9 | 10 => NumFmtKind::Percentage,
|
||||
11 => NumFmtKind::Scientific,
|
||||
14..=17 => NumFmtKind::Date,
|
||||
18..=21 => NumFmtKind::Time,
|
||||
22 => NumFmtKind::DateTime,
|
||||
45..=48 => NumFmtKind::Time,
|
||||
49 => NumFmtKind::Text,
|
||||
_ => NumFmtKind::General,
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify a custom format code string by inspecting its characters.
|
||||
fn classify_format_code(code: &str) -> NumFmtKind {
|
||||
let lower = code.to_ascii_lowercase();
|
||||
// Remove escaped sequences and quoted strings
|
||||
let cleaned = remove_quoted_sections(&lower);
|
||||
|
||||
let has_date = cleaned.contains('y') || cleaned.contains('d');
|
||||
let has_month = cleaned.contains('m');
|
||||
let has_time = cleaned.contains('h') || cleaned.contains('s');
|
||||
let has_ampm = cleaned.contains("am/pm") || cleaned.contains("a/p");
|
||||
|
||||
if has_date && has_time {
|
||||
return NumFmtKind::DateTime;
|
||||
}
|
||||
if has_date {
|
||||
return NumFmtKind::Date;
|
||||
}
|
||||
// 'm' alone with time indicators is minutes, not months
|
||||
if has_time || has_ampm {
|
||||
return NumFmtKind::Time;
|
||||
}
|
||||
// After ruling out date/time, check for m alone (month)
|
||||
if has_month && !cleaned.contains('#') && !cleaned.contains('0') {
|
||||
return NumFmtKind::Date;
|
||||
}
|
||||
|
||||
if cleaned.contains('%') {
|
||||
return NumFmtKind::Percentage;
|
||||
}
|
||||
if cleaned.contains("e+") || cleaned.contains("e-") {
|
||||
return NumFmtKind::Scientific;
|
||||
}
|
||||
if cleaned.contains('$')
|
||||
|| cleaned.contains('\u{20ac}')
|
||||
|| cleaned.contains('\u{00a3}')
|
||||
|| cleaned.contains('\u{00a5}')
|
||||
|| cleaned.contains("eur")
|
||||
|| cleaned.contains("usd")
|
||||
|| cleaned.contains("gbp")
|
||||
{
|
||||
return NumFmtKind::Currency;
|
||||
}
|
||||
if cleaned.contains('@') {
|
||||
return NumFmtKind::Text;
|
||||
}
|
||||
if cleaned.contains('#') || cleaned.contains('0') {
|
||||
return NumFmtKind::Number;
|
||||
}
|
||||
|
||||
NumFmtKind::General
|
||||
}
|
||||
|
||||
/// Remove quoted sections (e.g., "text") and escaped chars (e.g., \x) from a format code.
|
||||
fn remove_quoted_sections(code: &str) -> String {
|
||||
let mut result = String::with_capacity(code.len());
|
||||
let mut chars = code.chars().peekable();
|
||||
while let Some(ch) = chars.next() {
|
||||
if ch == '"' {
|
||||
// Skip until closing quote
|
||||
for c in chars.by_ref() {
|
||||
if c == '"' {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if ch == '\\' {
|
||||
// Skip next char (escaped literal)
|
||||
let _ = chars.next();
|
||||
} else {
|
||||
result.push(ch);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Parse a cell reference like "A1" or "AZ100" into (row, col) 0-based.
|
||||
#[must_use]
|
||||
pub fn parse_cell_ref(cell_ref: &str) -> Option<(u32, u32)> {
|
||||
let mut col_str = String::new();
|
||||
let mut row_str = String::new();
|
||||
|
||||
for ch in cell_ref.chars() {
|
||||
if ch.is_ascii_alphabetic() {
|
||||
col_str.push(ch.to_ascii_uppercase());
|
||||
} else if ch.is_ascii_digit() {
|
||||
row_str.push(ch);
|
||||
}
|
||||
}
|
||||
|
||||
if col_str.is_empty() || row_str.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let col = col_str
|
||||
.chars()
|
||||
.fold(0u32, |acc, c| acc * 26 + (c as u32 - b'A' as u32 + 1))
|
||||
.saturating_sub(1);
|
||||
let row = row_str.parse::<u32>().ok()?.saturating_sub(1);
|
||||
|
||||
Some((row, col))
|
||||
}
|
||||
|
||||
/// Parse a range reference like "A1:D10" into ((top_row, left_col), (bottom_row, right_col)).
|
||||
#[must_use]
|
||||
pub fn parse_range_ref(range_ref: &str) -> Option<((u32, u32), (u32, u32))> {
|
||||
let parts: Vec<&str> = range_ref.split(':').collect();
|
||||
if parts.len() != 2 {
|
||||
return None;
|
||||
}
|
||||
let start = parse_cell_ref(parts[0])?;
|
||||
let end = parse_cell_ref(parts[1])?;
|
||||
Some((start, end))
|
||||
}
|
||||
|
||||
/// Extract OOXML metadata from an XLSX file's bytes.
|
||||
///
|
||||
/// Parses styles.xml for number formats and worksheet XMLs for merged cells.
|
||||
/// Table definitions come from calamine's native table support (calamine 0.25+).
|
||||
pub fn parse_ooxml_metadata(xlsx_bytes: &[u8]) -> Result<OoxmlMetadata> {
|
||||
let cursor = Cursor::new(xlsx_bytes);
|
||||
let mut archive =
|
||||
ZipArchive::new(cursor).map_err(|err| crate::MemvidError::ExtractionFailed {
|
||||
reason: format!("failed to open xlsx zip: {err}").into(),
|
||||
})?;
|
||||
|
||||
let mut metadata = OoxmlMetadata::default();
|
||||
|
||||
// Seed built-in formats
|
||||
for id in 0..=49 {
|
||||
let kind = classify_builtin_fmt(id);
|
||||
if kind != NumFmtKind::General {
|
||||
metadata.num_fmts.insert(id, kind);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse styles.xml
|
||||
if let Ok(styles_xml) = read_zip_entry(&mut archive, "xl/styles.xml") {
|
||||
parse_styles_xml(&styles_xml, &mut metadata);
|
||||
}
|
||||
|
||||
// Parse worksheet XMLs for merged cells
|
||||
let sheet_names = collect_sheet_filenames(&mut archive);
|
||||
for (sheet_name, zip_path) in &sheet_names {
|
||||
if let Ok(sheet_xml) = read_zip_entry(&mut archive, zip_path) {
|
||||
let regions = parse_merge_cells_xml(&sheet_xml);
|
||||
if !regions.is_empty() {
|
||||
metadata
|
||||
.merged_regions
|
||||
.insert(sheet_name.clone(), regions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
/// Read a file entry from a zip archive into a string.
|
||||
fn read_zip_entry(
|
||||
archive: &mut ZipArchive<Cursor<&[u8]>>,
|
||||
path: &str,
|
||||
) -> std::result::Result<String, ()> {
|
||||
let mut file = archive.by_name(path).map_err(|_| ())?;
|
||||
let mut buf = String::new();
|
||||
file.read_to_string(&mut buf).map_err(|_| ())?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Collect worksheet file paths from the zip, mapping sheet index to zip path.
|
||||
/// Returns (sheet_display_name, zip_path) pairs.
|
||||
fn collect_sheet_filenames(archive: &mut ZipArchive<Cursor<&[u8]>>) -> Vec<(String, String)> {
|
||||
let mut sheets = Vec::new();
|
||||
|
||||
// First try to read workbook.xml for sheet names
|
||||
let sheet_names_from_wb = if let Ok(wb_xml) = read_zip_entry(archive, "xl/workbook.xml") {
|
||||
parse_workbook_sheet_names(&wb_xml)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// Match sheet names to worksheet files
|
||||
for i in 0..archive.len() {
|
||||
if let Ok(file) = archive.by_index(i) {
|
||||
let name = file.name().to_string();
|
||||
if name.starts_with("xl/worksheets/sheet") && name.ends_with(".xml") {
|
||||
// Extract sheet number from filename (e.g., "sheet1.xml" -> 0)
|
||||
let num_str = name
|
||||
.trim_start_matches("xl/worksheets/sheet")
|
||||
.trim_end_matches(".xml");
|
||||
if let Ok(num) = num_str.parse::<usize>() {
|
||||
let display_name = sheet_names_from_wb
|
||||
.get(num.saturating_sub(1))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| format!("Sheet{num}"));
|
||||
sheets.push((display_name, name));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sheets
|
||||
}
|
||||
|
||||
/// Parse workbook.xml to extract sheet display names in order.
|
||||
fn parse_workbook_sheet_names(xml: &str) -> Vec<String> {
|
||||
let mut reader = XmlReader::from_str(xml);
|
||||
reader.trim_text(true);
|
||||
let mut names = Vec::new();
|
||||
let mut buf = Vec::new();
|
||||
let mut in_sheets = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e) | Event::Empty(ref e))
|
||||
if e.name().as_ref() == b"sheets" =>
|
||||
{
|
||||
in_sheets = true;
|
||||
}
|
||||
Ok(Event::Start(ref e) | Event::Empty(ref e))
|
||||
if in_sheets && e.name().as_ref() == b"sheet" =>
|
||||
{
|
||||
for attr in e.attributes().flatten() {
|
||||
if attr.key.as_ref() == b"name" {
|
||||
if let Ok(val) = attr.unescape_value() {
|
||||
names.push(val.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Event::End(ref e)) if e.name().as_ref() == b"sheets" => {
|
||||
in_sheets = false;
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
names
|
||||
}
|
||||
|
||||
/// Parse styles.xml to extract numFmt definitions and cellXfs mappings.
|
||||
fn parse_styles_xml(xml: &str, metadata: &mut OoxmlMetadata) {
|
||||
let mut reader = XmlReader::from_str(xml);
|
||||
reader.trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut in_num_fmts = false;
|
||||
let mut in_cell_xfs = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e) | Event::Empty(ref e)) => {
|
||||
let tag = e.name();
|
||||
match tag.as_ref() {
|
||||
b"numFmts" => in_num_fmts = true,
|
||||
b"cellXfs" => in_cell_xfs = true,
|
||||
b"numFmt" if in_num_fmts => {
|
||||
let mut fmt_id = None;
|
||||
let mut fmt_code = None;
|
||||
for attr in e.attributes().flatten() {
|
||||
match attr.key.as_ref() {
|
||||
b"numFmtId" => {
|
||||
if let Ok(v) = attr.unescape_value() {
|
||||
fmt_id = v.parse::<u32>().ok();
|
||||
}
|
||||
}
|
||||
b"formatCode" => {
|
||||
if let Ok(v) = attr.unescape_value() {
|
||||
fmt_code = Some(v.to_string());
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if let (Some(id), Some(code)) = (fmt_id, fmt_code) {
|
||||
metadata.num_fmts.insert(id, classify_format_code(&code));
|
||||
}
|
||||
}
|
||||
b"xf" if in_cell_xfs => {
|
||||
let mut num_fmt_id = 0u32;
|
||||
for attr in e.attributes().flatten() {
|
||||
if attr.key.as_ref() == b"numFmtId" {
|
||||
if let Ok(v) = attr.unescape_value() {
|
||||
num_fmt_id = v.parse::<u32>().unwrap_or(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata.cell_xfs.push(num_fmt_id);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(Event::End(ref e)) => match e.name().as_ref() {
|
||||
b"numFmts" => in_num_fmts = false,
|
||||
b"cellXfs" => in_cell_xfs = false,
|
||||
_ => {}
|
||||
},
|
||||
Ok(Event::Eof) => break,
|
||||
Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a worksheet XML for `<mergeCells>` regions.
|
||||
fn parse_merge_cells_xml(xml: &str) -> Vec<MergedRegion> {
|
||||
let mut reader = XmlReader::from_str(xml);
|
||||
reader.trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut regions = Vec::new();
|
||||
let mut in_merge_cells = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) if e.name().as_ref() == b"mergeCells" => {
|
||||
in_merge_cells = true;
|
||||
}
|
||||
Ok(Event::Start(ref e) | Event::Empty(ref e))
|
||||
if in_merge_cells && e.name().as_ref() == b"mergeCell" =>
|
||||
{
|
||||
for attr in e.attributes().flatten() {
|
||||
if attr.key.as_ref() == b"ref" {
|
||||
if let Ok(val) = attr.unescape_value() {
|
||||
if let Some(((tr, lc), (br, rc))) = parse_range_ref(&val) {
|
||||
regions.push(MergedRegion {
|
||||
top_row: tr,
|
||||
left_col: lc,
|
||||
bottom_row: br,
|
||||
right_col: rc,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Event::End(ref e)) if e.name().as_ref() == b"mergeCells" => {
|
||||
in_merge_cells = false;
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
regions
|
||||
}
|
||||
|
||||
/// Convert an Excel serial date number to ISO-8601 string.
|
||||
///
|
||||
/// Excel dates are stored as days since 1900-01-00 (serial 1 = Jan 1, 1900).
|
||||
/// Excel has the Lotus 1-2-3 bug: serial 60 = Feb 29, 1900 (which doesn't exist).
|
||||
/// For serial > 60, subtract 1 to get the correct date.
|
||||
#[must_use]
|
||||
pub fn excel_serial_to_iso(serial: f64) -> Option<String> {
|
||||
if serial < 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let days_from_epoch = serial.floor() as i64;
|
||||
let frac = serial - serial.floor();
|
||||
|
||||
// Base: 1899-12-31 (so serial 1 = 1900-01-01)
|
||||
// For serial > 60, Excel's Lotus bug means the real date is one day earlier
|
||||
// than what the serial suggests, because Excel thinks Feb 29, 1900 exists.
|
||||
let base = chrono::NaiveDate::from_ymd_opt(1899, 12, 31)?;
|
||||
let adjusted_days = if days_from_epoch > 60 {
|
||||
days_from_epoch - 1
|
||||
} else {
|
||||
days_from_epoch
|
||||
};
|
||||
let date = base.checked_add_signed(chrono::Duration::days(adjusted_days))?;
|
||||
|
||||
if frac > 0.0001 {
|
||||
// Has a time component
|
||||
let total_seconds = (frac * 86400.0).round() as u32;
|
||||
let hours = total_seconds / 3600;
|
||||
let minutes = (total_seconds % 3600) / 60;
|
||||
let seconds = total_seconds % 60;
|
||||
let time = chrono::NaiveTime::from_hms_opt(hours, minutes, seconds)?;
|
||||
Some(
|
||||
chrono::NaiveDateTime::new(date, time)
|
||||
.format("%Y-%m-%d %H:%M:%S")
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
Some(date.format("%Y-%m-%d").to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Format a percentage value (0.153 -> "15.3%").
|
||||
#[must_use]
|
||||
pub fn format_percentage(val: f64) -> String {
|
||||
let pct = val * 100.0;
|
||||
if (pct - pct.round()).abs() < 0.001 {
|
||||
format!("{}%", pct.round() as i64)
|
||||
} else {
|
||||
format!("{pct:.1}%")
|
||||
}
|
||||
}
|
||||
|
||||
/// Format a currency value with the appropriate symbol.
|
||||
#[must_use]
|
||||
pub fn format_currency(val: f64, code: &str) -> String {
|
||||
let lower = code.to_ascii_lowercase();
|
||||
let symbol = if lower.contains('$') || lower.contains("usd") {
|
||||
"$"
|
||||
} else if lower.contains('\u{20ac}') || lower.contains("eur") {
|
||||
"\u{20ac}"
|
||||
} else if lower.contains('\u{00a3}') || lower.contains("gbp") {
|
||||
"\u{00a3}"
|
||||
} else if lower.contains('\u{00a5}') || lower.contains("jpy") || lower.contains("cny") {
|
||||
"\u{00a5}"
|
||||
} else {
|
||||
"$" // default
|
||||
};
|
||||
|
||||
if val < 0.0 {
|
||||
format!("-{symbol}{:.2}", val.abs())
|
||||
} else {
|
||||
format!("{symbol}{val:.2}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_classify_builtin_fmts() {
|
||||
assert_eq!(classify_builtin_fmt(0), NumFmtKind::General);
|
||||
assert_eq!(classify_builtin_fmt(1), NumFmtKind::Number);
|
||||
assert_eq!(classify_builtin_fmt(5), NumFmtKind::Currency);
|
||||
assert_eq!(classify_builtin_fmt(9), NumFmtKind::Percentage);
|
||||
assert_eq!(classify_builtin_fmt(11), NumFmtKind::Scientific);
|
||||
assert_eq!(classify_builtin_fmt(14), NumFmtKind::Date);
|
||||
assert_eq!(classify_builtin_fmt(18), NumFmtKind::Time);
|
||||
assert_eq!(classify_builtin_fmt(22), NumFmtKind::DateTime);
|
||||
assert_eq!(classify_builtin_fmt(49), NumFmtKind::Text);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_classify_custom_formats() {
|
||||
assert_eq!(classify_format_code("yyyy-mm-dd"), NumFmtKind::Date);
|
||||
assert_eq!(classify_format_code("mm/dd/yyyy"), NumFmtKind::Date);
|
||||
assert_eq!(classify_format_code("hh:mm:ss"), NumFmtKind::Time);
|
||||
assert_eq!(
|
||||
classify_format_code("yyyy-mm-dd hh:mm"),
|
||||
NumFmtKind::DateTime
|
||||
);
|
||||
assert_eq!(classify_format_code("0.00%"), NumFmtKind::Percentage);
|
||||
assert_eq!(classify_format_code("0.00E+00"), NumFmtKind::Scientific);
|
||||
assert_eq!(classify_format_code("$#,##0.00"), NumFmtKind::Currency);
|
||||
assert_eq!(
|
||||
classify_format_code("\u{20ac}#,##0.00"),
|
||||
NumFmtKind::Currency
|
||||
);
|
||||
assert_eq!(classify_format_code("#,##0.00"), NumFmtKind::Number);
|
||||
assert_eq!(classify_format_code("@"), NumFmtKind::Text);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_cell_ref() {
|
||||
assert_eq!(parse_cell_ref("A1"), Some((0, 0)));
|
||||
assert_eq!(parse_cell_ref("B5"), Some((4, 1)));
|
||||
assert_eq!(parse_cell_ref("Z1"), Some((0, 25)));
|
||||
assert_eq!(parse_cell_ref("AA1"), Some((0, 26)));
|
||||
assert_eq!(parse_cell_ref("AZ100"), Some((99, 51)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_range_ref() {
|
||||
let result = parse_range_ref("A1:D3");
|
||||
assert_eq!(result, Some(((0, 0), (2, 3))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_excel_serial_to_iso() {
|
||||
assert_eq!(excel_serial_to_iso(1.0), Some("1900-01-01".to_string()));
|
||||
assert_eq!(excel_serial_to_iso(44927.0), Some("2023-01-01".to_string()));
|
||||
assert!(excel_serial_to_iso(-1.0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_percentage() {
|
||||
assert_eq!(format_percentage(0.153), "15.3%");
|
||||
assert_eq!(format_percentage(0.5), "50%");
|
||||
assert_eq!(format_percentage(1.0), "100%");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_currency() {
|
||||
assert_eq!(format_currency(10.5, "$#,##0.00"), "$10.50");
|
||||
assert_eq!(format_currency(-10.5, "$#,##0.00"), "-$10.50");
|
||||
assert_eq!(
|
||||
format_currency(10.5, "\u{20ac}#,##0.00"),
|
||||
"\u{20ac}10.50"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_quoted_section_removal() {
|
||||
assert_eq!(
|
||||
remove_quoted_sections("yyyy\"year\"mm\"month\"dd\"day\""),
|
||||
"yyyymmdd"
|
||||
);
|
||||
assert_eq!(remove_quoted_sections("#,##0.00\"$\""), "#,##0.00");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,639 @@
|
||||
//! Table structure detection for XLSX sheets.
|
||||
//!
|
||||
//! Detects header rows, table boundaries, and column types for sheets
|
||||
//! not covered by OOXML table definitions.
|
||||
#![allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::xlsx_ooxml::{MergedRegion, NumFmtKind, TableDefinition};
|
||||
|
||||
/// Column type inferred from data sampling.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ColumnType {
|
||||
#[default]
|
||||
Text,
|
||||
Integer,
|
||||
Float,
|
||||
Date,
|
||||
DateTime,
|
||||
Time,
|
||||
Currency,
|
||||
Percentage,
|
||||
Boolean,
|
||||
Mixed,
|
||||
Empty,
|
||||
}
|
||||
|
||||
/// A detected table within a sheet.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DetectedTable {
|
||||
/// Table name (from OOXML or auto-generated)
|
||||
pub name: String,
|
||||
/// Sheet name this table belongs to
|
||||
pub sheet_name: String,
|
||||
/// Column headers (may be empty if no header detected)
|
||||
pub headers: Vec<String>,
|
||||
/// Column types inferred from data
|
||||
pub column_types: Vec<ColumnType>,
|
||||
/// First data row (0-based, the row after the header)
|
||||
pub first_data_row: u32,
|
||||
/// Last data row (inclusive, 0-based)
|
||||
pub last_data_row: u32,
|
||||
/// First column (0-based)
|
||||
pub first_col: u32,
|
||||
/// Last column (inclusive, 0-based)
|
||||
pub last_col: u32,
|
||||
/// Header row index (0-based), None if no header detected
|
||||
pub header_row: Option<u32>,
|
||||
/// Detection confidence (0.0 - 1.0)
|
||||
pub confidence: f64,
|
||||
}
|
||||
|
||||
/// A cell value representation for detection purposes.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum CellValue {
|
||||
Empty,
|
||||
Text(String),
|
||||
Number(f64),
|
||||
Integer(i64),
|
||||
Boolean(bool),
|
||||
DateTime(String),
|
||||
Error(String),
|
||||
}
|
||||
|
||||
impl CellValue {
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
matches!(self, Self::Empty)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_text(&self) -> bool {
|
||||
matches!(self, Self::Text(_))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_numeric(&self) -> bool {
|
||||
matches!(self, Self::Number(_) | Self::Integer(_))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_text(&self) -> String {
|
||||
match self {
|
||||
Self::Empty => String::new(),
|
||||
Self::Text(s) => s.clone(),
|
||||
Self::Number(v) => format!("{v}"),
|
||||
Self::Integer(v) => format!("{v}"),
|
||||
Self::Boolean(b) => if *b { "true" } else { "false" }.to_string(),
|
||||
Self::DateTime(s) => s.clone(),
|
||||
Self::Error(s) => s.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A grid of cell values representing one sheet.
|
||||
pub struct SheetGrid {
|
||||
pub sheet_name: String,
|
||||
pub rows: Vec<Vec<CellValue>>,
|
||||
/// Number format kinds per cell (row, col) if available from OOXML metadata.
|
||||
/// Outer vec is rows, inner is columns.
|
||||
pub num_fmt_kinds: Vec<Vec<NumFmtKind>>,
|
||||
pub num_rows: u32,
|
||||
pub num_cols: u32,
|
||||
}
|
||||
|
||||
impl SheetGrid {
|
||||
#[must_use]
|
||||
pub fn new(sheet_name: String) -> Self {
|
||||
Self {
|
||||
sheet_name,
|
||||
rows: Vec::new(),
|
||||
num_fmt_kinds: Vec::new(),
|
||||
num_rows: 0,
|
||||
num_cols: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get cell value at (row, col). Returns Empty if out of bounds.
|
||||
#[must_use]
|
||||
pub fn cell(&self, row: u32, col: u32) -> &CellValue {
|
||||
static EMPTY: CellValue = CellValue::Empty;
|
||||
self.rows
|
||||
.get(row as usize)
|
||||
.and_then(|r| r.get(col as usize))
|
||||
.unwrap_or(&EMPTY)
|
||||
}
|
||||
|
||||
/// Get number format kind at (row, col). Returns General if not available.
|
||||
#[must_use]
|
||||
pub fn num_fmt(&self, row: u32, col: u32) -> NumFmtKind {
|
||||
self.num_fmt_kinds
|
||||
.get(row as usize)
|
||||
.and_then(|r| r.get(col as usize))
|
||||
.copied()
|
||||
.unwrap_or(NumFmtKind::General)
|
||||
}
|
||||
|
||||
/// Check if a row is entirely empty.
|
||||
#[must_use]
|
||||
pub fn is_row_empty(&self, row: u32) -> bool {
|
||||
if let Some(r) = self.rows.get(row as usize) {
|
||||
r.iter().all(CellValue::is_empty)
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Count non-empty cells in a row.
|
||||
#[must_use]
|
||||
pub fn row_nonempty_count(&self, row: u32) -> usize {
|
||||
if let Some(r) = self.rows.get(row as usize) {
|
||||
r.iter().filter(|c| !c.is_empty()).count()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect tables within a sheet grid.
|
||||
///
|
||||
/// Uses cascading heuristics:
|
||||
/// 1. OOXML table definitions (confidence 1.0)
|
||||
/// 2. All-text row + typed data below (0.7)
|
||||
/// 3. Type consistency boost (+0.15)
|
||||
/// 4. First non-empty row fallback (0.4)
|
||||
#[must_use]
|
||||
pub fn detect_tables(
|
||||
grid: &SheetGrid,
|
||||
ooxml_tables: &[TableDefinition],
|
||||
merged_regions: &[MergedRegion],
|
||||
) -> Vec<DetectedTable> {
|
||||
let mut tables = Vec::new();
|
||||
|
||||
// Phase 1: Use OOXML table definitions for this sheet
|
||||
for tdef in ooxml_tables {
|
||||
if tdef.sheet_name == grid.sheet_name {
|
||||
let column_types =
|
||||
infer_column_types(grid, tdef.first_row + 1, tdef.last_row, tdef.first_col, tdef.last_col);
|
||||
tables.push(DetectedTable {
|
||||
name: tdef.name.clone(),
|
||||
sheet_name: grid.sheet_name.clone(),
|
||||
headers: tdef.headers.clone(),
|
||||
column_types,
|
||||
first_data_row: tdef.first_row + 1,
|
||||
last_data_row: tdef.last_row,
|
||||
first_col: tdef.first_col,
|
||||
last_col: tdef.last_col,
|
||||
header_row: Some(tdef.first_row),
|
||||
confidence: 1.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// If OOXML tables covered the whole sheet, we're done
|
||||
if !tables.is_empty() {
|
||||
return tables;
|
||||
}
|
||||
|
||||
// Phase 2: Heuristic detection — find table boundaries
|
||||
let table_ranges = find_table_boundaries(grid, merged_regions);
|
||||
let mut table_idx = 0;
|
||||
|
||||
for (start_row, end_row, start_col, end_col) in table_ranges {
|
||||
let (header_row, headers, confidence) =
|
||||
detect_header(grid, start_row, end_row, start_col, end_col);
|
||||
|
||||
let first_data_row = header_row.map_or(start_row, |hr| hr + 1);
|
||||
let column_types = infer_column_types(grid, first_data_row, end_row, start_col, end_col);
|
||||
|
||||
// Boost confidence if column types are consistent
|
||||
let type_boost = if column_types.iter().filter(|t| **t != ColumnType::Mixed && **t != ColumnType::Empty).count()
|
||||
> column_types.len() / 2
|
||||
{
|
||||
0.15
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
table_idx += 1;
|
||||
tables.push(DetectedTable {
|
||||
name: format!("Table{table_idx}"),
|
||||
sheet_name: grid.sheet_name.clone(),
|
||||
headers,
|
||||
column_types,
|
||||
first_data_row,
|
||||
last_data_row: end_row,
|
||||
first_col: start_col,
|
||||
last_col: end_col,
|
||||
header_row,
|
||||
confidence: (confidence + type_boost).min(1.0),
|
||||
});
|
||||
}
|
||||
|
||||
tables
|
||||
}
|
||||
|
||||
/// Find table boundaries by detecting gaps (2+ consecutive empty rows/cols).
|
||||
fn find_table_boundaries(
|
||||
grid: &SheetGrid,
|
||||
_merged_regions: &[MergedRegion],
|
||||
) -> Vec<(u32, u32, u32, u32)> {
|
||||
if grid.num_rows == 0 || grid.num_cols == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Find vertical boundaries (consecutive empty rows split tables)
|
||||
let mut row_groups: Vec<(u32, u32)> = Vec::new();
|
||||
let mut current_start: Option<u32> = None;
|
||||
let mut empty_streak = 0u32;
|
||||
|
||||
for row in 0..grid.num_rows {
|
||||
if grid.is_row_empty(row) {
|
||||
empty_streak += 1;
|
||||
if empty_streak >= 2 {
|
||||
if let Some(start) = current_start.take() {
|
||||
let end = row.saturating_sub(empty_streak);
|
||||
if end >= start {
|
||||
row_groups.push((start, end));
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if current_start.is_none() {
|
||||
current_start = Some(row);
|
||||
}
|
||||
empty_streak = 0;
|
||||
}
|
||||
}
|
||||
// Close the last group
|
||||
if let Some(start) = current_start {
|
||||
row_groups.push((start, grid.num_rows.saturating_sub(1)));
|
||||
}
|
||||
|
||||
// For each row group, find column boundaries
|
||||
let mut boundaries = Vec::new();
|
||||
for (start_row, end_row) in row_groups {
|
||||
let col_ranges = find_column_boundaries(grid, start_row, end_row);
|
||||
for (start_col, end_col) in col_ranges {
|
||||
boundaries.push((start_row, end_row, start_col, end_col));
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: if no boundaries detected, treat entire used area as one table
|
||||
if boundaries.is_empty() && grid.num_rows > 0 {
|
||||
boundaries.push((
|
||||
0,
|
||||
grid.num_rows.saturating_sub(1),
|
||||
0,
|
||||
grid.num_cols.saturating_sub(1),
|
||||
));
|
||||
}
|
||||
|
||||
boundaries
|
||||
}
|
||||
|
||||
/// Find horizontal table boundaries within a row range.
|
||||
fn find_column_boundaries(grid: &SheetGrid, start_row: u32, end_row: u32) -> Vec<(u32, u32)> {
|
||||
if grid.num_cols == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Check which columns have any data in the row range
|
||||
let mut col_has_data = vec![false; grid.num_cols as usize];
|
||||
for row in start_row..=end_row {
|
||||
if let Some(r) = grid.rows.get(row as usize) {
|
||||
for (ci, cell) in r.iter().enumerate() {
|
||||
if !cell.is_empty() {
|
||||
col_has_data[ci] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find contiguous ranges of columns with data
|
||||
let mut ranges = Vec::new();
|
||||
let mut current_start: Option<u32> = None;
|
||||
let mut empty_streak = 0u32;
|
||||
|
||||
for (ci, &has_data) in col_has_data.iter().enumerate() {
|
||||
if has_data {
|
||||
if current_start.is_none() {
|
||||
current_start = Some(ci as u32);
|
||||
}
|
||||
empty_streak = 0;
|
||||
} else {
|
||||
empty_streak += 1;
|
||||
if empty_streak >= 2 {
|
||||
if let Some(start) = current_start.take() {
|
||||
let end = (ci as u32).saturating_sub(empty_streak);
|
||||
if end >= start {
|
||||
ranges.push((start, end));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(start) = current_start {
|
||||
ranges.push((start, (grid.num_cols).saturating_sub(1)));
|
||||
}
|
||||
|
||||
// Fallback: whole range
|
||||
if ranges.is_empty() {
|
||||
ranges.push((0, grid.num_cols.saturating_sub(1)));
|
||||
}
|
||||
|
||||
ranges
|
||||
}
|
||||
|
||||
/// Detect header row within a table range.
|
||||
/// Returns (header_row_index, header_texts, confidence).
|
||||
fn detect_header(
|
||||
grid: &SheetGrid,
|
||||
start_row: u32,
|
||||
end_row: u32,
|
||||
start_col: u32,
|
||||
end_col: u32,
|
||||
) -> (Option<u32>, Vec<String>, f64) {
|
||||
// Heuristic 1: All-text row followed by typed (numeric/date) data below
|
||||
for row in start_row..=end_row.min(start_row + 3) {
|
||||
let nonempty = grid.row_nonempty_count(row);
|
||||
if nonempty == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let all_text = (start_col..=end_col).all(|col| {
|
||||
let cell = grid.cell(row, col);
|
||||
cell.is_empty() || cell.is_text()
|
||||
});
|
||||
|
||||
if !all_text {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if the next row has any numeric/date data
|
||||
let next_row = row + 1;
|
||||
if next_row > end_row {
|
||||
continue;
|
||||
}
|
||||
let has_typed_data = (start_col..=end_col).any(|col| {
|
||||
let cell = grid.cell(next_row, col);
|
||||
cell.is_numeric() || matches!(cell, CellValue::DateTime(_) | CellValue::Boolean(_))
|
||||
});
|
||||
|
||||
if has_typed_data {
|
||||
let headers: Vec<String> = (start_col..=end_col)
|
||||
.map(|col| grid.cell(row, col).as_text())
|
||||
.collect();
|
||||
return (Some(row), headers, 0.7);
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic 2: First non-empty row as fallback
|
||||
for row in start_row..=end_row.min(start_row + 5) {
|
||||
if grid.row_nonempty_count(row) > 0 {
|
||||
let headers: Vec<String> = (start_col..=end_col)
|
||||
.map(|col| grid.cell(row, col).as_text())
|
||||
.collect();
|
||||
return (Some(row), headers, 0.4);
|
||||
}
|
||||
}
|
||||
|
||||
(None, Vec::new(), 0.3)
|
||||
}
|
||||
|
||||
/// Infer column types by sampling data rows.
|
||||
fn infer_column_types(
|
||||
grid: &SheetGrid,
|
||||
first_data_row: u32,
|
||||
last_data_row: u32,
|
||||
first_col: u32,
|
||||
last_col: u32,
|
||||
) -> Vec<ColumnType> {
|
||||
let num_cols = (last_col - first_col + 1) as usize;
|
||||
let mut type_counts: Vec<[u32; 10]> = vec![[0; 10]; num_cols];
|
||||
let sample_limit = 100;
|
||||
for (sampled, row) in (first_data_row..=last_data_row).enumerate() {
|
||||
if sampled >= sample_limit as usize {
|
||||
break;
|
||||
}
|
||||
|
||||
for col_offset in 0..num_cols {
|
||||
let col = first_col + col_offset as u32;
|
||||
let cell = grid.cell(row, col);
|
||||
let fmt = grid.num_fmt(row, col);
|
||||
|
||||
let type_idx = match (cell, fmt) {
|
||||
(CellValue::Empty, _) => 9, // Empty
|
||||
(CellValue::Text(_), _) => 0,
|
||||
(CellValue::Integer(_), NumFmtKind::Date) => 2,
|
||||
(CellValue::Integer(_), NumFmtKind::DateTime) => 3,
|
||||
(CellValue::Integer(_), NumFmtKind::Time) => 4,
|
||||
(CellValue::Integer(_), NumFmtKind::Currency) => 5,
|
||||
(CellValue::Integer(_), NumFmtKind::Percentage) => 6,
|
||||
(CellValue::Integer(_), _) => 1,
|
||||
(CellValue::Number(_), NumFmtKind::Date) => 2,
|
||||
(CellValue::Number(_), NumFmtKind::DateTime) => 3,
|
||||
(CellValue::Number(_), NumFmtKind::Time) => 4,
|
||||
(CellValue::Number(_), NumFmtKind::Currency) => 5,
|
||||
(CellValue::Number(_), NumFmtKind::Percentage) => 6,
|
||||
(CellValue::Number(_), _) => 8, // Float
|
||||
(CellValue::Boolean(_), _) => 7,
|
||||
(CellValue::DateTime(_), _) => 2,
|
||||
(CellValue::Error(_), _) => 9,
|
||||
};
|
||||
type_counts[col_offset][type_idx] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
type_counts
|
||||
.iter()
|
||||
.map(|counts| {
|
||||
// Find the most common non-empty type
|
||||
let non_empty_total: u32 = counts.iter().take(9).sum();
|
||||
if non_empty_total == 0 {
|
||||
return ColumnType::Empty;
|
||||
}
|
||||
|
||||
let (max_idx, &max_count) = counts
|
||||
.iter()
|
||||
.take(9)
|
||||
.enumerate()
|
||||
.max_by_key(|&(_, c)| *c)
|
||||
.unwrap_or((0, &0));
|
||||
|
||||
// If >30% are a different type, it's mixed
|
||||
let threshold = (non_empty_total as f64 * 0.3).ceil() as u32;
|
||||
let other_count = non_empty_total - max_count;
|
||||
if other_count >= threshold && max_count < non_empty_total {
|
||||
return ColumnType::Mixed;
|
||||
}
|
||||
|
||||
match max_idx {
|
||||
0 => ColumnType::Text,
|
||||
1 => ColumnType::Integer,
|
||||
2 => ColumnType::Date,
|
||||
3 => ColumnType::DateTime,
|
||||
4 => ColumnType::Time,
|
||||
5 => ColumnType::Currency,
|
||||
6 => ColumnType::Percentage,
|
||||
7 => ColumnType::Boolean,
|
||||
8 => ColumnType::Float,
|
||||
_ => ColumnType::Text,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Propagate merged cell values into a grid.
|
||||
/// The top-left cell's value is copied to all cells in the merged region.
|
||||
pub fn propagate_merged_cells(grid: &mut SheetGrid, merged_regions: &[MergedRegion]) {
|
||||
for region in merged_regions {
|
||||
// Get the top-left cell value
|
||||
let value = grid.cell(region.top_row, region.left_col).clone();
|
||||
if value.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fill all cells in the region with the top-left value
|
||||
for row in region.top_row..=region.bottom_row {
|
||||
for col in region.left_col..=region.right_col {
|
||||
// Skip the top-left cell itself
|
||||
if row == region.top_row && col == region.left_col {
|
||||
continue;
|
||||
}
|
||||
if let Some(r) = grid.rows.get_mut(row as usize) {
|
||||
// Extend the row if necessary
|
||||
while r.len() <= col as usize {
|
||||
r.push(CellValue::Empty);
|
||||
}
|
||||
r[col as usize] = value.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_grid(data: Vec<Vec<CellValue>>, sheet_name: &str) -> SheetGrid {
|
||||
let num_rows = data.len() as u32;
|
||||
let num_cols = data.iter().map(|r| r.len()).max().unwrap_or(0) as u32;
|
||||
SheetGrid {
|
||||
sheet_name: sheet_name.to_string(),
|
||||
rows: data,
|
||||
num_fmt_kinds: Vec::new(),
|
||||
num_rows,
|
||||
num_cols,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_header_all_text_row() {
|
||||
let grid = make_grid(
|
||||
vec![
|
||||
vec![
|
||||
CellValue::Text("Name".into()),
|
||||
CellValue::Text("Age".into()),
|
||||
CellValue::Text("City".into()),
|
||||
],
|
||||
vec![
|
||||
CellValue::Text("Alice".into()),
|
||||
CellValue::Integer(30),
|
||||
CellValue::Text("Austin".into()),
|
||||
],
|
||||
vec![
|
||||
CellValue::Text("Bob".into()),
|
||||
CellValue::Integer(25),
|
||||
CellValue::Text("Boston".into()),
|
||||
],
|
||||
],
|
||||
"Sheet1",
|
||||
);
|
||||
|
||||
let tables = detect_tables(&grid, &[], &[]);
|
||||
assert_eq!(tables.len(), 1);
|
||||
assert_eq!(tables[0].headers, vec!["Name", "Age", "City"]);
|
||||
assert!(tables[0].confidence >= 0.7);
|
||||
assert_eq!(tables[0].header_row, Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detect_multi_table_gap() {
|
||||
let grid = make_grid(
|
||||
vec![
|
||||
vec![CellValue::Text("A".into()), CellValue::Integer(1)],
|
||||
vec![CellValue::Text("B".into()), CellValue::Integer(2)],
|
||||
vec![CellValue::Empty, CellValue::Empty],
|
||||
vec![CellValue::Empty, CellValue::Empty],
|
||||
vec![CellValue::Text("X".into()), CellValue::Integer(10)],
|
||||
vec![CellValue::Text("Y".into()), CellValue::Integer(20)],
|
||||
],
|
||||
"Sheet1",
|
||||
);
|
||||
|
||||
let tables = detect_tables(&grid, &[], &[]);
|
||||
assert_eq!(tables.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_propagate_merged_cells() {
|
||||
let mut grid = make_grid(
|
||||
vec![
|
||||
vec![
|
||||
CellValue::Text("Merged Title".into()),
|
||||
CellValue::Empty,
|
||||
CellValue::Empty,
|
||||
],
|
||||
vec![
|
||||
CellValue::Text("A".into()),
|
||||
CellValue::Text("B".into()),
|
||||
CellValue::Text("C".into()),
|
||||
],
|
||||
],
|
||||
"Sheet1",
|
||||
);
|
||||
|
||||
let regions = vec![MergedRegion {
|
||||
top_row: 0,
|
||||
left_col: 0,
|
||||
bottom_row: 0,
|
||||
right_col: 2,
|
||||
}];
|
||||
|
||||
propagate_merged_cells(&mut grid, ®ions);
|
||||
|
||||
assert!(matches!(grid.cell(0, 1), CellValue::Text(s) if s == "Merged Title"));
|
||||
assert!(matches!(grid.cell(0, 2), CellValue::Text(s) if s == "Merged Title"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_column_type_inference() {
|
||||
let grid = make_grid(
|
||||
vec![
|
||||
vec![
|
||||
CellValue::Text("Name".into()),
|
||||
CellValue::Text("Value".into()),
|
||||
],
|
||||
vec![CellValue::Text("A".into()), CellValue::Integer(100)],
|
||||
vec![CellValue::Text("B".into()), CellValue::Integer(200)],
|
||||
vec![CellValue::Text("C".into()), CellValue::Integer(300)],
|
||||
],
|
||||
"Sheet1",
|
||||
);
|
||||
|
||||
let types = infer_column_types(&grid, 1, 3, 0, 1);
|
||||
assert_eq!(types[0], ColumnType::Text);
|
||||
assert_eq!(types[1], ColumnType::Integer);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_grid() {
|
||||
let grid = make_grid(Vec::new(), "Empty");
|
||||
let tables = detect_tables(&grid, &[], &[]);
|
||||
assert!(tables.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,509 @@
|
||||
//! Integration tests for the structured XLSX extraction pipeline.
|
||||
//!
|
||||
//! Uses `/Users/olow/Desktop/memvid-org/arden.xlsx` — a real-world 1.7 MB
|
||||
//! real-estate pro forma with 19 sheets, merged cells, currency/date formats,
|
||||
//! and multi-table layouts.
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
use memvid_core::{
|
||||
DetectedTable, Memvid, PutOptions, SearchRequest, XlsxChunkingOptions, XlsxReader,
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
const ARDEN_PATH: &str = "/Users/olow/Desktop/memvid-org/arden.xlsx";
|
||||
|
||||
fn load_arden() -> Vec<u8> {
|
||||
std::fs::read(ARDEN_PATH).expect("arden.xlsx must exist at the expected path")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 1: Structured extraction speed + completeness
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn structured_extraction_completes_under_5s() {
|
||||
let bytes = load_arden();
|
||||
let start = Instant::now();
|
||||
let result = XlsxReader::extract_structured(&bytes).expect("extraction must succeed");
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
println!("Extraction time: {elapsed:?}");
|
||||
println!("Flat text length: {} chars", result.text.len());
|
||||
println!("Tables detected: {}", result.tables.len());
|
||||
println!("Chunks produced: {}", result.chunks.chunks.len());
|
||||
println!("Diagnostics warnings: {}", result.diagnostics.warnings.len());
|
||||
|
||||
assert!(
|
||||
elapsed.as_secs() < 5,
|
||||
"Structured extraction took {elapsed:?} — should be under 5s"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_multiple_tables_across_sheets() {
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
// 19 sheets — should detect at least several tables
|
||||
assert!(
|
||||
result.tables.len() >= 5,
|
||||
"Expected at least 5 tables from a 19-sheet workbook, got {}",
|
||||
result.tables.len()
|
||||
);
|
||||
|
||||
// Collect unique sheet names
|
||||
let sheet_names: std::collections::HashSet<&str> =
|
||||
result.tables.iter().map(|t| t.sheet_name.as_str()).collect();
|
||||
println!("Sheets with tables: {sheet_names:?}");
|
||||
|
||||
assert!(
|
||||
sheet_names.len() >= 3,
|
||||
"Tables should span at least 3 sheets, got {}",
|
||||
sheet_names.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunks_have_header_context() {
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
assert!(
|
||||
!result.chunks.chunks.is_empty(),
|
||||
"Should produce at least one chunk"
|
||||
);
|
||||
|
||||
// Every chunk should contain sheet context prefix
|
||||
let chunks_with_sheet = result
|
||||
.chunks
|
||||
.chunks
|
||||
.iter()
|
||||
.filter(|c| c.text.contains("[Sheet:"))
|
||||
.count();
|
||||
|
||||
let ratio = chunks_with_sheet as f64 / result.chunks.chunks.len() as f64;
|
||||
println!(
|
||||
"Chunks with [Sheet:] prefix: {chunks_with_sheet}/{} ({:.0}%)",
|
||||
result.chunks.chunks.len(),
|
||||
ratio * 100.0
|
||||
);
|
||||
|
||||
assert!(
|
||||
ratio > 0.8,
|
||||
"At least 80% of chunks should have sheet context, got {:.0}%",
|
||||
ratio * 100.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunks_respect_row_boundaries() {
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
for chunk in &result.chunks.chunks {
|
||||
// No chunk should end with a partial header:value pair mid-line
|
||||
// Each data line should have balanced pipes (Header: Value | Header: Value)
|
||||
for line in chunk.text.lines().skip(2) {
|
||||
// skip [Sheet:] prefix + header row
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Lines should not be cut mid-cell (no trailing ':' without a value)
|
||||
let trailing_colon = line.trim_end().ends_with(':');
|
||||
assert!(
|
||||
!trailing_colon,
|
||||
"Chunk has a line ending with bare colon (mid-row split?): {:?}",
|
||||
&line[..line.len().min(80)]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_sizes_near_target() {
|
||||
let bytes = load_arden();
|
||||
let opts = XlsxChunkingOptions {
|
||||
max_chars: 1200,
|
||||
max_chunks: 500,
|
||||
};
|
||||
let result = XlsxReader::extract_structured_with_options(&bytes, opts).unwrap();
|
||||
|
||||
let sizes: Vec<usize> = result.chunks.chunks.iter().map(|c| c.text.len()).collect();
|
||||
let avg = sizes.iter().sum::<usize>() as f64 / sizes.len().max(1) as f64;
|
||||
let max = sizes.iter().max().copied().unwrap_or(0);
|
||||
|
||||
println!(
|
||||
"Chunk count: {}, avg size: {avg:.0} chars, max: {max} chars",
|
||||
sizes.len()
|
||||
);
|
||||
|
||||
// Average should be in a reasonable range
|
||||
assert!(
|
||||
avg < 2000.0,
|
||||
"Average chunk is {avg:.0} chars — way over target"
|
||||
);
|
||||
|
||||
// No chunk should be absurdly large (allow 3x target for wide rows)
|
||||
assert!(
|
||||
max < 5000,
|
||||
"Max chunk is {max} chars — should be under 5000"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 2: OOXML metadata (merged cells, number formats)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn merged_regions_detected() {
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
let total_merged: usize = result.metadata.merged_regions.values().map(|v| v.len()).sum();
|
||||
println!("Total merged regions: {total_merged}");
|
||||
|
||||
// A complex real-estate pro forma with 19 sheets should have many merged cells
|
||||
assert!(
|
||||
total_merged > 0,
|
||||
"Expected merged regions in a complex workbook"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn number_formats_parsed() {
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
println!(
|
||||
"Number format entries: {}",
|
||||
result.metadata.num_fmts.len()
|
||||
);
|
||||
println!(
|
||||
"Cell XF entries: {}",
|
||||
result.metadata.cell_xfs.len()
|
||||
);
|
||||
|
||||
// Financial workbook should have custom number formats
|
||||
assert!(
|
||||
!result.metadata.num_fmts.is_empty() || !result.metadata.cell_xfs.is_empty(),
|
||||
"Expected number format metadata from a financial workbook"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 3: Flat text backward compatibility
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn flat_text_contains_key_data() {
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
let text = &result.text;
|
||||
assert!(
|
||||
text.len() > 1000,
|
||||
"Flat text should be substantial, got {} chars",
|
||||
text.len()
|
||||
);
|
||||
|
||||
// Check for known content from the arden.xlsx file
|
||||
let text_lower = text.to_lowercase();
|
||||
|
||||
// The file is a real estate deal for "TRG Apartments" in SLC, UT
|
||||
let key_terms = [
|
||||
"sheet:", // Should have sheet labels
|
||||
"248", // 248 units
|
||||
];
|
||||
|
||||
for term in &key_terms {
|
||||
assert!(
|
||||
text_lower.contains(&term.to_lowercase()),
|
||||
"Flat text should contain '{term}'"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 4: End-to-end ingestion + search accuracy
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Ingest the XLSX into a Memvid file and return the path + temp dir (to keep alive).
|
||||
fn ingest_arden() -> (std::path::PathBuf, TempDir) {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mv2_path = dir.path().join("arden.mv2");
|
||||
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
let mut mem = Memvid::create(&mv2_path).unwrap();
|
||||
mem.enable_lex().unwrap();
|
||||
|
||||
// Ingest each chunk as a separate frame with search_text set to the chunk content
|
||||
for (i, chunk) in result.chunks.chunks.iter().enumerate() {
|
||||
let opts = PutOptions {
|
||||
uri: Some(format!("mv2://arden/chunk/{i}")),
|
||||
title: Some(format!("Arden XLSX chunk {i}")),
|
||||
search_text: Some(chunk.text.clone()),
|
||||
auto_tag: false,
|
||||
extract_dates: false,
|
||||
extract_triplets: false,
|
||||
..Default::default()
|
||||
};
|
||||
mem.put_bytes_with_options(chunk.text.as_bytes(), opts)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
mem.commit().unwrap();
|
||||
(mv2_path, dir)
|
||||
}
|
||||
|
||||
fn search_arden(
|
||||
mem: &mut Memvid,
|
||||
query: &str,
|
||||
top_k: usize,
|
||||
) -> Vec<memvid_core::SearchHit> {
|
||||
mem.search(SearchRequest {
|
||||
query: query.to_string(),
|
||||
top_k,
|
||||
snippet_chars: 300,
|
||||
uri: None,
|
||||
scope: None,
|
||||
cursor: None,
|
||||
#[cfg(feature = "temporal_track")]
|
||||
temporal: None,
|
||||
as_of_frame: None,
|
||||
as_of_ts: None,
|
||||
no_sketch: false,
|
||||
acl_context: None,
|
||||
acl_enforcement_mode: memvid_core::types::AclEnforcementMode::Audit,
|
||||
})
|
||||
.unwrap()
|
||||
.hits
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "lex")]
|
||||
fn ingest_and_search_units() {
|
||||
let (path, _dir) = ingest_arden();
|
||||
let mut mem = Memvid::open_read_only(&path).unwrap();
|
||||
|
||||
// Search for unit count — the file has 248 multifamily units
|
||||
let hits = search_arden(&mut mem, "248 units", 5);
|
||||
|
||||
println!(
|
||||
"Query '248 units' — {} hits",
|
||||
hits.len()
|
||||
);
|
||||
for (i, h) in hits.iter().enumerate() {
|
||||
println!(
|
||||
" [{i}] score={:.3} uri={} text={:.120}",
|
||||
h.score.unwrap_or(0.0),
|
||||
h.uri,
|
||||
h.text.replace('\n', " ")
|
||||
);
|
||||
}
|
||||
|
||||
assert!(!hits.is_empty(), "Should find results for '248 units'");
|
||||
|
||||
// At least one hit should contain "248"
|
||||
let has_248 = hits.iter().any(|h| h.text.contains("248"));
|
||||
assert!(has_248, "At least one hit should contain '248'");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "lex")]
|
||||
fn ingest_and_search_financial_terms() {
|
||||
let (path, _dir) = ingest_arden();
|
||||
let mut mem = Memvid::open_read_only(&path).unwrap();
|
||||
|
||||
// The file contains construction costs, debt service, NOI, etc.
|
||||
let queries = [
|
||||
"construction",
|
||||
"debt",
|
||||
"occupancy",
|
||||
"revenue",
|
||||
"lease",
|
||||
];
|
||||
|
||||
let mut found_count = 0;
|
||||
for query in &queries {
|
||||
let hits = search_arden(&mut mem, query, 3);
|
||||
println!("Query '{query}': {} hits", hits.len());
|
||||
|
||||
if !hits.is_empty() {
|
||||
found_count += 1;
|
||||
println!(
|
||||
" Top hit: score={:.3} text={:.100}",
|
||||
hits[0].score.unwrap_or(0.0),
|
||||
hits[0].text.replace('\n', " ")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// At least 3 out of 5 financial queries should return results
|
||||
assert!(
|
||||
found_count >= 3,
|
||||
"Expected at least 3/5 financial queries to match, got {found_count}/5"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "lex")]
|
||||
fn search_hits_contain_header_context() {
|
||||
let (path, _dir) = ingest_arden();
|
||||
let mut mem = Memvid::open_read_only(&path).unwrap();
|
||||
|
||||
let hits = search_arden(&mut mem, "construction", 5);
|
||||
|
||||
if hits.is_empty() {
|
||||
println!("WARN: no hits for 'construction' — skipping header context check");
|
||||
return;
|
||||
}
|
||||
|
||||
// Check that hit text contains structured context (sheet/table prefix or header:value pairs)
|
||||
let has_context = hits.iter().any(|h| {
|
||||
h.text.contains("[Sheet:") || h.text.contains(':')
|
||||
});
|
||||
|
||||
assert!(
|
||||
has_context,
|
||||
"Search hits should contain structured context (sheet prefix or header:value pairs)"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 5: Full pipeline timing benchmark
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "lex")]
|
||||
fn full_pipeline_timing() {
|
||||
let bytes = load_arden();
|
||||
|
||||
// Step 1: Structured extraction
|
||||
let t0 = Instant::now();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
let extraction_time = t0.elapsed();
|
||||
|
||||
// Step 2: Memvid create + lex enable
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mv2_path = dir.path().join("bench.mv2");
|
||||
|
||||
let t1 = Instant::now();
|
||||
let mut mem = Memvid::create(&mv2_path).unwrap();
|
||||
mem.enable_lex().unwrap();
|
||||
|
||||
// Step 3: Ingest all chunks
|
||||
for (i, chunk) in result.chunks.chunks.iter().enumerate() {
|
||||
let opts = PutOptions {
|
||||
uri: Some(format!("mv2://arden/chunk/{i}")),
|
||||
title: Some(format!("Chunk {i}")),
|
||||
search_text: Some(chunk.text.clone()),
|
||||
auto_tag: false,
|
||||
extract_dates: false,
|
||||
extract_triplets: false,
|
||||
..Default::default()
|
||||
};
|
||||
mem.put_bytes_with_options(chunk.text.as_bytes(), opts)
|
||||
.unwrap();
|
||||
}
|
||||
mem.commit().unwrap();
|
||||
let ingest_time = t1.elapsed();
|
||||
|
||||
// Step 4: Search
|
||||
let mut mem = Memvid::open_read_only(&mv2_path).unwrap();
|
||||
let t2 = Instant::now();
|
||||
let hits = search_arden(&mut mem, "construction cost", 10);
|
||||
let search_time = t2.elapsed();
|
||||
|
||||
let total = extraction_time + ingest_time + search_time;
|
||||
|
||||
println!("=== Full Pipeline Timing ===");
|
||||
println!(" XLSX extraction: {extraction_time:?}");
|
||||
println!(" Memvid ingest: {ingest_time:?} ({} chunks)", result.chunks.chunks.len());
|
||||
println!(" Search query: {search_time:?} ({} hits)", hits.len());
|
||||
println!(" TOTAL: {total:?}");
|
||||
println!(" Tables detected: {}", result.tables.len());
|
||||
println!(" Flat text chars: {}", result.text.len());
|
||||
println!(" MV2 file size: {} KB", std::fs::metadata(&mv2_path).unwrap().len() / 1024);
|
||||
|
||||
// In release mode, target is under 40s. Debug mode gets 3x slack for
|
||||
// unoptimized Tantivy indexing on 500 individual put_bytes calls.
|
||||
let limit = if cfg!(debug_assertions) { 180 } else { 40 };
|
||||
assert!(
|
||||
total.as_secs() < limit,
|
||||
"Full pipeline took {total:?} — target is under {limit}s"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase 6: Table detection quality
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn tables_have_headers() {
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
let tables_with_headers: Vec<&DetectedTable> = result
|
||||
.tables
|
||||
.iter()
|
||||
.filter(|t| !t.headers.is_empty())
|
||||
.collect();
|
||||
|
||||
println!(
|
||||
"Tables with headers: {}/{}",
|
||||
tables_with_headers.len(),
|
||||
result.tables.len()
|
||||
);
|
||||
|
||||
for t in &tables_with_headers {
|
||||
println!(
|
||||
" [{}] '{}' — {} headers, {} rows, confidence={:.2}",
|
||||
t.sheet_name,
|
||||
t.name,
|
||||
t.headers.len(),
|
||||
t.last_data_row.saturating_sub(t.first_data_row) + 1,
|
||||
t.confidence
|
||||
);
|
||||
}
|
||||
|
||||
// Most tables should have detected headers
|
||||
let ratio = tables_with_headers.len() as f64 / result.tables.len().max(1) as f64;
|
||||
assert!(
|
||||
ratio > 0.5,
|
||||
"At least 50% of tables should have headers, got {:.0}%",
|
||||
ratio * 100.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_column_types_inferred() {
|
||||
let bytes = load_arden();
|
||||
let result = XlsxReader::extract_structured(&bytes).unwrap();
|
||||
|
||||
let tables_with_types: Vec<&DetectedTable> = result
|
||||
.tables
|
||||
.iter()
|
||||
.filter(|t| !t.column_types.is_empty())
|
||||
.collect();
|
||||
|
||||
println!(
|
||||
"Tables with column types: {}/{}",
|
||||
tables_with_types.len(),
|
||||
result.tables.len()
|
||||
);
|
||||
|
||||
for t in &tables_with_types[..tables_with_types.len().min(5)] {
|
||||
println!(
|
||||
" [{}] '{}' — types: {:?}",
|
||||
t.sheet_name, t.name, t.column_types
|
||||
);
|
||||
}
|
||||
|
||||
assert!(
|
||||
!tables_with_types.is_empty(),
|
||||
"At least some tables should have inferred column types"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user