github-featured/hmbown--codewhale
2
0
Fork 0
Code Issues Pull Requests Actions 16 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

fix(tui): keep display-width paths grapheme-safe

Browse Source 
Measure and split rendered text at extended grapheme boundaries so keycaps, ZWJ emoji, and combining sequences stay aligned with Ratatui cell accounting. Cover transcript wrapping, selection, slicing, and truncation regressions.

Signed-off-by: Hunter B <hmbown@gmail.com>
This commit is contained in:
Hunter B
2026-07-17 20:11:48 -07:00
parent fd2584f08f
commit 9be600bf45
3 changed files with 173 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/tui/src/tui/markdown_render.rs
+69 -54
View File
@@ -29,6 +29,7 @@ use std::cell::Cell;
use ratatui::style::{Modifier, Style};
use ratatui::text::{Line, Span};
use unicode_segmentation::UnicodeSegmentation;
use unicode_width::{UnicodeWidthChar, UnicodeWidthStr};
use crate::palette;
@@ -379,10 +380,10 @@ fn wrap_plain_line(line: &str, width: usize, style: Style) -> Vec<Line<'static>>
let mut current_width = 0usize;
let mut last_break_pos = None;
for ch in line.chars() {
for grapheme in line.graphemes(true) {
loop {
let ch_width = char_display_width(ch, current_width);
if current_width + ch_width <= width || current.is_empty() {
let grapheme_width = markdown_grapheme_width(grapheme, current_width);
if current_width + grapheme_width <= width || current.is_empty() {
break;
}
@@ -410,10 +411,10 @@ fn wrap_plain_line(line: &str, width: usize, style: Style) -> Vec<Line<'static>>
break;
}
let ch_width = char_display_width(ch, current_width);
current.push(ch);
current_width += ch_width;
if ch.is_whitespace() {
let grapheme_width = markdown_grapheme_width(grapheme, current_width);
current.push_str(grapheme);
current_width += grapheme_width;
if grapheme.chars().all(char::is_whitespace) {
last_break_pos = Some(current.len());
}
}
@@ -434,8 +435,8 @@ fn wrap_plain_line(line: &str, width: usize, style: Style) -> Vec<Line<'static>>
fn plain_display_width(text: &str) -> usize {
let mut width = 0usize;
for ch in text.chars() {
width += char_display_width(ch, width);
for grapheme in text.graphemes(true) {
width += markdown_grapheme_width(grapheme, width);
}
width
}
@@ -646,7 +647,7 @@ fn render_line_with_links_tagged(
continue;
}
// If the word itself is wider than an entire line, hard-break it at
// character boundaries so wrapping always makes progress (#1344,
// grapheme boundaries so wrapping always makes progress (#1344,
// #1351). Without this, long URLs / paths / hashes were placed on
// their own line whole and silently overflowed the right edge of
// the transcript.
@@ -666,9 +667,9 @@ fn render_line_with_links_tagged(
// current line so the next word can pack onto it.
let mut chunk = String::new();
let mut chunk_w = 0usize;
for ch in word.text.chars() {
let cw = ch.width().unwrap_or(1);
if chunk_w + cw > width && chunk_w > 0 {
for grapheme in word.text.graphemes(true) {
let grapheme_width = grapheme.width();
if chunk_w + grapheme_width > width && chunk_w > 0 {
let chunk = std::mem::take(&mut chunk);
let mut links = Vec::new();
record_inline_link(&mut links, &word, 0, chunk_w);
@@ -681,8 +682,8 @@ fn render_line_with_links_tagged(
});
chunk_w = 0;
}
chunk.push(ch);
chunk_w += cw;
chunk.push_str(grapheme);
chunk_w += grapheme_width;
}
if !chunk.is_empty() {
record_inline_link(&mut current_links, &word, 0, chunk_w);
@@ -1076,7 +1077,7 @@ fn split_table_cells(inner: &str) -> Vec<String> {
/// Word-wrap a single cell's text into one or more visual lines, each
/// constrained to `col_width` display columns. Whitespace is the preferred
/// break point; words wider than `col_width` are hard-broken at character
/// break point; words wider than `col_width` are hard-broken at grapheme
/// boundaries so wrapping always makes progress (no infinite loop on URLs
/// or paths). Returns at least one segment.
fn wrap_cell_text(cell: &str, col_width: usize) -> Vec<String> {
@@ -1091,7 +1092,13 @@ fn wrap_cell_text(cell: &str, col_width: usize) -> Vec<String> {
let word_w = word.width();
if current_w == 0 {
if word_w > col_width {
push_word_breaking_chars(word, col_width, &mut current, &mut current_w, &mut lines);
push_word_breaking_graphemes(
word,
col_width,
&mut current,
&mut current_w,
&mut lines,
);
} else {
current.push_str(word);
current_w = word_w;
@@ -1104,7 +1111,13 @@ fn wrap_cell_text(cell: &str, col_width: usize) -> Vec<String> {
lines.push(std::mem::take(&mut current));
current_w = 0;
if word_w > col_width {
push_word_breaking_chars(word, col_width, &mut current, &mut current_w, &mut lines);
push_word_breaking_graphemes(
word,
col_width,
&mut current,
&mut current_w,
&mut lines,
);
} else {
current.push_str(word);
current_w = word_w;
@@ -1252,22 +1265,22 @@ fn link_style() -> Style {
.add_modifier(Modifier::UNDERLINED)
}
/// Hard-wrap a code line at `width` display columns, preserving all
/// whitespace (including leading indentation). Unlike [`wrap_text`], this
/// does not split on word boundaries — code indentation is semantic.
/// Display-column width of a single character for the purposes of terminal
/// line-wrap calculations.
/// Display-column width of one extended grapheme for terminal line-wrap
/// calculations.
///
/// `UnicodeWidthChar::width` returns `None` for control characters, which
/// includes `\t`. A tab advances to the next 8-column tab stop, so we model
/// it as 8 columns here (a safe over-estimate that avoids terminal overflow).
/// Other control characters are counted as 1 column.
fn char_display_width(ch: char, col: usize) -> usize {
match ch {
'\t' => 8 - (col % 8), // advance to next 8-column tab stop
'\u{20E3}' => 1, // COMBINING ENCLOSING KEYCAP
_ => ch.width().unwrap_or(1),
/// A tab advances to the next 8-column tab stop. Single control characters
/// retain the previous one-column fallback; multi-codepoint emoji and
/// combining sequences use the same string-level width contract as Ratatui.
fn markdown_grapheme_width(grapheme: &str, col: usize) -> usize {
if grapheme == "\t" {
return 8 - (col % 8); // advance to next 8-column tab stop
}
if let Some(ch) = grapheme.chars().next()
&& ch.len_utf8() == grapheme.len()
{
return ch.width().unwrap_or(1);
}
grapheme.width()
}
/// Hard-wrap a code line at `width` display columns, preserving all
@@ -1281,15 +1294,15 @@ fn wrap_code_line(line: &str, width: usize) -> Vec<String> {
let mut current = String::new();
let mut current_width = 0usize;
for ch in line.chars() {
let ch_width = char_display_width(ch, current_width);
if current_width + ch_width > width && !current.is_empty() {
for grapheme in line.graphemes(true) {
let grapheme_width = markdown_grapheme_width(grapheme, current_width);
if current_width + grapheme_width > width && !current.is_empty() {
chunks.push(current);
current = String::new();
current_width = 0;
}
current.push(ch);
current_width += ch_width;
current.push_str(grapheme);
current_width += grapheme_width;
}
chunks.push(current);
chunks
@@ -1306,7 +1319,7 @@ fn wrap_text(text: &str, width: usize) -> Vec<String> {
for word in text.split_whitespace() {
let word_width = word.width();
// If this single word is wider than the entire line, hard-break it
// at character boundaries so wrapping always makes progress
// at grapheme boundaries so wrapping always makes progress
// (#1344, #1351). Without this, long URLs / paths / hashes overflow
// the right edge silently.
if word_width > width {
@@ -1314,7 +1327,7 @@ fn wrap_text(text: &str, width: usize) -> Vec<String> {
lines.push(std::mem::take(&mut current));
current_width = 0;
}
push_word_breaking_chars(word, width, &mut current, &mut current_width, &mut lines);
push_word_breaking_graphemes(word, width, &mut current, &mut current_width, &mut lines);
continue;
}
let additional = if current.is_empty() {
@@ -1345,26 +1358,26 @@ fn wrap_text(text: &str, width: usize) -> Vec<String> {
lines
}
/// Push characters from `word` into `current`, flushing to `lines` when the
/// running display width would exceed `width`. Width is computed at the
/// `unicode-width` char level, matching the rest of the rendering pipeline.
/// Push graphemes from `word` into `current`, flushing to `lines` when the
/// running display width would exceed `width`. String-level Unicode width
/// matches Ratatui for emoji and combining sequences.
/// Used by `wrap_text` and `wrap_cell_text` so a word longer than the
/// allotted width never silently overflows the right edge.
fn push_word_breaking_chars(
fn push_word_breaking_graphemes(
word: &str,
width: usize,
current: &mut String,
current_width: &mut usize,
lines: &mut Vec<String>,
) {
for ch in word.chars() {
let cw = ch.width().unwrap_or(1);
if *current_width + cw > width && *current_width > 0 {
for grapheme in word.graphemes(true) {
let grapheme_width = grapheme.width();
if *current_width + grapheme_width > width && *current_width > 0 {
lines.push(std::mem::take(current));
*current_width = 0;
}
current.push(ch);
*current_width += cw;
current.push_str(grapheme);
*current_width += grapheme_width;
}
}
@@ -1592,15 +1605,17 @@ mod tests {
}
#[test]
fn char_display_width_tab_uses_tab_stop() {
fn markdown_grapheme_width_uses_tab_stop_and_string_width() {
// At column 0 a tab fills to column 8.
assert_eq!(char_display_width('\t', 0), 8);
assert_eq!(markdown_grapheme_width("\t", 0), 8);
// At column 4 a tab fills to column 8 (4 remaining).
assert_eq!(char_display_width('\t', 4), 4);
assert_eq!(markdown_grapheme_width("\t", 4), 4);
// At column 8 a tab fills to the next stop at 16 (8 columns).
assert_eq!(char_display_width('\t', 8), 8);
assert_eq!(markdown_grapheme_width("\t", 8), 8);
// Regular ASCII is 1.
assert_eq!(char_display_width('a', 0), 1);
assert_eq!(markdown_grapheme_width("a", 0), 1);
// A fully-qualified keycap is one two-column grapheme.
assert_eq!(markdown_grapheme_width("1\u{fe0f}\u{20e3}", 0), 2);
}
#[test]
@@ -1932,7 +1947,7 @@ mod tests {
// width was placed alone on a line and silently overflowed the right
// edge of the transcript. Long URLs / paths / hashes / no-whitespace
// CJK runs all hit this. The fix hard-breaks overlong words at
// character boundaries; these tests pin that across widths 40/60/80/120.
// grapheme boundaries; these tests pin that across widths 40/60/80/120.
fn rendered_widths(rendered: &[Line<'static>]) -> Vec<usize> {
rendered
crates/tui/src/tui/ui_text.rs
+77 -59
View File
@@ -1,6 +1,7 @@
//! Shared text helpers for TUI selection and clipboard workflows.
use ratatui::text::{Line, Span};
use unicode_segmentation::UnicodeSegmentation;
use unicode_width::{UnicodeWidthChar, UnicodeWidthStr};
use crate::tui::history::HistoryCell;
@@ -28,20 +29,21 @@ pub(crate) fn truncate_line_to_width(text: &str, max_width: usize) -> String {
if max_width == 0 {
return String::new();
}
if UnicodeWidthStr::width(text) <= max_width {
if text_display_width(text) <= max_width {
return text.to_string();
}
// For very small budgets, take chars until we exceed the *display* width.
// For very small budgets, take whole graphemes until the next one would
// exceed the display width. Never split an emoji or combining sequence.
if max_width <= 3 {
let mut out = String::new();
let mut width = 0usize;
for ch in text.chars() {
let ch_width = char_display_width(ch);
if width + ch_width > max_width {
for grapheme in text.graphemes(true) {
let grapheme_width = grapheme_display_width(grapheme);
if width + grapheme_width > max_width {
break;
}
out.push(ch);
width += ch_width;
out.push_str(grapheme);
width += grapheme_width;
}
return out;
}
@@ -49,13 +51,13 @@ pub(crate) fn truncate_line_to_width(text: &str, max_width: usize) -> String {
let mut out = String::new();
let mut width = 0usize;
let limit = max_width.saturating_sub(3);
for ch in text.chars() {
let ch_width = char_display_width(ch);
if width + ch_width > limit {
for grapheme in text.graphemes(true) {
let grapheme_width = grapheme_display_width(grapheme);
if width + grapheme_width > limit {
break;
}
out.push(ch);
width += ch_width;
out.push_str(grapheme);
width += grapheme_width;
}
out.push_str("...");
out
@@ -81,14 +83,14 @@ pub(crate) fn semantic_truncate(text: &str, max_width: usize) -> String {
let mut cut_byte = 0usize;
let mut last_word_end = None;
let mut in_word = false;
for (byte_idx, ch) in text.char_indices() {
let ch_width = char_display_width(ch);
if width + ch_width > limit {
for (byte_idx, grapheme) in text.grapheme_indices(true) {
let grapheme_width = grapheme_display_width(grapheme);
if width + grapheme_width > limit {
break;
}
width += ch_width;
cut_byte = byte_idx + ch.len_utf8();
if ch.is_whitespace() {
width += grapheme_width;
cut_byte = byte_idx + grapheme.len();
if grapheme.chars().all(char::is_whitespace) {
if in_word {
last_word_end = Some(byte_idx);
in_word = false;
@@ -262,7 +264,7 @@ where
}
pub(crate) fn text_display_width(text: &str) -> usize {
text.chars().map(char_display_width).sum()
text.graphemes(true).map(grapheme_display_width).sum()
}
pub(super) fn slice_text(text: &str, start: usize, end: usize) -> String {
@@ -272,14 +274,14 @@ pub(super) fn slice_text(text: &str, start: usize, end: usize) -> String {
let mut out = String::new();
let mut col = 0usize;
for ch in text.chars() {
let ch_width = char_display_width(ch);
let ch_start = col;
let ch_end = col.saturating_add(ch_width);
if ch_end > start && ch_start < end {
out.push(ch);
for grapheme in text.graphemes(true) {
let grapheme_width = grapheme_display_width(grapheme);
let grapheme_start = col;
let grapheme_end = col.saturating_add(grapheme_width);
if grapheme_end > start && grapheme_start < end {
out.push_str(grapheme);
}
col = ch_end;
col = grapheme_end;
if col >= end {
break;
}
@@ -290,13 +292,6 @@ pub(super) fn slice_text(text: &str, start: usize, end: usize) -> String {
pub(super) fn char_display_width(ch: char) -> usize {
if ch == '\t' {
4
} else if ch == '\u{20E3}' {
// U+20E3 COMBINING ENCLOSING KEYCAP completes a keycap sequence
// (e.g. 1️⃣ = 1️⃣) that renders as 2 columns
// in terminals, but unicode-width reports it as 1 (base = 1,
// FE0F = 0, 20E3 = 0). Giving 20E3 a display width of 1 makes
// the total 2, matching the terminal.
1
} else {
// `width()` returns `None` for control/unassigned chars (default them to
// one column so layout doesn't collapse) and `Some(0)` for genuinely
@@ -307,6 +302,21 @@ pub(super) fn char_display_width(ch: char) -> usize {
}
}
/// Measure one extended grapheme using the same string-level Unicode rules as
/// Ratatui. String width intentionally differs from the sum of codepoint widths
/// for keycaps, ZWJ emoji, modifiers, and other terminal ligatures.
pub(super) fn grapheme_display_width(grapheme: &str) -> usize {
if grapheme == "\t" {
return 4;
}
if let Some(ch) = grapheme.chars().next()
&& ch.len_utf8() == grapheme.len()
{
return char_display_width(ch);
}
UnicodeWidthStr::width(grapheme)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -386,8 +396,8 @@ mod tests {
// combining marks or ZWJ emoji sequences.)
assert_eq!(text_display_width("e\u{0301}"), 1);
assert_eq!(text_display_width("cafe\u{0301}"), 4);
// ZWJ joiner itself is zero-width; the two emoji are 2 cols each.
assert_eq!(text_display_width("\u{1F469}\u{200D}\u{1F4BB}"), 4);
// The complete ZWJ emoji is one two-column grapheme, matching Ratatui.
assert_eq!(text_display_width("\u{1F469}\u{200D}\u{1F4BB}"), 2);
}
#[test]
@@ -546,34 +556,38 @@ mod tests {
}
}
// --- keycap / enclosing-keycap regression guard (#4479) -------------------
// Keycap sequences (digit + FE0F + U+20E3) render as 2 columns in Windows
// Terminal but were measured as 1 column by char_display_width. Proof that
// every code path listed below returns 2 after the fix.
// --- keycap / grapheme regression guard (#4479) ---------------------------
// Fully qualified keycap sequences render as two columns. Codepoint sums
// report one; the canonical string/grapheme contract reports two.
#[test]
fn text_display_width_treats_keycap_sequence_as_two_columns() {
assert_eq!(text_display_width("1\u{fe0f}\u{20e3}"), 2);
assert_eq!(text_display_width("9\u{fe0f}\u{20e3}"), 2);
// Without FE0F: digit + enclosing keycap still renders as a single
// keycap on many terminals (2 cols).
assert_eq!(text_display_width("1\u{20e3}"), 2);
assert_eq!(text_display_width("#\u{fe0f}\u{20e3}"), 2);
// Independent enclosing keycap codepoint: we give it 1 col to err
// on the safe side (extra space over cell collapse).
assert_eq!(text_display_width("\u{20e3}"), 1);
for keycap in [
"1\u{fe0f}\u{20e3}",
"9\u{fe0f}\u{20e3}",
"#\u{fe0f}\u{20e3}",
] {
assert_eq!(text_display_width(keycap), 2);
assert_eq!(text_display_width(keycap), UnicodeWidthStr::width(keycap));
}
// Preserve unicode-width's distinction between fully-qualified emoji
// presentation and text/standalone combining-mark forms.
assert_eq!(text_display_width("1\u{20e3}"), 1);
assert_eq!(text_display_width("\u{20e3}"), 0);
}
#[test]
fn slice_text_does_not_split_keycap_sequence() {
// Slicing a row that contains a keycap must keep the sequence intact.
let row = "step 1\u{fe0f}\u{20e3} done";
// Slice from column 0 to 6 display-width cols.
let sliced = slice_text(row, 0, 6);
assert!(sliced.contains("1\u{fe0f}\u{20e3}"),
"keycap was split: {sliced:?}");
assert_eq!(UnicodeWidthStr::width(sliced.as_str()), 6,
"slice width {sliced:?} != 6");
// The keycap occupies columns [5, 7). Any overlapping selection keeps
// the complete grapheme; no isolated FE0F/U+20E3 mark may escape.
for (start, end) in [(0, 7), (5, 6), (6, 7)] {
let sliced = slice_text(row, start, end);
assert!(
sliced.contains("1\u{fe0f}\u{20e3}"),
"range=({start}, {end}) split keycap: {sliced:?}"
);
}
}
#[test]
@@ -585,11 +599,15 @@ mod tests {
"step 2\u{fe0f}\u{20e3} and 3\u{fe0f}\u{20e3} continue",
];
for text in &cases {
for budget in 0..=text.chars().count() + 4 {
for budget in 0..=text_display_width(text) + 4 {
let out = truncate_line_to_width(text, budget);
let w = UnicodeWidthStr::width(out.as_str());
assert!(w <= budget,
"budget={budget} text={text:?} -> {out:?} (width={w})");
let width = text_display_width(&out);
assert!(
width <= budget,
"budget={budget} text={text:?} -> {out:?} (width={width})"
);
assert!(!out.ends_with('\u{fe0f}'));
assert!(!out.starts_with('\u{20e3}'));
}
}
}
crates/tui/src/tui/widgets/mod.rs
+27 -12
View File
@@ -40,7 +40,7 @@ use crate::tui::approval::{
};
use crate::tui::history::{GenericToolCell, HistoryCell, ToolCell, ToolRun, ToolStatus};
use crate::tui::scrolling::TranscriptLineMeta;
use crate::tui::ui_text::{char_display_width, text_display_width};
use crate::tui::ui_text::{grapheme_display_width, text_display_width};
use crate::tui::underwater::ShellPhase;
use ratatui::{
buffer::Buffer,
@@ -3087,20 +3087,20 @@ fn apply_selection_to_line(
let mut before = String::new();
let mut selected = String::new();
let mut after = String::new();
let mut ch_col = current_col;
let mut grapheme_col = current_col;
for ch in span_text.chars() {
let ch_width = char_display_width(ch);
let ch_start = ch_col;
let ch_end = ch_col.saturating_add(ch_width);
if ch_end <= col_start {
before.push(ch);
} else if ch_start >= col_end {
after.push(ch);
for grapheme in span_text.graphemes(true) {
let grapheme_width = grapheme_display_width(grapheme);
let grapheme_start = grapheme_col;
let grapheme_end = grapheme_col.saturating_add(grapheme_width);
if grapheme_end <= col_start {
before.push_str(grapheme);
} else if grapheme_start >= col_end {
after.push_str(grapheme);
} else {
selected.push(ch);
selected.push_str(grapheme);
}
ch_col = ch_end;
grapheme_col = grapheme_end;
}
if !before.is_empty() {
@@ -5200,6 +5200,21 @@ mod tests {
assert_eq!(styled[1].content.as_ref(), " world");
}
#[test]
fn selection_keeps_keycap_grapheme_intact() {
let line = Line::from(Span::raw("A1\u{fe0f}\u{20e3}B"));
let selection_style = Style::default().bg(palette::SELECTION_BG);
// Selecting the second display column of the two-column keycap must
// style the complete grapheme, never only FE0F/U+20E3.
let styled = apply_selection_to_line(&line, 2, 3, selection_style);
assert_eq!(styled.len(), 3);
assert_eq!(styled[0].content.as_ref(), "A");
assert_eq!(styled[1].content.as_ref(), "1\u{fe0f}\u{20e3}");
assert_eq!(styled[1].style.bg, Some(palette::SELECTION_BG));
assert_eq!(styled[2].content.as_ref(), "B");
}
#[test]
fn composer_layout_helpers_stay_consistent() {
let input = "line one wraps nicely\nline two wraps as well";