guyoron1
f496f59b77
Addresses the inline review on #2717. decode_process_output - Decode a line at a time instead of reinterpreting the whole buffer at the first bad byte. Valid UTF-8 lines keep their bytes; only lines that fail UTF-8 validation go through the code page, so one stray byte no longer mangles output that was almost entirely UTF-8. The line is the unit because a byte run is not one: GB18030's four-byte sequences embed bytes in the ASCII digit range, so any rule that ends a run below 0x80 splits them. \n cannot appear as a trail byte in any encoding handled here, and a process does not switch encoding mid-line. - A code page result is only accepted when it decodes cleanly, so a UTF-8 line with a corrupt byte falls back to lossy UTF-8 rather than mojibake. - Replace the hand-written code page table with the codepage crate, as suggested. That also fixes 54936, which was mapped to GBK and now correctly resolves to gb18030. - Add oem_cp for the legacy OEM/DOS pages (437, 850, 852, …) that plain cmd.exe still defaults to in many locales. encoding_rs implements only WHATWG encodings, so codepage alone returns None for them. - Fall back to GetACP when GetConsoleOutputCP reports no console, which is the piped case rtk normally runs in, and warn once instead of falling back to lossy silently. - Cache the code page lookup in a OnceLock. - The mapping and the walk take the code page as a parameter, so they are compiled and unit-tested on every platform rather than only Windows. Call sites - Route the remaining production sites through stream::exec_capture and exec_capture_stdin rather than decoding at each one, so future callers inherit decoding. git commit keeps inherited stdin via the _stdin variant. Test-only sites go back to from_utf8_lossy: they assert on rtk's own UTF-8 output, where a console code page has no meaning. - Decode the streamed path (read_lines_lossy) too — the OEM/ANSI lines its comment describes were still going straight to U+FFFD. - curl keeps its body on from_utf8_lossy: a response body is a network payload whose encoding comes from the HTTP charset, not the local console, and non-UTF-8 bodies already take the binary passthrough for #1087. Only curl's own stderr is code page decoded. git commit summary parsing - parse_commit_output sliced from byte 1, which panics when the first line starts with a multi-byte character — git prints hook output before its summary, and a lossily decoded line starts with a multi-byte U+FFFD. Locate the bracket pair with find instead, so both indices are character boundaries. Verified: unit tests for the walk, GBK, gb18030, CP437/850, mixed lines, truncated input and every byte value; a test pinning that output without a code page stays byte-identical to from_utf8_lossy; and the Windows-only lookup cross-compiled for x86_64-pc-windows-msvc.
86 lines
2.2 KiB
TOML
86 lines
2.2 KiB
TOML
[package]
|
|
name = "rtk"
|
|
version = "0.42.4"
|
|
edition = "2021"
|
|
rust-version = "1.91"
|
|
authors = ["Patrick Szymkowiak"]
|
|
description = "Rust Token Killer - High-performance CLI proxy to minimize LLM token consumption"
|
|
license = "Apache 2.0"
|
|
homepage = "https://www.rtk-ai.app"
|
|
repository = "https://github.com/rtk-ai/rtk"
|
|
readme = "README.md"
|
|
keywords = ["cli", "llm", "token", "filter", "productivity"]
|
|
categories = ["command-line-utilities", "development-tools"]
|
|
|
|
[dependencies]
|
|
clap = { version = "4", features = ["derive"] }
|
|
anyhow = "1.0"
|
|
# Console code page decoding for child-process output. Kept off the
|
|
# cfg(windows) target so the mapping and the incremental UTF-8 walk stay
|
|
# compiled — and unit-tested — on every platform; only the code page *lookup*
|
|
# is Windows-specific. codepage covers the ANSI/DBCS pages encoding_rs
|
|
# implements, oem_cp the legacy OEM/DOS pages (437, 850, …) it does not.
|
|
encoding_rs = "0.8"
|
|
codepage = "0.1"
|
|
oem_cp = "2"
|
|
ignore = "0.4"
|
|
walkdir = "2"
|
|
regex = "1"
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = { version = "1", features = ["preserve_order"] }
|
|
colored = "3"
|
|
dirs = "5"
|
|
rusqlite = { version = "0.31", features = ["bundled"] }
|
|
toml = "0.8"
|
|
chrono = "0.4"
|
|
tempfile = "3"
|
|
sha2 = "0.10"
|
|
ureq = "2"
|
|
getrandom = "0.4"
|
|
flate2 = "1.0"
|
|
quick-xml = "0.37"
|
|
which = "8"
|
|
automod = "1"
|
|
|
|
[target.'cfg(unix)'.dependencies]
|
|
libc = "0.2"
|
|
|
|
[target.'cfg(windows)'.dependencies]
|
|
windows-sys = { version = "0.59", features = [
|
|
"Win32_System_Console",
|
|
"Win32_Globalization",
|
|
] }
|
|
|
|
[build-dependencies]
|
|
toml = "0.8"
|
|
|
|
[dev-dependencies]
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
lto = true
|
|
codegen-units = 1
|
|
panic = "abort"
|
|
strip = true
|
|
|
|
# cargo-deb configuration
|
|
[package.metadata.deb]
|
|
maintainer = "Patrick Szymkowiak"
|
|
copyright = "2024 Patrick Szymkowiak"
|
|
license-file = ["LICENSE", "0"]
|
|
extended-description = "rtk filters and compresses command outputs before they reach your LLM context, saving 60-90% of tokens."
|
|
section = "utility"
|
|
priority = "optional"
|
|
assets = [
|
|
["target/release/rtk", "usr/bin/", "755"],
|
|
]
|
|
# cargo-generate-rpm configuration
|
|
[package.metadata.generate-rpm]
|
|
assets = [
|
|
{ source = "target/release/rtk", dest = "/usr/bin/rtk", mode = "755" },
|
|
]
|
|
|
|
[lints.rust]
|
|
unsafe_code = "deny"
|
|
warnings = "deny"
|