fix(streaming): wrap overlong no-whitespace text

Hard-break oversized streaming tokens so CJK runs, URLs, and other no-whitespace content stay within the target width. Includes regression coverage for long CJK text and first-token overflow.
This commit is contained in:
Reid
2026-05-14 20:11:38 +08:00
committed by GitHub
parent 96369a8d51
commit 8fd82be1ee
+58 -1
View File
@@ -10,7 +10,7 @@
use ratatui::style::{Modifier, Style};
use ratatui::text::{Line, Span};
use std::time::Instant;
use unicode_width::UnicodeWidthStr;
use unicode_width::{UnicodeWidthChar, UnicodeWidthStr};
use crate::palette;
@@ -184,6 +184,9 @@ fn wrap_line(line: &str, width: usize) -> Vec<String> {
if line.is_empty() {
return vec![String::new()];
}
if width == 0 {
return vec![line.to_string()];
}
let mut result = Vec::new();
let mut current_line = String::new();
@@ -192,6 +195,21 @@ fn wrap_line(line: &str, width: usize) -> Vec<String> {
for word in line.split_whitespace() {
let word_width = word.width();
if word_width > width {
if !current_line.is_empty() {
result.push(std::mem::take(&mut current_line));
current_width = 0;
}
push_word_breaking_chars(
word,
width,
&mut current_line,
&mut current_width,
&mut result,
);
continue;
}
if current_width == 0 {
// First word on line
current_line = word.to_string();
@@ -220,6 +238,24 @@ fn wrap_line(line: &str, width: usize) -> Vec<String> {
}
}
fn push_word_breaking_chars(
word: &str,
width: usize,
current_line: &mut String,
current_width: &mut usize,
result: &mut Vec<String>,
) {
for ch in word.chars() {
let ch_width = ch.width().unwrap_or(1);
if *current_width + ch_width > width && *current_width > 0 {
result.push(std::mem::take(current_line));
*current_width = 0;
}
current_line.push(ch);
*current_width += ch_width;
}
}
/// Per-block streaming substate: optional line-buffer feeding a collector +
/// chunker/policy for two-gear pacing.
///
@@ -521,6 +557,7 @@ impl StreamingState {
#[cfg(test)]
mod tests {
use super::*;
use unicode_width::UnicodeWidthStr;
#[test]
fn test_commit_complete_lines() {
@@ -552,6 +589,26 @@ mod tests {
assert!(result.len() > 1);
}
#[test]
fn wrap_line_breaks_no_whitespace_cjk_text() {
let text = "这是一个没有任何空格的中文长段落".repeat(12);
let result = wrap_line(&text, 40);
assert!(result.len() > 1);
assert!(result.iter().all(|line| line.width() <= 40));
assert_eq!(result.join(""), text);
}
#[test]
fn wrap_line_breaks_first_overlong_word() {
let text = "x".repeat(120);
let result = wrap_line(&text, 40);
assert_eq!(result.len(), 3);
assert!(result.iter().all(|line| line.width() <= 40));
assert_eq!(result.join(""), text);
}
#[test]
fn assistant_text_streams_before_newline() {
let mut state = StreamingState::new();