fix(tui): add stale busy-state watchdog (#1170)
This commit is contained in:
@@ -911,6 +911,8 @@ pub struct App {
|
||||
pub runtime_turn_id: Option<String>,
|
||||
/// Current runtime turn status (if known).
|
||||
pub runtime_turn_status: Option<String>,
|
||||
/// When the UI accepted a user message but has not observed `TurnStarted` yet.
|
||||
pub dispatch_started_at: Option<Instant>,
|
||||
|
||||
/// Cached git context snapshot for the footer.
|
||||
pub workspace_context: Option<String>,
|
||||
@@ -1408,6 +1410,7 @@ impl App {
|
||||
cumulative_turn_duration: std::time::Duration::ZERO,
|
||||
runtime_turn_id: None,
|
||||
runtime_turn_status: None,
|
||||
dispatch_started_at: None,
|
||||
workspace_context: None,
|
||||
workspace_context_cell: std::sync::Arc::new(std::sync::Mutex::new(None)),
|
||||
workspace_context_refreshed_at: None,
|
||||
|
||||
@@ -125,6 +125,7 @@ const CONTEXT_CRITICAL_THRESHOLD_PERCENT: f64 = 95.0;
|
||||
const UI_IDLE_POLL_MS: u64 = 48;
|
||||
const UI_ACTIVE_POLL_MS: u64 = 24;
|
||||
const WEB_CONFIG_POLL_MS: u64 = 16;
|
||||
const DISPATCH_WATCHDOG_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
// Forced repaint cadence while a turn is live (model loading, compacting,
|
||||
// sub-agents running). Drives the footer water-spout animation as well as
|
||||
// the per-tool spinner pulse — keep this fast enough that the spout reads as
|
||||
@@ -848,6 +849,7 @@ async fn run_event_loop(
|
||||
EngineEvent::TurnStarted { turn_id } => {
|
||||
app.is_loading = true;
|
||||
app.offline_mode = false;
|
||||
app.dispatch_started_at = None;
|
||||
current_streaming_text.clear();
|
||||
app.streaming_state.reset();
|
||||
app.streaming_message_index = None;
|
||||
@@ -887,6 +889,7 @@ async fn run_event_loop(
|
||||
app.flush_active_cell();
|
||||
}
|
||||
app.is_loading = false;
|
||||
app.dispatch_started_at = None;
|
||||
app.offline_mode = false;
|
||||
app.streaming_state.reset();
|
||||
// Capture elapsed before clearing turn_started_at so
|
||||
@@ -1443,6 +1446,9 @@ async fn run_event_loop(
|
||||
}
|
||||
|
||||
let has_running_agents = running_agent_count(app) > 0;
|
||||
if reconcile_turn_liveness(app, Instant::now(), has_running_agents) {
|
||||
app.needs_redraw = true;
|
||||
}
|
||||
if (app.is_loading || has_running_agents || app.is_compacting)
|
||||
&& last_status_frame.elapsed()
|
||||
>= Duration::from_millis(status_animation_interval_ms(app))
|
||||
@@ -2239,6 +2245,7 @@ async fn run_event_loop(
|
||||
if app.is_loading {
|
||||
engine_handle.cancel();
|
||||
app.is_loading = false;
|
||||
app.dispatch_started_at = None;
|
||||
app.streaming_state.reset();
|
||||
// Optimistically clear the turn-in-progress flag so
|
||||
// the footer wave animation halts immediately —
|
||||
@@ -2291,6 +2298,7 @@ async fn run_event_loop(
|
||||
app.backtrack.reset();
|
||||
engine_handle.cancel();
|
||||
app.is_loading = false;
|
||||
app.dispatch_started_at = None;
|
||||
app.streaming_state.reset();
|
||||
// Optimistically halt the wave + working label —
|
||||
// engine's TurnComplete will resync with the real
|
||||
@@ -3023,6 +3031,46 @@ fn queued_session_to_ui(msg: QueuedSessionMessage) -> QueuedMessage {
|
||||
}
|
||||
}
|
||||
|
||||
fn reconcile_turn_liveness(app: &mut App, now: Instant, has_running_agents: bool) -> bool {
|
||||
if app.is_loading
|
||||
&& app.runtime_turn_status.is_none()
|
||||
&& !has_running_agents
|
||||
&& !app.is_compacting
|
||||
&& app.dispatch_started_at.is_some_and(|started| {
|
||||
now.saturating_duration_since(started) > DISPATCH_WATCHDOG_TIMEOUT
|
||||
})
|
||||
{
|
||||
app.is_loading = false;
|
||||
app.dispatch_started_at = None;
|
||||
app.push_status_toast(
|
||||
"Turn dispatch timed out; the engine may have stopped. Please try again.",
|
||||
StatusToastLevel::Error,
|
||||
None,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if app.is_loading
|
||||
&& matches!(
|
||||
app.runtime_turn_status.as_deref(),
|
||||
Some("completed" | "interrupted" | "failed")
|
||||
)
|
||||
&& !has_running_agents
|
||||
&& !app.is_compacting
|
||||
{
|
||||
app.is_loading = false;
|
||||
app.dispatch_started_at = None;
|
||||
app.push_status_toast(
|
||||
"Recovered from an inconsistent busy state.",
|
||||
StatusToastLevel::Warning,
|
||||
None,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// Translate an `EngineEvent::Error` into UI state updates.
|
||||
///
|
||||
/// The engine's `recoverable` flag (mirrored on `ErrorEnvelope`) decides
|
||||
@@ -3064,6 +3112,7 @@ pub(crate) fn apply_engine_error_to_app(
|
||||
severity,
|
||||
});
|
||||
app.is_loading = false;
|
||||
app.dispatch_started_at = None;
|
||||
if matches!(
|
||||
envelope.category,
|
||||
crate::error_taxonomy::ErrorCategory::Authentication
|
||||
@@ -3663,8 +3712,11 @@ async fn dispatch_user_message(
|
||||
}
|
||||
|
||||
// Set immediately to prevent double-dispatch before TurnStarted event arrives.
|
||||
let dispatch_started_at = Instant::now();
|
||||
app.is_loading = true;
|
||||
app.last_send_at = Some(Instant::now());
|
||||
app.dispatch_started_at = Some(dispatch_started_at);
|
||||
app.runtime_turn_status = None;
|
||||
app.last_send_at = Some(dispatch_started_at);
|
||||
|
||||
let cwd = std::env::current_dir().ok();
|
||||
let references = crate::tui::file_mention::context_references_from_input(
|
||||
@@ -3781,6 +3833,7 @@ async fn dispatch_user_message(
|
||||
.await
|
||||
{
|
||||
app.is_loading = false;
|
||||
app.dispatch_started_at = None;
|
||||
app.last_send_at = None;
|
||||
return Err(err);
|
||||
}
|
||||
@@ -5784,6 +5837,7 @@ async fn handle_view_events(
|
||||
app.backtrack.reset();
|
||||
engine_handle.cancel();
|
||||
app.is_loading = false;
|
||||
app.dispatch_started_at = None;
|
||||
app.streaming_state.reset();
|
||||
app.runtime_turn_status = None;
|
||||
app.finalize_active_cell_as_interrupted();
|
||||
|
||||
@@ -1074,6 +1074,61 @@ async fn dispatch_user_message_failed_send_clears_loading_state() {
|
||||
"failed dispatch must not leave the composer in a permanent busy state"
|
||||
);
|
||||
assert!(app.last_send_at.is_none());
|
||||
assert!(app.dispatch_started_at.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn turn_liveness_watchdog_clears_stale_dispatch() {
|
||||
let mut app = create_test_app();
|
||||
app.is_loading = true;
|
||||
app.dispatch_started_at =
|
||||
Some(Instant::now() - DISPATCH_WATCHDOG_TIMEOUT - Duration::from_millis(1));
|
||||
|
||||
let recovered = reconcile_turn_liveness(&mut app, Instant::now(), false);
|
||||
|
||||
assert!(recovered);
|
||||
assert!(!app.is_loading);
|
||||
assert!(app.dispatch_started_at.is_none());
|
||||
let toast = app.status_toasts.back().expect("watchdog toast");
|
||||
assert_eq!(toast.level, StatusToastLevel::Error);
|
||||
assert!(toast.text.contains("Turn dispatch timed out"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn turn_liveness_reconciles_completed_busy_state() {
|
||||
let mut app = create_test_app();
|
||||
app.is_loading = true;
|
||||
app.runtime_turn_status = Some("completed".to_string());
|
||||
app.dispatch_started_at = Some(Instant::now());
|
||||
|
||||
let recovered = reconcile_turn_liveness(&mut app, Instant::now(), false);
|
||||
|
||||
assert!(recovered);
|
||||
assert!(!app.is_loading);
|
||||
assert!(app.dispatch_started_at.is_none());
|
||||
let toast = app.status_toasts.back().expect("reconciliation toast");
|
||||
assert_eq!(toast.level, StatusToastLevel::Warning);
|
||||
assert!(
|
||||
toast
|
||||
.text
|
||||
.contains("Recovered from an inconsistent busy state")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn turn_liveness_leaves_active_turn_running() {
|
||||
let mut app = create_test_app();
|
||||
app.is_loading = true;
|
||||
app.runtime_turn_status = Some("in_progress".to_string());
|
||||
app.dispatch_started_at =
|
||||
Some(Instant::now() - DISPATCH_WATCHDOG_TIMEOUT - Duration::from_secs(10));
|
||||
|
||||
let recovered = reconcile_turn_liveness(&mut app, Instant::now(), false);
|
||||
|
||||
assert!(!recovered);
|
||||
assert!(app.is_loading);
|
||||
assert!(app.dispatch_started_at.is_some());
|
||||
assert!(app.status_toasts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user