""" Hermes Web UI -- SSE streaming engine and agent thread runner. Includes Sprint 10 cancel support via CANCEL_FLAGS. """ import base64 import contextlib import contextvars import json import logging import math import mimetypes import os import queue import random import re import sqlite3 import shlex import sys import subprocess import threading import time import traceback import copy import inspect from pathlib import Path from typing import Optional logger = logging.getLogger(__name__) from api.config import ( get_config, STREAMS, STREAMS_LOCK, CANCEL_FLAGS, AGENT_INSTANCES, STREAM_PARTIAL_TEXT, STREAM_REASONING_TEXT, STREAM_LIVE_TOOL_CALLS, STREAM_GOAL_RELATED, PENDING_GOAL_CONTINUATION, STREAM_LAST_EVENT_ID, LOCK, SESSIONS, SESSIONS_MAX, SESSION_DIR, _get_session_agent_lock, _alias_session_agent_lock, _set_thread_env, _clear_thread_env, register_active_run, update_active_run, unregister_active_run, unregister_stream_owner, peek_stream, stream_owner_session_id, session_writeback_owner, clear_session_writeback_owner_if_owned, SESSION_AGENT_LOCKS, SESSION_AGENT_LOCKS_LOCK, resolve_model_provider, resolve_custom_provider_connection, apply_custom_provider_connection_authority, merge_custom_provider_runtime_bundle, CustomProviderRouteError, CUSTOM_ROUTE_NO_CREDENTIAL, CUSTOM_ROUTE_NO_ENDPOINT, custom_provider_route_error, raise_for_custom_provider_route, model_with_provider_context, warm_models_catalog_provenance_if_cold, load_settings, parse_reasoning_effort, coerce_reasoning_effort_for_model, _main_model_request_overrides, PROCESS_SESSION_INDEX, PROCESS_SESSION_INDEX_LOCK, ) from api.helpers import ( redact_session_data, scrub_internal_replay_fields, _redact_text, ) from api.compression_anchor import is_context_compression_marker, visible_messages_for_anchor from api.compression_recovery import stamp_compression_exhausted_recovery from api.gateway_chat import WEBUI_LOCAL_CHAT_BACKEND from api.metering import meter from api.run_journal import RunJournalWriter from api.todo_state import attach_todo_state, emit_todo_state from api.turn_journal import append_turn_journal_event_for_stream from api.usage import prompt_cache_hit_percent from api.models import ( StateDBSessionMessagesSnapshot, _WEBUI_TRUSTED_AGENT_INPUT_FIELD, _is_empty_partial_activity_message, _message_exact_timestamp_details, _message_private_identity_compatible, _validated_webui_pending_user_timestamp_identity, _evict_sessions_over_cap, clear_process_wakeup_pause, get_state_db_session_messages, record_process_wakeup_provider_unavailable_pause, reconciled_state_db_messages_for_session, ) from api.session_ops import mark_session_title_generated, session_has_manual_title from api.process_event_utils import ( build_active_turn_token, claim_async_delegation_delivery, complete_async_delegation_delivery, completion_delivery_id, release_async_delegation_delivery, requeue_async_delegation_event, schedule_async_delegation_claim_retry, stamp_message_source, ) def get_stream_runtime_snapshot() -> dict[str, object]: """Return aggregate stream observations without waiting on the registry. The registry is copied under a nonblocking ``STREAMS_LOCK`` acquire and the lock is released before any channel lock is touched, so registry and channel locks are never nested. Each channel is then read through its nonblocking owner method: a channel busy with its own producer or subscriber work counts as one unavailable channel and the loop keeps summing its siblings. """ result = { "available": False, "active": 0, "agent_instances": 0, "subscribers": 0, "offline_buffered_events": 0, "offline_dropped_events": 0, "subscriber_dropped_events": 0, "unavailable_channels": 0, } try: if not STREAMS_LOCK.acquire(blocking=False): return result try: channels = list(STREAMS.values()) agent_count = len(AGENT_INSTANCES) finally: STREAMS_LOCK.release() result["available"] = True result["active"] = len(channels) result["agent_instances"] = agent_count for channel in channels: try: snapshot = channel.try_diagnostic_snapshot() if snapshot is None: result["unavailable_channels"] += 1 continue result["subscribers"] += max(0, int(snapshot.get("subscriber_count", 0))) result["offline_buffered_events"] += max( 0, int(snapshot.get("offline_buffered_events", 0)) ) result["offline_dropped_events"] += max( 0, int(snapshot.get("offline_dropped_events", 0)) ) result["subscriber_dropped_events"] += max( 0, int(snapshot.get("subscriber_dropped_events", 0)) ) except Exception: result["unavailable_channels"] += 1 except Exception: # Keep the aggregates already summed from channels that read cleanly; a # late unexpected failure must not discard successful sibling counts. return result return result def _session_payload_with_full_messages(session, *, tool_calls=None): """Return compact session metadata plus the embedded full transcript. ``Session.compact()`` may intentionally use metadata-only counts from an index/sidebar load. A settled SSE payload that embeds ``session.messages`` must report the count of that embedded transcript, otherwise completion and reconcile paths can mistake a complete payload for a stale short window. """ messages = list(getattr(session, 'messages', None) or []) raw = session.compact() | { 'messages': messages, 'message_count': len(messages), } attach_todo_state(raw, messages) if tool_calls is not None: raw['tool_calls'] = tool_calls try: from api.session_ops import ( regeneration_authority, regeneration_state, ) canonical_messages, canonical_context = regeneration_state(session) revision = regeneration_authority( session, rows=canonical_messages, context=canonical_context, full_transcript=True, canonical_state=(canonical_messages, canonical_context), ) if revision: messages = list(canonical_messages) raw['messages'] = messages raw['message_count'] = len(messages) attach_todo_state(raw, messages) raw['regeneration_revision'] = revision else: raw.pop('regeneration_revision', None) except Exception: raw.pop('regeneration_revision', None) return raw def _compact_for_echo_compare(value: str) -> str: """Normalize visible stream text for duplicate echo detection.""" return re.sub(r'\s+', '', str(value or '')) class _CompactEchoIndex: """Incremental whitespace-folded index over a growing text buffer. The journal-rebuild echo check runs once per interim event against the whole accumulated reasoning transcript. Rescanning the raw buffer each time walks every whitespace character between the tail and the first non-whitespace character — on a whitespace-heavy transcript that span is megabytes, and the per-interim walk turns the replay quadratic (the ``_find_compact_echo_suffix_start`` shape measured ~55x slower than master on a production-shaped journal in the #7569 review). This class keeps the folded view and the raw cut offsets *incrementally*: * ``append`` folds each new chunk once and records, per folded character, its raw index — so the cost of a chunk is proportional to the chunk, never to the buffer. * ``matches_tail`` compares only the candidate against the end of the folded view: O(len(candidate)), with no raw-text walk at all. * ``cut_to`` converts a folded length back to a raw index by bisecting the recorded offsets, so one call returns the same cut point the raw backward walk would have produced. ``str.isspace`` is used for folding instead of the ``\\s`` pattern used by :func:`_compact_for_echo_compare`. The two agree on every Unicode code point, so the indexed view and the regex-folded view stay consistent. """ __slots__ = ('_compact', '_offsets', '_raw_len') def __init__(self) -> None: self._compact: list[str] = [] self._offsets: list[int] = [] self._raw_len = 0 def append(self, text: str) -> None: """Fold ``text`` onto the end of the index (one pass, no rescan).""" raw = str(text or '') if not raw: return compact = self._compact offsets = self._offsets base = self._raw_len for i, ch in enumerate(raw): if not ch.isspace(): compact.append(ch) offsets.append(base + i) self._raw_len = base + len(raw) def reset(self) -> None: """Drop all indexed state (used after the raw text is truncated).""" self._compact.clear() self._offsets.clear() self._raw_len = 0 @property def compact_length(self) -> int: return len(self._compact) def matches_tail(self, suffix: str) -> bool: """True when the folded ``suffix`` equals the folded tail.""" candidate = _compact_for_echo_compare(suffix) if not candidate: return False n = len(candidate) if n > len(self._compact): return False start = len(self._compact) - n compact = self._compact for i in range(n): if compact[start + i] != candidate[i]: return False return True def cut_to(self, suffix: str) -> int | None: """Raw index at which the echo of ``suffix`` starts, else ``None``. The returned index is the first raw character of the echo, matching the leftmost-cut semantics of the raw backward walk. Trailing whitespace before the echo is left to the caller's ``rstrip``. """ candidate = _compact_for_echo_compare(suffix) if not candidate: return None n = len(candidate) if n > len(self._compact): return None start = len(self._compact) - n compact = self._compact for i in range(n): if compact[start + i] != candidate[i]: return None return self._offsets[start] def compact_view(self) -> str: """The folded text (materialized on demand — never kept as a string).""" return ''.join(self._compact) def _find_compact_echo_suffix_start(value: str, suffix: str) -> int | None: """Return the index where a whitespace-folded ``suffix`` starts at the end of ``value``, or ``None`` when the tail does not echo it. The match walks ``value`` and ``suffix`` from the end, skipping whitespace in ``value``; only the echo span itself is inspected, so the cost is linear in the echo length and allocation-free. Unlike a fixed fold window, the walk cannot miss a compact-equivalent suffix whose raw span is stretched by interior whitespace. ``str.isspace`` is used for the walk instead of the ``\\s`` pattern used by :func:`_compact_for_echo_compare`. The two agree on every Unicode code point, so the folded view and the walk stay consistent. """ candidate = _compact_for_echo_compare(suffix) if not candidate: return None i = len(value) - 1 j = len(candidate) - 1 while j >= 0: while i >= 0 and value[i].isspace(): i -= 1 if i < 0 or value[i] != candidate[j]: return None i -= 1 j -= 1 return i + 1 def _strip_compact_echo_suffix(value: str, suffix: str) -> tuple[str, bool]: """Remove ``suffix`` from ``value`` when they match after whitespace folding. The cut point is located by the same backward walk as :func:`_find_compact_echo_suffix_start`: no fixed search window is involved, so a compact-equivalent suffix is removed no matter how much interior whitespace stretches its raw span. The previous windowed implementation folded a bounded tail on every call; its retired probing variant re-folded the remaining tail per candidate cut index, which is quadratic: a 6000-character final message cost seconds of CPU, held under the GIL, stalling every other stream in the process. Whitespace sitting between the kept text and the echo is removed by ``rstrip``, which lands on the same result as the leftmost cut index the probing loop used to return. """ raw = str(value or '') if not raw: return raw, False start = _find_compact_echo_suffix_start(raw, suffix) if start is None: return raw, False return raw[:start].rstrip(), True def _redacted_session_payload_with_full_messages(session, *, tool_calls=None) -> dict | None: """Best-effort terminal SSE session payload for already-persisted state.""" try: return redact_session_data( _session_payload_with_full_messages(session, tool_calls=tool_calls) ) except Exception: logger.debug("Failed to build redacted session payload", exc_info=True) return None def _ephemeral_session_payload(session_id: str, messages) -> dict: """Project the non-persistent ``/btw`` terminal session for public SSE.""" return redact_session_data( {'session_id': session_id, 'messages': messages if isinstance(messages, list) else []} ) def _cancel_event_payload( message: str = "Cancelled by user", *, session: dict | None = None, ) -> dict: """Return base cancel terminal event metadata.""" payload = { 'message': message, 'type': 'cancelled', 'status': 'cancelled', } if session: payload['session'] = session payload['session_id'] = session.get('session_id') return payload # Global lock for os.environ writes. Per-session locks (_agent_lock) prevent # concurrent runs of the SAME session, but two DIFFERENT sessions can still # interleave their os.environ writes. This global lock serializes the env # save/restore — held only briefly across the env-mutation critical section, # NOT for the entire agent run. The agent runs outside the lock; the finally # block re-acquires to atomically restore env vars. See narrow-lock pattern # in _run_agent_streaming (line ~2719) and profile_env_for_background_worker # (api/profiles.py:715). _ENV_LOCK = threading.Lock() _STREAM_WRITEBACK_DIAG_DEFAULT_THRESHOLD_MS = 250.0 _STREAMING_CRON_PROFILE_HOME: contextvars.ContextVar[str | None] = contextvars.ContextVar( "webui_streaming_cron_profile_home", default=None, ) _STREAMING_CRONJOB_WRAPPER_INSTALLED = False def _stream_writeback_diag_threshold_seconds(environ=None): if environ is None: environ = os.environ raw = str( environ.get( "HERMES_WEBUI_STREAM_WRITEBACK_DIAG_MS", _STREAM_WRITEBACK_DIAG_DEFAULT_THRESHOLD_MS, ) ).strip() try: threshold_ms = float(raw) except (TypeError, ValueError): threshold_ms = _STREAM_WRITEBACK_DIAG_DEFAULT_THRESHOLD_MS if threshold_ms < 0: return None return threshold_ms / 1000.0 @contextlib.contextmanager def _stream_writeback_stage(timings, name, *, clock=time.perf_counter): started = clock() try: yield finally: try: timings.append((str(name), max(0.0, float(clock() - started)))) except Exception: pass def _log_stream_writeback_timings( session_id, stream_id, timings, started, *, clock=time.perf_counter, log=logger, environ=None, ): threshold = _stream_writeback_diag_threshold_seconds(environ=environ) if threshold is None: return False try: total_seconds = max(0.0, float(clock() - started)) except Exception: return False if total_seconds < threshold: return False parts = [] for name, elapsed in timings or []: try: parts.append(f"{name}={float(elapsed) * 1000.0:.1f}ms") except Exception: continue log.debug( "stream final writeback timing session=%s stream=%s total=%.1fms stages=%s", session_id, stream_id, total_seconds * 1000.0, " ".join(parts), ) return True def _install_streaming_cronjob_profile_wrapper() -> None: """Wrap the agent cronjob tool so calls run under the streaming profile. The in-chat agent run already binds per-turn contextvars for other session-scoped state. Cron jobs are special because ``cron.jobs`` snapshots path constants at import time, so the model-facing ``cronjob`` tool must enter the existing WebUI cron profile context at the tool-call boundary. That context uses the cron-specific lock and restores the module caches as soon as the single cron tool call returns, avoiding long-lived global path mutation for the whole agent turn. """ global _STREAMING_CRONJOB_WRAPPER_INSTALLED if _STREAMING_CRONJOB_WRAPPER_INSTALLED: return try: from tools.registry import registry except Exception: logger.debug("streaming cronjob wrapper: tools registry unavailable", exc_info=True) return entry = registry.get_entry("cronjob") if entry is None: try: import tools.cronjob_tools # noqa: F401 except Exception: logger.debug("streaming cronjob wrapper: cronjob tool import failed", exc_info=True) entry = registry.get_entry("cronjob") if entry is None: logger.debug("streaming cronjob wrapper: cronjob tool not registered") return original_handler = entry.handler if getattr(original_handler, "_webui_streaming_profile_wrapper", False): _STREAMING_CRONJOB_WRAPPER_INSTALLED = True return # This relies on the agent tool executor's ``propagate_context_to_thread`` # using an unfiltered ``contextvars.copy_context()`` so this WebUI-owned # contextvar reaches the sync cronjob handler even when the tool call runs # on the agent's ThreadPoolExecutor worker. def _profile_scoped_cronjob_handler(args, **kwargs): profile_home = _STREAMING_CRON_PROFILE_HOME.get() if not profile_home: return original_handler(args, **kwargs) from api.profiles import cron_profile_context_for_home with cron_profile_context_for_home(Path(profile_home)): return original_handler(args, **kwargs) _profile_scoped_cronjob_handler.__dict__["_webui_streaming_profile_wrapper"] = True _profile_scoped_cronjob_handler.__dict__["_webui_original_handler"] = original_handler registry.register( name=entry.name, toolset=entry.toolset, schema=entry.schema, handler=_profile_scoped_cronjob_handler, check_fn=entry.check_fn, requires_env=entry.requires_env, is_async=entry.is_async, description=entry.description, emoji=entry.emoji, max_result_size_chars=entry.max_result_size_chars, dynamic_schema_overrides=entry.dynamic_schema_overrides, ) _STREAMING_CRONJOB_WRAPPER_INSTALLED = True def _supports_kwarg(func, kwarg_name: str) -> bool: """Return True if callable `func` accepts `kwarg_name` (explicitly or via **kwargs).""" try: sig = inspect.signature(func) for param in sig.parameters.values(): if param.kind == param.VAR_KEYWORD or param.name == kwarg_name: return True return False except Exception: return True _PERSISTENT_MEMORY_FILES = ( ("memory", ("memories", "MEMORY.md")), ("user", ("memories", "USER.md")), ("soul", ("SOUL.md",)), ) def _file_signature(path: Path) -> tuple[int, int] | None: try: st = path.stat() return (int(st.st_mtime_ns), int(st.st_size)) except OSError: return None def _persistent_state_snapshot(profile_home: str | None) -> dict: """Capture lightweight memory/skill file signatures for save toasts.""" if not profile_home: return {"memory": {}, "skills": {}} root = Path(profile_home) memory = {} for key, parts in _PERSISTENT_MEMORY_FILES: sig = _file_signature(root.joinpath(*parts)) if sig is not None: memory[key] = sig skills = {} skills_dir = root / "skills" try: for skill_md in skills_dir.rglob("SKILL.md"): try: rel = str(skill_md.relative_to(skills_dir)).replace("\\", "/") except ValueError: rel = str(skill_md) sig = _file_signature(skill_md) if sig is not None: skills[rel] = sig except OSError: pass return {"memory": memory, "skills": skills} def _persistent_state_changes(before: dict | None, after: dict | None) -> dict: before = before or {"memory": {}, "skills": {}} after = after or {"memory": {}, "skills": {}} memory_before = before.get("memory") or {} memory_after = after.get("memory") or {} skills_before = before.get("skills") or {} skills_after = after.get("skills") or {} memory_changed = any(memory_before.get(key) != sig for key, sig in memory_after.items()) skills = [] for rel, sig in skills_after.items(): old_sig = skills_before.get(rel) if old_sig == sig: continue name = Path(rel).parent.name or Path(rel).stem skills.append({ "name": name, "path": rel, "action": "created" if old_sig is None else "updated", }) return {"memory_saved": memory_changed, "skills": skills[:10]} def _apply_profile_provider_context_to_streaming_model( model: str | None, provider_context: str | None, profile_provider: str | None, profile_default_model: str | None, ) -> tuple[str | None, str | None, bool]: """Attach profile provider context and repair stale cross-provider models.""" if provider_context or not profile_provider: return model, provider_context, False provider_context = profile_provider.lower() if not profile_default_model: return model, provider_context, False from api.routes import _normalize_provider_id profile_provider_normalized = _normalize_provider_id(profile_provider) model_lower = (model or "").lower() # Only run the bare-prefix family match on un-namespaced model ids. A custom # namespace like "gemini_cli/..." or "claude-relay/..." merely *starts with* a # first-party token; matching it here would clobber the model to the profile # default on the send path (the #4278 collision — the slash-qualified branch # below routes through the fixed _normalize_provider_id instead). if "/" not in model_lower: for prefix in ("gpt", "claude", "gemini"): if model_lower.startswith(prefix): if _normalize_provider_id(prefix) != profile_provider_normalized: return profile_default_model, provider_context, True return model, provider_context, False if "/" in model_lower: slash_prefix = model_lower.split("/", 1)[0] if provider_context == "openai-codex" and slash_prefix == "openai": return profile_default_model, provider_context, True slash_provider = _normalize_provider_id(slash_prefix) if ( slash_provider and slash_provider != profile_provider_normalized and profile_provider_normalized not in {"openrouter", "custom", ""} ): return profile_default_model, provider_context, True return model, provider_context, False def _apply_profile_home_context_to_streaming_model( model: str | None, provider_context: str | None, profile_home: str | None, has_profile: bool, ) -> tuple[str | None, str | None, bool]: """Apply profile provider/model context from a profile config if present.""" if not (profile_home and has_profile and not provider_context): return model, provider_context, False try: import yaml as _yaml_pp _pp_cfg_path = Path(profile_home) / "config.yaml" if not _pp_cfg_path.is_file(): return model, provider_context, False _pp_cfg = _yaml_pp.safe_load(_pp_cfg_path.read_text(encoding="utf-8")) or {} if not isinstance(_pp_cfg, dict): return model, provider_context, False _pp = (_pp_cfg.get("model", {}).get("provider") or "").strip() if not _pp: return model, provider_context, False _pp_default = (_pp_cfg.get("model", {}).get("default") or "").strip() return _apply_profile_provider_context_to_streaming_model( model, provider_context, _pp, _pp_default, ) except Exception: logger.warning("profile provider read failed", exc_info=True) return model, provider_context, False def _resolve_custom_provider_connection_authority( resolved_provider: str | None, resolved_api_key: str | None, resolved_base_url: str | None, profile_name: str | None = None, custom_provider_lookup: str | None = None, ) -> tuple[str | None, str | None, str | None, bool]: """Return ``(provider, api_key, base_url, custom_owned)`` for ``custom:*``. Thin profile-scoped wrapper around :func:`api.config.apply_custom_provider_connection_authority`, which holds the atomic-replacement rules (exact ``custom_providers[]`` row owns BOTH the endpoint and the credential; keyed/``model:`` fallbacks only fill gaps, as one same-record bundle). ``profile_name`` binds the SESSION's own profile while the connection is resolved. That lookup reads the endpoint AND the credential from ONE ``get_config()``/env snapshot, so binding the profile here keeps them from splitting across profiles: on a detached worker thread (which doesn't inherit the request-profile TLS/env) an unbound call would read the DEFAULT profile and pair a named profile's endpoint with the default profile's API key (finding #3). No-op for the default/root profile; when ``profile_name`` is None the ambient scope (if any) is used. ``custom_provider_lookup`` carries the session's pre-canonicalization identity, so a provider already rewritten to plain ``"custom"`` by an earlier resolve still selects its own named record on a retry. ``custom_owned`` is True when a config-owned custom record supplied the connection; see :func:`_resolve_runtime_connection_bundle` for why callers must then clear the runtime-owned side fields. """ lookup_provider = custom_provider_lookup or resolved_provider if not (isinstance(lookup_provider, str) and lookup_provider.startswith("custom:")): return resolved_provider, resolved_api_key, resolved_base_url, False from api import profiles as _profiles_api with _profiles_api.profile_scope_for_detached_worker( profile_name, "custom provider connection", logger_override=logger ): return apply_custom_provider_connection_authority( resolved_provider, resolved_api_key, resolved_base_url, lookup_provider=lookup_provider, # Pass THIS module's binding so tests (and any future shim) that patch # ``streaming.resolve_custom_provider_connection`` still take effect. connection_resolver=resolve_custom_provider_connection, ) def _resolve_custom_provider_runtime_overrides( resolved_provider: str | None, resolved_api_key: str | None, resolved_base_url: str | None, profile_name: str | None = None, custom_provider_lookup: str | None = None, ) -> tuple[str | None, str | None, str | None]: """Return provider/key/base_url overrides for ``custom:*`` endpoints. Legacy three-value view of :func:`_resolve_custom_provider_connection_authority`. Prefer :func:`_resolve_runtime_connection_bundle` in agent-construction paths: the three fields alone are NOT a complete constructor bundle, so a caller that keeps the ambient runtime's ``credential_pool`` / ``api_mode`` / ACP command+args beside them still sends a mixed-authority agent to the custom endpoint. """ return _resolve_custom_provider_connection_authority( resolved_provider, resolved_api_key, resolved_base_url, profile_name=profile_name, custom_provider_lookup=custom_provider_lookup, )[:3] # The constructor-routing fields AIAgent takes from the resolved runtime # provider. They travel together: a bundle that replaces the endpoint/credential # but leaves these behind builds an agent whose transport (ACP subprocess), # wire protocol (api_mode) or credential source (pool) still points at the # previous authority. _RUNTIME_BUNDLE_FIELDS = ('api_mode', 'acp_command', 'acp_args', 'credential_pool') def _resolve_runtime_connection_bundle( resolved_provider: str | None, resolved_api_key: str | None, resolved_base_url: str | None, runtime_provider: dict | None, profile_name: str | None = None, custom_provider_lookup: str | None = None, ) -> dict: """Return the COMPLETE constructor-routing bundle for one send attempt. Keys: ``provider``, ``base_url``, ``api_key`` plus every field in :data:`_RUNTIME_BUNDLE_FIELDS`. Callers must apply the whole dict — that is the point: the endpoint/credential and the transport/protocol/pool fields are one authority, and the agent-cache signature must be derived from the same dict so a bundle change always mints a new agent. """ lookup_provider = custom_provider_lookup or resolved_provider from api import profiles as _profiles_api with _profiles_api.profile_scope_for_detached_worker( profile_name, "custom provider connection", logger_override=logger ): return merge_custom_provider_runtime_bundle( resolved_provider, resolved_api_key, resolved_base_url, runtime_provider, lookup_provider=lookup_provider, connection_resolver=resolve_custom_provider_connection, ) def _same_base_url_endpoint(url_a: str, url_b: str) -> bool: """True if two base URLs point at the same scheme+host+port endpoint. Used to decide whether a runtime base_url is just a normalized form of the configured one (e.g. OpenCode-Go's ``/v1`` de-duplication on the same host) versus a genuinely different endpoint (an explicit ``providers..base_url`` override at a different host/port that must be preserved). Path/query are intentionally ignored — the normalization #3895 fixes is path-only. """ from urllib.parse import urlsplit try: a = urlsplit((url_a or "").strip()) b = urlsplit((url_b or "").strip()) except Exception: return False _default_port = {"http": 80, "https": 443} a_host = (a.hostname or "").lower() b_host = (b.hostname or "").lower() a_scheme = (a.scheme or "").lower() b_scheme = (b.scheme or "").lower() a_port = a.port or _default_port.get(a_scheme) b_port = b.port or _default_port.get(b_scheme) return bool(a_host) and a_host == b_host and a_scheme == b_scheme and a_port == b_port def _runtime_preferred_base_url( runtime_provider: dict | None, resolved_provider: str | None, configured_base_url: str | None, session_requested_provider: str | None = None, ) -> str | None: """Prefer the runtime-normalized base_url, but never override an explicit configured endpoint that points somewhere genuinely different. The #3895 bug was that WebUI used the *configured* base_url (which can carry a duplicated ``/v1``) instead of the runtime provider's per-model-normalized base_url, 404ing OpenCode-Go. But blindly preferring the runtime URL would clobber a legitimate ``providers..base_url`` override (e.g. LM Studio at a LAN IP, an OpenRouter mirror). So: - no runtime URL -> keep configured - no configured URL -> use runtime (all we have) - named ``custom:`` endpoint -> configured wins (then runtime as fallback) - same scheme+host+port -> runtime wins (it's the normalized/corrected form of the same endpoint — the #3895 case) - different endpoint -> configured override wins (no regression) """ runtime_base_url = None if isinstance(runtime_provider, dict): runtime_base_url = runtime_provider.get("base_url") if not runtime_base_url: return configured_base_url if not configured_base_url: return runtime_base_url provider_id = str( session_requested_provider or resolved_provider or (runtime_provider or {}).get("provider") or "" ).strip().lower() if provider_id.startswith("custom:"): return configured_base_url or runtime_base_url # An explicit configured override at a DIFFERENT endpoint must be preserved; # only prefer the runtime URL when it's the same endpoint (path-normalized). if _same_base_url_endpoint(configured_base_url, runtime_base_url): return runtime_base_url return configured_base_url def _is_fallback_lifecycle_message(kind: str, message: str) -> bool: """Return True if an agent lifecycle status should surface as a fallback warning.""" k = str(kind or '').strip().lower() m = str(message or '').strip().lower() return ( k == 'lifecycle' and ( 'rate limited' in m or 'switching to fallback' in m or 'falling back' in m or 'fallback activated' in m or 'trying fallback' in m or 'model fallback:' in m or 'switched to fallback' in m or 'primary model restored:' in m ) ) # Session turn-lease notices emitted by the Agent (agent/turn_facade_lease.py) # while another Hermes process (gateway, CLI, cron) holds this session's turn # lease. Emitted via ``_emit_status`` (kind ``lifecycle``) while waiting and on # admission, and via ``_emit_warning`` (kind ``warn``) when the wait times out # and the message was not processed. _SESSION_LEASE_WAIT_MARKERS = ( 'another hermes process is using this session', 'still waiting for the other hermes process', 'another hermes process kept this session busy', 'session is free; loading the latest transcript', ) def _is_session_lease_wait_message(kind: str, message: str) -> bool: """Return True for Agent session turn-lease wait notices. Classification keys on the Agent status kind (``lifecycle`` / ``warn``) so user-authored text can never be promoted to a warning. """ k = str(kind or '').strip().lower() if k not in ('lifecycle', 'warn'): return False m = str(message or '').strip().lower() return any(marker in m for marker in _SESSION_LEASE_WAIT_MARKERS) def _is_agent_compression_start_status(kind: str, message: str) -> bool: """Return True only for real Hermes context-compression start notices. WebUI bridges matching lifecycle statuses into an SSE ``compressing`` event and paints the live "Compressing context" worklog divider. The previous matcher used broad substrings such as ``'compressing' in message`` and ``'preflight compression' in message``, which can false-positive on skip / cooldown / unrelated notices and make brand-new low-token turns look like auto-compression. Positive markers below match the agent emitters in hermes-agent (``conversation_loop`` pre-API / 413 / too-large, ``conversation_compression`` compaction status). Preflight compression (``turn_context``) is intentionally excluded — the later authoritative ``Compacting context`` marker is the signal that compression actually proceeded. Explicitly reject skip / defer notices so "Skipping preflight compression…" never surfaces as a running compress divider. """ k = str(kind or '').strip().lower() m = str(message or '').strip().lower() if k != 'lifecycle' or not m: return False # Skip / cooldown / defer logs must never look like a live compression start. if ( 'skipping' in m or 'defer' in m or 'cooldown' in m or 'will not start' in m ): return False # Post-compress retry chatter is not a start event. if 'compressed' in m and 'compressing' not in m and 'compression attempt' not in m: return False return ( 'pre-api compression:' in m or 'compacting context' in m or 'context too large' in m or '— compressing (' in m or '- compressing (' in m or 'compression attempt' in m ) def _prewarm_skill_tool_modules(): """Import tools.skills_tool and tools.skill_manager_tool outside any lock. First-time module imports can trigger heavy initialisation (disk I/O, transitive imports, plugin discovery). Performing those imports while holding ``_ENV_LOCK`` serialises every concurrent session behind the slowest import. Prewarming ensures the modules are already in ``sys.modules`` before the lock is acquired, so the lock body only does lightweight attribute patching. We cannot place these at module top-level because ``tools.*`` lives in the hermes-agent package which may not be on ``sys.path`` at import time (Docker volume-mount ordering). A dedicated helper keeps the lazy-import try/except in one place and makes the intent explicit. """ for _mod_name in ('tools.skills_tool', 'tools.skill_manager_tool'): try: __import__(_mod_name) except ImportError: pass # Lazy import to avoid circular deps -- hermes-agent is on sys.path via api/config.py from api.agent_runtime import ensure_agent_runtime_current, get_ai_agent_class # Eagerly attempt the import at startup, matching the pre-guard behavior. If # dependencies are not ready yet, _get_ai_agent() retries when a chat starts. AIAgent = get_ai_agent_class() def _get_ai_agent(): """Return AIAgent class, retrying the import if the initial attempt failed. auto_install_agent_deps() in server.py may install missing packages after this module is first imported (common in Docker with a volume-mounted agent). Re-attempting the import here picks up the newly installed packages without requiring a server restart. The shared runtime guard also refuses to reuse cached Agent modules after the source checkout changes. """ global AIAgent ensure_agent_runtime_current() if AIAgent is None: AIAgent = get_ai_agent_class() return AIAgent def _is_quota_error_text(err_text: str) -> bool: """Return True when provider text looks like quota/usage exhaustion.""" _err_lower = str(err_text or '').lower() return ( 'insufficient credit' in _err_lower or 'credit balance' in _err_lower or 'credits exhausted' in _err_lower or 'more credits' in _err_lower or 'can only afford' in _err_lower or 'fewer max_tokens' in _err_lower or 'quota_exceeded' in _err_lower or 'quota exceeded' in _err_lower or 'exceeded your current quota' in _err_lower # OpenAI Codex OAuth usage-exhaustion shapes (#1765). or 'plan limit reached' in _err_lower or 'usage_limit_exceeded' in _err_lower or 'usage limit exceeded' in _err_lower or 'reached the limit of messages' in _err_lower or 'used up your usage' in _err_lower or ('plan' in _err_lower and 'limit' in _err_lower and 'reached' in _err_lower) ) def _clarify_session_config(sid: str) -> dict | None: """Config dict for the clarify session's profile, or None to use ambient. The agent runs on a detached worker thread that does not inherit the per-request TLS profile context, so the ambient ``get_config()`` would resolve to the process-global profile (usually ``default``) even for a session running under a named profile (issue #3294 pattern). Resolve the session's own profile home instead, mirroring how the rest of the run is configured. """ try: from api.models import _get_profile_home from api.config import get_config_for_profile_home session = get_session(sid) if session is None: return None return get_config_for_profile_home(_get_profile_home(getattr(session, "profile", None))) except Exception: return None def _clarify_timeout_seconds(config: dict | None = None, default: int = 3600) -> int: """Resolve the clarify timeout (seconds) for WebUI clarify prompts. Uses the agent core's canonical resolver when available (``tools.clarify_gateway.resolve_clarify_timeout`` — the same function the CLI and messaging gateways use), falling back to an inline mirror of its resolution order when the agent package is not importable. Resolution: legacy top-level ``clarify.timeout`` if explicitly set, else the canonical ``agent.clarify_timeout``, else 3600. ``<= 0`` is preserved verbatim and means *unlimited*: the prompt waits until the user answers or the run is cancelled — it is never auto-skipped. ``config`` may be an explicit profile config dict (see :func:`_clarify_session_config`); when None, the ambient ``get_config()`` is used. """ cfg = config if config is not None else (get_config() or {}) try: from tools.clarify_gateway import resolve_clarify_timeout return int(resolve_clarify_timeout(cfg)) except Exception: pass try: raw = (cfg.get("clarify") or {}).get("timeout") if raw is None: raw = (cfg.get("agent") or {}).get("clarify_timeout", 3600) return int(raw) except Exception: return int(default) def _await_clarify_response(entry, timeout, cancel_evt) -> tuple[str, bool]: """Block until a clarify entry resolves, or (finite timeout only) expires. ``timeout <= 0`` means *unlimited*: no deadline — the prompt waits until the user answers or the run is cancelled. Polls the entry's event in ~1s slices so a cancelled run unblocks promptly, and still honours the ``is_interrupted``-style cancel check each slice. Returns ``(response, expired)``: ``expired`` is True when it returned because of cancellation or a finite timeout without a user response (the caller should clear pending and fall back), False when the user answered. """ deadline = None if timeout <= 0 else time.monotonic() + timeout while True: if cancel_evt.is_set(): return "", True wait_for = 1.0 if deadline is not None: remaining = deadline - time.monotonic() if remaining <= 0: return "", True wait_for = min(1.0, remaining) if entry.event.wait(timeout=wait_for): return str(entry.result or "").strip(), False _CANCEL_MARKER_PATTERNS = ('task cancelled', 'task canceled', 'response interrupted') _WEBUI_PROGRESS_PROMPT = """ WebUI progress guidance: - Match the normal Hermes messaging style, but do not let long tool-running WebUI turns appear silent. - For long multi-step work that uses tools, emit brief user-visible progress updates as normal assistant content, not only as hidden reasoning. - Before the first tool batch in a long task, say what you are about to inspect. - After each meaningful batch of tool calls, say what you just confirmed and what you will check next before continuing with more tools. - Do not run many independent tool batches back-to-back without visible assistant text between them when the task is still ongoing. - Do not keep progress only in reasoning, thinking, or tool-result channels; those are not a substitute for visible interim updates. - Each update should say what you are about to check, what you just confirmed, or why the next tool call is needed. - Keep updates concise, factual, and in the user's language. One or two short sentences are enough. - Do not reveal hidden reasoning, chain-of-thought, private scratchpads, secrets, raw logs, or long tool output. - Password, API-key, token, and secret fields are automatically redacted by the system. Treat masked values as intentional redaction, not placeholder text or user input errors, and do not tell the user a stored credential is wrong based on a masked value alone. - Final visible assistant replies must be clear, user-facing, and in the user's language, not private planning notes. - Do not include terse planning fragments or scratchpad shorthand in visible assistant text. Avoid fragments like "Need script", "Need check logs", "Need inspect email", or "maybe invite"; either omit them or rewrite them as clear user-facing progress. - For direct answers or very short tasks, skip progress updates and answer normally. """.strip() def _webui_surface_context_prompt(surface_context: Optional[dict]) -> str: """Return safe WebUI session metadata for the agent's ephemeral context. Messaging gateways inject platform/channel context before each run. Browser sessions do not have a chat platform wrapper, so provide an explicit, small surface description here instead of relying on the model to infer where it is running from the transcript alone. """ if not isinstance(surface_context, dict): return "" lines = [ "WebUI session context:", "- This browser session is not the same live transcript as Telegram, Discord, Slack, or other messaging surfaces.", "- Use durable memory, saved sessions, and available tools for cross-surface recall instead of assuming those transcripts are in this browser chat.", "- Do not copy or dump this browser transcript into external notes or durable memory by default.", "- Write to external notes or durable memory only for explicit captures, durable user preferences, decisions, blockers/open issues, runbook-worthy workflows, or other clearly reusable signals; otherwise leave notes unchanged.", "- When you do write or update a durable note, briefly tell the user what note/section changed so the write is reviewable.", ] fields = ( ("source", "Source"), ("session_id", "Session ID"), ("profile", "Profile"), ("workspace", "Workspace"), ) for key, label in fields: raw = surface_context.get(key) value = str(raw).strip() if raw is not None else "" if value: lines.append(f"- {label}: {value}") return "\n".join(lines) def _webui_ephemeral_system_prompt( personality_prompt: Optional[str], surface_context: Optional[dict] = None, config_data: Optional[dict] = None, ) -> str: """Build WebUI-only runtime instructions that are not persisted to history.""" parts = [] if personality_prompt: parts.append(str(personality_prompt).strip()) surface_prompt = _webui_surface_context_prompt(surface_context) if surface_prompt: parts.append(surface_prompt) parts.append(_WEBUI_PROGRESS_PROMPT) delivery_prompt = _webui_delivery_context_prompt(config_data) if delivery_prompt: parts.append(delivery_prompt) return "\n\n".join(part for part in parts if part) _SECRET_SHAPED_RE = re.compile( r"(?i)(api[_-]?key|token|password|secret)\s*[:=]\s*[^\s]+|" r"\b(?:sk-[A-Za-z0-9_-]{16,}|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b|" r"[A-Za-z0-9_-]{24,}\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}" ) def _redact_prefill_status_text(text: str) -> str: """Return a short, non-secret diagnostic string for prefill status.""" clean = _SECRET_SHAPED_RE.sub("[REDACTED]", str(text or "")) return " ".join(clean.split())[:240] def _valid_prefill_messages(value) -> list[dict]: """Normalize a prefill payload to role/content messages.""" if not isinstance(value, list): return [] messages: list[dict] = [] for item in value: if not isinstance(item, dict): continue role = item.get("role") content = item.get("content") if role not in {"system", "user", "assistant"} or not isinstance(content, str) or not content.strip(): continue messages.append({"role": role, "content": content}) return messages def _resolve_prefill_path(raw: str) -> Path: path = Path(str(raw)).expanduser() if not path.is_absolute(): try: from api.config import _get_config_path path = _get_config_path().parent / path except Exception: path = Path.cwd() / path return path _PREFILL_SCRIPT_OUTPUT_LIMIT = 262_144 _PREFILL_CONTEXT_DEFAULT_MAX_CHARS = 12_000 def _prefill_context_max_chars(config_data: dict) -> int: raw = os.getenv("HERMES_WEBUI_PREFILL_CONTEXT_MAX_CHARS", "") or str( config_data.get("webui_prefill_context_max_chars") or "" ) try: value = int(raw or _PREFILL_CONTEXT_DEFAULT_MAX_CHARS) except Exception: value = _PREFILL_CONTEXT_DEFAULT_MAX_CHARS return max(0, min(value, _PREFILL_SCRIPT_OUTPUT_LIMIT)) def _prefill_context_char_count(messages: list[dict]) -> int: return sum(len(str(message.get("content") or "")) for message in messages if isinstance(message, dict)) def _budget_compacted_prefill_context(context: dict, *, max_chars: int, char_count: int) -> dict: label = str(context.get("label") or "prefill context") message = ( "A configured WebUI startup prefill source was available, but it exceeded " f"the WebUI prefill context budget ({char_count} chars > {max_chars} chars), " "so the note/body payload was omitted from this new chat. If the user's " "request depends on prior decisions, durable notes, runbooks, current " "context, or open issues, use the available retrieval/search/note tools " "to fetch only the relevant details before answering." ) return { "status": "loaded", "source": "budget_compacted", "label": label, "messages": [{"role": "user", "content": message}], "message_count": 1, "compacted": True, "original_source": context.get("source", ""), "original_message_count": int(context.get("message_count") or 0), "original_char_count": char_count, "max_chars": max_chars, } def _apply_prefill_context_budget(context: dict, config_data: dict) -> dict: if context.get("status") != "loaded": return context max_chars = _prefill_context_max_chars(config_data) if max_chars <= 0: return context messages = context.get("messages") or [] char_count = _prefill_context_char_count(messages if isinstance(messages, list) else []) if char_count <= max_chars: return context file_raw = os.getenv("HERMES_PREFILL_MESSAGES_FILE", "") or str(config_data.get("prefill_messages_file") or "") if context.get("source") == "script" and file_raw: fallback = _load_prefill_messages_file(file_raw, source="file_budget_fallback") fallback_messages = fallback.get("messages") if isinstance(fallback, dict) else [] fallback_chars = _prefill_context_char_count(fallback_messages if isinstance(fallback_messages, list) else []) if fallback.get("status") == "loaded" and fallback_chars <= max_chars: fallback["compacted"] = True fallback["original_source"] = context.get("source", "") fallback["original_label"] = context.get("label", "") fallback["original_message_count"] = int(context.get("message_count") or 0) fallback["original_char_count"] = char_count fallback["max_chars"] = max_chars return fallback return _budget_compacted_prefill_context(context, max_chars=max_chars, char_count=char_count) def _prefill_not_configured() -> dict: return {"status": "not_configured", "source": "none", "label": "", "messages": [], "message_count": 0} def _load_prefill_messages_file(file_raw: str, *, source: str = "file", status: str = "loaded") -> dict: path = _resolve_prefill_path(file_raw) label = path.name or "prefill file" if not path.exists(): return {"status": "error", "source": source, "label": label, "messages": [], "message_count": 0, "error": "prefill file not found"} try: messages = _valid_prefill_messages(json.loads(path.read_text(encoding="utf-8"))) return {"status": status, "source": source, "label": label, "messages": messages, "message_count": len(messages)} except Exception as exc: return {"status": "error", "source": source, "label": label, "messages": [], "message_count": 0, "error": _redact_prefill_status_text(str(exc))} def _prefill_script_timeout(config_data: dict) -> float: raw = os.getenv("HERMES_WEBUI_PREFILL_MESSAGES_SCRIPT_TIMEOUT", "") or str(config_data.get("webui_prefill_messages_script_timeout") or "") try: return max(0.1, min(float(raw or 5), 30.0)) except Exception: return 5.0 def _prefill_script_command(raw) -> list[str]: if isinstance(raw, (list, tuple)): return [str(part) for part in raw if str(part)] parts = shlex.split(str(raw or "")) if not parts: return [] # A single script path mirrors prefill_messages_file path resolution. More # complex commands keep their argv untouched so admins can pass arguments. if len(parts) == 1: parts[0] = str(_resolve_prefill_path(parts[0])) return parts def _messages_from_prefill_script_output(text: str) -> list[dict]: stripped = str(text or "").strip() if not stripped: return [] try: payload = json.loads(stripped) except Exception: payload = None if isinstance(payload, dict): payload = payload.get("messages") messages = _valid_prefill_messages(payload) if messages: return messages return [{"role": "user", "content": stripped}] def _load_prefill_messages_script(config_data: dict) -> dict: script_raw = os.getenv("HERMES_WEBUI_PREFILL_MESSAGES_SCRIPT", "") or config_data.get("webui_prefill_messages_script") if not script_raw: return _prefill_not_configured() command = _prefill_script_command(script_raw) label = Path(command[0]).name if command else "prefill script" if not command: return {"status": "error", "source": "script", "label": label, "messages": [], "message_count": 0, "error": "prefill script is empty"} try: proc = subprocess.run( command, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=_prefill_script_timeout(config_data), check=False, ) except subprocess.TimeoutExpired: return {"status": "error", "source": "script", "label": label, "messages": [], "message_count": 0, "error": "prefill script timed out"} except Exception as exc: return {"status": "error", "source": "script", "label": label, "messages": [], "message_count": 0, "error": _redact_prefill_status_text(str(exc))} if proc.returncode != 0: err = _redact_prefill_status_text(proc.stderr or proc.stdout or f"prefill script exited {proc.returncode}") return {"status": "error", "source": "script", "label": label, "messages": [], "message_count": 0, "error": err} if len(proc.stdout.encode("utf-8")) > _PREFILL_SCRIPT_OUTPUT_LIMIT: return { "status": "error", "source": "script", "label": label, "messages": [], "message_count": 0, "error": f"prefill script output exceeded {_PREFILL_SCRIPT_OUTPUT_LIMIT} bytes", } messages = _messages_from_prefill_script_output(proc.stdout) return {"status": "loaded", "source": "script", "label": label, "messages": messages, "message_count": len(messages)} def _load_webui_prefill_context( config_data: Optional[dict] = None, ) -> dict: """Load configured WebUI session prefill messages. Supports the same bounded JSON-file shape used by Hermes Agent. WebUI also supports its own explicitly opt-in script hook so admins can bridge Joplin, Obsidian, Notion, llm-wiki, or another local notes source into ephemeral turn context without baking any one note provider into the WebUI. """ cfg = config_data if isinstance(config_data, dict) else get_config() script_context = _load_prefill_messages_script(cfg) file_raw = os.getenv("HERMES_PREFILL_MESSAGES_FILE", "") or str(cfg.get("prefill_messages_file") or "") if script_context.get("status") == "not_configured": if file_raw: return _apply_prefill_context_budget(_load_prefill_messages_file(file_raw), cfg) return _prefill_not_configured() if script_context.get("status") == "error" and file_raw: file_context = _load_prefill_messages_file(file_raw, source="file_fallback") if file_context.get("status") == "loaded": file_context["script_error"] = script_context.get("error", "") return _apply_prefill_context_budget(file_context, cfg) return _apply_prefill_context_budget(script_context, cfg) def _public_prefill_context_status(prefill_context: dict) -> dict: """Strip message bodies before sending context status to the browser.""" return { "status": prefill_context.get("status", "not_configured"), "source": prefill_context.get("source", "none"), "label": prefill_context.get("label", ""), "message_count": int(prefill_context.get("message_count") or 0), **({"error": prefill_context.get("error", "")} if prefill_context.get("error") else {}), **({"compacted": True} if prefill_context.get("compacted") else {}), **({"original_source": prefill_context.get("original_source", "")} if prefill_context.get("original_source") else {}), **({"original_message_count": int(prefill_context.get("original_message_count") or 0)} if prefill_context.get("original_message_count") else {}), **({"original_char_count": int(prefill_context.get("original_char_count") or 0)} if prefill_context.get("original_char_count") else {}), **({"max_chars": int(prefill_context.get("max_chars") or 0)} if prefill_context.get("max_chars") else {}), } def _webui_delivery_context_prompt(config_data: Optional[dict] = None) -> str: """Return platform/delivery context for the ephemeral system prompt. Connected platforms, home channels, and scheduled-task delivery hints are injected into the system prompt (safe for role alternation) rather than as a prefill ``user`` message, which strict chat templates (Mistral, Gemma) reject. NOTE: This function only covers platform/delivery info. The session framing (\"Source: WebUI\", \"Session ID\", \"Profile\", \"Workspace\") is emitted by ``_webui_surface_context_prompt()``, which is called from ``_webui_ephemeral_system_prompt()`` before this helper. If you refactor this area, keep that surface call in place — the two helpers together produce the full session context block. """ cfg = config_data if isinstance(config_data, dict) else get_config() lines: list[str] = [] display_hermes_home = None try: from hermes_constants import get_hermes_home, display_hermes_home as _dh display_hermes_home = _dh except Exception: get_hermes_home = None # type: ignore[assignment] connected = ["local (files on this machine)"] try: if get_hermes_home is not None: state_path = get_hermes_home() / "gateway_state.json" if state_path.exists(): raw_state = json.loads(state_path.read_text(encoding="utf-8")) platforms = raw_state.get("platforms") if isinstance(raw_state, dict) else {} if isinstance(platforms, dict): for name in sorted(platforms): pdata = platforms.get(name) or {} if isinstance(pdata, dict) and pdata.get("state") == "connected" and name != "local": connected.append(f"{name}: Connected ✓") except Exception: pass lines.append(f"**Connected Platforms:** {', '.join(connected)}") home_channels = {} try: platforms_cfg = cfg.get("platforms", {}) if isinstance(cfg, dict) else {} if isinstance(platforms_cfg, dict): for name, pdata in platforms_cfg.items(): if not isinstance(pdata, dict): continue if pdata.get("enabled") is False: continue home = pdata.get("home_channel") if isinstance(home, dict): home_channels[str(name)] = str(home.get("name") or name) except Exception: home_channels = {} if home_channels: lines.append("") lines.append("**Home Channels (default destinations):**") for platform, label in sorted(home_channels.items()): lines.append(f" - {platform}: {label}") lines.append("") lines.append("**Delivery options for scheduled tasks:**") lines.append("- `\"origin\"` → Back to this WebUI/browser session when the WebUI runtime supports origin delivery; otherwise prefer an explicit platform target.") try: home_display = display_hermes_home() if display_hermes_home else "~/.hermes" except Exception: home_display = "~/.hermes" lines.append(f"- `\"local\"` → Save to local files only ({home_display}/cron/output/)") for platform, label in sorted(home_channels.items()): lines.append(f"- `\"{platform}\"` → Home channel ({label})") lines.append("") lines.append("*For explicit targeting, use `\"platform:chat_id\"` format if the user provides a specific chat ID. Do not invent private IDs.*") return "\n".join(lines) def _prefill_messages_with_webui_context(prefill_context: dict, config_data: Optional[dict] = None) -> list[dict]: """Combine recall prefill with WebUI session context. The session context (connected platforms, delivery hints) is injected via ``_webui_ephemeral_system_prompt`` / ``ephemeral_system_prompt`` instead of as a prefill ``user`` message. Adding it as a user message creates two consecutive user turns (prefill + actual) which strict chat templates (Mistral, Gemma) reject with a Jinja 500. """ return list(prefill_context.get("messages") or []) def _normalize_prefill_messages_before_user_turn(prefill_messages: list[dict]) -> list[dict]: """Ensure WebUI prefill does not end with user role before an appended turn. Some upstream prefill sources can end with `role: user` (for example, session context or recall snippets). WebUI always appends the current user turn after prefill in the streaming path, so a terminal user role creates an adjacent user/user sequence that strict chat templates (Gemma, Mistral/Jinja) reject. To keep behavior scoped, only consecutive terminal user messages are removed just before that boundary; earlier roles remain untouched. """ sanitized = list(prefill_messages or []) n_dropped = 0 while sanitized: last_message = sanitized[-1] if not isinstance(last_message, dict): break if str(last_message.get("role") or "").strip().lower() != "user": break sanitized.pop() n_dropped += 1 if n_dropped: logger.debug("Dropped %d trailing user message(s) from prefill", n_dropped) return sanitized def _has_new_assistant_reply(all_messages: list, prev_count: int) -> bool: """Return True if *new* messages (beyond ``prev_count``) contain an assistant message with non-empty content. ``all_messages`` is ``result.get('messages')`` which includes the full conversation history. ``prev_count`` is ``len(_previous_context_messages)`` — the number of messages present before the current turn started. Only messages at index >= prev_count are inspected so that historical assistant replies don't mask a silent failure on the current turn. If ``len(all_messages) < prev_count`` (an edge-case shrink), there is no reliable new-message slice to inspect. Treat that as "no new assistant reply" so stale historical assistant replies cannot mask a silent failure. When ``len == prev_count``, there are no new messages and we return False. """ if len(all_messages) > prev_count: # Normal case: new messages appended beyond the pre-turn history. candidates = all_messages[prev_count:] elif len(all_messages) < prev_count: return False else: # Same length. In production this means no new messages were appended. # However, some test fixtures replace the entire message list rather # than appending, so check whether the tail changed. return False return any( m.get('role') == 'assistant' and str(m.get('content') or '').strip() for m in candidates ) def _preferred_agent_display_name() -> str: """Return the configured assistant display name for user-facing copy.""" try: name = str((load_settings() or {}).get('bot_name') or '').strip() except Exception: logger.debug("Failed to load bot_name for cancellation copy", exc_info=True) name = '' return name or 'Hermes' def _preferred_agent_display_name_for_session(session) -> str: profile = str(getattr(session, 'profile', '') or '').strip() if profile and profile != 'default': return profile[:1].upper() + profile[1:] return _preferred_agent_display_name() def _cancelled_turn_hint(agent_name: str | None = None) -> str: name = str(agent_name or _preferred_agent_display_name()).strip() or 'Hermes' return f'The run was cancelled by the user before {name} finished. No provider failure occurred.' def _provider_error_probe_text(value) -> tuple[str, int | None]: """Flatten structured provider-error payloads into searchable text.""" _texts: list[str] = [] _status_code: int | None = None _seen: set[int] = set() def _walk(node): nonlocal _status_code if node is None: return if isinstance(node, (dict, list, tuple, set)): _node_id = id(node) if _node_id in _seen: return _seen.add(_node_id) if isinstance(node, dict): for _key in ('type', 'code', 'message', 'detail', 'details', 'name', 'status', 'status_code'): _val = node.get(_key) if _val is None: continue if _key in ('status', 'status_code') and _status_code is None: try: _status_code = int(_val) except Exception: pass _texts.append(str(_val)) for _key, _val in node.items(): if _key in ('type', 'code', 'message', 'detail', 'details', 'name', 'status', 'status_code'): continue _walk(_val) return if isinstance(node, (list, tuple, set)): for _item in node: _walk(_item) return _texts.append(str(node)) _walk(value) return ' '.join(t for t in _texts if t).strip(), _status_code def _result_reports_compression_snapshot_stale(result) -> bool: """Return whether an Agent result carries an exact stale-snapshot marker.""" return ( isinstance(result, dict) and ( result.get('error') == 'compression_snapshot_stale' or result.get('compression_snapshot_stale') is True ) ) def _custom_provider_route_classification(error) -> dict: """WebUI apperror classification for a terminal custom-provider route. ``error`` is either a :class:`CustomProviderRouteError` or the verdict dict :func:`custom_provider_route_error` read off a bundle — the retry paths hold the verdict without ever raising, so both shapes classify identically. """ if isinstance(error, dict): reason = error.get('reason') hint = error.get('hint') or '' message = error.get('message') or '' else: reason = getattr(error, 'reason', None) hint = getattr(error, 'hint', '') or '' message = getattr(error, 'message', None) or str(error) if reason == CUSTOM_ROUTE_NO_CREDENTIAL: label = 'Provider credential unavailable' elif reason == CUSTOM_ROUTE_NO_ENDPOINT: label = 'Provider endpoint unavailable' else: label = 'Provider not configured' return { 'label': label, 'type': 'provider_unroutable', # The STRUCTURED verdict (``custom_provider_endpoint_unresolved`` / # ``custom_provider_no_credential``) travels beside the prose so the # emitted apperror can carry it too. ``type`` alone says only "this route # is unroutable"; which of the two settings to fix lives here, and the # non-streaming /api/chat refusal already answers it as ``reason``. 'reason': reason, 'hint': hint, 'message': message, } def _classify_provider_error( err_str: str, exc=None, *, silent_failure: bool = False, result=None, ) -> dict: """Classify provider/agent failure text for WebUI apperror UX. Keep this string-based until hermes-agent exposes stable structured provider error classes for Codex OAuth plan limits. """ _probe_text, _probe_status_code = _provider_error_probe_text(err_str) if exc is not None: _exc_probe_text, _exc_status_code = _provider_error_probe_text(exc) if _exc_probe_text: _probe_text = f"{_probe_text} {_exc_probe_text}".strip() if _probe_status_code is None: _probe_status_code = _exc_status_code err_str = str(_probe_text or err_str or '') _err_lower = err_str.lower() _exc_name = type(exc).__name__ if exc is not None else '' if isinstance(exc, CustomProviderRouteError): # An unroutable named ``custom:`` route. Classified BEFORE any text # matching so it can never be read as a 401 — the auth branch would run # credential self-heal, which re-resolves a provider and is exactly the # re-routing this verdict exists to prevent. return _custom_provider_route_classification(exc) _result_is_compression_snapshot_stale = ( _result_reports_compression_snapshot_stale(result) ) try: from agent.conversation_compression import CompressionSnapshotStaleError # type: ignore[attr-defined] except ImportError: # Paired deployments export the typed exception. The name-only fallback # keeps a rolling WebUI/Agent upgrade actionable without text matching. _is_compression_snapshot_stale = ( _result_is_compression_snapshot_stale or _exc_name == 'CompressionSnapshotStaleError' ) else: _is_compression_snapshot_stale = ( _result_is_compression_snapshot_stale or isinstance(exc, CompressionSnapshotStaleError) ) if _is_compression_snapshot_stale: return { 'label': 'Conversation changed during compression', 'type': 'compression_snapshot_stale', 'hint': ( 'The durable conversation changed while compression was being prepared. ' 'No automatic retry was attempted. Send your next message to reload and ' 'reconcile the latest conversation state.' ), } _is_cancelled = ( 'cancelled by user' in _err_lower or 'canceled by user' in _err_lower or 'user cancelled' in _err_lower or 'user canceled' in _err_lower or 'task cancelled' in _err_lower or 'task canceled' in _err_lower or 'cancellederror' in _err_lower or (exc is not None and _exc_name in ('CancelledError', 'CanceledError')) ) _is_interrupted = ( not _is_cancelled and ( 'interrupted by user' in _err_lower or 'response interrupted' in _err_lower or 'operation interrupted' in _err_lower or 'operation was interrupted' in _err_lower or 'operation aborted' in _err_lower or 'request was aborted' in _err_lower or 'aborterror' in _err_lower or (exc is not None and type(exc).__name__ in ('KeyboardInterrupt', 'AbortError')) ) ) if _is_cancelled: return { 'label': 'Task cancelled', 'type': 'cancelled', 'hint': _cancelled_turn_hint(), } if _is_interrupted: return { 'label': 'Response interrupted', 'type': 'interrupted', 'hint': 'The run stopped before a provider response completed. If you did not cancel it, try again.', } _is_quota = _is_quota_error_text(err_str) # A credential-POOL exhaustion ("All 0 credential(s) exhausted for ") # is a distinct shape from account/plan quota: it means the profile's # credential pool has no usable keys for that provider (a config problem), # not that a funded account ran out of credits. It is NOT matched by # _is_quota_error_text ('credential(s) exhausted' != 'credits exhausted'), so # without this it fell through to the generic error label/hint. Classify it # explicitly so the user gets a pool-specific, actionable hint. (#3929) _is_credential_pool_empty = ( 'credential(s) exhausted' in _err_lower or 'credentials exhausted' in _err_lower or ('credential' in _err_lower and 'exhausted' in _err_lower) ) _is_auth = ( not _is_quota and not _is_credential_pool_empty and ( _probe_status_code == 401 or '401' in err_str or (exc is not None and 'AuthenticationError' in _exc_name) or 'authentication' in _err_lower or 'unauthorized' in _err_lower or 'invalid api key' in _err_lower or 'invalid_api_key' in _err_lower or 'no cookie auth credentials' in _err_lower ) ) _is_not_found = ( # model_not_found hints mention Settings / `hermes model` below. '404' in err_str or 'not found' in _err_lower or 'does not exist' in _err_lower or 'model not found' in _err_lower or 'model_not_found' in _err_lower # hint below points to Settings / `hermes model` or 'invalid model' in _err_lower or 'does not match any known model' in _err_lower or 'unknown model' in _err_lower ) _is_rate_limit = (not _is_quota) and ( 'rate limit' in _err_lower or '429' in err_str or (exc is not None and 'RateLimitError' in _exc_name) ) _is_compression_exhausted = ( 'compression_exhausted' in _err_lower or 'compression exhausted' in _err_lower or ('context length exceeded' in _err_lower and 'cannot compress further' in _err_lower) or ('context compression' in _err_lower and 'max compression attempts' in _err_lower) ) if _is_credential_pool_empty: return { 'label': 'No usable credentials', 'type': 'credential_pool_empty', 'hint': 'The credential pool for this provider has no usable keys left (all entries exhausted or unconfigured). Add or refresh a key for this provider in your Hermes config / credential pool, or switch providers via `hermes model`.', } if _is_quota: return { 'label': 'Out of credits', 'type': 'quota_exhausted', 'hint': 'Your provider account is out of credits or usage. Top up, wait for the plan window to reset, or switch providers via `hermes model`.', } if _is_rate_limit: return { 'label': 'Rate limit reached', 'type': 'rate_limit', 'hint': 'Rate limit reached. The fallback model (if configured) was also exhausted. Try again in a moment.', } if _is_auth: return { 'label': 'Authentication failed', 'type': 'auth_mismatch', 'hint': 'The selected model may not be supported by your configured provider or your API key is invalid. Run `hermes model` in your terminal to update credentials, then restart the WebUI.', } if _is_not_found: return { 'label': 'Model not found', 'type': 'model_not_found', 'hint': 'The selected model was not found by the provider. Check the model ID in Settings or run `hermes model` to verify it exists for your provider.', } if _is_compression_exhausted: return { 'label': 'Context compression exhausted', 'type': 'compression_exhausted', 'hint': 'The conversation context is too large to compress safely. Start a new conversation or retry with a narrower task.', } if silent_failure: return { 'label': 'No response from provider', # Preserve the existing no_response event type (#373) while making # the catch-all silent-failure message more specific for #1765. 'type': 'no_response', 'hint': 'The provider returned no content and no error. This often means a usage/rate limit was hit silently. Check provider status, switch providers via `hermes model`, or try again in a moment.', } return {'label': 'Error', 'type': 'error', 'hint': ''} def _provider_error_payload(message: str, err_type: str, hint: str = '') -> dict: """Build a bounded, redacted apperror payload with provider details.""" _message = str(message or '') _safe_message = _redact_text(_message).strip() if _message else '' payload: dict = {'message': _safe_message or _message, 'type': err_type} if hint: payload['hint'] = hint if _safe_message: _details = _safe_message if len(_details) > 1200: _details = _details[:1197].rstrip() + '…' if _details: payload['details'] = _details return payload _MAX_ITERATION_SUMMARY_REQUEST = ( "You've reached the maximum number of tool-calling iterations allowed. " "Please provide a final response summarizing what you've found and accomplished " "so far, without calling any more tools." ) def _is_synthetic_max_iteration_summary_request(message) -> bool: """Return True for Hermes Agent's internal max-iteration summary prompt.""" if not isinstance(message, dict) or message.get('role') != 'user': return False text = " ".join(_message_text(message.get('content', '')).split()) expected = " ".join(_MAX_ITERATION_SUMMARY_REQUEST.split()) return text == expected def _drop_synthetic_max_iteration_summary_requests(messages, *, enabled: bool = True): """Remove Agent-internal max-iteration summary prompts from WebUI state.""" if not enabled: return list(messages or []) return [ msg for msg in list(messages or []) if not _is_synthetic_max_iteration_summary_request(msg) ] # Structured markers the Hermes Agent stamps on synthetic scaffolding turns that # drive its internal verify-before-finish loop. The agent appends BOTH a # synthetic assistant "premature done" answer AND a synthetic ``user`` nudge # (e.g. "[System: You edited code in this turn, but the workspace does not have # fresh passing verification evidence yet...]") to preserve role alternation for # the next API turn, and flags each with one of these keys. They exist only to # run the loop; they must never surface as visible user/assistant turns in the # WebUI transcript. This mirrors ``run_agent._EPHEMERAL_SCAFFOLDING_FLAGS`` on # the agent side (which keeps them out of the durable session store); WebUI # honors the same markers when building the visible transcript. Keep roughly in # sync with the agent set. (#5334; same class as #3320/#3821/#4373/#4875) _SYNTHETIC_CONTROL_MESSAGE_FLAGS = ( "_verification_stop_synthetic", "_pre_verify_synthetic", ) def _is_synthetic_control_message(message) -> bool: """Return True for an Agent-internal synthetic scaffolding turn flagged by marker.""" return isinstance(message, dict) and any( message.get(flag) for flag in _SYNTHETIC_CONTROL_MESSAGE_FLAGS ) def _drop_synthetic_control_messages(messages): """Remove Agent-internal synthetic scaffolding turns from the WebUI transcript. Honors the structured ``_verification_stop_synthetic`` / ``_pre_verify_synthetic`` markers the agent already sets, rather than string-matching the nudge copy. """ return [ msg for msg in list(messages or []) if not _is_synthetic_control_message(msg) ] def _clean_synthetic_control_messages_with_provenance(messages): """Remove synthetic rows while retaining marked verification-nudge provenance.""" raw_messages = list(messages or []) has_verification_nudge = any( isinstance(message, dict) and message.get('role') == 'user' and _is_synthetic_control_message(message) for message in raw_messages ) return _drop_synthetic_control_messages(raw_messages), has_verification_nudge def _active_turn_authority(session, stream_id, msg_text): """Capture the stream-owned pending turn before settlement mutates it.""" pending_text = getattr(session, 'pending_user_message', None) token = build_active_turn_token(stream_id, getattr(session, 'pending_started_at', None)) checkpoint = ( next( ( copy.deepcopy(message) for message in reversed(list(getattr(session, 'messages', None) or [])) if isinstance(message, dict) and message.get('role') == 'user' and message.get('_active_turn_token') == token ), None, ) if token else None ) return { 'session_id': getattr(session, 'session_id', None), 'token': token, 'text': pending_text if pending_text is not None else msg_text, 'timestamp': getattr(session, 'pending_started_at', None), 'source': getattr(session, 'pending_user_source', None) or 'webui', 'attachments': copy.deepcopy(getattr(session, 'pending_attachments', None) or []), 'checkpoint': checkpoint, 'current_turn_user_idx': None, 'turn_id': '', } def _coerce_current_turn_user_idx(value): if isinstance(value, bool) or value is None: return None try: idx = int(value) except (TypeError, ValueError): return None return idx if idx >= 0 else None def _resolve_active_turn_authority(identity, *, result=None, agent=None): if not isinstance(identity, dict): return identity resolved = dict(identity) resolved.pop('agent_turn_boundary_resolved', None) resolved.pop('agent_turn_boundary_source', None) # Treat the boundary as one coherent pair. In particular, do not retain the # failed Agent instance's index/turn when credential self-heal creates a # fresh Agent whose compacted projection has a different current-user index. # The returned-result pair is authoritative when exported; current paired # Agents keep the same pair on the instance instead. _boundary_idx = None _boundary_turn_id = '' _boundary_source = '' if isinstance(result, dict): _result_idx = _coerce_current_turn_user_idx(result.get('current_turn_user_idx')) _result_turn_id = str(result.get('turn_id') or '').strip() if _result_idx is not None and _result_turn_id: _boundary_idx = _result_idx _boundary_turn_id = _result_turn_id _boundary_source = 'result' if agent is not None: _agent_idx = _coerce_current_turn_user_idx( getattr(agent, '_persist_user_message_idx', None) ) _agent_turn_id = str(getattr(agent, '_current_turn_id', '') or '').strip() if _agent_idx is not None and _agent_turn_id and not _boundary_source: _boundary_idx = _agent_idx _boundary_turn_id = _agent_turn_id _boundary_source = 'agent' if _boundary_source: resolved['current_turn_user_idx'] = _boundary_idx resolved['turn_id'] = _boundary_turn_id resolved['agent_turn_boundary_resolved'] = True resolved['agent_turn_boundary_source'] = _boundary_source return resolved def _active_turn_boundary_is_valid(identity): if not isinstance(identity, dict): return False if not str(identity.get('turn_id') or '').strip(): return False return ( identity.get('agent_turn_boundary_resolved') is True and isinstance(identity.get('current_turn_user_idx'), int) ) def _active_turn_token_matches(message, identity): token = identity.get('token') if isinstance(identity, dict) else None if not token: return False return ( isinstance(message, dict) and message.get('role') == 'user' and message.get('_active_turn_token') == token ) def _active_turn_has_checkpoint(messages, identity): return any(_active_turn_token_matches(message, identity) for message in messages or []) def _mark_active_turn_checkpoint(message, identity): if ( isinstance(message, dict) and message.get('role') == 'user' and isinstance(identity, dict) and identity.get('token') ): message['_active_turn_token'] = identity['token'] return message def _mark_active_turn_checkpoint_in_history(messages, identity, msg_text, *, allow_index_fallback=True): messages = list(messages or []) if not isinstance(identity, dict): return messages, False if _active_turn_has_checkpoint(messages, identity): return messages, True if not allow_index_fallback: return messages, False if not _active_turn_boundary_is_valid(identity): return messages, False idx = identity['current_turn_user_idx'] if idx < 0 or idx >= len(messages): return messages, False message = messages[idx] expected_text = identity.get('text') if identity.get('text') is not None else msg_text if ( not isinstance(message, dict) or message.get('role') != 'user' or _normalize_user_text(_message_text(message.get('content'))) != _normalize_user_text(expected_text) ): return messages, False _mark_active_turn_checkpoint(message, identity) return messages, True def _active_turn_user_text_matches(message, expected_text): """Validate the Agent's declared user row against the submitted prompt.""" if not isinstance(message, dict) or message.get('role') != 'user' or not isinstance(expected_text, str): return False content = message.get('content') if isinstance(content, list): leading_text = [] for part in content: if not isinstance(part, dict): return False part_type = str(part.get('type') or '').lower() if part_type in {'image', 'image_url', 'input_image'}: break if part_type not in {'', 'text', 'input_text', 'output_text'}: return False text = part.get('text', part.get('input_text', part.get('output_text', ''))) if not isinstance(text, str): return False leading_text.append(text) if not content: return False actual_text = '\n'.join(leading_text) elif isinstance(content, str): actual_text = content else: return False if ( isinstance(content, str) and " ".join(actual_text.split()) == " ".join(expected_text.split()) ): return True return _submitted_user_text_matches(actual_text, expected_text) def _submitted_user_text_matches(actual_text, expected_text): """Match exact prompt text with at most its generated workspace prefix.""" if actual_text == expected_text: return True if not isinstance(actual_text, str) or not isinstance(expected_text, str): return False if not actual_text.endswith(expected_text): return False prefix = actual_text[:-len(expected_text)] if expected_text else actual_text return bool( prefix and ( _WORKSPACE_PREFIX_RE.fullmatch(prefix) or _LEGACY_WORKSPACE_PREFIX_RE.fullmatch(prefix) ) ) def _owner_projection_current_turn_row(messages, identity): messages = list(messages or []) if not isinstance(identity, dict): return None for message in messages: if _active_turn_token_matches(message, identity): return copy.deepcopy(message) return None def _find_active_turn_checkpoint_index(result_messages, previous_context, identity, msg_text): """Locate the current turn's user row inside ``result_messages``. Exactly one declared index domain is supported. The WebUI token is the strongest proof and wins when it survives the Agent projection. Otherwise the Agent-resolved ``current_turn_user_idx`` addresses ``result["messages"]`` directly, so only that exact index is validated. With a repeated prompt a shifted probe (``current_turn_user_idx - len(previous_context)``) can land on an identical historical user row and leak historical assistant prose into the current turn, so no alternative projection is ever probed. If a shifted projection is ever genuinely required, the Agent must carry an explicit projection-origin/base-length discriminator instead. ``previous_context`` is retained for call-site compatibility only; it does not participate in index resolution. """ del previous_context # single declared index domain: result_messages only result_messages = list(result_messages or []) if not isinstance(identity, dict): return None if _active_turn_has_checkpoint(result_messages, identity): for idx, message in enumerate(result_messages): if _active_turn_token_matches(message, identity): return idx if not _active_turn_boundary_is_valid(identity): return None idx = identity['current_turn_user_idx'] if idx < 0 or idx >= len(result_messages): return None message = result_messages[idx] expected_text = identity.get('text') if identity.get('text') is not None else msg_text if _active_turn_user_text_matches(message, expected_text): return idx trusted_agent_input = identity.get('trusted_agent_input_text') if ( identity.get('text') == msg_text and isinstance(trusted_agent_input, str) and trusted_agent_input != expected_text and _active_turn_user_text_matches(message, trusted_agent_input) ): return idx return None def _active_turn_boundary(result_messages, previous_context, identity, msg_text): """Index in ``result_messages`` where the current turn starts (0 = all current). Proof: active-turn token/index authority, else the last prompt-matching user row AT OR AFTER a content-matching previous-context prefix; else 0 (fail closed). """ result_messages = list(result_messages or []) if not result_messages: return 0 checkpoint_idx = _find_active_turn_checkpoint_index( result_messages, previous_context, identity, msg_text, ) if checkpoint_idx is not None: return checkpoint_idx # A content-only prefix is NOT ownership proof: _message_identity ignores # ids/timestamps, so a compacted current-only result can echo old context. previous_context = list(previous_context or []) candidate_start = 0 if ( previous_context and len(result_messages) > len(previous_context) and _messages_have_prefix(result_messages, previous_context) ): candidate_start = len(previous_context) expected_text = identity.get('text') if isinstance(identity, dict) else None expected = _normalize_user_text(expected_text if expected_text is not None else msg_text) if expected: for idx in range(len(result_messages) - 1, candidate_start - 1, -1): message = result_messages[idx] if ( isinstance(message, dict) and message.get('role') == 'user' and _normalize_user_text(_message_text(message.get('content'))) == expected ): return idx return 0 def _materialize_active_turn_user(identity, msg_text, source): checkpoint = identity.get('checkpoint') if isinstance(identity, dict) else None message = ( copy.deepcopy(checkpoint) if isinstance(checkpoint, dict) and checkpoint.get('role') == 'user' else { 'role': 'user', 'content': identity.get('text') if isinstance(identity, dict) else msg_text, } ) if isinstance(identity, dict): if identity.get('timestamp') is not None and message.get('timestamp') is None: message['timestamp'] = identity['timestamp'] if identity.get('attachments'): message['attachments'] = copy.deepcopy(identity['attachments']) stamp_message_source( message, identity.get('source') or source or 'webui', active_turn_token=identity.get('token'), ) if str(identity.get('source') or source or '').strip().lower() == 'fork': child_session_id = identity.get('session_id') if child_session_id: message['_fork_child_turn'] = child_session_id else: stamp_message_source( message, source, ) return message def _settle_current_turn_boundary(previous_context, result_messages, identity, msg_text, source): """Insert the pending turn before assistant/tool output when it is absent.""" result_messages = list(result_messages or []) if not result_messages or not isinstance(identity, dict): return result_messages _checkpoint_idx = _find_active_turn_checkpoint_index( result_messages, previous_context, identity, msg_text, ) if _checkpoint_idx is not None: existing_checkpoint = result_messages[_checkpoint_idx] _mark_active_turn_checkpoint(existing_checkpoint, identity) checkpoint = identity.get('checkpoint') if isinstance(checkpoint, dict): for key in ('id', 'timestamp'): if existing_checkpoint.get(key) is None and checkpoint.get(key) is not None: existing_checkpoint[key] = copy.deepcopy(checkpoint[key]) if checkpoint.get('attachments'): existing_checkpoint['attachments'] = copy.deepcopy(checkpoint['attachments']) stamp_message_source( existing_checkpoint, identity.get('source') or source or 'webui', active_turn_token=identity.get('token'), ) return result_messages previous_context = list(previous_context or []) if _messages_have_prefix(result_messages, previous_context): insert_at = len(previous_context) elif _active_turn_boundary_is_valid(identity): insert_at = identity['current_turn_user_idx'] - len(previous_context) if insert_at < 0 or insert_at > len(result_messages): insert_at = identity['current_turn_user_idx'] if insert_at < 0 or insert_at > len(result_messages): insert_at = None else: insert_at = None if insert_at is None and all( _is_context_compression_marker(message) or (isinstance(message, dict) and message.get('role') in ('assistant', 'tool')) for message in result_messages ): insert_at = 0 while insert_at < len(result_messages) and _is_context_compression_marker(result_messages[insert_at]): insert_at += 1 if insert_at is None: return result_messages return ( result_messages[:insert_at] + [_materialize_active_turn_user(identity, msg_text, source)] + result_messages[insert_at:] ) def _align_current_turn_display(previous_display, previous_context, identity): """Make a context-only exact checkpoint visible before shared settlement.""" display = list(previous_display or []) context = list(previous_context or []) if not isinstance(identity, dict): return display, context display, _ = _mark_active_turn_checkpoint_in_history( display, identity, identity.get('text'), allow_index_fallback=False, ) if _active_turn_has_checkpoint(display, identity): return display, context checkpoint = _owner_projection_current_turn_row( context, identity, ) if checkpoint is None and identity.get('token'): checkpoint = _materialize_active_turn_user( identity, identity.get('text'), identity.get('source') or 'webui', ) context.append(copy.deepcopy(checkpoint)) if checkpoint is not None and not _active_turn_has_checkpoint(display, identity): display.append(copy.deepcopy(checkpoint)) return display, context def _prepare_marker_clean_writeback( previous_context_messages, result_messages, active_turn_identity=None, msg_text=None, ): """Return marker-cleaned rows, next context rows, nudge provenance, boundary.""" cleaned, has_verification_nudge = _clean_synthetic_control_messages_with_provenance( result_messages ) # Same internal-control class, second home: a consumed mid-turn /steer is # appended to the turn's last tool result wrapped in # [OUT-OF-BAND USER MESSAGE ...] ... [/OUT-OF-BAND USER MESSAGE]. Strip it # here, on the rows both writebacks are built from, so neither # session.messages (rendered verbatim) nor session.context_messages keeps # the raw wrapper. Stripping the incoming rows too keeps them identity-equal # to the marker-free rows persisted by earlier turns. (#7600) cleaned = _strip_oob_markers_from_messages(cleaned) provenance = { 'verification_nudge_seen': has_verification_nudge, 'active_turn_identity': copy.deepcopy(active_turn_identity), } if not result_messages: return ( cleaned, list(previous_context_messages or []), provenance, 0, ) if cleaned: # The boundary is resolved BEFORE any restoration and reused by every # restore below, so the contract is decided once per settle. boundary = _active_turn_boundary( cleaned, previous_context_messages, active_turn_identity, msg_text, ) return ( cleaned, _restore_reasoning_metadata_before_boundary( previous_context_messages, cleaned, boundary, ), provenance, boundary, ) return [], list(previous_context_messages or []), provenance, 0 def _annotate_media_snapshots_for_settled_messages(messages) -> None: """Freeze local-file MEDIA: bytes at settle time so historical previews keep showing the version the turn emitted, even after the file is overwritten in place (#6922 follow-up). Runs AFTER the display merge so the annotation rides the exact messages the frontend will render. The store is content-addressed and the annotator is idempotent (already-stored digests short-circuit), so re-settling the same transcript is cheap. Never raises — snapshotting is best-effort durability. """ try: from api.media_snapshots import annotate_media_snapshots annotate_media_snapshots(messages) except Exception: logger.debug("Media snapshot annotation failed during settle", exc_info=True) def _settle_result_messages( session, previous_messages, previous_context_messages, result_messages, msg_text, source, active_turn_identity, ): ( result_messages, next_context_messages, verification_nudge_provenance, current_turn_boundary, ) = _prepare_marker_clean_writeback( previous_context_messages, result_messages, active_turn_identity, msg_text, ) if result_messages: _assign_stable_message_ids( result_messages, previous_messages, previous_context_messages, ) next_context_messages = _dedupe_replayed_context_messages( previous_context_messages, next_context_messages, msg_text, ) next_context_messages = _settle_current_turn_boundary( previous_context_messages, next_context_messages, active_turn_identity, msg_text, source, ) session.context_messages = ( _deduplicate_context_messages(next_context_messages) if result_messages else list(next_context_messages or []) ) if result_messages: session.context_messages = _settle_current_turn_boundary( previous_context_messages, session.context_messages, active_turn_identity, msg_text, source, ) previous_display_for_writeback, session.context_messages = _align_current_turn_display( previous_messages, session.context_messages, active_turn_identity, ) session.messages = _merge_display_messages_after_agent_result( previous_display_for_writeback, previous_context_messages, _restore_display_reasoning_metadata( previous_messages, result_messages, current_turn_boundary=current_turn_boundary, ), msg_text, source=source, verification_nudge_provenance=verification_nudge_provenance, ) trusted_agent_input = ( active_turn_identity.get('trusted_agent_input_text') if isinstance(active_turn_identity, dict) else None ) if ( isinstance(trusted_agent_input, str) and active_turn_identity.get('text') == msg_text and active_turn_identity.get('token') ): current_display_rows = [ message for message in session.messages if _active_turn_token_matches(message, active_turn_identity) ] current_context_rows = [ message for message in session.context_messages if _active_turn_token_matches(message, active_turn_identity) ] if ( len(current_display_rows) == 1 and current_display_rows[0].get('content') == msg_text and len(current_context_rows) == 1 ): context_user = current_context_rows[0] context_content = context_user.get('content') if ( isinstance(context_content, list) and any( isinstance(part, dict) and str(part.get('type') or '').lower() in {'image', 'image_url', 'input_image'} for part in context_content ) and _active_turn_user_text_matches( context_user, trusted_agent_input, ) ): context_user[_WEBUI_TRUSTED_AGENT_INPUT_FIELD] = trusted_agent_input # The merge carries earlier display rows across turns verbatim, so a row # settled before this guard existed would keep its raw wrapper forever. # Scrub the persisted display copy too — after the merge, so identity # matching above still saw the rows unchanged. (#7600) session.messages = _strip_oob_markers_from_messages(session.messages) _annotate_media_snapshots_for_settled_messages(session.messages) _compact_session_image_parts_for_persistence(session) _advance_truncation_watermark_after_commit(session) # #3831 return result_messages def _current_turn_already_has_visible_assistant_answer(messages, *, active_turn_identity=None): """Return True only when the token-owned current turn already has visible assistant prose.""" if not isinstance(active_turn_identity, dict) or not active_turn_identity.get('token'): return False current_turn_seen = False for msg in messages or []: if not isinstance(msg, dict) or _is_synthetic_control_message(msg): continue if _active_turn_token_matches(msg, active_turn_identity): current_turn_seen = True continue if not current_turn_seen: continue role = msg.get('role') if role == 'assistant' and _assistant_message_has_final_visible_text(msg) and not msg.get('_error'): return True if role == 'user': return False return False def _trusted_native_image_context_display_mirror_tokens(previous_display, previous_context): """Find unique token-paired rich image rows already represented in display.""" display_by_token = {} context_by_token = {} for messages, rows_by_token in ( (previous_display, display_by_token), (previous_context, context_by_token), ): for message in messages or []: if not isinstance(message, dict) or message.get('role') != 'user': continue token = message.get('_active_turn_token') if not isinstance(token, str) or not token: continue rows_by_token.setdefault(token, []).append(message) mirrored_tokens = set() for token, context_rows in context_by_token.items(): display_rows = display_by_token.get(token, []) if len(context_rows) != 1 or len(display_rows) != 1: continue context, display = context_rows[0], display_rows[0] content = context.get('content') if ( not isinstance(content, list) or not any( isinstance(part, dict) and str(part.get('type') or '').lower() in {'image', 'image_url', 'input_image'} for part in content ) or not isinstance( context.get(_WEBUI_TRUSTED_AGENT_INPUT_FIELD), str, ) or not _active_turn_user_text_matches( context, context.get(_WEBUI_TRUSTED_AGENT_INPUT_FIELD), ) or not _message_private_identity_compatible(display, context) ): continue display_ts, display_ts_valid = _message_exact_timestamp_details(display) context_ts, context_ts_valid = _message_exact_timestamp_details(context) if ( not display_ts_valid or not context_ts_valid or display_ts is None or context_ts is None or display_ts != context_ts ): continue mirrored_tokens.add(token) return mirrored_tokens def _agent_result_tool_limit_reached(result) -> bool: """Return True when current-turn metadata says the tool iteration cap fired.""" if not isinstance(result, dict): return False fields = [ result.get('turn_exit_reason'), result.get('terminal_reason'), result.get('status'), result.get('state'), result.get('error'), ] haystack = " ".join(str(value or '') for value in fields).lower() if ( 'max_iterations_reached' in haystack or 'maximum number of tool-calling iterations' in haystack or ('tool-calling iterations' in haystack and 'maximum' in haystack) ): return True return False def _maybe_inject_max_iteration_summary_fallback(messages, result) -> list: """Append the agent's graceful summary text as an assistant turn when one is missing. When ``AIAgent`` exhausts its iteration budget, ``agent.handle_max_iterations`` always returns a non-empty ``final_response`` — either the model-generated summary or a graceful fallback (e.g. ``"I reached the iteration limit and couldn't generate a summary."``). Hermes Agent surfaces that string as the final answer to the user; the WebUI, by contrast, reads only ``messages``, so an empty summary (common with reasoning-only responses) left the user with a bare ``tool_limit_reached`` error instead of any closure text. When ``_tool_limit_reached`` is true and ``messages`` ends without a final assistant answer, inject ``result['final_response']`` as a new assistant turn so ``_mark_latest_assistant_tool_limit_status`` can attach the status card in the normal flow and the user sees the same closure text as hermes-agent. Returns the (possibly new) messages list; does nothing when a usable assistant answer already exists or when ``result`` carries no graceful fallback text. """ if not isinstance(result, dict): return list(messages or []) fallback = result.get('final_response') if not isinstance(fallback, str) or not fallback.strip(): return list(messages or []) out = list(messages or []) if not _session_lacks_final_assistant_answer(out): return out # Append a synthetic summary turn. Tag it so downstream consumers can # distinguish it from model-emitted assistant turns if needed; mirrors the # synthetic-scaffolding flag convention already used elsewhere (#5334). out.append({"role": "assistant", "content": fallback, "_max_iteration_summary_fallback": True}) return out def _mark_latest_assistant_tool_limit_status(messages) -> bool: """Annotate the latest usable assistant final answer as limit-stopped.""" for msg in reversed(list(messages or [])): if not isinstance(msg, dict): continue if msg.get('_error') or msg.get('role') != 'assistant': continue content = msg.get('content') if isinstance(content, list): text = '\n'.join( str(part.get('text') or part.get('content') or '') for part in content if isinstance(part, dict) ) else: text = str(content or '') if msg.get('tool_calls') or not text.strip(): continue msg['_terminal_state'] = 'tool_limit_reached' msg['_terminal_reason'] = 'max_iterations' msg.setdefault('_statusCard', { 'title': 'Tool iteration limit reached', 'subtitle': 'Stopped because the tool iteration limit was reached.', 'rows': [ {'label': 'State', 'value': 'Limit reached'}, {'label': 'Next step', 'value': 'Start a new turn to continue.'}, ], }) return True return False def _session_has_cancel_marker(session) -> bool: """Return True if a visible cancel/interrupted marker is already persisted.""" for msg in reversed(getattr(session, 'messages', None) or []): if not isinstance(msg, dict): continue if msg.get('role') == 'user': return False if msg.get('role') != 'assistant': continue content = msg.get('content') text = '' if isinstance(content, str): text = content elif isinstance(content, list): parts = [] for part in content: if isinstance(part, dict): parts.append(str(part.get('text') or part.get('content') or '')) text = '\n'.join(parts) normalized = text.strip().lower() if any(pattern in normalized for pattern in _CANCEL_MARKER_PATTERNS): return True return False def _cancelled_turn_content(message: str = 'Task cancelled.', agent_name: str | None = None) -> str: """Return cancelled-turn copy matching the verbose provider-error layout.""" _message = str(message or 'Task cancelled.').strip() if not _message.endswith('.'): _message += '.' return ( f"**Task cancelled:** {_message}\n\n" f"*{_cancelled_turn_hint(agent_name)}*" ) def _persist_cancelled_turn(session, *, message: str = 'Task cancelled.') -> None: """Persist a user-cancelled terminal state without provider-error wording. cancel_stream() usually writes this marker first, but the streaming thread can later unwind through the silent-failure or exception path. Those paths must not append a misleading provider no-response error after an explicit cancel. """ _materialize_pending_user_turn_before_error(session) session.active_stream_id = None session.pending_user_message = None session.pending_attachments = [] session.pending_started_at = None session.pending_user_source = None if not _session_has_cancel_marker(session): agent_name = _preferred_agent_display_name_for_session(session) session.messages.append({ 'role': 'assistant', 'content': _cancelled_turn_content(message, agent_name), '_error': True, 'provider_details': str(message or 'Task cancelled.').strip(), 'provider_details_label': 'Cancellation details', 'timestamp': int(time.time()), }) def _cleanup_ephemeral_cancelled_turn(session) -> None: """Remove transient /btw session state after a cancel without saving it.""" session.active_stream_id = None session.pending_user_message = None session.pending_attachments = [] session.pending_started_at = None session.pending_user_source = None try: import pathlib pathlib.Path(session.path).unlink(missing_ok=True) except Exception: logger.debug("Failed to clean up ephemeral cancelled session", exc_info=True) def _resolve_current_session_for_write(session): """Resolve the CURRENT session object for a delayed-cancel write. The worker thread holds a snapshot ``Session`` object captured when the turn started. ``cancel_stream()`` clears ``active_stream_id`` on the object it resolves and saves it, after which the session is eligible for LRU eviction; a successor turn may then be admitted through a DISTINCT object lazily reloaded by ``get_session()``. Writing to the worker's snapshot after that point serializes stale state over the successor (#6623 re-gate), so delayed-cancel writes MUST target the object that ``get_session()`` currently resolves under the canonical per-session lock. Returns ``None`` (fail closed) when the current object cannot be resolved — missing/deleted session, transient error, or no session id. Callers MUST no-op on ``None`` and must NEVER fall back to the worker-held snapshot: that object may be a detached LRU-evicted generation whose save would overwrite the current session state. """ sid = getattr(session, "session_id", None) if not sid: return None try: return get_session(sid) except Exception: logger.debug( "Failed to resolve current session %s for delayed-cancel write; " "failing closed (no write)", sid, exc_info=True, ) return None def _merge_process_wakeup_pause_into_current_session( session, *, classification: str, model=None, provider=None, ) -> dict | None: """Record a process-wakeup provider-unavailable pause on the CURRENT session. The worker-held ``session`` may be a detached snapshot (LRU-evicted and replaced by a distinct object through which a successor was admitted and saved). The pause is session-wide suppression metadata that must survive regardless of stream ownership, so it is merged into the current object — resolved under the canonical per-session lock — and saved there. Saving the worker's snapshot instead would serialize stale state over the successor even when the generation-guarded finalizer later no-ops (#6623 re-gate). If the current object cannot be resolved the pause is skipped (fail closed) — it is never written through the worker-held snapshot. """ current = _resolve_current_session_for_write(session) if current is None: logger.info( "Skipping process-wakeup pause for unresolvable session %s " "(fail closed — no detached-snapshot save)", getattr(session, "session_id", None), ) return None recorded = record_process_wakeup_provider_unavailable_pause( current, classification=classification, model=model, provider=provider, ) if recorded: try: current.save(touch_updated_at=False) except Exception: logger.debug("Failed to save process-wakeup pause", exc_info=True) return recorded def _finalize_cancelled_turn( session, *, ephemeral: bool = False, message: str = 'Task cancelled.', stream_id: str | None = None, ) -> None: """Finalize a cancelled turn for persistent or ephemeral sessions. Generation-guarded (#6623 re-gate): when ``stream_id`` is provided, the finalizer only acts while the session has not advanced past that stream. The worker-held ``session`` object is only a snapshot: cancel_stream() clears ``active_stream_id`` eagerly, and the session may have been LRU-evicted and lazily reloaded as a DISTINCT object through which a successor turn was admitted and saved. Finalization authority is therefore bound to the per-session writeback-ownership record (``SESSION_WRITEBACK_OWNERS``), which survives cancel cleanup, AND to the CURRENT session object resolved by canonical session id under the per-session agent lock (callers hold it). A successor owns the session now and a delayed finalizer from the old worker MUST no-op against it instead of clearing pending fields, materializing the pending user turn, appending markers, unlinking the session, or saving. ``active_stream_id is None`` on a worker-held snapshot is NOT proof that no successor exists — cancel cleanup clears that field eagerly. And a MISSING ownership record is NOT proof that the session is idle: the successor's own teardown clears the record when it completes, so ``owner is None`` must fail closed too. """ if stream_id: session_id = getattr(session, "session_id", None) current = _resolve_current_session_for_write(session) if session_id else None if current is None: logger.info( "Skipping stale cancelled-turn finalize for session %s stream %s; " "current session cannot be resolved (fail closed)", session_id, stream_id, ) return # Immutable stream/generation authority: the writeback-ownership record # survives cancel cleanup and is replaced only by a successor admission, # so authority must EQUAL this worker's exact stream_id. ``None`` — # successor completed and its teardown cleared the entry, record never # registered, or already reaped/deleted — must FAIL CLOSED: it is not # proof that no successor exists, and the old worker would otherwise # append/save its obsolete cancellation over the completed successor. owner = session_writeback_owner(session_id) if session_id else None if owner != stream_id: logger.info( "Skipping stale cancelled-turn finalize for session %s stream %s; " "writeback owner=%r (missing, replaced, or unresolvable — fail closed)", session_id, stream_id, owner, ) return _current = getattr(current, "active_stream_id", None) if _current is not None and _current != stream_id: logger.info( "Skipping stale cancelled-turn finalize for session %s stream %s; " "active_stream_id=%s (successor owns the writeback)", session_id, stream_id, _current, ) return # Finalize against the CURRENT object — never the worker's snapshot. session = current if ephemeral: _cleanup_ephemeral_cancelled_turn(session) return _persist_cancelled_turn(session, message=message) try: session.save() except Exception: logger.debug("Failed to persist cancelled turn", exc_info=True) def _aiagent_import_error_detail() -> str: """Return a multi-line diagnostic string for the "AIAgent not available" path. The bare ImportError ("AIAgent not available -- check that hermes-agent is on sys.path") leaves users guessing at which python is running, where it's looking, and what to fix. We assemble the same evidence a maintainer would ask for first (issue #1695): the python that's running, the agent_dir env var if set, the sys.path entries that mention 'hermes', and the most-common fix (`pip install -e .` in the agent dir). Kept as a separate helper so it stays out of the hot path until we actually need to raise — building it on every successful import would be wasted work. """ import os as _os import sys as _sys lines = ["AIAgent not available -- check that hermes-agent is on sys.path"] lines.append("") lines.append(f" python: {_sys.executable}") agent_dir = _os.environ.get("HERMES_WEBUI_AGENT_DIR") if agent_dir: lines.append(f" HERMES_WEBUI_AGENT_DIR: {agent_dir}") else: lines.append(" HERMES_WEBUI_AGENT_DIR: (not set)") # Show only the sys.path entries that look relevant — full sys.path is noisy. relevant = [p for p in _sys.path if "hermes" in p.lower() or "agent" in p.lower()] if relevant: lines.append(" sys.path entries mentioning hermes/agent:") for entry in relevant[:6]: lines.append(f" - {entry}") if len(relevant) > 6: lines.append(f" ... and {len(relevant) - 6} more") else: lines.append(" sys.path: (no entries mention hermes or agent)") lines.append("") lines.append(" Most common fix: install the agent in editable mode so its modules") lines.append(" appear on sys.path:") lines.append("") lines.append(" cd /path/to/hermes-agent") lines.append(" pip install -e .") lines.append("") lines.append(" Then restart the WebUI.") lines.append("") lines.append(' Full troubleshooting: docs/troubleshooting.md ("AIAgent not available")') return "\n".join(lines) from api.models import get_session, title_from from api.workspace import _resolve_path # Fields that are safe to send to LLM provider APIs. # Everything else (attachments, timestamp, _ts, etc.) is display-only # metadata added by the webui and must be stripped before the API call. # `reasoning_content` is provider-facing for reasoning-capable models. Display # metadata such as `reasoning`, `thinking`, and `_reasoning` stays omitted here. _API_SAFE_MSG_KEYS = {'role', 'content', 'tool_calls', 'tool_call_id', 'name', 'refusal', 'reasoning_content'} _NATIVE_IMAGE_MAX_BYTES = 20 * 1024 * 1024 _GATEWAY_ROUTING_TOP_LEVEL_KEYS = { 'used_provider', 'used_model', 'requested_provider', 'requested_model', } _GATEWAY_ROUTING_CONTAINER_KEYS = ( 'llm_gateway', 'gateway', 'metadata', 'response_metadata', 'routing_metadata', 'usage', ) _GATEWAY_ROUTING_ATTEMPT_KEYS = { 'provider', 'model', 'status', 'reason', 'selection_reason', 'score', 'latency_ms', 'error', 'timestamp', 'selected', 'attempt', 'attempt_index', } def _clean_gateway_routing_scalar(value): if value is None: return None if isinstance(value, (str, int, float, bool)): text = str(value).strip() if not text: return None return value if isinstance(value, (int, float, bool)) else text[:240] return None def _find_gateway_metadata_payload(payload): if not isinstance(payload, dict): return None if any(k in payload for k in _GATEWAY_ROUTING_TOP_LEVEL_KEYS) or isinstance(payload.get('routing'), list): return payload for key in _GATEWAY_ROUTING_CONTAINER_KEYS: nested = payload.get(key) found = _find_gateway_metadata_payload(nested) if found: return found return None def _normalize_gateway_routing_metadata(payload, requested_model=None, requested_provider=None): """Return safe LLM Gateway routing metadata, or None when absent. LLM Gateway response metadata can contain provider/model routing details, but WebUI must only persist display-safe scalars and a bounded routing list. Secrets or provider-specific request objects are deliberately ignored. """ src = _find_gateway_metadata_payload(payload) if not src: return None normalized = {} for key in _GATEWAY_ROUTING_TOP_LEVEL_KEYS: value = _clean_gateway_routing_scalar(src.get(key)) if value is not None: normalized[key] = value if 'requested_model' not in normalized: fallback_model = _clean_gateway_routing_scalar(requested_model) if fallback_model is not None: normalized['requested_model'] = fallback_model if 'requested_provider' not in normalized: fallback_provider = _clean_gateway_routing_scalar(requested_provider) if fallback_provider is not None: normalized['requested_provider'] = fallback_provider routing = [] raw_routing = src.get('routing') if isinstance(raw_routing, list): for attempt in raw_routing[:12]: if not isinstance(attempt, dict): continue clean_attempt = {} for key in _GATEWAY_ROUTING_ATTEMPT_KEYS: value = _clean_gateway_routing_scalar(attempt.get(key)) if value is not None: clean_attempt[key] = value if clean_attempt: routing.append(clean_attempt) if routing: normalized['routing'] = routing used_provider = str(normalized.get('used_provider') or '').strip().lower() requested_provider_norm = str(normalized.get('requested_provider') or '').strip().lower() used_model = str(normalized.get('used_model') or '').strip().lower() requested_model_norm = str(normalized.get('requested_model') or '').strip().lower() provider_changed = bool(used_provider and requested_provider_norm and used_provider != requested_provider_norm) model_changed = bool(used_model and requested_model_norm and used_model != requested_model_norm) attempted_providers = [ str(a.get('provider') or '').strip().lower() for a in routing if a.get('provider') ] distinct_attempted_providers = {p for p in attempted_providers if p} failed_before_selection = any( str(a.get('status') or '').strip().lower() in {'failed', 'error', 'timeout', 'rejected'} for a in routing ) has_failover = bool(provider_changed or len(distinct_attempted_providers) > 1 or failed_before_selection) if not ( normalized.get('used_provider') or normalized.get('used_model') or routing or provider_changed or model_changed ): return None normalized['provider_changed'] = provider_changed normalized['model_changed'] = model_changed normalized['has_failover'] = has_failover return normalized def _extract_gateway_routing_metadata(agent, result, requested_model=None, requested_provider=None): candidates = [] if isinstance(result, dict): candidates.extend([ result.get('llm_gateway'), result.get('gateway'), result.get('metadata'), result.get('response_metadata'), result.get('routing_metadata'), result.get('usage'), result, ]) for attr in ( 'llm_gateway_metadata', 'gateway_metadata', 'last_response_metadata', 'response_metadata', 'routing_metadata', 'last_usage', ): if agent is not None: candidates.append(getattr(agent, attr, None)) for candidate in candidates: normalized = _normalize_gateway_routing_metadata( candidate, requested_model=requested_model, requested_provider=requested_provider, ) if normalized: return normalized return None def _build_agent_thread_env(profile_runtime_env: dict | None, workspace: str, session_id: str, profile_home: str) -> dict: """Build thread-local agent env with per-run values overriding profile defaults. Profile runtime env may include TERMINAL_CWD from config.yaml. Passing it as **kwargs alongside an explicit TERMINAL_CWD raises TypeError before the agent starts, so merge into one dict first and let the active workspace win. """ env = dict(profile_runtime_env or {}) env.update({ 'TERMINAL_CWD': str(workspace), 'HERMES_EXEC_ASK': '1', 'HERMES_SESSION_KEY': session_id, 'HERMES_SESSION_ID': session_id, 'HERMES_SESSION_PLATFORM': 'webui', # process_complete agent-wakeup wiring (ours-original, Option B): the # terminal_tool watcher routing gate (terminal_tool.py:~1940) reads # HERMES_SESSION_CHAT_ID to populate pending_watchers for WebUI # sessions so notify_on_complete completions enqueue and the agent # can be woken. HERMES_SESSION_ID/PLATFORM come from upstream #2279. 'HERMES_SESSION_CHAT_ID': str(session_id), 'HERMES_HOME': profile_home, }) return env _streaming_hermes_home_override_available = None def _resolve_streaming_hermes_home_override(): """Return hermes_constants module if context-local home override APIs exist. Cached import-safe resolver mirrors the optional pattern used by `api.profiles` so older agent versions safely degrade to the process-global env mirror fallback. """ global _streaming_hermes_home_override_available import sys as _sys if _streaming_hermes_home_override_available is False: return None mod = _sys.modules.get('hermes_constants') if mod is None and _streaming_hermes_home_override_available is None: try: import hermes_constants # noqa: F401 mod = _sys.modules.get('hermes_constants') except Exception: _streaming_hermes_home_override_available = False return None if ( mod is not None and hasattr(mod, 'set_hermes_home_override') and hasattr(mod, 'reset_hermes_home_override') ): _streaming_hermes_home_override_available = True return mod _streaming_hermes_home_override_available = False return None def _set_streaming_hermes_home_override(profile_home: str): """Install the context-local home override if available. Returns ``(module, token, installed)`` so callers can restore it with the matching module in the inverse order. """ if not profile_home: return None, None, False _home_override_mod = _resolve_streaming_hermes_home_override() if _home_override_mod is None: return None, None, False try: _token = _home_override_mod.set_hermes_home_override(profile_home) return _home_override_mod, _token, True except Exception: logger.debug( "Failed to set streaming Hermes home override; continuing with os.environ mirror", exc_info=True, ) return None, None, False def _reset_streaming_hermes_home_override(override_mod, override_token, override_installed: bool) -> None: """Reset the context-local home override if it was installed.""" if override_mod is None or not override_installed: return try: override_mod.reset_hermes_home_override(override_token) except Exception: logger.debug("Failed to reset streaming Hermes home override", exc_info=True) # ── Per-turn session identity (xsession wakeup misroute root fix — Option 1) ─ # WebUI bound per-turn session identity ONLY to the process-global # os.environ['HERMES_SESSION_KEY'] (turn-start, line ~3263) and released the # env lock BEFORE the agent ran. WebUI never called any contextvar setter, so # gateway.session_context._SESSION_KEY stayed _UNSET and # tools.approval.get_current_session_key (the EXACT call a # notify_on_complete background spawn makes in terminal_tool.py:~1928) fell # back to that racy process-global slot. Two concurrent WebUI turns therefore # raced on one slot: session A's spawn could capture session B's id, and at # completion the server-side wakeup turn started for the WRONG session # (RCA t_f62ff1e8, agent.log:6632). The agent worker runs synchronously inside # the _run_agent_streaming thread (concurrent tool batches use # contextvars.copy_context() so children inherit this binding); binding the # context-local here makes the capture task/thread-local and race-immune. def _set_turn_session_identity(session_id: str, workspace: str = ""): """Bind THIS turn's session identity to the current (task/thread-local) context and return an opaque token for _reset_turn_session_identity. Binds four context-locals so every session-key / UI-owner consumer is covered without a race: * ``tools.approval._approval_session_key`` — checked FIRST by ``get_current_session_key`` (the exact call terminal_tool.py makes for a notify_on_complete background spawn: the bug path). * ``gateway.session_context._SESSION_KEY`` — read by direct ``get_session_env("HERMES_SESSION_KEY")`` consumers. * ``gateway.session_context._SESSION_UI_SESSION_ID`` — exact browser-tab return address stamped onto ProcessSession.origin_ui_session_id and completion events by modern hermes-agent builds. Authoritative for wakeup routing when present (see ``_resolve_completion_target``). * ``agent.runtime_cwd._SESSION_CWD`` — this turn's workspace, when *workspace* is given. The WebUI runs the agent IN-PROCESS, so ``os.getcwd()`` is the server's launch directory, not the workspace the user selected. Anything resolving a default working directory from the process therefore lands in the Hermes install tree: measured, every conductor child spawned from a WebUI session recorded ``workdir=~/.hermes/hermes-agent`` while the selected workspace was ``~/workspace``, which also fed those children the install tree's own contributor ``AGENTS.md`` as workspace doctrine. ``TERMINAL_CWD`` is already set per turn for the same purpose but is a process-global that concurrent turns overwrite; this contextvar is task-local, so it is race-immune for exactly the reason the three bindings above are. It deliberately does NOT call ``gateway.session_context.set_session_vars``: that blanket setter also zeroes the platform/chat_id/user contextvars, flipping ``HERMES_SESSION_PLATFORM`` from its env fallback (``'webui'``, still written to os.environ at turn-start) to an explicit ``""`` — which would break the ``notify_on_complete`` watcher registration gate. """ sid = str(session_id or "") tokens: dict = {} try: from api.agent_compat import agent_attr set_current_session_key = agent_attr( "tools.approval", "set_current_session_key", "tools.approval_context" ) tokens["approval"] = set_current_session_key(sid) except Exception: logger.debug("per-turn approval session-key bind failed", exc_info=True) try: from gateway.session_context import _SESSION_KEY as _SK tokens["session_key"] = _SK.set(sid) except Exception: logger.debug("per-turn _SESSION_KEY bind failed", exc_info=True) try: from gateway.session_context import _SESSION_UI_SESSION_ID as _UI_SID tokens["ui_session_id"] = _UI_SID.set(sid) except Exception: logger.debug("per-turn _SESSION_UI_SESSION_ID bind failed", exc_info=True) if workspace: # Bound via the ContextVar directly, not ``set_session_cwd``, so the # reset below can use reset-token semantics like every sibling above. # ``clear_session_cwd()`` would instead pin "" and mask the CLI/cron # env fallback for a reused thread-pool worker. try: from agent.runtime_cwd import _SESSION_CWD as _SCWD tokens["session_cwd"] = _SCWD.set(str(workspace)) except Exception: logger.debug("per-turn _SESSION_CWD bind failed", exc_info=True) return tokens def _reset_turn_session_identity(tokens) -> None: """Restore the context-locals bound by ``_set_turn_session_identity`` via contextvars reset-token semantics. Reset-token (not a blanket clear) is the canonical idiom: it composes correctly under nesting and restores ``_UNSET`` for the top-level turn so a reused thread-pool worker leaks no identity and CLI/cron env fallback resumes. Order mirrors the bind in reverse. """ if not tokens: return tok = tokens.get("session_cwd") if tok is not None: try: from agent.runtime_cwd import _SESSION_CWD as _SCWD _SCWD.reset(tok) except Exception: logger.debug("per-turn _SESSION_CWD reset failed", exc_info=True) tok = tokens.get("ui_session_id") if tok is not None: try: from gateway.session_context import _SESSION_UI_SESSION_ID as _UI_SID _UI_SID.reset(tok) except Exception: logger.debug("per-turn _SESSION_UI_SESSION_ID reset failed", exc_info=True) tok = tokens.get("session_key") if tok is not None: try: from gateway.session_context import _SESSION_KEY as _SK _SK.reset(tok) except Exception: logger.debug("per-turn _SESSION_KEY reset failed", exc_info=True) tok = tokens.get("approval") if tok is not None: try: from api.agent_compat import agent_attr reset_current_session_key = agent_attr( "tools.approval", "reset_current_session_key", "tools.approval_context" ) reset_current_session_key(tok) except Exception: logger.debug("per-turn approval session-key reset failed", exc_info=True) @contextlib.contextmanager def _bind_turn_session_identity(session_id: str): """Context-manager form of the per-turn session-identity binding. The ``_run_agent_streaming`` worker uses the explicit ``_set``/``_reset`` pair directly because its single ``try/finally`` already spans the whole turn (~2k lines) and the binding must cover every mid-turn background spawn; this wrapper is the canonical single-call API for other callers and for tests, and shares the exact same code path. """ tokens = _set_turn_session_identity(session_id) try: yield finally: _reset_turn_session_identity(tokens) def _stale_completion_max_age_seconds() -> float: """Max age (seconds) a background-process completion may sit in the queue before the WebUI drain treats it as stale and drops it instead of prepending it to the user's next turn. Completions older than this are silently consumed (not requeued) so a notification that finally fires long after the user moved on cannot contaminate an unrelated later turn. See nesquena/hermes-webui#4029. Configurable via HERMES_WEBUI_STALE_COMPLETION_MAX_AGE_SECONDS. A value of 0 (or negative) disables age-gating and restores the legacy drain-all behavior. Defaults to 6 hours. """ raw = os.environ.get("HERMES_WEBUI_STALE_COMPLETION_MAX_AGE_SECONDS") if raw is not None: try: return float(raw) except (TypeError, ValueError): logger.warning( "Invalid HERMES_WEBUI_STALE_COMPLETION_MAX_AGE_SECONDS=%r; using default", raw, ) return 6 * 60 * 60 # 6 hours def _format_process_notification(evt: dict) -> str: """Format a completed background process notification for agent input.""" if not isinstance(evt, dict): return '' if evt.get('type') == 'async_delegation': try: from tools.process_registry import format_process_notification return format_process_notification(evt) or '' except Exception: logger.debug("Failed to format async delegation notification", exc_info=True) return '' if evt.get('type') != 'completion': return '' _sid = evt.get('session_id', '') _cmd = evt.get('command', '') _exit = evt.get('exit_code', '') _out = evt.get('output') or '' if len(_out) > 4000: _out = _out[:4000] + '\n... (truncated)' return ( f"[IMPORTANT: Background process {_sid} completed (exit code {_exit}).\n" f"Command: {_cmd}\n" f"Output:\n{_out}]" ) def _mark_process_completion_consumed(process_registry, process_id: str) -> None: """Best-effort bridge to the agent registry's private completion marker.""" try: with process_registry._lock: process_registry._completion_consumed.add(process_id) except Exception: logger.debug("Failed to mark process completion consumed", exc_info=True) def _completion_event_targets_webui_session(evt_session_key: str, session_id: str) -> bool: """Return whether a completion event belongs to this WebUI session. WebUI normally registers ``PROCESS_SESSION_INDEX[session_id] = session_id``. Gateway/agent session keys can differ, so match the direct WebUI case first and otherwise resolve through the same session-key index used by the background wakeup path. """ if not evt_session_key or not session_id: return False if evt_session_key == session_id: return True try: with PROCESS_SESSION_INDEX_LOCK: return PROCESS_SESSION_INDEX.get(evt_session_key) == session_id except Exception: logger.debug("Failed to resolve completion event session key", exc_info=True) return False def _drain_webui_process_notifications( session_id: str, *, pending_async_acceptances: list | None = None, ) -> list[str]: """Return completion notifications that belong to this WebUI session. The agent registry completion queue is process-wide and events do not carry the WebUI session key directly. Look up the live process session before delivery so completions from other tabs remain queued for their owners. """ if not session_id: return [] try: from tools.process_registry import process_registry except Exception: return [] notifications: list[str] = [] skipped_events: list[dict] = [] async_retry_events: list[tuple[dict, bool]] = [] completion_queue = getattr(process_registry, 'completion_queue', None) if completion_queue is None: return [] # Computed once per drain (not per event): reads/validates the env cap a # single time so an invalid value logs at most one warning per drain. stale_completion_max_age = _stale_completion_max_age_seconds() while True: try: evt = completion_queue.get_nowait() except queue.Empty: break except Exception: logger.debug("Failed to drain process completion queue", exc_info=True) break evt_sid = completion_delivery_id(evt) if isinstance(evt, dict) else '' if not evt_sid: skipped_events.append(evt) continue is_async_delegation = ( isinstance(evt, dict) and evt.get('type') == 'async_delegation' ) try: if ( not is_async_delegation and process_registry.is_completion_consumed(evt_sid) ): continue evt_session_key = str(evt.get('session_key') or '') if isinstance(evt, dict) else '' evt_origin_ui_session_id = ( str(evt.get('origin_ui_session_id') or '') if isinstance(evt, dict) else '' ) if not evt_session_key or not evt_origin_ui_session_id: proc = process_registry.get(evt_sid) if not evt_session_key: evt_session_key = str(getattr(proc, 'session_key', '') or '') if not evt_origin_ui_session_id: evt_origin_ui_session_id = ( str(getattr(proc, 'origin_ui_session_id', '') or '') or str(getattr(proc, 'spawn_session_id', '') or '') ) except Exception: evt_session_key = '' evt_origin_ui_session_id = '' # origin_ui_session_id is the exact, immutable return address and is # authoritative over the mutable session-key index (mirrors the # background _process_one path via _resolve_completion_target). When it # is present, this drain claims/ACKs the event ONLY for the origin # session — otherwise the next-turn drain could win the shared-queue # race and deliver+ACK a completion to the wrong (session-key-index) # session, leaving the true origin empty. Fall back to the session-key # target check only for legacy events that carry no origin address. if evt_origin_ui_session_id: if evt_origin_ui_session_id != session_id: skipped_events.append(evt) continue elif not _completion_event_targets_webui_session(evt_session_key, session_id): skipped_events.append(evt) continue # Age-gate stale completions: a completion that fires long after the # user moved on must not be prepended to an unrelated later turn # (nesquena/hermes-webui#4029). Drop (consume, do not requeue) any # completion whose enqueue time is older than the configured cap. # Events without a 'completed_at' (older agent builds) are never # dropped here, preserving backward-compatible behavior. is_stale = False stale_age = 0.0 if stale_completion_max_age > 0 and isinstance(evt, dict): completed_at = evt.get('completed_at') if isinstance(completed_at, (int, float)) and completed_at > 0: stale_age = time.time() - completed_at is_stale = stale_age > stale_completion_max_age if is_async_delegation: try: claim = claim_async_delegation_delivery(evt, "webui-next-turn") except Exception: skipped_events.append(evt) continue if claim is None: schedule_async_delegation_claim_retry(evt, completion_queue) continue notification_added = False try: if is_stale: notification = '' else: notification = _format_process_notification(evt) if not notification: raise ValueError( "async delegation formatter returned an empty notification" ) if notification: notifications.append(notification) notification_added = True if is_stale: # Stale async events are an explicit terminal disposition. complete_async_delegation_delivery(evt, claim) elif pending_async_acceptances is not None: pending_async_acceptances.append( (evt, claim, notification, completion_queue) ) else: # Direct callers without a live agent turn retain the # historical synchronous acceptance behavior used by # CLI-style drains. complete_async_delegation_delivery(evt, claim) except Exception: if notification_added: notifications.pop() release_async_delegation_delivery(evt, claim) async_retry_events.append( (evt, bool(getattr(claim, "durable", False))) ) logger.warning( "Failed to accept async delegation completion for session %s", session_id, exc_info=True, ) continue if is_stale: logger.info( "Dropping stale async-delegation completion for session %s " "(age %.0fs > cap %.0fs)", evt_sid, stale_age, stale_completion_max_age, ) continue if is_stale: logger.info( "Dropping stale background-process completion for " "session %s (age %.0fs > cap %.0fs)", evt_sid, stale_age, stale_completion_max_age, ) _mark_process_completion_consumed(process_registry, evt_sid) continue notification = _format_process_notification(evt) if notification: notifications.append(notification) # Matched but unformattable process completions are consumed rather than # replayed forever on later turns. _mark_process_completion_consumed(process_registry, evt_sid) for evt, durable in async_retry_events: requeue_async_delegation_event( evt, completion_queue, durable=durable, ) for evt in skipped_events: try: completion_queue.put(evt) except Exception: logger.debug("Failed to requeue process completion event", exc_info=True) break return notifications def _accept_pending_async_delegations( pending_async_acceptances: list, *, session_id: str, ) -> list[str]: """ACK turn-bound delegation claims and return rejected notifications.""" rejected_notifications: list[str] = [] for evt, claim, notification, completion_queue in pending_async_acceptances: try: complete_async_delegation_delivery(evt, claim) except Exception: release_async_delegation_delivery(evt, claim) requeue_async_delegation_event( evt, completion_queue, durable=bool(getattr(claim, "durable", False)), ) rejected_notifications.append(notification) logger.warning( "Async delegation was not accepted into session %s; retrying later", session_id, exc_info=True, ) return rejected_notifications def _attachment_name(att) -> str: if isinstance(att, dict): return str(att.get('name') or att.get('filename') or att.get('path') or '').strip() return str(att or '').strip() _IMAGE_MAGIC: dict[bytes | None, frozenset[str]] = { b'\x89PNG\r\n\x1a\n': frozenset({'image/png'}), b'\xff\xd8\xff': frozenset({'image/jpeg'}), b'GIF87a': frozenset({'image/gif'}), b'GIF89a': frozenset({'image/gif'}), b'RIFF': frozenset({'image/webp'}), b'BM': frozenset({'image/bmp'}), None: frozenset({'image/svg+xml'}), } def _is_valid_image(path: Path, mime: str) -> bool: """Check that the file's first bytes match the expected image MIME type. Uses simple magic-number detection (no external dependency). SVG is allowed through because it is text-based and has no binary signature. """ if not mime.startswith('image/'): return False mime_base = mime.split(';', 1)[0] if mime_base == 'image/svg+xml': return True try: with path.open('rb') as fh: head = fh.read(16) except OSError: return False for magic, mimes in _IMAGE_MAGIC.items(): if magic is not None and head.startswith(magic) and mime_base in mimes: return True return False def _explicit_text_signal(cfg: dict) -> bool: """True when the user has explicitly opted into the text (vision_analyze) image pipeline. Two explicit signals, either of which means "the user chose text on purpose" and we must honour it rather than forwarding images natively: * ``agent.image_input_mode: text`` — a direct mode override. * a configured ``auxiliary.vision`` backend (provider not ``auto``/empty, or an explicit model / base_url) — the user is paying for a dedicated vision model and wants the text pipeline regardless of the main model. This mirrors the explicit-signal portion of ``agent/image_routing.py:decide_image_input_mode`` and is used both to interpret *why* the canonical router returned ``"text"`` (so the unknown-model carve-out only fires when there's no explicit user choice) and as the fallback decision when the agent package is unavailable. """ if not isinstance(cfg, dict): return False agent_cfg = cfg.get("agent") or {} if isinstance(agent_cfg, dict): mode = str(agent_cfg.get("image_input_mode", "auto") or "auto").strip().lower() if mode == "text": return True aux = cfg.get("auxiliary") or {} vision = (aux.get("vision") or {}) if isinstance(aux, dict) else {} if not isinstance(vision, dict): return False provider = str(vision.get("provider") or "").strip().lower() model_name = str(vision.get("model") or "").strip() base_url = str(vision.get("base_url") or "").strip() return provider not in ("", "auto") or bool(model_name) or bool(base_url) def _resolve_image_input_mode(cfg: dict, active_provider: str = "", active_model: str = "", *, requested_provider: str = "") -> str: """Return ``"native"`` or ``"text"`` for current-turn image uploads. Delegates the routing decision to ``agent/image_routing.py: decide_image_input_mode`` — the single source of truth — instead of the local re-implementation that previously lived here. That copy had DIVERGED from the canonical function: it returned ``"text"`` (dropping the image) in cases the canonical router would have forwarded natively, because it never consulted the active model's vision capability and instead hard-coded a handful of config heuristics. The WebUI keeps one deliberate carve-out on top of the canonical decision: for UNKNOWN / custom models (no models.dev capability data) the canonical router conservatively returns ``"text"``, but the WebUI historically forwards images NATIVELY and relies on the agent's strip-and-retry guard (``run_agent._try_shrink_image_parts_in_messages`` / ``_strip_images_from_messages``) to downgrade on a provider rejection. We preserve that behaviour here: a canonical ``"text"`` verdict is only honoured when there is a real signal — an explicit user choice (``image_input_mode: text`` or a configured ``auxiliary.vision`` backend) or a model KNOWN to lack vision. Otherwise we forward native. When the agent package is unavailable (e.g. the WebUI standalone test environment, where ``import agent`` fails), we fall back to the historical WebUI behaviour: honour an explicit text signal, otherwise native. Args: active_provider: the provider selected by the current session (e.g. ``"custom:mygateway"``). When provided, it beats the config/global default so image routing matches the model the user actually picked. active_model: the model selected by the current session. requested_provider: the provider identity before runtime canonicalization (e.g. the original ``"custom:mygateway"`` when ``active_provider`` was normalized to ``"custom"`` by ``_resolve_custom_provider_runtime_overrides``). Lets capability lookup select the exact ``custom_providers``/``providers`` entry. """ if not isinstance(cfg, dict): cfg = {} try: from agent.image_routing import decide_image_input_mode, _lookup_supports_vision from agent.auxiliary_client import _read_main_provider, _read_main_model if active_provider and active_model: provider = active_provider model = active_model else: provider = (_read_main_provider() or "").strip() model = (_read_main_model() or "").strip() mode = decide_image_input_mode(provider, model, cfg, requested_provider=requested_provider) if mode == "native": return "native" # Canonical returned "text". Honour it only when it reflects a genuine # signal; otherwise apply the WebUI unknown-model native carve-out. if _explicit_text_signal(cfg): return "text" if _lookup_supports_vision(provider, model, cfg, requested_provider=requested_provider) is False: # Model is KNOWN to be text-only — respect the canonical verdict. return "text" # Unknown / custom model (capability is None): WebUI forwards native # and lets the agent's strip-and-retry guard downgrade on rejection. return "native" except Exception: # Agent package unavailable or import error — preserve historical WebUI # behaviour: explicit text signal wins, otherwise native. pass if _explicit_text_signal(cfg): return "text" return "native" def _build_native_multimodal_message(workspace_ctx: str, msg_text: str, attachments, workspace: str, *, cfg: dict = None, active_provider: str = "", active_model: str = "", requested_provider: str = "", profile: str | Path | None = None): """Build native multimodal content parts for current-turn image uploads. WebUI uploads files into the active workspace. For image files, pass the bytes to Hermes as OpenAI-style image_url data URLs so vision-capable main models can consume them in the same request. Non-image files intentionally stay as text path attachments so the agent can inspect them with file tools. When *cfg* is provided, respects ``agent.image_input_mode`` — if the resolved mode is ``"text"``, returns a plain string (attachments are not embedded) so the agent's text-mode pipeline (``vision_analyze``) handles images. """ if not attachments: return workspace_ctx + msg_text # ── Check image_input_mode before embedding anything ── if cfg is not None and _resolve_image_input_mode(cfg, active_provider, active_model, requested_provider=requested_provider) == "text": return workspace_ctx + msg_text parts = [{'type': 'text', 'text': workspace_ctx + msg_text}] workspace_root = _resolve_path(workspace, profile=profile) # Stage-361 maintainer fix (Opus SHOULD-FIX): chat uploads from #2319 now # land in ~/.hermes/webui/attachments// (outside workspace_root by # design). The pre-existing `path.relative_to(workspace_root)` guard would # silently reject every image upload for vision-capable models. Allow the # configured attachment root in addition to workspace_root so native # multimodal embeds still build the base64 image_url part. The # _attachment_root() helper applies expanduser+resolve and is also reused # by _upload_destination — single source of truth for the inbox root. try: from api.upload import _attachment_root attachment_root = _attachment_root() _allowed_roots = (workspace_root, attachment_root) except Exception: _allowed_roots = (workspace_root,) image_count = 0 for att in attachments or []: if not isinstance(att, dict): continue raw_path = str(att.get('path') or '').strip() if not raw_path: continue try: path = Path(raw_path).expanduser().resolve() # Uploads should live inside the selected workspace OR the # session attachment inbox (#2319). Do not read arbitrary paths # from client-provided attachment metadata. if not any(path.is_relative_to(r) for r in _allowed_roots): continue if not path.is_file(): continue size = path.stat().st_size if size <= 0 or size > _NATIVE_IMAGE_MAX_BYTES: continue mime = str(att.get('mime') or '').strip() or (mimetypes.guess_type(path.name)[0] or '') if not mime.startswith('image/') or not _is_valid_image(path, mime): continue data = base64.b64encode(path.read_bytes()).decode('ascii') except Exception: continue parts.append({ 'type': 'image_url', 'image_url': {'url': f'data:{mime};base64,{data}'}, }) image_count += 1 return parts if image_count else workspace_ctx + msg_text _INLINE_THINKING_TAG_PAIRS = ( ('', ''), ('<|channel>thought\n', ''), ('<|turn|>thinking\n', ''), ) def _inline_thinking_fence_marker_at(text, index): # A fenced code block opener may be indented up to 3 spaces in Markdown # (4+ spaces is an indented code block, handled separately). The marker is # only a fence when it sits at the start of a line (after optional 1-3 # spaces of indentation). if index > 0 and text[index - 1] != '\n': # Allow up to 3 leading spaces: walk back over spaces to a line start. back = index - 1 spaces = 0 while back >= 0 and text[back] == ' ' and spaces < 3: back -= 1 spaces += 1 if not (back < 0 or text[back] == '\n'): return '' if text.startswith('```', index): return '```' if text.startswith('~~~', index): return '~~~' return '' def _next_inline_thinking_opener(text, start): """Index of the earliest complete thinking opener at/after `start`, or -1. Cheap str.find per opener — lets the scanner bulk-skip plain trailing content instead of walking it char-by-char (#3633 Codex per-token perf catch).""" best = -1 for open_tag, _close in _INLINE_THINKING_TAG_PAIRS: i = text.find(open_tag, start) if i != -1 and (best == -1 or i < best): best = i return best def _text_tail_is_partial_opener(text): """True when the END of `text` is a non-empty proper prefix of some thinking opener (e.g. ````). Used to decide whether a streaming tail might be a forming block worth code-aware handling.""" for open_tag, _close in _INLINE_THINKING_TAG_PAIRS: m = min(len(open_tag) - 1, len(text)) for n in range(m, 0, -1): if open_tag.startswith(text[-n:]): return True return False def _line_is_indented_code(text, line_start): """True when the line beginning at `line_start` is a markdown indented code block line (>=4 leading spaces or a leading tab, and not blank). `line_start` must be the index of the first character of the line. O(1)-ish: only inspects the line's leading characters, not the whole document (the per-character variant was O(n^2) on long no-newline content — #3633 Codex perf catch).""" if line_start >= len(text): return False if text[line_start] == '\t': # A leading tab is indented code only if the line isn't otherwise blank. nl = text.find('\n', line_start) seg = text[line_start:(nl if nl != -1 else len(text))] return bool(seg.strip()) if text.startswith(' ', line_start): nl = text.find('\n', line_start) seg = text[line_start:(nl if nl != -1 else len(text))] return bool(seg.strip()) return False def _merge_inline_thinking_reasoning(existing_reasoning, extracted_parts): out = str(existing_reasoning or '').strip() for part in extracted_parts or (): item = str(part or '').strip() if not item: continue if not out: out = item continue if out == item or any(existing.strip() == item for existing in out.split('\n\n')): continue out = out + '\n\n' + item return out def _extract_inline_thinking_from_content(raw_content, existing_reasoning='', *, streaming=False): """Split inline thinking blocks out of assistant content. Code-aware: thinking tags inside a triple-fence (``` / ~~~), an inline single-backtick code span, or an indented (>=4-space / tab) code block are LEFT VISIBLE — they are literal text a user typed/pasted, not a real thinking trace. (#3633 deep-review / Codex catch: the earlier full-scan version only protected triple fences, so a literal `` in an inline code span got silently extracted.) ``streaming`` gates partial/unclosed-block handling: during live streaming an unmatched open tag means "still thinking" and its tail is shown as reasoning; on the persist/reload path (streaming=False) an unclosed tag is LEFT VISIBLE so prose after a literal ```` is never silently truncated on save. """ text = '' if raw_content is None else str(raw_content) if not text: return text, str(existing_reasoning or '').strip() # Fast path (#3633 Codex perf catch — _parseStreamState / syncInflight call # this on the FULL accumulator on every streamed token, so the common no-tag # case must not do the O(length) char walk per call). If the text contains no # complete thinking opener AND — when streaming — its tail is not a prefix of # any opener (a partial opener mid-stream), there is nothing to extract: # return the text unchanged. Two cheap substring scans instead of a full walk. if not any(open_tag in text for open_tag, _close in _INLINE_THINKING_TAG_PAIRS): tail_is_partial_opener = False if streaming: for open_tag, _close in _INLINE_THINKING_TAG_PAIRS: # Does the END of text look like the START of an opener? max_prefix = min(len(open_tag) - 1, len(text)) for n in range(max_prefix, 0, -1): if open_tag.startswith(text[-n:]): tail_is_partial_opener = True break if tail_is_partial_opener: break if not tail_is_partial_opener: return text, str(existing_reasoning or '').strip() visible = [] extracted = [] cursor = 0 index = 0 fence = '' in_backtick = False length = len(text) # Incremental, O(1)-per-iteration line state (the previous per-character line # scan made the whole pass O(n^2) on long no-newline content — #3633 Codex # perf catch). `line_is_indented_code` is recomputed only at a line start. line_is_indented_code = _line_is_indented_code(text, 0) # Whether any non-whitespace char appeared in text[:index] — the cheap # equivalent of the old `text[:index].strip() != ''` leading check. seen_nonspace = False # Whether a LEADING thinking block/prefix was removed — only then do we # lstrip the final content (so a reply that legitimately starts with # indented code / whitespace and has NO leading thinking wrapper keeps its # leading whitespace — #3633 Codex catch). leading_removed = False # Index of the next opener at/after `index` (recomputed only when we pass it). # When no opener remains ahead, the rest of the text is plain and can be # appended in one slice — this keeps a stream that DID contain a leading # thinking block from re-walking the whole growing answer tail every token # (#3633 Codex perf catch: the per-token full walk was O(n^2) over a stream). next_opener = _next_inline_thinking_opener(text, 0) while index < length: if next_opener == -1 or index > next_opener: next_opener = _next_inline_thinking_opener(text, index) if next_opener == -1: # No further COMPLETE opener ahead. The remaining tail is plain # visible content and can be appended in one slice — EXCEPT during # streaming when the tail is a prefix of an opener (e.g. "... 0 and text[index - 1] == '\n': line_is_indented_code = _line_is_indented_code(text, index) marker = _inline_thinking_fence_marker_at(text, index) if marker: fence = '' if fence == marker else (fence or marker) # Inline single-backtick code span toggles on each lone backtick that is # not part of a triple fence. Only tracked outside a triple fence. if not fence and not marker and ch == '`': in_backtick = not in_backtick in_code = bool(fence) or in_backtick or line_is_indented_code if not in_code: pair = None for open_tag, close_tag in _INLINE_THINKING_TAG_PAIRS: if text.startswith(open_tag, index): pair = (open_tag, close_tag) break if pair: open_tag, close_tag = pair close_index = text.find(close_tag, index + len(open_tag)) if close_index == -1: # Unclosed open tag. A LEADING unclosed block (nothing # visible before it) is a genuine thinking trace that got # cut off / persisted mid-thought → reasoning (master #3455 # leading-only intent, and the live-stream "still thinking" # case). An unclosed tag AFTER visible content on the persist # path is almost always a literal typed tag — leave it (and # the prose after it) visible so nothing is silently # truncated (#3633 Codex catch). During live streaming any # unmatched open tag is treated as in-progress thinking. leading = not seen_nonspace if not streaming and not leading: break if leading: leading_removed = True visible.append(text[cursor:index]) partial = text[index + len(open_tag):] if partial: extracted.append(partial) cursor = length index = length break visible.append(text[cursor:index]) extracted.append(text[index + len(open_tag):close_index]) if not seen_nonspace: leading_removed = True seen_nonspace = True # the extracted tag span is non-whitespace index = close_index + len(close_tag) cursor = index continue if streaming: matched_partial = False for open_tag, _close_tag in _INLINE_THINKING_TAG_PAIRS: rest = text[index:] if len(rest) < len(open_tag) and open_tag.startswith(rest): if not seen_nonspace: leading_removed = True visible.append(text[cursor:index]) cursor = length index = length matched_partial = True break if matched_partial or index >= length: break if not ch.isspace(): seen_nonspace = True index += 1 if cursor < length: visible.append(text[cursor:]) content = ''.join(visible) if leading_removed: content = content.lstrip() reasoning = _merge_inline_thinking_reasoning(existing_reasoning, extracted) return content, reasoning def _split_thinking_from_content(raw_content, existing_reasoning=''): """Split inline thinking blocks out of assistant content for persistence. Persistence path: streaming=False, so an unclosed tag stays visible content (a partial block only means "still thinking" during a live stream). """ return _extract_inline_thinking_from_content( raw_content, existing_reasoning=existing_reasoning, streaming=False, ) def _strip_thinking_markup(text: str) -> str: """Remove common reasoning/thinking wrappers from model text.""" if not text: return '' s = str(text) # Treat provider thinking wrappers as metadata only when they lead the # response. Literal discussion of these tags later in normal prose should # stay visible (#2152). s = re.sub(r'^\s*.*?\s*', ' ', s, flags=re.IGNORECASE | re.DOTALL) s = re.sub(r'^\s*<\|channel\|?>thought\n?.*?\s*', ' ', s, flags=re.IGNORECASE | re.DOTALL) s = re.sub(r'^\s*<\|turn\|>thinking\n.*?\s*', ' ', s, flags=re.IGNORECASE | re.DOTALL) # Gemma 4 s = re.sub(r'^\s*(the|ther)\s+user\s+is\s+asking[^\n]*(?:\n|$)', ' ', s, flags=re.IGNORECASE) # Strip plain-text thinking preambles from models that don't use tags (e.g. Qwen3). # These appear as the very first sentence of the assistant response and are not useful as titles. s = re.sub( r"^\s*(?:here(?:'s| is) (?:a |my )?(?:thinking|thought) (?:process|trace|through)\b[^\n]*\n?" r"|let me (?:think|work|reason|analyze|walk) (?:through|about|this|step)\b[^\n]*\n?" r"|i(?:'ll| will) (?:think|work|reason|analyze|break this down)\b[^\n]*\n?" r"|(?:okay|alright|sure|of course),?\s+let me\b[^\n]*\n?)", ' ', s, flags=re.IGNORECASE ) s = re.sub(r'\s+', ' ', s).strip() return s def _strip_xml_tool_calls(text: str) -> str: """Strip XML-style function_calls blocks that DeepSeek and similar models emit in their raw response text. These blocks are processed separately as tool calls; leaving them in the assistant content causes them to render visibly in the chat bubble. Handles both complete blocks (…) and partial/orphaned opening tags that may appear at the tail of a stream. Also handles variants like <｜DSML｜function_calls> from DeepSeek on Bedrock. """ if not text: return text s = str(text) # Check if contains any function_calls/DSML marker (case-insensitive) _lo = s.lower() if 'function_calls' not in _lo and 'dsml' not in _lo: return text _dsml_prefix = r'(?:\s*｜\s*DSML\s*[｜|]\s*)?' open_tag = rf'<{_dsml_prefix}function_calls' close_tag = rf'' # Strip complete blocks for both and <｜DSML｜function_calls>. s = re.sub( rf'{open_tag}>.*?{close_tag}', '', s, flags=re.IGNORECASE | re.DOTALL ) # Strip orphaned/truncated opening tags, including missing ">" at stream tail. s = re.sub( rf'{open_tag}(?:>|$).*$', '', s, flags=re.IGNORECASE | re.DOTALL ) # Remove malformed DSML fragments like "<｜DSML |" that can leak in tokens. s = re.sub(r'<\s*｜\s*DSML\s*[｜|]\s*', '', s, flags=re.IGNORECASE) return s.strip() def _sanitize_generated_title(text: str) -> str: """Sanitize LLM-generated title text before persisting to session.""" s = _strip_thinking_markup(text or '') s = re.sub( r'^\s*(?:[*_`~]+\s*)?(?:session\s+title|title)\s*:\s*(?:[*_`~]+\s*)?', '', s, flags=re.IGNORECASE, ) s = re.sub(r'^\s*title\s*:\s*', '', s, flags=re.IGNORECASE) s = s.strip(" \t\r\n\"'`*_~") s = re.sub(r'\s+', ' ', s).strip() # Guard against chain-of-thought leakage, meta-reasoning, and trivial echo. if _is_bad_new_title(s): return '' return s[:80] def _looks_invalid_generated_title(text: str) -> bool: """True when an existing/persisted title is structurally invalid (CoT leak). Intentionally does NOT reject short conversational words like "Done" or "Cool" — those can be legitimate stored titles. Use ``_is_bad_new_title`` when validating a freshly generated candidate. """ s = str(text or '') if not s.strip(): return True return bool( re.search(r'|<\|channel\|>thought|<\|turn\|>thinking', s, flags=re.IGNORECASE) or re.search(r'^\s*(the|ther)\s+user\s+', s, flags=re.IGNORECASE) or re.search(r'^\s*user\s+\w+\s+', s, flags=re.IGNORECASE) or re.search(r'\b(they|user)\s+want(s)?\s+me\s+to\b', s, flags=re.IGNORECASE) or re.search(r'^\s*(i|we)\s+(should|need to|will|can)\b', s, flags=re.IGNORECASE) or re.search(r'^\s*let me\b', s, flags=re.IGNORECASE) or re.search(r"^\s*here(?:'s| is) (?:a |my )?(?:thinking|thought)", s, flags=re.IGNORECASE) ) def _is_bad_new_title(text: str) -> bool: """True when a freshly generated title candidate should be discarded. Includes structural CoT rejection plus trivial single-token echo replies (pong, yes, done, cool, …) that are useless as first-generation titles. """ if _looks_invalid_generated_title(text): return True s = str(text or '').strip() _token = re.sub(r'[\s.!?]+$', '', s, flags=re.IGNORECASE) if re.fullmatch( r'(?:pong|ping|yes|no|yep|nope|hi|hello|hey|thanks|thank you|sure|k|kk|cool|nice|lol|ok|okay|done)', _token, flags=re.IGNORECASE, ): return True return bool( re.search( r'^\s*(ok|okay|done|all set|complete|completed|finished)\b[\s.!?]*$', s, flags=re.IGNORECASE, ) ) def _message_content_part_text(part) -> str: """Extract visible text from a structured content part.""" if not isinstance(part, dict): return '' return str( part.get('text') or part.get('content') or part.get('input_text') or part.get('output_text') or '' ) def _message_text(value) -> str: """Extract plain text from mixed message content payloads.""" if isinstance(value, list): parts = [] for p in value: if not isinstance(p, dict): continue ptype = str(p.get('type') or '').lower() if ptype in ('', 'text', 'input_text', 'output_text'): parts.append(_message_content_part_text(p)) return _strip_thinking_markup('\n'.join(parts).strip()) return _strip_thinking_markup(str(value or '').strip()) def _assistant_content_part_is_tool_use(part) -> bool: """Return True when a content[] part represents a tool invocation boundary.""" if not isinstance(part, dict): return False part_type = str(part.get('type') or '').lower() if part_type in {'tool_use', 'tool_call'}: return True if part_type: return False if _message_content_part_text(part).strip(): return False return any(key in part for key in ('tool_use_id', 'tool_call_id', 'call_id')) and any( key in part for key in ('name', 'tool_name', 'input', 'args') ) def _assistant_message_has_final_visible_text(message) -> bool: """Return True when an assistant row carries a settled visible answer.""" if not isinstance(message, dict) or message.get('role') != 'assistant': return False content = message.get('content', '') if isinstance(content, list): last_tool_idx = -1 for idx, part in enumerate(content): if _assistant_content_part_is_tool_use(part): last_tool_idx = idx if last_tool_idx >= 0: tail_parts = content[last_tool_idx + 1:] return bool(_message_text(tail_parts).strip()) if message.get('tool_calls'): return False return bool(_message_text(content).strip()) if message.get('tool_calls'): return False return bool(_message_text(content).strip()) _WORKSPACE_PREFIX_RE = re.compile(r'^\s*\[Workspace::v1:\s*(?:\\.|[^\]\\])+\]\s*') _LEGACY_WORKSPACE_PREFIX_RE = re.compile(r'^\s*\[Workspace:[^\]]+\]\s*') _WORKSPACE_PREFIX_ANY_RE = re.compile(r'\[Workspace::v1:\s*(?:\\.|[^\]\\])+\]\s*') _LEGACY_WORKSPACE_PREFIX_ANY_RE = re.compile(r'\[Workspace:[^\]]+\]\s*') def _escape_workspace_prefix_path(path: str) -> str: return str(path or '').replace('\\', '\\\\').replace(']', '\\]') def _workspace_context_prefix(path: str) -> str: return f"[Workspace::v1: {_escape_workspace_prefix_path(path)}]\n" def _strip_workspace_prefix(text: str, *, include_legacy: bool = False) -> str: """Remove WebUI-injected workspace tags without eating user-typed text.""" value = str(text or '') stripped = _WORKSPACE_PREFIX_RE.sub('', value, count=1) if include_legacy and stripped == value: stripped = _LEGACY_WORKSPACE_PREFIX_RE.sub('', value, count=1) return stripped.strip() _TITLE_ATTACHMENT_SUFFIX_RE = re.compile( r'(?:\n\n|\r\n\r\n)\[Attached files(?: for this steer)?: [^\]]+\]\s*$' ) def _strip_title_attachment_suffix(text) -> str: """Remove one exact WebUI-generated terminal attachment suffix.""" return _TITLE_ATTACHMENT_SUFFIX_RE.sub('', str(text or '')) def _strip_title_input_metadata(text) -> str: """Remove only internal title metadata from one selected text value.""" value = _strip_title_attachment_suffix(text) return _strip_workspace_prefix(value).strip() def _title_structured_text_parts( content, allowed_types, *, normalize_types: bool = False ) -> list[str]: """Sanitize accepted structured title parts without mutating the content.""" parts = [] workspace_prefix_stripped = False for part in content: if not isinstance(part, dict): continue part_type = str(part.get('type') or '').lower() if normalize_types else part.get('type') if part_type not in allowed_types: continue text = ( str(part.get('text') or '') if not normalize_types else _message_content_part_text(part) ) if text.strip() and not workspace_prefix_stripped: text = _strip_title_input_metadata(text) workspace_prefix_stripped = bool(text) parts.append(text) return parts def _title_input_text(content) -> str: """Extract raw title text using the same content rules as title_from.""" if content is None: return '' if isinstance(content, list): return ' '.join( _title_structured_text_parts(content, ('text',), normalize_types=False) ).strip() return _strip_title_input_metadata(str(content)) _TITLE_MIXED_PART_TYPES = ('', 'text', 'input_text', 'output_text') def _title_exchange_input_text(content) -> str: """Extract sanitized first/latest-exchange title input text.""" if isinstance(content, list): return '\n'.join( _title_structured_text_parts( content, _TITLE_MIXED_PART_TYPES, normalize_types=True ) ).strip() return _strip_title_input_metadata(str(content or '').strip()) def _looks_like_current_user_turn(msg, msg_text) -> bool: """Match the current human turn even if an internal workspace tag leaked mid-text. Normal model-facing messages start with the workspace sentinel. A failed retry/merge path can also return an optimistic draft followed by the sentinel and the real prompt. Only treat that shape as the current turn when the text after the sentinel exactly matches the submitted prompt. """ if not isinstance(msg, dict) or msg.get('role') != 'user': return False needle = " ".join(str(msg_text or '').split()) if not needle: return False text = _message_text(msg.get('content', '')) candidates = [_strip_workspace_prefix(text, include_legacy=True)] for pattern in (_WORKSPACE_PREFIX_ANY_RE, _LEGACY_WORKSPACE_PREFIX_ANY_RE): for match in pattern.finditer(text): candidates.append(text[match.end():]) return any(" ".join(str(candidate or '').split()) == needle for candidate in candidates) def _first_exchange_snippets(messages, *, scan_past_consecutive_users: bool = False): """Return (first_user_text, first_assistant_text) snippets for title generation. Prefer the first substantive assistant answer in the opening exchange, skipping empty placeholders and assistant tool-call preambles. ``scan_past_consecutive_users`` (opt-in) keeps scanning past consecutive opening user rows (queued first turns) until the first COMPLETE user+assistant pair — needed by the manual "Regenerate title" path (#7543), which otherwise aborted at the second user row with an empty assistant snippet and silently persisted the local fallback. It defaults False so the automatic in-stream background-title path keeps its exact prior behavior (a transcript with no assistant text before the second user row yields an empty assistant snippet, which _background_title_generation_inputs treats as "not yet eligible" — the stream teardown then emits stream_end on its synchronous path unchanged). """ user_text = '' asst_text = '' for m in messages or []: if not isinstance(m, dict): continue role = m.get('role') if role == 'user': candidate = _strip_thinking_markup(_title_exchange_input_text(m.get('content'))) if candidate and not user_text: user_text = candidate elif user_text and candidate and not scan_past_consecutive_users: # Legacy/default behavior: a second populated user row before any # assistant text ends the opening exchange (asst_text stays empty). break # When scan_past_consecutive_users is set, keep going past consecutive # user rows (#7543) until the first user+assistant pair. elif role == 'assistant' and user_text: candidate = _message_text(m.get('content')) # Skip tool-call preambles *only* when content is empty or looks # like meta-reasoning ("Let me check my memory first.", "The user # is asking...", etc.). Assistant rows that carry tool_calls but # also contain a substantive answer text are kept — those are # agentic first-turn plans that are legitimate title candidates. if m.get('tool_calls') and (not candidate or _is_bad_new_title(candidate)): continue if candidate: asst_text = candidate if user_text and asst_text: break return user_text[:500], asst_text[:500] def _latest_exchange_snippets(messages): """Return (last_user_text, last_assistant_text) snippets for title refresh. Walks the message list backwards to find the last user+assistant pair, skipping empty or tool-call-only assistant messages. """ user_text = '' asst_text = '' for m in reversed(messages or []): if not isinstance(m, dict): continue role = m.get('role') if role == 'assistant' and not asst_text: candidate = _message_text(m.get('content')) # Skip tool-call-only preambles if m.get('tool_calls') and (not candidate or _is_bad_new_title(candidate)): continue if candidate: asst_text = candidate elif role == 'user': candidate = _strip_thinking_markup(_title_exchange_input_text(m.get('content'))) if not candidate: user_text = '' asst_text = '' break if not user_text: user_text = candidate if user_text and asst_text: break return user_text[:500], asst_text[:500] def _count_exchanges(messages): """Count the number of user messages (rough exchange count).""" count = 0 for m in messages or []: if isinstance(m, dict) and m.get('role') == 'user': content = m.get('content', '') if isinstance(content, list): content = ' '.join(p.get('text', '') for p in content if isinstance(p, dict) and p.get('type') == 'text') if str(content).strip(): count += 1 return count def _get_title_refresh_interval() -> int: """Read the auto_title_refresh_every setting (0 = disabled).""" try: from api.config import load_settings settings = load_settings() val = settings.get('auto_title_refresh_every', '0') return int(val) if str(val).strip().isdigit() and int(val) > 0 else 0 except Exception: return 0 def _is_provisional_title(current_title: str, messages) -> bool: """Heuristic: title equals first-message substring placeholder.""" first_user_text = '' for message in messages or []: if not isinstance(message, dict) or message.get('role') != 'user': continue first_user_text = _title_input_text(message.get('content')) if first_user_text: break if not first_user_text: return False sanitized_derived = title_from([{'role': 'user', 'content': first_user_text}], '') or '' raw_derived = title_from(messages or [], '') or '' if not sanitized_derived: return False def _normalize_candidate(value): return re.sub(r'\s+', ' ', str(value or '')[:64]).strip() current = re.sub(r'\s+', ' ', str(current_title or '')).strip() candidates = ( _normalize_candidate(sanitized_derived), _normalize_candidate(raw_derived), ) if not current: return False return any(candidate and current == candidate for candidate in candidates) def _background_title_generation_inputs(session): """Return sanitized first-exchange inputs when background title generation is eligible.""" messages = getattr(session, 'messages', None) or [] title = getattr(session, 'title', '') invalid_existing_title = _looks_invalid_generated_title(title) eligible_title = ( title == 'Untitled' or title == 'New Chat' or not title or _is_provisional_title(title, messages) or invalid_existing_title ) if not eligible_title or ( getattr(session, 'llm_title_generated', False) and not invalid_existing_title ): return None user_text, assistant_text = _first_exchange_snippets(messages) return (user_text, assistant_text) if user_text and assistant_text else None def _detect_title_language(text: str) -> str: """Best-effort language hint for title generation/validation.""" s = re.sub(r'\s+', ' ', str(text or '')).strip().lower() if not s: return '' german_markers = { 'warum', 'werden', 'wird', 'wurde', 'hier', 'nicht', 'mehr', 'alte', 'alten', 'bilder', 'angezeigt', 'prüfe', 'ich', 'und', 'oder', 'mit', 'für', 'von', 'zu', 'ist', 'sind', 'bitte', 'kannst', } tokens = re.findall(r'[A-Za-zÀ-ÖØ-öø-ÿ]+', s) german_hits = sum(1 for tok in tokens if tok in german_markers) if re.search(r'[äöüß]', s) or german_hits >= 3: return 'de' return '' def _script_counts(text: str) -> dict: """Return per-script alphabetic character counts for *text*. Buckets: ``latin``, ``cjk`` (Han/Hiragana/Katakana/Hangul), ``cyrillic``, ``arabic``, ``hebrew``, ``greek``, ``devanagari``. Non-alphabetic and unclassified characters are ignored. """ counts: dict[str, int] = {} for ch in str(text or ''): if not ch.isalpha(): continue o = ord(ch) if (0x0041 <= o <= 0x024F) or (0x1E00 <= o <= 0x1EFF): bucket = 'latin' elif ( (0x4E00 <= o <= 0x9FFF) or (0x3400 <= o <= 0x4DBF) # Han or (0x3040 <= o <= 0x30FF) # Hiragana/Katakana or (0xAC00 <= o <= 0xD7A3) or (0x1100 <= o <= 0x11FF) # Hangul ): bucket = 'cjk' elif 0x0400 <= o <= 0x04FF: bucket = 'cyrillic' elif (0x0600 <= o <= 0x06FF) or (0x0750 <= o <= 0x077F): bucket = 'arabic' elif 0x0590 <= o <= 0x05FF: bucket = 'hebrew' elif 0x0370 <= o <= 0x03FF: bucket = 'greek' elif 0x0900 <= o <= 0x097F: bucket = 'devanagari' else: continue counts[bucket] = counts.get(bucket, 0) + 1 return counts def _dominant_script(text: str) -> str: """Return a coarse writing-script bucket for *text*, or '' when undecidable. Script-level (not language-level) classification is cheap and dependency-free. Returns the dominant script only when it holds a clear (≥60%) majority of the alphabetic characters, so mixed/borrowed text doesn't flip the bucket. Used to establish the conversation start's expected script for cross-script title drift detection (#3293). """ counts = _script_counts(text) total = sum(counts.values()) if total < 2: return '' top, top_n = max(counts.items(), key=lambda kv: kv[1]) if top_n / total >= 0.6: return top return '' def _title_prompt_language_rule(user_text: str) -> str: return "Match the language of the user question.\n" def _title_language_mismatch(user_text: str, title: str) -> bool: """Reject titles whose language clearly diverges from the conversation start. Two independent signals: 1. Cross-script drift (#3293): when the conversation start has a clear dominant writing script (e.g. latin/English) and the generated title introduces a *substantial* amount of a different script (e.g. CJK or Cyrillic), reject. This is language-agnostic and catches the common "English chat -> Chinese/Spanish/Russian title" drift. Because titles are short and frequently embed a borrowed Latin technical term (e.g. a CJK title containing the word "Python"), the title side uses a proportion threshold (>=35% of the title's alphabetic characters in a non-start script, min 2 chars) rather than a strict majority. CJK titles with borrowed Latin terms are allowed when the title also contains CJK characters (#7693), but pure-Latin titles for CJK conversations are still rejected. 2. The legacy German-start → English-title heuristic, preserved verbatim so the original behavior keeps working for same-script (latin) drift that the script check can't see. """ candidate = str(title or '').strip() if not candidate: return False # (1) Cross-script mismatch — language-agnostic. # CJK text routinely borrows Latin product/technical terms (e.g. "WeChat # Pay", "Python", "ProRes RAW"), so when the user writes in CJK, Latin # characters in the title are acceptable as long as the title also # contains CJK — i.e. the title is genuinely mixed, not pure drift. # A pure-Latin title for a CJK conversation is still rejected. # Unrelated scripts (Cyrillic, Arabic, Greek …) are always flagged. user_script = _dominant_script(user_text) if user_script: title_counts = _script_counts(candidate) title_total = sum(title_counts.values()) if title_total >= 2: for script, n in title_counts.items(): if script == user_script: continue # When user writes in CJK, Latin in the title is a borrowed # term as long as the title also contains CJK characters. if user_script == 'cjk' and script == 'latin': if title_counts.get('cjk', 0) >= 2: continue if n >= 2 and (n / title_total) >= 0.35: return True # (2) Legacy same-script German→English heuristic. if _detect_title_language(user_text) != 'de': return False candidate_lower = candidate.lower() if _detect_title_language(candidate_lower) == 'de': return False english_markers = { 'old', 'image', 'display', 'issue', 'problem', 'discussion', 'conversation', 'session', 'title', 'fix', 'bug', 'attachment', 'attachments', 'context', } tokens = re.findall(r'[a-z]+', candidate_lower) english_hits = sum(1 for tok in tokens if tok in english_markers) return english_hits >= 2 def _title_prompts(user_text: str, assistant_text: str) -> tuple[str, list[str]]: qa = f"User question:\n{user_text[:500]}\n\nAssistant answer:\n{assistant_text[:500]}" language_rule = _title_prompt_language_rule(user_text) prompts = [ ( "Generate a short session title from this conversation start.\n" "Use BOTH the user's question and the assistant's visible answer.\n" f"{language_rule}" "Return only the title text, 3-8 words, as a topic label.\n" "Do not use markdown, bullets, labels, or prefixes like Session Title:.\n" "Do not output a full sentence.\n" "Do not output acknowledgements or completion phrases like OK, done, or all set.\n" "Do not describe internal reasoning.\n" "Bad: The user is asking..., OK, all set.\n" "Good: Title Generation Test, Clarify Dialog Layout, GitHub Issue Triage" ), ( "Rewrite this conversation start as a concise noun-phrase title.\n" "Use the actual topic, not the task outcome.\n" f"{language_rule}" "Return title text only.\n" "Do not use markdown, bullets, labels, or prefixes like Session Title:.\n" "Never output acknowledgements, completion status, or meta commentary." ), ] return qa, prompts def _is_minimax_route(provider: str = '', model: str = '', base_url: str = '') -> bool: text = ' '.join([ str(provider or '').lower(), str(model or '').lower(), str(base_url or '').lower(), ]) return 'minimax' in text or 'minimaxi.com' in text def _route_rejects_reasoning_extra(provider: str = '', model: str = '', base_url: str = '') -> bool: """Routes known to reject an ``extra_body`` ``reasoning`` parameter with HTTP 400. Title generation injects ``extra_body={"reasoning": {"enabled": False}}`` to suppress thinking on reasoning-capable models (#2083). But OpenAI Chat Completions (and Azure OpenAI) reject unknown top-level params with a 400, so that inject silently fails the title call and falls back to a low-quality heuristic title (#4161). Skip the inject for those routes. OpenRouter Anthropic mandatory-reasoning models (Claude Sonnet 4.6 / Opus 4.8) are reasoning-capable but reject a reasoning *disable* — title gen only needs reasoning off, so skip the inject for them too rather than risk the same 400. """ provider_lower = str(provider or '').strip().lower() model_lower = str(model or '').strip().lower() # Hostname-based match (not substring) so a proxy URL that merely *contains* # one of these strings in a path segment isn't mis-classified. host = '' try: from urllib.parse import urlsplit host = (urlsplit(str(base_url or '').strip()).hostname or '').lower() except Exception: host = '' if host == 'api.openai.com' or host.endswith('.openai.azure.com'): return True # Azure AI Foundry chat-completions hosts (also reject the reasoning param). if host.endswith('.services.ai.azure.com') or host.endswith('.cognitiveservices.azure.com'): return True if provider_lower in ('openai', 'openai-api', 'openai-codex'): return True if ( provider_lower in ('azure', 'azure-foundry', 'azure-ai-foundry', 'azure-ai') or provider_lower.startswith('azure/') or provider_lower.startswith('azure-') ): return True if (host == 'openrouter.ai' or host.endswith('.openrouter.ai')) and model_lower.startswith('anthropic/'): # Anthropic on OpenRouter: mandatory-reasoning families reject a disable. return True return False def _get_aux_title_config() -> dict: """Return title_generation auxiliary config, or an empty dict on errors.""" try: from agent.auxiliary_client import _get_auxiliary_task_config tg = _get_auxiliary_task_config('title_generation') return tg if isinstance(tg, dict) else {} except Exception: return {} def _aux_title_generation_enabled() -> bool: """Return whether automatic title generation is enabled (default: enabled). Mirrors Hermes Agent's ``auxiliary.title_generation.enabled`` contract byte-for-byte via its canonical ``is_truthy_value(value, default=True)`` semantics (agent/title_generator.py -> utils.is_truthy_value): * ``None`` / missing -> default (True) * ``bool`` -> unchanged * ``str`` -> True ONLY when normalized into {"1", "true", "yes", "on"} (an empty or unrecognized string is False, matching the agent — an allowlist, NOT a "disable on false/0/no/off" blocklist) * any other type -> ``bool(value)`` """ tg = _get_aux_title_config() val = tg.get('enabled', True) if val is None: return True if isinstance(val, bool): return val if isinstance(val, str): return val.strip().lower() in ('1', 'true', 'yes', 'on') return bool(val) def _aux_title_configured() -> bool: """Return True when any auxiliary title_generation config field is meaningfully set.""" tg = _get_aux_title_config() provider = tg.get('provider', '') or '' model = tg.get('model', '') or '' base_url = tg.get('base_url', '') or '' return bool(model or base_url or (provider and provider.lower() != 'auto')) def _aux_title_timeout(default: float = 15.0) -> float: """Return the configured timeout (seconds) for auxiliary title generation. Only accepts positive numeric values. Falls back to *default* when the value is ``None``, non-numeric, zero, or negative, and emits a debug log so mis-configurations are visible in server output. """ try: tg = _get_aux_title_config() raw = tg.get('timeout') if raw is None: return default try: value = float(raw) except (ValueError, TypeError): logger.debug("aux title timeout: non-numeric value %r, falling back to %s", raw, default) return default if value > 0: return value logger.debug("aux title timeout: non-positive value %s, falling back to %s", value, default) return default except Exception: return default def _title_completion_budget(provider: str = '', model: str = '', base_url: str = '') -> int: # Title generation is a small auxiliary task, but reasoning models may # spend a surprising amount of the completion budget before emitting final # content. Keep the budget high enough for MiniMax/Kimi-style reasoning # responses without making title generation depend on provider-specific # one-off branches. return 512 def _title_retry_completion_budget(provider: str = '', model: str = '', base_url: str = '') -> int: return max(1024, _title_completion_budget(provider, model, base_url) * 2) def _title_retry_status(status: str) -> bool: # Whether to grant a second budget attempt within the same prompt+model # combination. ``llm_length`` indicates the model would have produced # content with more headroom, so doubling the budget can help. # # ``llm_empty_reasoning`` historically also triggered a retry, but for # reasoning models (Qwen3-thinking, DeepSeek-R1, Kimi-K2, etc.) that # status means the model burned its entire budget on hidden reasoning # tokens and emitted nothing visible. Doubling the budget in that case # just doubles the GPU/credit cost without changing the outcome — the # next attempt produces the same shape. We skip the retry for empty- # reasoning statuses and let the title path fall through to the local # fallback summary. See issue #2083 for the LM Studio + Qwen3 repro. return status in { 'llm_length', 'llm_length_aux', } def _title_should_skip_remaining_attempts(status: str) -> bool: """Statuses where re-issuing the next prompt against the same model produces the same failing shape (model burned its budget on hidden reasoning, hit a hard provider gate, etc.). Short-circuit the prompt-iteration loop so we don't issue a second full-budget LLM call (and twice the GPU/credit burn) only to land in the same fallback path. See issue #2083. Add a status here only when retrying the next prompt is provably wasted work (single-call signal already establishes that the next call will return the same shape). Length-truncation WITHOUT reasoning is NOT in the set — that's legitimately recoverable by a larger budget on a different prompt and stays in :func:`_title_retry_status`. """ return status in { 'llm_empty_reasoning', 'llm_empty_reasoning_aux', } def _safe_obj_value(obj, key: str): if obj is None: return None if isinstance(obj, dict): return obj.get(key) value = getattr(obj, key, None) # Missing MagicMock attrs stringify as mock reprs and look truthy. Treat # them as absent so tests model real provider objects accurately. if value.__class__.__module__.startswith('unittest.mock'): return None return value def _safe_text_value(value) -> str: if value is None: return '' if value.__class__.__module__.startswith('unittest.mock'): return '' return str(value or '').strip() def _extract_title_response(resp, *, aux: bool = False) -> tuple[str, str]: """Return (content, empty_status) from an OpenAI-compatible response.""" suffix = '_aux' if aux else '' try: choices = _safe_obj_value(resp, 'choices') or [] choice = choices[0] if choices else None message = _safe_obj_value(choice, 'message') content = _safe_text_value(_safe_obj_value(message, 'content')) if content: return content, '' finish_reason = _safe_text_value(_safe_obj_value(choice, 'finish_reason')).lower() reasoning = ( _safe_text_value(_safe_obj_value(message, 'reasoning')) or _safe_text_value(_safe_obj_value(message, 'reasoning_content')) or _safe_text_value(_safe_obj_value(message, 'thinking')) ) # When the model emitted reasoning tokens but no visible content, it # burned its budget on hidden thinking — retrying with a larger budget # almost never recovers a useful title (see issue #2083: Qwen3-thinking # via LM Studio loops indefinitely on auto-title generation). Report # this case distinctly so callers can short-circuit instead of double- # billing the GPU/credit on a near-certain repeat. if reasoning: return '', f'llm_empty_reasoning{suffix}' if finish_reason == 'length': return '', f'llm_length{suffix}' return '', f'llm_empty{suffix}' except Exception: return '', f'llm_empty{suffix}' def generate_title_raw_via_aux( user_text: str, assistant_text: str, provider: str = '', model: str = '', base_url: str = '', ) -> tuple[Optional[str], str]: """Return (raw_text, status) via auxiliary LLM route.""" if not user_text or not assistant_text: return None, 'missing_exchange' qa, prompts = _title_prompts(user_text, assistant_text) configured = _get_aux_title_config() caller_supplied_route = bool(provider or model or base_url) provider = provider or configured.get('provider', '') or '' if str(provider).strip().lower() == 'auto': provider = '' model = model or configured.get('model', '') or '' base_url = base_url or configured.get('base_url', '') or '' try: from api.profiles import _split_webui_provider_model_value normalized_model, normalized_provider = _split_webui_provider_model_value( model or None, provider or None, ) model = normalized_model or '' provider = normalized_provider or '' except ValueError: pass api_key = '' if not caller_supplied_route: api_key = str(configured.get('api_key', '') or '').strip() base_max_tokens = _title_completion_budget(provider, model, base_url) reasoning_extra = {} if not _route_rejects_reasoning_extra(provider, model, base_url): reasoning_extra["reasoning"] = {"enabled": False} if _is_minimax_route(provider, model, base_url): reasoning_extra["reasoning_split"] = True try: _timeout = _aux_title_timeout() from agent.auxiliary_client import call_llm last_status = 'llm_error_aux' for idx, prompt in enumerate(prompts): messages = [ {"role": "system", "content": prompt}, {"role": "user", "content": qa}, ] budgets = [base_max_tokens] try: for budget_idx, max_tokens in enumerate(budgets): resp = call_llm( task='title_generation', provider=provider or None, model=model or None, base_url=base_url or None, api_key=api_key or None, messages=messages, max_tokens=max_tokens, temperature=0.2, timeout=_timeout, extra_body=reasoning_extra or None, ) raw, empty_status = _extract_title_response(resp, aux=True) if raw: return raw, ('llm_aux' if idx == 0 and budget_idx == 0 else 'llm_aux_retry') last_status = empty_status or 'llm_empty_aux' if budget_idx == 0 and _title_retry_status(last_status): budgets.append(_title_retry_completion_budget(provider, model, base_url)) except Exception as e: last_status = 'llm_error_aux' logger.debug("Aux title generation attempt %s failed: %s", idx + 1, e) # If the model just burned its budget on hidden reasoning, retrying # the next prompt against the same model produces the same shape. # Short-circuit to the local fallback path (#2083). if _title_should_skip_remaining_attempts(last_status): logger.debug( "Aux title generation short-circuiting after %s (reasoning-only response).", last_status, ) break return None, last_status except Exception as e: logger.debug("Aux title generation failed: %s", e) return None, 'llm_error_aux' def generate_title_raw_via_agent(agent, user_text: str, assistant_text: str) -> tuple[Optional[str], str]: """Return (raw_text, status) via active-agent route.""" if not user_text or not assistant_text: return None, 'missing_exchange' if agent is None: return None, 'missing_agent' qa, prompts = _title_prompts(user_text, assistant_text) base_max_tokens = _title_completion_budget( getattr(agent, 'provider', ''), getattr(agent, 'model', ''), getattr(agent, 'base_url', ''), ) disabled_reasoning = {"enabled": False} prev_reasoning = getattr(agent, 'reasoning_config', None) try: agent.reasoning_config = disabled_reasoning for idx, prompt in enumerate(prompts): api_messages = [ {"role": "system", "content": prompt}, {"role": "user", "content": qa}, ] budgets = [base_max_tokens] try: last_status = 'llm_empty' for budget_idx, max_tokens in enumerate(budgets): raw = "" empty_status = '' if getattr(agent, 'api_mode', '') == 'codex_responses': codex_kwargs = agent._build_api_kwargs(api_messages) codex_kwargs.pop('tools', None) if 'max_output_tokens' in codex_kwargs: codex_kwargs['max_output_tokens'] = max_tokens resp = agent._run_codex_stream(codex_kwargs) normalized = agent._get_transport('codex_responses').normalize_response(resp) raw = (normalized.content or '') if normalized else '' if not raw: empty_status = 'llm_empty' elif getattr(agent, 'api_mode', '') == 'anthropic_messages': from agent.anthropic_adapter import build_anthropic_kwargs ant_kwargs = build_anthropic_kwargs( model=agent.model, messages=api_messages, tools=None, max_tokens=max_tokens, reasoning_config=disabled_reasoning, is_oauth=getattr(agent, '_is_anthropic_oauth', False), preserve_dots=agent._anthropic_preserve_dots(), base_url=getattr(agent, '_anthropic_base_url', None), ) resp = agent._anthropic_messages_create(ant_kwargs) normalized = agent._get_transport().normalize_response( resp, strip_tool_prefix=getattr(agent, '_is_anthropic_oauth', False) ) raw = (normalized.content or '') if normalized else '' if not raw: empty_status = 'llm_empty' else: api_kwargs = agent._build_api_kwargs(api_messages) api_kwargs.pop('tools', None) api_kwargs['temperature'] = 0.1 api_kwargs['timeout'] = 15.0 # Reasoning suppression for title gen is already handled # route-correctly by `_build_api_kwargs()` from the # `agent.reasoning_config = {"enabled": False}` set above — # each provider profile applies (or deliberately omits) the # disable in the form its endpoint accepts (OpenAI/Nous omit # the field; LM Studio uses top-level reasoning_effort; # OpenRouter Anthropic mandatory-reasoning is omitted). Do NOT # re-inject a generic `reasoning:{enabled:False}` here — that # re-adds a 400-rejected param on top of the profile output # (#4161). MiniMax still needs reasoning_split, which the # profile path does not add. _tg_extra = dict(api_kwargs.get('extra_body') or {}) if _is_minimax_route(getattr(agent, 'provider', ''), getattr(agent, 'model', ''), getattr(agent, 'base_url', '')): _tg_extra['reasoning_split'] = True if _tg_extra: api_kwargs['extra_body'] = _tg_extra if 'max_completion_tokens' in api_kwargs: api_kwargs['max_completion_tokens'] = max_tokens else: api_kwargs['max_tokens'] = max_tokens resp = agent._ensure_primary_openai_client(reason='title_generation').chat.completions.create( **api_kwargs, ) raw, empty_status = _extract_title_response(resp) raw = str(raw or '').strip() if raw: return raw, ('llm' if idx == 0 and budget_idx == 0 else 'llm_retry') last_status = empty_status or 'llm_empty' if budget_idx == 0 and _title_retry_status(last_status): budgets.append(_title_retry_completion_budget( getattr(agent, 'provider', ''), getattr(agent, 'model', ''), getattr(agent, 'base_url', ''), )) except Exception as e: last_status = 'llm_error' logger.debug( "Agent title generation attempt %s failed: provider=%s model=%s error=%s", idx + 1, getattr(agent, 'provider', None), getattr(agent, 'model', None), e, ) # If the model just burned its budget on hidden reasoning, retrying # the next prompt against the same model produces the same shape. # Short-circuit to the local fallback path (#2083). if _title_should_skip_remaining_attempts(last_status): logger.debug( "Agent title generation short-circuiting after %s (reasoning-only response).", last_status, ) break return None, last_status except Exception as e: logger.debug("Agent title generation failed: %s", e) return None, 'llm_error' finally: agent.reasoning_config = prev_reasoning def _generate_llm_session_title_for_agent(agent, user_text: str, assistant_text: str) -> tuple[Optional[str], str, str]: """Generate a title via active-agent route, then sanitize/validate result.""" raw, status = generate_title_raw_via_agent(agent, user_text, assistant_text) if not raw: return None, status, '' title = _sanitize_generated_title(raw) if title: if _title_language_mismatch(user_text, title): return None, 'llm_language_mismatch', str(raw)[:120] return title, status, '' return None, 'llm_invalid', str(raw)[:120] def _generate_llm_session_title_via_aux(user_text: str, assistant_text: str, agent=None, *, use_agent_model: bool = False, conversation_id: str = '') -> tuple[Optional[str], str, str]: """Generate a title via dedicated auxiliary LLM route, then sanitize/validate result. When use_agent_model is False (default), the auxiliary client resolves provider/model/base_url from config.yaml auxiliary.title_generation, which prevents the session's chat model (e.g. a Chinese model) from overriding the dedicated title model. When True, the agent's attrs are passed through (legacy fallback behaviour). conversation_id republishes the webui session id as the Agent's ambient conversation context for the duration of the aux call, so OpenCode relay targets receive the same ``x-opencode-session`` sticky key as the session's main turns (#7470). The context is reset in all exit paths. """ if use_agent_model and agent: provider = getattr(agent, 'provider', '') model = getattr(agent, 'model', '') base_url = getattr(agent, 'base_url', '') else: provider = '' model = '' base_url = '' ctx_token = None if conversation_id: try: from agent.portal_tags import set_conversation_context ctx_token = set_conversation_context(str(conversation_id)) except Exception: # Older/absent agent runtime: proceed without conversation context # (previous behaviour) rather than failing title generation. ctx_token = None try: raw, status = generate_title_raw_via_aux( user_text, assistant_text, provider=provider, model=model, base_url=base_url, ) finally: if ctx_token is not None: try: from agent.portal_tags import reset_conversation_context reset_conversation_context(ctx_token) except Exception: pass if not raw: return None, status, '' title = _sanitize_generated_title(raw) if title: if _title_language_mismatch(user_text, title): return None, 'llm_language_mismatch_aux', str(raw)[:120] return title, status, '' return None, 'llm_invalid_aux', str(raw)[:120] def _put_title_status(put_event, session_id: str, status: str, reason: str = '', title: str = '', raw_preview: str = '') -> None: payload = {'session_id': session_id, 'status': status} if reason: payload['reason'] = reason if title: payload['title'] = title if raw_preview: payload['raw_preview'] = raw_preview put_event('title_status', payload) logger.info( "title_status session=%s status=%s reason=%s title=%r raw_preview=%r", session_id, status, reason or '-', title or '', (raw_preview or '')[:120], ) def _fallback_title_from_exchange(user_text: str, assistant_text: str) -> Optional[str]: """Generate a readable local fallback title when LLM title generation fails.""" user_text = (user_text or '').strip() assistant_text = _strip_thinking_markup(assistant_text or '').strip() if not user_text: return None user_text = _strip_workspace_prefix(user_text) user_text = re.sub(r'\s+', ' ', user_text).strip() assistant_text = re.sub(r'\s+', ' ', assistant_text).strip() combined = f"{user_text} {assistant_text}".strip().lower() combined_raw = f"{user_text} {assistant_text}".strip() def _contains_latin(text: str) -> bool: return bool(re.search(r'[A-Za-z]', text or '')) def _extract_named_topic(text: str) -> str: m = re.search(r'"([^"\n]{2,24})"', text) if m: return (m.group(1) or '').strip() m = re.search(r'“([^”\n]{2,24})”', text) if m: return (m.group(1) or '').strip() return '' topic_name = _extract_named_topic(combined_raw) if topic_name: if not _contains_latin(topic_name): if any(k in combined for k in ('time', 'schedule', 'efficiency', 'manage', 'fitness', 'singing', 'calligraphy')): return 'Time management discussion' if any(k in combined for k in ('hermes', 'codex', 'ai')): return 'AI productivity discussion' return 'Conversation topic' if any(k in combined for k in ('time', 'schedule', 'efficiency', 'manage', 'fitness', 'singing', 'calligraphy')): return f'{topic_name} time management' if any(k in combined for k in ('hermes', 'codex', 'ai')): return f'{topic_name} AI productivity' return f'{topic_name} discussion' if any(k in combined for k in ('title', 'session title')) and any(k in combined for k in ('summary', 'summar', 'short title')): if any(k in combined for k in ('test', 'ok', 'reply ok')): return 'Session title auto-summary test' return 'Session title auto-summary' if any(k in combined for k in ('clarify', 'clarification')) and any(k in combined for k in ('dialog', 'card')): return 'Clarify dialog card' if any(k in combined for k in ('issue', 'github', 'pr')) and any(k in combined for k in ('triage', 'bug', 'review')): return 'GitHub Issue Triage' head = re.split(r'[.!?\n]', user_text)[0].strip() if not head: return None stop_en = { 'the', 'this', 'that', 'with', 'from', 'into', 'just', 'reply', 'please', 'need', 'needs', 'want', 'wants', 'user', 'assistant', 'could', 'would', 'should', 'about', 'there', 'here', 'test', 'testing', 'title', 'summary', } # Unicode-aware Latin tokenization: keep the old "no leading underscore" # and non-Latin placeholder behavior while allowing letters such as ä/ö/ü/ß. # The previous ASCII-only pattern turned "führe" into "f" + "hre"; the short # "f" was filtered and the broken "hre" became part of the title. latin_word = r'A-Za-z0-9À-ÖØ-öø-ÿ' tokens = re.findall(rf'[{latin_word}][{latin_word}_./+-]*', head) if not tokens: return 'Conversation topic' picked = [] for tok in tokens: lower_tok = tok.lower() if lower_tok in stop_en or len(lower_tok) < 3: continue if tok not in picked: picked.append(tok) if len(picked) >= 4: break if picked: return ' '.join(picked)[:60] return 'Conversation topic' def _is_generic_fallback_title(title: str) -> bool: """Return True for low-information fallback labels that should not be persisted.""" return str(title or '').strip().lower() in {'conversation topic'} def _run_background_title_update(session_id: str, user_text: str, assistant_text: str, placeholder_title: str, put_event, agent=None): """Generate and publish a better title after `done`, then end the stream.""" try: try: s = get_session(session_id) except KeyError: _put_title_status(put_event, session_id, 'skipped', 'missing_session') return # Allow self-heal when a previously generated title leaked thinking text. _invalid_existing = _looks_invalid_generated_title(s.title) if getattr(s, 'llm_title_generated', False) and not _invalid_existing: _put_title_status(put_event, session_id, 'skipped', 'already_generated', str(s.title or '')) return current = str(s.title or '').strip() if session_has_manual_title(s): _put_title_status(put_event, session_id, 'skipped', 'manual_title', current) return still_auto = ( current == placeholder_title or current in ('Untitled', 'New Chat', '') or _is_provisional_title(current, s.messages) or _invalid_existing ) if not still_auto: _put_title_status(put_event, session_id, 'skipped', 'manual_title', current) return from api import profiles as profiles_api with profiles_api.profile_env_for_background_worker(s, "background title", logger_override=logger): if not _aux_title_generation_enabled(): _put_title_status(put_event, session_id, 'skipped', 'title_generation_disabled', current) return aux_title_configured = _aux_title_configured() if agent and not aux_title_configured: next_title, llm_status, raw_preview = _generate_llm_session_title_for_agent(agent, user_text, assistant_text) if not next_title and llm_status in ('llm_error', 'llm_invalid'): next_title, llm_status, raw_preview = _generate_llm_session_title_via_aux(user_text, assistant_text, agent=agent, use_agent_model=True, conversation_id=session_id) else: next_title, llm_status, raw_preview = _generate_llm_session_title_via_aux(user_text, assistant_text, conversation_id=session_id) if not next_title and agent and llm_status in ('llm_error_aux', 'llm_invalid_aux'): next_title, llm_status, raw_preview = _generate_llm_session_title_for_agent(agent, user_text, assistant_text) source = llm_status if not next_title: fallback_title = _fallback_title_from_exchange(user_text, assistant_text) if fallback_title and not _is_generic_fallback_title(fallback_title): logger.debug("Using local fallback for session title generation") next_title = fallback_title source = 'fallback' elif fallback_title: logger.debug("Skipping generic local fallback for session title generation: %r", fallback_title) fallback_reason = ( f'local_summary:{llm_status}' if source == 'fallback' and llm_status else 'local_summary' ) wrote_title = False effective_title = current if next_title: with _get_session_agent_lock(session_id): with LOCK: cached_session = SESSIONS.get(session_id) if cached_session is not None and getattr(cached_session, 'session_id', None) == session_id: s = cached_session effective_title = str(s.title or '').strip() manual_title = session_has_manual_title(s) invalid_existing_now = _looks_invalid_generated_title(s.title) still_auto = ( effective_title == placeholder_title or effective_title in ('Untitled', 'New Chat', '') or _is_provisional_title(effective_title, s.messages) or invalid_existing_now ) if manual_title or not still_auto: _put_title_status(put_event, session_id, 'skipped', 'manual_title', effective_title) return if next_title != effective_title: s.title = next_title mark_session_title_generated(s) # Keep chronological ordering stable in the sidebar. s.save(touch_updated_at=False) effective_title = s.title wrote_title = True if wrote_title: if source == 'fallback': _put_title_status(put_event, session_id, source, fallback_reason, effective_title, raw_preview) else: _put_title_status(put_event, session_id, source, llm_status, effective_title, raw_preview) put_event('title', {'session_id': session_id, 'title': effective_title}) # Sync the generated title to state.db so `hermes sessions list` shows it. try: from api.state_sync import sync_session_title sync_session_title(session_id, effective_title, profile=getattr(s, 'profile', None) or 'default') except Exception: logger.debug("Failed to sync title to state.db after generation for %s", session_id) else: _put_title_status(put_event, session_id, 'skipped', source or 'unchanged', effective_title, raw_preview) finally: put_event('stream_end', {'session_id': session_id}) def _run_background_title_refresh(session_id: str, user_text: str, assistant_text: str, current_title: str, put_event, agent=None): """Refresh an existing LLM-generated title using the latest exchange text. Unlike _run_background_title_update, this does NOT guard on llm_title_generated — it assumes the title was already LLM-generated and the session has progressed enough to warrant a refresh. It does NOT emit stream_end (the caller already did). """ try: try: s = get_session(session_id) except KeyError: return # Safety: skip if user manually renamed since the check effective = str(s.title or '').strip() if session_has_manual_title(s): _put_title_status(put_event, session_id, 'skipped', 'manual_title', effective) return if effective != current_title: _put_title_status(put_event, session_id, 'skipped', 'manual_title', effective) return if not effective or effective in ('Untitled', 'New Chat'): return from api import profiles as profiles_api with profiles_api.profile_env_for_background_worker(s, "background title", logger_override=logger): if not _aux_title_generation_enabled(): _put_title_status(put_event, session_id, 'refresh_skipped', 'title_generation_disabled', effective) return aux_title_configured = _aux_title_configured() if agent and not aux_title_configured: next_title, llm_status, raw_preview = _generate_llm_session_title_for_agent(agent, user_text, assistant_text) if not next_title and llm_status in ('llm_error', 'llm_invalid'): next_title, llm_status, raw_preview = _generate_llm_session_title_via_aux(user_text, assistant_text, agent=agent, use_agent_model=True, conversation_id=session_id) else: next_title, llm_status, raw_preview = _generate_llm_session_title_via_aux(user_text, assistant_text, conversation_id=session_id) if not next_title and agent and llm_status in ('llm_error_aux', 'llm_invalid_aux'): next_title, llm_status, raw_preview = _generate_llm_session_title_for_agent(agent, user_text, assistant_text) if not next_title: _put_title_status(put_event, session_id, 'refresh_skipped', llm_status or 'empty', effective, raw_preview) return # Skip if the new title is essentially the same (after normalization) normalized_current = re.sub(r'\s+', ' ', effective).strip().lower() normalized_new = re.sub(r'\s+', ' ', next_title).strip().lower() if normalized_current == normalized_new: _put_title_status(put_event, session_id, 'refresh_skipped', 'same_title', effective, raw_preview) return with _get_session_agent_lock(session_id): with LOCK: cached_session = SESSIONS.get(session_id) if cached_session is not None and getattr(cached_session, 'session_id', None) == session_id: s = cached_session # Re-check: user may have renamed while we were generating if session_has_manual_title(s) or str(s.title or '').strip() != current_title: _put_title_status(put_event, session_id, 'skipped', 'manual_title', str(s.title or '').strip()) return s.title = next_title mark_session_title_generated(s) effective_title = s.title # Session.save() calls _write_session_index(), which acquires LOCK. # Keep the per-session agent lock for mutation serialization, but # release the global session LOCK before persisting to avoid a # self-deadlock in the background title-refresh thread. s.save(touch_updated_at=False) _put_title_status(put_event, session_id, 'refreshed', llm_status, effective_title, raw_preview) put_event('title', {'session_id': session_id, 'title': effective_title}) # Sync the refreshed title to state.db so `hermes sessions list` stays current. try: from api.state_sync import sync_session_title sync_session_title(session_id, effective_title, profile=getattr(s, 'profile', None) or 'default') except Exception: logger.debug("Failed to sync refreshed title to state.db for %s", session_id) logger.info("Adaptive title refresh: session=%s new_title=%r", session_id, effective_title) except Exception: logger.debug("Background title refresh failed for session %s", session_id, exc_info=True) def generate_session_title_for_session(session, *, prefer_latest: bool = False, agent=None) -> tuple[Optional[str], str, str]: """Generate a session title on demand from persisted conversation messages. This helper powers explicit UI title-regeneration controls. It intentionally does not inspect or mutate ``llm_title_generated``; callers decide whether replacing the current title is allowed, then persist the returned title. """ messages = getattr(session, 'messages', None) or [] if prefer_latest: user_text, assistant_text = _latest_exchange_snippets(messages) else: # Manual "Regenerate title" (#7543): scan past consecutive opening user # rows to the first complete user+assistant pair, so a transcript that # opens with queued user turns still reaches the aux LLM instead of # silently persisting the local fallback. The automatic in-stream path # keeps the default (no scan-past) so its stream teardown is unchanged. user_text, assistant_text = _first_exchange_snippets(messages, scan_past_consecutive_users=True) if not user_text: return None, 'empty_user_message', '' from api import profiles as profiles_api with profiles_api.profile_env_for_background_worker(session, "manual title regeneration", logger_override=logger): if not _aux_title_generation_enabled(): return None, 'title_generation_disabled', '' next_title, llm_status, raw_preview = _generate_llm_session_title_via_aux(user_text, assistant_text, agent=agent, conversation_id=getattr(session, 'session_id', '') or '') if next_title: return next_title, llm_status, raw_preview fallback_title = _fallback_title_from_exchange(user_text, assistant_text) if fallback_title and not _is_generic_fallback_title(fallback_title): reason = f'local_summary:{llm_status}' if llm_status else 'local_summary' return fallback_title, reason, raw_preview return None, llm_status or 'empty_title', raw_preview def _preserve_pre_compression_snapshot(s, old_sid: str) -> None: """Persist old_sid as a read-only pre-compression snapshot. Context compression rotates the active WebUI session id from old_sid to the agent's new continuation id. The old JSON must remain on disk for lineage traversal, but it should not continue to appear as an active sidebar row. """ old_path = SESSION_DIR / f'{old_sid}.json' if not old_path.exists(): return try: existing_text = old_path.read_text(encoding='utf-8') try: existing = json.loads(existing_text) existing_msgs = len(existing.get('messages') or []) existing_snapshot = bool(existing.get('pre_compression_snapshot')) except (json.JSONDecodeError, ValueError): # Treat corrupt/malformed old JSON as missing history and rewrite it # from the in-memory pre-compression messages below. That is safer # than leaving an unreadable recovery snapshot behind. existing_msgs = -1 existing_snapshot = False if len(s.messages) > existing_msgs: # In-memory messages are newer than the file; save the full old # snapshot from the current session object while preserving its # pre-existing parent_session_id lineage. saved_sid = s.session_id saved_snapshot = bool(getattr(s, 'pre_compression_snapshot', False)) saved_pinned = bool(getattr(s, 'pinned', False)) s.session_id = old_sid s.pre_compression_snapshot = True s.pinned = False # Stage-359 / PR #2295: clear runtime stream-state fields on the # archived snapshot so the sidebar does not reopen the parent as # a permanently-running session while the child already holds the # completed answer. The continuation session's live state is # restored from saved_* locals in the finally block. saved_active_stream_id = getattr(s, 'active_stream_id', None) saved_pending_user_message = getattr(s, 'pending_user_message', None) saved_pending_attachments = list(getattr(s, 'pending_attachments', []) or []) saved_pending_started_at = getattr(s, 'pending_started_at', None) saved_pending_user_source = getattr(s, 'pending_user_source', None) s.active_stream_id = None s.pending_user_message = None s.pending_attachments = [] s.pending_started_at = None s.pending_user_source = None try: # skip_index=False so the snapshot appears in _index.json with # the pre_compression_snapshot marker. The sidebar projection # (#2285) reads that marker to hide the snapshot from active # rows while keeping the JSON discoverable for lineage traversal. s.save(touch_updated_at=False, skip_index=False) logger.info( "Preserved pre-compression session %s (%d messages) to disk", old_sid, len(s.messages), ) finally: s.session_id = saved_sid s.pre_compression_snapshot = saved_snapshot s.pinned = saved_pinned s.active_stream_id = saved_active_stream_id s.pending_user_message = saved_pending_user_message s.pending_attachments = saved_pending_attachments s.pending_started_at = saved_pending_started_at s.pending_user_source = saved_pending_user_source return # Existing file is already at least as complete as memory; stamp only # the snapshot marker so index/sidebar projection can hide it without # rewriting a shorter messages array over a fuller transcript. from api.models import Session snapshot = Session.load(old_sid) if snapshot: snapshot.pre_compression_snapshot = True snapshot.pinned = False # Stage-359 Opus SHOULD-FIX: clear runtime fields on the loaded # snapshot too. If the disk snapshot was last persisted while the # parent was live, it could carry a stale active_stream_id / # pending_* over to disk. The sidebar projection filters snapshot # rows so this is latent today, but the contract should match the # primary branch above so future readers can trust snapshot files # to never contain live runtime state. snapshot.active_stream_id = None snapshot.pending_user_message = None snapshot.pending_attachments = [] snapshot.pending_started_at = None snapshot.pending_user_source = None snapshot.save(touch_updated_at=False, skip_index=False) logger.info( "Marked pre-compression session %s as sidebar-hidden snapshot", old_sid, ) except OSError: logger.debug("Could not read old session file before preservation") except Exception: logger.debug("Failed to preserve pre-compression session file", exc_info=True) def _maybe_schedule_title_refresh(session, put_event, agent): """Check if the session is due for an adaptive title refresh and schedule it.""" refresh_interval = _get_title_refresh_interval() if refresh_interval <= 0: return current_title = str(session.title or '').strip() if not current_title or current_title in ('Untitled', 'New Chat'): return if session_has_manual_title(session): return if not getattr(session, 'llm_title_generated', False): return exchange_count = _count_exchanges(session.messages) if exchange_count <= 0 or exchange_count % refresh_interval != 0: return last_u, last_a = _latest_exchange_snippets(session.messages) if not last_u and not last_a: return threading.Thread( target=_run_background_title_refresh, args=(session.session_id, last_u, last_a, current_title, put_event, agent), daemon=True, ).start() def _strip_native_image_parts_from_content(content): """Return provider-safe content with native image parts removed. Text-only provider endpoints (for example DeepSeek/OpenAI-compatible text models) reject historical OpenAI-style ``image_url`` parts before the agent can recover. When WebUI is configured for text-mode image handling, preserve textual content from mixed content arrays and drop only the native image blocks from replayed history. """ if not isinstance(content, list): return content clean_parts = [] for part in content: if not isinstance(part, dict): continue if part.get('type') == 'image_url' or 'image_url' in part: continue clean_parts.append(copy.deepcopy(part)) if not clean_parts: return '' if len(clean_parts) == 1 and clean_parts[0].get('type') == 'text': return str(clean_parts[0].get('text') or '') return clean_parts _OOB_USER_MESSAGE_BLOCK_RE = re.compile( r'\[OUT-OF-BAND\s+USER\s+MESSAGE(?:\s*(?:—|-)\s*.*?)?\]\s*?.*?\[/OUT-OF-BAND\s+USER\s+MESSAGE\]', re.DOTALL | re.IGNORECASE, ) def _strip_oob_blocks(content): """Remove consumed [OUT-OF-BAND USER MESSAGE ...] blocks from content. These markers are internal control data that should never reach the model. They can appear as plain strings or inside list-based content parts. """ if isinstance(content, str): return _OOB_USER_MESSAGE_BLOCK_RE.sub('', content) if isinstance(content, list): return [_strip_oob_blocks(part) for part in content] if isinstance(content, dict): return { key: _strip_oob_blocks(value) if isinstance(value, (str, list, dict)) else copy.deepcopy(value) for key, value in content.items() } return content _OOB_ANY_OPEN_RE = re.compile( r'\[OUT-OF-BAND\s+USER\s+MESSAGE(?:\s*(?:—|-)\s*.*?)?\]', re.IGNORECASE, ) _OOB_ANY_CLOSE_RE = re.compile( r'\[/OUT-OF-BAND\s+USER\s+MESSAGE\]', re.IGNORECASE, ) def _unwrap_single_oob_frame(content: str) -> str | None: """Unwrap exactly one fully-validated [OUT-OF-BAND USER MESSAGE] frame. Returns the extracted inner user text if and only if ``content`` consists of exactly one valid opening tag and one valid closing tag wrapping the user content. If markers are multiple, nested, incomplete, or ambiguous, returns None so caller preserves the row byte-for-byte. """ if not isinstance(content, str): return None stripped = content.strip() if not stripped: return None open_matches = list(_OOB_ANY_OPEN_RE.finditer(stripped)) close_matches = list(_OOB_ANY_CLOSE_RE.finditer(stripped)) # Must have exactly one opening marker and one closing marker if len(open_matches) != 1 or len(close_matches) != 1: return None open_m = open_matches[0] close_m = close_matches[0] # Opening marker must be at the very start of stripped content if open_m.start() != 0: return None # Closing marker must be at the very end of stripped content if close_m.end() != len(stripped): return None # Opening marker must end before closing marker starts if open_m.end() > close_m.start(): return None inner = stripped[open_m.end():close_m.start()] # Strip surrounding whitespace/newlines from the extracted user text return inner.strip('\r\n').strip() def _unwrap_steer_row_oob_marker(message: dict) -> None: """Extract inner steer text from a typed steer row in place (#7600). The Hermes Agent emits mid-turn steers as standalone typed user rows (`role == 'user'`, `display_kind == 'steer'`). The control wrapper `[OUT-OF-BAND USER MESSAGE ...] ... [/OUT-OF-BAND USER MESSAGE]` is extracted to preserve only the user-authored instruction. Mutates caller-row in place to maintain object identity. If the marker frame is malformed, nested, multiple, or legacy, preserves the row byte-for-byte. """ if not isinstance(message, dict): return if message.get('role') != 'user' or message.get('display_kind') != 'steer': return content = message.get('content') if isinstance(content, str): unwrapped = _unwrap_single_oob_frame(content) if unwrapped is not None: message['content'] = unwrapped elif isinstance(content, list): if len(content) == 1 and isinstance(content[0], dict): part = content[0] if part.get('type') == 'text' and isinstance(part.get('text'), str): unwrapped = _unwrap_single_oob_frame(part['text']) if unwrapped is not None: part['text'] = unwrapped def _strip_oob_markers_from_messages(messages): """Unwrap OOB steer markers from typed steer rows in place (#7600).""" for message in messages or []: _unwrap_steer_row_oob_marker(message) return messages def _content_has_reasoning_only_parts(content) -> bool: if not isinstance(content, list) or not content: return False saw_reasoning = False for part in content: if not isinstance(part, dict): continue part_type = part.get('type') if part_type in {'thinking', 'reasoning'}: text = part.get('thinking') or part.get('reasoning') or part.get('text') or '' if str(text).strip(): saw_reasoning = True continue if part_type == 'text' and str(part.get('text') or part.get('content') or '').strip(): return False if part_type not in {'text', 'thinking', 'reasoning'}: return False return saw_reasoning def _is_reasoning_only_assistant_message(msg) -> bool: """Return True for display-only assistant Thinking entries. These entries keep partial Thinking cards visible after reload/cancel, but they are not API-safe history: providers only see a blank assistant turn. Visible assistant replies that also carry reasoning metadata are kept. """ if not isinstance(msg, dict) or msg.get('role') != 'assistant': return False if msg.get('tool_calls'): return False content = msg.get('content', '') if _message_text(content).strip(): return False if str(msg.get('reasoning') or msg.get('reasoning_content') or '').strip(): return True return _content_has_reasoning_only_parts(content) def _is_local_reasoning_replay_base_url(base_url: str | None) -> bool: """Return True when a custom provider base URL confidently points at localhost.""" if not base_url: return False try: from urllib.parse import urlsplit raw = str(base_url or '').strip() if not raw: return False parsed = urlsplit(raw) if not parsed.hostname and '://' not in raw: parsed = urlsplit(f"http://{raw}") host = (parsed.hostname or '').strip().lower() except Exception: return False return host in {'localhost', '127.0.0.1', '::1', 'localhost.localdomain'} def _should_strip_reasoning_content( cfg: dict | None, *, mode: str | None = None, effective_model: str | None = None, effective_provider: str | None = None, effective_base_url: str | None = None, ) -> bool: """Decide whether historical assistant reasoning_content should be stripped from model-facing history. This is a provider/protocol decision, not a model-capability heuristic. Local/generic backends (LM Studio, llama.cpp, Ollama, and custom localhost OpenAI-compatible endpoints) do not require historical reasoning replay and receive stale content when it's preserved. Unknown providers preserve by default so replay does not break providers that require reasoning/tool-call continuity. Args: cfg: Config dict from get_config(), expected to contain webui.reasoning_content_replay. mode: Explicit override mode ("strip", "preserve", "auto"). If provided, bypasses config lookup. effective_model: Runtime-resolved model for the current session/request. effective_provider: Runtime-resolved provider for the current session/request. effective_base_url: Runtime-resolved base URL for custom providers. Returns: True if reasoning_content should be stripped from sanitized output. False if it should be preserved in sanitized output. """ # Explicit mode override takes priority if mode is not None: return mode == "strip" # Config lookup. Missing/invalid config preserves shipped behavior: do not # strip reasoning_content unless config explicitly requests it or auto can # identify a local/generic effective backend. if cfg is None: return False webui_cfg = cfg.get("webui", {}) or {} if not isinstance(webui_cfg, dict): return False replay_mode = webui_cfg.get("reasoning_content_replay") if not isinstance(replay_mode, str): return False normalized = replay_mode.strip().lower() if normalized == "preserve": return False if normalized == "strip": return True if normalized == "auto": # Auto mode: prefer runtime-resolved provider/model because profile # defaults may differ from a per-session/request override. model_cfg = cfg.get("model", {}) or {} if not isinstance(model_cfg, dict): model_cfg = {} provider_id = str(effective_provider or model_cfg.get("provider", "") or "").strip().lower() model_id = str( effective_model or model_cfg.get("default") or model_cfg.get("name") or "" ).strip().lower() base_url = effective_base_url or model_cfg.get("base_url") if not provider_id: return False # Known providers that require historical reasoning_content replay: # - DeepSeek thinking mode distinguishes normal history from tool-call reasoning chains # - Anthropic Claude 4+/3.7+ uses structured reasoning in tool-use contexts # - OpenAI GPT-5+/o-series requires reasoning replay for tool-use continuity if provider_id == "deepseek": return False # preserve for DeepSeek if provider_id == "anthropic" and model_id.startswith("claude"): return False # preserve for Claude models if provider_id == "openai": # Preserve for GPT-5+ and o-series (not GPT-4o, etc.) # Use exact match + dash-prefixed suffix to avoid broad substring matches. _is_reasoning_model = ( model_id == "gpt-5" or model_id.startswith("gpt-5-") or model_id in {"o1", "o3", "o4"} or model_id.startswith(("o1-", "o3-", "o4-")) ) if _is_reasoning_model: return False # preserve for reasoning-capable OpenAI models if provider_id in {"lmstudio", "ollama", "llamacpp", "llama.cpp"}: return True if provider_id == "custom" or provider_id.startswith("custom:"): return _is_local_reasoning_replay_base_url(base_url) # Unknown/cloud providers preserve by default. return False # Unknown mode -- preserve default behavior. return False def _compact_image_parts_for_persistence(messages) -> int: """Replace persisted image parts with text placeholders after a completed turn. The active model receives native image parts while a tool call is running. Once the turn has completed, retaining base64 data URLs in both the visible transcript and ``context_messages`` makes every JSON sidecar save/load and session API response scale with the image bytes. Hermes Agent's durable session store applies the same text-only policy for completed multimodal tool results. Keep text parts and the surrounding tool-call chain intact so future turns retain the conversational record and can re-open the original image from the preceding tool-call arguments when needed. This intentionally mutates the owned session message rows in place. It is called only after ``run_conversation()`` has returned, so it never removes image parts that the current model invocation still needs. """ changed = 0 for message in messages or (): # Mirror Hermes Agent's durable-session policy: native *tool* results # are transient input for the current model call. User attachments are # a separate product contract and must remain intact here. if not isinstance(message, dict) or message.get('role') != 'tool': continue content = message.get('content') if not isinstance(content, list): continue compacted_content = [] image_parts = 0 for part in content: if not isinstance(part, dict): # A provider can legally return scalar content alongside typed # parts. Preserve it unchanged rather than flattening/reordering # the structured result during durable-session compaction. compacted_content.append(part) continue part_type = part.get('type') # Guard the set-membership with an isinstance check: a JSON-valid # part can carry an unhashable ``type`` (e.g. a list), and # ``unhashable in {...}`` raises TypeError — which would turn an # otherwise-complete streaming send into the error path before the # session is saved. Only the three string image types are compacted; # every other part (including non-string ``type`` values) is preserved. if isinstance(part_type, str) and part_type in {'image', 'image_url', 'input_image'}: compacted_content.append({'type': 'text', 'text': '[screenshot]'}) image_parts += 1 else: compacted_content.append(part) if image_parts: message['content'] = compacted_content changed += image_parts return changed def _compact_session_image_parts_for_persistence(session) -> int: """Compact completed native-vision tool results in both durable histories.""" changed = ( _compact_image_parts_for_persistence(getattr(session, 'context_messages', None)) + _compact_image_parts_for_persistence(getattr(session, 'messages', None)) ) if changed: logger.info( "Compacted %d completed image message part(s) for session %s", changed, getattr(session, 'session_id', None), ) return changed def _sanitize_messages_for_api( messages, *, cfg: dict = None, effective_model: str | None = None, effective_provider: str | None = None, effective_base_url: str | None = None, preserve_api_content: bool = False, requested_provider: str = "", ): """Return a deep copy of messages with only API-safe fields. The webui stores extra metadata on messages (attachments, timestamp, _ts) for display purposes. Some providers (e.g. Z.AI/GLM) reject unknown fields instead of ignoring them, causing HTTP 400 errors on subsequent messages. Also strips orphaned tool-role messages whose tool_call_id cannot be linked to a preceding assistant message with tool_calls. Strictly-conformant providers (Mercury-2/Inception, newer OpenAI models) reject histories containing dangling tool results with a 400 error: "Message has tool role, but there was no previous assistant message with a tool call." If ``agent.image_input_mode`` resolves to ``text``, native historical ``image_url`` content parts are stripped too. Current-turn uploads already respect text mode in ``_build_native_multimodal_message``; this closes the remaining replay gap where an older native image in the saved transcript kept causing 400s on every later text-only turn (#2297). """ strip_native_images = cfg is not None and _resolve_image_input_mode( cfg, effective_provider or "", effective_model or "", requested_provider=requested_provider, ) == "text" allowed_keys = _API_SAFE_MSG_KEYS if preserve_api_content: # ``api_content`` is an internal Agent replay sidecar. It is opt-in # here because direct provider/compression projections must continue to # reject unknown bookkeeping fields. allowed_keys = _API_SAFE_MSG_KEYS | {"api_content"} # First pass: collect all tool_call_ids declared by assistant messages. # Handles both OpenAI ('id') and Anthropic ('call_id') field names. valid_tool_call_ids: set = set() for msg in messages: if not isinstance(msg, dict): continue if msg.get('role') == 'assistant': for tc in msg.get('tool_calls') or []: if isinstance(tc, dict): tid = tc.get('id') or tc.get('call_id') or '' if tid: valid_tool_call_ids.add(tid) # Second pass: build the sanitized list, dropping orphaned tool messages. clean = [] for msg in messages: if not isinstance(msg, dict): continue # Skip display-only Thinking entries. They are visible transcript # metadata, not provider-facing assistant turns. if _is_reasoning_only_assistant_message(msg): continue # Skip persisted error markers — never send them to the LLM as prior context. if msg.get('_error'): continue # Skip _partial markers with no visible content. Partial messages that # carry actual text (e.g. "Python is a high-level…") are kept so the # model can continue from the cut-off point (#893). But empty partials # (reasoning-only or tool-only cancellations where thinking markup was # stripped) have nothing for the model to continue from and cause # API 400 errors on strict providers (empty assistant content). if msg.get('_partial') and not str(msg.get('content') or '').strip(): continue # Note: _recovered user messages are NOT skipped here — they may need # to be retained to preserve role alternation when a kept assistant # follows. The _recovered skip happens in a final pass after orphaned # tool_calls are stripped, so the anchor check is exact (#4283). # Temporarily mark _recovered users so the final pass can find them. is_recovered = msg.get('_recovered') and msg.get('role') == 'user' role = msg.get('role') if role == 'tool': tid = msg.get('tool_call_id') or '' if not tid or tid not in valid_tool_call_ids: # Orphaned tool result — skip to avoid 400 from strict providers. continue sanitized = {k: v for k, v in msg.items() if k in allowed_keys} sanitized = scrub_internal_replay_fields( [sanitized], preserve_message_api_content=preserve_api_content, message_records=True, )[0] if sanitized.get("role") not in {"user", "assistant"}: sanitized.pop("api_content", None) elif not isinstance(sanitized.get("api_content"), str) or not sanitized.get("api_content"): sanitized.pop("api_content", None) # Drop empty tool_calls — strict providers (DeepSeek, newer OpenAI) # reject tool_calls: [] with HTTP 400 even when no orphaned calls exist. if 'tool_calls' in sanitized and not sanitized['tool_calls']: del sanitized['tool_calls'] # Provider-aware reasoning_content stripping from model-facing history. # Historical assistant reasoning_content is stripped only when the user # explicitly requests strip mode or auto mode identifies a local/generic # effective backend. if msg.get('role') == 'assistant' and 'reasoning_content' in sanitized: if _should_strip_reasoning_content( cfg, effective_model=effective_model, effective_provider=effective_provider, effective_base_url=effective_base_url, ): del sanitized['reasoning_content'] if is_recovered: sanitized['_recovered'] = True # temporary marker — stripped before return if 'content' in sanitized: sanitized['content'] = _strip_oob_blocks(sanitized['content']) if strip_native_images and 'content' in sanitized: sanitized['content'] = _strip_native_image_parts_from_content(sanitized.get('content')) if sanitized.get('role'): clean.append(sanitized) # Third pass: strip orphaned tool_calls from assistant messages — calls whose id # has no matching tool-role response in the clean list. Strict providers (DeepSeek, # newer OpenAI) reject with 400 when an assistant message references a tool call that # was never answered (e.g. session aborted before results flushed). answered_ids: set = set() for msg in clean: if msg.get('role') == 'tool': tid = msg.get('tool_call_id') or '' if tid: answered_ids.add(tid) filtered_clean = [] for msg in clean: if msg.get('role') == 'assistant' and msg.get('tool_calls'): kept = [ tc for tc in msg['tool_calls'] if isinstance(tc, dict) and (tc.get('id') or tc.get('call_id') or '') in answered_ids ] if not kept: # All calls orphaned: drop tool_calls key; if no content, drop message. msg = {k: v for k, v in msg.items() if k != 'tool_calls'} if not str(msg.get('content') or '').strip(): continue else: msg = dict(msg, tool_calls=kept) filtered_clean.append(msg) # Fourth pass: drop _recovered user messages unless removing one would fuse # two same-role neighbours. Operating on filtered_clean (post orphaned-tool/ # tool_calls stripping) means the neighbour check is exact (#4283). The # decision uses the ACTUAL kept sequence: the previously-kept message's role # (`final[-1]`) and the next surviving message's role. A _recovered user is # kept ONLY when it separates two assistants (prev kept == assistant AND next # surviving == assistant) — i.e. it is an answered turn whose removal would # leave `assistant, assistant` adjacency. In every other case dropping it is # safe and correct: it would either leave a clean `user, assistant` pair, or # (if next is a user) it is a stale unanswered prompt that must not replay. # Deciding only on "an assistant follows" (ignoring the prev kept role) is the # bug that re-introduced `user, _recovered user, assistant` → adjacent users # → strict-provider 400 once the anchoring assistant's predecessor was a user. final = [] for i, msg in enumerate(filtered_clean): if msg.get('_recovered') and msg.get('role') == 'user': prev_role = final[-1].get('role') if final else None next_role = None for j in range(i + 1, len(filtered_clean)): next_role = filtered_clean[j].get('role') break # Keep only if this recovered user actually separates two assistants. if not (prev_role == 'assistant' and next_role == 'assistant'): continue # drop — fusing the neighbours is clean, or it's a stale prompt # Keep but strip the temporary marker msg = {k: v for k, v in msg.items() if k != '_recovered'} final.append(msg) return final def _sanitize_messages_for_agent( messages, *, cfg: dict = None, effective_model: str | None = None, effective_provider: str | None = None, effective_base_url: str | None = None, requested_provider: str = "", ): """Build the internal Agent replay projection with ``api_content`` intact. ``api_content`` is a durable provider-facing sidecar, not a direct-provider payload field. Keep this opt-in at one named boundary so every Agent history call uses the same contract while ordinary API/compression callers continue to use the default-stripping sanitizer. """ return _sanitize_messages_for_api( messages, cfg=cfg, effective_model=effective_model, effective_provider=effective_provider, effective_base_url=effective_base_url, preserve_api_content=True, requested_provider=requested_provider, ) def _api_safe_message_positions(messages): """Return [(original_index, sanitized_message)] for API-safe messages.""" valid_tool_call_ids: set = set() for msg in messages: if not isinstance(msg, dict): continue if msg.get('role') == 'assistant': for tc in msg.get('tool_calls') or []: if isinstance(tc, dict): tid = tc.get('id') or tc.get('call_id') or '' if tid: valid_tool_call_ids.add(tid) out = [] for idx, msg in enumerate(messages): if not isinstance(msg, dict): continue if _is_reasoning_only_assistant_message(msg): continue if msg.get('_error'): continue if msg.get('_partial') and not str(msg.get('content') or '').strip(): continue # Note: _recovered user messages are NOT skipped here — deferred to # a final pass after orphaned tool_calls stripping (#4283). is_recovered = msg.get('_recovered') and msg.get('role') == 'user' role = msg.get('role') if role == 'tool': tid = msg.get('tool_call_id') or '' if not tid or tid not in valid_tool_call_ids: continue sanitized = {k: v for k, v in msg.items() if k in _API_SAFE_MSG_KEYS} sanitized = scrub_internal_replay_fields( [sanitized], message_records=True, )[0] if 'tool_calls' in sanitized and not sanitized['tool_calls']: del sanitized['tool_calls'] if is_recovered: sanitized['_recovered'] = True # temporary marker — stripped before return if 'content' in sanitized: sanitized['content'] = _strip_oob_blocks(sanitized['content']) if sanitized.get('role'): out.append((idx, sanitized)) # Third pass: strip orphaned tool_calls from assistant messages (mirrors # _sanitize_messages_for_api pass 3). answered_ids: set = set() for _idx, msg in out: if msg.get('role') == 'tool': tid = msg.get('tool_call_id') or '' if tid: answered_ids.add(tid) filtered_out = [] for idx, msg in out: if msg.get('role') == 'assistant' and msg.get('tool_calls'): kept = [ tc for tc in msg['tool_calls'] if isinstance(tc, dict) and (tc.get('id') or tc.get('call_id') or '') in answered_ids ] if not kept: msg = {k: v for k, v in msg.items() if k != 'tool_calls'} if not str(msg.get('content') or '').strip(): continue else: msg = dict(msg, tool_calls=kept) filtered_out.append((idx, msg)) # Fourth pass: drop _recovered user messages unless removing one would fuse # two same-role neighbours — mirrors _sanitize_messages_for_api pass 4 (#4283). # Decide on the ACTUAL kept sequence: prev kept role (final_out[-1]) + next # surviving role. Keep ONLY when it separates two assistants; otherwise drop. final_out = [] for i, (idx, msg) in enumerate(filtered_out): if msg.get('_recovered') and msg.get('role') == 'user': prev_role = final_out[-1][1].get('role') if final_out else None next_role = None for j in range(i + 1, len(filtered_out)): next_role = filtered_out[j][1].get('role') break if not (prev_role == 'assistant' and next_role == 'assistant'): continue msg = {k: v for k, v in msg.items() if k != '_recovered'} final_out.append((idx, msg)) return final_out def _deduplicate_context_messages(messages): """Remove duplicate messages from context by identity, keeping first occurrence. Prevents the agent from seeing the same message twice in conversation_history when result_messages contain duplicates that weren't caught by display-merge. Compression/reference markers are internal recovery material: keep at most one canonical assistant reference so a mis-role ``user`` marker cannot become the next active user instruction. """ if not messages: return messages seen = set() deduped = [] user_exact_index = {} for msg in messages: if _is_context_compression_marker(msg): marker_key = ( '__context_compression_marker__', " ".join(_message_text(msg.get('content', '')).split())[:500], ) if marker_key in seen: continue seen.add(marker_key) if isinstance(msg, dict) and msg.get('role') != 'assistant': msg = copy.deepcopy(msg) msg['role'] = 'assistant' deduped.append(msg) continue if _is_compressed_context_tool_result_summary_message(msg) and not msg.get('tool_call_id'): deduped.append(msg) continue # Context ownership is provider-facing: two rows with identical visible # text but different durable ``api_content`` sidecars are distinct turns. # Keep the display identity unchanged so ordinary transcript dedup still # collapses the same visible row. key = _message_replay_key(msg) if isinstance(msg, dict) and msg.get('role') == 'user' and key is not None: user_exact_key = ( key, msg.get('timestamp'), msg.get('_ts'), msg.get('_source') or 'webui', json.dumps(msg.get('attachments') or [], sort_keys=True, ensure_ascii=False), ) prior_exact_idx = user_exact_index.get(user_exact_key) if msg.get('_active_turn_token'): if prior_exact_idx is not None: deduped[prior_exact_idx] = msg continue if key in seen: deduped.append(msg) user_exact_index[user_exact_key] = len(deduped) - 1 continue elif prior_exact_idx is not None: continue user_exact_index[user_exact_key] = len(deduped) if key is not None and key in seen: continue if key is not None: seen.add(key) deduped.append(msg) return deduped def _assign_stable_message_ids(result_messages, *existing_arrays): """Mint a stable, session-unique integer ``id`` on model-result rows lacking one. Both ``messages`` (display transcript) and ``context_messages`` (model-facing history) are derived from the *same* per-turn ``result['messages']`` dicts. Stamping the id on those shared dicts here makes each logical row carry an identical id in both arrays, so the fork/truncate aligner (``session_ops.truncate_context_for_display_keep``) can match rows by its preferred ``id`` key instead of the fragile content-signature fallback that goes blind on large sessions full of structurally-identical rows. Ids are monotonic within a session (max existing id + 1, seeded from the result rows plus any ``existing_arrays`` passed for collision-avoidance). Across sessions ids may repeat, which is safe: alignment only ever compares rows within one session, and a fork copies both arrays together so the child stays internally consistent. Rows whose historical id was carried forward by ``_restore_reasoning_metadata`` keep it; only genuinely new rows are minted. Returns the number of rows newly stamped. Mutates ``result_messages`` in place. """ if not result_messages: return 0 seed = 0 for arr in (result_messages, *existing_arrays): for m in arr or []: if isinstance(m, dict): mid = m.get('id') if _is_stable_message_id(mid) and mid > seed: seed = mid stamped = 0 for m in result_messages: # Invalid ids (True, 1.0, "3", 0, -1) are re-minted, never kept: they # compare equal to minted integers and would forge successor identity. if isinstance(m, dict) and not _is_stable_message_id(m.get('id')): seed += 1 m['id'] = seed stamped += 1 return stamped def _is_stable_message_id(value) -> bool: """Stable-id contract: a positive ``int`` only (no bool/float/str/0/<0).""" return type(value) is int and value > 0 def _stable_id_counts(messages) -> dict: """Count how many rows carry each valid stable id (1 == sole owner).""" counts: dict = {} for m in messages: if isinstance(m, dict) and _is_stable_message_id(m.get('id')): counts[m['id']] = counts.get(m['id'], 0) + 1 return counts _POST_COMPRESSION_TOOL_RESULT_TOTAL_TOKENS = 4096 _POST_COMPRESSION_TOOL_RESULT_MIN_SNIPPET_TOKENS = 256 _POST_COMPRESSION_TOOL_RESULT_SUMMARY_FLAG = "_webui_pruned_tool_result_summary" _POST_COMPRESSION_TOOL_RESULT_MARKER = "[WebUI compressed-context budget:" _ROUGH_TOKEN_CHARS = 4 def _positive_int_value(value, default: int = 0) -> int: try: parsed = int(value or 0) except (TypeError, ValueError): return default return parsed if parsed > 0 else default def _rough_text_token_count(text: str) -> int: text = str(text or "") if not text: return 0 return max(1, (len(text) + (_ROUGH_TOKEN_CHARS - 1)) // _ROUGH_TOKEN_CHARS) def _post_compression_tool_result_budget(compressor) -> int: budget = _positive_int_value( getattr(compressor, 'tail_token_budget', None), _POST_COMPRESSION_TOOL_RESULT_TOTAL_TOKENS, ) threshold = _positive_int_value(getattr(compressor, 'threshold_tokens', None), 0) if threshold: budget = min(budget, max(512, threshold // 2)) return max(512, budget) def _compressed_context_tool_result_summary(text: str, *, original_tokens: int, keep_tokens: int) -> str: text = str(text or "") keep_chars = max(0, int(keep_tokens or 0) * _ROUGH_TOKEN_CHARS) note_prefix = f"{_POST_COMPRESSION_TOOL_RESULT_MARKER} omitted " note_suffix = ( f" (~{int(original_tokens or 0)} rough tokens) from this tool result; " "the full output remains in the visible transcript/tool log.]" ) if keep_chars < (_POST_COMPRESSION_TOOL_RESULT_MIN_SNIPPET_TOKENS * _ROUGH_TOKEN_CHARS): return f"{note_prefix}{len(text)} chars{note_suffix}" note_len_for_budget = len(f"{note_prefix}{len(text)} of {len(text)} chars{note_suffix}") snippet_limit = max(0, keep_chars - note_len_for_budget - 2) snippet = text[:snippet_limit].rstrip() if not snippet: return f"{note_prefix}{len(text)} chars{note_suffix}" omitted_chars = max(0, len(text) - len(snippet)) note = f"{note_prefix}{omitted_chars} of {len(text)} chars{note_suffix}" return f"{snippet}\n\n{note}" def _is_compressed_context_tool_result_summary_message(msg) -> bool: if not isinstance(msg, dict) or msg.get('role') != 'tool': return False return msg.get(_POST_COMPRESSION_TOOL_RESULT_SUMMARY_FLAG) is True def _hard_prune_post_compression_tool_results(messages, *, compressor=None): if not messages: return messages, 0 budget = _post_compression_tool_result_budget(compressor) raw_tool_tokens = 0 replacements = [] for idx in range(len(messages) - 1, -1, -1): msg = messages[idx] if not isinstance(msg, dict) or msg.get('role') != 'tool': continue content = msg.get('content', '') text = _raw_message_text(content) if not text.strip(): continue token_count = _rough_text_token_count(text) if _is_compressed_context_tool_result_summary_message(msg): raw_tool_tokens += token_count continue remaining = max(0, budget - raw_tool_tokens) if token_count <= remaining: raw_tool_tokens += token_count continue replacements.append((idx, text, token_count, remaining)) # Once a tool row exceeds the residual budget, older tool rows should # not be allowed to spend that same residual budget again. raw_tool_tokens = budget if not replacements: return messages, 0 pruned = copy.deepcopy(list(messages)) for idx, text, token_count, keep_tokens in replacements: msg = pruned[idx] msg['content'] = _compressed_context_tool_result_summary( text, original_tokens=token_count, keep_tokens=keep_tokens, ) msg[_POST_COMPRESSION_TOOL_RESULT_SUMMARY_FLAG] = True return pruned, len(replacements) def _prune_context_tool_results_after_compression(agent, context_messages): """Run the active compressor's cheap tool-result pruning on model context. Auto-compression can happen mid-turn and then the agent may run more tools before producing the final answer. Those completed tail tool results are model-facing context, but they were produced after the compression pass and therefore did not go through the compressor's tool-output pruning. Apply the same cheap pruning once more after a confirmed compression event, then apply a WebUI hard cap to retained tool-result payloads. This keeps the visible transcript untouched while preventing the next turn from seeing raw post-compression tool dumps that the compressor protected as recent tail. """ if not context_messages: return context_messages compressor = getattr(agent, 'context_compressor', None) prune = getattr(compressor, '_prune_old_tool_results', None) pruned_messages = context_messages if callable(prune): try: candidate_messages, pruned_count = prune( copy.deepcopy(context_messages), protect_tail_count=getattr(compressor, 'protect_last_n', 20), protect_tail_tokens=getattr(compressor, 'tail_token_budget', None), ) if pruned_count: pruned_messages = _deduplicate_context_messages(candidate_messages) except Exception: logger.debug("post-compression context tool-result pruning failed", exc_info=True) hard_pruned_messages, hard_pruned_count = _hard_prune_post_compression_tool_results( pruned_messages, compressor=compressor, ) if hard_pruned_count: return _deduplicate_context_messages(hard_pruned_messages) return pruned_messages def _estimate_post_compression_context_tokens(agent, context_messages, system_message): """Return a display-only estimate for the prepared post-compression request.""" try: from agent import model_metadata messages = context_messages or [] tools = getattr(agent, 'tools', None) or None request_estimator = getattr(model_metadata, 'estimate_request_tokens_rough', None) if callable(request_estimator): estimate = request_estimator(messages, system_prompt=system_message or '', tools=tools) else: message_estimator = getattr(model_metadata, 'estimate_messages_tokens_rough', None) if callable(message_estimator): estimate = message_estimator(messages) if system_message: estimate += message_estimator([{'role': 'system', 'content': system_message}]) if tools: estimate += message_estimator([{'role': 'system', 'content': str(tools)}]) else: try: from agent.context_compressor import _estimate_msg_budget_tokens except ImportError: return None estimate = sum( _estimate_msg_budget_tokens(message) for message in messages if isinstance(message, dict) ) if system_message: estimate += _estimate_msg_budget_tokens({'role': 'system', 'content': system_message}) if tools: estimate += _estimate_msg_budget_tokens({'role': 'system', 'content': str(tools)}) return estimate if isinstance(estimate, int) and estimate > 0 else None except Exception: logger.debug("post-compression context estimate failed", exc_info=True) return None def _restore_reasoning_metadata(previous_messages, updated_messages): """Carry forward display-only metadata lost during API-safe history sanitization. The provider-facing history strips WebUI-only fields like `reasoning`. When the agent returns its new full message history, prior assistant messages come back without that metadata unless we merge it back in by API-history position. This also preserves existing timestamps for unchanged historical messages. Without that, older turns that come back from the agent without `_ts` / `timestamp` can be re-stamped with the current time on every new assistant response, making prior messages appear to "move" in time. """ return _restore_reasoning_metadata_before_boundary(previous_messages, updated_messages) def _restore_reasoning_metadata_before_boundary( previous_messages, updated_messages, current_turn_boundary=None, ): """Boundary-aware core: rows at/after ``current_turn_boundary`` get nothing historical.""" if not previous_messages or not updated_messages: return updated_messages updated_messages = list(updated_messages) prev_safe = _api_safe_message_positions(previous_messages) def _safe_projection(msg): if not isinstance(msg, dict): return None projected = {k: v for k, v in msg.items() if k in _API_SAFE_MSG_KEYS and msg.get('role')} # Mirror the empty-tool_calls drop applied by _api_safe_message_positions # (#5737) so this projection matches the API-safe positions it's aligned # against — otherwise a row stored with tool_calls: [] projects # differently here than in prev_safe and loses its metadata carry-forward. if 'tool_calls' in projected and not projected['tool_calls']: del projected['tool_calls'] if 'content' in projected: projected['content'] = _strip_oob_blocks(projected['content']) return projected safe_pos = 0 # Rows at/after the active-turn boundary belong to the current turn: never # carry historical ids/metadata onto them (same-content successor forgery). while safe_pos < len(prev_safe) and (current_turn_boundary is None or safe_pos < current_turn_boundary): prev_idx, _ = prev_safe[safe_pos] prev_msg = previous_messages[prev_idx] cur_msg = updated_messages[safe_pos] if safe_pos < len(updated_messages) else None if isinstance(prev_msg, dict) and isinstance(cur_msg, dict) and _safe_projection(prev_msg) == _safe_projection(cur_msg): if prev_msg.get('role') == 'assistant' and prev_msg.get('reasoning') and not cur_msg.get('reasoning'): cur_msg['reasoning'] = prev_msg['reasoning'] # Carry the stable per-message id (#context-message-stable-id) forward # the same way timestamp is carried. The agent rebuilds result rows # without our id every turn; without this, historical context rows # would be re-minted a fresh id each turn and drift out of alignment # with their display counterpart. if prev_msg.get('id') is not None and cur_msg.get('id') is None: cur_msg['id'] = prev_msg['id'] if ( prev_msg.get(_POST_COMPRESSION_TOOL_RESULT_SUMMARY_FLAG) is True and cur_msg.get(_POST_COMPRESSION_TOOL_RESULT_SUMMARY_FLAG) is not True ): cur_msg[_POST_COMPRESSION_TOOL_RESULT_SUMMARY_FLAG] = True if prev_msg.get('timestamp') and not cur_msg.get('timestamp'): cur_msg['timestamp'] = prev_msg['timestamp'] elif prev_msg.get('_ts') and not cur_msg.get('_ts') and not cur_msg.get('timestamp'): cur_msg['_ts'] = prev_msg['_ts'] safe_pos += 1 continue safe_pos += 1 return updated_messages def _restore_display_reasoning_metadata(previous_messages, updated_messages, *, current_turn_boundary=None): """Restore display-only thinking rows for visible transcript persistence.""" updated_messages = _restore_reasoning_metadata_before_boundary( previous_messages, updated_messages, current_turn_boundary, ) if not previous_messages or not updated_messages: return updated_messages prev_safe = _api_safe_message_positions(previous_messages) safe_indices = {idx for idx, _ in prev_safe} # Stable-id ownership (#context-message-stable-id) must be one-to-one: # an id reused by another API-safe row in either projection proves nothing. prev_ids = _stable_id_counts(previous_messages[idx] for idx in safe_indices) updated_ids = _stable_id_counts( updated_messages[idx] for idx, _ in _api_safe_message_positions(updated_messages) ) inserted_reasoning_only = 0 for prev_idx, prev_msg in enumerate(previous_messages): if _is_empty_partial_activity_message(prev_msg): continue if prev_idx in safe_indices or not _is_reasoning_only_assistant_message(prev_msg): continue anchor_pos = sum(1 for idx, _ in prev_safe if idx < prev_idx) # A historical reasoning-only row never restores into the current-turn slice. if current_turn_boundary is not None and anchor_pos >= current_turn_boundary: continue safe_pos = anchor_pos + inserted_reasoning_only existing = updated_messages[safe_pos] if safe_pos < len(updated_messages) else None if isinstance(existing, dict) and _is_reasoning_only_assistant_message(existing): continue # Restore only in front of the row's own API-safe successor. A compacted # result has no aligned slot; inserting past its end would append the # historical row after the new reply and re-add it every turn. if anchor_pos >= len(prev_safe) or not isinstance(existing, dict): continue successor = previous_messages[prev_safe[anchor_pos][0]] # Stable ids (#context-message-stable-id) win over content identity: # repeated prompts ("continue") make a distinct current row look like # the historical successor and misplace every anchor before it. successor_id = successor.get('id') existing_id = existing.get('id') if 'id' in successor or 'id' in existing: if not ( _is_stable_message_id(successor_id) and _is_stable_message_id(existing_id) and successor_id == existing_id and prev_ids.get(successor_id) == 1 and updated_ids.get(successor_id) == 1 ): continue else: anchor_key = _message_identity(successor) if anchor_key is None or anchor_key != _message_identity(existing): continue updated_messages.insert(safe_pos, copy.deepcopy(prev_msg)) inserted_reasoning_only += 1 return updated_messages def _session_context_messages(session): """Return model-facing history without assuming it matches the UI transcript.""" context_messages = getattr(session, 'context_messages', None) if isinstance(context_messages, list) and context_messages: return context_messages return session.messages or [] def _message_identity(msg): if not isinstance(msg, dict): return None role = str(msg.get('role') or '') content = msg.get('content', '') text = _message_text(content) if role == 'user': # WebUI sends the model a workspace-prefixed user_message while the # visible optimistic bubble contains only the human text. Treat them as # the same turn for merge/dedup purposes; otherwise compaction results # render two adjacent user bubbles ("Ok" and "[Workspace...]\nOk"). text = _strip_workspace_prefix(text, include_legacy=True) if not text and not msg.get('tool_call_id') and not msg.get('tool_calls'): # Empty assistant messages (e.g. _partial markers with no visible # content) previously returned None, making them invisible to the # merge dedup in _merge_display_messages_after_agent_result. This # caused exponential accumulation: each turn's merge copied ALL # prior _partial messages because they had no identity to track. # Now, _partial messages with empty text get a stable identity # keyed on their role + _partial flag + reasoning/tool metadata, # so the merge can dedup identical empty partials. if msg.get('_partial'): reasoning_key = " ".join(str(msg.get('reasoning') or '').split())[:200] return ( role, '', # empty text '', # no tool_call_id '__partial__' + reasoning_key, ) return None return ( role, " ".join(str(text or '').split())[:500], str(msg.get('tool_call_id') or ''), json.dumps(msg.get('tool_calls') or [], sort_keys=True, ensure_ascii=False), ) def _messages_have_prefix(messages, prefix, *, key_fn=None): key_fn = key_fn or _message_identity if len(messages or []) < len(prefix or []): return False for idx, expected in enumerate(prefix or []): if key_fn((messages or [])[idx]) != key_fn(expected): return False return True def _message_replay_key(msg): """Return a stable comparison key for replay/overlap de-duplication.""" identity = _message_identity(msg) # ``api_content`` is a provider-facing replay sidecar. It must participate # in context/replay overlap identity or two same-visible turns can collapse # before the Agent sees the original wire bytes. Keep synthetic/adjacent # partial collapse on the existing identity path; partial rows are display # bookkeeping rather than durable provider turns. raw_sidecar = ( msg.get("api_content") if isinstance(msg, dict) and not msg.get("_partial") else None ) sidecar = raw_sidecar if isinstance(raw_sidecar, str) and raw_sidecar else None if identity is not None: if sidecar is not None: return (*identity, sidecar) return identity if not isinstance(msg, dict): return None key = ( str(msg.get('role') or ''), _message_text(msg.get('content', '')), str(msg.get('tool_call_id') or ''), json.dumps(msg.get('tool_calls') or [], sort_keys=True, ensure_ascii=False), ) return (*key, sidecar) if sidecar is not None else key def _strip_replayed_prefix(existing_messages, candidates): """Drop a candidate prefix that is already the suffix of existing_messages. Compression/continuation can replay the active tail from state.db after the previous WebUI context/display already contains it. Prefix-only merge logic then treats that replayed tail as a fresh delta and duplicates a whole turn. Strip the largest exact suffix/prefix overlap before appending. """ existing_messages = list(existing_messages or []) candidates = list(candidates or []) max_overlap = min(len(existing_messages), len(candidates)) for overlap in range(max_overlap, 0, -1): left = [_message_replay_key(m) for m in existing_messages[-overlap:]] right = [_message_replay_key(m) for m in candidates[:overlap]] if left == right: return candidates[overlap:] return candidates def _looks_like_replayed_session_arc_summary(previous_msg, candidate_msg): """Return True for repeated LCM/session summaries with refreshed hints. LCM summary cards can be re-injected with the same long recovered context and a different tail such as an expand hint. Exact identity misses those, but appending both copies bloats every later model prompt. """ if not isinstance(previous_msg, dict) or not isinstance(candidate_msg, dict): return False if previous_msg.get('role') != candidate_msg.get('role'): return False previous_api_content = previous_msg.get("api_content") candidate_api_content = candidate_msg.get("api_content") normalized_previous_api_content = ( " ".join(previous_api_content.split()) if isinstance(previous_api_content, str) and previous_api_content else None ) normalized_candidate_api_content = ( " ".join(candidate_api_content.split()) if isinstance(candidate_api_content, str) and candidate_api_content else None ) if normalized_previous_api_content != normalized_candidate_api_content: return False previous_text = " ".join(_message_text(previous_msg.get('content', '')).split()) candidate_text = " ".join(_message_text(candidate_msg.get('content', '')).split()) if len(previous_text) < 2000 or len(candidate_text) < 2000: return False marker = '[Session Arc Summary' if not previous_text.startswith(marker) or not candidate_text.startswith(marker): return False return previous_text[:1500] == candidate_text[:1500] def _strip_replayed_context_items(existing_messages, candidates): """Drop replayed non-adjacent context blocks before persisting context.""" existing_messages = list(existing_messages or []) candidates = list(candidates or []) if not existing_messages or not candidates: return candidates existing_keys = [_message_replay_key(m) for m in existing_messages] candidate_keys = [_message_replay_key(m) for m in candidates] existing_large = [m for m in existing_messages if isinstance(m, dict)] cleaned = [] idx = 0 min_block = 3 while idx < len(candidates): msg = candidates[idx] if any(_looks_like_replayed_session_arc_summary(prev, msg) for prev in existing_large): idx += 1 continue best = 0 for start in range(len(existing_keys)): length = 0 while ( idx + length < len(candidate_keys) and start + length < len(existing_keys) and candidate_keys[idx + length] == existing_keys[start + length] ): length += 1 if length > best: best = length if best >= min_block: idx += best continue cleaned.append(msg) idx += 1 return cleaned def _dedupe_replayed_context_messages(previous_context, result_messages, msg_text=None): """Keep model context append-only without replayed blocks/summaries.""" previous_context = list(previous_context or []) result_messages = list(result_messages or []) if not previous_context or not result_messages: return result_messages previous_user_tail = _stale_user_tail_candidate(_last_user_row(previous_context)) if not _messages_have_prefix( result_messages, previous_context, key_fn=_message_replay_key, ): # Agent-side role-sequence repair can replace the last prior user row # with a repaired current-user row. In that shape the result no longer # has `previous_context` as an exact prefix, but it should still be # merged as: previous context + clean current turn + assistant/tool delta. if ( msg_text and len(previous_context) >= 1 and len(result_messages) >= len(previous_context) and _messages_have_prefix( result_messages, previous_context[:-1], key_fn=_message_replay_key, ) ): boundary_idx = len(previous_context) - 1 boundary_row = result_messages[boundary_idx] is_stale_merge = bool( previous_user_tail and _detect_stale_user_merge( boundary_row, msg_text, previous_user_tail, previous_context=previous_context, ) ) if is_stale_merge or _looks_like_current_user_turn(boundary_row, msg_text): if is_stale_merge: # Clean only the stale-merged boundary row; leave all prior # history in previous_context untouched. cleaned_boundary = copy.deepcopy(boundary_row) cleaned_boundary['content'] = msg_text # The stale row's provider-facing payload describes the # discarded prefix + current turn, not the rewritten # visible turn. Do not let those obsolete bytes survive # into Agent context. cleaned_boundary.pop('api_content', None) candidates = [cleaned_boundary] + result_messages[boundary_idx + 1:] else: candidates = result_messages[boundary_idx:] candidates = _strip_replayed_prefix(previous_context, candidates) if candidates: candidates = _strip_replayed_context_items(previous_context, candidates) return previous_context + candidates assistant_or_tool_only_result = bool(result_messages) and all( _is_context_compression_marker(m) or ( isinstance(m, dict) and m.get('role') in ('assistant', 'tool') ) for m in result_messages ) if assistant_or_tool_only_result: candidates = _strip_replayed_prefix(previous_context, result_messages) if candidates: candidates = _strip_replayed_context_items(previous_context, candidates) return previous_context + candidates return result_messages candidates = result_messages[len(previous_context):] # Strip stale merges only from the new-turn candidate slice so that # legitimate historical user rows in the already-committed previous_context # prefix are never rewritten. if msg_text and previous_user_tail: candidates = _strip_stale_user_merge_from_messages( candidates, msg_text, previous_user_tail, previous_context=previous_context, ) candidates = _strip_replayed_prefix(previous_context, candidates) if candidates: candidates = _strip_replayed_context_items(previous_context, candidates) return previous_context + candidates def _dedupe_replayed_active_context(previous_context, result_messages, msg_text=None): """Keep model context append-only without re-appending a replayed tail.""" return _dedupe_replayed_context_messages(previous_context, result_messages, msg_text) def _is_context_compression_marker(msg): return is_context_compression_marker(msg) def _compact_summary_text(raw_text: str | None) -> str | None: """Normalize a text blob used in compression summary cards.""" if not isinstance(raw_text, str): return None txt = raw_text.strip() if not txt: return None return re.sub(r"\s+", " ", txt).strip() def _compression_anchor_message_key(message): if not isinstance(message, dict): return None role = str(message.get('role') or '') if not role or role == 'tool': return None content = message.get('content', '') text = _message_text(content) if len(text) > 160: text = text[:160] ts = message.get('_ts') or message.get('timestamp') attachments = message.get('attachments') attach_count = len(attachments) if isinstance(attachments, list) else 0 if not text and not attach_count and not ts: return None return {'role': role, 'ts': ts, 'text': text, 'attachments': attach_count} def _compression_summary_from_messages(messages): for m in reversed(messages or []): if not isinstance(m, dict): continue if not _is_context_compression_marker(m): continue text = _message_text(m.get('content')) if text: return text return None def _find_current_user_turn(messages, msg_text): needle = " ".join(str(msg_text or '').split()) last_strong_match = None # _looks_like_current_user_turn (high confidence) last_weak_match = None # needle substring match (lower confidence) fallback = None for idx, msg in enumerate(messages or []): if not isinstance(msg, dict) or msg.get('role') != 'user': continue fallback = idx if _looks_like_current_user_turn(msg, msg_text): last_strong_match = idx continue text = " ".join( _strip_workspace_prefix( _message_text(msg.get('content', '')), include_legacy=True, ).split() ) if needle and (needle in text or text in needle): last_weak_match = idx # Return the LAST matching user turn. After context compression the agent's # result_messages contain the full conversation history; if the user asked a # similar question in an earlier turn, first-match would return that old # index, causing the merge to replay the entire history from that point. # Last-match anchors on the current turn instead. # # Prefer the last STRONG match (an exact `_looks_like_current_user_turn` # hit) over the last WEAK substring match. The agent loop appends synthetic # `role:"user"` continuation prompts (e.g. "Continue", empty-recovery nudges # — see conversation_loop.py) AFTER the real user turn; those can weak-match # `msg_text` and, if weak matches were allowed to win, would anchor the merge # PAST the real turn and drop the assistant/tool output in between. The real # current turn is the last strong match, so it must take priority. if last_strong_match is not None: return last_strong_match if last_weak_match is not None: return last_weak_match return fallback def _drop_checkpointed_current_user_from_context(messages, msg_text): """Return model history without an eager-checkpointed current user turn.""" history = list(messages or []) if not history: return history current_user_key = _message_identity({'role': 'user', 'content': msg_text}) if current_user_key and _message_identity(history[-1]) == current_user_key: return history[:-1] return history def _strip_workspace_prefixes_for_compare(text: str) -> str: """Remove WebUI workspace sentinels anywhere before text comparison.""" value = _strip_workspace_prefix(text, include_legacy=True) for pattern in (_WORKSPACE_PREFIX_ANY_RE, _LEGACY_WORKSPACE_PREFIX_ANY_RE): value = pattern.sub('', value) return value.strip() def _normalize_user_text(text): """Collapse whitespace and strip workspace sentinels for tail comparisons.""" if not isinstance(text, str): return "" return " ".join(_strip_workspace_prefixes_for_compare(text).split()) def _raw_message_text(value) -> str: """Extract text from a message content payload without stripping markup. Used for the stale-user-merge detector so the literal boundary between the prior tail and the current turn survives into the comparison. The thinking-markup strip in ``_message_text`` collapses newlines, which would defeat the boundary check. """ if isinstance(value, list): return ' '.join( _message_content_part_text(p) for p in value if isinstance(p, dict) ) return str(value or '') def _stale_user_tail_candidate(msg): """Return normalized text if msg is a user row that could be a stale tail.""" if not isinstance(msg, dict) or msg.get('role') != 'user': return None raw = _raw_message_text(msg.get('content', '')) if not raw.strip(): return None return _normalize_user_text(raw) def _last_user_row(messages): """Return the last user-role row in `messages`, or None.""" for msg in reversed(list(messages or [])): if isinstance(msg, dict) and msg.get('role') == 'user': return msg return None def _stale_prefix_matches_prior_user_context(stale_prefix, stale_segments, previous_context): """Return True when a stale prefix is explainable by prior user context. First-generation repair usually produces segments matching consecutive prior user rows. Once a session is already contaminated, later repair can replay a stable stale prefix from an older polluted row even after newer clean user turns have moved the context tail forward. Handle both shapes while still requiring all evidence to come from prior user-role rows. """ prior_rows = [ _stale_user_tail_candidate(msg) for msg in previous_context or [] ] prior_rows = [row for row in prior_rows if row] if not prior_rows: return False if stale_segments: segment_count = len(stale_segments) for start in range(0, len(prior_rows) - segment_count + 1): if prior_rows[start:start + segment_count] == stale_segments: return True # Already-polluted sessions may replay paragraphs from older polluted # rows after newer clean turns have advanced the context tail. In that # shape the stale paragraphs are still all prior user content, but they # are substrings within older merged rows rather than standalone rows. # # NOTE: this substring match is intentionally loose (a stale segment can # coincidentally appear inside an unrelated prior row). Correctness does # NOT depend on it being precise — the caller (_detect_stale_user_merge) # only reaches here once the row's suffix already normalizes to the # ENTIRE submitted turn, so anything this branch flags has a prefix that # is extra-to-the-submission. Cleaning therefore only ever rewrites the # row to the user's actual current turn; it can never drop legitimate # current-turn content even on a coincidental substring hit. row_index = 0 row_offset = 0 matched_all_segments = True for segment in stale_segments: matched_segment = False while row_index < len(prior_rows): row = prior_rows[row_index] pos = row.find(segment, row_offset) if pos >= 0: row_offset = pos + len(segment) matched_segment = True break row_index += 1 row_offset = 0 if not matched_segment: matched_all_segments = False break if matched_all_segments: return True prefix_norm = _normalize_user_text(stale_prefix) if not prefix_norm: return False for row in prior_rows: if row == prefix_norm or row.startswith(f'{prefix_norm} '): return True return False def _detect_stale_user_merge(message, msg_text, previous_user_tail, previous_context=None): """Return True if `message` is the current user turn with a stale prefix merged in. The agent's defensive repair path can concatenate prior user context with the submitted current turn as ``\\n\\n``. The stale portion can be either the immediate prior user tail or a replayed prefix from an older already-polluted user row. The literal ``\\n\\n`` boundary must survive into the comparison; a single-newline or space-only join is not the repair shape and must not match. Workspace sentinels may be present on either or both halves and are stripped before comparison. """ if not isinstance(message, dict) or message.get('role') != 'user': return False current_norm = _normalize_user_text(msg_text) if not current_norm: return False merged = _raw_message_text(message.get('content', '')).replace("\r\n", "\n") if "\n\n" not in merged: return False # The user's current turn can itself contain paragraph breaks. Find a repair # boundary whose suffix normalizes to the *entire* submitted turn, then treat # only the prefix as stale context. A plain split-and-last-segment check would # miss ``\n\n\n\n``. stale_segments = [] stale_prefix = '' search_end = len(merged) while search_end > 0: boundary_idx = merged.rfind("\n\n", 0, search_end) if boundary_idx < 0: break suffix = merged[boundary_idx + 2:] if _normalize_user_text(suffix) == current_norm: prefix = merged[:boundary_idx] candidate_segments = [ _normalize_user_text(segment) for segment in prefix.split("\n\n") ] if candidate_segments and all(candidate_segments): stale_segments = candidate_segments stale_prefix = prefix break # The suffix matched the submitted turn, but this boundary leaves # blank prefix segments; try the next candidate boundary to the left. search_end = boundary_idx if not stale_segments: return False if previous_context is not None and _stale_prefix_matches_prior_user_context( stale_prefix, stale_segments, previous_context, ): return True return bool( previous_context is None and len(stale_segments) == 1 and _normalize_user_text(previous_user_tail) == stale_segments[0] ) def _strip_stale_user_merge_from_messages( messages, msg_text, previous_user_tail, previous_context=None, ): """Return messages with stale-prefixed current user turns replaced by clean ones. Both context-merge (model-facing) and display-merge (visible transcript) callers funnel through this so a single detection rule governs persistence. The current user row is replaced with a clean copy using `msg_text` so the displayed bubble matches what the human submitted, never the polluted pair. """ if not messages or not msg_text: return messages out = [] for msg in messages: if _detect_stale_user_merge( msg, msg_text, previous_user_tail, previous_context=previous_context, ): cleaned = copy.deepcopy(msg) if isinstance(msg, dict) else {'role': 'user', 'content': msg_text} cleaned['content'] = msg_text # The stale row's provider-facing payload describes the discarded # merged prefix, so it is invalid once the visible content is # rewritten to the submitted turn. cleaned.pop('api_content', None) out.append(cleaned) else: out.append(msg) return out def _save_streaming_checkpoint(session): """Persist a streaming checkpoint under the session's profile context.""" from api import profiles as profiles_api with profiles_api.profile_env_for_background_worker( session, "streaming checkpoint", logger_override=logger, scope_skill_modules=False, ): session.save(skip_index=True) # A checkpoint rewrite re-serializes the whole session (messages included) while # holding the GIL, so its cost grows with transcript size: ~100 ms for a 1.8 MB # sidecar, ~300 ms for a 36 MB one. Because the HTTP server and the agent workers # share one interpreter, every one of those milliseconds is added latency for all # concurrent requests. Skip the rewrite when it would persist byte-identical state. _CHECKPOINT_IDLE_REFRESH_SECONDS = 300 def _streaming_checkpoint_fingerprint(session): """Return a cheap fingerprint of the state a streaming checkpoint persists. The periodic checkpoint fires on tool completion, but a completed tool call does not by itself mutate the session: ``run_conversation()`` works on an internal copy of ``messages`` and the turn bookkeeping fields are written once before the run starts. So the recurring rewrite normally re-persists state that is already on disk. This fingerprint covers every field the checkpoint can legitimately advance mid-run — session identity (compression rotation), turn bookkeeping, and the length of the persisted message projections — without serializing anything. Comparing it lets the checkpoint thread skip redundant work while still writing within one interval of any real change. Fails closed: if a field cannot be read, return ``None`` so the caller keeps the unconditional write. """ try: return ( getattr(session, 'session_id', None), getattr(session, 'parent_session_id', None), getattr(session, 'profile', None), getattr(session, 'active_stream_id', None), getattr(session, 'pending_user_message', None), getattr(session, 'pending_started_at', None), getattr(session, 'pending_user_source', None), len(getattr(session, 'pending_attachments', None) or ()), len(getattr(session, 'messages', None) or ()), len(getattr(session, 'context_messages', None) or ()), getattr(session, 'compression_anchor_visible_idx', None), bool(getattr(session, 'pre_compression_snapshot', None)), ) except Exception: # pragma: no cover - defensive: never block a checkpoint return None def _normalize_fresh_chat_text(text): text = _strip_workspace_prefix(str(text or ''), include_legacy=True) text = re.sub(r"\s+", " ", text).strip().lower() return text.strip(" \t\r\n.!?。！？,，~～") def _is_casual_fresh_chat_message(msg_text): """Return True for short opener messages that should not resume old tasks.""" text = _normalize_fresh_chat_text(msg_text) if not text or len(text) > 24: return False continuation_terms = ( "continue", "resume", "carry on", "go on", # CJK continuation terms (zh-CN): jixu, jiezhe, wangxia, xiayibu. # Encoded as Python escape sequences (not literal CJK) so api/streaming.py # passes tests/test_title_sanitization.py::test_title_generation_source_has_no_cjk_literals, # which scans this file for any U+4E00-U+9FFF code points. Runtime # comparisons still use the real CJK strings — Python decodes the # escapes at compile time. "\u7ee7\u7eed", "\u63a5\u7740", "\u5f80\u4e0b", "\u4e0b\u4e00\u6b65", ) if any(term in text for term in continuation_terms): return False return text in { "hi", "hello", "hey", "hello there", "hi there", # CJK greetings (zh-CN): nihao, ninhao, hai, haluo, zaima, zaime. # Same escape-sequence rationale as the continuation block above. "\u4f60\u597d", # nihao "\u60a8\u597d", # ninhao "\u55e8", # hai (was \u5616 = "click of tongue", not a greeting) "\u54c8\u55bd", # haluo (was \u54c8\u5582 = uncommon "ha-wei" variant) "\u5728\u5417", # zaima "\u5728\u4e48", # zaime } def _has_task_resume_compaction_marker(messages): """Detect compacted model context that tells the agent to resume an old task.""" for msg in messages or []: if not isinstance(msg, dict): continue text = _message_text(msg.get('content', '')).lower() if not text: continue if "context compaction" not in text and "context compression" not in text: continue if ( "active task" in text or "resume exactly" in text or "current task" in text or "task list was preserved" in text or "in_progress" in text ): return True return False def _new_turn_context_from_messages(messages, msg_text): """Return provider-facing history for a new user turn from a message list.""" history = _drop_checkpointed_current_user_from_context(messages, msg_text) if _is_casual_fresh_chat_message(msg_text) and _has_task_resume_compaction_marker(history): return [] return history def _context_messages_for_new_turn(session, msg_text): """Return provider-facing history for a new user turn. Compacted agent sessions can carry a hidden "resume the active task" summary in context_messages. If the user starts a fresh casual greeting in that old session, do not feed that stale active-task summary back to the model. """ return _new_turn_context_from_messages(_session_context_messages(session), msg_text) def _stream_writeback_is_current(session, stream_id): """Return True only while a worker still owns the session writeback. cancel_stream() intentionally clears ``active_stream_id`` early so the UI can accept a follow-up turn while the old worker is unwinding. That old worker must not later persist its stale result over the newer transcript. """ return bool(stream_id) and getattr(session, 'active_stream_id', None) == stream_id def _stream_writeback_can_supersede_recovery_marker(session, msg_text): """Allow a finishing worker to replace its own stale-repair marker. The stale-pending repair path can occasionally run while the original worker is still alive but temporarily missing from the in-memory stream registry. It clears ``active_stream_id`` and appends a "Response interrupted" marker. If the original worker later finishes, treating ``active_stream_id is None`` as stale drops the real answer and leaves the misleading marker visible. This is intentionally narrow: only a session with no active/pending turn and whose last visible row is the recovery marker for this exact user prompt may be superseded. If a newer turn has appended anything after the marker, the normal stale-writeback guard still wins. """ if getattr(session, 'active_stream_id', None): return False if getattr(session, 'pending_user_message', None): return False if getattr(session, 'pending_attachments', None): return False messages = list(getattr(session, 'messages', None) or []) if len(messages) < 2: return False last = messages[-1] if not isinstance(last, dict) or not last.get('_error'): return False if last.get('type') != 'interrupted': return False content = str(last.get('content') or '') if 'Response interrupted' not in content or 'before this turn finished' not in content: return False expected = ' '.join(str(msg_text or '').split()) if not expected: return False for msg in reversed(messages[:-1]): if not isinstance(msg, dict): continue if msg.get('_error'): continue if msg.get('role') != 'user': continue actual = ' '.join(str(msg.get('content') or '').split()) return actual == expected return False def _advance_truncation_watermark_after_commit(session) -> None: """Advance a positive truncation watermark once a new user turn is committed to ``session.messages`` (#3831). retry/undo/Edit set a positive watermark to suppress the *replaced* tail from the append-only state.db merge; Session.save() deliberately never auto-clears it (#2914). Once the new turn is durably in messages we advance the watermark to the newest user message timestamp so that state.db rows newer than the watermark are still merged in, while the replaced pre-edit tail remains filtered. Never 0.0 (the truncate-to-empty sentinel that must keep blocking replay, #2914). """ if not getattr(session, 'truncation_watermark', None): return messages = getattr(session, 'messages', None) or [] # Walk backwards to find the newest user message timestamp for msg in reversed(messages): if isinstance(msg, dict) and msg.get('role') == 'user': ts = msg.get('timestamp') if isinstance(ts, (int, float)) and ts > 0: session.truncation_watermark = float(ts) return session.truncation_watermark = time.time() def _merge_display_messages_after_agent_result( previous_display, previous_context, result_messages, msg_text, source: str = "webui", verification_nudge_provenance=None, ): """Keep UI transcript durable while allowing model context to compact. If Hermes Agent returns a normal append-only history, append that delta to the UI transcript. If the model/context history was compacted and no longer has the prior context as a prefix, keep the previous UI transcript and append the current user turn onward. Synthetic compaction/reference markers remain internal recovery material and must not become visible user/assistant turns. """ previous_display = [ m for m in list(previous_display or []) if not _is_context_compression_marker(m) and not _is_compressed_context_tool_result_summary_message(m) ] # Drop Hermes Agent internal verify-loop scaffolding (synthetic "premature # done" answer + the "[System: ...verification evidence...]" nudge) before # it can become a visible user/assistant turn. The agent flags these with # structured markers (_verification_stop_synthetic / _pre_verify_synthetic) # and already keeps them out of its own durable store; honor the same # markers here so they never leak into the WebUI transcript. Filter all # three inputs consistently so prefix/delta detection below stays aligned. # (#5334; same internal-control-message class as #3320/#3821/#4373/#4875) previous_display = _drop_synthetic_control_messages(previous_display) # Deduplicate stale _partial messages that accumulated in previous_display. # A bug in cancel_stream() could insert multiple identical _partial messages # when _stripped was empty but _has_reasoning/_has_tools was True. The # merge's _message_identity previously returned None for empty _partial # messages, so the seen-set couldn't catch them — they doubled each turn. # Scan backwards and keep only the LAST occurrence of each unique _partial # identity, then reverse back to original order. _partial_seen = set() _deduped_rev = [] for m in reversed(previous_display): if isinstance(m, dict) and m.get('_partial'): key = _message_identity(m) if key is not None: if key in _partial_seen: continue _partial_seen.add(key) _deduped_rev.append(m) _deduped = list(reversed(_deduped_rev)) if len(_deduped) < len(previous_display): logger.debug( "Deduplicated %d stale _partial messages from previous_display (was %d, now %d)", len(previous_display) - len(_deduped), len(previous_display), len(_deduped), ) previous_display = _deduped previous_context = list(previous_context or []) result_messages = list(result_messages or []) if isinstance(verification_nudge_provenance, dict): _writeback_provenance = verification_nudge_provenance else: _writeback_provenance = { 'verification_nudge_seen': bool(verification_nudge_provenance), 'active_turn_identity': None, } _verification_nudge_seen = _writeback_provenance.get('verification_nudge_seen', False) or any( isinstance(message, dict) and message.get('role') == 'user' and _is_synthetic_control_message(message) for message in result_messages ) _active_turn_identity = _writeback_provenance.get('active_turn_identity') previous_display, _ = _align_current_turn_display( previous_display, previous_context, _active_turn_identity, ) # Same marker filter for the model-history inputs: the synthetic verify-loop # answer/nudge live in the agent's returned messages and prior context, and # would otherwise slip into the merged transcript as a real delta. (#5334) previous_context = _drop_synthetic_control_messages(previous_context) result_messages = _drop_synthetic_control_messages(result_messages) if not result_messages: return previous_display active_turn_row_index = _find_active_turn_checkpoint_index( result_messages, previous_context, _active_turn_identity, msg_text, ) active_turn_row = ( result_messages[active_turn_row_index] if active_turn_row_index is not None else None ) active_turn_display_text = None active_content = active_turn_row.get('content') if isinstance(active_turn_row, dict) else None if ( isinstance(active_turn_row, dict) and active_turn_row.get('role') == 'user' and isinstance(active_content, list) and any( isinstance(part, dict) and isinstance(part.get('type'), str) and part.get('type') in {'image', 'image_url', 'input_image'} for part in active_content ) ): active_turn_display_text = ( _active_turn_identity.get('text') if _active_turn_identity.get('text') is not None else msg_text ) if not isinstance(active_turn_display_text, str): active_turn_display_text = None else: for display_message in previous_display: if _active_turn_token_matches(display_message, _active_turn_identity): display_message['content'] = active_turn_display_text display_message.pop('api_content', None) for key in ('id', 'timestamp', '_row_id'): if display_message.get(key) is None and active_turn_row.get(key) is not None: display_message[key] = copy.deepcopy(active_turn_row[key]) if not display_message.get('attachments') and active_turn_row.get('attachments'): display_message['attachments'] = copy.deepcopy( active_turn_row['attachments'] ) previous_user_tail = _stale_user_tail_candidate(_last_user_row(previous_context)) # ── Backfill normal turns from previous_context that are missing from # previous_display. After context compression recovery, previous_context # can contain user/assistant turns that were never rendered in the visible # transcript (they were behind a compression marker). On the next # append-only merge those turns sit inside the shared prefix and get # stripped, leaving them permanently invisible. Reinsert them now. # # Use display as the backbone to preserve visible order. Walk display in # order and for each display message search for its identity in context # at/after a cursor. Any context messages between the cursor and that # match are context-only gaps that get spliced in before the display msg. if previous_display and previous_context: _displayed_native_image_context_tokens = ( _trusted_native_image_context_display_mirror_tokens( previous_display, previous_context, ) ) _display_id_set = {_message_identity(m) for m in previous_display} _context_id_set = { _message_identity(m) for m in previous_context if not ( isinstance(m, dict) and m.get('_active_turn_token') in _displayed_native_image_context_tokens ) if not _is_context_compression_marker(m) and not _is_compressed_context_tool_result_summary_message(m) } _has_context_only_turns = bool(_context_id_set - _display_id_set) if _has_context_only_turns: context_keys = [ None if ( isinstance(m, dict) and m.get('_active_turn_token') in _displayed_native_image_context_tokens ) else _message_identity(m) for m in previous_context ] # Precompute display keys once; avoids repeated json.dumps calls inside # the inner any() loop (was O(D²·C) — see perf fix below). _display_keys = [_message_identity(m) for m in previous_display] # Multiset mirror of context_keys[_cursor:] kept in sync as _cursor # advances. Enables O(1) membership tests in the any() check instead # of an O(N) list scan, while preserving EXACT list-slice semantics: # _message_identity intentionally returns duplicate keys for # identical-content turns (and None for empty rows), so a plain set # would drop a key still present later in the slice. A count-keyed # dict (including None) matches `in context_keys[_cursor:]` exactly. _remaining_ck_counts = {} for _ck in context_keys: _remaining_ck_counts[_ck] = _remaining_ck_counts.get(_ck, 0) + 1 _backfilled = [] # #3300 fix: track ONLY context rows we splice in, so the # visible-display backbone is never suppressed. Sharing one set # between context inserts and display rows (and _message_identity # ignoring timestamps) dropped a legitimate second identical visible # user turn. Display rows are always appended in order; a context # row is backfilled only if it isn't already a display row and # hasn't already been inserted. _context_inserted = set() _cursor = 0 for _display_idx, _dmsg in enumerate(previous_display): _dkey = _display_keys[_display_idx] if _dkey is not None: _j = _cursor while _j < len(context_keys) and context_keys[_j] != _dkey: _j += 1 if _j < len(context_keys): for _k in range(_cursor, _j): _ckey = context_keys[_k] _cmsg = previous_context[_k] if ( _ckey is not None and _ckey not in _context_inserted and _ckey not in _display_id_set and not _is_context_compression_marker(_cmsg) and not _is_compressed_context_tool_result_summary_message(_cmsg) ): _backfilled.append(copy.deepcopy(_cmsg)) _context_inserted.add(_ckey) # Sync multiset: decrement keys consumed by advancing # the cursor to _j+1 (delete at zero so membership matches # the list slice exactly). for _k in range(_cursor, _j + 1): _consumed_ck = context_keys[_k] _ck_n = _remaining_ck_counts.get(_consumed_ck, 0) - 1 if _ck_n <= 0: _remaining_ck_counts.pop(_consumed_ck, None) else: _remaining_ck_counts[_consumed_ck] = _ck_n _cursor = _j + 1 elif not any( _display_keys[_fi] in _remaining_ck_counts for _fi in range(_display_idx + 1, len(_display_keys)) ): for _k in range(_cursor, len(context_keys)): _ckey = context_keys[_k] _cmsg = previous_context[_k] if ( _ckey is not None and _ckey not in _context_inserted and _ckey not in _display_id_set and not _is_context_compression_marker(_cmsg) and not _is_compressed_context_tool_result_summary_message(_cmsg) ): _backfilled.append(copy.deepcopy(_cmsg)) _context_inserted.add(_ckey) _cursor = len(context_keys) _remaining_ck_counts.clear() # The display row is the visible backbone — always preserve it, # in order, even when an earlier (identical-content) turn or a # backfilled context row shares its timestamp-less identity. _backfilled.append(_dmsg) while _cursor < len(context_keys): _ckey = context_keys[_cursor] _cmsg = previous_context[_cursor] _cursor += 1 if ( _ckey is not None and _ckey not in _context_inserted and _ckey not in _display_id_set and not _is_context_compression_marker(_cmsg) and not _is_compressed_context_tool_result_summary_message(_cmsg) ): _backfilled.append(copy.deepcopy(_cmsg)) _context_inserted.add(_ckey) if len(_backfilled) > len(previous_display): logger.debug( "Backfilled %d context-only turns into previous_display (was %d, now %d)", len(_backfilled) - len(previous_display), len(previous_display), len(_backfilled), ) previous_display = _backfilled if _messages_have_prefix(result_messages, previous_context): candidates = result_messages[len(previous_context):] # Normalize stale merges only in the new-turn slice; never rewrite # historical rows in the already-committed previous_context prefix. if msg_text and previous_user_tail: candidates = _strip_stale_user_merge_from_messages( candidates, msg_text, previous_user_tail, previous_context=previous_context, ) current_user_key = _message_identity({'role': 'user', 'content': msg_text}) current_user_in_candidates = any( _message_identity(m) == current_user_key or _looks_like_current_user_turn(m, msg_text) for m in candidates ) assistant_or_tool_only_candidates = bool(candidates) and all( _is_context_compression_marker(m) or ( isinstance(m, dict) and m.get('role') in ('assistant', 'tool') ) for m in candidates ) if not (assistant_or_tool_only_candidates and not current_user_in_candidates): candidates = _strip_replayed_prefix(previous_display, candidates) candidates = _strip_replayed_prefix(previous_context, candidates) else: current_user_idx = _find_current_user_turn(result_messages, msg_text) assistant_or_tool_only_result = bool(result_messages) and all( _is_context_compression_marker(m) or ( isinstance(m, dict) and m.get('role') in ('assistant', 'tool') ) for m in result_messages ) turn_candidates = ( result_messages[current_user_idx:] if current_user_idx is not None else result_messages if assistant_or_tool_only_result else [] ) # Normalize stale merges only in the current-turn slice. if msg_text and previous_user_tail: turn_candidates = _strip_stale_user_merge_from_messages( turn_candidates, msg_text, previous_user_tail, previous_context=previous_context, ) candidates = turn_candidates merged = previous_display[:] seen = {_message_identity(m) for m in merged} current_user_key = _message_identity({'role': 'user', 'content': msg_text}) current_user_in_candidates = any( _message_identity(m) == current_user_key or _looks_like_current_user_turn(m, msg_text) for m in candidates ) current_user_already_checkpointed = ( _active_turn_has_checkpoint(merged, _active_turn_identity) if _active_turn_identity else False ) verification_user_already_checkpointed = bool( _verification_nudge_seen and _active_turn_identity and ( _active_turn_has_checkpoint(previous_display, _active_turn_identity) or _active_turn_has_checkpoint(previous_context, _active_turn_identity) ) ) if ( current_user_key is not None and not current_user_in_candidates and not current_user_already_checkpointed and not verification_user_already_checkpointed and any( isinstance(m, dict) and m.get('role') in ('assistant', 'tool') for m in candidates ) ): # Some provider retry/fallback paths can return an assistant/tool delta # without echoing the current user turn. In deferred session-save mode # the prompt exists only in pending_user_message, so appending that delta # directly would make the assistant bubble appear attached to the prior # exchange and then clear the pending prompt. Materialize the current # turn at the transcript boundary before the assistant/tool response. current_user_msg = _materialize_active_turn_user( _active_turn_identity, msg_text, source ) insert_at = 0 while insert_at < len(candidates) and _is_context_compression_marker(candidates[insert_at]): insert_at += 1 candidates = candidates[:insert_at] + [current_user_msg] + candidates[insert_at:] for msg in candidates: if ( _is_context_compression_marker(msg) or _is_compressed_context_tool_result_summary_message(msg) ): continue is_active_image_row = ( active_turn_display_text is not None and isinstance(msg, dict) and msg.get('role') == 'user' and ( msg is active_turn_row or _active_turn_token_matches(msg, _active_turn_identity) ) ) if is_active_image_row: display_row = _materialize_active_turn_user( _active_turn_identity, active_turn_display_text, source, ) display_row['content'] = active_turn_display_text display_row.pop('api_content', None) for key in ('id', 'timestamp', '_row_id'): if display_row.get(key) is None and msg.get(key) is not None: display_row[key] = copy.deepcopy(msg[key]) if not display_row.get('attachments') and msg.get('attachments'): display_row['attachments'] = copy.deepcopy(msg['attachments']) _mark_active_turn_checkpoint(display_row, _active_turn_identity) existing_idx = next( ( idx for idx, existing in enumerate(merged) if _active_turn_token_matches(existing, _active_turn_identity) ), None, ) if existing_idx is not None: existing = merged[existing_idx] existing['content'] = active_turn_display_text existing.pop('api_content', None) for key in ('id', 'timestamp', '_row_id'): if existing.get(key) is None and msg.get(key) is not None: existing[key] = copy.deepcopy(msg[key]) if not existing.get('attachments') and msg.get('attachments'): existing['attachments'] = copy.deepcopy(msg['attachments']) if not existing.get('attachments') and display_row.get('attachments'): existing['attachments'] = copy.deepcopy(display_row['attachments']) _mark_active_turn_checkpoint(existing, _active_turn_identity) continue msg = display_row key = _message_identity(msg) is_current_user_turn = _looks_like_current_user_turn(msg, msg_text) if ( not is_active_image_row and ((key is not None and key == current_user_key) or is_current_user_turn) and merged and ( _message_identity(merged[-1]) == current_user_key or _looks_like_current_user_turn(merged[-1], msg_text) ) ): # Eager session-save mode can checkpoint the current user turn # before the agent runs. When the agent returns that same user turn # in result_messages, keep the durable checkpoint and append only # the assistant/tool delta. # The eager checkpoint was written before _assign_stable_message_ids # stamped the result rows, so it has no `id` while the context copy # does — which would silently defeat id-based fork/truncate alignment # for eager-mode users. Carry the minted id onto the kept checkpoint # so display and context share it (#5564). if ( isinstance(msg, dict) and msg.get('id') is not None and isinstance(merged[-1], dict) and merged[-1].get('id') is None ): merged[-1]['id'] = msg['id'] continue if ( key is not None and isinstance(msg, dict) and msg.get('role') == 'assistant' and merged and _message_identity(merged[-1]) == key ): # Some provider/result replay paths can include the same assistant # message twice in the current delta. Treat only adjacent identity # matches as replay duplicates so identical answers in separate # user turns remain visible. continue if _is_context_compression_marker(msg) and key is not None and key in seen: continue display_msg = msg if ( ((key is not None and key == current_user_key) or is_current_user_turn or is_active_image_row) and isinstance(msg, dict) and msg.get('role') == 'user' ): display_msg = copy.deepcopy(msg) display_msg['content'] = ( active_turn_display_text if is_active_image_row else msg_text ) stamp_message_source(display_msg, source) merged.append(copy.deepcopy(display_msg)) if key is not None: seen.add(key) return merged def _stamp_missing_message_timestamps(messages, *, now: float | None = None) -> int: """Stamp missing message timestamps without collapsing transcript order. Compacted/reconciled rows can arrive without timestamps. Assigning one integer seconds value to the whole batch makes later timestamp-based display merges unstable; use a subsecond sequence instead. """ base = time.time() if now is None else float(now) stamped = 0 for msg in messages or []: if isinstance(msg, dict) and not msg.get('timestamp') and not msg.get('_ts'): msg['timestamp'] = base + (stamped * 0.000001) stamped += 1 return stamped def _assistant_reply_added_after_current_turn(result_messages, previous_context, msg_text) -> bool: """Return True only when the just-finished turn produced assistant text.""" result_messages = list(result_messages or []) previous_context = list(previous_context or []) if _messages_have_prefix(result_messages, previous_context): candidates = result_messages[len(previous_context):] else: current_user_idx = _find_current_user_turn(result_messages, msg_text) candidates = result_messages[current_user_idx + 1:] if current_user_idx is not None else result_messages return any( isinstance(m, dict) and m.get('role') == 'assistant' and not m.get('_error') and _assistant_message_has_final_visible_text(m) for m in candidates ) def _session_lacks_final_assistant_answer(messages) -> bool: """Return True when the persisted transcript ends before a final answer.""" for msg in reversed(list(messages or [])): if not isinstance(msg, dict): continue if msg.get('_error'): return False if _is_context_compression_marker(msg): continue role = msg.get('role') if role == 'tool': return True if role == 'assistant': if _assistant_message_has_final_visible_text(msg): return False continue if role == 'user': return True return True def _turn_transcript_lacks_final_assistant_answer( merged_messages, previous_display, msg_text, source: str = "webui", drop_replayed_assistant: bool = False, active_turn_identity=None, ) -> bool: """Return True when an already-merged transcript still lacks a final assistant answer.""" merged_messages = list(merged_messages or []) previous_display = list(previous_display or []) current_user_token_idx = next( ( idx for idx, message in enumerate(merged_messages) if _active_turn_token_matches(message, active_turn_identity) ), None, ) current_user_idx = current_user_token_idx if current_user_idx is None: current_user_idx = _find_current_user_turn(merged_messages, msg_text) if current_user_idx is None or ( current_user_token_idx is None and current_user_idx < len(previous_display) ): # The active turn lives after the durable transcript boundary. If the # merged display only exposes an older user row, materialize the pending # prompt so a replayed assistant row cannot satisfy the wrong turn. pending_user = { 'role': 'user', 'content': msg_text, } if source and source != 'webui': pending_user['_source'] = source merged_messages.append(pending_user) current_user_idx = len(merged_messages) - 1 current_user_key = _message_identity(merged_messages[current_user_idx]) filtered_messages = merged_messages[:current_user_idx + 1] if drop_replayed_assistant: prior_id_set = { _message_identity(msg) for msg in merged_messages[:current_user_idx] if isinstance(msg, dict) } for msg in merged_messages[current_user_idx + 1:]: if not isinstance(msg, dict): filtered_messages.append(msg) continue if msg.get('role') == 'assistant': key = _message_identity(msg) if key is not None and key in prior_id_set: continue filtered_messages.append(msg) else: filtered_messages.extend(merged_messages[current_user_idx + 1:]) if current_user_key is not None: filtered_messages = [ msg for msg in filtered_messages if _message_identity(msg) != current_user_key or msg is merged_messages[current_user_idx] ] return _session_lacks_final_assistant_answer(filtered_messages) def _merged_transcript_lacks_final_assistant_answer( previous_display, previous_context, result_messages, msg_text, source: str = "webui", drop_replayed_assistant: bool = False, active_turn_identity=None, ) -> bool: """Return True when the current turn still lacks a final assistant answer.""" previous_display = list(previous_display or []) _verification_nudge_seen = any( isinstance(message, dict) and message.get('role') == 'user' and _is_synthetic_control_message(message) for message in result_messages or [] ) merged_messages = _merge_display_messages_after_agent_result( previous_display, previous_context, _restore_reasoning_metadata(previous_display, result_messages), msg_text, source=source, verification_nudge_provenance={ 'verification_nudge_seen': _verification_nudge_seen, 'active_turn_identity': active_turn_identity, }, ) return _turn_transcript_lacks_final_assistant_answer( merged_messages, previous_display, msg_text, source=source, drop_replayed_assistant=drop_replayed_assistant, active_turn_identity=active_turn_identity, ) def _agent_result_terminal_failure(result) -> bool: """Return True for agent results that must not be finalized as done.""" if not isinstance(result, dict): return False status = str(result.get('status') or result.get('state') or '').strip().lower() if status in {'failed', 'error', 'partial', 'compression_exhausted'}: return True if result.get('compression_exhausted'): return True if result.get('failed') or result.get('partial'): return True return False def _self_heal_result_succeeded( result, previous_context, active_turn_identity, msg_text, ) -> bool: """Accept a credential retry only after authoritative turn success.""" if not isinstance(result, dict): return False if _result_reports_compression_snapshot_stale(result): return False if _agent_result_terminal_failure(result): return False result_error = result.get('error') if isinstance(result_error, str): if result_error.strip(): return False elif result_error: return False if result.get('completed') is False: return False messages = result.get('messages') or [] previous_context = list(previous_context or []) if _messages_have_prefix(messages, previous_context): current_turn_rows = list(messages[len(previous_context):]) current_user_idx = next( ( index for index, row in enumerate(current_turn_rows) if isinstance(row, dict) and row.get('role') == 'user' and _normalize_user_text(_message_text(row.get('content'))) == _normalize_user_text(msg_text) ), None, ) if current_user_idx is not None: current_turn_rows = current_turn_rows[current_user_idx + 1:] else: current_user_idx = _find_active_turn_checkpoint_index( messages, previous_context, active_turn_identity, msg_text, ) if current_user_idx is None: current_turn_rows = [] else: current_turn_rows = list(messages[current_user_idx + 1:]) next_user_idx = next( ( index for index, row in enumerate(current_turn_rows) if isinstance(row, dict) and row.get('role') == 'user' ), None, ) if next_user_idx is not None: current_turn_rows = current_turn_rows[:next_user_idx] return any( isinstance(row, dict) and row.get('role') == 'assistant' and not row.get('_error') and _assistant_message_has_final_visible_text(row) for row in current_turn_rows ) _TOOL_RESULT_SNIPPET_MAX = 4000 # Tool-arg keys whose values are card content / diff-reconstruction inputs. # These must not be capped to the short incidental-arg limit (#4928), or long # commands/paths get cut and recovery-rebuilt diffs (built from old_string/ # new_string/patch) break. Matched case-insensitively against the arg key. _TOOL_ARG_CONTENT_KEYS = frozenset({ 'command', 'cmd', 'script', 'code', 'patch', 'diff', 'old_string', 'new_string', 'content', 'path', 'file_path', }) _TOOL_ARG_CONTENT_CAP = _TOOL_RESULT_SNIPPET_MAX _LIVE_TOOL_PROMPT_DELTA_MAX = 12_000 _LIVE_TOOL_PROMPT_TURN_MAX = 24_000 def _bounded_live_tool_prompt_delta(messages, *, cap: int = _LIVE_TOOL_PROMPT_DELTA_MAX) -> int: """Return a bounded rough token delta for live tool metering. Tool-result callbacks can fire before the agent's next exact prompt accounting is available. The live usage ring should show a conservative in-flight hint, not replay a full large tool payload into `last_prompt_tokens`. """ if not messages: return 0 try: from agent.model_metadata import estimate_messages_tokens_rough delta = int(estimate_messages_tokens_rough(messages) or 0) except Exception: delta = 0 if delta <= 0: return 0 return min(delta, int(cap or 0)) def live_usage_prompt_estimate_after_tool_delta( *, base_prompt_tokens: int, exact_prompt_tokens: int = 0, messages=None, cap: int = _LIVE_TOOL_PROMPT_DELTA_MAX, turn_tool_prompt_tokens: int = 0, turn_cap: int = _LIVE_TOOL_PROMPT_TURN_MAX, ) -> dict: """Compute the live `last_prompt_tokens` estimate after a tool update. Exact compressor/provider prompt accounting wins. When no newer exact prompt is available, add only bounded live tool deltas to the persisted base. """ base = int(base_prompt_tokens or 0) exact = int(exact_prompt_tokens or 0) if exact and exact != base: return { 'last_prompt_tokens': exact, 'estimated': False, 'turn_tool_prompt_tokens': 0, } prior_turn_delta = max(0, int(turn_tool_prompt_tokens or 0)) turn_ceiling = max(0, int(turn_cap or 0)) next_turn_delta = min( prior_turn_delta + _bounded_live_tool_prompt_delta(messages, cap=cap), turn_ceiling, ) return { 'last_prompt_tokens': base + next_turn_delta, 'estimated': True, 'turn_tool_prompt_tokens': next_turn_delta, } def _live_usage_session_snapshot(session_id, current_session, cache_ref, *, loader=get_session): """Return a session object for hot live-metering paths without repeated loads.""" if current_session is not None: try: cache_ref[0] = current_session except Exception: pass return current_session try: cached = cache_ref[0] except Exception: cached = None if cached is not None: return cached try: loaded = loader(session_id) except Exception: return None try: cache_ref[0] = loaded except Exception: pass return loaded def _tool_result_snippet(raw, limit: int = _TOOL_RESULT_SNIPPET_MAX) -> str: """Extract a bounded result preview from a stored tool message payload.""" if limit <= 0: return '' text = str(raw or '') try: data = raw if isinstance(raw, dict) else json.loads(text) if isinstance(data, dict): preview = data.get('output') or data.get('result') or data.get('error') or text text = str(preview) except Exception: pass return text[:limit] def _truncate_tool_args(args, limit: int = 6) -> dict: """Truncate tool args for compact session persistence. Incidental args keep a short 120-char cap, but content/diff-bearing keys (the command, code, and patch fields that tool cards and recovery-rebuilt diffs are reconstructed from) get a much larger cap so a long command, file path, or reconstructed diff is not silently corrupted (#4928). A hard cap is still applied for storage safety, aligned with the result snippet cap. """ out = {} if not isinstance(args, dict): return out for k, v in list(args.items())[:limit]: s = str(v) cap = _TOOL_ARG_CONTENT_CAP if str(k).lower() in _TOOL_ARG_CONTENT_KEYS else 120 out[k] = s[:cap] + ('...' if len(s) > cap else '') return out def _nearest_assistant_msg_idx(messages, msg_idx: int) -> int: """Find the closest preceding assistant message index for a tool result.""" for idx in range(msg_idx - 1, -1, -1): msg = messages[idx] if isinstance(msg, dict) and msg.get('role') == 'assistant': return idx return -1 def _extract_tool_calls_from_messages(messages, live_tool_calls=None): """Build persisted tool-call summaries from final messages plus live progress fallback.""" tool_calls = [] pending_names = {} pending_args = {} pending_asst_idx = {} tool_msg_sequence = [] for msg_idx, m in enumerate(messages or []): if not isinstance(m, dict): continue role = m.get('role') if role == 'assistant': content = m.get('content', '') if isinstance(content, list): for part in content: if isinstance(part, dict) and part.get('type') == 'tool_use': tid = part.get('id', '') if tid: pending_names[tid] = part.get('name', '') pending_args[tid] = part.get('input', {}) pending_asst_idx[tid] = msg_idx for tc in m.get('tool_calls') or []: if not isinstance(tc, dict): continue tid = tc.get('id', '') or tc.get('call_id', '') fn = tc.get('function', {}) name = fn.get('name', '') try: args = json.loads(fn.get('arguments', '{}') or '{}') except Exception: args = {} if tid and name: pending_names[tid] = name pending_args[tid] = args pending_asst_idx[tid] = msg_idx elif role == 'tool': tid = m.get('tool_call_id') or m.get('tool_use_id', '') raw = m.get('content', '') seq = {'msg_idx': msg_idx, 'raw': raw, 'resolved': False} if tid: name = pending_names.get(tid, '') if name and name != 'tool': tool_calls.append({ 'name': name, 'snippet': _tool_result_snippet(raw), 'tid': tid, 'assistant_msg_idx': pending_asst_idx.get(tid, -1), 'args': _truncate_tool_args(pending_args.get(tid, {})), }) seq['resolved'] = True tool_msg_sequence.append(seq) live = [tc for tc in (live_tool_calls or []) if isinstance(tc, dict) and tc.get('name') and tc.get('name') != 'clarify'] if live: for seq_idx, seq in enumerate(tool_msg_sequence): if seq.get('resolved'): continue if seq_idx >= len(live): break live_tc = live[seq_idx] tool_calls.append({ 'name': live_tc.get('name', 'tool'), 'snippet': _tool_result_snippet(seq.get('raw', '')), 'tid': live_tc.get('tid', '') or '', 'assistant_msg_idx': _nearest_assistant_msg_idx(messages, seq.get('msg_idx', -1)), 'args': _truncate_tool_args(live_tc.get('args', {}), limit=4), }) return tool_calls def _partial_message_signature(message: dict) -> tuple: """Return a stable identity for a persisted partial assistant marker.""" if not isinstance(message, dict): return ('', '', ()) tool_sig = [] for tool_call in message.get('_partial_tool_calls') or []: if not isinstance(tool_call, dict): continue try: args_sig = json.dumps( tool_call.get('args') or {}, ensure_ascii=False, sort_keys=True, default=str, ) except Exception: args_sig = str(tool_call.get('args') or '') tool_sig.append(( str(tool_call.get('name') or ''), args_sig, bool(tool_call.get('done', False)), bool(tool_call.get('is_error', False)), str(tool_call.get('preview') or tool_call.get('snippet') or ''), )) return ( str(message.get('content') or '').strip(), str(message.get('reasoning') or '').strip(), tuple(tool_sig), ) def _partial_marker_already_present(messages, candidate: dict, *, before_idx: int | None = None) -> bool: """Check for an equivalent partial marker in the current user turn only.""" if not isinstance(messages, list) or not isinstance(candidate, dict): return False end = before_idx if isinstance(before_idx, int) else len(messages) end = max(0, min(end, len(messages))) start = 0 for idx in range(end - 1, -1, -1): msg = messages[idx] if isinstance(msg, dict) and msg.get('role') == 'user': start = idx + 1 break candidate_sig = _partial_message_signature(candidate) for msg in messages[start:end]: if isinstance(msg, dict) and msg.get('_partial') and _partial_message_signature(msg) == candidate_sig: return True return False def _partial_snapshot_prefers_candidate_text(current, candidate) -> bool: """Choose the later-looking monotonic snapshot without dropping a suffix.""" current_text = str(current or '').strip() candidate_text = str(candidate or '').strip() if not candidate_text: return False if not current_text or candidate_text.startswith(current_text): return True if current_text.startswith(candidate_text): return False # Provider adapters can normalize a partial between the callback and the # returned result. They are no longer prefix-comparable; retain whichever # carries more delivered text, preferring the later result on an exact tie. return len(candidate_text) >= len(current_text) def _merge_partial_snapshots(current: dict, candidate: dict) -> dict: """Merge two sources for one partial response into ``current`` in place.""" if _partial_snapshot_prefers_candidate_text( current.get('content'), candidate.get('content') ): current['content'] = candidate.get('content') or '' if _partial_snapshot_prefers_candidate_text( current.get('reasoning'), candidate.get('reasoning') ): current['reasoning'] = candidate.get('reasoning') candidate_tools = candidate.get('_partial_tool_calls') or [] current_tools = current.get('_partial_tool_calls') or [] if candidate_tools and len(candidate_tools) >= len(current_tools): current['_partial_tool_calls'] = copy.deepcopy(candidate_tools) current['_partial'] = True current.setdefault('role', 'assistant') current.setdefault('timestamp', candidate.get('timestamp') or int(time.time())) return current def _upsert_current_turn_partial( messages, candidate: dict, *, active_turn_identity=None, ) -> dict | None: """Persist exactly one partial assistant snapshot in the current turn. Live callback buffers and ``result["messages"]`` are two observations of the same Agent response. Their bytes can differ when one observation is a slightly newer prefix. Reconcile them into one row rather than treating content inequality as proof of two assistant turns. """ if not isinstance(messages, list) or not isinstance(candidate, dict): return None current_user_idx = next( ( index for index in range(len(messages) - 1, -1, -1) if _active_turn_token_matches(messages[index], active_turn_identity) ), None, ) if current_user_idx is None: # Error settlement materializes the pending WebUI user row before this # helper, but that recovered row intentionally has no provider-facing # token. The last user row is therefore the display boundary. current_user_idx = next( ( index for index in range(len(messages) - 1, -1, -1) if isinstance(messages[index], dict) and messages[index].get('role') == 'user' ), None, ) start = (current_user_idx + 1) if current_user_idx is not None else 0 end = len(messages) for index in range(start, len(messages)): if isinstance(messages[index], dict) and messages[index].get('role') == 'user': end = index break candidate_content = str(candidate.get('content') or '') canonical_idx = next( ( index for index in range(start, end) if isinstance(messages[index], dict) and messages[index].get('role') == 'assistant' and not messages[index].get('_error') and str(messages[index].get('content') or '') == candidate_content ), None, ) partial_indices = [ index for index in range(start, end) if isinstance(messages[index], dict) and messages[index].get('role') == 'assistant' and messages[index].get('_partial') is True and not messages[index].get('_error') ] if canonical_idx is None and partial_indices: canonical_idx = partial_indices[-1] if canonical_idx is None: messages.append(dict(candidate)) return messages[-1] canonical = messages[canonical_idx] for partial_idx in partial_indices: if partial_idx != canonical_idx: _merge_partial_snapshots(canonical, messages[partial_idx]) _merge_partial_snapshots(canonical, candidate) # Collapse any partials left by an earlier source while preserving ordinary # assistant/tool history in the same turn. for index in reversed(partial_indices): if index != canonical_idx: del messages[index] return canonical def _sse(handler, event, data): """Write one SSE event to the response stream.""" payload = f"event: {event}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n" handler.wfile.write(payload.encode('utf-8')) handler.wfile.flush() # ── SSE write deadline (Defect A: per-connection thread exhaustion) ───────── # server.py runs QuietHTTPServer(ThreadingHTTPServer): one OS thread per # connection, no pool cap (request_queue_size=64). Every SSE endpoint holds # its thread for the connection's whole lifetime. If a tab is slow or # backgrounded its TCP receive window fills; the next handler.wfile.write()/ # flush() then blocks *indefinitely* (sockets have no write timeout by # default). That thread is pinned forever — it never reaches its # `finally: unsubscribe`, so the SessionChannel reaper can never reclaim the # channel either. N such tabs * M sessions pile threads up until new # requests queue past request_queue_size and the UI shows "streaming # pending". # # Fix: arm a socket-level timeout on the connection. A genuinely healthy # keepalive/event write completes in well under a millisecond, so a # multi-second deadline never trips for a live tab; only a backpressured # (stuck) socket blocks past it. When it trips, the write raises # socket.timeout — which on Python 3.10+ *is* TimeoutError, already a member # of api.routes._CLIENT_DISCONNECT_ERRORS — so each SSE handler's existing # `except _CLIENT_DISCONNECT_ERRORS:` breaks the loop, `finally` drops the # subscriber, the browser's EventSource auto-reconnects, and the OS thread # is released. SessionChannel already supports reconnect + offline buffer, # so no events are lost for a tab that comes back. Operators behind unusual # proxies can tune the deadline without code changes. try: _raw_deadline = os.getenv("HERMES_WEBUI_SSE_WRITE_DEADLINE") or os.getenv("HERMES_SSE_WRITE_DEADLINE") SSE_WRITE_DEADLINE_SECONDS = float(_raw_deadline or "20.0") except (TypeError, ValueError): SSE_WRITE_DEADLINE_SECONDS = 20.0 if SSE_WRITE_DEADLINE_SECONDS <= 0: SSE_WRITE_DEADLINE_SECONDS = 20.0 def _sse_set_write_deadline(handler, seconds=None): """Best-effort: arm a socket write deadline on an SSE handler. Call once, right after end_headers(), in every long-lived SSE endpoint. Never raises — an unusual/missing transport just keeps the pre-fix (no-deadline) behaviour for that single connection rather than breaking the stream setup. """ if seconds is None: seconds = SSE_WRITE_DEADLINE_SECONDS try: conn = getattr(handler, "connection", None) if conn is not None and hasattr(conn, "settimeout"): conn.settimeout(seconds) except Exception: logger.debug("Failed to arm SSE write deadline", exc_info=True) def _materialize_pending_user_turn_before_error( session, *, active_turn_identity=None, ) -> bool: """Persist the pending user prompt before clearing runtime stream state. Error paths often clear ``pending_user_message`` before appending an assistant error marker. In deferred session-save mode that pending field can be the only durable copy of the user's current turn, so clearing it makes the user bubble disappear on reload/reconcile. Return True when a recovered user turn was appended. """ pending_text = str(getattr(session, 'pending_user_message', None) or '') if not pending_text: return False recovered_ts = int(time.time()) pending_started_at = getattr(session, 'pending_started_at', None) if isinstance(pending_started_at, (int, float)) and pending_started_at > 0: recovered_ts = int(pending_started_at) pending_source = getattr(session, 'pending_user_source', None) or 'webui' pending_attachments = list(getattr(session, 'pending_attachments', None) or []) # A returned Agent result is settled into ``session.messages`` before its # terminal classification runs. In that path the current user row can # already own a result assistant while no longer being the final row. Do # not append a second recovered copy: that artificial user boundary would # split the live-buffer and result partial snapshots into two turns and # defeat exact-once consolidation below. Only the unforgeable WebUI turn # token is accepted here; matching repeated prompt text is not authority. active_turn_token = ( active_turn_identity.get('token') if isinstance(active_turn_identity, dict) else build_active_turn_token( getattr(session, 'active_stream_id', None), pending_started_at, ) ) session_messages = getattr(session, 'messages', None) if active_turn_token and isinstance(session_messages, list): for index in range(len(session_messages) - 1, -1, -1): message = session_messages[index] if not ( isinstance(message, dict) and message.get('role') == 'user' and message.get('_active_turn_token') == active_turn_token ): continue normalized_user = _normalize_user_text( _message_text(message.get('content')) ) if normalized_user != _normalize_user_text(pending_text): break if not any( isinstance(later, dict) and later.get('role') == 'user' for later in session_messages[index + 1 :] ): return False break def is_exact_checkpoint(messages): if not isinstance(messages, list) or not messages: return False existing = messages[-1] if not isinstance(existing, dict) or existing.get('role') != 'user': return False existing_source = existing.get('_source') or 'webui' try: existing_ts = int(existing.get('timestamp')) except (TypeError, ValueError): return False return ( _normalize_user_text(_message_text(existing.get('content'))) == _normalize_user_text(pending_text) and existing_ts == recovered_ts and existing_source == pending_source and list(existing.get('attachments') or []) == pending_attachments ) if is_exact_checkpoint(getattr(session, 'messages', None)): return False recovered = { 'role': 'user', 'content': pending_text, 'timestamp': recovered_ts, '_recovered': True, } if str(pending_source or '').strip().lower() == 'fork': recovered['_fork_child_turn'] = session.session_id stamp_message_source(recovered, pending_source) if pending_attachments: recovered['attachments'] = pending_attachments session.messages.append(recovered) # Mirror to context_messages so the _recovered flag survives the state.db # round-trip (#4283). state.db has no _recovered column, so without this # mirror the next turn's reconciled_state_db_messages_for_session( # prefer_context=True) finds the recovered user as a flagless state.db # delta and _sanitize_messages_for_api cannot filter it — causing the # interrupted turn's prompt to be prepended to every subsequent turn. # Placing the mirror here (rather than in _persist_cancelled_turn) covers # all three callers: cancel, provider-error, and exception paths. ctx = getattr(session, 'context_messages', None) if isinstance(ctx, list) and ctx and not is_exact_checkpoint(ctx): ctx.append(dict(recovered)) # The new user turn is now committed to messages (#3831): advance a positive # truncation watermark left over from a prior retry/undo/edit so that # merge_session_messages_append_only() still filters out replaced pre-edit # rows from state.db. The merge's sidecar_advanced_past_watermark guard # allows state.db rows newer than the watermark, so post-edit turns are not # dropped. Never 0.0 (the truncate-to-empty sentinel, #2914). if getattr(session, 'truncation_watermark', None): session.truncation_watermark = float(recovered_ts) return True def _terminal_turn_duration(session, *, now: float | None = None) -> float | None: """Freeze a valid turn timer before terminal cleanup clears its origin.""" started_at = getattr(session, 'pending_started_at', None) try: started = float(started_at) ended = float(time.time() if now is None else now) except (TypeError, ValueError, OverflowError): return None if not math.isfinite(started) or not math.isfinite(ended) or started <= 0: return None elapsed = ended - started if elapsed < 0: return None return round(elapsed, 3) def _build_partial_message(content_text, reasoning_text, tool_calls) -> dict | None: """Build a _partial assistant message from raw streaming buffers. Shared by cancel_stream() and _snapshot_and_append_partial_on_error(). Strips thinking/reasoning markup, builds the dict, returns None when there is nothing meaningful to preserve. """ import re as _re partial_text = (content_text or '').strip() _stripped = '' if partial_text: # First pass: remove complete ... blocks. _stripped = _re.sub(r']*>.*?', '', partial_text, flags=_re.DOTALL | _re.IGNORECASE).strip() # Second pass: strip trailing UNCLOSED think/thinking block (the common # cancel/error case — user stops mid-reasoning before the close tag appears). _stripped = _re.sub(r']*>.*', '', _stripped, flags=_re.DOTALL | _re.IGNORECASE).strip() _has_reasoning = bool(reasoning_text and reasoning_text.strip()) _has_tools = bool(tool_calls) if not (_stripped or _has_reasoning or _has_tools): return None _msg: dict = { 'role': 'assistant', 'content': _stripped, # may be empty for reasoning/tool-only turns '_partial': True, 'timestamp': int(time.time()), } if _has_reasoning: _msg['reasoning'] = reasoning_text.strip() if _has_tools: _msg['_partial_tool_calls'] = list(tool_calls) return _msg def _snapshot_and_append_partial_on_error( session, stream_id, *, active_turn_identity=None, ) -> dict | None: """Snapshot streaming buffers under STREAMS_LOCK and append a _partial message. Uses _build_partial_message() for the shared thinking-strip + dict-build logic. """ from api import config as _live_config streams_lock = STREAMS_LOCK partial_texts = STREAM_PARTIAL_TEXT reasoning_texts = STREAM_REASONING_TEXT live_tool_calls = STREAM_LIVE_TOOL_CALLS # Defensive check for live config (similar to cancel_stream) if getattr(_live_config, 'STREAMS_LOCK', streams_lock) is not streams_lock: streams_lock = _live_config.STREAMS_LOCK partial_texts = getattr(_live_config, 'STREAM_PARTIAL_TEXT', partial_texts) reasoning_texts = getattr(_live_config, 'STREAM_REASONING_TEXT', reasoning_texts) live_tool_calls = getattr(_live_config, 'STREAM_LIVE_TOOL_CALLS', live_tool_calls) _snap_partial_text = None _snap_reasoning = None _snap_tool_calls = None # The streaming thread mirrors these three buffers lock-free (GIL-atomic; see # the STREAMS_LOCK contract note at on_token in the streaming loop). We take # STREAMS_LOCK here to read atomically w.r.t. the worker's cleanup `finally` # (which pops all three under STREAMS_LOCK) — NOT w.r.t. the writer, which # holds no lock, so the three reads may still reflect slightly different # points in time. That is acceptable: each individual read is complete (never # torn) and a slightly-stale partial is reconciled by later journal/SSE events. with streams_lock: _snap_partial_text = partial_texts.get(stream_id, '') if not _snap_partial_text: _live_partials = getattr(_live_config, 'STREAM_PARTIAL_TEXT', partial_texts) if _live_partials is not partial_texts: _snap_partial_text = _live_partials.get(stream_id, '') _snap_reasoning = reasoning_texts.get(stream_id, '') if not _snap_reasoning: _live_reasoning = getattr(_live_config, 'STREAM_REASONING_TEXT', reasoning_texts) if _live_reasoning is not reasoning_texts: _snap_reasoning = _live_reasoning.get(stream_id, '') _snap_tool_calls = list(live_tool_calls.get(stream_id, []) or []) if not _snap_tool_calls: _live_tools = getattr(_live_config, 'STREAM_LIVE_TOOL_CALLS', live_tool_calls) if _live_tools is not live_tool_calls: _snap_tool_calls = list(_live_tools.get(stream_id, []) or []) # Seal projected live tool rows so the durable scene cannot disagree with # the settled terminal state (#6309). Any tool call that was still running # at error time is marked as completed with the terminal-error metadata. if _snap_tool_calls: for _tc in _snap_tool_calls: if isinstance(_tc, dict) and not _tc.get('done'): _tc['done'] = True _tc['_sealed_by_terminal_error'] = True _partial_msg = _build_partial_message(_snap_partial_text, _snap_reasoning, _snap_tool_calls) if _partial_msg is None: return None if not isinstance(session.messages, list): session.messages = [] return _upsert_current_turn_partial( session.messages, _partial_msg, active_turn_identity=active_turn_identity, ) def _append_result_partial_on_error( session, result, pre_call_context, msg_text, *, active_turn_identity=None, ) -> dict | None: """Retain only a current-turn partial assistant row from an Agent result.""" if not isinstance(result, dict) or result.get('partial') is not True: return None messages = result.get('messages') if not isinstance(messages, list) or not isinstance(pre_call_context, list): return None normalized_msg_text = _normalize_user_text(msg_text) if _messages_have_prefix(messages, pre_call_context): # Append-only result: only rows after the pre-call baseline can # belong to this call. current_turn_rows = messages[len(pre_call_context):] baseline_has_current_user = bool( pre_call_context and isinstance(pre_call_context[-1], dict) and pre_call_context[-1].get('role') == 'user' and _normalize_user_text(_message_text(pre_call_context[-1].get('content'))) == normalized_msg_text ) if not baseline_has_current_user: current_user_index = next( ( index for index, row in enumerate(current_turn_rows) if isinstance(row, dict) and row.get('role') == 'user' and _normalize_user_text(_message_text(row.get('content'))) == normalized_msg_text ), None, ) if current_user_index is None: return None current_turn_rows = current_turn_rows[current_user_index + 1:] else: # Compacted/replayed results can REPLACE the pre-call baseline instead # of appending to it; a numeric suffix slice can then expose a # historical assistant row as this turn's partial. Require the unique # Agent-resolved turn boundary. WebUI's private token is preferred when # it survives, but compacted Agent projections legitimately omit it. # In that case require the authoritative Agent turn id/index and validate # that the indexed row is this normalized current user prompt. current_user_idx = _find_active_turn_checkpoint_index( messages, pre_call_context, active_turn_identity, msg_text, ) if current_user_idx is None: return None current_turn_rows = messages[current_user_idx + 1:] # A later user row starts another turn. Never cross that boundary while # selecting a partial assistant response for the current turn. next_user_idx = next( ( index for index, row in enumerate(current_turn_rows) if isinstance(row, dict) and row.get('role') == 'user' ), None, ) if next_user_idx is not None: current_turn_rows = current_turn_rows[:next_user_idx] assistant_row = next( ( row for row in reversed(current_turn_rows) if isinstance(row, dict) and row.get('role') == 'assistant' and row.get('content') and not row.get('_error') ), None, ) if assistant_row is None: return None partial_msg = _build_partial_message( str(assistant_row.get('content') or ''), None, None, ) if partial_msg is None: return None if not isinstance(session.messages, list): session.messages = [] return _upsert_current_turn_partial( session.messages, partial_msg, active_turn_identity=active_turn_identity, ) def _last_resort_sync_from_core(session, stream_id, agent_lock): """Final-exit guard: if the stream exits with pending_user_message still set, sync messages from the core transcript or add an error marker. Called from the outer finally block of _run_agent_streaming. Must never raise. """ from api.models import _get_profile_home, _apply_core_sync_or_error_marker try: # Guard: if a cancel was already requested, bail out — cancel_stream() has # already saved partial content and we must not double-append error markers. if stream_id in CANCEL_FLAGS and CANCEL_FLAGS[stream_id].is_set(): return profile_home = _get_profile_home(session.profile) core_path = profile_home / 'sessions' / f'session_{session.session_id}.json' _lock_ctx = agent_lock if agent_lock is not None else contextlib.nullcontext() with _lock_ctx: _apply_core_sync_or_error_marker( session, core_path, stream_id_for_recheck=stream_id, require_stream_dead=False, ) except Exception: logger.exception( "_last_resort_sync_from_core failed for session %s", getattr(session, 'session_id', '?'), ) def _session_db_is_open(session_db) -> bool: """True when *session_db* still has a live sqlite connection. SessionDB.close() sets ``_conn = None``. Subagents capture the parent's SessionDB object by reference at spawn time (delegate_tool), so closing that object mid-parent-turn makes every subsequent child ``append_message`` fail with ``'NoneType' object has no attribute 'execute'``. """ if session_db is None: return False return getattr(session_db, "_conn", None) is not None def _adopt_session_db_for_cached_agent(agent, new_session_db): """Attach a SessionDB to a reused cached agent without breaking subagents. Historical behaviour (PR #1421 FD-leak fix): create a fresh SessionDB every stream request and close the previous handle before replacing ``agent._session_db``. That stops EMFILE growth, but a server-side wakeup / new turn for the same parent session will close the shared handle while background subagents are still writing into it. Policy now: - If the cached agent already holds an *open* SessionDB, keep it and close the unused new handle (no FD leak; live subagents keep working). - If the existing handle is missing or already closed, adopt *new_session_db*. - If *new_session_db* is None, leave the existing handle alone. """ if agent is None: return new_session_db existing = getattr(agent, "_session_db", None) if new_session_db is None: return existing if existing is new_session_db: return existing if _session_db_is_open(existing): try: new_session_db.close() except Exception: # Same observability as _replace_session_db_in_kwargs: a failed # close here reintroduces the EMFILE pressure PR #1421 fixed. logger.debug( "Failed to close unused session_db handle in adopt helper", exc_info=True, ) return existing if existing is not None: try: existing.close() except Exception: logger.debug( "Failed to close previous session_db handle in adopt helper", exc_info=True, ) agent._session_db = new_session_db return new_session_db def _build_session_db_for_stream(state_db_path): """Build a per-request SessionDB handle for WebUI session search. Returns ``None`` if the helper module or constructor fails so callers can continue without session_search rather than propagating a hard failure. """ try: from hermes_state import SessionDB _attempts = 3 _last_error = None for _attempt in range(_attempts): try: return SessionDB(db_path=state_db_path) except sqlite3.OperationalError as _db_err: _db_err_text = str(_db_err).lower() if not ( "locked" in _db_err_text or "busy" in _db_err_text ): raise _last_error = _db_err if _attempt < _attempts - 1: print( f"[webui] WARNING: SessionDB init attempt {_attempt + 1}/{_attempts} failed, retrying: {_db_err}", flush=True, ) time.sleep(0.05 * (2 ** _attempt) + random.uniform(0, 0.05)) raise _last_error or RuntimeError("SessionDB construction exhausted all attempts") except Exception as _db_err: print(f"[webui] WARNING: SessionDB init failed - session_search will be unavailable: {_db_err}", flush=True) return None def _replace_session_db_in_kwargs(agent_kwargs, state_db_path): """Build a fresh SessionDB and replace ``agent_kwargs['session_db']`` safely. Does not close an existing open handle that may still be shared with live subagents; only replaces when the prior handle is missing or already closed. """ if not isinstance(agent_kwargs, dict): return None _old_session_db = agent_kwargs.get("session_db") _next_session_db = _build_session_db_for_stream(state_db_path) if _next_session_db is None: # Replacement construction failed. Keep the prior handle only if it is # still open (live subagents may hold it by reference); otherwise # degrade cleanly to None — as master did — so the rebuilt agent lazily # reinitialises its SessionDB instead of reusing a closed handle and # failing every persist/search with # "'NoneType' object has no attribute 'execute'". if _session_db_is_open(_old_session_db): return _old_session_db agent_kwargs["session_db"] = None return None if _session_db_is_open(_old_session_db): # Keep the live handle; discard the unused new one. try: if _next_session_db is not _old_session_db: _next_session_db.close() except Exception: logger.debug("Failed to close unused session_db handle during self-heal") agent_kwargs["session_db"] = _old_session_db return _old_session_db if _old_session_db is not None and _old_session_db is not _next_session_db: try: _old_session_db.close() except Exception: logger.debug("Failed to close previous session_db handle during self-heal") agent_kwargs["session_db"] = _next_session_db return _next_session_db def _add_supported_run_conversation_kwarg(callable_obj, kwargs, name, value): """Add a run-conversation kwarg only when the live Agent accepts it.""" if not isinstance(kwargs, dict): return False try: import inspect parameters = inspect.signature(callable_obj).parameters except (TypeError, ValueError): return False if name in parameters or any( parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() ): kwargs[name] = value return True return False def _register_pending_user_timestamp_identity( callable_obj, session, persist_user_timestamp, ): """Persist the pending-turn proof while the caller owns the session lock.""" if session is None: return try: timestamp_parameter = inspect.signature(callable_obj).parameters.get( "persist_user_timestamp" ) except (TypeError, ValueError): timestamp_parameter = None has_timestamp_contract = ( timestamp_parameter is not None and timestamp_parameter.kind is not inspect.Parameter.POSITIONAL_ONLY ) identity = ( _validated_webui_pending_user_timestamp_identity( session, (getattr(session, "active_stream_id", None), persist_user_timestamp), ) if has_timestamp_contract else None ) previous = getattr(session, "_webui_pending_user_timestamp_identity", None) session._webui_pending_user_timestamp_identity = identity session_id = getattr(session, "session_id", None) save = getattr(session, "save", None) if callable(save) and session_id and previous != identity: # Persist before run_conversation can write this turn to state.db. save(touch_updated_at=False, skip_index=True) def _build_run_conversation_kwargs( callable_obj, *, user_message, system_message, conversation_history, conversation_history_revision, task_id, persist_user_message, persist_user_timestamp, ): """Build one rolling-compatible Agent invocation contract without mutation. ``persist_user_timestamp`` is signature-gated (#6935): an older hermes-agent whose ``run_conversation()`` predates the kwarg must not receive it, or the call trips a TypeError before the turn starts. """ kwargs = { "user_message": user_message, "system_message": system_message, "conversation_history": conversation_history, "task_id": task_id, "persist_user_message": persist_user_message, } _add_supported_run_conversation_kwarg( callable_obj, kwargs, "persist_user_timestamp", persist_user_timestamp, ) _add_supported_run_conversation_kwarg( callable_obj, kwargs, "conversation_history_revision", conversation_history_revision, ) return kwargs def _attempt_credential_self_heal( provider_id, session_id, _agent_lock_ref, *, target_model=None, ): """Try to silently refresh credentials after a 401/auth error (#1401). Returns a new ``(agent, rt_dict)`` tuple on success so the caller can retry the conversation. Returns ``None`` when self-heal is not applicable (e.g. auth.json unchanged, provider unresolvable). Steps: 1. Re-read ``~/.hermes/auth.json`` to pick up fresh credentials that may have been written by a concurrent ``hermes model`` CLI invocation. 2. Evict the session's cached agent so it is rebuilt with fresh keys. 3. Evict the provider's credential-pool cache entry. 4. Re-resolve the runtime provider. 5. Return a new agent + resolved-provider dict (the caller must re-invoke ``run_conversation`` with these). """ try: from api.oauth import ( read_auth_json, resolve_runtime_provider_with_anthropic_env_lock, ) from api.config import ( SESSION_AGENT_CACHE, SESSION_AGENT_CACHE_LOCK, invalidate_credential_pool_cache, ) from hermes_cli.runtime_provider import resolve_runtime_provider # 1. Re-read auth.json (triggers a fresh credential scan) _fresh_auth = read_auth_json() if not _fresh_auth: logger.debug('[webui] self-heal: auth.json empty or missing, skipping') return None # 2. Evict the cached agent for this session _evicted_entry = None with SESSION_AGENT_CACHE_LOCK: _evicted_entry = SESSION_AGENT_CACHE.pop(session_id, None) if _evicted_entry is not None: _close_cached_agent_entry_at_session_boundary(session_id, _evicted_entry) # 3. Invalidate the credential pool for this provider invalidate_credential_pool_cache(provider_id) # 4. Re-resolve runtime provider with fresh credentials _new_rt = resolve_runtime_provider_with_anthropic_env_lock( resolve_runtime_provider, requested=provider_id, target_model=target_model, ) logger.info( '[webui] self-heal: credential refresh succeeded for provider=%s session=%s', provider_id, session_id, ) return _new_rt except Exception as _heal_err: logger.warning( '[webui] self-heal: failed for provider=%s session=%s: %s', provider_id, session_id, _heal_err, ) return None def _agent_cache_api_key_sig(resolved_api_key, credential_pool) -> str: """Return the cache-signature component for runtime credentials. Credential-pool providers and callable key_cmd sources can legitimately hand WebUI a different runtime token on each request (round-robin pools, OAuth refresh, auth self-heal). The AIAgent object is also where cross-turn memory-provider state lives, so using a volatile token in the cache signature silently defeats the per-session agent cache and drops warmed Hindsight prefetch results. """ if credential_pool is not None: return 'credential-pool' if not isinstance(resolved_api_key, str) and callable(resolved_api_key): # key_cmd credentials are resolved by the client at request time; do # not mint or stringify a potentially secret token for cache identity. return 'dynamic-credential' import hashlib as _hashlib return _hashlib.sha256((resolved_api_key or '').encode()).hexdigest()[:16] def _compute_agent_cache_signature( resolved_model: str | None, resolved_api_key: str | None, resolved_base_url: str | None, resolved_provider: str | None, runtime_bundle: dict | None, max_iterations_cfg=None, max_tokens_cfg=None, fallback_resolved=None, toolsets=None, reasoning_config=None, main_request_overrides=None, _main_request_overrides=None, prefill_context=None, profile_home: str | None = None, safe_profile_runtime_env: dict | None = None, ) -> str: """Return the SHA256 signature identifying an agent's constructor routing state.""" import hashlib as _hashlib import json as _json _bundle = runtime_bundle if isinstance(runtime_bundle, dict) else {} _credential_pool = _bundle.get('credential_pool') _env = safe_profile_runtime_env if isinstance(safe_profile_runtime_env, dict) else {} _main_request_overrides = main_request_overrides if main_request_overrides is not None else _main_request_overrides _sig_blob = _json.dumps([ resolved_model or '', _agent_cache_api_key_sig(resolved_api_key, _credential_pool), resolved_base_url or '', resolved_provider or '', _bundle.get('api_mode') or '', _bundle.get('acp_command') or '', _bundle.get('acp_args') or [], bool(_credential_pool), max_iterations_cfg or '', max_tokens_cfg or '', fallback_resolved or {}, sorted(toolsets) if toolsets else [], reasoning_config or {}, _main_request_overrides or {}, _public_prefill_context_status(prefill_context), profile_home or '', _env.get('TERMINAL_ENV', '') or '', _env.get('TERMINAL_SSH_HOST', '') or '', _env.get('TERMINAL_SSH_USER', '') or '', ], sort_keys=True) return _hashlib.sha256(_sig_blob.encode()).hexdigest()[:16] def _lifecycle_commit_session_memory(session_id: str, *, agent=None, wait: bool = False) -> bool: from api.session_lifecycle import commit_session_memory return commit_session_memory(session_id, agent=agent, wait=wait) def _lifecycle_has_uncommitted_work(session_id: str) -> bool: from api.session_lifecycle import has_uncommitted_work return has_uncommitted_work(session_id) def _lifecycle_unregister_agent(session_id: str) -> None: from api.session_lifecycle import unregister_agent unregister_agent(session_id) def _lifecycle_discard_session(session_id: str) -> bool: from api.session_lifecycle import discard_session return discard_session(session_id) def _close_evicted_agent_at_session_boundary(session_id: str, agent) -> bool: """Commit and tear down an evicted cached agent at a WebUI session boundary. WebUI keeps AIAgent instances in an LRU cache so memory providers can carry state across turns. When an agent is evicted, commit pending memory first; if the lifecycle entry is clean afterwards, unregister and call shutdown_memory_provider(messages) so provider-owned clients such as Hindsight's aiohttp session are closed instead of being garbage-collected later. Passing the cached transcript mirrors gateway cleanup semantics for providers that use on_session_end(messages) during shutdown. """ if agent is None: return True should_close_evicted_agent = True try: _lifecycle_commit_session_memory(session_id, agent=agent, wait=True) if not _lifecycle_has_uncommitted_work(session_id): _lifecycle_unregister_agent(session_id) # Drop the lifecycle dict entry now that the LRU-evicted agent is # gone and no uncommitted work remains, so the dict tracks only live # sessions instead of growing unbounded (issue #3506). _lifecycle_discard_session(session_id) else: should_close_evicted_agent = False except Exception: should_close_evicted_agent = False logger.debug("Lifecycle commit on eviction failed for %s", session_id, exc_info=True) if not should_close_evicted_agent: return False try: shutdown_memory_provider = getattr(agent, 'shutdown_memory_provider', None) if callable(shutdown_memory_provider): session_messages = vars(agent).get('_session_messages', []) shutdown_memory_provider(session_messages) except Exception: logger.debug("Failed to shut down evicted agent memory provider for session %s", session_id, exc_info=True) try: session_db = getattr(agent, '_session_db', None) if session_db is not None: session_db.close() except Exception: logger.debug("Failed to close evicted agent session DB for session %s", session_id, exc_info=True) return True def _close_cached_agent_entry_at_session_boundary(session_id: str, cache_entry) -> bool: """Commit and tear down a popped SESSION_AGENT_CACHE entry outside the cache lock.""" agent = cache_entry[0] if isinstance(cache_entry, tuple) else None return _close_evicted_agent_at_session_boundary(session_id, agent) def _refresh_cached_agent_runtime(agent, agent_kwargs: dict) -> bool: """Refresh volatile runtime credentials on a reused cached AIAgent. The cache key intentionally ignores credential-pool token churn, but the cached agent's LLM client still needs the latest selected/refreshed runtime key. Keep long-lived provider/session state (memory prefetch, turn counters, tool state) while swapping only the runtime credential/client. """ if agent is None or not isinstance(agent_kwargs, dict): return False new_pool = agent_kwargs.get('credential_pool') if new_pool is not None: try: agent._credential_pool = new_pool except Exception: pass new_key = agent_kwargs.get('api_key') or '' if not new_key: return True new_base = agent_kwargs.get('base_url') or getattr(agent, 'base_url', '') or '' if getattr(agent, '_fallback_activated', False): # Avoid mixing a refreshed primary credential into a live fallback # runtime. Rebuilding is safer than mutating a fallback-active agent # whose restore/cooldown state has not run yet for this turn. return False if new_key == (getattr(agent, 'api_key', '') or ''): _refresh_cached_agent_primary_runtime_snapshot(agent) return True try: if getattr(agent, 'api_mode', None) == 'anthropic_messages': # Native Anthropic-style clients have their own construction path; # switch_model() already handles token/client refresh there. if hasattr(agent, 'switch_model'): agent.switch_model( agent_kwargs.get('model') or getattr(agent, 'model', None), agent_kwargs.get('provider') or getattr(agent, 'provider', None), api_key=new_key, base_url=new_base, api_mode=agent_kwargs.get('api_mode') or getattr(agent, 'api_mode', ''), ) return True return False if not hasattr(agent, '_client_kwargs') or not hasattr(agent, '_replace_primary_openai_client'): # Test/fake-agent fallback: keep metadata accurate even if no real # OpenAI client exists to rebuild. agent.api_key = new_key if new_base: agent.base_url = new_base _refresh_cached_agent_primary_runtime_snapshot(agent) return True client_kwargs = dict(getattr(agent, '_client_kwargs', {}) or {}) client_kwargs['api_key'] = new_key if new_base: client_kwargs['base_url'] = new_base agent._client_kwargs = client_kwargs agent.api_key = new_key if new_base: agent.base_url = new_base if hasattr(agent, '_apply_client_headers_for_base_url'): agent._apply_client_headers_for_base_url(agent.base_url) rebuilt = bool(agent._replace_primary_openai_client(reason='webui_credential_refresh')) if rebuilt: _refresh_cached_agent_primary_runtime_snapshot(agent) return rebuilt except Exception: logger.debug('[webui] Failed to refresh cached agent runtime credentials', exc_info=True) return False def _cached_agent_session_identity(agent) -> str | None: """Best-effort session id carried by a cached AIAgent. The cache key is only safe when it agrees with the object's own session identity. Some old/fake agents may not expose an identity; keep those backwards-compatible and treat them as unverifiable rather than mismatched. """ if agent is None: return None for attr in ('session_id', '_session_id'): value = getattr(agent, attr, None) if isinstance(value, str) and value: return value session_db = getattr(agent, '_session_db', None) if session_db is not None: for attr in ('session_id', '_session_id'): value = getattr(session_db, attr, None) if isinstance(value, str) and value: return value return None def _cached_agent_matches_session(agent, session_id: str) -> bool: identity = _cached_agent_session_identity(agent) return identity is None or identity == str(session_id) def _refresh_cached_agent_primary_runtime_snapshot(agent) -> None: """Keep AIAgent's primary-runtime snapshot aligned with refreshed creds. Long-lived AIAgent instances use `_primary_runtime` to restore the preferred provider after fallback/transport recovery. If WebUI refreshes a cached agent's runtime token but leaves that snapshot stale, a later restore can resurrect the old credential and undo the refresh. """ rt = getattr(agent, '_primary_runtime', None) if not isinstance(rt, dict): return base_url = getattr(agent, 'base_url', rt.get('base_url')) api_key = getattr(agent, 'api_key', rt.get('api_key', '')) client_kwargs = dict(getattr(agent, '_client_kwargs', None) or rt.get('client_kwargs', {}) or {}) rt['base_url'] = base_url rt['api_key'] = api_key rt['client_kwargs'] = client_kwargs # The default context compressor usually tracks the primary runtime too; # keep both the live compressor fields and the fallback-restoration # snapshot aligned when those attributes exist. cc = getattr(agent, 'context_compressor', None) if cc is not None: if hasattr(cc, 'base_url'): cc.base_url = base_url if hasattr(cc, 'api_key'): cc.api_key = api_key if 'compressor_base_url' in rt: rt['compressor_base_url'] = getattr(cc, 'base_url', base_url) if 'compressor_api_key' in rt: rt['compressor_api_key'] = getattr(cc, 'api_key', api_key) else: if 'compressor_base_url' in rt: rt['compressor_base_url'] = base_url if 'compressor_api_key' in rt: rt['compressor_api_key'] = api_key if getattr(agent, 'api_mode', None) == 'anthropic_messages': if hasattr(agent, '_anthropic_api_key'): rt['anthropic_api_key'] = getattr(agent, '_anthropic_api_key') if hasattr(agent, '_anthropic_base_url'): rt['anthropic_base_url'] = getattr(agent, '_anthropic_base_url') if hasattr(agent, '_is_anthropic_oauth'): rt['is_anthropic_oauth'] = getattr(agent, '_is_anthropic_oauth') def _run_agent_streaming( session_id, msg_text, model, workspace, stream_id, attachments=None, *, ephemeral=False, model_provider=None, goal_related=False, moa_config=None, ): """Run agent in background thread, writing SSE events to STREAMS[stream_id]. When ephemeral=True, session mutations are skipped — used by /btw to get a streaming answer without persisting to the parent session. """ _turn_route_model = model _turn_route_provider = model_provider cancel_event = threading.Event() q = peek_stream(stream_id) if q is not None: # Snapshot lookup is not admission: Stop can detach the stream before # we publish the initial run. Register ownership and its retained cancel # signal on the same STREAMS_LOCK edge used by cancellation. with STREAMS_LOCK: cancel_event = CANCEL_FLAGS.get(stream_id, cancel_event) if stream_id not in STREAMS or cancel_event.is_set(): q = None else: CANCEL_FLAGS[stream_id] = cancel_event STREAM_PARTIAL_TEXT[stream_id] = '' STREAM_REASONING_TEXT[stream_id] = '' STREAM_LIVE_TOOL_CALLS[stream_id] = [] register_active_run( stream_id, session_id=session_id, started_at=time.time(), phase="starting", workspace=str(workspace), model=model, provider=model_provider, ephemeral=bool(ephemeral), backend=WEBUI_LOCAL_CHAT_BACKEND, ) if q is None: # The stream was cancelled before the worker started; the route layer # already registered the stream owner, so release it here to avoid # leaking a STREAM_SESSION_OWNERS entry that the teardown finally never sees. unregister_stream_owner(stream_id) try: clear_session_writeback_owner_if_owned(session_id, stream_id) except Exception: logger.debug( "Failed to clear session writeback owner for stream %s", stream_id, exc_info=True, ) return try: run_journal = RunJournalWriter(session_id, stream_id) except Exception: run_journal = None logger.debug("Failed to initialize run journal for stream %s", stream_id, exc_info=True) if not ephemeral: try: append_turn_journal_event_for_stream( session_id, stream_id, {"event": "worker_started", "created_at": time.time()}, ) except Exception: logger.debug("Failed to append worker_started turn journal event", exc_info=True) s = None _rt = {} old_cwd = None old_exec_ask = None old_session_key = None old_session_id = None old_session_platform = None old_hermes_home = None old_profile_env = {} result = None _result_partial_pre_call_context = [] # MCP discovery moved to AFTER the per-profile HERMES_HOME mutation below # (was here at v0.51.30) — the previous placement always read the default # profile's mcp_servers because os.environ['HERMES_HOME'] hadn't been # rewritten yet. See https://github.com/nesquena/hermes-webui/issues/1968. agent = None _live_prompt_estimate_tokens = [0] _live_prompt_exact_tokens = [0] _live_prompt_estimate_tool_delta_tokens = [0] _live_prompt_estimate_seen_ids = set() # Per-stream cache for the real per-model context_length (#3256 perf). # _live_usage_snapshot() runs on every metering tick (~10x/sec during # streaming); recomputing get_model_context_length() there triggered a # config read + potential metadata/network probe on every token for # non-default models (e.g. claude-opus-4.7-1m), freezing the stream while # the default model was unaffected. The value is constant for a given # (model, base_url, provider) within one stream, so resolve it at most # once. Sentinel: None=not computed, 0=not applicable/failed, >0=real cap. _real_ctx_cache = [None] _live_usage_session_cache = [None] def _current_live_usage_session(): return _live_usage_session_snapshot( session_id, s, _live_usage_session_cache, ) def _seed_live_prompt_estimate() -> int: """Capture the latest exact prompt size before adding live tool deltas.""" if _live_prompt_estimate_tokens[0] > 0: return _live_prompt_estimate_tokens[0] _base = 0 _agent = agent if _agent is not None: try: _cc = getattr(_agent, 'context_compressor', None) if _cc: _base = getattr(_cc, 'last_prompt_tokens', 0) or 0 except Exception: _base = 0 if not _base: try: _session_obj = _current_live_usage_session() _base = getattr(_session_obj, 'last_prompt_tokens', 0) or 0 except Exception: _base = 0 _live_prompt_estimate_tokens[0] = int(_base or 0) _live_prompt_exact_tokens[0] = _live_prompt_estimate_tokens[0] return _live_prompt_estimate_tokens[0] def _bump_live_prompt_estimate(messages) -> int: """Increment a rough next-prompt estimate from live tool activity.""" if not messages: return _live_prompt_estimate_tokens[0] _seed_live_prompt_estimate() _usage = live_usage_prompt_estimate_after_tool_delta( base_prompt_tokens=_live_prompt_exact_tokens[0], exact_prompt_tokens=_live_prompt_exact_tokens[0], messages=messages, turn_tool_prompt_tokens=_live_prompt_estimate_tool_delta_tokens[0], ) _live_prompt_estimate_tokens[0] = _usage['last_prompt_tokens'] _live_prompt_estimate_tool_delta_tokens[0] = _usage['turn_tool_prompt_tokens'] return _live_prompt_estimate_tokens[0] def _live_usage_snapshot(): """Best-effort live usage payload for mid-stream UI updates. During tool execution the final `done` event has not fired yet, but the frontend still benefits from seeing the latest known token / context values. These are exact for the most recent model call and a truthful lower bound for the pending next call after a tool result is appended. """ _usage = { 'input_tokens': 0, 'output_tokens': 0, 'estimated_cost': 0, 'cache_read_tokens': 0, 'cache_write_tokens': 0, 'cache_hit_percent': None, 'context_length': 0, 'threshold_tokens': 0, 'last_prompt_tokens': 0, 'post_compression_context_tokens_estimate': None, } _session_obj = _current_live_usage_session() _agent = agent if _agent is not None: try: _usage['input_tokens'] = getattr(_agent, 'session_prompt_tokens', 0) or 0 _usage['output_tokens'] = getattr(_agent, 'session_completion_tokens', 0) or 0 _usage['estimated_cost'] = getattr(_agent, 'session_estimated_cost_usd', 0) or 0 _usage['cache_read_tokens'] = getattr(_agent, 'session_cache_read_tokens', 0) or 0 _usage['cache_write_tokens'] = getattr(_agent, 'session_cache_write_tokens', 0) or 0 except Exception: pass try: _cc = getattr(_agent, 'context_compressor', None) if _cc: _cc_cl_u = getattr(_cc, 'context_length', 0) or 0 # Stale-compressor self-heal (#3256, broadened): the # agent-side compressor caches a context_length from the # model it was *built/last-updated* with. After an in-place # model switch (or when agent_init seeded it with the global # model.context_length cap), that cached value can be the # WRONG model's window — e.g. a session on claude-opus-4.8 # (1M / 936k prompt on Copilot) whose compressor still holds # claude-opus-4.5's 168k. The original guard only corrected # the narrow case where the cached value equalled the config # cap exactly; a leftover *other-model* value (168k) slipped # straight through to the live usage payload. Broaden it: # ALWAYS resolve the real per-model window for the agent's # CURRENT model and, when that differs from the cached value, # surface the real one. Frontend hydration (GET /api/session) # already does this; this aligns the streaming path with it # so "refresh shows 1M, send-a-message drops to 168k" can't # happen. # PERF: resolve at most once per stream (cached in # _real_ctx_cache). This snapshot runs on every metering # tick; doing the config read + metadata lookup per tick # froze non-default-model streams. if _real_ctx_cache[0] is None: _resolved_real = 0 # 0 = no correction / lookup failed try: _sm_u = str(getattr(_agent, 'model', '') or '').strip() _prov_u = str(getattr(_agent, 'provider', '') or '').strip() _base_u = str(getattr(_agent, 'base_url', '') or '').strip() _key_u = getattr(_agent, 'api_key', '') or '' if _sm_u: # Resolve the real window through the SAME helper # hydration uses (routes._context_length_lookup_inputs_for_model # + get_model_context_length). This honors the # nested per-model config override # (model..models..context_length, # e.g. claude-opus-4.8 -> 1,000,000) and custom- # provider keys, so the streaming/SSE path and the # GET /api/session path land on the IDENTICAL value. # Reusing the helper (instead of hand-reading the # flat top-level model.context_length, which is # None here) is what prevents a new mismatch like # "refresh shows 1M, send-a-message shows 936k". try: from api.routes import ( _context_length_lookup_inputs_for_model as _cli_u, _should_accept_session_context_length_refresh as _accept_u, ) from agent.model_metadata import get_model_context_length as _g_u # Resolve the SESSION's own profile config, not # the ambient one. This worker is a detached # thread that does NOT inherit the per-request # thread-local profile context, so a bare # get_config() resolves the process-global # (default) profile (#3294) — for a non-default # profile that pins a different per-model # context_length, that would surface the WRONG # profile's window in the live payload. Read the # session's profile home explicitly, mirroring # the worker's own _cfg resolution below. try: from api.config import get_config_for_profile_home as _gch_u from api.profiles import get_hermes_home_for_profile as _ghp_u _ph_u = _ghp_u(getattr(_session_obj, 'profile', None)) _cfg_u = _gch_u(_ph_u) except Exception: from api.config import get_config as _gc_u _cfg_u = _gc_u() _lk_u = _cli_u( _sm_u, _prov_u, base_url=_base_u, api_key=_key_u, cfg=_cfg_u if isinstance(_cfg_u, dict) else {}, ) _real_u = _g_u( _sm_u, _lk_u.base_url, api_key=_lk_u.api_key, config_context_length=_lk_u.config_context_length, provider=_lk_u.provider or _prov_u or '', custom_providers=_lk_u.custom_providers, ) or 0 # Only treat it as a correction when the real # window is valid AND disagrees with the # compressor's cached value. Equal => nothing # to fix, leave the fast path untouched. # #4248: never let a low-confidence 256k metadata # fallback clobber a LARGER cached window — that # would reintroduce the very "drops to a smaller # window mid-stream" regression this guard fixes. # Reuse the exact acceptance gate hydration uses. # NOTE: we deliberately omit model_changed (=False # default) here, unlike hydration. The streaming # path can't cheaply know if the model changed # since the compressor was seeded, so we err # toward the LARGER window (auto-compress fires # late, not early — the safe direction), and the # next GET /api/session hydration self-heals any # genuine downward 256k case via model_changed. if ( _real_u and _real_u != _cc_cl_u and _accept_u(_cc_cl_u, _real_u) ): _resolved_real = _real_u except TypeError: # Older hermes-agent: legacy 2-arg form. try: from api.routes import ( _should_accept_session_context_length_refresh as _accept2_u, ) from agent.model_metadata import get_model_context_length as _g2_u _real_u = _g2_u(_sm_u, _base_u) or 0 if ( _real_u and _real_u != _cc_cl_u and _accept2_u(_cc_cl_u, _real_u) ): _resolved_real = _real_u except Exception: pass except Exception: pass except Exception: _resolved_real = 0 _real_ctx_cache[0] = _resolved_real # Apply the cached real cap when the guard determined one. if _real_ctx_cache[0]: # Also rescale threshold_tokens by the same ratio so the # auto-compress trigger reflects the real window, not # the stale global cap (e.g. 197.2k @ 232K cap → ~850k # @ 1M real cap). _orig_cc_cl = getattr(_cc, 'context_length', 0) or 0 _orig_thresh = getattr(_cc, 'threshold_tokens', 0) or 0 _cc_cl_u = _real_ctx_cache[0] if _orig_cc_cl > 0 and _orig_thresh > 0: _scaled_thresh = int(_orig_thresh * _real_ctx_cache[0] / _orig_cc_cl) _usage['context_length'] = _cc_cl_u _usage['threshold_tokens'] = _scaled_thresh _usage['last_prompt_tokens'] = getattr(_cc, 'last_prompt_tokens', 0) or 0 else: _usage['context_length'] = _cc_cl_u _usage['threshold_tokens'] = _orig_thresh _usage['last_prompt_tokens'] = getattr(_cc, 'last_prompt_tokens', 0) or 0 else: _usage['context_length'] = _cc_cl_u _usage['threshold_tokens'] = getattr(_cc, 'threshold_tokens', 0) or 0 _usage['last_prompt_tokens'] = getattr(_cc, 'last_prompt_tokens', 0) or 0 except Exception: pass if _session_obj is not None: for _field in ('input_tokens', 'output_tokens', 'estimated_cost', 'cache_read_tokens', 'cache_write_tokens', 'context_length', 'threshold_tokens', 'last_prompt_tokens'): if not _usage.get(_field): try: _usage[_field] = getattr(_session_obj, _field, 0) or 0 except Exception: pass _post_compression_estimate = getattr( _session_obj, 'post_compression_context_tokens_estimate', None, ) if isinstance(_post_compression_estimate, int) and _post_compression_estimate > 0: _usage['post_compression_context_tokens_estimate'] = _post_compression_estimate _real_prompt_tokens = int(_usage.get('last_prompt_tokens') or 0) _usage['cache_hit_percent'] = prompt_cache_hit_percent( _usage.get('cache_read_tokens') or 0, _usage.get('input_tokens') or 0, ) if _real_prompt_tokens and _real_prompt_tokens != _live_prompt_exact_tokens[0]: _live_prompt_exact_tokens[0] = _real_prompt_tokens _live_prompt_estimate_tokens[0] = _real_prompt_tokens _live_prompt_estimate_tool_delta_tokens[0] = 0 elif _live_prompt_estimate_tokens[0] > _real_prompt_tokens: _usage['last_prompt_tokens'] = _live_prompt_estimate_tokens[0] return _usage # Metering ticker — emits a metering event at 1 Hz while sessions are active. # When get_interval() returns >= 10.0 (no active sessions), the ticker exits # so no idle readings are emitted and the SSE consumer sees nothing. # # #4633/#2476: begin_session() and the ticker .start() are deferred into the # outer `try` below so the outer `finally` (which pops STREAMS/CANCEL_FLAGS) # always runs its paired end_session()/_metering_stop.set() teardown. A raise # between here and that `try` would otherwise leak the _sessions[stream_id] # entry — get_stats() only prunes sessions with first_token_ts > 0, so a # zero-token turn (pre-flight cancel, setup raise) is never reclaimed and its # count inflates the SSE `active` field. Deferring .start() until after `put` # is defined also removes a latent start-before-put ordering window. _metering_stop = threading.Event() def _metering_ticker(): while True: interval = meter().get_interval() if interval >= 10.0: break # nothing active — stop the ticker if _metering_stop.wait(interval): break # stream was cancelled or ended — exit stats = meter().get_stats(stream_id) stats['session_id'] = session_id stats['usage'] = _live_usage_snapshot() put('metering', stats) _metering_thread = threading.Thread(target=_metering_ticker, daemon=True) _success_writeback_committed = False _steer_settled = False _returned_pending_steer = [] def _remember_pending_steer_result(value): text = value.get('pending_steer') if isinstance(value, dict) else None if isinstance(text, str) and text: _returned_pending_steer.append(text) def _settle_pending_steer(): # Single terminal boundary for success, returned errors, exceptions and # successful self-heal. Admission closes before the last slot consumer; # registry locks never cover SSE writes or Agent drain callbacks. nonlocal _steer_settled with STREAMS_LOCK: if _steer_settled: return _steer_settled = True target = AGENT_INSTANCES.get(stream_id) or agent if stream_id in STREAMS and not cancel_event.is_set(): update_active_run(stream_id, phase="finalizing") leftovers = list(_returned_pending_steer) try: drain = getattr(target, '_drain_pending_steer', None) text = drain() if callable(drain) else None if text: leftovers.append(str(text)) except Exception: logger.debug("Failed to drain pending steer for session %s", session_id) if leftovers: put('pending_steer_leftover', { 'session_id': session_id, 'text': '\n'.join(leftovers), }) def put(event, data): if event in ('done', 'apperror', 'stream_end'): _settle_pending_steer() # If cancelled, drop all further events except the cancel event itself if cancel_event.is_set() and not _success_writeback_committed and event not in ('cancel', 'apperror'): return event_id = None if run_journal is not None: try: journaled = run_journal.append_sse_event(event, data) # Carry the exact journal id for this queued frame. A global # "latest event" side channel is still kept for legacy queues, # but StreamChannel subscribers need the per-item id so a # queued backlog cannot advance the browser cursor past an # undelivered event. event_id = (journaled or {}).get('event_id') if isinstance(journaled, dict) else None if event_id: STREAM_LAST_EVENT_ID[stream_id] = event_id except Exception: logger.debug("Failed to append run journal event %s for stream %s", event, stream_id, exc_info=True) if event_id and hasattr(q, "note_last_event_id"): try: q.note_last_event_id(event_id) except Exception: logger.debug("Failed to note event_id %s for stream %s", event_id, stream_id, exc_info=True) try: queue_item = (event, data, event_id) if hasattr(q, "subscribe_with_snapshot") else (event, data) q.put_nowait(queue_item) except Exception: logger.debug("Failed to put event to queue") _last_runtime_model_identity = None _runtime_model_session_id = session_id def _observe_runtime_model(): """Publish this turn's Agent identity at output, never at an attempted route.""" nonlocal _last_runtime_model_identity raw_model = getattr(agent, 'model', None) if not isinstance(raw_model, str) or not raw_model.strip(): return raw_provider = getattr(agent, 'provider', None) provider = str(raw_provider).strip().lstrip('@').lower() if isinstance(raw_provider, str) else '' # The Agent may carry a provider-qualified routing hint in its model. from api.config import _parse_provider_qualified_model_id parsed = _parse_provider_qualified_model_id(raw_model.strip()) model_id = (parsed[0] if parsed else raw_model).strip() if not model_id: return fallback_active = getattr(agent, '_provider_fallback_active', None) is True identity = (provider, model_id.lower(), fallback_active) if identity == _last_runtime_model_identity: return payload = { 'session_id': _runtime_model_session_id, 'stream_id': stream_id, 'model': model_id, 'fallback_active': fallback_active, 'phase': 'observed_output', } if provider: payload['provider'] = provider put('runtime_model', payload) _last_runtime_model_identity = identity # #5940: capture a terminal (non-retryable) provider error the Agent emits via # its lifecycle status_callback. The Agent aborts a non-retryable API error # (e.g. HTTP 400 "invalid model / no credentials") with # `_emit_status("❌ Non-retryable error (HTTP ): ")` but the run # result / agent._last_error are empty for that path, so turn-completion below # fell through to the misleading `no_response` "silent rate limit, try again" # message. Stash the emitted terminal error here (single-element list = closure # write without nonlocal) so it can seed `_last_err` and let the classifier # surface the real, actionable cause (model_not_found / auth_mismatch). _captured_terminal_error = [None] def _agent_status_callback(kind, message): """Bridge Agent lifecycle status into WebUI SSE. Passes compression events as 'compressing' events and rate-limit/fallback events as 'warning' events so the frontend can surface them to the user. Also captures a terminal non-retryable provider error (#5940) so the turn-completion classifier can report the real cause instead of the generic no_response fallback. All other lifecycle messages are dropped. """ nonlocal _last_runtime_model_identity _message = str(message or '').strip() _kind = str(kind or '').strip().lower() if not _message: return _lower = _message.lower() # #5940: a non-retryable terminal provider error the Agent aborted on. Keep # the FIRST one seen this turn (the original cause; later fallback notices # are handled separately below). Matched on the Agent's emitted shape. if ( _captured_terminal_error[0] is None and 'non-retryable error' in _lower and 'http' in _lower ): _captured_terminal_error[0] = _message if _is_agent_compression_start_status(_kind, _message): put('compressing', { 'session_id': session_id, 'message': 'Compressing context', }) return # Pass through rate-limit and fallback messages so the frontend can # show them as warnings via the existing messages.js 'warning' listener. # Session turn-lease waits (another Hermes process owns this session) # use the same channel so a delayed turn explains itself. if _is_session_lease_wait_message(_kind, _message): put('warning', {'type': 'session_lease_wait', 'message': _message}) return _is_fallback_notice = _is_fallback_lifecycle_message(_kind, _message) if _is_fallback_notice: _last_runtime_model_identity = None put('warning', {'type': 'fallback', 'message': _message}) # xsession wakeup misroute root fix (Option 1): pre-init so the outer # finally can always reset even if an exception fires before the bind. # Placed ABOVE the _checkpoint_stop cluster so that cluster stays adjacent # to the `try:` (preserves the Issue #765 static-locator invariant). _turn_session_identity_tokens = None _streaming_cron_profile_home_token = None _turn_pending_source = 'webui' _streaming_hermes_home_override_ctx = (None, None, False) _streaming_skill_home_snapshot = None _restore_streaming_skill_home_modules = False _acquired_streaming_skill_home_patch_lock = False def _register_agent_if_current(candidate, cache_signature=None): nonlocal agent # Every constructor, including both credential self-heal branches, must # share the Stop publication edge. CANCEL_FLAGS may already be detached; # the worker-retained event and stream membership remain authoritative. with STREAMS_LOCK: cancelled = cancel_event.is_set() or stream_id not in STREAMS if not cancelled: AGENT_INSTANCES[stream_id] = candidate # Check and publication share one admission, not a check followed # by an unlocked cache/lifecycle write that could undo Stop. if cache_signature is not None: from api.config import SESSION_AGENT_CACHE, SESSION_AGENT_CACHE_LOCK with SESSION_AGENT_CACHE_LOCK: SESSION_AGENT_CACHE[session_id] = (candidate, cache_signature) SESSION_AGENT_CACHE.move_to_end(session_id) if not ephemeral: try: # Pure in-memory bookkeeping; no Agent/provider call. from api.session_lifecycle import register_agent register_agent(session_id, candidate) except Exception: logger.debug("Lifecycle register_agent failed for session %s", session_id, exc_info=True) if not cancelled: return True # A cache hit is a borrowed reusable object, not this worker's private # candidate. Stop may already have let a successor reuse it. Rejected # cache-hit admission must not interrupt that other turn. if cache_signature is None and not ephemeral: if agent is candidate: agent = None return False # Newly constructed candidates were never published on this failed # admission; they cannot have been borrowed by a successor from us. # No Agent call, session persistence or SSE write under registry locks. try: candidate.interrupt("Cancelled before start") except Exception: logger.debug("Failed to interrupt cancelled candidate agent") return False def _agent_can_invoke(candidate): nonlocal agent # Prompt preparation and LRU cleanup can yield after registration. Admit # each invocation again at its point of use; never hold registry locks # across run_conversation or interrupt. A later Stop interrupts the # already-admitted invocation through the registered Agent as usual. from api.config import ( SESSION_AGENT_CACHE, SESSION_AGENT_CACHE_LOCK, SESSION_WRITEBACK_OWNERS, SESSION_WRITEBACK_OWNERS_LOCK, ) with STREAMS_LOCK: current = (not cancel_event.is_set() and stream_id in STREAMS and AGENT_INSTANCES.get(stream_id) is candidate) if not current: if AGENT_INSTANCES.get(stream_id) is candidate: AGENT_INSTANCES.pop(stream_id, None) # Agent identity is not turn identity: a successor can reuse the # same cached object. Use the existing cancellation-surviving # ownership record, holding its lock through cache retirement. # Missing ownership also fails closed (successor may have ended). with SESSION_WRITEBACK_OWNERS_LOCK: if SESSION_WRITEBACK_OWNERS.get(session_id) == stream_id: with SESSION_AGENT_CACHE_LOCK: entry = SESSION_AGENT_CACHE.get(session_id) if entry and entry[0] is candidate: SESSION_AGENT_CACHE.pop(session_id, None) # Dirty memory segments keep their original owner. from api.session_lifecycle import unregister_agent unregister_agent(session_id) # This invocation never started. Stop owns any required interrupt of the # running Agent; a late duplicate interrupt could hit its new borrower. # Drop our local borrowed handle too: final Steer drain must not reach # the successor through the old worker's fallback `agent` reference. if not current and agent is candidate: agent = None return current # Initialised here (before any code that may raise) so the outer `finally` # block can safely check `if _checkpoint_stop is not None` even when an # exception fires before the checkpoint thread is created (Issue #765). _checkpoint_stop = None _ckpt_thread = None _agent_lock = None try: # Register this stream with the global streaming meter and start the 1 Hz # metering ticker. Kept INSIDE the outer try so the outer `finally`'s # end_session()/_metering_stop.set() teardown is always paired (#4633/#2476). meter().begin_session(stream_id) _metering_thread.start() # Bind THIS turn's session identity to the worker thread/context BEFORE # any agent work (so every mid-turn notify_on_complete background spawn # captures THIS session, not a concurrent turn's process-global env). # Co-located with the existing env-restore lifecycle: set here, reset # in the outer finally next to _clear_thread_env(). # Resolved the same way as `s.workspace` below so the bound cwd and the # session's own record cannot disagree. Guarded: a turn must not die # here because a workspace path is malformed. s = get_session(session_id) try: _turn_workspace_cwd = str(_resolve_path(workspace, profile=getattr(s, 'profile', None))) except Exception: _turn_workspace_cwd = "" logger.debug("per-turn workspace cwd resolve failed", exc_info=True) _turn_session_identity_tokens = _set_turn_session_identity( session_id, workspace=_turn_workspace_cwd ) _turn_pending_source = getattr(s, 'pending_user_source', None) or 'webui' _active_turn_identity = _active_turn_authority(s, stream_id, msg_text) update_active_run(stream_id, phase="running", session_id=session_id) s.workspace = _turn_workspace_cwd _last_persisted_model = None _last_persisted_provider = None _turn_owns_persisted_model = False provider_context = ( str(model_provider).strip().lower() if model_provider is not None else getattr(s, "model_provider", None) ) provider_context = str(provider_context).strip().lower() if provider_context else None _agent_lock = _get_session_agent_lock(session_id) # #4251: the route layer already persisted this turn's model under the # session lock before dispatch, so a mismatch here means a newer picker # write won the race and must not be clobbered by the worker thread. with _agent_lock: _last_persisted_model = getattr(s, "model", None) _last_persisted_provider = getattr(s, "model_provider", None) if _last_persisted_provider is not None: _last_persisted_provider = str(_last_persisted_provider).strip().lower() or None _persisted_model_is_empty = _last_persisted_model in (None, "") _provider_matches = _last_persisted_provider in (None, provider_context) if _persisted_model_is_empty or ( _last_persisted_model == model and _provider_matches ): s.model = model s.model_provider = provider_context _last_persisted_model = model _last_persisted_provider = provider_context _turn_owns_persisted_model = True # TD1: set thread-local env context so concurrent sessions don't clobber globals # Check for pre-flight cancel (user cancelled before agent even started) if cancel_event.is_set(): with _agent_lock: _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) put('cancel', _cancel_event_payload('Cancelled before start')) return # Resolve profile home for this agent run — use the session's own profile # (stamped at new_session() time from the client's S.activeProfile) so that # two concurrent tabs on different profiles don't clobber each other via the # process-level active-profile global. Falls back gracefully. try: from api.profiles import ( filter_runtime_env_for_gateway_parity, patch_skill_home_modules, restore_skill_home_modules, snapshot_skill_home_modules, get_hermes_home_for_profile, get_profile_runtime_env, _skill_modules_support_profile_home, _SKILL_HOME_MODULE_PATCH_LOCK, ) _profile_home_path = get_hermes_home_for_profile(getattr(s, 'profile', None)) _profile_home = str(_profile_home_path) _streaming_cron_profile_home_token = _STREAMING_CRON_PROFILE_HOME.set(_profile_home) _profile_runtime_env = get_profile_runtime_env(_profile_home_path) _safe_profile_runtime_env = filter_runtime_env_for_gateway_parity(_profile_runtime_env) except ImportError: _profile_home = os.environ.get('HERMES_HOME', '') _profile_runtime_env = {} _safe_profile_runtime_env = {} patch_skill_home_modules = None snapshot_skill_home_modules = None restore_skill_home_modules = None _skill_modules_support_profile_home = None _SKILL_HOME_MODULE_PATCH_LOCK = None # Profile-aware provider/model enrichment: when the session belongs # to a profile that specifies model.provider and model.default, use # those to set provider_context and repair stale models. model, provider_context, _repaired = _apply_profile_home_context_to_streaming_model( model=model, provider_context=provider_context, profile_home=_profile_home, has_profile=bool(getattr(s, "profile", None)), ) # #4251: only apply the profile-repair persistence if this turn still # owns the session model/provider pair it last wrote. provider_context = str(provider_context).strip().lower() if provider_context else None with _agent_lock: _current_provider = getattr(s, "model_provider", None) if _current_provider is not None: _current_provider = str(_current_provider).strip().lower() or None if ( _turn_owns_persisted_model and getattr(s, "model", None) == _last_persisted_model and _current_provider == _last_persisted_provider ): s.model_provider = provider_context if _repaired and model != (s.model or ""): s.model = model # Capture the resolved profile name now, while profile context is # reliable. Used in the compression migration block to stamp s.profile # on the continuation session. We resolve it here rather than calling # get_active_profile_name() at compression time because that function # reads thread-local storage (_tls.profile) set by set_request_profile() # on the HTTP handler thread. The streaming thread is a separate # threading.Thread and does not inherit TLS. At compression time, # get_active_profile_name() would fall back to the process-global # _active_profile, which may belong to a different concurrent tab. _resolved_profile_name = getattr(s, 'profile', None) if not _resolved_profile_name: try: from api.profiles import get_active_profile_name _resolved_profile_name = get_active_profile_name() except Exception: _resolved_profile_name = None _thread_env = _build_agent_thread_env( _profile_runtime_env, str(s.workspace), session_id, _profile_home, ) _streaming_hermes_home_override_ctx = _set_streaming_hermes_home_override(_profile_home) _set_thread_env(**_thread_env) # process_complete agent-wakeup wiring (ours-original, Option B): bind # this session's HERMES_SESSION_KEY to its WebUI session_id so the # drain thread can route notify_on_complete events back to the right # SSE channel / server-side wakeup. try: from api.background_process import register_process_session register_process_session(session_id, session_id) except Exception: logger.debug("register_process_session failed", exc_info=True) # first-time module initialisation (which can be slow) does not # block other concurrent sessions waiting on _ENV_LOCK (#2024). ensure_agent_runtime_current() _prewarm_skill_tool_modules() _install_streaming_cronjob_profile_wrapper() # Full-turn serialization is only needed for static/legacy skill-module # resolution, where process-global skill-module globals are still used. # Dynamic-capable modules continue concurrent execution. _streaming_override_installed = bool(_streaming_hermes_home_override_ctx[2]) _streaming_modules_are_dynamic = False if patch_skill_home_modules is not None and snapshot_skill_home_modules is not None: if _streaming_override_installed and _skill_modules_support_profile_home is not None: try: _streaming_modules_are_dynamic = bool( _skill_modules_support_profile_home(_profile_home_path) ) except Exception: logger.debug( "Failed to evaluate streaming skill-module home capability for profile %r", _profile_home, exc_info=True, ) _streaming_modules_are_dynamic = False if not (_streaming_override_installed and _streaming_modules_are_dynamic): _restore_streaming_skill_home_modules = True _SKILL_HOME_MODULE_PATCH_LOCK.acquire() _acquired_streaming_skill_home_patch_lock = True # Still set process-level env as fallback for tools that bypass thread-local # Acquire lock only for the env mutation, then release before the agent runs. # The finally block re-acquires to restore — keeping critical sections short # and preventing a deadlock where the restore would re-enter the same lock. with _ENV_LOCK: if _restore_streaming_skill_home_modules: # Snapshot and patch before mutating process env so setup # failures can unwind without leaking either state. _streaming_skill_home_snapshot = snapshot_skill_home_modules() patch_skill_home_modules(Path(_profile_home)) old_profile_env = {key: os.environ.get(key) for key in _safe_profile_runtime_env} old_cwd = os.environ.get('TERMINAL_CWD') old_exec_ask = os.environ.get('HERMES_EXEC_ASK') old_session_key = os.environ.get('HERMES_SESSION_KEY') old_session_id = os.environ.get('HERMES_SESSION_ID') old_session_platform = os.environ.get('HERMES_SESSION_PLATFORM') old_session_chat_id = os.environ.get('HERMES_SESSION_CHAT_ID') old_hermes_home = os.environ.get('HERMES_HOME') os.environ.update(_safe_profile_runtime_env) os.environ['TERMINAL_CWD'] = str(s.workspace) os.environ['HERMES_EXEC_ASK'] = '1' os.environ['HERMES_SESSION_KEY'] = session_id os.environ['HERMES_SESSION_ID'] = session_id os.environ['HERMES_SESSION_PLATFORM'] = 'webui' # process_complete wiring (ours-original, Option B): see # _build_agent_thread_env above. os.environ['HERMES_SESSION_CHAT_ID'] = str(session_id) if _profile_home: os.environ['HERMES_HOME'] = _profile_home # Prefer context-local Hermes-home overrides when available. # In that mode, tools.skills_tool._skills_dir() and # tools.skill_manager_tool._skills_dir() can resolve the active # profile from get_hermes_home() and keep per-thread isolation # without mutating module globals. If override installation # succeeds for both modules, skip process-cache patching. # If either module is static/missing/raises, the legacy path # above has already snapshotted and patched under this lock. # Lock released — agent runs without holding it # ── MCP Server Discovery (lazy import, idempotent) ── # MUST run AFTER the HERMES_HOME mutation above — `discover_mcp_tools()` # reads `~/.hermes/config.yaml` via `get_hermes_home()`, which uses # `os.environ['HERMES_HOME']`. Calling it before the mutation always # loaded the default profile's `mcp_servers`, even when the session # was stamped with a non-default profile. See issue #1968. # # NOTE: hermes-agent keys MCP connections by `(profile_home, name)` only # for a routed profile (override != process home). The HERMES_HOME # mirror above would make this turn's own profile look like the process # profile, falling back to bare, cross-profile names, unless the agent # honours the pin from `api.profiles._pin_process_profile_home()`. try: from api.agent_compat import agent_attr discover_mcp_tools = agent_attr("tools.mcp_tool", "discover_mcp_tools", "tools.mcp_tool_discovery") discover_mcp_tools() except Exception: pass # MCP not available or not configured — non-fatal # Register a gateway-style notify callback so the approval system can # push the `approval` SSE event the moment a dangerous command is # detected, without waiting for the next on_tool() poll cycle. # Without this, the agent thread blocks inside the terminal tool # waiting for approval that the UI never knew to ask for, leaving # the chat stuck in "Thinking…" forever. _approval_registered = False _unreg_notify = None _cleanup_gateway_pending_mirror = None try: try: from api.route_approvals import ( settle_gateway_pending_local_notification as _settle_pending_for_polling, retire_gateway_pending_mirror as _retire_gateway_pending_mirror, ) def _cleanup_gateway_pending_mirror(): _retire_gateway_pending_mirror(session_id) except ImportError: _settle_pending_for_polling = None _cleanup_gateway_pending_mirror = None from tools.approval import ( register_gateway_notify as _reg_notify, unregister_gateway_notify as _unreg_notify, ) def _approval_notify_cb(approval_data): if _settle_pending_for_polling is not None: try: auto_resolved, head, total = _settle_pending_for_polling( session_id, approval_data, ) if auto_resolved and head is None: return approval_data = {**(head or approval_data), "pending_count": total} except Exception: logger.warning("Failed to mirror approval into WebUI polling state", exc_info=True) put('approval', approval_data) _reg_notify(session_id, _approval_notify_cb) _approval_registered = True except ImportError: logger.debug("Approval module not available, falling back to polling") _clarify_registered = False _unreg_clarify_notify = None try: from api.clarify import ( register_gateway_notify as _reg_clarify_notify, unregister_gateway_notify as _unreg_clarify_notify, ) def _clarify_notify_cb(clarify_data): put('clarify', clarify_data) _reg_clarify_notify(session_id, _clarify_notify_cb) _clarify_registered = True except ImportError: logger.debug("Clarify module not available, falling back to polling") def _clarify_callback_impl(question, choices, sid, cancel_evt, put_event): """Bridge Hermes clarify prompts to the WebUI.""" timeout = _clarify_timeout_seconds(_clarify_session_config(sid)) choices_list = [str(choice) for choice in (choices or [])] data = { 'question': str(question or ''), 'choices_offered': choices_list, 'session_id': sid, 'kind': 'clarify', 'requested_at': time.time(), 'timeout_seconds': timeout, } try: from api.clarify import submit_pending as _submit_clarify_pending, clear_pending as _clear_clarify_pending except ImportError: return ( "The user did not provide a response within the time limit. " "Use your best judgement to make the choice and proceed." ) entry = _submit_clarify_pending(sid, data) response, expired = _await_clarify_response(entry, timeout, cancel_evt) if expired: _clear_clarify_pending(sid) return ( response or "The user did not provide a response within the time limit. " "Use your best judgement to make the choice and proceed." ) try: _token_sent = False # tracks whether any streamed tokens were sent _self_healed = False # (#1401) prevents infinite self-heal retries # Per-message reasoning: dict maps assistant-message index → accumulated text # (#3587) replaces the flat _reasoning_text string so each intermediate # assistant turn (before tool calls) keeps its own reasoning segment. _reasoning_segments: dict = {} # Incremental folded indexes mirroring each reasoning buffer. The # live echo strip consults them instead of re-walking the raw # buffers: an unbounded raw scan of the growing transcript is # quadratic on whitespace-heavy output (#7569 review). Each index # is fed on every append and re-indexed after a strip. _reasoning_segment_indexes: dict = {} _stream_reasoning_index = _CompactEchoIndex() _reasoning_buffer_index = _CompactEchoIndex() _current_reasoning_idx = 0 _tool_boundary_advanced = False _live_tool_calls = [] # tool progress fallback when final messages omit tool IDs # Throttle: emit metering events at most every 100 ms so the per-message # TPS label feels live during fast token streams without flooding SSE. _metering_last_emit = [time.monotonic() - 1] # fire immediately on first token _reasoning_last_put = [0.0] _reasoning_buffer = [''] _metering_output_deltas = [0] _metering_reasoning_deltas = [0] def _flush_reasoning_buffer(): # #4729: emit any coalesced-but-not-yet-flushed reasoning text immediately. # The ~10 Hz throttle in on_reasoning leaves a sub-100ms tail in the buffer; # the agent never calls reasoning_callback(None), and reasoning can transition # to tool calls / visible output, so we must flush at every boundary that # closes or reorders the live reasoning stream — otherwise the tail is # silently lost from the live Thinking view (the frontend appends deltas). if _reasoning_buffer[0]: put('reasoning', {'text': _reasoning_buffer[0]}) _reasoning_buffer[0] = '' # The folded index mirrors this buffer 1:1 — dropping the # text without dropping the index would leave it describing # text that is no longer there. _reasoning_buffer_index.reset() def _emit_metering(): now = time.monotonic() if now - _metering_last_emit[0] < 0.1: return _metering_last_emit[0] = now stats = meter().get_stats(stream_id) stats['session_id'] = session_id stats['usage'] = _live_usage_snapshot() stats.setdefault('tps_available', False) stats.setdefault('estimated', False) put('metering', stats) def _is_visible_output_echo(text: str) -> bool: candidate = _compact_for_echo_compare(text) if not candidate: return False visible_output = STREAM_PARTIAL_TEXT.get(stream_id, '') visible_tail = _compact_for_echo_compare( visible_output[-max(len(str(text)) * 2, 512):] ) if visible_tail and visible_tail.endswith(candidate): return True # Some runtimes can report a prefix of the already-streamed final # answer through reasoning after visible output has completed. That # prefix is not a tail echo, so catch only substantial chunks that # are already present in the visible assistant stream. Short text # stays on the stricter suffix path to avoid hiding genuine # reasoning that happens to reuse an answer phrase. if len(candidate) < 80: return False visible_compact = _compact_for_echo_compare(visible_output) return bool(visible_compact and candidate in visible_compact) def _strip_reasoning_output_echo(text: str) -> bool: nonlocal _reasoning_segments removed = False if stream_id in STREAM_REASONING_TEXT: cut = _stream_reasoning_index.cut_to(text) if cut is not None: next_text = STREAM_REASONING_TEXT.get(stream_id, '')[:cut].rstrip() STREAM_REASONING_TEXT[stream_id] = next_text _stream_reasoning_index.reset() _stream_reasoning_index.append(next_text) removed = True cut = _reasoning_buffer_index.cut_to(text) if cut is not None: next_buffer = _reasoning_buffer[0][:cut].rstrip() _reasoning_buffer[0] = next_buffer _reasoning_buffer_index.reset() _reasoning_buffer_index.append(next_buffer) removed = True for idx in (_current_reasoning_idx, _current_reasoning_idx - 1): if idx not in _reasoning_segments: continue segment_index = _reasoning_segment_indexes.get(idx) if segment_index is None: continue cut = segment_index.cut_to(text) if cut is None: continue next_segment = _reasoning_segments.get(idx, '')[:cut].rstrip() if next_segment: _reasoning_segments[idx] = next_segment segment_index.reset() segment_index.append(next_segment) else: _reasoning_segments.pop(idx, None) _reasoning_segment_indexes.pop(idx, None) removed = True break return removed def on_token(text): nonlocal _token_sent if text is None: return # end-of-stream sentinel if text: _observe_runtime_model() # #4729: visible output is starting — flush any buffered reasoning tail # first so the live Thinking stream is complete before/at the transition. _flush_reasoning_buffer() _token_sent = True # Accumulate partial text so cancel_stream() can persist it (#893). # # STREAMS_LOCK contract for the three STREAM_* buffers (partial text, # reasoning, live tool calls): the per-token hot path below mirrors # into them WITHOUT holding STREAMS_LOCK, while snapshot readers hold # STREAMS_LOCK (_snapshot_and_append_partial_on_error / cancel_stream). # This is deliberate and safe for two reasons, NOT because `+=` is one # bytecode (it is not — `d[k] += s` compiles to load/add/STORE_SUBSCR): # 1. Single writer: only this streaming thread ever writes a given # stream_id's buffers, so there is no writer/writer race to tear. # 2. Reader/writer atomicity under the GIL: `+=` builds a complete new # immutable str, then the final STORE_SUBSCR that binds it into the # dict is a single atomic bytecode; a concurrent reader's dict.get # therefore returns either the old or the new *complete* string # object (strings are immutable — never a half-built one). Same for # the atomic list.append / single-key dict writes on the other two. # So a reader sees a complete-but-possibly-stale value, never a torn one. # The snapshot is a best-effort partial that later journal/SSE events # reconcile, so exact-latest is not required. Taking STREAMS_LOCK per # token would add real contention against readers copying large buffers # and would entangle the documented LOCK -> STREAMS_LOCK ordering — not # worth it for a recoverable staleness window. if stream_id in STREAM_PARTIAL_TEXT: STREAM_PARTIAL_TEXT[stream_id] += str(text) put('token', {'text': text}) # Update live throughput from stream delta callbacks, not from # byte/character length. If a backend cannot provide live deltas, # the frontend hides TPS rather than showing an estimate. _metering_output_deltas[0] += 1 meter().record_token(stream_id, _metering_output_deltas[0]) _emit_metering() def on_reasoning(text): nonlocal _reasoning_segments, _current_reasoning_idx, _tool_boundary_advanced if text is None: # Flush any remaining coalesced reasoning buffer so the last # partial window is not lost when the reasoning phase ends. _flush_reasoning_buffer() return if text: _observe_runtime_model() _tool_boundary_advanced = False reasoning_delta = str(text) # Some runtimes mirror user-visible progress text through the # reasoning channel after it already streamed as normal assistant # output. Treat that as an echo, otherwise the UI renders the # same sentence again inside a Thinking card. if _is_visible_output_echo(reasoning_delta): return # Accumulate into the current message's segment (#3587) _reasoning_segments[_current_reasoning_idx] = ( _reasoning_segments.get(_current_reasoning_idx, '') + reasoning_delta ) # Keep the folded index in step with the segment text. _reasoning_segment_indexes.setdefault( _current_reasoning_idx, _CompactEchoIndex() ).append(reasoning_delta) # Mirror full concatenation to shared dict so cancel_stream() can persist # it (#1361 §A). Cancel only creates one partial message, so the flat # concatenation is correct there. # Lock-free GIL-atomic mirror — see the STREAMS_LOCK contract in on_token. if stream_id in STREAM_REASONING_TEXT: STREAM_REASONING_TEXT[stream_id] += reasoning_delta _stream_reasoning_index.append(reasoning_delta) # Accumulate into a coalescing buffer so every delta reaches the # browser — reasoning deltas are incremental, not idempotent. _reasoning_buffer[0] += reasoning_delta _reasoning_buffer_index.append(reasoning_delta) # Throttle reasoning SSE events to ~10 Hz to avoid overwhelming the # frontend renderer. Each event triggers _parseStreamState() which # scans the full accumulated text — 10k+ reasoning tokens/second # builds up and locks the JS main thread. The user still sees live # Thinking updates, just at a sustainable rate. now = time.monotonic() if now - _reasoning_last_put[0] >= 0.1: _reasoning_last_put[0] = now put('reasoning', {'text': _reasoning_buffer[0]}) _reasoning_buffer[0] = '' # The folded index mirrors this buffer 1:1 — dropping the # text without dropping the index would leave it describing # text that is no longer there. _reasoning_buffer_index.reset() # Track reasoning deltas in the meter so live TPS reflects all AI output. _metering_reasoning_deltas[0] += 1 meter().record_reasoning(stream_id, _metering_reasoning_deltas[0]) _emit_metering() def on_interim_assistant(text, **cb_kwargs): nonlocal _current_reasoning_idx # Advance the per-message reasoning index unconditionally (#3587): # even if this callback fires with empty text, a new assistant # segment is starting and subsequent reasoning must be attributed # to the next message. _current_reasoning_idx += 1 if text is None: return visible = str(text).strip() if not visible: return reasoning_echo = _strip_reasoning_output_echo(visible) already_streamed = bool(cb_kwargs.get('already_streamed', False)) or _is_visible_output_echo(visible) payload = { 'text': visible, 'already_streamed': already_streamed, } if reasoning_echo: payload['reasoning_echo'] = True put('interim_assistant', payload) # Pre-initialise the activity counter here so on_tool (which # closes over it) never captures an unbound name even if this # block is reordered later (Issue #765). _checkpoint_activity = [0] _live_tool_event_start_ids = set() _live_tool_event_complete_ids = set() def _tool_args_snapshot(args): args_snap = {} if isinstance(args, dict): for k, v in list(args.items())[:4]: s2 = str(v) cap = _TOOL_ARG_CONTENT_CAP if str(k).lower() in _TOOL_ARG_CONTENT_KEYS else 120 args_snap[k] = s2[:cap] + ('...' if len(s2) > cap else '') return args_snap def _record_live_tool_start(tool_call_id, name, args): if not tool_call_id or tool_call_id in _live_prompt_estimate_seen_ids: return False _live_prompt_estimate_seen_ids.add(tool_call_id) _tool_call = { 'id': tool_call_id, 'type': 'function', 'function': { 'name': str(name or ''), 'arguments': json.dumps(args if isinstance(args, dict) else {}, ensure_ascii=False, sort_keys=True), }, } _bump_live_prompt_estimate([{ 'role': 'assistant', 'content': '', 'tool_calls': [_tool_call], }]) return True def _record_live_tool_complete(tool_call_id, name, function_result): if not tool_call_id: return False _result_text = _tool_result_snippet(function_result) _bump_live_prompt_estimate([{ 'role': 'tool', 'name': str(name or ''), 'tool_call_id': tool_call_id, 'content': _result_text, }]) return True def on_tool(*cb_args, **cb_kwargs): nonlocal _reasoning_segments, _current_reasoning_idx, _tool_boundary_advanced # #4729: a tool boundary closes/reorders the live reasoning stream — flush # any buffered reasoning tail first so it isn't stranded behind the tool event. _flush_reasoning_buffer() event_type = None name = None preview = None args = None if len(cb_args) >= 4: event_type, name, preview, args = cb_args[:4] elif len(cb_args) == 3: name, preview, args = cb_args event_type = 'tool.started' elif len(cb_args) == 2: event_type, name = cb_args elif len(cb_args) == 1: name = cb_args[0] event_type = 'tool.started' if event_type in ('reasoning.available', '_thinking'): reason_text = preview if event_type == 'reasoning.available' else name if reason_text: reason_delta = str(reason_text) # Older tool-progress paths can mirror the same visible # progress text already emitted through stream_delta_callback. # Suppress those echoes like the dedicated reasoning callback. if _is_visible_output_echo(reason_delta): return # Accumulate into the current message's segment (#3587) _reasoning_segments[_current_reasoning_idx] = ( _reasoning_segments.get(_current_reasoning_idx, '') + reason_delta ) _reasoning_segment_indexes.setdefault( _current_reasoning_idx, _CompactEchoIndex() ).append(reason_delta) # Mirror full concatenation to shared dict (#1361 §A) # Lock-free GIL-atomic mirror — see STREAMS_LOCK contract in on_token. if stream_id in STREAM_REASONING_TEXT: STREAM_REASONING_TEXT[stream_id] += reason_delta _stream_reasoning_index.append(reason_delta) put('reasoning', {'text': reason_delta}) _metering_reasoning_deltas[0] += 1 meter().record_reasoning(stream_id, _metering_reasoning_deltas[0]) _emit_metering() return # (#3587) Advance reasoning index at tool-call boundaries. # on_interim_assistant is suppressed for contentless tool-call # messages (run_agent.py:3834), so the index never advances # there. The first tool.started event after reasoning indicates # a new assistant message boundary. if not _tool_boundary_advanced and _current_reasoning_idx in _reasoning_segments: _current_reasoning_idx += 1 _tool_boundary_advanced = True args_snap = _tool_args_snapshot(args) # Modern Hermes Agent builds can call both tool_progress_callback # and the structured tool_start/tool_complete callbacks for the # same tool. Prefer the structured path when it is supported so # the browser receives one tid-tagged tool card per real call. if event_type in (None, 'tool.started') and 'tool_start_callback' in _agent_params: return if event_type in (None, 'tool.started'): _live_tool_calls.append({ 'name': name, 'args': args if isinstance(args, dict) else {}, }) # Mirror to shared dict so cancel_stream() can persist it (#1361 §B) # Lock-free GIL-atomic mirror — see STREAMS_LOCK contract in on_token. if stream_id in STREAM_LIVE_TOOL_CALLS: STREAM_LIVE_TOOL_CALLS[stream_id].append({ 'name': name, 'args': args if isinstance(args, dict) else {}, 'done': False, }) put('tool', { 'event_type': event_type or 'tool.started', 'name': name, 'preview': preview, 'args': args_snap, }) _tool_stats = meter().get_stats(stream_id) _tool_stats['session_id'] = session_id _tool_stats['usage'] = _live_usage_snapshot() put('metering', _tool_stats) # Fallback: poll for pending approval in case notify_cb wasn't # registered (e.g. older approval module without gateway support). try: from api.route_approvals import ( _gateway_queues as _approval_gateway_queues, _lock as _approval_lock, _pending as _approval_pending, reconcile_gateway_pending_mirror_locked as _reconcile_gateway_pending_mirror_locked, ) from tools.approval import has_blocking_approval as _has_blocking_approval if _has_blocking_approval(session_id): p = None with _approval_lock: p, pending_count, _changed = _reconcile_gateway_pending_mirror_locked(session_id) if p: p = {**p, "pending_count": pending_count} if p: put('approval', p) except ImportError: pass return if event_type == 'tool.completed' and 'tool_complete_callback' in _agent_params: return if event_type == 'tool.completed': for live_tc in reversed(_live_tool_calls): if live_tc.get('done'): continue if not name or live_tc.get('name') == name: live_tc['done'] = True live_tc['duration'] = cb_kwargs.get('duration') live_tc['is_error'] = bool(cb_kwargs.get('is_error', False)) break # Mirror done state to shared dict (#1361 §B) if stream_id in STREAM_LIVE_TOOL_CALLS: for shared_tc in reversed(STREAM_LIVE_TOOL_CALLS[stream_id]): if shared_tc.get('done'): continue if not name or shared_tc.get('name') == name: shared_tc['done'] = True shared_tc['duration'] = cb_kwargs.get('duration') shared_tc['is_error'] = bool(cb_kwargs.get('is_error', False)) break # Signal the checkpoint thread that new work has completed (Issue #765). # Each completed tool call is a meaningful unit of progress worth persisting. _checkpoint_activity[0] += 1 put('tool_complete', { 'event_type': event_type, 'name': name, 'preview': preview, 'args': args_snap, 'duration': cb_kwargs.get('duration'), 'is_error': bool(cb_kwargs.get('is_error', False)), }) # Mirror the todo tool's in-memory state into a # dedicated SSE event so the Todos panel can update # in real-time without waiting for the turn to # settle. The helper guards on name=='todo', sends # the full snapshot (idempotent under SSE replay) # and swallows internal errors so emission never # breaks tool delivery. Prefer the structured # `result` kwarg from modern Hermes builds; fall # back to the truncated `preview` only when the # callback was invoked without one (older builds). # # Graceful degradation on old builds: `preview` is a # truncated snippet, so its JSON is usually unparseable. # parse_todo_tool_result() then returns None and NO # todo_state event is emitted — live panel updates are # silently unavailable on pre-`result` builds. This is # intended: the panel still hydrates via cold-load on the # next session GET; it just won't update mid-stream. emit_todo_state( put, name=name, function_result=( cb_kwargs.get('result') if cb_kwargs.get('result') is not None else preview ), session_id=session_id, stream_id=stream_id, ) _tool_stats = meter().get_stats(stream_id) _tool_stats['session_id'] = session_id _tool_stats['usage'] = _live_usage_snapshot() put('metering', _tool_stats) return def on_tool_start(tool_call_id, name, args): try: _record_live_tool_start(tool_call_id, name, args) if tool_call_id and tool_call_id not in _live_tool_event_start_ids: _live_tool_event_start_ids.add(tool_call_id) _live_tool_calls.append({ 'name': name, 'args': args if isinstance(args, dict) else {}, 'tid': tool_call_id, }) # Mirror to shared dict so cancel_stream() can persist it (#1361 §B) # Lock-free GIL-atomic mirror — see STREAMS_LOCK contract in on_token. if stream_id in STREAM_LIVE_TOOL_CALLS: STREAM_LIVE_TOOL_CALLS[stream_id].append({ 'name': name, 'args': args if isinstance(args, dict) else {}, 'done': False, 'tid': tool_call_id, }) put('tool', { 'event_type': 'tool.started', 'name': name, 'preview': None, 'args': _tool_args_snapshot(args), 'tid': tool_call_id, }) _tool_stats = meter().get_stats(stream_id) _tool_stats['session_id'] = session_id _tool_stats['usage'] = _live_usage_snapshot() put('metering', _tool_stats) except Exception: logger.debug('Failed to update live prompt estimate on tool start', exc_info=True) def on_tool_complete(tool_call_id, name, args, function_result): try: _record_live_tool_complete(tool_call_id, name, function_result) if tool_call_id and tool_call_id not in _live_tool_event_complete_ids: _live_tool_event_complete_ids.add(tool_call_id) result_snippet = _tool_result_snippet(function_result) for live_tc in reversed(_live_tool_calls): if live_tc.get('done'): continue if live_tc.get('tid') == tool_call_id or (not live_tc.get('tid') and live_tc.get('name') == name): live_tc['done'] = True live_tc['snippet'] = result_snippet break if stream_id in STREAM_LIVE_TOOL_CALLS: for shared_tc in reversed(STREAM_LIVE_TOOL_CALLS[stream_id]): if shared_tc.get('done'): continue if shared_tc.get('tid') == tool_call_id or (not shared_tc.get('tid') and shared_tc.get('name') == name): shared_tc['done'] = True shared_tc['snippet'] = result_snippet break _checkpoint_activity[0] += 1 put('tool_complete', { 'event_type': 'tool.completed', 'name': name, 'preview': result_snippet, 'args': _tool_args_snapshot(args), 'tid': tool_call_id, 'is_error': False, }) # Mirror the todo tool's in-memory state into # a dedicated SSE event so the Todos panel can # update in real-time without waiting for the # turn to settle. See the legacy path above # for the contract; the helper handles the # name guard, payload shape, and swallow-all # error policy. emit_todo_state( put, name=name, function_result=function_result, session_id=session_id, stream_id=stream_id, ) _tool_stats = meter().get_stats(stream_id) _tool_stats['session_id'] = session_id _tool_stats['usage'] = _live_usage_snapshot() put('metering', _tool_stats) except Exception: logger.debug('Failed to update live prompt estimate on tool completion', exc_info=True) _AIAgent = _get_ai_agent() if _AIAgent is None: raise ImportError(_aiagent_import_error_detail()) # Initialize SessionDB so session_search works in WebUI sessions _state_db_path = (Path(_profile_home) / "state.db") if _profile_home else None _session_db = _build_session_db_for_stream(_state_db_path) # #5979: publish catalog provenance from the durable disk cache when # memory is cold, so the custom-proxy resolver below sees the # endpoint-advertised model ids (non-blocking, disk-only, never # live-rebuilds). Both the warm and the resolve read profile-keyed # config (cache path + source fingerprint via get_active_profile_name), # but this streaming worker is a separate thread that does NOT inherit # the HTTP handler's request-profile TLS — without binding it, a cold # send from a NAMED profile would resolve against the DEFAULT profile's # config and route to the wrong provider/base_url. Bind the captured # owning-session profile across warm + resolve so both see the right # profile (no-op for the default/root profile). from api import profiles as profiles_api # #5979: treat this send as a deliberate pick ONLY when the persisted # explicit-pick signature matches the CURRENT model+provider routing # context. Storing/comparing a signature (not a bare bool) means a # later model/provider change via /api/chat/start, /api/session/update, # normalization, or provider repair automatically invalidates a stale # pick — so a #433 first-party leftover is never wrongly preserved on # a cold catalog. Only affects the cold custom-proxy branch; warm # endpoint-advertised provenance always wins over this flag. from api.models import model_explicit_pick_signature as _mk_sig _picked_sig = getattr(s, "model_explicit_pick_signature", None) # Compare against the session's persisted model+provider — the exact # fields /api/chat/start stamped the signature from (it persists the # resolved model+provider onto the session before dispatch). Falls # back to the worker's model/provider_context if the session fields # are unset. A mismatch (any later model/provider change) yields a # different signature → treated as NOT a deliberate pick. _sig_model = getattr(s, "model", None) or model _sig_provider = getattr(s, "model_provider", None) or provider_context _current_sig = _mk_sig(_sig_model, _sig_provider) _explicitly_picked = bool(_picked_sig) and _picked_sig == _current_sig # Resolve the endpoint AND the credential inside ONE profile scope so # they always come from the same profile-owned snapshot. Previously # only resolve_model_provider() ran under the scope; the runtime-key # and custom-provider resolution below ran after it closed, so a # detached worker for a NAMED profile paired that profile's endpoint # with the DEFAULT profile's API key (finding #3). No-op for the # default/root profile. with profiles_api.profile_scope_for_detached_worker( _resolved_profile_name, "model + credential resolution", logger_override=logger ): warm_models_catalog_provenance_if_cold() resolved_model, resolved_provider, resolved_base_url = resolve_model_provider( model_with_provider_context(model, provider_context), explicitly_picked=_explicitly_picked, ) configured_base_url = resolved_base_url # Resolve API key via Hermes runtime provider (matches gateway behaviour). # Pass the resolved provider so non-default providers get their own credentials. resolved_api_key = None # Default to an empty runtime dict so the constructor-routing # bundle below stays buildable when resolution raises. _rt = {} try: from api.oauth import resolve_runtime_provider_with_anthropic_env_lock from hermes_cli.runtime_provider import resolve_runtime_provider _rt = resolve_runtime_provider_with_anthropic_env_lock( resolve_runtime_provider, requested=resolved_provider, target_model=resolved_model, ) resolved_api_key = _rt.get("api_key") if not resolved_provider: resolved_provider = _rt.get("provider") resolved_base_url = _runtime_preferred_base_url( _rt, resolved_provider, configured_base_url ) except Exception as _e: print(f"[webui] WARNING: resolve_runtime_provider failed: {_e}", flush=True) # Named custom providers (custom:slug) may not be resolvable by # hermes_cli.runtime_provider directly. Fall back to config.yaml # custom_providers[] so WebUI can pass explicit creds/base_url. # Preserve the pre-canonicalization identity so image routing can # still select the exact custom_providers entry after the rewrite # to "custom" below. # # Resolve ONE complete constructor-routing bundle here and let # every downstream field (including the agent-cache signature) # read from it. Resolving only provider/key/base_url left the # runtime-owned credential_pool / api_mode / ACP command+args # beside a replaced custom endpoint, so the agent was built with # a mixed authority: the list row's URL and key, but the ambient # provider's transport, wire protocol and credential source. _session_requested_provider = resolved_provider _runtime_bundle = _resolve_runtime_connection_bundle( resolved_provider, resolved_api_key, resolved_base_url, _rt, profile_name=_resolved_profile_name, custom_provider_lookup=_session_requested_provider, ) # Stop HERE on a terminal route verdict — before the agent # kwargs, before _AIAgent(), before the agent-cache write. A # named custom: that resolved no complete # (api_key, base_url) pair is not "fail closed" at the # constructor: AIAgent honours an explicit pair only when BOTH # are truthy and otherwise calls _routed_client_kwargs(), which # re-resolves a provider and can land on the ambient endpoint or # the init-time fallback chain. Raising sends this turn to the # outer handler, which classifies it via # _custom_provider_route_classification() and emits a controlled # provider / missing-credential error — and, because we never # reach SESSION_AGENT_CACHE, the cache is never poisoned with an # agent built on an unroutable bundle that later turns would # silently reuse. raise_for_custom_provider_route(_runtime_bundle) resolved_provider = _runtime_bundle['provider'] resolved_api_key = _runtime_bundle['api_key'] resolved_base_url = _runtime_bundle['base_url'] # Read per-profile config at call time (not module-level snapshot). # The streaming worker is a detached thread that does NOT inherit the # per-request thread-local profile context, so the ambient # get_config() would resolve the process-global (default) profile and # leak the wrong profile's toolsets / prefill / fallback config into # this run (issue #3294). Read the SESSION's own profile home # explicitly so toolsets and context match the profile the session # actually runs under. from api.config import get_config_for_profile_home as _get_config_for_home try: _cfg = _get_config_for_home(_profile_home) except Exception: from api.config import get_config as _get_config _cfg = _get_config() _prefill_context = _load_webui_prefill_context(_cfg) _prefill_messages = _prefill_messages_with_webui_context(_prefill_context, _cfg) _prefill_messages = _normalize_prefill_messages_before_user_turn(_prefill_messages) _main_request_overrides = _main_model_request_overrides( _cfg, effective_model=resolved_model, effective_provider=resolved_provider, ) put('context_status', { 'session_id': session_id, 'prefill': _public_prefill_context_status(_prefill_context), }) # Per-profile toolsets — use _resolve_cli_toolsets() so MCP # server toolsets are included, matching native CLI behaviour. from api.config import _resolve_cli_toolsets _toolsets = _resolve_cli_toolsets(_cfg) # Per-session toolset override (#493): if the session has # enabled_toolsets set, use that instead of the global config. try: from api.models import Session, SESSION_DIR _session_path = SESSION_DIR / f"{session_id}.json" if _session_path.exists(): _session_meta = Session.load_metadata_only(session_id) # load_metadata_only returns a Session INSTANCE, not a dict. # The previous .get('enabled_toolsets') raised AttributeError # which was swallowed by the bare except below — the entire # per-session toolset override silently no-op'd. Use # getattr() to read the attribute correctly. # (Opus pre-release advisor finding for v0.50.257.) _override = getattr(_session_meta, 'enabled_toolsets', None) if _session_meta else None if _override: _toolsets = _override except Exception as _ts_err: print(f"[webui] WARNING: failed to read per-session toolsets for {session_id}: {_ts_err}", flush=True) # Fallback model chain from profile config (e.g. for rate-limit or # provider recovery). Match Hermes CLI/gateway semantics: # fallback_providers entries are tried first, then legacy # fallback_model entries are appended unless they duplicate an # earlier provider/model/base_url route. def _fallback_entries(_raw): if isinstance(_raw, dict): _items = [_raw] elif isinstance(_raw, list): _items = _raw else: return [] _entries = [] for _entry in _items: if not isinstance(_entry, dict): continue _provider = str(_entry.get('provider') or '').strip() _model = str(_entry.get('model') or '').strip() if not _provider or not _model: continue _entries.append({ 'model': _model, 'provider': _provider, 'base_url': _entry.get('base_url'), 'api_key': _entry.get('api_key'), 'key_env': _entry.get('key_env'), }) return _entries _fallback_chain = [] _fallback_seen = set() _fallback_resolved = None for _fallback_key in ('fallback_providers', 'fallback_model'): for _fb_entry in _fallback_entries(_cfg.get(_fallback_key)): _identity = ( str(_fb_entry.get('provider') or '').strip().lower(), str(_fb_entry.get('model') or '').strip().lower(), str(_fb_entry.get('base_url') or '').strip().rstrip('/').lower(), ) if _identity in _fallback_seen: continue _fallback_seen.add(_identity) _fallback_chain.append(_fb_entry) _fallback_resolved = _fallback_chain or None # Build kwargs defensively — guard newer params so the WebUI # degrades gracefully when run against an older hermes-agent build. # (fixes: TypeError: AIAgent.__init__() got an unexpected keyword # argument 'credential_pool' — issue #772) import inspect as _inspect _agent_params = set(_inspect.signature(_AIAgent.__init__).parameters) # CLI-parity max-iteration budget: read config.yaml's # agent.max_turns and pass it to AIAgent when supported. Without # this WebUI-created agents silently use AIAgent's constructor # default (90), so long browser-originated tasks hit the # "maximum number of tool-calling iterations" summary path even # after the operator raises Hermes' global turn budget. _max_iterations_cfg = None try: _raw_max_iterations = None _agent_cfg_for_iterations = _cfg.get('agent', {}) if isinstance(_cfg, dict) else {} if isinstance(_agent_cfg_for_iterations, dict): _raw_max_iterations = _agent_cfg_for_iterations.get('max_turns') if _raw_max_iterations is None and isinstance(_cfg, dict): # Back-compat for older Hermes config files that used a # root-level max_turns key. _raw_max_iterations = _cfg.get('max_turns') if _raw_max_iterations is not None: _parsed_max_iterations = int(_raw_max_iterations) if _parsed_max_iterations > 0: _max_iterations_cfg = _parsed_max_iterations except Exception: _max_iterations_cfg = None # CLI-parity max output cap: read config.yaml's max_tokens and pass # it to AIAgent when supported. Without this WebUI-created agents use # provider-native output ceilings (e.g. Claude via OpenRouter can # request 64k), which may turn an otherwise usable fallback into a # 402 "more credits / fewer max_tokens" failure. _max_tokens_cfg = None try: _raw_max_tokens = _cfg.get('max_tokens') if _raw_max_tokens is None: _agent_cfg_for_tokens = _cfg.get('agent', {}) if isinstance(_agent_cfg_for_tokens, dict): _raw_max_tokens = _agent_cfg_for_tokens.get('max_tokens') if _raw_max_tokens is not None: _parsed_max_tokens = int(_raw_max_tokens) if _parsed_max_tokens > 0: _max_tokens_cfg = _parsed_max_tokens except Exception: _max_tokens_cfg = None # CLI-parity reasoning effort: read agent.reasoning_effort from the # active profile's config.yaml (the same key the CLI writes via # `/reasoning `) and hand the parsed dict to AIAgent. When # the key is absent or invalid, pass None → agent uses its default. try: _effort_cfg = _cfg.get('agent', {}) if isinstance(_cfg, dict) else {} _effort_raw = _effort_cfg.get('reasoning_effort') if isinstance(_effort_cfg, dict) else None _effort = coerce_reasoning_effort_for_model( _effort_raw, resolved_model, provider_id=resolved_provider, base_url=resolved_base_url, ) _reasoning_config = parse_reasoning_effort(_effort) except Exception: _reasoning_config = None _agent_kwargs = dict( model=resolved_model, provider=resolved_provider, base_url=resolved_base_url, api_key=resolved_api_key, # Identify browser-originated sessions as WebUI so Hermes Agent # does not inject CLI-specific terminal/output guidance. platform='webui', quiet_mode=True, enabled_toolsets=_toolsets, fallback_model=_fallback_resolved, session_id=session_id, session_db=_session_db, prefill_messages=_prefill_messages, stream_delta_callback=on_token, reasoning_callback=on_reasoning, tool_progress_callback=on_tool, clarify_callback=( lambda question, choices: _clarify_callback_impl( question, choices, session_id, cancel_event, put ) ), ) # reasoning_config has been an AIAgent param for several releases, # but guard defensively to avoid TypeError on an older agent build. if 'reasoning_config' in _agent_params and _reasoning_config is not None: _agent_kwargs['reasoning_config'] = _reasoning_config if 'prefill_messages' not in _agent_params: _agent_kwargs.pop('prefill_messages', None) if 'interim_assistant_callback' in _agent_params: _agent_kwargs['interim_assistant_callback'] = on_interim_assistant if 'tool_start_callback' in _agent_params: _agent_kwargs['tool_start_callback'] = on_tool_start if 'tool_complete_callback' in _agent_params: _agent_kwargs['tool_complete_callback'] = on_tool_complete if 'status_callback' in _agent_params: _agent_kwargs['status_callback'] = _agent_status_callback if 'max_iterations' in _agent_params and _max_iterations_cfg is not None: _agent_kwargs['max_iterations'] = _max_iterations_cfg if 'max_tokens' in _agent_params and _max_tokens_cfg is not None: _agent_kwargs['max_tokens'] = _max_tokens_cfg if 'request_overrides' in _agent_params and _main_request_overrides: _agent_kwargs['request_overrides'] = _main_request_overrides # Params added in newer hermes-agent — skip if not supported. # Read from the resolved bundle, NOT from _rt: a custom-provider # override clears these, and taking them straight off the runtime # provider would re-introduce the authority it replaced. if 'api_mode' in _agent_params: _agent_kwargs['api_mode'] = _runtime_bundle['api_mode'] if 'acp_command' in _agent_params: _agent_kwargs['acp_command'] = _runtime_bundle['acp_command'] if 'acp_args' in _agent_params: _agent_kwargs['acp_args'] = _runtime_bundle['acp_args'] if 'credential_pool' in _agent_params: _agent_kwargs['credential_pool'] = _runtime_bundle['credential_pool'] # Pin Honcho memory sessions to the stable WebUI session ID. # Without this, 'per-session' Honcho strategy creates a new Honcho # session on every streaming request because HonchoSessionManager is # re-instantiated fresh each turn (#855). if 'gateway_session_key' in _agent_params: _agent_kwargs['gateway_session_key'] = session_id # ── Agent cache: reuse across messages in the same session ── # Mirrors gateway _agent_cache. Keeps _user_turn_count alive so # injectionFrequency: "first-turn" actually suppresses after turn 1. _cache_new_agent = False if ephemeral: agent = _AIAgent(**_agent_kwargs) logger.debug('[webui] Created ephemeral agent for session %s', session_id) else: from api.config import SESSION_AGENT_CACHE, SESSION_AGENT_CACHE_LOCK # Signature fields come from the SAME bundle the constructor # received, so a cleared/overridden field always mints a new # agent instead of reusing one built on the prior authority. _agent_sig = _compute_agent_cache_signature( resolved_model, resolved_api_key, resolved_base_url, resolved_provider, _runtime_bundle, max_iterations_cfg=_max_iterations_cfg, max_tokens_cfg=_max_tokens_cfg, fallback_resolved=_fallback_resolved, toolsets=_toolsets, reasoning_config=_reasoning_config, main_request_overrides=_main_request_overrides, prefill_context=_prefill_context, profile_home=_profile_home, safe_profile_runtime_env=_safe_profile_runtime_env, ) agent = None _identity_mismatch_entry = None with SESSION_AGENT_CACHE_LOCK: _cached = SESSION_AGENT_CACHE.get(session_id) if _cached and _cached[1] == _agent_sig: _cached_agent = _cached[0] if _cached_agent_matches_session(_cached_agent, session_id): agent = _cached_agent SESSION_AGENT_CACHE.move_to_end(session_id) # LRU: mark as recently used logger.debug('[webui] Reusing cached agent for session %s', session_id) else: _identity_mismatch_entry = SESSION_AGENT_CACHE.pop(session_id, None) logger.warning( '[webui] Evicted cached agent with mismatched session identity: cache_key=%s agent_session_id=%s', session_id, _cached_agent_session_identity(_cached_agent), ) if _identity_mismatch_entry is not None: try: _close_cached_agent_entry_at_session_boundary(session_id, _identity_mismatch_entry) except Exception: logger.debug("Failed to close identity-mismatched cached agent for session %s", session_id, exc_info=True) if agent is not None: # Refresh volatile runtime credentials selected from provider # pools without discarding cross-turn agent/provider state. if not _refresh_cached_agent_runtime(agent, _agent_kwargs): logger.warning( '[webui] Cached agent runtime could not be safely refreshed; rebuilding agent for session %s', session_id, ) _stale_runtime_entry = None with SESSION_AGENT_CACHE_LOCK: _stale_runtime_entry = SESSION_AGENT_CACHE.pop(session_id, None) if _stale_runtime_entry is not None: try: _close_cached_agent_entry_at_session_boundary(session_id, _stale_runtime_entry) except Exception: logger.debug("Failed to close stale-runtime cached agent for session %s", session_id, exc_info=True) agent = None if agent is not None: # Refresh per-turn callbacks — these close over request-scoped # objects (put queue, cancel_event) that are new each request. agent.stream_delta_callback = _agent_kwargs.get('stream_delta_callback') agent.tool_progress_callback = _agent_kwargs.get('tool_progress_callback') if hasattr(agent, 'tool_start_callback'): agent.tool_start_callback = _agent_kwargs.get('tool_start_callback') if hasattr(agent, 'tool_complete_callback'): agent.tool_complete_callback = _agent_kwargs.get('tool_complete_callback') if hasattr(agent, 'status_callback'): agent.status_callback = _agent_kwargs.get('status_callback') if hasattr(agent, 'interim_assistant_callback'): agent.interim_assistant_callback = _agent_kwargs.get('interim_assistant_callback') if hasattr(agent, 'reasoning_callback'): agent.reasoning_callback = _agent_kwargs.get('reasoning_callback') if hasattr(agent, 'clarify_callback'): agent.clarify_callback = _agent_kwargs.get('clarify_callback') if 'prefill_messages' in _agent_kwargs and hasattr(agent, 'prefill_messages'): agent.prefill_messages = list(_agent_kwargs.get('prefill_messages') or []) if _session_db is not None: # Prefer reusing a still-open SessionDB on the cached # agent. Closing it mid-turn breaks background # subagents that hold a reference to the same object # (delegate_tool copies parent._session_db by ref) — # they then fail with # 'NoneType' object has no attribute 'execute'. # When the existing handle is already closed/missing, # adopt the fresh per-request SessionDB (and close the # dead one) so we still avoid the EMFILE FD-leak from # PR #1421. _session_db = _adopt_session_db_for_cached_agent( agent, _session_db ) agent._session_db = _session_db if hasattr(agent, '_api_call_count'): agent._api_call_count = 0 # Reset interrupt state from a prior cancel so the reused # agent does not think it is still interrupted. if hasattr(agent, '_interrupted'): agent._interrupted = False if hasattr(agent, '_interrupt_message'): agent._interrupt_message = None else: agent = _AIAgent(**_agent_kwargs) _cache_new_agent = True if not _register_agent_if_current(agent, _agent_sig if _cache_new_agent else None): with _agent_lock: _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) put('cancel', _cancel_event_payload('Cancelled by user')) return # Cache publication is already fenced with Stop above. Keep LRU # eviction and any memory-provider close outside the stream lock. if _cache_new_agent: _evicted_items = [] # Snapshot the set of session_ids with a LIVE agent worker # BEFORE taking SESSION_AGENT_CACHE_LOCK, so LRU eviction never # closes an agent mid-run AND we never nest ACTIVE_RUNS_LOCK # inside SESSION_AGENT_CACHE_LOCK (avoids any lock-ordering # deadlock). A cancel/reconnect can drop STREAMS while the # worker is still unwinding or blocked in a provider call, so # ACTIVE_RUNS (worker lifecycle) is the authoritative liveness # signal, not STREAMS. (#3536 review round 2) _active_sids = set() try: from api.config import ACTIVE_RUNS, ACTIVE_RUNS_LOCK with ACTIVE_RUNS_LOCK: for _entry in (ACTIVE_RUNS or {}).values(): _sid = (_entry or {}).get("session_id") if _sid: _active_sids.add(_sid) except Exception: _active_sids = set() with SESSION_AGENT_CACHE_LOCK: from api.config import SESSION_AGENT_CACHE_MAX # Evict the oldest INACTIVE entries first. Walk LRU order # (front = oldest); skip any session with a live run. If # every over-cap entry is active, leave the cache # temporarily above cap rather than close a live worker's # agent — a later insertion/finalization trims it once the # run ends. while len(SESSION_AGENT_CACHE) > SESSION_AGENT_CACHE_MAX: _evictable_sid = None for _sid in list(SESSION_AGENT_CACHE.keys()): if _sid not in _active_sids: _evictable_sid = _sid break if _evictable_sid is None: break # all over-cap entries are active; defer evicted_entry = SESSION_AGENT_CACHE.pop(_evictable_sid) _evicted_items.append((_evictable_sid, evicted_entry)) # Commit and close evicted agents outside the cache lock so # concurrent cache users are not blocked by provider I/O. for _evicted_sid, _evicted_entry in _evicted_items: try: _evicted_agent = _evicted_entry[0] if isinstance(_evicted_entry, tuple) else None _close_evicted_agent_at_session_boundary(_evicted_sid, _evicted_agent) except Exception: logger.debug("Failed to close evicted agent for session %s", _evicted_sid, exc_info=True) logger.debug('[webui] Evicted LRU agent from cache: %s', _evicted_sid) logger.debug('[webui] Created new agent for session %s', session_id) # Prepend workspace context so the agent always knows which directory # to use for file operations, regardless of session age or AGENTS.md defaults. workspace_ctx = _workspace_context_prefix(str(s.workspace)) # #6672: interpolate the session-CREATION workspace (immutable), never # the live s.workspace, into the system prompt. s.workspace changes on # every mid-session workspace switch in the WebUI header; mutating the # system prompt would rewrite msg[0] and invalidate the LLM prefix # cache (APC/Radix Tree) for the whole 50k+ token transcript. Active # switches still reach the model via the [Workspace::v1: ...] tag on # the current user turn (workspace_ctx above), which lives in msg[-1]. _session_workspace_frozen = getattr(s, 'created_workspace', None) or str(s.workspace) workspace_system_msg = ( f"Active workspace at session start: {_session_workspace_frozen}\n" "Every user message is prefixed with [Workspace::v1: /absolute/path] indicating the " "workspace the user has selected in the web UI at the time they sent that message. " "This tag is the single authoritative source of the active workspace and updates " "with every message. It overrides any prior workspace mentioned in this system " "prompt, memory, or conversation history. Always use the value from the most recent " "[Workspace::v1: ...] tag as your default working directory for ALL file operations: " "write_file, read_file, search_files, terminal workdir, and patch. " "Never fall back to a hardcoded path when this tag is present." ) # Resolve personality prompt from config.yaml agent.personalities # (matches hermes-agent CLI behavior — passes via ephemeral_system_prompt) _personality_prompt = None _pname = getattr(s, 'personality', None) if _pname: _agent_cfg = _cfg.get('agent', {}) _personalities = _agent_cfg.get('personalities', {}) if isinstance(_personalities, dict) and _pname in _personalities: _pval = _personalities[_pname] if isinstance(_pval, dict): _parts = [_pval.get('system_prompt', '') or _pval.get('prompt', '')] if _pval.get('tone'): _parts.append(f'Tone: {_pval["tone"]}') if _pval.get('style'): _parts.append(f'Style: {_pval["style"]}') _personality_prompt = '\n'.join(p for p in _parts if p) else: _personality_prompt = str(_pval) # Pass WebUI-only runtime guidance via ephemeral_system_prompt # (agent's own mechanism). This preserves any selected personality # while making long tool runs emit real user-visible interim text # through interim_assistant_callback instead of frontend guesses. agent.ephemeral_system_prompt = _webui_ephemeral_system_prompt( _personality_prompt, surface_context={ 'source': 'webui', 'session_id': session_id, 'profile': getattr(s, 'profile', None), # #6672: frozen session-creation workspace — see # workspace_system_msg above. Live workspace switches stay out # of msg[0] so LLM prefix caches are not invalidated. 'workspace': _session_workspace_frozen, }, config_data=_cfg, ) _pending_started_at = getattr(s, 'pending_started_at', None) meter().set_pending_started_at(stream_id, _pending_started_at) # Normal chat-start sets pending_started_at before spawning this thread; # fallback to now only for recovered/legacy flows where that marker is absent # or has been zeroed out (e.g. via a buggy migration / manual file edit). # Truthy-check covers None, missing-attr, and 0 uniformly. _turn_started_at = _pending_started_at if _pending_started_at else time.time() _external_state_snapshot = get_state_db_session_messages( session_id, profile=getattr(s, 'profile', None), with_revision=True, ) def _context_and_revision_from_state_snapshot(state_snapshot): reconciled_snapshot = reconciled_state_db_messages_for_session( s, prefer_context=True, state_messages=state_snapshot, with_revision=True, ) if not isinstance(reconciled_snapshot, StateDBSessionMessagesSnapshot): raise TypeError( "state.db context reconciliation did not return a revision snapshot" ) context_messages = _new_turn_context_from_messages( reconciled_snapshot.messages, msg_text, ) return ( _deduplicate_context_messages(context_messages), reconciled_snapshot.revision, ) def _refresh_context_and_revision_from_state_db(): fresh_state_snapshot = get_state_db_session_messages( session_id, profile=getattr(s, 'profile', None), with_revision=True, ) return _context_and_revision_from_state_snapshot(fresh_state_snapshot) _previous_messages = list( reconciled_state_db_messages_for_session( s, state_messages=_external_state_snapshot, ) or [] ) # Keep the owner/context projection distinct from the display projection. # Settlement and stale-result attribution require the exact pre-run owner # context even when state.db reconciliation changes the visible transcript. _previous_owner_context_messages = list( reconciled_state_db_messages_for_session( s, prefer_context=True, state_messages=_external_state_snapshot, ) or [] ) _previous_owner_context_messages = _deduplicate_context_messages( _previous_owner_context_messages ) ( _previous_context_messages, _conversation_history_revision, ) = _context_and_revision_from_state_snapshot( _external_state_snapshot, ) # Dedup before feeding to agent — merge_session_messages_append_only # can produce duplicates when context_messages and state.db share # messages with different timestamps. _pre_compression_count = getattr( getattr(agent, 'context_compressor', None), 'compression_count', 0, ) # ── Periodic checkpoint during streaming (Issue #765) ── # The agent works on an internal copy of s.messages during run_conversation() # so we cannot watch s.messages for growth. Instead, on_tool() increments # _checkpoint_activity[0] each time a tool call completes — that is the real # signal that progress has been made worth persisting. # # What gets saved on each checkpoint: # - s.pending_user_message (already written before run starts) # - s.pending_started_at / s.active_stream_id (turn bookkeeping) # On a server restart the UI will see a session with a pending message and no # response — better than a silent loss of the entire conversation turn. # The final s.save() at task completion handles the full session update + index. # (_checkpoint_stop is pre-initialised at the top of the outer try.) # (_checkpoint_activity is already initialised before on_tool().) def _periodic_checkpoint(): last_saved_activity = 0 last_fingerprint = None last_write_at = 0.0 while not _checkpoint_stop.wait(15): try: cur = _checkpoint_activity[0] if cur > last_saved_activity: with _agent_lock: fingerprint = _streaming_checkpoint_fingerprint(s) # A completed tool call is the trigger, but not # proof that anything the checkpoint persists # actually changed. Rewriting a multi-megabyte # sidecar to re-persist identical bytes stalls # every concurrent HTTP request behind the GIL, # so only write when the persisted state moved. # Fail closed: an unreadable fingerprint (None) # always writes, and a periodic refresh keeps # updated_at from going stale on a long turn. now = time.time() stale = (now - last_write_at) >= _CHECKPOINT_IDLE_REFRESH_SECONDS if ( fingerprint is None or fingerprint != last_fingerprint or stale ): _save_streaming_checkpoint(s) last_fingerprint = fingerprint last_write_at = now last_saved_activity = cur except Exception as e: logger.debug("Periodic checkpoint save failed: %s", e) _checkpoint_stop = threading.Event() # Persist the user message BEFORE streaming starts so it's durable even if # the server crashes before the first checkpoint fires (every 15s). with _agent_lock: s.save(touch_updated_at=True, skip_index=False) _ckpt_thread = threading.Thread( target=_periodic_checkpoint, daemon=True, name=f"ckpt-{session_id[:8]}", ) _ckpt_thread.start() _pending_async_acceptances = [] _process_notifications = _drain_webui_process_notifications( session_id, pending_async_acceptances=_pending_async_acceptances, ) _agent_msg_text = msg_text if _process_notifications: _agent_msg_text = "\n\n".join([*_process_notifications, msg_text]).strip() user_message = _build_native_multimodal_message(workspace_ctx, _agent_msg_text, attachments, workspace, cfg=_cfg, active_provider=(resolved_provider or ""), active_model=(resolved_model or ""), requested_provider=(_session_requested_provider or ""), profile=(getattr(s, "profile", None) or Path(_profile_home))) _persistent_state_before = _persistent_state_snapshot(_profile_home) with _agent_lock: _persist_user_timestamp = getattr(s, 'pending_started_at', None) _register_pending_user_timestamp_identity( agent.run_conversation, s, _persist_user_timestamp ) _run_conversation_kwargs = _build_run_conversation_kwargs( agent.run_conversation, user_message=user_message, system_message=workspace_system_msg, conversation_history=_sanitize_messages_for_agent( _previous_context_messages, cfg=_cfg, effective_model=resolved_model, effective_provider=resolved_provider, effective_base_url=resolved_base_url, requested_provider=(_session_requested_provider or ""), ), conversation_history_revision=_conversation_history_revision, task_id=session_id, persist_user_message=msg_text, persist_user_timestamp=_persist_user_timestamp, ) # Only pass moa_config when a /moa override is actually active, so a # normal send never trips a TypeError on an older hermes-agent whose # run_conversation() predates the moa_config kwarg. if moa_config is not None: _run_conversation_kwargs["moa_config"] = moa_config # Finalize durable delegation claims at the current-turn acceptance # boundary: immediately before invoking the agent with the message # that contains their notifications. A failed ACK is removed from # this turn and requeued so retry cannot create a duplicate prompt. _rejected_async_notifications = _accept_pending_async_delegations( _pending_async_acceptances, session_id=session_id, ) if _rejected_async_notifications: for _notification in _rejected_async_notifications: try: _process_notifications.remove(_notification) except ValueError: pass _agent_msg_text = msg_text if _process_notifications: _agent_msg_text = "\n\n".join( [*_process_notifications, msg_text] ).strip() user_message = _build_native_multimodal_message( workspace_ctx, _agent_msg_text, attachments, workspace, cfg=_cfg, active_provider=(resolved_provider or ""), active_model=(resolved_model or ""), requested_provider=(_session_requested_provider or ""), # Legacy fallback wraps the home STRING in Path: string # profiles are logical ids only (round-5 grammar gate), # explicit homes must arrive as Path values. profile=(getattr(s, "profile", None) or Path(_profile_home)), ) _run_conversation_kwargs["user_message"] = user_message # This exact WebUI-composed input is identity evidence for the # Agent-indexed current row. Keep identity['text'] as the clean # submitted prompt for display. Assignment follows notification # rejection/rebuild so rejected text cannot claim a returned row. _active_turn_identity['trusted_agent_input_text'] = _agent_msg_text _result_partial_pre_call_context = list(_previous_context_messages) if not _agent_can_invoke(agent): with _agent_lock: _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) put('cancel', _cancel_event_payload('Cancelled by user')) return result = agent.run_conversation(**_run_conversation_kwargs) _remember_pending_steer_result(result) _active_turn_identity = _resolve_active_turn_authority( _active_turn_identity, result=result, agent=agent, ) # #4729: the run is done — flush any reasoning tail still in the coalescing # buffer (the agent never calls reasoning_callback(None), and a turn can end on # reasoning with no trailing token/tool boundary to trigger a flush) so the last # sub-100ms window reaches the live Thinking view before the terminal done event. _flush_reasoning_buffer() if cancel_event.is_set(): if _checkpoint_stop is not None: _checkpoint_stop.set() if _ckpt_thread is not None: _ckpt_thread.join(timeout=15) if ephemeral: with _agent_lock: _finalize_cancelled_turn(s, ephemeral=True, stream_id=stream_id) else: with _agent_lock: _finalize_cancelled_turn(s, ephemeral=False, stream_id=stream_id) try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return # ── Ephemeral mode (/btw): deliver answer, skip persistence, cleanup ── if ephemeral: _answer = '' for _m in reversed(result.get('messages') or []): if isinstance(_m, dict) and _m.get('role') == 'assistant': _answer = str(_m.get('content', '')) break if (_answer.strip() and not result.get('error') and not _agent_result_terminal_failure(result) and not getattr(agent, '_last_error', None) and not _captured_terminal_error[0]): _observe_runtime_model() # /btw is intentionally non-persistent, but its terminal SSE # payload is still public output. Project the ephemeral # session before enqueueing it so raw Agent ``api_content`` or # provenance aliases cannot cross the wire. _ephemeral_session = _ephemeral_session_payload( session_id, result.get('messages', []) ) put('done', { 'session': _ephemeral_session, 'usage': {'input_tokens': 0, 'output_tokens': 0}, 'ephemeral': True, 'answer': _answer, }) if _checkpoint_stop is not None: _checkpoint_stop.set() try: import pathlib pathlib.Path(s.path).unlink(missing_ok=True) except Exception: pass return # skip all normal persistence for ephemeral sessions if _checkpoint_stop is not None: _checkpoint_stop.set() if _ckpt_thread is not None: _ckpt_thread.join(timeout=15) if cancel_event.is_set(): with _agent_lock: _finalize_cancelled_turn(s, ephemeral=False, stream_id=stream_id) try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return _writeback_timings = [] _writeback_started = time.perf_counter() with _agent_lock: if not ephemeral and not _stream_writeback_is_current(s, stream_id): if _stream_writeback_can_supersede_recovery_marker(s, msg_text): logger.info( "Superseding stale recovery marker for session %s stream %s", getattr(s, 'session_id', session_id), stream_id, ) else: logger.info( "Skipping stale stream writeback for session %s stream %s; active_stream_id=%s", getattr(s, 'session_id', session_id), stream_id, getattr(s, 'active_stream_id', None), ) return with _stream_writeback_stage(_writeback_timings, "merge_result"): _tool_limit_reached = _agent_result_tool_limit_reached(result) _result_messages = result.get('messages') if _result_messages is None: _result_messages = _previous_context_messages _result_messages = _drop_synthetic_max_iteration_summary_requests( _result_messages, enabled=_tool_limit_reached, ) # #5494 — parity with hermes-agent's handle_max_iterations() return # value. When the agent produced no usable summary assistant # message but result['final_response'] carries a graceful fallback # string, inject it as a final assistant turn so the user sees # closure text instead of a bare tool_limit_reached error. Apply # the synthesis to result['messages'] AND _result_messages so the # downstream _all_result_messages checks (silent-failure detection # at api/streaming.py:_assistant_reply_added_after_current_turn) # see the fallback too. `finalize_turn` in the agent always returns # messages as a list, but we write back unconditionally so the # contract is "if we built a result-messages list, the silent-failure # classifier reads the augmented version." if _tool_limit_reached: _result_messages = _maybe_inject_max_iteration_summary_fallback( _result_messages, result ) if isinstance(result, dict): result = {**result, 'messages': _result_messages} if cancel_event.is_set(): _finalize_cancelled_turn(s, ephemeral=False, stream_id=stream_id) try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return _result_messages = _settle_result_messages( s, _previous_messages, _previous_owner_context_messages, _result_messages, msg_text, _turn_pending_source, _active_turn_identity, ) # Strip XML tool-call blocks from assistant message content. # DeepSeek and some other providers emit ... # in the raw response text; this must be removed before the content is # saved to the session and displayed in the chat bubble. (#702) for _m in s.messages: if isinstance(_m, dict) and _m.get('role') == 'assistant': _raw_content = _m.get('content') if isinstance(_raw_content, str): _cleaned = _strip_xml_tool_calls(_raw_content) if _cleaned != _raw_content: _m['content'] = _cleaned elif isinstance(_raw_content, list): for _part in _raw_content: if isinstance(_part, dict) and isinstance(_part.get('text'), str): _part['text'] = _strip_xml_tool_calls(_part['text']) # ── Handle context compression side effects ── # If compression fired inside run_conversation, the agent may have # rotated its session_id. Detect and fix the mismatch before any # terminal-failure return so snapshot preservation, continuation # registration, and subsequent error persistence all target the # continuation session instead of the stale parent. # # Lock migration: when session_id rotates, alias both old and new # IDs to the *same* _agent_lock. Keeping the old alias ensures a # late old-ID request cannot create a second mutation lock while # the streaming holder or an earlier waiter is still active. The # weak registry reclaims both aliases after the final strong # reference to the Lock is released. _compression_origin_session_id = session_id _compression_continuation_session_id = None _agent_sid = getattr(agent, 'session_id', None) _compressed = False if _agent_sid and _agent_sid != session_id: old_sid = session_id new_sid = _agent_sid _compression_origin_session_id = old_sid _compression_continuation_session_id = new_sid s.session_id = new_sid # Carry profile identity across the compression boundary. # Without this, s.profile stays None on the continuation # session. On the next request, _run_agent_streaming calls # get_hermes_home_for_profile(getattr(s, 'profile', None)) # which falls back to the default profile's HERMES_HOME. # Memory writes then land in the wrong profile's MEMORY.md. # Stamping here also ensures s.save() persists a non-null # profile field to the continuation session's JSON file, # covering the case where the session is later evicted from # SESSIONS and reconstructed from disk via Session.load(). if not s.profile and _resolved_profile_name: s.profile = _resolved_profile_name logger.info( "Stamped profile=%r on continuation session %s after compression", _resolved_profile_name, new_sid, ) # Preserve the original session file so the full pre-compression # history survives even when summarisation fails. The previous # implementation renamed old_sid.json → new_sid.json, which # destroyed the only persistent copy of the uncompressed history # before the new (possibly summary-only) session had been saved. # If the LLM summariser also failed, the user was left with zero # recoverable messages. (#2223) # --- # Archive the old session: write its current state to disk so # the full conversation history survives even when context # compression removes messages from the model's context. Skip # the write when the file already contains up-to-date data # (i.e. it was just saved by a checkpoint). _preserve_pre_compression_snapshot(s, old_sid) # The continuation is the live/tip session, not another archived # snapshot. If the in-memory object was itself loaded from a # pre-compression snapshot (possible on repeated compression chains # or stale-cache repair paths), _preserve_pre_compression_snapshot() # intentionally restores that old flag; clear it before saving the # new continuation so sidebar/discoverability code does not hide the # session that owns the completed turn. s.pre_compression_snapshot = False # Always link the continuation session to its immediate predecessor # (the preserved snapshot). This OVERRIDES any prior # parent_session_id because the new continuation IS the next link # in the chain: traversal walks new → old → old.parent → ... root. # Stage-353 Opus SHOULD-FIX: previous `if not s.parent_session_id` # guard skipped this stamp on fork-of-fork compressions, so a # subsequent traversal from the new continuation would jump # over the just-preserved snapshot back to the original fork # parent, losing access to the recoverable history in old_sid.json. s.parent_session_id = old_sid with LOCK: cached_old_session = SESSIONS.pop(old_sid, None) if cached_old_session is not None and cached_old_session is not s: cached_old_sid = str(getattr(cached_old_session, 'session_id', '') or '') if cached_old_sid == str(old_sid): SESSIONS[old_sid] = cached_old_session else: logger.warning( "compression cache migration skipped stale object: old_sid=%s new_sid=%s cached_session_id=%s", old_sid, new_sid, cached_old_sid or None, ) SESSIONS[new_sid] = s SESSIONS.move_to_end(new_sid) _evict_sessions_over_cap() # #4765: safe LRU eviction (never active/unsaved) # Migrate the per-session lock by aliasing new_sid to the # held _agent_lock reference directly. Keep old_sid aliased # too until the weak registry can reclaim both safely after # all old-ID holders and waiters release the lock. _alias_session_agent_lock(old_sid, new_sid, _agent_lock) # Migrate cached agent to the new session ID so the turn # count survives context compression. from api.config import SESSION_AGENT_CACHE, SESSION_AGENT_CACHE_LOCK _skipped_agent_migration_entry = None with SESSION_AGENT_CACHE_LOCK: _cached_entry = SESSION_AGENT_CACHE.pop(old_sid, None) if _cached_entry: _cached_agent = _cached_entry[0] if _cached_agent_matches_session(_cached_agent, new_sid): SESSION_AGENT_CACHE[new_sid] = _cached_entry else: _skipped_agent_migration_entry = _cached_entry logger.warning( '[webui] Skipped cached agent migration with mismatched session identity: old_sid=%s new_sid=%s agent_session_id=%s', old_sid, new_sid, _cached_agent_session_identity(_cached_agent), ) if _skipped_agent_migration_entry is not None: try: _close_cached_agent_entry_at_session_boundary(old_sid, _skipped_agent_migration_entry) except Exception: logger.debug("Failed to close skipped compression-migration cached agent for session %s", old_sid, exc_info=True) _compressed = True # ── Detect silent agent failure (no assistant reply produced) ── # When the agent catches an auth/network error internally it may return # an empty final_response without raising — the stream would end with # a done event containing zero assistant messages, leaving the user with # no feedback. Emit an apperror so the client shows an inline error. # Keep the current-turn assistant detection aligned with the # display-merge logic. A compacted or replayed result payload # is not always a simple append-only suffix, so use the # workspace-aware helper from this branch while still # preserving the pre-turn length for downstream self-heal # checks introduced on master. _all_result_messages = _drop_synthetic_control_messages(result.get('messages') or []) _prev_len = len(_previous_context_messages) _assistant_added = _assistant_reply_added_after_current_turn( _all_result_messages, _previous_context_messages, msg_text, ) _last_err = getattr(agent, '_last_error', None) or result.get('error') or '' # #5940: if the Agent aborted on a non-retryable provider error # (captured from its lifecycle status_callback) but left no error on # the result/agent, use the captured message so the classifier can # surface the real cause (model_not_found / auth) instead of the # misleading no_response "silent rate limit, try again" fallback. _captured_terminal_failure = bool(_captured_terminal_error[0]) if not _last_err and _captured_terminal_failure: _last_err = _captured_terminal_error[0] _classification = _classify_provider_error( str(_last_err) if _last_err else '', _last_err, silent_failure=not bool(_last_err), result=result, ) _is_quota = _classification['type'] == 'quota_exhausted' _is_auth = _classification['type'] == 'auth_mismatch' _drop_replayed_assistant = ( _captured_terminal_failure or _agent_result_terminal_failure(result) or bool(getattr(agent, '_last_error', None)) or ('error' in result and result.get('error') is not None) ) _saved_transcript_lacks_final_answer = _merged_transcript_lacks_final_assistant_answer( _previous_messages, _previous_owner_context_messages, _all_result_messages, msg_text, source=getattr(s, 'pending_user_source', None) or 'webui', drop_replayed_assistant=_drop_replayed_assistant, active_turn_identity=_active_turn_identity, ) if ( not _all_result_messages and _current_turn_already_has_visible_assistant_answer( _align_current_turn_display( _previous_messages, _previous_owner_context_messages, _active_turn_identity, )[0], active_turn_identity=_active_turn_identity, ) ): _saved_transcript_lacks_final_answer = False if not _assistant_added and not _saved_transcript_lacks_final_answer: _assistant_added = True _is_agent_result_terminal = _agent_result_terminal_failure(result) _terminal_failure = ( _captured_terminal_failure or _is_agent_result_terminal or ( _saved_transcript_lacks_final_answer and _classification['type'] not in {'cancelled', 'interrupted'} ) ) _result_status = str(result.get('status') or result.get('state') or '').strip().lower() _soft_partial_terminal_failure = ( _is_agent_result_terminal and (_result_status == 'partial' or bool(result.get('partial'))) and _result_status not in {'failed', 'error', 'compression_exhausted'} and not result.get('failed') and not result.get('compression_exhausted') and not _tool_limit_reached and not _last_err ) if ( _terminal_failure and (_soft_partial_terminal_failure or _tool_limit_reached) and _classification['type'] == 'no_response' and not _saved_transcript_lacks_final_answer ): _terminal_failure = False if _terminal_failure: _assistant_added = False elif _tool_limit_reached and not _session_lacks_final_assistant_answer(s.messages): _mark_latest_assistant_tool_limit_status(s.messages) # _token_sent tracks whether on_token() was called (any streamed text) if _terminal_failure or (not _assistant_added and not _token_sent): if cancel_event.is_set(): _finalize_cancelled_turn(s, ephemeral=ephemeral, stream_id=stream_id) if not ephemeral: try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return _err_str = str(_last_err) if _last_err else '' if _is_quota: _err_label = _classification['label'] _err_type = _classification['type'] _err_hint = _classification['hint'] elif _is_auth and not _self_healed: # ── Credential self-heal on 401 (#1401) ── # Before emitting the error, try re-reading credentials # and retrying once with a fresh agent. _heal_result = None _heal_stale_classification = None # Bind the session's profile so the self-heal re-resolve # AND the custom-provider override below read one # profile-owned snapshot (finding #3): otherwise a named # profile's endpoint pairs with the default profile's key. from api import profiles as _profiles_api with _profiles_api.profile_scope_for_detached_worker( _resolved_profile_name, "credential self-heal", logger_override=logger ): _heal_rt = _attempt_credential_self_heal( resolved_provider or '', session_id, _agent_lock, target_model=resolved_model, ) if _heal_rt is not None: logger.info('[webui] self-heal: retrying stream after credential refresh') # Rebuild runtime variables from the refreshed resolve _rt = _heal_rt resolved_api_key = _heal_rt.get('api_key') if not resolved_provider: resolved_provider = _heal_rt.get('provider') resolved_base_url = _runtime_preferred_base_url( _heal_rt, resolved_provider, configured_base_url, session_requested_provider=_session_requested_provider, ) # Preserve the session's original pre-canonicalization # provider identity (captured at first resolve) so a # named custom:slug retry can still select its exact # vision-capability entry. Only initialize when empty # (e.g. the provider was first discovered on this heal). if not _session_requested_provider: _session_requested_provider = resolved_provider # Rebuild the COMPLETE bundle, not just the three # connection fields: the self-heal re-resolve can # return a different runtime authority, and a # custom-provider override must clear its # credential_pool / api_mode / ACP fields here too. _runtime_bundle = _resolve_runtime_connection_bundle( resolved_provider, resolved_api_key, resolved_base_url, _heal_rt, profile_name=_resolved_profile_name, custom_provider_lookup=_session_requested_provider, ) # The re-resolve can turn a previously routable named # route terminal (the record's key_cmd stopped # minting, the pool drained, the row was edited # mid-turn). Abandon the retry BEFORE _AIAgent and # before the SESSION_AGENT_CACHE write below: healing # a 401 by handing the constructor an incomplete pair # is exactly the re-routing this verdict exists to # stop, and caching that agent would carry it into # every later turn. Raising reaches the outer handler, # which emits the controlled provider / # missing-credential error instead of the 401. raise_for_custom_provider_route(_runtime_bundle) resolved_provider = _runtime_bundle['provider'] resolved_api_key = _runtime_bundle['api_key'] resolved_base_url = _runtime_bundle['base_url'] # Rebuild agent kwargs and create a fresh agent _agent_kwargs['api_key'] = resolved_api_key _agent_kwargs['base_url'] = resolved_base_url _agent_kwargs['model'] = resolved_model _agent_kwargs['provider'] = resolved_provider _replace_session_db_in_kwargs(_agent_kwargs, _state_db_path) if 'api_mode' in _agent_params: _agent_kwargs['api_mode'] = _runtime_bundle['api_mode'] if 'acp_command' in _agent_params: _agent_kwargs['acp_command'] = _runtime_bundle['acp_command'] if 'acp_args' in _agent_params: _agent_kwargs['acp_args'] = _runtime_bundle['acp_args'] if 'credential_pool' in _agent_params: _agent_kwargs['credential_pool'] = _runtime_bundle['credential_pool'] agent = _AIAgent(**_agent_kwargs) _agent_sig = _compute_agent_cache_signature( resolved_model, resolved_api_key, resolved_base_url, resolved_provider, _runtime_bundle, max_iterations_cfg=_max_iterations_cfg, max_tokens_cfg=_max_tokens_cfg, fallback_resolved=_fallback_resolved, toolsets=_toolsets, reasoning_config=_reasoning_config, main_request_overrides=_main_request_overrides, prefill_context=_prefill_context, profile_home=_profile_home, safe_profile_runtime_env=_safe_profile_runtime_env, ) if not _register_agent_if_current(agent, _agent_sig): # Returned-error settlement already owns _agent_lock. _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) put('cancel', _cancel_event_payload('Cancelled by user')) return # Retry the conversation once with fresh credentials _self_healed = True _token_sent = False try: ( _heal_context_messages, _heal_conversation_history_revision, ) = _refresh_context_and_revision_from_state_db() _heal_persist_user_timestamp = getattr( s, 'pending_started_at', None ) # The returned-error path already owns _agent_lock. _register_pending_user_timestamp_identity( agent.run_conversation, s, _heal_persist_user_timestamp, ) _heal_kwargs = _build_run_conversation_kwargs( agent.run_conversation, user_message=user_message, system_message=workspace_system_msg, conversation_history=_sanitize_messages_for_agent( _heal_context_messages, cfg=_cfg, effective_model=resolved_model, effective_provider=resolved_provider, effective_base_url=resolved_base_url, requested_provider=(_session_requested_provider or ""), ), conversation_history_revision=( _heal_conversation_history_revision ), task_id=session_id, persist_user_message=msg_text, persist_user_timestamp=_heal_persist_user_timestamp, ) if moa_config is not None: _heal_kwargs["moa_config"] = moa_config _result_partial_pre_call_context = list( _heal_context_messages ) if not _agent_can_invoke(agent): # Returned-error settlement already owns the lock. _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) put('cancel', _cancel_event_payload('Cancelled by user')) return _heal_result = agent.run_conversation(**_heal_kwargs) _remember_pending_steer_result(_heal_result) _active_turn_identity = _resolve_active_turn_authority( _active_turn_identity, result=_heal_result, agent=agent, ) if _result_reports_compression_snapshot_stale(_heal_result): _heal_stale_classification = _classify_provider_error( '', result=_heal_result, ) result = _heal_result _heal_ok = _self_heal_result_succeeded( _heal_result, _heal_context_messages, _active_turn_identity, msg_text, ) except Exception as _retry_exc: logger.warning( '[webui] self-heal: retry also failed: %s', _retry_exc, ) _heal_ok = False if _heal_ok and _heal_result is not None: # Retry succeeded — replace result and skip error result = _heal_result # Fall through past the error-emission block; # the post-result persistence code below will # process ``result`` normally. We jump past # the ``put('apperror', ...)`` + ``return`` by # NOT entering the ``if not _assistant_added`` # guard again — but we are already inside it. # Solution: set _assistant_added so the guard # evaluates False on next conceptual pass. # Since we're in a flat block, directly run the # post-result merge logic here. _result_messages = result.get('messages') if _result_messages is None: _result_messages = _previous_context_messages _result_messages = _drop_synthetic_max_iteration_summary_requests( _result_messages, enabled=_agent_result_tool_limit_reached(result), ) _result_messages = _settle_result_messages( s, _previous_messages, _previous_owner_context_messages, _result_messages, msg_text, _turn_pending_source, _active_turn_identity, ) # normal post-result persistence path by # leaving _assistant_added truthy (set below). _assistant_added = True # prevent re-entering guard if not _assistant_added: # Self-heal didn't apply or retry failed — emit error. if _heal_stale_classification is not None: _err_label = _heal_stale_classification['label'] _err_type = _heal_stale_classification['type'] _err_hint = _heal_stale_classification['hint'] else: _err_label = 'Authentication failed' _err_type = 'auth_mismatch' _err_hint = ( 'The selected model may not be supported by your configured provider or ' 'your API key is invalid. Run `hermes model` in your terminal to ' 'update credentials, then restart the WebUI.' ) elif _is_auth: _err_label = 'Authentication failed' _err_type = 'auth_mismatch' _err_hint = ( 'The selected model may not be supported by your configured provider or ' 'your API key is invalid. Run `hermes model` in your terminal to ' 'update credentials, then restart the WebUI.' ) elif _tool_limit_reached: _err_label = 'Tool iteration limit reached' _err_type = 'tool_limit_reached' _err_hint = ( 'The agent reached its configured tool iteration limit before producing ' 'a final answer. Start a narrower follow-up or increase agent.max_turns.' ) _err_str = ( 'The agent reached its configured tool iteration limit before producing ' 'a final answer.' ) else: _err_label = _classification['label'] _err_type = _classification['type'] _err_hint = _classification['hint'] # Skip error emission if credential self-heal succeeded # (#1401) — _assistant_added is set True on successful retry. if _assistant_added: # Self-heal succeeded: messages are already merged into s, # fall through to normal post-result persistence below. pass else: _result_public_error = _err_str or f'{_err_label}.' if _err_type == 'compression_snapshot_stale': _result_public_error = ( 'The conversation changed while context compression was being prepared.' ) _error_payload = _provider_error_payload( _result_public_error, _err_type, _err_hint, ) if _turn_pending_source == 'process_wakeup': _recorded_pause = record_process_wakeup_provider_unavailable_pause( s, classification=_err_type, model=_turn_route_model, provider=_turn_route_provider, ) # Disclose the suppression so the silence reads as # intentional, not a stuck agent (#3929 UX) — but ONLY # when a pause was actually recorded (credential-pool # exhaustion), never for a rate-limit/other wakeup # failure that doesn't pause. Keep the SSE payload hint # in sync with the persisted bubble. if _recorded_pause: _err_hint = ( (_err_hint + ' ' if _err_hint else '') + 'Automatic retries for this conversation are paused until you ' + 'send a message, switch the model/provider, or fix the credentials.' ) _error_payload['hint'] = _err_hint _turn_duration = _terminal_turn_duration(s) _materialize_pending_user_turn_before_error( s, active_turn_identity=_active_turn_identity, ) s.active_stream_id = None s.pending_user_message = None s.pending_attachments = [] s.pending_started_at = None s.pending_user_source = None try: _snapshot_and_append_partial_on_error( s, stream_id, active_turn_identity=_active_turn_identity, ) _append_result_partial_on_error( s, result, _result_partial_pre_call_context, msg_text, active_turn_identity=_active_turn_identity, ) except Exception: logger.debug("Failed to snapshot partials on error for %s", stream_id, exc_info=True) _error_content = ( f'**{_err_label}:** {_error_payload.get("message") or _err_label}' + (f'\n\n*{_err_hint}*' if _err_hint else '') ) _error_message = { 'role': 'assistant', 'content': _error_content, 'timestamp': int(time.time()), '_error': True, } if _turn_duration is not None: _error_message['_turnDuration'] = _turn_duration if _err_type == 'compression_exhausted': _recovery = stamp_compression_exhausted_recovery( s, message=_error_payload.get('message') or _err_label, details=_error_payload.get('details') or '', ) _error_message['_compressionRecovery'] = _recovery _error_payload['compression_recovery'] = _recovery _error_payload['recommended_recovery_action'] = _recovery.get('recommended_action') if _error_payload.get('details'): _error_message['provider_details'] = _error_payload['details'] if _err_type == 'cancelled': _error_message['provider_details_label'] = 'Cancellation details' elif _err_type == 'interrupted': _error_message['provider_details_label'] = 'Interruption details' elif _err_type == 'tool_limit_reached': _error_message['provider_details_label'] = 'Terminal state details' s.messages.append(_error_message) try: s.save() except Exception: pass _error_payload['session'] = redact_session_data( _session_payload_with_full_messages(s, tool_calls=s.tool_calls) ) _error_payload['session_id'] = s.session_id _error_payload['old_session_id'] = _compression_origin_session_id if _compression_continuation_session_id is not None: _error_payload['new_session_id'] = _compression_continuation_session_id _error_payload['continuation_session_id'] = _compression_continuation_session_id if _err_type == 'tool_limit_reached': _error_payload['terminal_state'] = 'tool_limit_reached' _error_payload['terminal_reason'] = 'max_iterations' put('apperror', _error_payload) # Legacy #373 source tests and clients look for the # no_response type; #1765 keeps that type but improves # the catch-all label, hint, and provider details. return # apperror already closes the stream on the client side _observe_runtime_model() # ── Handle context compression side effects ── # Also detect compression via the result dict or compressor state if not _compressed: _compressor = getattr(agent, 'context_compressor', None) if _compressor and getattr(_compressor, 'compression_count', 0) > _pre_compression_count: _compressed = True # Notify the frontend that compression happened if _compressed: s.context_messages = _prune_context_tool_results_after_compression( agent, s.context_messages, ) s.post_compression_context_tokens_estimate = _estimate_post_compression_context_tokens( agent, s.context_messages, workspace_system_msg, ) visible_after = visible_messages_for_anchor(s.messages, auto_compression=True) # Find the LAST [CONTEXT COMPACTION] marker in s.messages # and count visible messages before it. This is the correct # anchor — it points to the compression boundary regardless # of how many turns have been added since the boundary was # established. Using len(visible_before)-1 is fragile when # _previous_messages doesn't include markers or when extra # messages accumulate between compression and the done event. _last_marker_raw_idx = None for _mi, _m in enumerate(s.messages): if _is_context_compression_marker(_m): _last_marker_raw_idx = _mi if _last_marker_raw_idx is not None: _visible_before_marker = visible_messages_for_anchor( s.messages[:_last_marker_raw_idx], auto_compression=True, ) s.compression_anchor_visible_idx = max(0, len(_visible_before_marker) - 1) logger.info( '[ANCHOR-MARKER] session=%s marker_raw=%d vis_before=%d anchor=%d', getattr(s, 'session_id', '?'), _last_marker_raw_idx, len(_visible_before_marker), s.compression_anchor_visible_idx, ) else: # Fallback: use pre-turn display messages visible_before = visible_messages_for_anchor( _previous_messages, auto_compression=True, ) if visible_before: s.compression_anchor_visible_idx = max(0, len(visible_before) - 1) elif visible_after: s.compression_anchor_visible_idx = 0 else: s.compression_anchor_visible_idx = None logger.info( '[ANCHOR-FALLBACK] session=%s vis_before=%d anchor=%d', getattr(s, 'session_id', '?'), len(visible_before) if visible_before else 0, s.compression_anchor_visible_idx if s.compression_anchor_visible_idx is not None else -1, ) # Pick anchor_msg for _compression_anchor_message_key _anchor_vis_idx = s.compression_anchor_visible_idx if _anchor_vis_idx is not None and visible_after and _anchor_vis_idx < len(visible_after): anchor_msg = visible_after[_anchor_vis_idx] elif visible_after: anchor_msg = visible_after[-1] else: anchor_msg = None s.compression_anchor_message_key = ( _compression_anchor_message_key(anchor_msg) if anchor_msg else None ) s.compression_anchor_summary = _compact_summary_text( _compression_summary_from_messages(s.messages) or _compression_summary_from_messages(s.context_messages) ) if _compression_continuation_session_id is None: _compression_continuation_session_id = s.session_id put('compressed', { 'session_id': _compression_origin_session_id, 'old_session_id': _compression_origin_session_id, 'new_session_id': _compression_continuation_session_id, 'continuation_session_id': _compression_continuation_session_id, 'message': 'Compression finished', 'usage': _live_usage_snapshot(), }) # Stamp 'timestamp' on any messages that don't have one yet, # preserving transcript order across compacted/reconciled batches. _stamp_missing_message_timestamps(s.messages) # Only auto-generate title when still default; preserves user renames if s.title == 'Untitled' or s.title == 'New Chat' or not s.title: s.title = title_from(s.messages, s.title) _bg_title_inputs = _background_title_generation_inputs(s) # Read token/cost usage from the agent object (if available). # Per-turn overwrite (#1857): replace cumulative session totals with the # agent's most recent values, which already represent the current turn's # full prompt+completion (input_tokens are the entire context, not delta). # Defensive: only overwrite when the agent reports non-zero / non-None # values. A rebuilt-from-cache-miss agent (post-restart, post-LRU-eviction) # starts at zero; without this guard, the next turn would zero out the # persisted disk total before any new tokens were spent. Per Opus advisor # on stage-320: prevents restart-induced regression of session usage data. input_tokens = getattr(agent, 'session_prompt_tokens', 0) or 0 output_tokens = getattr(agent, 'session_completion_tokens', 0) or 0 estimated_cost = getattr(agent, 'session_estimated_cost_usd', None) cache_read_tokens = getattr(agent, 'session_cache_read_tokens', 0) or 0 cache_write_tokens = getattr(agent, 'session_cache_write_tokens', 0) or 0 prev_input_tokens = getattr(s, 'input_tokens', 0) or 0 prev_cache_read_tokens = getattr(s, 'cache_read_tokens', 0) or 0 turn_input_tokens = max(0, input_tokens - prev_input_tokens) turn_cache_read_tokens = max(0, cache_read_tokens - prev_cache_read_tokens) # Per-turn percent is computed server-side from persisted session # counters so the message label uses the same denominator as the # final usage payload even if the browser missed an intermediate event. cache_hit_percent = prompt_cache_hit_percent(cache_read_tokens, input_tokens) turn_cache_hit_percent = prompt_cache_hit_percent(turn_cache_read_tokens, turn_input_tokens) if input_tokens > 0: s.input_tokens = input_tokens if output_tokens > 0: s.output_tokens = output_tokens if estimated_cost is not None: s.estimated_cost = estimated_cost if cache_read_tokens > 0: s.cache_read_tokens = cache_read_tokens if cache_write_tokens > 0: s.cache_write_tokens = cache_write_tokens # Persist tool-call summaries even when the final message history only # kept bare tool rows and omitted explicit assistant tool_call IDs. tool_calls = _extract_tool_calls_from_messages( s.messages, live_tool_calls=_live_tool_calls, ) s.tool_calls = tool_calls s.active_stream_id = None s.pending_user_message = None s.pending_attachments = [] s.pending_started_at = None s.pending_user_source = None # Tag the matching user message with attachment filenames for display on reload # Only tag a user message whose content relates to this turn's text # (msg_text is the full message including the [Attached files: ...] suffix) if attachments: display_attachments = [_attachment_name(a) for a in attachments if _attachment_name(a)] for m in reversed(s.messages): if _active_turn_token_matches(m, _active_turn_identity): m['attachments'] = display_attachments break # Persist reasoning trace in the session so it survives reload. # Must run BEFORE s.save() — otherwise the mutation lives only in # memory until the next turn's save, and the last-turn thinking card # is lost when the user reloads immediately after a response. # # #3455/#3599: split inline thinking blocks out of the saved # assistant content into m['reasoning'] (server-side twin of the JS # _splitThinkFromContent). Inline-thinking providers (e.g. MiniMax-M3) # otherwise leave the thinking trace in m['content'], bloating the # persisted session file 30-50% and bypassing the thinking card. The # #3587: use per-message segments so intermediate assistant turns # (before tool calls) each receive their own reasoning trace rather # than all reasoning being written only to the last assistant message. # Scope the walk to this turn's newly-appended assistant messages # to prevent cross-turn reasoning clobber (multi-turn off-by-N). if s.messages: _prev_asst = sum( 1 for m in (_previous_messages or []) if isinstance(m, dict) and m.get('role') == 'assistant' ) _asst_count = 0 for _rm in s.messages: if not (isinstance(_rm, dict) and _rm.get('role') == 'assistant'): continue _turn_idx = _asst_count _asst_count += 1 if _turn_idx < _prev_asst: continue # prior-turn message — never touch its reasoning _seg_reasoning = _reasoning_segments.get(_turn_idx - _prev_asst, '') _existing_reasoning = _seg_reasoning or _rm.get('reasoning') or '' _content = _rm.get('content') if isinstance(_content, str) and _content: _new_content, _merged_reasoning = _split_thinking_from_content( _content, _existing_reasoning ) _rm['content'] = _new_content if _merged_reasoning: _rm['reasoning'] = _merged_reasoning elif _existing_reasoning: _rm['reasoning'] = _existing_reasoning try: _turn_duration_seconds = max(0.0, time.time() - float(_turn_started_at)) except Exception: _turn_duration_seconds = 0.0 _turn_tps = None if output_tokens and _turn_duration_seconds > 0: _turn_tps = round(float(output_tokens) / _turn_duration_seconds, 1) _gateway_routing = _extract_gateway_routing_metadata( agent, result, requested_model=resolved_model or model, requested_provider=resolved_provider, ) # #6068: the served model must be read AFTER agent.run — the agent # mutates agent.model when a fallback fires, so the pre-run # resolved_model would mis-attribute exactly the turns where # attribution matters most. # The configured selection is not proof that it served this turn. _observed_model = getattr(agent, 'model', None) _used_model = _observed_model.strip() if isinstance(_observed_model, str) else None if _gateway_routing: s.gateway_routing = _gateway_routing _history = list(getattr(s, 'gateway_routing_history', None) or []) _history.append(_gateway_routing) s.gateway_routing_history = _history[-50:] if s.messages: for _dm in reversed(s.messages): if isinstance(_dm, dict) and _dm.get('role') == 'assistant': _dm['_turnDuration'] = round(_turn_duration_seconds, 3) if _turn_tps is not None: _dm['_turnTps'] = _turn_tps if _gateway_routing: _dm['_gatewayRouting'] = _gateway_routing _ttft_ms = meter().get_ttft_ms(stream_id) if _ttft_ms is not None: _dm['_firstTokenMs'] = _ttft_ms if _used_model: _dm['_usedModel'] = _used_model break # Persist context window data on the session so the context-ring # indicator survives a page reload (#1318). Must run BEFORE # s.save() for the same reason as the reasoning trace above. # The fields are captured into the SSE usage payload below; this # block writes them to the session itself so GET /api/session # returns them on reload instead of falling back to 0. _cc_for_save = getattr(agent, 'context_compressor', None) # Initialized before the compressor block so the #3256/#3263 # threshold-rescale below is safe even when there is no # compressor (fresh agent / interrupted stream): _skip_cc_cl # stays False and _cc_cl stays 0, so the rescale is a no-op. _skip_cc_cl = False _cc_cl = 0 if _cc_for_save: _cc_cl = getattr(_cc_for_save, 'context_length', 0) or 0 # Same guard as routes._resolve_context_length_for_session_model: # the agent-side context_compressor was constructed with the # global model.context_length applied to EVERY model. If the # session's model isn't model.default, that value is a stale # cap (e.g. 232K) that would clobber the real 1M metadata # on every stream end. In that case skip the compressor # value and let the fallback resolver below recompute. # #4618: broaden the stale-compressor guard the same way the # live-usage snapshot does. The OLD test only skipped the # compressor value when it equalled the config cap EXACTLY # (a non-default model carrying the global cap). But a # compressor can hold a DIFFERENT model's window after an # in-place model switch (e.g. opus-4.5's 168k lingering on an # opus-4.8 1M session) — that value != the config cap, so the # old guard let it persist to s.context_length and the SSE # payload, snapping the indicator back to 168k at turn-end. # Resolve the real per-model window via the SAME helper the # live path + hydration use and skip the compressor value # whenever the real window differs, honoring the #4248 # acceptance gate (never let a low-confidence 256k fallback # clobber a larger cached window). _skip_cc_cl = False try: from api.routes import ( _context_length_lookup_inputs_for_model as _cli_cc, _should_accept_session_context_length_refresh as _accept_cc, ) from agent.model_metadata import get_model_context_length as _g_cc _sess_model_cc = str(getattr(agent, 'model', resolved_model or '') or '').strip() if _sess_model_cc and _cc_cl > 0: _lk_cc = _cli_cc( _sess_model_cc, resolved_provider or '', base_url=getattr(agent, 'base_url', '') or resolved_base_url or '', api_key=getattr(agent, 'api_key', '') or resolved_api_key or '', cfg=_cfg if isinstance(_cfg, dict) else {}, ) try: _real_cc = _g_cc( _sess_model_cc, _lk_cc.base_url, api_key=_lk_cc.api_key, config_context_length=_lk_cc.config_context_length, provider=_lk_cc.provider or resolved_provider or '', custom_providers=_lk_cc.custom_providers, ) or 0 except TypeError: _real_cc = _g_cc(_sess_model_cc, _lk_cc.base_url) or 0 if _real_cc and _real_cc != _cc_cl and _accept_cc(_cc_cl, _real_cc): _skip_cc_cl = True except Exception: pass if not _skip_cc_cl: s.context_length = _cc_cl s.threshold_tokens = getattr(_cc_for_save, 'threshold_tokens', 0) or 0 s.last_prompt_tokens = getattr(_cc_for_save, 'last_prompt_tokens', 0) or 0 # Fallback: if the compressor didn't report a context_length # (fresh agent, interrupted stream, or compressor missing the # attribute), resolve it from the model's static metadata so # the indicator can still show a meaningful percentage. # Sourced from PR #1344 (@jasonjcwu) — extracted to a focused # follow-up after PR #1344 was closed as superseded by #1341. # # #1896: pass config_context_length, provider, and # custom_providers so explicit config overrides win over the # 256K default fallback. Without these, users on 1M-context # models who set `model.context_length: 1048576` (or rely on # a `custom_providers` per-model override) get a 256K # window in the persisted session and the SSE payload — # which then trips LCM auto-compress at ~25% of the wrong # value, cascading into 429 floods. # # #3256/#3263: ALSO run this fallback when _skip_cc_cl is true # (non-default model whose compressor carried the stale global # cap). Without this, a session that already had a stale 232K # context_length persisted keeps it forever — skipping the # compressor write removes the re-clobber but never recomputes # the real per-model window. Recompute and overwrite in that case. if (not getattr(s, 'context_length', 0)) or _skip_cc_cl: try: from agent.model_metadata import get_model_context_length from api.routes import _context_length_lookup_inputs_for_model _cfg_base_url = getattr(agent, 'base_url', '') or resolved_base_url or '' _ctx_lookup = _context_length_lookup_inputs_for_model( getattr(agent, 'model', resolved_model or '') or '', resolved_provider, base_url=_cfg_base_url, cfg=_cfg if isinstance(_cfg, dict) else {}, ) _cfg_ctx_len = _ctx_lookup.config_context_length _cfg_custom_providers = _ctx_lookup.custom_providers _cfg_api_key = _ctx_lookup.api_key or getattr(agent, 'api_key', '') or resolved_api_key or '' _cfg_base_url = _ctx_lookup.base_url or _cfg_base_url _cfg_provider = _ctx_lookup.provider or resolved_provider or '' _resolved_cl = get_model_context_length( getattr(agent, 'model', resolved_model or '') or '', _cfg_base_url, api_key=_cfg_api_key, config_context_length=_cfg_ctx_len, provider=_cfg_provider, custom_providers=_cfg_custom_providers, ) if _resolved_cl: s.context_length = _resolved_cl except TypeError: # Older hermes-agent builds whose get_model_context_length # signature pre-dates the config_context_length / # custom_providers kwargs. Retry with the legacy 2-arg # form so the indicator still resolves *something*. try: from agent.model_metadata import get_model_context_length as _legacy_cl _resolved_cl = _legacy_cl( getattr(agent, 'model', resolved_model or '') or '', _cfg_base_url, ) if _resolved_cl: s.context_length = _resolved_cl except Exception: pass except Exception: # Older hermes-agent builds may not expose this helper. # Better to leave context_length=0 than crash the save. pass # #3256/#3263: when we skipped the stale compressor cap for a # non-default model and recomputed the real per-model window # above, rescale the persisted threshold_tokens to that real cap # so the auto-compress trigger and the reloaded context-ring # match the live snapshot (which already rescales). Without this, # a reload shows a smaller compression trigger than streaming did. # Only rescale when both the original cap and threshold are # positive; otherwise clear the threshold to 0 (consistent with # the live-snapshot path) rather than leave a stale value. if _skip_cc_cl: _orig_cap = _cc_cl # the stale global cap the compressor reported _orig_thresh = getattr(s, 'threshold_tokens', 0) or 0 _real_cap = getattr(s, 'context_length', 0) or 0 if _real_cap > 0 and _orig_cap > 0 and _orig_thresh > 0: s.threshold_tokens = int(_orig_thresh * _real_cap / _orig_cap) else: s.threshold_tokens = 0 if not ephemeral and s.messages: _latest_assistant_idx = next( (idx for idx in range(len(s.messages) - 1, -1, -1) if isinstance(s.messages[idx], dict) and s.messages[idx].get('role') == 'assistant'), None, ) if _latest_assistant_idx is not None: _latest_assistant = s.messages[_latest_assistant_idx] try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "assistant_started", "created_at": float(_latest_assistant.get('timestamp') or time.time()), "assistant_message_index": _latest_assistant_idx, }, ) except Exception: logger.debug("Failed to append assistant_started turn journal event", exc_info=True) if cancel_event.is_set(): _finalize_cancelled_turn(s, ephemeral=False, stream_id=stream_id) try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return with _stream_writeback_stage(_writeback_timings, "session_save"): s.save() if cancel_event.is_set(): _finalize_cancelled_turn(s, ephemeral=False, stream_id=stream_id) try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return if not ephemeral: try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "completed", "created_at": time.time(), "assistant_message_index": next( (idx for idx in range(len(s.messages) - 1, -1, -1) if isinstance(s.messages[idx], dict) and s.messages[idx].get('role') == 'assistant'), None, ), }, ) except Exception: logger.debug("Failed to append completed turn journal event", exc_info=True) if not ephemeral: # ── Memory-provider lifecycle: mark turn completed (CLI parity) ── # Completed, non-ephemeral turns are marked dirty/uncommitted so # boundary drains know there is work. Per CLI semantics, the # actual memory extraction/commit happens only at session boundaries # (new session creation, LRU eviction, shutdown drain) — NOT after # every completed turn. This mirrors Hermes CLI where # run_agent.py::_sync_external_memory_for_turn() records messages # but only AIAgent.commit_memory_session()/shutdown_memory_provider() # trigger extraction via provider on_session_end(). The mark is # in-memory bookkeeping, not provider I/O, so keep it inside the # per-session writeback lock to preserve completed-turn ordering. try: from api.session_lifecycle import mark_turn_completed mark_turn_completed(s.session_id, agent=agent) except Exception: logger.debug("Memory lifecycle mark failed for session %s", s.session_id, exc_info=True) with _stream_writeback_stage(_writeback_timings, "persistent_state_scan"): try: _persistent_changes = _persistent_state_changes( _persistent_state_before, _persistent_state_snapshot(_profile_home), ) if _persistent_changes.get("memory_saved"): put("state_saved", { "session_id": session_id, "kind": "memory", "action": "saved", }) for _skill_change in _persistent_changes.get("skills") or []: put("state_saved", { "session_id": session_id, "kind": "skill", "action": _skill_change.get("action") or "updated", "name": _skill_change.get("name") or "", }) except Exception: logger.debug("Persistent state change detection failed for session %s", s.session_id, exc_info=True) # Sync to state.db for /insights (opt-in setting) with _stream_writeback_stage(_writeback_timings, "state_sync"): try: from api.config import load_settings as _load_settings if _load_settings().get('sync_to_insights'): from api.state_sync import sync_session_usage sync_session_usage( session_id=s.session_id, input_tokens=s.input_tokens or 0, output_tokens=s.output_tokens or 0, estimated_cost=s.estimated_cost, model=model, title=s.title, message_count=len(s.messages), cache_read_tokens=s.cache_read_tokens or 0, cache_write_tokens=s.cache_write_tokens or 0, api_call_count=getattr(agent, 'session_api_calls', None), # #2762: pass the session's profile explicitly so the # background-thread state.db lookup doesn't fall # through to the process-global active profile and # write to the wrong DB (TLS profile is set on the # HTTP thread but not propagated to this worker). profile=getattr(s, 'profile', None), ) except Exception: logger.debug("Failed to sync session to insights") # A late cancel can land during memory/state-sync writeback. Do not # clear a credential-exhausted process-wakeup pause unless this run # is still settling as a normal completion. The pause re-read, clear, # restore, and save must stay under the session lock so a concurrent # suppression cannot observe stale pause state or lose its update. _lock_ctx = _agent_lock if _agent_lock is not None else contextlib.nullcontext() with _lock_ctx: if cancel_event.is_set(): _finalize_cancelled_turn(s, ephemeral=False, stream_id=stream_id) try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return try: _latest_pause_owner = get_session(getattr(s, 'session_id', session_id)) if _latest_pause_owner is not None: s = _latest_pause_owner except Exception: logger.debug( "Failed to re-read process wakeup pause before success clear", exc_info=True, ) _process_wakeup_pause_before_clear = dict(getattr(s, 'process_wakeup_pause', {}) or {}) if clear_process_wakeup_pause(s, reason='run_completed'): if cancel_event.is_set(): s.process_wakeup_pause = dict(_process_wakeup_pause_before_clear) try: s.save(touch_updated_at=False) except Exception: logger.debug("Failed to persist restored process wakeup pause", exc_info=True) _finalize_cancelled_turn(s, ephemeral=False, stream_id=stream_id) try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return with _stream_writeback_stage(_writeback_timings, "process_wakeup_pause_clear_save"): s.save(touch_updated_at=False) if cancel_event.is_set(): s.process_wakeup_pause = dict(_process_wakeup_pause_before_clear) try: s.save(touch_updated_at=False) except Exception: logger.debug("Failed to persist restored process wakeup pause", exc_info=True) _finalize_cancelled_turn(s, ephemeral=False, stream_id=stream_id) try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return _success_writeback_committed = True usage = { 'input_tokens': input_tokens, 'output_tokens': output_tokens, 'estimated_cost': estimated_cost, 'cache_read_tokens': cache_read_tokens, 'cache_write_tokens': cache_write_tokens, 'cache_hit_percent': cache_hit_percent, 'turn_cache_hit_percent': turn_cache_hit_percent, 'duration_seconds': round(_turn_duration_seconds, 3), } if _turn_tps is not None: usage['tps'] = _turn_tps if _gateway_routing: usage['gateway_routing'] = _gateway_routing _ttft_ms = meter().get_ttft_ms(stream_id) if _ttft_ms is not None: usage['ttft_ms'] = _ttft_ms if _used_model: usage['used_model'] = _used_model # Include context window data from the agent's compressor for the UI indicator. # The session-level persistence happens above (before s.save()) so the values # survive a page reload; this block only populates the live SSE usage payload. _cc = getattr(agent, 'context_compressor', None) if _cc: _cc_cl_sse = getattr(_cc, 'context_length', 0) or 0 # #3256/#3263: remember the original compressor cap + threshold # so that if we drop the stale cap below and the fallback # resolves the real per-model window, we can rescale the # threshold consistently (the live snapshot already does this). _orig_cc_cl_sse = _cc_cl_sse _orig_cc_thresh_sse = getattr(_cc, 'threshold_tokens', 0) or 0 _dropped_stale_cap_sse = False # Default-only guard (#3256), broadened (#4618): the agent-side # context_compressor caches a context_length from the model it # was built/last-updated with. For a non-default model it may be # the stale global cap (e.g. 232K); after an in-place model switch # it may be a DIFFERENT model's window (e.g. opus-4.5's 168k on an # opus-4.8 1M session). Either way, surfacing it via the terminal # `done` SSE makes the indicator REVERT to the wrong window on # stream end (messages.js overwrites S.lastUsage with this payload) # — the exact "send a message reverts to 168k" symptom. Resolve # the real per-model window via the SAME helper the live path + # hydration use; drop the compressor value whenever the real # window differs, honoring the #4248 acceptance gate (never let a # low-confidence 256k fallback clobber a larger cached window). try: from api.routes import ( _context_length_lookup_inputs_for_model as _cli_sse, _should_accept_session_context_length_refresh as _accept_sse, ) from agent.model_metadata import get_model_context_length as _g_sse _sess_model_sse = str(getattr(agent, 'model', resolved_model or '') or '').strip() if _sess_model_sse and _cc_cl_sse > 0: _lk_sse = _cli_sse( _sess_model_sse, resolved_provider or '', base_url=getattr(agent, 'base_url', '') or resolved_base_url or '', api_key=getattr(agent, 'api_key', '') or resolved_api_key or '', cfg=_cfg if isinstance(_cfg, dict) else {}, ) try: _real_sse = _g_sse( _sess_model_sse, _lk_sse.base_url, api_key=_lk_sse.api_key, config_context_length=_lk_sse.config_context_length, provider=_lk_sse.provider or resolved_provider or '', custom_providers=_lk_sse.custom_providers, ) or 0 except TypeError: _real_sse = _g_sse(_sess_model_sse, _lk_sse.base_url) or 0 if _real_sse and _real_sse != _cc_cl_sse and _accept_sse(_cc_cl_sse, _real_sse): _cc_cl_sse = 0 _dropped_stale_cap_sse = True except Exception: pass if _cc_cl_sse: usage['context_length'] = _cc_cl_sse usage['threshold_tokens'] = getattr(_cc, 'threshold_tokens', 0) or 0 usage['last_prompt_tokens'] = getattr(_cc, 'last_prompt_tokens', 0) or 0 # Fallback: when the compressor is absent or reports context_length=0, # resolve the model's context window from metadata so the UI indicator # shows the correct percentage rather than overflowing against the 128K # JS default. Mirrors the session-save fallback above (lines ~2205-2217). # # #1896: pass config_context_length, provider, and custom_providers so # explicit config overrides win over the 256K default fallback. The # SSE payload's `context_length` is what feeds the live token-usage # indicator, so a stale 256K here surfaces as the same wrong-window # display that motivates this fix. if not usage.get('context_length'): try: from agent.model_metadata import get_model_context_length as _get_cl from api.routes import _context_length_lookup_inputs_for_model _ctx_lookup = _context_length_lookup_inputs_for_model( getattr(agent, 'model', resolved_model or '') or '', resolved_provider, base_url=getattr(agent, 'base_url', '') or resolved_base_url or '', cfg=_cfg if isinstance(_cfg, dict) else {}, ) _cfg_ctx_len = _ctx_lookup.config_context_length _cfg_custom_providers = _ctx_lookup.custom_providers _cfg_api_key = _ctx_lookup.api_key or getattr(agent, 'api_key', '') or resolved_api_key or '' _cfg_base_url = _ctx_lookup.base_url _cfg_provider = _ctx_lookup.provider or resolved_provider or '' try: _fb_cl = _get_cl( getattr(agent, 'model', resolved_model or '') or '', _cfg_base_url, api_key=_cfg_api_key, config_context_length=_cfg_ctx_len, provider=_cfg_provider, custom_providers=_cfg_custom_providers, ) except TypeError: # Older hermes-agent builds: fall back to legacy 2-arg form. _fb_cl = _get_cl( getattr(agent, 'model', resolved_model or '') or '', _cfg_base_url, ) if _fb_cl: usage['context_length'] = _fb_cl # #3256/#3263: if we dropped the stale compressor cap # for a non-default model, the threshold_tokens written # above is still the stale compressor value. Rescale it # to the real resolved window so the terminal `done` # payload matches the live snapshot (which rescales) — # otherwise messages.js overwrites S.lastUsage with the # stale threshold and the indicator reverts on stream end. if _dropped_stale_cap_sse and _orig_cc_cl_sse > 0 and _orig_cc_thresh_sse > 0: usage['threshold_tokens'] = int(_orig_cc_thresh_sse * _fb_cl / _orig_cc_cl_sse) except Exception: pass # Fallback: when last_prompt_tokens is missing (no compressor), use the # session-persisted value rather than letting the frontend fall back to # the cumulative input_tokens counter, which overflows for long sessions. if not usage.get('last_prompt_tokens'): _sess_lpt = getattr(s, 'last_prompt_tokens', 0) or 0 if _sess_lpt: usage['last_prompt_tokens'] = _sess_lpt _post_compression_estimate = getattr(s, 'post_compression_context_tokens_estimate', None) usage['post_compression_context_tokens_estimate'] = ( _post_compression_estimate if isinstance(_post_compression_estimate, int) and _post_compression_estimate > 0 else None ) # (reasoning trace already attached + saved above, before s.save()) _settle_pending_steer() # /goal parity: after a successful assistant turn, run the Hermes # GoalManager judge before terminal done/stream_end events. The # frontend surfaces the status line and queues continuation_prompt as # a normal next user message so /queue and user input keep priority. # #1932: only evaluate when the turn was goal-related (set via # STREAM_GOAL_RELATED or goal_related parameter). try: from api.goals import evaluate_goal_after_turn, has_active_goal if not goal_related or not has_active_goal(session_id, profile_home=_profile_home): _goal_decision = {} else: _last_goal_response = '' for _goal_msg in reversed(s.messages or []): if not isinstance(_goal_msg, dict) or _goal_msg.get('role') != 'assistant': continue _goal_content = _goal_msg.get('content', '') if isinstance(_goal_content, list): _goal_parts = [] for _goal_part in _goal_content: if isinstance(_goal_part, dict): _goal_text = _goal_part.get('text') or _goal_part.get('content') if _goal_text: _goal_parts.append(str(_goal_text)) _last_goal_response = '\n'.join(_goal_parts) else: _last_goal_response = str(_goal_content or '') break put('goal', { 'session_id': session_id, 'state': 'evaluating', 'message': 'Evaluating goal progress…', 'message_key': 'goal_evaluating_progress', }) _goal_decision = evaluate_goal_after_turn( session_id, _last_goal_response, user_initiated=True, profile_home=_profile_home, ) decision = _goal_decision or {} _goal_message = str(decision.get('message') or '').strip() if _goal_message: put('goal', { 'session_id': session_id, 'state': 'continuing' if decision.get('should_continue') else 'idle', 'message': _goal_message, 'message_key': decision.get('message_key') or ('goal_continuing' if _goal_message else ''), 'message_args': decision.get('message_args') or [], 'decision': decision, }) if decision.get('should_continue'): continuation_prompt = str(decision.get('continuation_prompt') or '').strip() if continuation_prompt: # #1932: mark this session as pending a goal continuation # so the next /chat/start creates a goal-related stream. PENDING_GOAL_CONTINUATION.add(session_id) put('goal_continue', { 'session_id': session_id, 'continuation_prompt': continuation_prompt, 'text': continuation_prompt, 'message': _goal_message, 'message_key': decision.get('message_key') or 'goal_continuing', 'message_args': decision.get('message_args') or [], 'decision': decision, }) except Exception as _goal_exc: logger.debug("Goal continuation hook failed for session %s: %s", session_id, _goal_exc) with _stream_writeback_stage(_writeback_timings, "done_payload"): raw_session = _session_payload_with_full_messages(s, tool_calls=tool_calls) _done_payload = {'session': redact_session_data(raw_session), 'usage': usage} if _tool_limit_reached: _done_payload['terminal_state'] = 'tool_limit_reached' _done_payload['terminal_reason'] = 'max_iterations' put('done', _done_payload) # Emit one last metering packet for the live message-header TPS label. meter_stats = meter().get_stats(stream_id) meter_stats['session_id'] = session_id meter_stats.setdefault('tps_available', False) meter_stats.setdefault('estimated', False) put('metering', meter_stats) try: _log_stream_writeback_timings( getattr(s, 'session_id', session_id), stream_id, _writeback_timings, _writeback_started, ) except Exception: # Diagnostics must never affect the stream lifecycle: a # misbehaving log handler here would otherwise skip the # background-title thread spawn below. (#4923 gate hardening) pass if _bg_title_inputs: threading.Thread( target=_run_background_title_update, args=(s.session_id, *_bg_title_inputs, str(s.title or '').strip(), put, agent), daemon=True, ).start() else: # Use the original session_id parameter (never reassigned), not s.session_id # which may be rotated during context compression. The client captured # activeSid = original session_id so they must match for stream_end to close. put('stream_end', {'session_id': session_id}) # Adaptive title refresh: re-generate title from latest exchange # every N exchanges (when enabled in settings). Runs after stream_end # so it doesn't block the stream. _maybe_schedule_title_refresh(s, put, agent) finally: # #4729: guaranteed-exit flush of any reasoning tail still buffered. On the # normal path the on_token/on_tool/post-run flushes already emptied it (no-op # here); on an exception or retry path that bypassed those, this emits the tail # before the outer handler sends apperror — so the live Thinking view never # loses its last coalesced chunk. Runs before stream teardown; STREAM_REASONING_TEXT # already mirrors the full text for persistence regardless. try: _flush_reasoning_buffer() except Exception: pass # Stop the live metering ticker _metering_stop.set() # Unregister the gateway approval callback and unblock any threads # still waiting on approval (e.g. stream cancelled mid-approval). if _approval_registered and _unreg_notify is not None: try: _unreg_notify(session_id) except Exception: logger.debug("Failed to unregister approval callback") if _cleanup_gateway_pending_mirror is not None: try: _cleanup_gateway_pending_mirror() except Exception: logger.debug("Failed to reconcile gateway approval mirror") if _clarify_registered and _unreg_clarify_notify is not None: try: _unreg_clarify_notify(session_id) except Exception: logger.debug("Failed to unregister clarify callback") with _ENV_LOCK: for _key, _old_value in old_profile_env.items(): if _old_value is None: os.environ.pop(_key, None) else: os.environ[_key] = _old_value if old_cwd is None: os.environ.pop('TERMINAL_CWD', None) else: os.environ['TERMINAL_CWD'] = old_cwd if old_exec_ask is None: os.environ.pop('HERMES_EXEC_ASK', None) else: os.environ['HERMES_EXEC_ASK'] = old_exec_ask if old_session_key is None: os.environ.pop('HERMES_SESSION_KEY', None) else: os.environ['HERMES_SESSION_KEY'] = old_session_key if old_session_id is None: os.environ.pop('HERMES_SESSION_ID', None) else: os.environ['HERMES_SESSION_ID'] = old_session_id if old_session_platform is None: os.environ.pop('HERMES_SESSION_PLATFORM', None) else: os.environ['HERMES_SESSION_PLATFORM'] = old_session_platform if old_session_chat_id is None: os.environ.pop('HERMES_SESSION_CHAT_ID', None) else: os.environ['HERMES_SESSION_CHAT_ID'] = old_session_chat_id if old_hermes_home is None: os.environ.pop('HERMES_HOME', None) else: os.environ['HERMES_HOME'] = old_hermes_home except Exception as e: print('[webui] stream error:\n' + traceback.format_exc(), flush=True) err_str = str(e) # Sanitize HTML from provider error responses — some providers return # full HTML pages (e.g. nginx "404 page not found") instead of JSON errors. # Strip HTML tags to avoid rendering raw markup in the chat message. _stripped = re.sub(r'<[^>]+>', ' ', err_str) _stripped = re.sub(r'\s+', ' ', _stripped).strip() if _stripped != err_str: err_str = _stripped _exc_lower = err_str.lower() _classification = _classify_provider_error(err_str, e) _exc_is_credential_pool_empty = _classification['type'] == 'credential_pool_empty' if cancel_event.is_set(): if s is not None: if _checkpoint_stop is not None: _checkpoint_stop.set() if _ckpt_thread is not None: _ckpt_thread.join(timeout=15) _lock_ctx = _agent_lock if _agent_lock is not None else contextlib.nullcontext() with _lock_ctx: if ( not ephemeral and _turn_pending_source == 'process_wakeup' and _exc_is_credential_pool_empty ): # Merge the pause into the CURRENT session object under # the canonical lock. The worker-held ``s`` may be a # detached snapshot (LRU-evicted + replaced by a # distinct object through which a successor was # admitted); saving it would serialize stale state over # the successor even though the generation-guarded # finalizer below later no-ops. The pause is # session-wide suppression metadata that must survive # regardless of stream ownership (#6623 re-gate). _wakeup_pause_recorded = _merge_process_wakeup_pause_into_current_session( s, classification=_classification['type'], model=_turn_route_model, provider=_turn_route_provider, ) _finalize_cancelled_turn(s, ephemeral=ephemeral, stream_id=stream_id) if not ephemeral: try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": "cancelled", }, ) except Exception: logger.debug("Failed to append cancelled turn journal event", exc_info=True) put('cancel', _cancel_event_payload('Cancelled by user')) return _exc_is_quota = _classification['type'] == 'quota_exhausted' # Exception quota text still includes: 'more credits' in _exc_lower, 'can only afford' in _exc_lower, 'fewer max_tokens' in _exc_lower. # Rate-limit detection remains guarded as: (not _exc_is_quota). _exc_is_rate_limit = (_classification['type'] == 'rate_limit') and (not _exc_is_quota) _exc_is_auth = _classification['type'] == 'auth_mismatch' # detects '401' and 'unauthorized' via _classify_provider_error. _exc_is_not_found = _classification['type'] == 'model_not_found' # detects '404', 'not found', 'does not exist', and 'invalid model'. _exc_is_cancelled = _classification['type'] == 'cancelled' _exc_is_interrupted = _classification['type'] == 'interrupted' _exc_is_compression_exhausted = _classification['type'] == 'compression_exhausted' _exc_is_compression_snapshot_stale = ( _classification['type'] == 'compression_snapshot_stale' ) _exc_is_provider_unroutable = _classification['type'] == 'provider_unroutable' # The structured terminal reason for an unroutable named custom route, # stamped onto the emitted payload below. Kept separate from _exc_type # because every unroutable route shares one type and differs only here. _exc_route_reason = None # The user hint still points to Settings / `hermes model` from _classify_provider_error(). if _exc_is_provider_unroutable: # Checked FIRST so the terminal route verdict can never be flattened # into the generic 'Error' tail of this chain: its hint names the # exact provider setting to fix, which is the only actionable thing # the user gets on an unroutable custom:. _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) _exc_route_reason = _classification.get('reason') elif _exc_is_quota: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) elif _exc_is_credential_pool_empty: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) elif _exc_is_rate_limit: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) elif _exc_is_auth: _heal_stale_classification = None # Set when the self-heal re-resolve produces a TERMINAL route # verdict. Unlike the two in-flight sites, this branch runs INSIDE # the outer exception handler, so raising here would escape the # generator with no error event at all. Record the verdict instead # and let the abandoned heal fall through to the emission below, # which prefers this classification over the generic # "Authentication error" — the 401 is a symptom, the unroutable # named provider is the cause the user can actually fix. _heal_route_classification = None if not _self_healed: # ── Credential self-heal on 401 (#1401) ── # Bind the session's profile so the self-heal re-resolve AND the # custom-provider override below read one profile-owned snapshot # (finding #3): otherwise a named profile's endpoint pairs with # the default profile's key. from api import profiles as _profiles_api with _profiles_api.profile_scope_for_detached_worker( _resolved_profile_name, "credential self-heal", logger_override=logger ): _heal_rt = _attempt_credential_self_heal( resolved_provider or '', session_id, _agent_lock, target_model=resolved_model, ) if _heal_rt is not None: logger.info('[webui] self-heal (except path): retrying stream after credential refresh') _self_healed = True # Rebuild runtime variables _rt = _heal_rt resolved_api_key = _heal_rt.get('api_key') if not resolved_provider: resolved_provider = _heal_rt.get('provider') resolved_base_url = _runtime_preferred_base_url( _heal_rt, resolved_provider, configured_base_url, session_requested_provider=_session_requested_provider, ) # Preserve the session's original pre-canonicalization provider # identity (captured at first resolve) so a named custom:slug # retry can still select its exact vision-capability entry. # Only initialize when empty. if not _session_requested_provider: _session_requested_provider = resolved_provider # Rebuild the COMPLETE bundle (see the returned-error retry # above): replacing only provider/key/base_url would leave # the pre-heal credential_pool / api_mode / ACP fields on a # custom endpoint that owns none of them. _runtime_bundle = _resolve_runtime_connection_bundle( resolved_provider, resolved_api_key, resolved_base_url, _heal_rt, profile_name=_resolved_profile_name, custom_provider_lookup=_session_requested_provider, ) _heal_route_verdict = custom_provider_route_error(_runtime_bundle) if _heal_route_verdict is not None: # The refreshed credential did not produce a routable # named connection. Abandon the heal BEFORE _AIAgent and # before the SESSION_AGENT_CACHE write below — building # here would hand the constructor an incomplete pair and # let _routed_client_kwargs() pick a provider of its own, # and caching that agent would keep doing so on every # later turn in this session. _heal_route_classification = ( _custom_provider_route_classification(_heal_route_verdict) ) logger.warning( '[webui] self-heal (except path): abandoning retry — %s', _heal_route_classification['message'], ) # Clearing _heal_rt closes the retry block below. We must # NOT raise from inside the outer exception handler: that # would escape the generator and the client would get no # error event at all. _heal_rt = None if _heal_rt is not None: resolved_provider = _runtime_bundle['provider'] resolved_api_key = _runtime_bundle['api_key'] resolved_base_url = _runtime_bundle['base_url'] # Build a fresh agent with the new credentials _heal_kwargs = dict(_agent_kwargs) if '_agent_kwargs' in dir() else {} _heal_kwargs['api_key'] = resolved_api_key _heal_kwargs['base_url'] = resolved_base_url _heal_kwargs['model'] = resolved_model _heal_kwargs['provider'] = resolved_provider _replace_session_db_in_kwargs(_heal_kwargs, _state_db_path) if 'api_mode' in _agent_params: _heal_kwargs['api_mode'] = _runtime_bundle['api_mode'] if 'acp_command' in _agent_params: _heal_kwargs['acp_command'] = _runtime_bundle['acp_command'] if 'acp_args' in _agent_params: _heal_kwargs['acp_args'] = _runtime_bundle['acp_args'] if 'credential_pool' in _agent_params: _heal_kwargs['credential_pool'] = _runtime_bundle['credential_pool'] _heal_agent = _AIAgent(**_heal_kwargs) # Delta callbacks close over `agent`; the replacement, not # the failed original, owns any successful retry output. agent = _heal_agent _agent_sig = _compute_agent_cache_signature( resolved_model, resolved_api_key, resolved_base_url, resolved_provider, _runtime_bundle, max_iterations_cfg=_max_iterations_cfg, max_tokens_cfg=_max_tokens_cfg, fallback_resolved=_fallback_resolved, toolsets=_toolsets, reasoning_config=_reasoning_config, main_request_overrides=_main_request_overrides, prefill_context=_prefill_context, profile_home=_profile_home, safe_profile_runtime_env=_safe_profile_runtime_env, ) if not _register_agent_if_current(_heal_agent, _agent_sig): with _agent_lock: _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) put('cancel', _cancel_event_payload('Cancelled by user')) return # Retry the conversation _token_sent = False try: ( _heal_context_messages, _heal_conversation_history_revision, ) = _refresh_context_and_revision_from_state_db() with _agent_lock: _heal_persist_user_timestamp = getattr( s, 'pending_started_at', None ) _register_pending_user_timestamp_identity( _heal_agent.run_conversation, s, _heal_persist_user_timestamp, ) _heal_kwargs2 = _build_run_conversation_kwargs( _heal_agent.run_conversation, user_message=user_message, system_message=workspace_system_msg, conversation_history=_sanitize_messages_for_agent( _heal_context_messages, cfg=_cfg, effective_model=resolved_model, effective_provider=resolved_provider, effective_base_url=resolved_base_url, requested_provider=(_session_requested_provider or ""), ), conversation_history_revision=( _heal_conversation_history_revision ), task_id=session_id, persist_user_message=msg_text, persist_user_timestamp=_heal_persist_user_timestamp, ) if moa_config is not None: _heal_kwargs2["moa_config"] = moa_config _result_partial_pre_call_context = list( _heal_context_messages ) if not _agent_can_invoke(_heal_agent): with _agent_lock: _finalize_cancelled_turn(s, ephemeral=ephemeral, message='Task cancelled before start.', stream_id=stream_id) put('cancel', _cancel_event_payload('Cancelled by user')) return _heal_result = _heal_agent.run_conversation(**_heal_kwargs2) _remember_pending_steer_result(_heal_result) _active_turn_identity = _resolve_active_turn_authority( _active_turn_identity, result=_heal_result, agent=_heal_agent, ) _heal_stale_classification = None if _result_reports_compression_snapshot_stale(_heal_result): _heal_stale_classification = _classify_provider_error( '', result=_heal_result, ) result = _heal_result elif _self_heal_result_succeeded( _heal_result, _heal_context_messages, _active_turn_identity, msg_text, ): # Retry succeeded — persist the result normally. _done_session_payload = None if s is not None: if _checkpoint_stop is not None: _checkpoint_stop.set() if _ckpt_thread is not None: _ckpt_thread.join(timeout=15) _lock_ctx = _agent_lock if _agent_lock is not None else contextlib.nullcontext() with _lock_ctx: if not ephemeral and not _stream_writeback_is_current(s, stream_id): logger.info( "Skipping stale stream self-heal writeback for session %s stream %s; active_stream_id=%s", getattr(s, 'session_id', session_id), stream_id, getattr(s, 'active_stream_id', None), ) return _result_messages = _heal_result.get('messages') if _result_messages is None: _result_messages = _previous_context_messages _result_messages = _settle_result_messages( s, _previous_messages, _previous_owner_context_messages, _result_messages, msg_text, _turn_pending_source, _active_turn_identity, ) # Terminal self-heal success must finalize the # turn exactly once: clear the pending markers # so the last-resort recovery sync in the outer # ``finally`` cannot re-materialize this user # turn and append a spurious "Response # interrupted" marker after the settled answer. s.tool_calls = _extract_tool_calls_from_messages( s.messages, live_tool_calls=_live_tool_calls, ) s.active_stream_id = None s.pending_user_message = None s.pending_attachments = [] s.pending_started_at = None s.pending_user_source = None s.save() _done_session_payload = redact_session_data( _session_payload_with_full_messages( s, tool_calls=s.tool_calls ) ) _observe_runtime_model() if _done_session_payload is not None: put('done', { 'session': _done_session_payload, 'usage': {'input_tokens': 0, 'output_tokens': 0}, }) put('stream_end', {'session_id': session_id}) logger.info('[webui] self-heal (except path): retry succeeded') return # skip error emission except Exception as _retry_exc2: logger.warning('[webui] self-heal (except path): retry failed: %s', _retry_exc2) # Fall through to emit the original error if _heal_route_classification is not None: # The heal was abandoned because the named route is unroutable. # Report THAT, not the 401 that triggered the heal: telling the # user to check credentials when the provider resolves no # endpoint (or declares a credential source that yields nothing) # sends them to the wrong setting. _exc_label = _heal_route_classification['label'] _exc_type = _heal_route_classification['type'] _exc_hint = _heal_route_classification['hint'] _exc_route_reason = _heal_route_classification.get('reason') if _heal_route_classification['message']: err_str = _heal_route_classification['message'] elif _heal_stale_classification is not None: _exc_label = _heal_stale_classification['label'] _exc_type = _heal_stale_classification['type'] _exc_hint = _heal_stale_classification['hint'] _exc_is_compression_snapshot_stale = True else: # Self-heal didn't apply or retry failed — emit the auth error. _exc_label, _exc_type, _exc_hint = ( 'Authentication error', 'auth_mismatch', 'The selected model may not be supported by your configured provider. ' 'Run `hermes model` in your terminal to switch providers, then restart the WebUI.', ) elif _exc_is_not_found: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) elif _exc_is_cancelled or _exc_is_interrupted: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) elif _exc_is_compression_snapshot_stale: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) elif _exc_is_compression_exhausted: _exc_label, _exc_type, _exc_hint = ( _classification['label'], _classification['type'], _classification['hint'], ) else: _exc_label, _exc_type, _exc_hint = 'Error', 'error', '' if _exc_is_compression_snapshot_stale: # The typed exception may carry expected/observed revision details for # diagnostics. Keep those out of the persisted transcript and SSE UI. err_str = ( 'The conversation changed while context compression was being prepared.' ) _error_payload = _provider_error_payload(err_str, _exc_type, _exc_hint) if _exc_route_reason: # Clients (and regressions) branch on the exact verdict rather than # on the human-readable message, which is free to be reworded. _error_payload['reason'] = _exc_route_reason if s is not None: if _checkpoint_stop is not None: _checkpoint_stop.set() if _ckpt_thread is not None: _ckpt_thread.join(timeout=15) # Persist the error so it survives page reload. # _error=True ensures _sanitize_messages_for_api excludes it from subsequent # API calls so the LLM never sees its own error as prior context on the next turn. _lock_ctx = _agent_lock if _agent_lock is not None else contextlib.nullcontext() with _lock_ctx: if not ephemeral and not _stream_writeback_is_current(s, stream_id): if _turn_pending_source == 'process_wakeup': # #6623 re-gate: merge the pause into the CURRENT # session object under the canonical lock — never save # the worker's detached snapshot. The helper fails # closed (no write) when the current object cannot be # resolved. _merge_process_wakeup_pause_into_current_session( s, classification=_exc_type, model=_turn_route_model, provider=_turn_route_provider, ) logger.info( "Skipping stale stream error writeback for session %s stream %s; active_stream_id=%s", getattr(s, 'session_id', session_id), stream_id, getattr(s, 'active_stream_id', None), ) return if _turn_pending_source == 'process_wakeup': _recorded_pause = record_process_wakeup_provider_unavailable_pause( s, classification=_exc_type, model=_turn_route_model, provider=_turn_route_provider, ) # #3929 UX: disclose the pause in the error card ONLY when a # pause was actually recorded (credential-pool exhaustion), # keeping the SSE payload hint in sync with the persisted bubble. if _recorded_pause: _exc_hint = ( (_exc_hint + ' ' if _exc_hint else '') + 'Automatic retries for this conversation are paused until you ' + 'send a message, switch the model/provider, or fix the credentials.' ) _error_payload['hint'] = _exc_hint _turn_duration = _terminal_turn_duration(s) # Keep the canonical one-argument error-settlement shape pinned # by #1361/#2136. The helper derives the same stream-owned turn # token from active_stream_id + pending_started_at when the # explicit identity is omitted, so repeated prompts remain # fenced without weakening pending-turn durability. _materialize_pending_user_turn_before_error(s) s.active_stream_id = None s.pending_user_message = None s.pending_attachments = [] s.pending_started_at = None s.pending_user_source = None try: _snapshot_and_append_partial_on_error( s, stream_id, active_turn_identity=_active_turn_identity, ) _append_result_partial_on_error( s, result, _result_partial_pre_call_context, msg_text, active_turn_identity=_active_turn_identity, ) except Exception: logger.debug("Failed to snapshot partials on error for %s", stream_id, exc_info=True) _error_message = { 'role': 'assistant', 'content': f'**{_exc_label}:** {_error_payload.get("message") or err_str}' + (f'\n\n*{_exc_hint}*' if _exc_hint else ''), 'timestamp': int(time.time()), '_error': True, } if _turn_duration is not None: _error_message['_turnDuration'] = _turn_duration if _exc_type == 'compression_exhausted': _recovery = stamp_compression_exhausted_recovery( s, message=_error_payload.get('message') or err_str, details=_error_payload.get('details') or '', ) _error_message['_compressionRecovery'] = _recovery _error_payload['compression_recovery'] = _recovery _error_payload['recommended_recovery_action'] = _recovery.get('recommended_action') if _error_payload.get('details'): _error_message['provider_details'] = _error_payload['details'] if _exc_type == 'cancelled': _error_message['provider_details_label'] = 'Cancellation details' elif _exc_type == 'interrupted': _error_message['provider_details_label'] = 'Interruption details' s.messages.append(_error_message) try: s.save() except Exception: pass if not ephemeral: try: append_turn_journal_event_for_stream( s.session_id, stream_id, { "event": "interrupted", "created_at": time.time(), "reason": _exc_type, }, ) except Exception: logger.debug("Failed to append interrupted turn journal event", exc_info=True) _error_payload['session_id'] = getattr(s, 'session_id', session_id) _error_payload['old_session_id'] = session_id put('apperror', _error_payload) finally: _settle_pending_steer() # #4633/#2476: symmetric metering teardown. begin_session() (top of the # outer try) had no paired end_session(), so zero-token turns leaked a # _sessions[stream_id] entry that get_stats() pruning never reclaims (its # criterion requires first_token_ts > 0). end_session() is idempotent — # it just pops _sessions[stream_id]; the metering payload is unchanged. # _metering_stop.set() deterministically stops the ticker (the inner # finally also sets it on the normal path; setting twice is harmless). try: # 0: end_session() currently ignores final_output_tokens — it only # pops _sessions[stream_id]. If it is ever extended to consume the # count (e.g. persisting final output tokens to a billing ledger), # this teardown caller will need to supply the real total; the outer # finally doesn't have easy access to it today. meter().end_session(stream_id, 0) except Exception: logger.debug("Failed to end metering session for stream %s", stream_id, exc_info=True) _metering_stop.set() # Stop the periodic checkpoint thread before the final recovery path. # The checkpoint thread also uses the per-session lock; joining it first # avoids contending with checkpoint writes during stale-pending repair. if _checkpoint_stop is not None: _checkpoint_stop.set() if _ckpt_thread is not None: _ckpt_thread.join(timeout=15) if (s is not None and getattr(s, 'active_stream_id', None) == stream_id and getattr(s, 'pending_user_message', None)): _last_resort_sync_from_core(s, stream_id, _agent_lock) _clear_thread_env() # TD1: always clear thread-local context if _streaming_cron_profile_home_token is not None: _STREAMING_CRON_PROFILE_HOME.reset(_streaming_cron_profile_home_token) if _restore_streaming_skill_home_modules and _streaming_skill_home_snapshot is not None: with _ENV_LOCK: if restore_skill_home_modules is not None: try: restore_skill_home_modules(_streaming_skill_home_snapshot) except Exception: logger.debug("Failed to restore skill module state for streaming profile", exc_info=True) _streaming_skill_home_snapshot = None _restore_streaming_skill_home_modules = False if _acquired_streaming_skill_home_patch_lock: _SKILL_HOME_MODULE_PATCH_LOCK.release() _acquired_streaming_skill_home_patch_lock = False _reset_streaming_hermes_home_override(*_streaming_hermes_home_override_ctx) # xsession wakeup misroute root fix (Option 1): restore the per-turn # session-identity context-locals (reset-token semantics). MUST run on # every exit path so a reused thread-pool worker leaks no identity and # CLI/cron env fallback resumes — same lifecycle slot as the env # restore above. _reset_turn_session_identity(_turn_session_identity_tokens) with STREAMS_LOCK: STREAMS.pop(stream_id, None) CANCEL_FLAGS.pop(stream_id, None) AGENT_INSTANCES.pop(stream_id, None) # Clean up agent instance reference STREAM_PARTIAL_TEXT.pop(stream_id, None) # Clean up partial text buffer (#893) STREAM_REASONING_TEXT.pop(stream_id, None) # Clean up reasoning trace (#1361 §A) STREAM_LIVE_TOOL_CALLS.pop(stream_id, None) # Clean up tool calls (#1361 §B) STREAM_GOAL_RELATED.pop(stream_id, None) # Clean up goal-related flag (#1932) STREAM_LAST_EVENT_ID.pop(stream_id, None) # Clean up event_id pointer (stage-364) unregister_active_run(stream_id) # Clean up the stream-owner registry so stale stream_id→session_id # mappings do not accumulate over thousands of completed streams (#6351). unregister_stream_owner(stream_id) # Release the session's writeback-ownership entry only while this # stream still owns it (#6623 re-gate): a successor admitted after # cancel must keep its registry claim. try: clear_session_writeback_owner_if_owned(session_id, stream_id) except Exception: logger.debug( "Failed to clear session writeback owner for stream %s", stream_id, exc_info=True, ) # NOTE: do NOT discard PENDING_GOAL_CONTINUATION here. The marker # is set by goal_continue (line ~3328) inside the SAME function # call and consumed atomically by `_start_chat_stream_for_session` # in routes.py (around line 6522) when the next stream starts. # Discarding here in the streaming worker's `finally` would # almost always race ahead of the frontend's SSE-receive → # POST /api/chat/start round-trip and erase the marker before # the next stream can read it, breaking the goal-continuation # chain. Stage-326 critical fix per Opus advisor review. # ── Defer-path fix: turn-teardown idle-hook ──────────────────────── # The session has just transitioned active→idle: unregister_active_run # above cleared this stream's ACTIVE_RUNS row (under ACTIVE_RUNS_LOCK, # independent of STREAMS_LOCK), so _session_has_active_turn() is now # False for this session unless a *different* stream is still active # (cancel/reconnect — drain_deferred_wakeups_for_session guards on # that and leaves the marker for the later teardown). A FAST # background task that completed while this turn was tearing down was # deferred by api/background_process._process_one (it could not start # a turn → would 409) and its wakeup_prompt persisted in # DEFERRED_PROCESS_WAKEUPS. For an autonomous agent there is no next # user turn, so the PR #2279 next-turn drain never runs; without this # hook the deferred wakeup is lost forever (the Test B failure). This # makes the busy-at-completion case symmetric with the idle case: # idle now → fire now (Option Z idle branch); busy now → fire here at # turn-end. claim_deferred_wakeups pops atomically, so this is # idempotent with the next-turn drain (no double-fire) and the wakeup # turn's own teardown finds nothing claimed (no wakeup loop). The # drain spawns its own daemon thread, so teardown never blocks. try: from api.background_process import drain_deferred_wakeups_for_session drain_deferred_wakeups_for_session(session_id) except Exception: logger.debug( "turn-teardown deferred-wakeup drain failed for session %s", session_id, exc_info=True, ) # ============================================================ # SECTION: HTTP Request Handler # do_GET: read-only API endpoints + SSE stream + static HTML # do_POST: mutating endpoints (session CRUD, chat, upload, approval) # Routing is a flat if/elif chain. See ARCHITECTURE.md section 4.1. # ============================================================ # Local lifecycle phases with a pending-steer consumer still ahead of them. # Gateway uses its own phase vocabulary and never falls through to local delivery. _LOCAL_STEERABLE_PHASES = frozenset({"starting", "running"}) def _steer_bound_stream(sid: str, stream_id: str, text: str) -> dict | None: """Deliver to a verified live worker; None retains cache-only compatibility. Never perform HTTP writes or cache/database teardown under stream locks. """ from api import config as cfg with cfg.STREAMS_LOCK: agent = cfg.AGENT_INSTANCES.get(stream_id) owner = cfg.stream_owner_session_id(stream_id) with cfg.ACTIVE_RUNS_LOCK: run = dict(cfg.ACTIVE_RUNS.get(stream_id) or {}) # Gateway owns transport even when no in-process worker is registered. # A reusable local cache entry must never override that authority. if run.get("backend") == "gateway": if (stream_id not in cfg.STREAMS or owner != sid or run.get("session_id") != sid or run.get("phase") == "cancelling"): return {"accepted": False, "fallback": "stream_dead", "stream_id": None} return {"accepted": False, "fallback": "gateway_steer_queued", "stream_id": stream_id} if (stream_id in cfg.STREAMS and owner == sid and run.get("session_id") == sid and run.get("backend") == WEBUI_LOCAL_CHAT_BACKEND and run.get("phase") == "finalizing"): return {"accepted": False, "fallback": "not_running", "stream_id": stream_id} if agent is None: if ((owner and owner != sid) or (run.get("session_id") and run["session_id"] != sid) or (run and run.get("phase") not in _LOCAL_STEERABLE_PHASES)): return {"accepted": False, "fallback": "stream_dead", "stream_id": None} return None # A cache hit cannot override missing or conflicting worker ownership. if (stream_id not in cfg.STREAMS or owner != sid or run.get("session_id") != sid or run.get("phase") not in _LOCAL_STEERABLE_PHASES or run.get("backend") == "gateway"): return {"accepted": False, "fallback": "stream_dead", "stream_id": None} if not callable(getattr(agent, "steer", None)): return {"accepted": False, "fallback": "agent_lacks_steer", "stream_id": None} # steer() only stashes input; serializing it with stream teardown keeps # cancel from detaching the selected worker before delivery. try: accepted = bool(agent.steer(text)) except Exception: logger.debug("Stream-bound steer failed for session %s", sid, exc_info=True) return {"accepted": False, "fallback": "steer_error", "stream_id": stream_id} return {"accepted": accepted, "fallback": None, "stream_id": stream_id} def _handle_chat_steer(handler, body: dict) -> bool: """Inject a /steer payload into the active agent for a session. Mirrors the CLI's `/steer ` command (cli.py:6140-6155): - Prefer the active stream's registered AIAgent, with explicit stream and worker ownership; use the cache only for legacy cache-only runs. - Verify a stream is currently active for this session. - Never evict or close an agent while delivering steering input. - Call agent.steer(text) — thread-safe, stashes text in _pending_steer for application at the next tool-result boundary. The agent's loop calls _apply_pending_steer_to_tool_results() at the end of every tool batch and appends the steer text to the last tool result's content with a marker, so the model sees the steer as part of the tool output on its next iteration. The user's stream is NOT interrupted. If no agent is cached, the agent is too old to support steer, or no stream is active, return {"accepted": False, "fallback": ""}. The frontend must surface that failure without cancelling the active run; Steer is active-run guidance, not implicit permission to Queue, Interrupt, or Stop-and-send. Returns 200 with {"accepted": bool, "fallback": str|None, "stream_id": str|None}. """ from api.helpers import j, bad from api import config as _cfg sid = str((body or {}).get("session_id", "") or "").strip() text = str((body or {}).get("text", "") or "").strip() if not sid: return bad(handler, "session_id required") if not text: return bad(handler, "text required") # A compression rotates agent.session_id without replacing the running # worker. Resolve the stream-bound instance BEFORE consulting the reusable # cache. Steer never owns cache eviction or agent/database teardown. try: session = get_session(sid) except KeyError: session = None stream_id = getattr(session, "active_stream_id", None) or None if stream_id: result = _steer_bound_stream(sid, stream_id, text) if result is not None: return j(handler, result) with _cfg.SESSION_AGENT_CACHE_LOCK: cached = _cfg.SESSION_AGENT_CACHE.get(sid) if cached and not _cached_agent_matches_session(cached[0], sid): cached = None if not cached: try: s = get_session(sid) active_stream_id = getattr(s, "active_stream_id", None) or None except KeyError: active_stream_id = None if active_stream_id: with _cfg.STREAMS_LOCK: stream_alive = active_stream_id in _cfg.STREAMS if stream_alive: try: with _cfg.ACTIVE_RUNS_LOCK: active_run = dict((_cfg.ACTIVE_RUNS or {}).get(str(active_stream_id)) or {}) if active_run.get("backend") == "gateway": return j(handler, {"accepted": False, "fallback": "gateway_steer_queued", "stream_id": active_stream_id}) except Exception: logger.warning( "Gateway ownership lookup failed before steer fallback for session=%s stream_id=%s", sid, active_stream_id, exc_info=True, ) # No active local agent for this session — caller surfaces a steer failure # without cancelling the active run. return j(handler, {"accepted": False, "fallback": "no_cached_agent", "stream_id": None}) agent = cached[0] if not hasattr(agent, "steer"): # Older hermes-agent that pre-dates the steer() method return j(handler, {"accepted": False, "fallback": "agent_lacks_steer", "stream_id": None}) # Verify the agent is currently running. Use the session's # active_stream_id rather than calling load_session_locked() which # would block on the streaming thread's lock. try: s = get_session(sid) except KeyError: return j(handler, {"accepted": False, "fallback": "session_not_found", "stream_id": None}) active_stream_id = getattr(s, "active_stream_id", None) or None if not active_stream_id: return j(handler, {"accepted": False, "fallback": "not_running", "stream_id": None}) # Cache-only compatibility: no registered worker for this stream. The # liveness/ownership revalidation and the enqueue must share the same # stream-ownership edge as Stop (STREAMS_LOCK -> ACTIVE_RUNS_LOCK). A Stop # that claims cancellation between an unlocked check and agent.steer() # would strand guidance this response still reports as accepted. steer() # only stashes input; interrupt, persistence, and HTTP writes stay outside # the lock, and the cached agent is never evicted or closed here. # # Ownership must be proven positively, not merely unrefuted: BOTH the # stream owner AND the active-run session must equal the requesting # session. Missing metadata is ambiguous and fails closed (stream_dead); # an unfenced cache object is never steered on absent ownership. # # The active-run backend is revalidated the same way. With no registered # worker, the backend tag is the only proof that an in-process runtime # owns this run: Gateway resolves to its own outcome, the explicit local # tag may enqueue, and a missing, empty, or foreign backend fails closed. result = {"accepted": False, "fallback": "stream_dead", "stream_id": None} with _cfg.STREAMS_LOCK: if active_stream_id in _cfg.STREAMS: owner = _cfg.stream_owner_session_id(active_stream_id) with _cfg.ACTIVE_RUNS_LOCK: run = dict((_cfg.ACTIVE_RUNS or {}).get(str(active_stream_id)) or {}) owned_stream = bool(owner) and owner == sid owned_run = bool(run.get("session_id")) and run["session_id"] == sid if owned_stream and owned_run and run.get("phase") != "cancelling": backend = run.get("backend") if backend == "gateway": # Gateway owns transport; a local cache object is never steered. result = {"accepted": False, "fallback": "gateway_steer_queued", "stream_id": active_stream_id} elif backend == WEBUI_LOCAL_CHAT_BACKEND and run.get("phase") == "finalizing": result = {"accepted": False, "fallback": "not_running", "stream_id": active_stream_id} elif (backend == WEBUI_LOCAL_CHAT_BACKEND and run.get("phase") in _LOCAL_STEERABLE_PHASES): try: accepted = bool(agent.steer(text)) except Exception as exc: logger.debug("agent.steer() raised for session=%s: %s", sid, exc) result = {"accepted": False, "fallback": "steer_error", "stream_id": active_stream_id} else: result = {"accepted": accepted, "fallback": None, "stream_id": active_stream_id} return j(handler, result) def cancel_stream(stream_id: str) -> bool: """Signal an in-flight stream to cancel. Returns True if work was found. Eagerly releases the session lock (pops STREAMS/CANCEL_FLAGS/AGENT_INSTANCES and clears session.active_stream_id) so new /api/chat/start requests succeed immediately after cancel, even if the agent thread is still blocked. The worker thread's finally block uses .pop(key, None), so the double-pop is a safe no-op. Session cleanup runs outside STREAMS_LOCK to preserve lock ordering (streaming thread does LOCK → STREAMS_LOCK; inverting would deadlock). """ from api import config as _live_config # Use module-level aliases (imported from api.config at startup). # In production these are always the same objects as api.config.STREAMS etc. # The fallback below handles a hypothetical future case where api.config's # state dicts are replaced at runtime (e.g. a future profile-reload path). # No production code currently does this; the fallback is defensive only. streams = STREAMS cancel_flags = CANCEL_FLAGS agent_instances = AGENT_INSTANCES partial_texts = STREAM_PARTIAL_TEXT streams_lock = STREAMS_LOCK if stream_id not in streams and getattr(_live_config, 'STREAMS', streams) is not streams: streams = _live_config.STREAMS cancel_flags = _live_config.CANCEL_FLAGS agent_instances = _live_config.AGENT_INSTANCES partial_texts = _live_config.STREAM_PARTIAL_TEXT streams_lock = _live_config.STREAMS_LOCK active_run_entry = None active_run_session_id = None stream_present = False agent = None q = None # Snapshots captured UNDER streams_lock so the worker's finally block (which # pops STREAM_PARTIAL_TEXT/REASONING/TOOL_CALLS under STREAMS_LOCK) cannot # race agent.interrupt() and clear these buffers before we read them — a # cancelled turn would otherwise silently lose its already-streamed # partial text / reasoning / tool-calls (Codex pre-release finding). _snap_partial_text = None _snap_reasoning = None _snap_tool_calls = None _snap_flag = None _snap_agent = None _snap_owner_session_id = None _cancel_session_payload = None with streams_lock: stream_present = stream_id in streams # Snapshot everything the worker's finally could pop, WHILE the lock is # held and before any interrupt lets that finally run — for BOTH the # STREAMS-present and the ACTIVE_RUNS-only (detached) paths. The buffers # are keyed by stream_id independent of STREAMS membership, so a detached # cancel must snapshot them too or it loses the already-streamed text. _snap_flag = cancel_flags.get(stream_id) _snap_agent = agent_instances.get(stream_id) # Capture the stream owner WHILE the stream still exists (#6623). The # just-starting worker takes its `q is None -> unregister_stream_owner` # early path (api/streaming.py `_run_agent_streaming` and # api/gateway_chat.py `_run_gateway_chat_streaming`) the instant # STREAMS[stream_id] is popped below, so reading the owner AFTER the # pop can race that teardown and yield None — leaving the session's # active_stream_id/pending_* stuck while cancel still returns True. # Reading it here, under streams_lock and before any pop, gives cancel # a stable owner to resolve session cleanup against. _snap_owner_session_id = stream_owner_session_id(stream_id) _snap_partial_text = partial_texts.get(stream_id, '') if not _snap_partial_text: _live_partials = getattr(_live_config, 'STREAM_PARTIAL_TEXT', partial_texts) if _live_partials is not partial_texts: _snap_partial_text = _live_partials.get(stream_id, '') _snap_reasoning = STREAM_REASONING_TEXT.get(stream_id, '') if not _snap_reasoning: _live_reasoning = getattr(_live_config, 'STREAM_REASONING_TEXT', STREAM_REASONING_TEXT) if _live_reasoning is not STREAM_REASONING_TEXT: _snap_reasoning = _live_reasoning.get(stream_id, '') _snap_tool_calls = list(STREAM_LIVE_TOOL_CALLS.get(stream_id, []) or []) if not _snap_tool_calls: _live_tools = getattr(_live_config, 'STREAM_LIVE_TOOL_CALLS', STREAM_LIVE_TOOL_CALLS) if _live_tools is not STREAM_LIVE_TOOL_CALLS: _snap_tool_calls = list(_live_tools.get(stream_id, []) or []) if stream_present: q = streams.get(stream_id) else: try: with _live_config.ACTIVE_RUNS_LOCK: active_run_entry = dict((_live_config.ACTIVE_RUNS or {}).get(stream_id) or {}) except Exception: active_run_entry = None if not active_run_entry: return False active_run_session_id = str(active_run_entry.get("session_id") or "").strip() or None if active_run_entry is None: try: with _live_config.ACTIVE_RUNS_LOCK: active_run_entry = dict((_live_config.ACTIVE_RUNS or {}).get(stream_id) or {}) except Exception: active_run_entry = None if active_run_entry and not active_run_session_id: active_run_session_id = str(active_run_entry.get("session_id") or "").strip() or None # Mark the worker lifecycle registry immediately. The SSE maps may be popped # below while the worker is still unwinding; ACTIVE_RUNS is what recovery / # health polling sees during that detached window. Stamp cancelled_at so # _clear_stale_stream_state() can eventually reclaim the session if the # worker is stuck in C-level I/O and never reaches its finally (#6623). update_active_run(stream_id, phase="cancelling", cancelled_at=time.time()) # Stop and Steer share STREAMS_LOCK -> ACTIVE_RUNS_LOCK ordering. # Publish cancellation and detach ownership before releasing the edge; # later Steer cannot enqueue into a turn already claimed by Stop. if stream_present: streams.pop(stream_id, None) cancel_flags.pop(stream_id, None) agent_instances.pop(stream_id, None) # Set WebUI layer cancel flag. Prefer the snapshot captured under the lock; # fall back to a fresh lookup for the ACTIVE_RUNS-only path (stream absent). flag = _snap_flag if _snap_flag is not None else cancel_flags.get(stream_id) if flag: flag.set() # Interrupt the AIAgent instance to stop tool execution. Use the # lock-snapshot agent when the stream was present; otherwise fall back to # the session agent cache via the active-run session id. agent = _snap_agent if _snap_agent is not None else agent_instances.get(stream_id) if agent is None and active_run_session_id: try: with _live_config.SESSION_AGENT_CACHE_LOCK: cached = _live_config.SESSION_AGENT_CACHE.get(active_run_session_id) if cached and _cached_agent_matches_session(cached[0], active_run_session_id): agent = cached[0] except Exception: pass if agent: try: agent.interrupt("Cancelled by user") except Exception as e: # Log but don't block the cancel flow import logging logging.getLogger(__name__).debug( f"Failed to interrupt agent for stream {stream_id}: {e}" ) elif stream_present: # Agent not yet stored - cancel_event flag will be checked by agent thread import logging logging.getLogger(__name__).debug( f"Cancel requested for stream {stream_id} before agent ready - " f"cancel_event flag set, will be checked on agent startup" ) # Clear any pending clarify prompt so the blocked tool call can unwind. try: from api.clarify import clear_pending as _clear_clarify_pending _clarify_session_id = getattr(agent, "session_id", None) if agent else active_run_session_id if _clarify_session_id: _clear_clarify_pending(_clarify_session_id) except Exception: logger.debug("Failed to clear clarify prompt during cancel") # Capture the queue while the stream still exists, but do not emit the # terminal cancel event until the session cleanup below confirms the turn # is still active. Otherwise a late Stop click can race with a successful # worker save and show cancel in the client while persistence says done. _emit_cancel_event = True # Stream ownership was detached under streams_lock before interrupting. # STREAM_PARTIAL_TEXT is intentionally NOT popped here — the agent thread may # still be appending tokens, and the streaming finally block handles cleanup # when the thread exits. We already snapshotted the buffers under streams_lock # at the top of this function (see _snap_*), so they're safe to read below # even if the worker's finally has since popped the live maps. # Resolve the cancel session id and reuse the under-lock snapshots. # Session cleanup (get_session + save) must happen OUTSIDE the lock — # get_session() acquires LOCK, and the streaming thread does LOCK first # then STREAMS_LOCK, so inverting the order here would cause deadlock. _cancel_session_id = getattr(agent, 'session_id', None) if agent else None if not _cancel_session_id and active_run_session_id: _cancel_session_id = active_run_session_id # Third fallback: stream owner registry — populated before the worker # thread starts, so it's always available even for early cancels that # race ahead of AGENT_INSTANCES and ACTIVE_RUNS (#6623). The owner is # read UNDER streams_lock above (while STREAMS[stream_id] still exists), # NOT here after the eager pop: the just-starting worker unregisters the # owner the instant the stream map entry disappears, so a post-pop lookup # would race that teardown and return None. if not _cancel_session_id and _snap_owner_session_id: _cancel_session_id = _snap_owner_session_id # Use the snapshots captured under streams_lock above (the worker's finally # may have popped the live buffers by now via agent.interrupt()). For the # ACTIVE_RUNS-only path (stream absent) the snapshots are None → fall back to # a best-effort live read. _cancel_partial_text = _snap_partial_text if _snap_partial_text is not None else partial_texts.get(stream_id, '') if not _cancel_partial_text: live_partials = getattr(_live_config, 'STREAM_PARTIAL_TEXT', partial_texts) if live_partials is not partial_texts: _cancel_partial_text = live_partials.get(stream_id, '') # Capture reasoning trace and live tool calls (#1361 §A + §B) _cancel_reasoning = _snap_reasoning if _snap_reasoning is not None else STREAM_REASONING_TEXT.get(stream_id, '') if not _cancel_reasoning: live_reasoning = getattr(_live_config, 'STREAM_REASONING_TEXT', STREAM_REASONING_TEXT) if live_reasoning is not STREAM_REASONING_TEXT: _cancel_reasoning = live_reasoning.get(stream_id, '') _cancel_tool_calls = _snap_tool_calls if _snap_tool_calls is not None else STREAM_LIVE_TOOL_CALLS.get(stream_id, []) if not _cancel_tool_calls: live_tools = getattr(_live_config, 'STREAM_LIVE_TOOL_CALLS', STREAM_LIVE_TOOL_CALLS) if live_tools is not STREAM_LIVE_TOOL_CALLS: _cancel_tool_calls = live_tools.get(stream_id, []) # Session cleanup outside STREAMS_LOCK to preserve lock ordering. # Acquire the per-session _agent_lock too, mirroring every other session # writer (streaming success/error paths, periodic checkpoint, POST endpoints) # so the cancel-path mutation races neither the checkpoint thread nor # concurrent undo/retry calls. if _cancel_session_id: with _get_session_agent_lock(_cancel_session_id): try: _cs = get_session(_cancel_session_id) if not isinstance(getattr(_cs, 'messages', None), list): _cs.messages = [] if not _stream_writeback_is_current(_cs, stream_id): # The stream has rotated to a different stream id (newer # turn started, or the worker already finalized this one). # Skip the cancel-marker append AND suppress the terminal # cancel event so we don't contradict a possibly-already- # delivered done payload (#2151 + #2154 / PR #2136). logger.info( "Skipping stale cancel writeback for session %s stream %s; active_stream_id=%s", _cancel_session_id, stream_id, getattr(_cs, 'active_stream_id', None), ) _emit_cancel_event = False return True # ── Preserve the user's typed message before clearing pending state (#1298) ── # The agent's internal messages list (where the user message was appended at # the start of run_conversation()) may not have been merged back into # _cs.messages yet — cancel_stream() races with the streaming thread's final # _merge_display_messages_after_agent_result() call. Without this guard, the # user's message is lost: pending_user_message gets cleared below, and # _cs.messages still only contains messages from prior turns. The reporter # of #1298 sees their typed text vanish from chat after clicking Stop. # # Recovery rule: if pending_user_message is set AND the latest message in # _cs.messages isn't already a matching user turn, synthesize one. The # match check guards against double-append when the streaming thread DID # reach its merge step before cancel_stream() got the session lock. # # Wrapped in its own try/except so an unexpected _cs.messages shape (e.g. # in unit tests using Mock sessions) cannot escape and skip the rest of # the cleanup. try: _pending_user = getattr(_cs, 'pending_user_message', None) _pending_source = getattr(_cs, 'pending_user_source', None) _pending_atts_raw = getattr(_cs, 'pending_attachments', None) _pending_atts = list(_pending_atts_raw) if isinstance(_pending_atts_raw, (list, tuple)) else [] _pending_started = getattr(_cs, 'pending_started_at', None) or 0 _msgs_for_recovery = _cs.messages if isinstance(_cs.messages, list) else None if _pending_user and _msgs_for_recovery is not None: _last_user = None for _m in reversed(_msgs_for_recovery): if isinstance(_m, dict) and _m.get('role') == 'user': _last_user = _m break _already_persisted = False if _last_user is not None: _last_content = _last_user.get('content') _last_ts = _last_user.get('timestamp') or 0 # Only treat as already-persisted if the latest user turn # was created AT OR AFTER the current turn's pending_started_at. # An earlier turn whose content happens to be a substring # (e.g. prior reply was "ok", user now types "ok please continue") # must NOT short-circuit synthesis — that would re-introduce # the data-loss bug this guard is supposed to prevent. if isinstance(_last_content, str) and _last_ts >= _pending_started: # Tolerate the workspace prefix the streaming thread prepends. if _pending_user == _last_content or _pending_user in _last_content: _already_persisted = True if not _already_persisted: _recovered_ts = int(time.time()) if isinstance(_pending_started, (int, float)) and _pending_started > 0: _recovered_ts = int(_pending_started) _user_turn: dict = { 'role': 'user', 'content': _pending_user, 'timestamp': _recovered_ts, } stamp_message_source(_user_turn, _pending_source) if _pending_atts: _user_turn['attachments'] = _pending_atts _msgs_for_recovery.append(_user_turn) except Exception: logger.debug( "Failed to recover pending user message on cancel for %s", _cancel_session_id, ) _cs.active_stream_id = None _cs.pending_user_message = None _cs.pending_attachments = [] _cs.pending_started_at = None _cs.pending_user_source = None # Persist any partial assistant text that was streamed before cancel (#893). # Preserving partial content means the user sees what the agent had # produced rather than losing it entirely. The marker is _partial=True # (for session/UI identification only) — NOT _error=True — so the partial # content IS kept in the history sent to the agent on the next user # message, letting the model continue from where it was cut off. # See the inner comment on the append call below for the rationale. # # #1361: Also persist reasoning trace and live tool calls that were # accumulated in thread-local variables but invisible to the cancel path. # This prevents paid-token data loss when cancelling mid-reasoning or # mid-tool-execution. # NOTE on _partial_tool_calls: the captured entries use the WebUI # internal shape {name, args, done, duration, is_error} — they do # NOT carry the OpenAI/Anthropic API id + function: {name, arguments} # envelope. Storing under 'tool_calls' would cause # _sanitize_messages_for_api to forward them to the next-turn LLM # call and strict providers would 400 on the malformed entries. # The underscore-prefixed key is not in the whitelist, so sanitize # strips it. The UI reads it via static/messages.js. (v0.50.251.) _partial_msg = _build_partial_message( _cancel_partial_text, _cancel_reasoning, _cancel_tool_calls, ) _cancel_marker_exists = _session_has_cancel_marker(_cs) _cancel_marker_idx = len(_cs.messages) if _cancel_marker_exists: for _idx in range(len(_cs.messages) - 1, -1, -1): _m = _cs.messages[_idx] if not isinstance(_m, dict) or _m.get('role') != 'assistant': continue _content = str(_m.get('content') or '').strip().lower() if any(pattern in _content for pattern in _CANCEL_MARKER_PATTERNS): _cancel_marker_idx = _idx break if _partial_msg is not None: # Deduplicate against the full partial payload, not just # non-empty content. Tool-only/reasoning-only partials have # empty content, so a content-gated check can append the same # failed turn repeatedly during cancel/replay recovery (#2592). if not _partial_marker_already_present( _cs.messages, _partial_msg, before_idx=_cancel_marker_idx, ): _cs.messages.insert(_cancel_marker_idx, _partial_msg) # Cancel marker — flagged _error=True so it is stripped from conversation # history on the next turn (prevents model from seeing "Task cancelled." # as a prior assistant reply). if not _cancel_marker_exists: _cs.messages.append({ 'role': 'assistant', 'content': _cancelled_turn_content( 'Task cancelled.', _preferred_agent_display_name_for_session(_cs), ), '_error': True, 'provider_details': 'Task cancelled.', 'provider_details_label': 'Cancellation details', 'timestamp': int(time.time()), }) _cs.save() _cancel_session_payload = _redacted_session_payload_with_full_messages(_cs) except Exception: logger.debug("Failed to clear session state on cancel for %s", _cancel_session_id) if _emit_cancel_event and q: _cancel_event_id = STREAM_LAST_EVENT_ID.get(stream_id) if _cancel_event_id and hasattr(q, "note_last_event_id"): try: q.note_last_event_id(_cancel_event_id) except Exception: logger.debug("Failed to note cancel event_id %s for stream %s", _cancel_event_id, stream_id, exc_info=True) try: _payload = _cancel_event_payload('Cancelled by user', session=_cancel_session_payload) q.put_nowait(('cancel', _payload)) except Exception: logger.debug("Failed to put cancel event to queue") return True