# NousResearch/hermes-agent · Stale editable finder refresh during update ## Open now - Nothing open on GitHub. A bug where hermes updates leave the venv editable finder stale, causing ModuleNotFoundError crashes, along with a fix that refreshes the finder before skipping reinstall. Related tests for linger and schtasks are also addressed. ## TL;DR - 3 member(s): 0 open issue(s), 1 closed issue(s), 0 open PR(s), 1 merged PR(s), 1 PR(s) closed without merging. - A fix was merged to main in PR #120127. Verify it addresses your case. - **Recommended:** read PR #120127 (and any competing PRs below) before filing. Confirm the merged change covers your symptom. ## Maintainer resolution verdict - Internal-tracker verdict over this cluster's issue members: `resolved` · deciding PR(s): `pr-120127`. This is distinct from the fix status above. Weigh it yourself. ## How this cluster was grouped - Link evidence (`STRUCTURAL_SPANNING`): every member is connected by structural evidence: shared changed files, or a closing/superseded reference. - Pairwise link density `0.67`: the share of member pairs that carry a link, counting a structural link in full and a relevance-only link at 0.36. `1.00` means every pair is linked. ## Closed / duplicate issues - #119466: \[Bug\] hermes update can leave the venv editable-finder stale — gateway crash-loops with ModuleNotFoundError: No module named 'hermes_platform' ## Prior & competing PRs - **#120127: hermes update: in-gateway cron updates no longer exit STALE, stale editable finder is refreshed, linger/schtasks tests fixed (#119597, #119466, #119786, #119845; salvage #119525 #119790 #119847)**. merged to main · fix tier `verified` - maintainer direction: category `merged` - files: `agent/agent_init.py`, `agent/relay_runtime.py`, `agent/turn_iteration_prep.py`, `agent/turn_overflow.py`, `agent/turn_preflight.py`, `agent/turn_tool_round.py`, `apps/desktop/src/app/contrib/hooks/use-session-tile-delegate.ts`, `apps/desktop/src/app/session/hooks/use-session-actions.test.tsx`, `apps/desktop/src/app/session/hooks/use-session-actions/index.ts`, `apps/desktop/src/app/session/hooks/use-session-actions/transcript-provenance.test.ts`, `apps/desktop/src/app/session/hooks/use-session-actions/transcript-provenance.ts`, `apps/desktop/src/app/session/hooks/use-session-state-cache.test.tsx`, `apps/desktop/src/app/session/hooks/use-session-state-cache.ts`, `apps/desktop/src/store/gateway-connection-lifecycle.test.ts`, `apps/desktop/src/store/gateway.ts`, `apps/desktop/src/store/session-states.test.ts`, `apps/desktop/src/store/session-states.ts`, `contributors/emails/[redacted:email]`, `gateway/AGENTS.md`, `gateway/config_env.py` (+68 more) - fix hunks: - `agent/agent_init.py` ```diff @@ -1256,7 +1256,7 @@ def _memory_provider_init_kwargs(agent, platform) -> Dict[str, Any]: return kwargs -def _init_memory(agent, _agent_cfg, skip_memory, platform): +def _init_memory(agent, _agent_cfg, skip_memory, platform, memory_manager=None): # Persistent memory (MEMORY.md + USER.md) — loaded from disk agent._memory_store = None agent._memory_enabled = False ``` ```diff @@ -1297,7 +1297,12 @@ def _init_memory(agent, _agent_cfg, skip_memory, platform): # External memory provider plugin (one at a time, alongside built-in): memory.provider. agent._memory_manager = None - if not skip_memory: + if memory_manager is not None and not skip_memory: + # A caller that rebuilds the agent per turn (gateway api_server) hands back the session's + # already-initialized manager: providers keep their prefetch/retain state across turns instead + # of being re-initialized (#120116). No initialize_all — the providers are already bound. + agent._memory_manager = memory_manager + elif not skip_memory: try: _mem_provider_name = mem_config.get("provider", "") if mem_config else "" if not is_core_memory_provider(_mem_provider_name): ``` ```diff @@ -2345,7 +2350,7 @@ def init_agent( checkpoint_max_snapshots: int = 20, checkpoint_max_total_size_mb: int = 500, checkpoint_max_file_size_mb: int = 10, pass_session_id: bool = False, requested_provider: str = None, capabilities: Optional[Dict[str, bool]] = None, cwd: Optional[str] = None, - side_agent: bool = False, + side_agent: bool = False, memory_manager=None, ): _install_safe_stdio() ``` ```diff @@ -2424,7 +2429,7 @@ def init_agent( _agent_cfg = {} _apply_display_config(agent, _agent_cfg, platform) - _init_memory(agent, _agent_cfg, skip_memory, platform) + _init_memory(agent, _agent_cfg, skip_memory, platform, memory_manager=memory_manager) _apply_agent_section(agent, _agent_cfg) cs = _parse_compression_config(agent, _agent_cfg) _config_context_length, _custom_providers, _effective_context_length, _model_cfg = _resolve_context_length( ``` - `agent/relay_runtime.py` ```diff @@ -177,8 +177,12 @@ def _load_segments_config() -> dict[str, Any]: on_compaction = False max_turns = 0 try: - from gateway.run import _load_gateway_config # late import - telemetry = (_load_gateway_config().get("gateway") or {}).get("telemetry") or {} + # Never import gateway.run here: its import-time env setup (_HERMES_GATEWAY, HERMES_QUIET, + # TERMINAL_CWD := home) rebinds a CLI/TUI/cron host — hung approvals (#87183), `hermes -z` + # running in $HOME without the launch dir's AGENTS.md (#95577). Same reader it delegates to. + from hermes_cli.config_effective import load_user_config_effective + + telemetry = (load_user_config_effective().get("gateway") or {}).get("telemetry") or {} segments = telemetry.get("session_segments") or {} on_compaction = bool(segments.get("on_compaction", False)) try: ``` - `agent/turn_iteration_prep.py` ```diff @@ -22,6 +22,21 @@ logger = logging.getLogger("agent.conversation_loop") + +def _anchors_current_turn(messages: Any, idx: Any, user_message: Any) -> bool: + """True when ``messages[idx]`` is this turn's user row (verbatim, or its user-originated view).""" + if not isinstance(idx, int) or not 0 <= idx < len(messages): + return False + msg = messages[idx] + if not (isinstance(msg, dict) and msg.get("role") == "user"): + return False + if msg.get("content") == user_message: + return True + from agent.context_compressor import user_originated_turn_view + + view = user_originated_turn_view(msg) + return view is not None and view.get("content") == user_message + ITERATION_BUDGET_WARNING_TEMPLATE = ( "[SYSTEM NOTICE — iteration budget checkpoint] You have used {used} of {maximum} " "iterations. Checkpoint durable progress now, then continue the task; do not stop " ``` ```diff @@ -205,6 +220,18 @@ def _is_scaffold_ghost(msg: Dict[str, Any]) -> bool: current_turn_user_idx, _reanchored_idx, agent.session_id or "-", ) current_turn_user_idx = _reanchored_idx + # Mid-turn compaction (post-tool gate, overflow restart, recovery) rebuilds ``messages`` without + # handing back a new index. A stale index splits the request's replay prefix inside this turn's + # tool rows: prefix canonicalization then drops the assistant tool_call whose result fell past the + # split, the orphaned result is sanitized away, and the model silently loses tool output that + # state.db still holds. A valid index always lands on this turn's user row; re-anchor otherwise. + if user_message is not None and not _anchors_current_turn(messages, current_turn_user_idx, user_message): + _reanchored_idx = _reanchor(agent, messages, user_message) + request_logger.info( + "Re-anchored stale current_turn_user_idx %s -> %s (session=%s)", + current_turn_user_idx, _reanchored_idx, agent.session_id or "-", + ) + current_turn_user_idx = _reanchored_idx return IterationPrep( action="fallthrough", messages=messages, request_logger=request_logger, current_turn_user_idx=current_turn_user_idx, ``` - `agent/turn_overflow.py` ```diff @@ -23,7 +23,7 @@ from agent.error_classifier import FailoverReason from agent.message_sanitization import serialized_messages_bytes from agent.model_metadata import ( - get_context_length_from_provider_error, is_output_cap_error, + get_context_length_from_provider_error, is_local_endpoint, is_output_cap_error, parse_available_output_tokens_from_error, ) from agent.turn_failure_copy import site_copy, stamp_failure ``` ```diff @@ -402,11 +402,14 @@ def _recover_context_length(st: _Recovery, _retry: TurnRetryState, error_msg: st # request — a background review from an earlier session — holds their context. Name that, # keep the turn retryable and transient: no "conversation too long", no gateway auto-reset. # Only when the server quoted NO measurement of its own: "prompt is too long: 233153 tokens - # > 200000" is the server's count and beats the local estimate. + # > 200000" is the server's count and beats the local estimate. Local endpoints only: a hosted + # route has no shared slot, so the same rejection means its real window is smaller than the one + # Hermes assumes, and "wait and /retry" would fail identically forever; compress instead. window = agent.context_compressor.context_length request_tokens = st.request_tokens() + max(0, int(getattr(agent, "max_tokens", 0) or 0)) if ( - not re.search(r"\d{4,}", error_msg) + is_local_endpoint(agent.base_url) + and not re.search(r"\d{4,}", error_msg) and isinstance(window, int) and window > 0 and request_tokens < window * _UNEXPLAINED_REJECTION_FRACTION ): ``` - `agent/turn_preflight.py` ```diff @@ -22,7 +22,7 @@ from agent.turn_context import _review_fork_first_request_pending from agent.turn_context_compaction import ( _apply_grown_window, _blocked_compress_reason, _clear_overflow_warn, _refund_api_call, - _reset_retry_state_after_compaction, + _reanchor, _reset_retry_state_after_compaction, ) logger = logging.getLogger("agent.conversation_loop") ``` ```diff @@ -238,13 +238,14 @@ class PostToolCompressionVerdict: compression_attempts: int final_response: Any turn_exit_reason: Any + current_turn_user_idx: int def compress_after_tool_results( agent: Any, *, messages: List[Dict[str, Any]], system_message: Any, user_message: Any, active_system_prompt: Any, conversation_history: Any, compression_attempts: int, max_compression_attempts: int, effective_task_id: Any, final_response: Any, - turn_exit_reason: Any, + turn_exit_reason: Any, current_turn_user_idx: int, ) -> PostToolCompressionVerdict: """Post-tool-call compression decision. Pressure comes from API-reported ``prompt_tokens`` (a tight lower bound; thinking models inflate completion tokens), ``` ```diff @@ -263,6 +264,7 @@ def _verdict(end_turn: bool) -> PostToolCompressionVerdict: end_turn=end_turn, messages=messages, active_system_prompt=active_system_prompt, conversation_history=conversation_history, compression_attempts=compression_attempts, final_response=final_response, turn_exit_reason=turn_exit_reason, + current_turn_user_idx=current_turn_user_idx, ) _compressor = agent.context_compressor ``` ```diff @@ -355,6 +357,7 @@ def _verdict(end_turn: bool) -> PostToolCompressionVerdict: final_response = _HANDOFF_SKIP_FINAL_RESPONSE turn_exit_reason = "compaction_handoff_not_actionable" return _verdict(True) + current_turn_user_idx = _reanchor(agent, messages, user_message) elif agent.compression_enabled: # Over threshold but compression blocked (cooldown/anti-thrash): deduped # warning so context can't silently overflow. ``attempts_spent`` names the ``` - `agent/turn_tool_round.py` ```diff @@ -39,6 +39,7 @@ class ToolRoundVerdict: failed: Any _turn_exit_reason: Any truncated_tool_call_retries: Any + current_turn_user_idx: Any result: Optional[Dict[str, Any]] = None ``` ```diff @@ -47,7 +48,7 @@ def run_tool_round( conversation_history: Any, api_call_count: Any, effective_task_id: Any, user_message: Any, system_message: Any, active_system_prompt: Any, compression_attempts: Any, max_compression_attempts: Any, final_response: Any, failed: Any, _turn_exit_reason: Any, - truncated_tool_call_retries: Any, + truncated_tool_call_retries: Any, current_turn_user_idx: Any, ) -> ToolRoundVerdict: """Execute one tool round in the exact original order. Persist-before-execute is a durability invariant: resume must see the executed block if a destructive tool restarts ``` ```diff @@ -60,7 +61,8 @@ def _verdict(action: str, result: Optional[Dict[str, Any]] = None) -> ToolRoundV action=action, messages=messages, conversation_history=conversation_history, active_system_prompt=active_system_prompt, compression_attempts=compression_attempts, final_response=final_response, failed=failed, _turn_exit_reason=_turn_exit_reason, - truncated_tool_call_retries=truncated_tool_call_retries, result=result, + truncated_tool_call_retries=truncated_tool_call_retries, + current_turn_user_idx=current_turn_user_idx, result=result, ) if not agent.quiet_mode: ``` ```diff @@ -191,6 +193,7 @@ def _verdict(action: str, result: Optional[Dict[str, Any]] = None) -> ToolRoundV compression_attempts=compression_attempts, max_compression_attempts=max_compression_attempts, effective_task_id=effective_task_id, final_response=final_response, turn_exit_reason=_turn_exit_reason, + current_turn_user_idx=current_turn_user_idx, ) messages = _ptc.messages active_system_prompt = _ptc.active_system_prompt ``` ```diff @@ -198,6 +201,7 @@ def _verdict(action: str, result: Optional[Dict[str, Any]] = None) -> ToolRoundV compression_attempts = _ptc.compression_attempts final_response = _ptc.final_response _turn_exit_reason = _ptc.turn_exit_reason + current_turn_user_idx = _ptc.current_turn_user_idx if _ptc.end_turn: return _verdict("break") ``` - `apps/desktop/src/app/contrib/hooks/use-session-tile-delegate.ts` ```diff @@ -218,6 +218,11 @@ export function useSessionTileDelegate({ } } }, + dropRuntimeBindings: storedSessionIds => { + for (const storedSessionId of storedSessionIds) { + runtimeIdByStoredSessionIdRef.current.delete(storedSessionId) + } + }, // Reconnect reconcile (#93059): retire an orphaned runtime's busy claim // through updateSessionState so the cache, focused view, busyRef and // tile mirrors settle together. A runtime this cache never held reports ``` - `apps/desktop/src/app/session/hooks/use-session-actions.test.tsx` ```diff @@ -93,6 +93,8 @@ import { NEW_CHAT_ROUTE, sessionRoute } from '../../routes' import type { ClientSessionState } from '../../types' import { useSessionActions } from './use-session-actions' +import { suppressTranscriptForView, transcriptRowContentKey } from './use-session-actions/transcript-provenance' +import type { TranscriptViewCutoff } from './use-session-actions/transcript-provenance' import { useSessionStateCache } from './use-session-state-cache' vi.mock('@/hermes', async importOriginal => ({ ``` ```diff @@ -992,6 +994,10 @@ function ResumeHarness({ const ref = (value: T): MutableRefObject => ({ current: value }) const runtimeMapRef = runtimeIdByStoredSessionIdRef ?? ref(new Map()) const stateMapRef = sessionStateByRuntimeIdRef ?? ref(new Map()) + // Stand in for the real hook's gate: without it this stub would publish the + // unproven warm cache before REST authority lands. Kept here (not per-test) + // so every resume through this harness sees one shared suppression policy. + const heldCutoffsByRuntimeIdRef = ref(new Map()) const actions = useSessionActions({ activeSessionId: null, ``` ```diff @@ -1008,7 +1014,25 @@ function ResumeHarness({ selectedStoredSessionId, selectedStoredSessionIdRef: ref(selectedStoredSessionId), sessionStateByRuntimeIdRef: stateMapRef, - syncSessionStateToView: (sessionId, state) => onViewSync?.(sessionId, state), + holdSessionTranscriptView: runtimeId => { + const rows = stateMapRef.current.get(runtimeId)?.messages ?? [] + + const cutoff: TranscriptViewCutoff = { + cutoffIds: new Set(rows.map(message => message.id)), + cutoffKeys: new Set(rows.map(transcriptRowContentKey)) + } + + heldCutoffsByRuntimeIdRef.current.set(runtimeId, cutoff) + + return () => { + heldCutoffsByRuntimeIdRef.current.delete(runtimeId) + } + }, + syncSessionStateToView: (sessionId, state) => { + const cutoff = heldCutoffsByRuntimeIdRef.current.get(sessionId) ?? null + + onViewSync?.(sessionId, suppressTranscriptForView(state, cutoff)) + }, updateSessionState: (sessionId, updater, storedSessionId) => { // Full default shape (not a bare {} cast) so seeded/derived fields like // turnStartedAt behave as in production state updates. ``` - `apps/desktop/src/app/session/hooks/use-session-actions/index.ts` ```diff @@ -147,7 +147,6 @@ import { projectPendingConnection, restorePendingConnectionFromSnapshot } from ' import { createPersistedDisplayTranscriptProvenance, hasPersistedDisplayTranscriptProvenance, - suppressTranscriptForView, withoutTranscriptProvenance } from './transcript-provenance' import { ``` ```diff @@ -1244,14 +1243,16 @@ export function useSessionActions({ dropSessionState(cachedRuntimeId) } else { // Bind the warm runtime immediately so cwd/workspace ownership don't - // wait on session.activate (#71254). Unproven cache entries (no - // persisted-display provenance) stay off the view until REST - // authority lands — a compressed runtime tail is legal in cache and - // is exactly the session-switch flicker (#73646). Proven caches and - // same-session re-resumes still paint immediately. The persisted - // refresh itself still starts after activate reattaches the live - // transport, so a turn finishing between snapshot and reattach - // cannot leave a stale partial on screen. + // wait on session.activate (#71254). The armed transcript gate is the + // single suppression authority: it hides only the unproven cached + // prefix (rows captured at arm time, including a compressed runtime + // tail — exactly the session-switch flicker, #73646) while rows that + // arrive live during the hold still paint (#117867). Proven caches + // and same-session re-resumes never arm the gate, so they paint + // immediately. The persisted refresh itself still starts after + // activate reattaches the live transport, so a turn finishing + // between snapshot and reattach cannot leave a stale partial on + // screen. const shouldRefreshPersistedTranscript = !isWatchWindow() const suppressUnprovenWarmTranscript = ``` ```diff @@ -1277,10 +1278,7 @@ export function useSessionActions({ selectedStoredSessionIdRef.current = storedSessionId setActiveSessionId(cachedRuntimeId) activeSessionIdRef.current = cachedRuntimeId - syncSessionStateToView( - cachedRuntimeId, - suppressTranscriptForView(cachedViewState, suppressUnprovenWarmTranscript) - ) + syncSessionStateToView(cachedRuntimeId, cachedViewState) setCurrentCwdTransient(cachedViewState.cwd) // The warm cache IS this conversation's own workspace truth, so the // switch is already re-homed here. This claim cannot wait for ``` ```diff @@ -1433,10 +1431,7 @@ export function useSessionActions({ busyRef.current = running setBusy(running) setAwaitingResponse(running && !pendingClarify) - syncSessionStateToView( - cachedRuntimeId, - suppressTranscriptForView(activatedLivenessState, suppressUnprovenWarmTranscript) - ) + syncSessionStateToView(cachedRuntimeId, activatedLivenessState) // session.activate is the ordering barrier for reconnect recovery: // it atomically rebinds a running turn before returning. If the ``` - `apps/desktop/src/app/session/hooks/use-session-actions/transcript-provenance.test.ts` ```diff @@ -52,8 +52,26 @@ describe('transcript provenance', () => { const state = createClientSessionState('stored-1') state.messages = [{ id: 'u1', role: 'user', parts: [{ type: 'text', text: 'hi' }] }] - expect(suppressTranscriptForView(state, false)).toBe(state) - expect(suppressTranscriptForView(state, true).messages).toEqual([]) + // No gate held: the state reaches the view verbatim. + expect(suppressTranscriptForView(state, null)).toBe(state) + + // Gate held with no captured prefix: fail-closed, everything hides. + expect(suppressTranscriptForView(state, { cutoffIds: new Set() }).messages).toEqual([]) expect(state.messages).toHaveLength(1) }) + + it('drops only the cached prefix and keeps rows appended after the arm (#117867)', () => { + const state = createClientSessionState('stored-1') + state.messages = [ + { id: 'cached-1', role: 'user', parts: [{ type: 'text', text: 'old' }] }, + { id: 'live-1', role: 'assistant', parts: [{ type: 'text', text: 'streaming' }] } + ] + + const suppressed = suppressTranscriptForView(state, { cutoffIds: new Set(['cached-1']) }) + + expect(suppressed.messages.map(message => message.id)).toEqual(['live-1']) + + // Nothing else in the state is touched, and the input is not mutated. + expect(state.messages).toHaveLength(2) + }) }) ``` - `apps/desktop/src/app/session/hooks/use-session-actions/transcript-provenance.ts` ```diff @@ -1,3 +1,5 @@ +import type { ChatMessage } from '@/lib/chat-messages' + import type { ClientSessionState, PersistedDisplayTranscriptProvenance } from '../../../types' export type TranscriptProvenanceScope = ``` ```diff @@ -60,10 +62,56 @@ export function invalidatePersistedDisplayTranscriptAuthority(state: ClientSessi } } -export function suppressTranscriptForView(state: ClientSessionState, suppress: boolean): ClientSessionState { - if (!suppress || state.messages.length === 0) { +export interface TranscriptViewCutoff { + cutoffIds: ReadonlySet + // Content fingerprints of the arm-time rows (optional). Compaction + // re-sequences the cached tail with FRESH row ids mid-hold + // (archive_and_compact: "consumers that reference durable row ids + // re-resolve by content"), so an id-only cutoff would pass the whole + // re-sequenced cached prefix as if it were live and paint the exact + // compressed tail the hold exists to hide (#73646 via #117867). + cutoffKeys?: ReadonlySet +} + +// Volatile-free content fingerprint: role + text of text parts, JSON of the +// rest. Deliberately ignores row ids and per-row timestamps so a re-sequenced +// copy of the same content fingerprints identically. Ceiling: a genuinely new +// row whose content is byte-identical to an arm-time row stays hidden until +// the hold releases (bounded by the REST window). +export function transcriptRowContentKey(message: ChatMessage): string { + return `${message.role}:${ + (message.parts ?? []) + .map(part => + 'text' in part && typeof part.text === 'string' ? `${part.type}:${part.text}` : JSON.stringify(part) + ) + .join('|') + }` +} + +export function suppressTranscriptForView( + state: ClientSessionState, + cutoff: TranscriptViewCutoff | null +): ClientSessionState { + if (cutoff === null) { + return state + } + + if (cutoff.cutoffIds.size === 0) { + // Fail-closed: the gate was armed before any cached row existed, so there + // is no unproven prefix to hide selectively — everything stays off the + // view until REST authority lands (#73646). + return { ...state, messages: [] } + } + + const messages = state.messages.filter( + message => + !cutoff.cutoffIds.has(message.id) && + !(cutoff.cutoffKeys?.has(transcriptRowContentKey(message)) ?? false) + ) + + if (messages.length === state.messages.length) { return state } - return { ...state, messages: [] } + return { ...state, messages } } ``` - `apps/desktop/src/app/session/hooks/use-session-state-cache.test.tsx` ```diff @@ -742,6 +742,169 @@ describe('useSessionStateCache — cross-thread error isolation', () => { }) }) +// #117867: the warm-resume transcript gate used to empty the view entirely +// while held, so rows arriving LIVE during the hold (the user's in-flight turn) +// never painted until REST authority landed. The gate now hides only the cached +// prefix captured when the hold was armed. +describe('useSessionStateCache — held transcript gate keeps live rows (#117867)', () => { + const runtime = 'hold-runtime' + const stored = 'hold-stored' + + beforeEach(() => { + clearAllSessionStates() + setActiveSessionId(runtime) + }) + + afterEach(() => { + cleanup() + $messages.set([]) + clearAllSessionStates() + setActiveSessionId(null) + }) + + it('paints rows appended after the arm and hides only the cached prefix', () => { + let cache!: Cache + let release: (() => void) | undefined + + render( (cache = value)} />) + + // Seed the warm cache entry with the unproven cached prefix — exactly what + // resumeSession has in hand when it arms the hold. + act(() => { + cache.updateSessionState( + runtime, + state => ({ + ...state, + messages: [userMessage('cached-1', 'cached prompt'), assistantText('cached-2', 'cached tail')] + }), + stored + ) + }) + + // The real production arm path, after the cache entry exists. + act(() => { + release = cache.holdSessionTranscriptView(runtime) + }) + + // A live row arrives mid-hold. busy:false forces the critical-transition + // sync flush (no rAF), so $messages is observable here. + act(() => { + cache.updateSessionState( + runtime, + state => ({ + ...state, + busy: false, + messages: [ + userMessage('cached-1', 'cached prompt'), + assistantText('cached-2', 'cached tail'), + userMessage('live-1', 'live turn') + ] + }), + stored + ) + }) + + const ids = $messages.get().map(message => message.id) + + expect(ids).toContain('live-1') + expect(ids).not.toContain('cached-1') + expect(ids).not.toContain('cached-2') + + // Releasing restores the full transcript. + act(() => release?.()) + act(() => { + cache.updateSessionState( + runtime, + state => ({ ...state, messages: state.messages }), + stored + ) + }) + + expect($messages.get().map(message => message.id)).toEqual(['cached-1', 'cached-2', 'live-1']) + }) + + it('fails closed when the hold was armed with no cached prefix', () => { + let cache!: Cache + let release: (() => void) | undefined + + render( (cache = value)} />) + + act(() => { + release = cache.holdSessionTranscriptView(runtime) + }) + + act(() => { + cache.updateSessionState( + runtime, + state => ({ + ...state, + busy: false, + messages: [userMessage('late-1', 'arrived after the arm')] + }), + stored + ) + }) + + // No baseline was captured, so nothing is trustworthy yet: today's + // hide-everything behavior is preserved until REST authority lands. + expect($messages.get()).toEqual([]) + + act(() => release?.()) + }) + + it('keeps a re-sequenced cached tail hidden when compaction assigns fresh ids mid-hold (#117867)', () => { + let cache!: Cache + let release: (() => void) | undefined + + render( (cache = value)} />) + + act(() => { + cache.updateSessionState( + runtime, + state => ({ + ...state, + messages: [userMessage('cached-1', 'cached prompt'), assistantText('cached-2', 'cached tail')] + }), + stored + ) + }) + + act(() => { + release = cache.holdSessionTranscriptView(runtime) + }) + + // A mid-hold compaction re-sequences the cached tail with FRESH row ids + // while keeping the content (archive_and_compact contract: consumers that + // reference durable row ids re-resolve by content). An id-only cutoff + // would pass `seq-*` through as if it were live and paint the compressed + // tail the hold exists to hide (#73646); the content fingerprint keeps it + // hidden while a genuinely new row still paints. + act(() => { + cache.updateSessionState( + runtime, + state => ({ + ...state, + busy: false, + messages: [ + userMessage('seq-71', 'cached prompt'), + assistantText('seq-72', 'cached tail'), + userMessage('live-1', 'live turn') + ] + }), + stored + ) + }) + + const ids = $messages.get().map(message => message.id) + + expect(ids).toContain('live-1') + expect(ids).not.toContain('seq-71') + expect(ids).not.toContain('seq-72') + + act(() => release?.()) + }) +}) + // #93059: reconnect used to downgrade the $sessionStates mirror only, leaving // this cache (which warm resume ORs over `running: false`) still busy. describe('useSessionStateCache — reconnect busy reconcile (#93059)', () => { ``` - `apps/desktop/src/app/session/hooks/use-session-state-cache.ts` ```diff @@ -35,10 +35,20 @@ import { SessionStateCache } from '../session-state-cache' import { invalidatePersistedDisplayTranscriptAuthority, - suppressTranscriptForView + suppressTranscriptForView, + transcriptRowContentKey } from './use-session-actions/transcript-provenance' import { chatMessageArraysEquivalent } from './use-session-actions/utils' +// A held transcript gate hides only the unproven CACHED prefix (the ids +// captured when the hold was armed) — rows that arrive live during the hold +// still paint, which is the point of the fix (#117867). +interface TranscriptViewGate { + cutoffIds: ReadonlySet + cutoffKeys: ReadonlySet + token: symbol +} + interface SessionStateCacheOptions { activeSessionId: string | null busyRef: MutableRefObject ``` ```diff @@ -143,7 +153,7 @@ export function useSessionStateCache({ const sessionStateCache = sessionStateByRuntimeIdRef.current const pendingViewStateRef = useRef<{ sessionId: string; state: ClientSessionState } | null>(null) const viewSyncRafRef = useRef(null) - const transcriptViewGateByRuntimeIdRef = useRef(new Map()) + const transcriptViewGateByRuntimeIdRef = useRef(new Map()) // Runtime id whose transcript currently occupies `$messages` — lets the // flush below tell a same-session refresh from a thread switch. const viewSessionIdRef = useRef(null) ``` ```diff @@ -226,16 +236,25 @@ export function useSessionStateCache({ } }, []) - const holdSessionTranscriptView = useCallback((runtimeId: string): (() => void) => { - const token = [redacted:secret-assignment] - transcriptViewGateByRuntimeIdRef.current.set(runtimeId, token) + const holdSessionTranscriptView = useCallback( + (runtimeId: string): (() => void) => { + const token = [redacted:secret-assignment] + const cached = sessionStateCache.get(runtimeId) - return () => { - if (transcriptViewGateByRuntimeIdRef.current.get(runtimeId) === token) { - transcriptViewGateByRuntimeIdRef.current.delete(runtimeId) + transcriptViewGateByRuntimeIdRef.current.set(runtimeId, { + cutoffIds: new Set((cached?.messages ?? []).map(message => message.id)), + cutoffKeys: new Set((cached?.messages ?? []).map(transcriptRowContentKey)), + token + }) + + return () => { + if (transcriptViewGateByRuntimeIdRef.current.get(runtimeId)?.token === token) { + transcriptViewGateByRuntimeIdRef.current.delete(runtimeId) + } } - } - }, []) + }, + [sessionStateCache] + ) const flushPendingViewState = useCallback(() => { const pending = pendingViewStateRef.current ``` ```diff @@ -300,7 +319,8 @@ export function useSessionStateCache({ return } - const viewState = suppressTranscriptForView(state, transcriptViewGateByRuntimeIdRef.current.has(sessionId)) + const gate = transcriptViewGateByRuntimeIdRef.current.get(sessionId) + const viewState = suppressTranscriptForView(state, gate ?? null) syncRuntimeMetadataToView(viewState) pendingViewStateRef.current = { sessionId, state: viewState } ``` - `apps/desktop/src/store/gateway-connection-lifecycle.test.ts` ```diff @@ -31,6 +31,7 @@ const gatewayMocks = vi.hoisted(() => { const reconnectStateMocks = vi.hoisted(() => ({ reconcileBusyStatesOnReconnect: vi.fn(), + resetRouteOwnedTileRuntimeBindings: vi.fn(), resetTileRuntimeBindings: vi.fn() })) ``` ```diff @@ -352,11 +353,14 @@ describe('secondary reconnect runtime scope', () => { const reopening = openGatewayForAgent('homelab', 'writer') await vi.waitFor(() => expect(gatewayMocks.connect).toHaveBeenCalledTimes(2)) - expect(reconnectStateMocks.resetTileRuntimeBindings).toHaveBeenCalledWith({ + // Not the window's ambient gateway: only tiles owned by this route can hold + // its runtime ids, so the reset is route-owned rather than window-wide. + expect(reconnectStateMocks.resetRouteOwnedTileRuntimeBindings).toHaveBeenCalledWith({ connectionId: 'homelab', profile: 'writer' }) - expect(reconnectStateMocks.resetTileRuntimeBindings.mock.invocationCallOrder[0]).toBeLessThan( + expect(reconnectStateMocks.resetTileRuntimeBindings).not.toHaveBeenCalled() + expect(reconnectStateMocks.resetRouteOwnedTileRuntimeBindings.mock.invocationCallOrder[0]).toBeLessThan( gatewayMocks.connect.mock.invocationCallOrder[1] ) ``` ```diff @@ -364,6 +368,30 @@ describe('secondary reconnect runtime scope', () => { await reopening }) + it('does not re-resume unrelated tiles when a background request lease reopens its route', async () => { + // The Bot relay drains every registered connection on a 30s tick through + // requestGatewayForAgent. A local route is exempt from relay retention, so + // each tick dials a fresh socket and disposes it after the RPC. Treating + // every such reopen as a window-wide backend restart dropped every open + // tile's runtime binding, remounting its composer (caret reset, layout + // shift, model pick reverted) every 30 seconds. + installDesktop({ + getConnectionFor: vi.fn(async ({ connectionId, profile }) => descriptorFor(connectionId, profile)) + }) + + await requestGatewayForAgent('local', 'writer', 'bot_relay.outbox.drain') + expect(gatewayMocks.instances[0]?.close).toHaveBeenCalledOnce() + + await requestGatewayForAgent('local', 'writer', 'bot_relay.outbox.drain') + + expect(gatewayMocks.connect).toHaveBeenCalledTimes(2) + expect(reconnectStateMocks.resetTileRuntimeBindings).not.toHaveBeenCalled() + expect(reconnectStateMocks.resetRouteOwnedTileRuntimeBindings).toHaveBeenCalledWith({ + connectionId: 'local', + profile: 'writer' + }) + }) + it('rebinds only Bot runtimes owned by the reconnected profile route', async () => { installDesktop({ getConnectionFor: vi.fn(async ({ connectionId, profile }) => descriptorFor(connectionId, profile)) ``` - `apps/desktop/src/store/gateway.ts` ```diff @@ -696,12 +696,21 @@ async function openSecondary(entry: Secondary, spawnPriority: SpawnPriority = 'b if (reopening) { try { - const { reconcileBusyStatesOnReconnect, resetTileRuntimeBindings } = await import('@/store/session-states') + const { reconcileBusyStatesOnReconnect, resetRouteOwnedTileRuntimeBindings, resetTileRuntimeBindings } = + await import('@/store/session-states') + + const scope = { connectionId: entry.connectionId || 'local', profile: entry.profile } + + // Only the window's ambient gateway carries un-owned tiles and the main + // thread. A background route (e.g. a relay request lease that disposed + // its socket after the last tick) can only have minted runtimes for + // tiles that name it as their owner route. + if (g.activeKey === entry.scope) { + resetTileRuntimeBindings(scope) + } else { + resetRouteOwnedTileRuntimeBindings(scope) + } - resetTileRuntimeBindings({ - connectionId: entry.connectionId || 'local', - profile: entry.profile - }) reconcileBusyAfterOpen = () => reconcileBusyStatesOnReconnect(entry.scope) } catch { // Best effort for partial test/HMR graphs. Production always loads the ``` - `apps/desktop/src/store/session-states.test.ts` ```diff @@ -43,6 +43,7 @@ import { recordSessionEventScope, releaseSessionTranscript, requestForOwnedSession, + resetRouteOwnedTileRuntimeBindings, resetTileRuntimeBindings, selectionHomesToWorkspace, type SessionTileDelegate, ``` ```diff @@ -261,6 +262,58 @@ describe('resetTileRuntimeBindings', () => { }) }) +describe('resetRouteOwnedTileRuntimeBindings', () => { + afterEach(() => { + $sessionTiles.set([]) + }) + + it('drops only tiles owned by the reopened route and leaves ambient tiles bound', () => { + const invalidateRuntimeBindings = vi.fn() + const dropRuntimeBindings = vi.fn() + setSessionTileDelegate({ dropRuntimeBindings, invalidateRuntimeBindings } as unknown as SessionTileDelegate) + $sessionTiles.set([ + { + ownerRoute: { connectionId: 'local', mode: 'local', profile: 'writer', targetProfile: 'writer' }, + runtimeId: 'runtime-writer-bot', + storedSessionId: 'stored-writer-bot', + workspaceMode: 'bots' + }, + { + ownerRoute: { connectionId: 'local', mode: 'local', profile: 'coder', targetProfile: 'coder' }, + runtimeId: 'runtime-coder-bot', + storedSessionId: 'stored-coder-bot', + workspaceMode: 'bots' + }, + { runtimeId: 'runtime-ambient', storedSessionId: 'stored-ambient' } + ]) + + resetRouteOwnedTileRuntimeBindings({ connectionId: 'local', profile: 'writer' }) + + const [writerBot, coderBot, ambient] = $sessionTiles.get() + + expect(writerBot).toMatchObject({ storedSessionId: 'stored-writer-bot' }) + expect(writerBot).not.toHaveProperty('runtimeId') + expect(coderBot).toMatchObject({ runtimeId: 'runtime-coder-bot' }) + expect(ambient).toMatchObject({ runtimeId: 'runtime-ambient' }) + expect(dropRuntimeBindings).toHaveBeenCalledWith(new Set(['stored-writer-bot'])) + expect(invalidateRuntimeBindings).not.toHaveBeenCalled() + }) + + it('is a no-op when no open tile belongs to the reopened route', () => { + const invalidateRuntimeBindings = vi.fn() + const dropRuntimeBindings = vi.fn() + setSessionTileDelegate({ dropRuntimeBindings, invalidateRuntimeBindings } as unknown as SessionTileDelegate) + const tiles = [{ runtimeId: 'runtime-ambient', storedSessionId: 'stored-ambient' }] + $sessionTiles.set(tiles) + + resetRouteOwnedTileRuntimeBindings({ connectionId: 'local', profile: 'writer' }) + + expect($sessionTiles.get()).toBe(tiles) + expect(dropRuntimeBindings).not.toHaveBeenCalled() + expect(invalidateRuntimeBindings).not.toHaveBeenCalled() + }) +}) + describe('SessionTile workspace scope', () => { afterEach(() => { $activeGatewayProfile.set('default') ``` - `apps/desktop/src/store/session-states.ts` ```diff @@ -1527,6 +1527,43 @@ export function resetTileRuntimeBindings( } } +/** Reset for a pooled secondary route that reopened while it was NOT the + * window's ambient gateway. Only tiles whose exact owner route names that + * runtime can hold ids it minted; un-owned tiles and the main thread ride the + * ambient socket, whose own reconnect path runs `resetTileRuntimeBindings`. + * Background request leases (the Bot relay drain) reopen such a route every + * tick, and a window-wide reset there re-resumed every open tile each time — + * remounting its composer (caret reset, layout shift, model pick reverted). */ +export function resetRouteOwnedTileRuntimeBindings(scope: RuntimeReconnectScope) { + const connectionId = scope.connectionId.trim() + const profile = scope.profile?.trim() || null + const tiles = $sessionTiles.get() + + const ownedStoredIds = new Set( + tiles + .filter(tile => { + const route = tile.ownerRoute + + return ( + Boolean(connectionId) && + route?.connectionId === connectionId && + (!profile || (route.targetProfile || route.profile) === profile) + ) + }) + .map(tile => tile.storedSessionId) + ) + + if (ownedStoredIds.size === 0) { + return + } + + sessionTileDelegate()?.dropRuntimeBindings?.(ownedStoredIds) + + if (tiles.some(tile => tile.runtimeId && ownedStoredIds.has(tile.storedSessionId))) { + $sessionTiles.set(tiles.map(tile => (ownedStoredIds.has(tile.storedSessionId) ? toStored(tile) : tile))) + } +} + /** Unbind ONE reclaimed runtime from whichever tile holds it — the targeted * sibling of resetTileRuntimeBindings. The reconnect-time reset can't cover a * backend reclaim: the WS re-dials immediately, but the orphan reaper fires a ``` ```diff @@ -1569,6 +1606,11 @@ export interface SessionTileDelegate { * warm path re-binds tiles to dead runtime ids (the sleep/wake "empty * right pane" bug). Bindings re-record from live post-reconnect events. */ invalidateRuntimeBindings?(preserveStoredSessionIds?: ReadonlySet): void + /** Drop ONLY these stored→runtime bindings from the wiring cache — the + * route-scoped twin of invalidateRuntimeBindings for a reopened secondary + * (`resetRouteOwnedTileRuntimeBindings`). Bindings for every other stored + * session, including the main thread, stay warm. */ + dropRuntimeBindings?(storedSessionIds: ReadonlySet): void /** Bind a live runtime id for a stored session (resume without touching * the main view). Returns the runtime id, or throws. * `refreshTranscript` forces a REST merge even when a warm cached ``` - `contributors/emails/[redacted:email]` ```diff @@ -0,0 +1 @@ +a1398394385 ``` - `gateway/AGENTS.md` ```diff @@ -210,6 +210,13 @@ gateway under the backend, and do NOT "fix" update locks by widening the tree-ki Resolve the owning home from the session record (`profile_home`, `agent::` key), never from `os.environ`, which holds the launch profile. Why: eviction that flushed under the launch scope wrote a secondary profile's memories into the default profile's store, silently. +- **`api_server` rebuilds the agent per request but not the memory provider.** The adapter bypasses + `TurnRunner` and the agent cache (per-request callbacks, model route, ephemeral prompt), so + `platforms/api_server_memory_sessions.py` parks each session's initialised `MemoryManager` between + requests (exclusive check-out in `_create_agent`, check-in in the turn's `finally`, keyed by profile + home + `agent.session_id`; idle/LRU eviction shuts down under the owning home) and + `AIAgent(memory_manager=...)` adopts it without a second provider init. Without it the previous + turn's queued recall never reaches the next request (#120116). - **`multiplex_profiles: false` is not "no scope ever".** A native hosted room serving a second profile flips the process-wide guard (`tui_gateway/launch_profile_policy.py:: activate_multi_profile_hosting`) inside the gateway process, after the adapters were wired; every ``` - `gateway/config_env.py` ```diff @@ -316,9 +316,15 @@ def _api_server(config: GatewayConfig) -> None: def _webhook(config: GatewayConfig) -> None: - if is_truthy_value(getenv("WEBHOOK_ENABLED")): - extra = _enable_from_env(config, Platform.WEBHOOK, pop_marker=True, warn=False).extra - _env_extras(extra, (("port", "WEBHOOK_PORT", _INT), ("secret", "WEBHOOK_SECRET"))) + enabled = is_truthy_value(getenv("WEBHOOK_ENABLED")) + if not (enabled or Platform.WEBHOOK in config.platforms): + return + webhook_config = ( + _enable_from_env(config, Platform.WEBHOOK, pop_marker=True, warn=False) + if enabled + else config.platforms[Platform.WEBHOOK] + ) + _env_extras(webhook_config.extra, (("port", "WEBHOOK_PORT", _INT), ("secret", "WEBHOOK_SECRET"))) def _msgraph_webhook(config: GatewayConfig) -> None: ``` - `gateway/config_loader.py` ```diff @@ -393,6 +393,14 @@ def read_yaml_layers(home: Path) -> dict: with open(config_yaml_path, encoding="utf-8") as f: yaml_cfg = yaml.safe_load(f) or {} + from hermes_cli.config import _expand_env_vars + + # ${VAR} / ${env:VAR} expansion — the same primitive the CLI loader applies, so platform + # adapter settings (webhook secret, api_server key, teams credentials) arrive resolved + # instead of as literal refs. User layer first: the managed overlay is expanded by + # apply_managed_overlay itself and must not be re-resolved (config_effective._effective order). + yaml_cfg = _expand_env_vars(yaml_cfg) + # Managed scope: overlay administrator-pinned values (this loader bypasses # hermes_cli.config.load_config, so managed quick_commands / stt would otherwise be ignored). from hermes_cli import managed_scope ``` - `gateway/platforms/api_server.py` ```diff @@ -122,6 +122,7 @@ def _approval_event_choices(*, smart_denied: bool, allow_session: bool, allow_pe from gateway.platforms import api_server_room_grants as _room_grants from gateway.platforms import api_server_runs as _api_runs from gateway.platforms.api_server_openai_routes import OpenAICompatRoutesMixin +from gateway.platforms.api_server_memory_sessions import ApiServerMemorySessions from gateway.platforms.base import ( MEDIA_TAG_CLEANUP_RE, BasePlatformAdapter, SendResult, _terminal_sentinel_start, is_network_accessible, validate_media_delivery_path) ``` ```diff @@ -1214,6 +1215,8 @@ def __init__(self, config: PlatformConfig): # Every agent inside _run_agent() for shutdown interrupt, keyed by id() (the strong ref # keeps the id() from recycling); distinct from the run_id-keyed _active_run_agents. self._shutdown_interruptible_agents: Dict[int, Any] = {} + # One memory provider per session across requests (this surface rebuilds the agent per turn). + self._memory_sessions = ApiServerMemorySessions() self.gateway_runner: Optional[Any] = None # set by gateway/run.py # Admitted requests not yet in agent bookkeeping, so shutdown drain counts them. self._pending_agent_requests: int = 0 ``` ```diff @@ -2236,7 +2239,10 @@ def _create_agent( # Same fallback provider chain as Telegram/Discord/Slack. "fallback_model": None if confirmed_runtime_lock else GatewayRunner._load_fallback_model(), "reasoning_config": request_reasoning_config, - "gateway_session_key": gateway_session_key} + "gateway_session_key": gateway_session_key, + # The session's provider from the previous request, so its queued recall reaches this turn + # (#120116); checked back in by the turn's finally. + "memory_manager": self._memory_sessions.checkout(session_id)} if request_service_tier is not _REQUEST_OPTION_MISSING: agent_kwargs["service_tier"] = request_service_tier agent = AIAgent(**agent_kwargs) ``` ```diff @@ -4034,6 +4040,7 @@ def _run(): if agent is not None: _clear_turn_process_ownership(agent) self._shutdown_interruptible_agents.pop(id(agent), None) + self._memory_sessions.checkin(agent) # Bind the declared key to the row the turn actually ended on # (agent.session_id carries a mid-turn rotation). Opt-in per route. # Record the declared conversation on the row the turn actually ended on — ``` ```diff @@ -4288,6 +4295,8 @@ async def disconnect(self) -> None: except Exception: logger.debug("Failed to close response store for %s", self.name, exc_info=True) _api_runs._close_run_state(self) + with suppress(Exception): + await asyncio.to_thread(self._memory_sessions.close_all) try: if self._site: await self._site.stop() ``` - `gateway/platforms/api_server_memory_sessions.py` ```diff @@ -0,0 +1,136 @@ +"""Memory-provider continuity across api_server requests (#120116). + +The api_server adapter builds a fresh ``AIAgent`` per request (per-request callbacks, model +route, ephemeral prompt), unlike the messaging platforms, whose cached agent — and with it the +memory provider — lives for the whole session. External providers deliver recall as the +PREVIOUS turn's background prefetch held on the provider instance, so a provider that is +re-initialised per request never has anything to inject, and each init re-runs the provider's +startup (for an embedded daemon: a restart that also kills the retain still in flight). + +This registry keeps one initialised ``MemoryManager`` per (profile home, session id): a request +checks the session's manager out before building its agent (``AIAgent(memory_manager=...)`` +skips provider init) and checks it back in when the turn ends. Check-out is exclusive, so two +concurrent requests on one session never share a manager; the loser's fresh manager is shut down +when it checks in behind the winner. Idle entries and LRU overflow are shut down under the owning +profile's scope, like the gateway agent cache's eviction. +""" + +from __future__ import annotations + +import logging +import threading +import time +from collections import OrderedDict +from contextlib import nullcontext, suppress +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +logger = logging.getLogger(__name__) + + +class ApiServerMemorySessions: + """Session-keyed ``MemoryManager`` registry with exclusive check-out/check-in.""" + + def __init__(self, *, max_size: Optional[int] = None, idle_ttl_secs: Optional[float] = None) -> None: + self._entries: "OrderedDict[Tuple[str, str], Tuple[Any, Optional[Path], float]]" = OrderedDict() + self._lock = threading.Lock() + self._max_size = max_size + self._idle_ttl_secs = idle_ttl_secs + + # -- bounds (same knobs as the gateway agent cache, resolved lazily) ------------------------- + + def _bounds(self) -> Tuple[int, float]: + if self._max_size is None or self._idle_ttl_secs is None: + from gateway.run import _AGENT_CACHE_IDLE_TTL_SECS, _AGENT_CACHE_MAX_SIZE, _load_gateway_config + from gateway.agent_cache_pressure import resolve_agent_cache_bounds + configured = None + with suppress(Exception): + configured = resolve_agent_cache_bounds(_load_gateway_config()) + if self._max_size is None: + self._max_size = getattr(configured, "max_size", None) or _AGENT_CACHE_MAX_SIZE + if self._idle_ttl_secs is None: + self._idle_ttl_secs = getattr(configured, "idle_ttl_secs", None) or _AGENT_CACHE_IDLE_TTL_SECS + return self._max_size, self._idle_ttl_secs + + @staticmethod + def _owner_home() -> Tuple[str, Optional[Path]]: + """(registry key, profile home to re-enter on eviction) for the CURRENT scope. Callers run + inside ``_profile_scope`` (or a single-profile gateway), so the ambient home is the owner's.""" + from hermes_constants import get_hermes_home, hermes_home_key + home = Path(get_hermes_home()) + return hermes_home_key(home), home + + # -- check-out / check-in ---------------------------------------------------------------- + + def checkout(self, session_id: Optional[str]) -> Optional[Any]: + """The manager a previous request on ``session_id`` checked in, or None (build a new one).""" + if not session_id: + return None + home_key, _home = self._owner_home() + with self._lock: + entry = self._entries.pop((home_key, session_id), None) + return entry[0] if entry else None + + def checkin(self, agent: Any) -> None: + """Park ``agent``'s manager under the session the turn ended on (``agent.session_id`` carries a + mid-turn compression rotation) and shut down whatever this displaces or has gone idle.""" + manager = getattr(agent, "_memory_manager", None) + session_id = str(getattr(agent, "session_id", "") or "") + if manager is None or not session_id: + return + home_key, home = self._owner_home() + max_size, idle_ttl = self._bounds() + now = time.monotonic() + doomed: List[Tuple[Any, Optional[Path]]] = [] + with self._lock: + displaced = self._entries.pop((home_key, session_id), None) + if displaced is not None and displaced[0] is not manager: + doomed.append((displaced[0], displaced[1])) + self._entries[(home_key, session_id)] = (manager, home, now) + for key, (mgr, owner, last_used) in list(self._entries.items()): + if mgr is manager: + continue + if now - last_used > idle_ttl or len(self._entries) > max_size: + del self._entries[key] + doomed.append((mgr, owner)) + for mgr, owner in doomed: + self._shutdown_async(mgr, owner) + + def close_all(self) -> None: + """Adapter shutdown: drain and shut down every parked manager (inline: the process is ending).""" + with self._lock: + entries = list(self._entries.values()) + self._entries.clear() + for mgr, owner, _ in entries: + self._shutdown(mgr, owner) + + # -- teardown ------------------------------------------------------------------------------- + + def _shutdown_async(self, manager: Any, owner: Optional[Path]) -> None: + """Eviction runs inside a request's own turn: never make that reply wait on a provider drain.""" + from agent.memory_provider import spawn_context_thread + spawn_context_thread(self._shutdown, args=(manager, owner), name="api-server-memory-evict").start() + + @staticmethod + def _shutdown(manager: Any, owner: Optional[Path]) -> None: + """Bounded drain then provider shutdown, under the OWNING profile's scope: eviction runs inside + whichever request happened to trigger it, and a provider reads its home/credentials at call time.""" + scope: Any = nullcontext() + with suppress(Exception): + from agent.secret_scope import is_multiplex_active + if owner is not None and is_multiplex_active(): + from gateway.run import _profile_runtime_scope + scope = _profile_runtime_scope(owner) + try: + with scope: + with suppress(Exception): + manager.flush_pending(timeout=10) + manager.shutdown_all() + except Exception: + logger.debug("api_server memory manager shutdown failed", exc_info=True) + + # -- introspection (tests) -------------------------------------------------------------------- + + def parked(self) -> Dict[Tuple[str, str], Any]: + with self._lock: + return {key: entry[0] for key, entry in self._entries.items()} ``` - `gateway/platforms/api_server_runs.py` ```diff @@ -780,6 +780,7 @@ def _run_agent_sync(self, run: _RunLaunch, agent, approval_notify, *, _api_serve finally: # Clear ownership now so a later stop can't reap work this run left running. _api_server._clear_turn_process_ownership(agent) + self._memory_sessions.checkin(agent) # Declared-conversation binding, same precedence gate as _run_agent. if run.declared_selected: self._bind_declared_conversation( ``` - `gateway/run.py` ```diff @@ -5264,6 +5264,8 @@ def restart_signal_handler(): def _start_gateway_make_shutdown_signal_handler(runner, _signal_initiated_shutdown: list): """Build the SIGINT/SIGTERM handler; ``_signal_initiated_shutdown[0]`` records an unplanned signal.""" + planned_stop_seen = [False] + def shutdown_signal_handler(received_signal=None): # Planned --replace takeover (sibling marked this PID): exit 0 so systemd won't revive us. def _takeover() -> bool: ``` ```diff @@ -5283,6 +5285,12 @@ def _snapshot(): planned_takeover = bool(_best_effort(_takeover, "Takeover marker check failed: %s")) planned_stop = received_signal == signal.SIGINT or ( not planned_takeover and bool(_best_effort(_planned_stop, "Planned stop marker check failed: %s"))) + # `hermes gateway stop` writes the marker, THEN signals: the planned-stop watcher can consume + # the marker in between, and the CLI's own SIGTERM must not then read as an external kill. + if planned_stop: + planned_stop_seen[0] = True + elif planned_stop_seen[0] and not planned_takeover: + planned_stop = True _shutdown_ctx = _best_effort(_snapshot, "snapshot_shutdown_context failed: %s") sig_name = _shutdown_ctx["signal"] if _shutdown_ctx else None ``` - `gateway/run_agent_cache.py` ```diff @@ -160,10 +160,13 @@ def _rehydrate_session_model_override(self, session_key: str) -> None: return override: Dict[str, Any] = {k: persisted.get(k) for k in ("model", "provider", "base_url")} provider = persisted.get("provider") + from hermes_cli.runtime_provider import is_foreign_provider_endpoint + if is_foreign_provider_endpoint(provider, override.get("base_url")): + override["base_url"] = None # left over from a switch that kept the previous provider's URL if provider: # Re-resolve credentials for the persisted provider. On failure (e.g. credentials removed # since the switch) keep the credential-less override — _resolve_session_agent_runtime - # falls back to env resolution and layers model/provider. + # retries the resolution for that provider on each turn (default route + notice meanwhile). try: runtime = _resolve_runtime_agent_kwargs_for_provider(provider, target_model=persisted.get("model") or None) for k in ("api_key", "api_mode", "credential_pool", "requested_provider", "max_tokens"): ``` - `gateway/run_turn.py` ```diff @@ -206,7 +206,8 @@ def _resolve_session_agent_runtime( skey or "", model, override_model, override_runtime.get("provider"), ) return override_model, override_runtime - # No api_key on the override: env-based resolution below, override model/provider on top. + # No api_key on the override (credentials failed to re-resolve at rehydrate): resolve them + # for the override's own provider below, never layer it over the default provider's runtime. logger.debug( "Session model override (no api_key, fallback): session=%s config_model=%s override_model=%s", skey or "", model, override_model, ``` ```diff @@ -221,7 +222,19 @@ def _resolve_session_agent_runtime( ][:5] or "[]", ) - runtime_kwargs = _resolve_runtime_agent_kwargs() + runtime_kwargs, unavailable_override = None, None + if override and override.get("provider"): + try: + runtime_kwargs = _resolve_runtime_agent_kwargs_for_provider( + override["provider"], target_model=override.get("model") or None) + except Exception as exc: + # Layering the override on the default runtime sent its model to the default provider's + # endpoint (openai-codex on the Nous URL). Run this turn on the whole default route and say + # so; the persisted override is kept, so the next turn retries it. + logger.warning("Session /model override provider %s unavailable: %s", override["provider"], exc) + unavailable_override, override = override, None + if runtime_kwargs is None: + runtime_kwargs = _resolve_runtime_agent_kwargs() # Private notice metadata must never reach an ``AIAgent(**runtime_kwargs)`` spread; the turn # runner surfaces it through the agent's one-shot fallback notice (#74349). self._pre_agent_fallback_notice = runtime_kwargs.pop("_fallback_notice", None) ``` ```diff @@ -229,6 +242,10 @@ def _resolve_session_agent_runtime( if runtime_model: logger.info("Runtime provider supplied explicit model override: %s -> %s", model, runtime_model) model = runtime_model + if unavailable_override and not self._pre_agent_fallback_notice: + from hermes_cli.fallback_config import pre_agent_fallback_notice + self._pre_agent_fallback_notice = pre_agent_fallback_notice( + unavailable_override["provider"], unavailable_override.get("model"), runtime_kwargs.get("provider"), model) cfg = getattr(self, "config", None) # getattr: bare object.__new__ test runners if cfg and source is not None: ``` - `gateway/slash_commands_model.py` ```diff @@ -215,6 +215,7 @@ async def _record_model_switch( _sess_entry.was_auto_reset = False await _sess_db.update_session_model( _sess_entry.session_id, result.new_model, provider=result.target_provider, + base_url=result.base_url, api_mode=result.api_mode, ) except Exception as exc: logger.debug("Failed to persist model switch to DB: %s", exc) ``` - `hermes_cli/AGENTS.md` ```diff @@ -152,10 +152,15 @@ it guards. `plan → snapshot → apply → restart-per-kind → verify → repo none of them; without the graft the swap deletes them). Post-swap, the Desktop rebuild decision also trusts the build stamp under HERMES_HOME, so an install that already lost its artifacts in an earlier update is rebuilt instead of "forgotten" (#90495). -- **Restart-per-kind**: systemd and launchd restarts are FLEET-WIDE (every `hermes-gateway*` unit / - `ai.hermes.gateway*` LaunchAgent), drain-first (SIGUSR1), with per-unit/per-label failure - isolation. Restarting only the invoking profile's service leaves siblings on stale `sys.modules` - until they crash — the largest dupe-PR cluster in the repo's history came from that bug. +- **Restart-per-kind**: systemd and launchd restarts are FLEET-WIDE within the updating install (every + `hermes-gateway*` unit / `ai.hermes.gateway*` LaunchAgent whose home is the updating root or one of its + `profiles/`), drain-first (SIGUSR1), with per-unit/per-label failure isolation. Restarting only the + invoking profile's service leaves siblings on stale `sys.modules` until they crash — the largest dupe-PR + cluster in the repo's history came from that bug. The fleet is bounded by HOME, not by namespace: + `hermes_cli/update_fleet_scope.py` judges every unit/label/process by the home it actually runs on + (live environ, unit `Environment=`, plist `HERMES_HOME`), and a runtime of another `HERMES_HOME` on the + same account — a sibling install, the real `hermes-gateway.service` seen from a scratch home — is named and + left alone, never restarted (#93349). - **Verify**: gateways stamp `code_sha`/`code_version` into `gateway_state.json` on every runtime-status write (`gateway/status.py`); the updater compares each live gateway against the fresh checkout and prints a fleet version matrix. A provably-stale gateway fails the update ``` - `hermes_cli/cli_model_switch_mixin.py` ```diff @@ -61,6 +61,10 @@ def stored_session_route(session_meta, *, current_model, current_provider): if stored_model == current_model and not provider_changed: return None api_mode = runtime.get("api_mode") or None + from hermes_cli.runtime_provider import is_foreign_provider_endpoint + if is_foreign_provider_endpoint(provider, base_url): + # The endpoint and its wire belong to the provider this chat left; resolve the stored one's own. + base_url = api_mode = None # A row's api_mode/base_url were written for whichever model the session last ran. Providers that # pick the wire per model (OpenCode Zen/Go, Copilot, Nous) re-derive both from the stored model, or a # resumed opencode-go session keeps a MiniMax-era anthropic_messages route for a chat_completions ``` - `hermes_cli/profile_cmd.py` ```diff @@ -215,6 +215,10 @@ def _profile_create(args): print(f"Full copy from {source_label} (excluding session history, cron jobs, backups, and snapshots).") else: print(f"Cloned config, .env, SOUL.md, and skills from {source_label}.") + from hermes_cli.profile_memory_config import cloned_memory_provider + memory_provider = cloned_memory_provider(profile_dir) + if memory_provider: + print(f"Cloned memory provider config ({memory_provider}) too.") if sync_imports: print(f"Import sources carried over — `hermes -p {name} import-agent --sync` " "keeps pulling the same Claude Code / Codex trees.") ``` - `hermes_cli/profile_memory_config.py` ```diff @@ -0,0 +1,71 @@ +"""Carry the ACTIVE memory provider's own config into a ``--clone`` (#120115). + +``--clone`` copies ``config.yaml`` — and with it ``memory.provider: hindsight`` — but the +provider keeps its settings outside config.yaml, so the clone booted with the provider +selected and silently unavailable. Providers store per-home config by convention (the same +convention ``hermes_cli.web_routers.memory_providers`` reads): a ``//`` +directory (hindsight) or a flat ``/.json`` (mem0, honcho, supermemory). Copying +by convention keeps this free of plugin imports: the provider may live in the catalog, not in +tree, so a hook the plugin must implement could not fix the reported case. +""" + +import contextlib +import os +import re +import shutil +from pathlib import Path +from typing import Optional + +# A provider name is a bare directory/file stem; anything else (path separators, ``..``, spaces) +# would let a hand-edited config.yaml aim the copy outside the source profile. +_PROVIDER_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def active_memory_provider(config: Optional[dict]) -> Optional[str]: + """The external ``memory.provider`` named in a parsed config.yaml, or None for the built-in + store or an unsafe name.""" + from agent.memory_provider import is_core_memory_provider + + memory = (config or {}).get("memory") + name = memory.get("provider") if isinstance(memory, dict) else None + if not isinstance(name, str) or is_core_memory_provider(name): + return None + name = name.strip() + if name in {".", ".."} or not _PROVIDER_NAME_RE.match(name): + return None + return name + + +def clone_memory_provider_config(source_dir: Path, profile_dir: Path, provider: Optional[str]) -> bool: + """Copy ``/`` and/or ``.json`` from *source_dir* into *profile_dir* when + present. Files land owner-only like ``.env``: they can hold an API key. Returns True when + anything was copied.""" + if not provider: + return False + copied = False + src_dir = source_dir / provider + if src_dir.is_dir(): + shutil.copytree(src_dir, profile_dir / provider, dirs_exist_ok=True) + for root, _dirs, files in os.walk(profile_dir / provider): + for filename in files: + with contextlib.suppress(OSError): + os.chmod(os.path.join(root, filename), 0o600) + copied = True + src_file = source_dir / f"{provider}.json" + if src_file.is_file(): + dst = profile_dir / f"{provider}.json" + shutil.copy2(src_file, dst) + with contextlib.suppress(OSError): + os.chmod(str(dst), 0o600) + copied = True + return copied + + +def cloned_memory_provider(profile_dir: Path) -> Optional[str]: + """Name of the external provider whose config *profile_dir* now carries, for the CLI notice.""" + from hermes_cli.profiles import _load_yaml_dict + + provider = active_memory_provider(_load_yaml_dict(profile_dir / "config.yaml")) + if provider and ((profile_dir / provider).is_dir() or (profile_dir / f"{provider}.json").is_file()): + return provider + return None ``` - `hermes_cli/profiles.py` ```diff @@ -1196,6 +1196,9 @@ def _bootstrap_profile_dir(profile_dir: Path, source_dir: Optional[Path], _copytree_keep_junctions(source_skills, profile_dir / "skills", _non_exportable_entries, dirs_exist_ok=True) for relpath in _CLONE_SUBDIR_FILES: _clone_file(source_dir, profile_dir, relpath) + from hermes_cli.profile_memory_config import active_memory_provider, clone_memory_provider_config + clone_memory_provider_config(source_dir, profile_dir, + active_memory_provider(_load_yaml_dict(source_dir / "config.yaml"))) if sync_imports: from hermes_cli.agent_import_sync import SYNC_MANIFEST_NAME # lazy: keeps yaml/utils off the hot startup path _clone_file(source_dir, profile_dir, SYNC_MANIFEST_NAME) ``` - `hermes_cli/runtime_provider.py` ```diff @@ -317,6 +317,20 @@ def _config_base_url_for_provider(model_cfg: Dict[str, Any], provider: str) -> s return str(model_cfg.get("base_url") or "").strip().rstrip("/") if _same_registered_provider(provider, configured_provider) else "" +def is_foreign_provider_endpoint(provider: Optional[str], base_url: Optional[str]) -> bool: + """True when ``base_url`` is another built-in provider's canonical endpoint, not ``provider``'s. + + A persisted session route that pairs one provider with another's endpoint is left over from a + switch that kept the old URL (openai-codex + the Nous Portal URL sent the Codex slug to the Portal). + Only registered providers are judged: a custom or proxy URL is never another provider's canonical one. + """ + pconfig = PROVIDER_REGISTRY.get(str(provider or "").strip().lower()) + url = str(base_url or "").strip().rstrip("/") + if pconfig is None or not url or url == (pconfig.inference_base_url or "").rstrip("/"): + return False + return any(url == (other.inference_base_url or "").rstrip("/") for other in PROVIDER_REGISTRY.values()) + + def _anthropic_base_url_override_ok(base_url: str) -> bool: """Whether a configured ``model.base_url`` plausibly speaks the Anthropic Messages protocol: official Anthropic/Claude hosts, Azure Foundry, or ``/anthropic`` / Kimi ``/coding`` proxies ``` - `hermes_cli/update_cmd_deps.py` ```diff @@ -4,12 +4,14 @@ import logging from contextlib import suppress +import ast import hashlib import json import os import shutil import subprocess import sys +import tomllib from pathlib import Path from typing import Optional from hermes_constants import project_venv_dir, venv_python_path ``` ```diff @@ -22,11 +24,83 @@ _INSTALL_DEFINING_FILES = "pyproject.toml", "setup.py", "setup.cfg", "MANIFEST.in", "uv.lock" +def _mapping_literal(tree: ast.AST): + """The setuptools finder uses ``MAPPING: dict[str, str] = {...}``, not a bare assign.""" + for node in tree.body: + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.target.id == "MAPPING": + return ast.literal_eval(node.value) + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "MAPPING" for target in node.targets + ): + return ast.literal_eval(node.value) + return None + + +def _checkout_import_names(root: Path) -> set[str]: + """Top-level names an editable install records: root modules plus configured packages.""" + names = {path.stem for path in root.glob("*.py") if path.name != "setup.py"} + with (root / "pyproject.toml").open("rb") as stream: + includes = ( + tomllib.load(stream) + .get("tool", {}) + .get("setuptools", {}) + .get("packages", {}) + .get("find", {}) + .get("include", []) + ) + prefixes = {item.removesuffix(".*") for item in includes if isinstance(item, str)} + names.update( + path.name + for path in root.iterdir() + if path.is_dir() and (path / "__init__.py").is_file() and path.name in prefixes + ) + return names + + +def _editable_finder_files(venv: Path) -> list[Path]: + sites = [*venv.glob("lib/python*/site-packages"), venv / "Lib" / "site-packages"] + return [ + finder + for site in sites + if site.is_dir() + for finder in site.glob("__editable__*hermes_agent*finder.py") + ] + + +def _editable_finder_mapping_current(cwd) -> bool | None: + """None when the install venv has no static finder; False when its map misses the checkout. + + Module entries point at the stem (``cli``, not ``cli.py``). Existence of that + path is not the contract — the key set is. A dangling path is a different repair. + """ + root = Path(cwd) + venv = project_venv_dir(root) + if venv is None: + return None + finders = _editable_finder_files(venv) + if not finders: + return None + try: + inventory = _checkout_import_names(root) + for finder in finders: + mapping = _mapping_literal(ast.parse(finder.read_text(encoding="utf-8"))) + if not isinstance(mapping, dict) or set(mapping) != inventory: + return False + except (OSError, SyntaxError, ValueError, TypeError, AttributeError): + return False + return True + + def _editable_install_is_current(git_cmd, cwd, pre_pull_sha: str | None) -> bool: - """True when the pulled commits cannot have invalidated the editable install: ``uv pip install - -e .`` always rewrites console-script shims (Windows: ``hermes.exe`` quarantine, ``os error 32`` - on a lost race), so skip it when only non-install files changed. Safe because the editable - finder uses a *static* module list. Fails closed: no pre-pull SHA or failed diff -> False.""" + """True when the pull cannot invalidate the editable install. + + ``uv pip install -e .`` rewrites console-script shims. On Windows that rewrite + quarantines the running ``hermes.exe``, and a lost race is the ``os error 32`` + family, so skip it only when packaging files are unchanged and the installed + finder still names every top-level import the checkout exposes. No finder keeps + the packaging-file gate. An unreadable finder, or a map that disagrees with the + checkout, is not current. No pre-pull SHA or a failed diff fails closed. + """ if not pre_pull_sha: return False try: ``` ```diff @@ -35,7 +109,10 @@ def _editable_install_is_current(git_cmd, cwd, pre_pull_sha: str | None) -> bool cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace") except OSError: return False - return result.returncode == 0 and not result.stdout.strip() + if result.returncode != 0 or result.stdout.strip(): + return False + mapping_current = _editable_finder_mapping_current(cwd) + return mapping_current is not False # Modules imported on every startup. Unlike _UPDATE_CRITICAL_FILES (only parsed) these are ``` ```diff @@ -95,19 +172,28 @@ def _critical_module_import_failures( report_runtime_errors, marker)) try: interpreter = sys.executable + argv = [interpreter, "-c", probe] with suppress(Exception): venv_dir = project_venv_dir(root) or Path(root) / "venv" venv_python = venv_python_path(venv_dir, windows=_m()._is_windows()) if venv_python.exists(): interpreter = str(venv_python) + argv = [interpreter, "-c", probe] + # ``-c`` puts the cwd (the checkout) at sys.path[0], which masks the installed + # editable finder — the exact thing a gateway started from ``/`` imports through. + # A stale finder MAPPING (new top-level package since the install) then reports + # green here and crash-loops the gateway (#119466). ``-P`` makes the probe see what + # the venv sees; only when an editable install exists, so a bare dev checkout that + # is importable through its cwd alone keeps its advisory verdict. + if _editable_finder_files(venv_dir): + argv = [interpreter, "-P", "-c", probe] # The candidate stays importable through the probe's cwd and its editable install; # the scrub only removes paths the guard never meant to vouch for. probe_env = dict(os.environ) for denied_key in _PROBE_ENV_DENYLIST: probe_env.pop(denied_key, None) result = bounded_probe_run( - [interpreter, "-c", probe], timeout=120, cwd=str(root), raise_on_spawn_failure=True, - env=probe_env, + argv, timeout=120, cwd=str(root), raise_on_spawn_failure=True, env=probe_env, ) except (OSError, subprocess.SubprocessError): # Keep this guard advisory: a probe we could not even spawn (unreadable venv ``` - `hermes_cli/update_cmd_fleet.py` ```diff @@ -581,6 +581,8 @@ def _restart_systemd_gateway_units_best_effort(failed: list, listings) -> None: for key in keys: scope, scope_cmd, svc_name = targets[key] + if not _systemd_unit_owned_by_update(scope_cmd, svc_name): + continue manage_cmd = list(scope_cmd) + ["--no-ask-password"] if _needs_sudo(scope): manage_cmd = ["sudo", "-n"] + manage_cmd ``` ```diff @@ -867,6 +869,34 @@ def _systemctl_reset_and_restart(manage_cmd: list, svc_name: str, *, scope_cmd: return _systemctl(manage_cmd + ["restart", svc_name], timeout=timeout) +def _systemd_unit_owned_by_update(scope_cmd: list, svc_name: str) -> bool: + """Gate a unit restart on the unit's home being one this update owns (#93349). + + ``hermes-gateway*`` is an account-wide namespace: a second install's ``hermes update`` used + to drain and restart the account's real ``hermes-gateway.service`` because the unit was + listed, not because it ran the updated code. Foreign or unreadable ownership prints a notice + and leaves the unit alone; it is not a failed restart. + """ + from hermes_cli.update_fleet_scope import describe_skipped_runtime, systemd_unit_hermes_home, home_in_update_scope + home = systemd_unit_hermes_home(scope_cmd, svc_name) + if home is not None and home_in_update_scope(home): + return True + print(describe_skipped_runtime("systemd unit", svc_name, home)) + return False + + +def _scoped_manual_gateway_pids(pids, *, keep=(), quiet: bool = False) -> list[int]: + """*pids* whose live home this update owns (plus *keep*, PIDs already mapped to this + install's profile PID files); every other gateway process is named and left running.""" + from hermes_cli.update_fleet_scope import describe_skipped_runtime, partition_gateway_pids_by_scope + keep = set(keep) + owned, foreign = partition_gateway_pids_by_scope([pid for pid in pids if pid not in keep]) + if not quiet: + for pid, home in foreign: + print(describe_skipped_runtime("gateway process", f"PID {pid}", home)) + return [pid for pid in pids if pid in keep or pid in owned] + + def _is_hermes_gateway_unit(unit: str) -> bool: """Exact base unit or hyphenated profile family only: ``startswith("hermes-serve")`` would accept ``hermes-server.service``.""" ``` ```diff @@ -945,7 +975,9 @@ def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None: print(" launchctl kickstart -k gui/$UID/