{"repo":"NousResearch/hermes-agent","public_id":"14154","display_label":"verify-on-stop bugs: message loss, leakage, delivery","summary":"These issues and fixes address a cluster of bugs around the verify-on-stop feature, including final answers being discarded or replaced by verification noise, verification output leaking into messaging surfaces, and gateway/cron delivery correctness problems.","subsystem":"agent","counts":{"members":11,"open_issues":2,"open_issues_distinct":2,"open_prs":1,"closed_issues":4,"merged_prs":2,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":14159},"resolution":{"state":"partially_resolved","by":"pr-52412"},"freshness":{"state":"verified","last_verified_at":"2026-06-26T07:30:09+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.13672727272727273},"fix_edges":[{"issue_number":14154,"pr_number":14159,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":14158,"pr_number":14159,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":52411,"pr_number":52412,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":14154,"title":"[Bug]: Gateway can show a visible interim success before an invalid final-response failure","url":null,"state":"closed"},{"kind":"issue","number":52411,"title":"[Bug]: verify-on-stop nudge leaks ad-hoc verification receipt into messaging (Telegram) replies","url":null,"state":"closed"},{"kind":"issue","number":53828,"title":"verify_on_stop should preserve the final answer and attach verification, not replace it","url":null,"state":"open"},{"kind":"issue","number":58075,"title":"[Bug] Cron job delivery messages contain agent self-verification noise instead of task output","url":null,"state":"open"},{"kind":"issue","number":62142,"title":"fix(tui): verification-stop can discard streamed final answers and cron reports","url":null,"state":"closed"},{"kind":"issue","number":68231,"title":"Desktop hides substantive response after follow-up ad-hoc verification message","url":null,"state":"closed"},{"kind":"pr","number":14155,"title":"fix(gateway): add failure context after visible interim replies","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#14159","number":14159},{"kind":"pr","value":"#14159","number":14159}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":14159,"title":"fix(agent): accept empty content with stop_reason=end_turn as valid anthropic response","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/transports/anthropic.py","tests/agent/transports/test_transport.py"],"fix_hunks":[{"file":"agent/transports/anthropic.py","hunks":["@@ -87,14 +87,20 @@ def normalize_response(self, response: Any, **kwargs) -> NormalizedResponse:\n         return normalize_anthropic_response_v2(response, strip_tool_prefix=strip_tool_prefix)\n \n     def validate_response(self, response: Any) -> bool:\n-        \"\"\"Check Anthropic response structure is valid.\"\"\"\n+        \"\"\"Check Anthropic response structure is valid.\n+\n+        An empty content list is legitimate when ``stop_reason == \"end_turn\"``\n+        — the model's canonical way of signalling \"nothing more to add\" after\n+        a tool turn that already delivered the user-facing text. Treating it\n+        as invalid falsely retries a completed response.\n+        \"\"\"\n         if response is None:\n             return False\n         content_blocks = getattr(response, \"content\", None)\n         if not isinstance(content_blocks, list):\n             return False\n         if not content_blocks:\n-            return False\n+            return getattr(response, \"stop_reason\", None) == \"end_turn\"\n         return True\n \n     def extract_cache_stats(self, response: Any) -> Optional[Dict[str, int]]:"]},{"file":"tests/agent/transports/test_transport.py","hunks":["@@ -114,6 +114,14 @@ def test_validate_response_empty_content(self, transport):\n         r = SimpleNamespace(content=[])\n         assert transport.validate_response(r) is False\n \n+    def test_validate_response_empty_content_with_end_turn_is_valid(self, transport):\n+        r = SimpleNamespace(content=[], stop_reason=\"end_turn\")\n+        assert transport.validate_response(r) is True\n+\n+    def test_validate_response_empty_content_with_tool_use_is_invalid(self, transport):\n+        r = SimpleNamespace(content=[], stop_reason=\"tool_use\")\n+        assert transport.validate_response(r) is False\n+\n     def test_validate_response_valid(self, transport):\n         r = SimpleNamespace(content=[SimpleNamespace(type=\"text\", text=\"hello\")])\n         assert transport.validate_response(r) is True\n"]}]},{"kind":"pr","number":52412,"title":"fix(agent): gate verify-on-stop nudge off for messaging surfaces","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/verification_stop.py","cli-config.yaml.example","hermes_cli/config.py","tests/agent/test_verification_stop.py"],"fix_hunks":[{"file":"agent/verification_stop.py","hunks":["@@ -15,9 +15,77 @@\n \n _MAX_CHANGED_PATHS_IN_NUDGE = 8\n \n+# Session identities (platform or source) that are NOT human conversational\n+# messaging surfaces: interactive coding surfaces (CLI, TUI, desktop, codex,\n+# local, gateway) and programmatic callers (API server, webhooks, tools).\n+# Verify-on-stop stays ON by default for these. Any other resolved gateway\n+# platform is a conversational messaging surface (Telegram, Discord, WhatsApp,\n+# Signal, Slack, etc.) where the verification narrative would reach a human as\n+# chat noise, so it defaults OFF. Mirrors LOCAL_SESSION_SOURCE_IDS in\n+# apps/desktop/src/lib/session-source.ts; keep roughly in sync when adding a\n+# local or programmatic surface. Default-deny by design: an unrecognized\n+# identity is treated as messaging (OFF) so a new chat platform never leaks the\n+# verification receipt before this set is updated.\n+_NON_MESSAGING_SESSION_SURFACES = frozenset(\n+    {\n+        \"\",\n+        \"cli\",\n+        \"codex\",\n+        \"desktop\",\n+        \"gateway\",\n+        \"local\",\n+        \"tui\",\n+        \"tool\",\n+        \"api_server\",\n+        \"webhook\",\n+        \"msgraph_webhook\",\n+    }\n+)\n+\n+\n+def _session_is_messaging_surface() -> bool:\n+    \"\"\"Return whether this turn is delivered over a human messaging channel.\n+\n+    The gateway binds the platform value (e.g. ``telegram``) to\n+    ``HERMES_SESSION_PLATFORM``; the CLI and TUI set ``HERMES_SESSION_SOURCE``\n+    (e.g. ``cli``, ``tui``) instead. Both are consulted via the session-context\n+    helper (with an ``os.environ`` fallback), alongside the ``HERMES_PLATFORM``\n+    override, matching the sibling platform resolution in\n+    ``agent/skill_commands.py`` and ``agent/prompt_builder.py``. A turn is a\n+    messaging surface when a resolved identity is present and is not a known\n+    non-messaging surface.\n+    \"\"\"\n+    try:\n+        from gateway.session_context import get_session_env\n+\n+        platform = (\n+            os.getenv(\"HERMES_PLATFORM\")\n+            or get_session_env(\"HERMES_SESSION_PLATFORM\", \"\")\n+        )\n+        source = get_session_env(\"HERMES_SESSION_SOURCE\", \"\")\n+    except Exception:\n+        platform = os.getenv(\"HERMES_PLATFORM\", \"\") or os.environ.get(\n+            \"HERMES_SESSION_PLATFORM\", \"\"\n+        )\n+        source = os.environ.get(\"HERMES_SESSION_SOURCE\", \"\")\n+    for identity in (platform, source):\n+        identity = str(identity or \"\").strip().lower()\n+        if identity and identity not in _NON_MESSAGING_SESSION_SURFACES:\n+            return True\n+    return False\n+\n \n def verify_on_stop_enabled(config: dict[str, Any] | None = None) -> bool:\n-    \"\"\"Return whether edit -> verify-before-finish behavior is enabled.\"\"\"\n+    \"\"\"Return whether edit -> verify-before-finish behavior is enabled.\n+\n+    Precedence: an explicit ``HERMES_VERIFY_ON_STOP`` env var wins, then an\n+    explicit boolean ``agent.verify_on_stop`` config value, then a surface-aware\n+    default. The config default is the sentinel ``\"auto\"`` (see\n+    ``DEFAULT_CONFIG``), which resolves to ON for interactive coding surfaces\n+    (CLI, TUI, desktop) and programmatic callers, and OFF for conversational\n+    messaging surfaces (Telegram, Discord, etc.) where the verification\n+    narrative would otherwise reach a human as chat noise.\n+    \"\"\"\n     env = os.environ.get(\"HERMES_VERIFY_ON_STOP\")\n     if env is not None:\n         return env.strip().lower() not in {\"0\", \"false\", \"no\", \"off\"}","@@ -29,9 +97,17 @@ def verify_on_stop_enabled(config: dict[str, Any] | None = None) -> bool:\n         except Exception:\n             config = {}\n     agent_cfg = (config or {}).get(\"agent\") if isinstance(config, dict) else None\n-    if isinstance(agent_cfg, dict) and \"verify_on_stop\" in agent_cfg:\n-        return bool(agent_cfg.get(\"verify_on_stop\"))\n-    return True\n+    cfg_val = agent_cfg.get(\"verify_on_stop\") if isinstance(agent_cfg, dict) else None\n+    if isinstance(cfg_val, bool):\n+        return cfg_val\n+    if isinstance(cfg_val, str):\n+        token = [redacted:secret-assignment]\n+        if token in {\"1\", \"true\", \"yes\", \"on\"}:\n+            return True\n+        if token in {\"0\", \"false\", \"no\", \"off\"}:\n+            return False\n+    # \"auto\", missing, or any other value -> surface-aware default.\n+    return not _session_is_messaging_surface()\n \n \n def _candidate_cwds(paths: Iterable[str]) -> list[Path]:"]},{"file":"cli-config.yaml.example","hunks":["@@ -637,7 +637,15 @@ agent:\n   # primaries (default 3). The OpenAI SDK does its own low-level retries\n   # underneath this wrapper — this is the Hermes-level loop.\n   # api_max_retries: 3\n-  \n+\n+  # After the agent edits code without fresh passing verification, nudge it to\n+  # verify before finishing. The default \"auto\" enables it on interactive\n+  # coding surfaces (CLI, TUI, desktop) and programmatic callers, and disables\n+  # it on conversational messaging surfaces (Telegram, Discord, etc.) where the\n+  # verification summary would reach a human as chat noise. Set true or false to\n+  # force it on or off; the HERMES_VERIFY_ON_STOP env var (1/0) takes precedence.\n+  # verify_on_stop: auto\n+\n   # Enable verbose logging\n   verbose: false\n   "]},{"file":"hermes_cli/config.py","hunks":["@@ -982,8 +982,12 @@ def _ensure_hermes_home_managed(home: Path):\n         # Verification closure: after the agent edits files in a code workspace,\n         # do not accept a final answer until fresh verification evidence exists\n         # or the agent explains why it cannot run checks. The loop is bounded\n-        # and uses the passive verification ledger. Set false to disable.\n-        \"verify_on_stop\": True,\n+        # and uses the passive verification ledger. The default \"auto\" enables\n+        # it on interactive coding surfaces (CLI, TUI, desktop) and programmatic\n+        # callers, and disables it on conversational messaging surfaces\n+        # (Telegram, Discord, etc.) where the verification summary would reach a\n+        # human as chat noise. Set true or false to force it on or off.\n+        \"verify_on_stop\": \"auto\",\n         # Staged inactivity warning: send a warning to the user at this\n         # threshold before escalating to a full timeout.  The warning fires\n         # once per run and does not interrupt the agent.  0 = disable warning."]},{"file":"tests/agent/test_verification_stop.py","hunks":["@@ -2,6 +2,8 @@\n import tempfile\n from pathlib import Path\n \n+import pytest\n+\n from agent.verification_evidence import (\n     mark_workspace_edited,\n     record_terminal_result,","@@ -25,21 +27,117 @@ def _make_project(root: Path) -> None:\n     _node_project(root)\n \n \n-def test_verify_on_stop_default_is_on(monkeypatch):\n-    monkeypatch.delenv(\"HERMES_VERIFY_ON_STOP\", raising=False)\n[redacted:email]\n+def clear_verify_env(monkeypatch):\n+    \"\"\"Clear every env signal verify_on_stop_enabled consults.\n+\n+    Tests then set only the variable they exercise, mirroring how the CLI/TUI\n+    set HERMES_SESSION_SOURCE and the gateway sets HERMES_SESSION_PLATFORM.\n+    \"\"\"\n+    for var in (\n+        \"HERMES_VERIFY_ON_STOP\",\n+        \"HERMES_PLATFORM\",\n+        \"HERMES_SESSION_PLATFORM\",\n+        \"HERMES_SESSION_SOURCE\",\n+    ):\n+        monkeypatch.delenv(var, raising=False)\n+    return monkeypatch\n+\n+\n+def test_verify_on_stop_default_is_on(clear_verify_env):\n+    # No env, no messaging identity, no explicit config -> default ON.\n     assert verify_on_stop_enabled({\"agent\": {}}) is True\n \n \n-def test_verify_on_stop_env_can_disable(monkeypatch):\n-    monkeypatch.setenv(\"HERMES_VERIFY_ON_STOP\", \"0\")\n+def test_verify_on_stop_auto_sentinel_resolves_to_surface_default(clear_verify_env):\n+    # The DEFAULT_CONFIG sentinel must fall through to the surface-aware default,\n+    # not be coerced to a truthy string.\n+    assert verify_on_stop_enabled({\"agent\": {\"verify_on_stop\": \"auto\"}}) is True\n+    clear_verify_env.setenv(\"HERMES_SESSION_PLATFORM\", \"telegram\")\n+    assert verify_on_stop_enabled({\"agent\": {\"verify_on_stop\": \"auto\"}}) is False\n+\n+\n+def test_verify_on_stop_env_can_disable(clear_verify_env):\n+    clear_verify_env.setenv(\"HERMES_VERIFY_ON_STOP\", \"0\")\n     assert verify_on_stop_enabled({\"agent\": {\"verify_on_stop\": True}}) is False\n \n \n-def test_verify_on_stop_config_can_disable(monkeypatch):\n-    monkeypatch.delenv(\"HERMES_VERIFY_ON_STOP\", raising=False)\n+def test_verify_on_stop_config_can_disable(clear_verify_env):\n     assert verify_on_stop_enabled({\"agent\": {\"verify_on_stop\": False}}) is False\n \n \n+def test_verify_on_stop_off_on_gateway_messaging_platform(clear_verify_env):\n+    # The gateway binds the platform value to HERMES_SESSION_PLATFORM and leaves\n+    # HERMES_SESSION_SOURCE empty, so a real Telegram turn must default OFF.\n+    clear_verify_env.setenv(\"HERMES_SESSION_PLATFORM\", \"telegram\")\n+    assert verify_on_stop_enabled({\"agent\": {}}) is False\n+\n+\n[redacted:email](\n+    \"platform\",\n+    [\"discord\", \"whatsapp_cloud\", \"signal\", \"slack\", \"matrix\", \"email\", \"sms\"],\n+)\n+def test_verify_on_stop_off_for_each_messaging_platform(clear_verify_env, platform):\n+    clear_verify_env.setenv(\"HERMES_SESSION_PLATFORM\", platform)\n+    assert verify_on_stop_enabled({\"agent\": {}}) is False\n+\n+\n+def test_verify_on_stop_messaging_platform_is_case_insensitive(clear_verify_env):\n+    clear_verify_env.setenv(\"HERMES_SESSION_PLATFORM\", \"  Telegram  \")\n+    assert verify_on_stop_enabled({\"agent\": {}}) is False\n+\n+\n+def test_verify_on_stop_uses_hermes_platform_override(clear_verify_env):\n+    # HERMES_PLATFORM mirrors the sibling platform resolution and also flags a\n+    # messaging surface.\n+    clear_verify_env.setenv(\"HERMES_PLATFORM\", \"discord\")\n+    assert verify_on_stop_enabled({\"agent\": {}}) is False\n+\n+\n[redacted:email](\"source\", [\"cli\", \"tui\", \"desktop\", \"codex\", \"local\"])\n+def test_verify_on_stop_on_for_interactive_surfaces(clear_verify_env, source):\n+    # CLI/TUI/desktop set HERMES_SESSION_SOURCE; these are coding surfaces -> ON.\n+    clear_verify_env.setenv(\"HERMES_SESSION_SOURCE\", source)\n+    assert verify_on_stop_enabled({\"agent\": {}}) is True\n+\n+\n[redacted:email](\"platform\", [\"api_server\", \"webhook\", \"msgraph_webhook\"])\n+def test_verify_on_stop_on_for_programmatic_surfaces(clear_verify_env, platform):\n+    clear_verify_env.setenv(\"HERMES_SESSION_PLATFORM\", platform)\n+    assert verify_on_stop_enabled({\"agent\": {}}) is True\n+\n+\n+def test_env_forces_verify_on_stop_on_for_messaging(clear_verify_env):\n+    clear_verify_env.setenv(\"HERMES_SESSION_PLATFORM\", \"telegram\")\n+    clear_verify_env.setenv(\"HERMES_VERIFY_ON_STOP\", \"1\")\n+    assert verify_on_stop_enabled({\"agent\": {}}) is True\n+\n+\n+def test_config_forces_verify_on_stop_on_for_messaging(clear_verify_env):\n+    clear_verify_env.setenv(\"HERMES_SESSION_PLATFORM\", \"telegram\")\n+    assert verify_on_stop_enabled({\"agent\": {\"verify_on_stop\": True}}) is True\n+\n+\n+def test_verify_on_stop_default_path_through_load_config(tmp_path, clear_verify_env):\n+    # E2E: the sole production caller passes no config, so verify_on_stop_enabled\n+    # resolves through load_config() + DEFAULT_CONFIG. The \"auto\" sentinel must\n+    # reach the surface-aware default rather than being shadowed by a static\n+    # True. This is the path the unit-level tests above cannot exercise.\n+    clear_verify_env.setenv(\"HERMES_HOME\", str(tmp_path / \".hermes\"))\n+\n+    from hermes_cli.config import load_config\n+\n+    merged = load_config()\n+    assert merged[\"agent\"][\"verify_on_stop\"] == \"auto\"\n+\n+    # Interactive (no messaging identity) resolves ON through the real loader.\n+    assert verify_on_stop_enabled() is True\n+\n+    # A messaging platform resolves OFF, proving the sentinel flows through.\n+    clear_verify_env.setenv(\"HERMES_SESSION_PLATFORM\", \"telegram\")\n+    assert verify_on_stop_enabled() is False\n+\n+\n def test_no_nudge_after_fresh_pass(tmp_path, monkeypatch):\n     monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path / \".hermes\"))\n     _node_project(tmp_path)\n"]}]},{"kind":"pr","number":58089,"title":"fix(cron): require deliverable-only final output","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":62184,"title":"fix(tui): verification-stop can discard streamed final answers (#62142)","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}