{"repo":"NousResearch/hermes-agent","public_id":"17712","display_label":"Langfuse observability missing LLM and tool I/O","summary":"Multiple bug reports and corresponding fixes address Langfuse tracing not capturing LLM input/output and tool outputs. The issues and pull requests appear to target the same observability gap in the Langfuse integration.","subsystem":null,"counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":26320},"resolution":{"state":"partially_resolved","by":"pr-26320"},"freshness":{"state":"verified","last_verified_at":"2026-05-15T12:04:03+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.3933333333333333},"fix_edges":[{"issue_number":22342,"pr_number":26320,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":22763,"pr_number":26320,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":23823,"pr_number":26320,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":17712,"title":"[Bug]: Langfuse plugin  cannot see llm call input and output","url":null,"state":"closed"},{"kind":"issue","number":22342,"title":"[Bug]: Langfuse traces missing LLM input/output and tool output","url":null,"state":"closed"},{"kind":"pr","number":22345,"title":"fix(langfuse): fixes langfuse observability - llm input/output and tool output missing","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#26320","number":26320},{"kind":"pr","value":"#23831","number":23831},{"kind":"pr","value":"#12928","number":12928}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":26320,"title":"fix(langfuse): complete observability fix — trace I/O, tool outputs, placeholder credentials (closes #22342, #22763)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/observability/langfuse/__init__.py","run_agent.py","tests/plugins/test_langfuse_plugin.py","tests/run_agent/test_run_agent.py"],"fix_hunks":[{"file":"plugins/observability/langfuse/__init__.py","hunks":["@@ -47,6 +47,7 @@ class TraceState:\n     root_span: Any\n     generations: Dict[str, Any] = field(default_factory=dict)\n     tools: Dict[str, Any] = field(default_factory=dict)\n+    pending_tools_by_name: Dict[str, list] = field(default_factory=dict)\n     turn_tool_calls: list[dict[str, Any]] = field(default_factory=list)\n     last_updated_at: float = field(default_factory=time.time)\n ","@@ -58,6 +59,17 @@ class TraceState:\n _READ_FILE_HEAD_LINES = 25\n _READ_FILE_TAIL_LINES = 15\n \n+# Langfuse-issued keys always carry these prefixes (cloud or self-hosted —\n+# the prefix is baked into the server-side issuance flow, not a UI hint).\n+# Anything else (`placeholder`, `test-key`, `your-langfuse-key`, etc.) is a\n+# leftover template value and would cause the SDK to silently accept the\n+# credentials at construction time but drop every trace at flush time.\n+# See #23823 — the silent-failure bug this guard fixes.\n+_LANGFUSE_KEY_PREFIXES: Dict[str, str] = {\n+    \"HERMES_LANGFUSE_PUBLIC_KEY\": \"pk-lf-\",\n+    \"HERMES_LANGFUSE_SECRET_KEY\": \"sk-lf-\",\n+}\n+\n \n def _env(name: str, default: str = \"\") -> str:\n     return os.environ.get(name, default).strip()","@@ -82,10 +94,49 @@ def _debug(message: str) -> None:\n \n # Sentinel: \"_get_langfuse() has tried and failed\". Lets us short-circuit\n # every subsequent hook call without re-checking env vars or re-attempting\n-# SDK init. Cleared by reset_cache_for_tests().\n+# SDK init. Tests clear this by reloading the module via\n+# ``sys.modules.pop(...) + importlib.import_module(...)`` rather than via a\n+# dedicated reset function. Runtime callers cannot reset the cache; if an\n+# operator fixes a misconfigured credential they must restart the process.\n _INIT_FAILED = object()\n \n \n+def _redact_key_preview(value: str) -> str:\n+    \"\"\"Return a brief, log-safe preview of a credential value.\n+\n+    Keeps enough characters to disambiguate common placeholders\n+    (``placeholder``, ``test-key``, ``your-key``) without echoing a\n+    real secret in full if an operator pasted one into the wrong env\n+    var.  Used only for the once-per-process placeholder-detection\n+    warning in :func:`_get_langfuse`.\n+    \"\"\"\n+    if not value:\n+        return \"<empty>\"\n+    if len(value) <= 12:\n+        return repr(value)\n+    return repr(value[:6] + \"...\")\n+\n+\n+def _validate_langfuse_key(env_name: str, value: str) -> Optional[str]:\n+    \"\"\"Return an error message if ``value`` is not a real Langfuse key.\n+\n+    Returns ``None`` when the value matches the documented Langfuse\n+    prefix for ``env_name``, or when no prefix is registered for the\n+    name (in which case we trust the operator).  When validation\n+    fails the returned string is suitable for direct inclusion in a\n+    single log line — it names the env var and shows a safe preview.\n+    \"\"\"\n+    expected = _LANGFUSE_KEY_PREFIXES.get(env_name, \"\")\n+    if not expected:\n+        return None\n+    if value.startswith(expected):\n+        return None\n+    return (\n+        f\"{env_name}={_redact_key_preview(value)} \"\n+        f\"(expected {expected!r} prefix)\"\n+    )\n+\n+\n def _get_langfuse() -> Optional[Langfuse]:\n     \"\"\"Return a cached Langfuse client, or ``None`` if unavailable.\n ","@@ -111,6 +162,33 @@ def _get_langfuse() -> Optional[Langfuse]:\n         _LANGFUSE_CLIENT = _INIT_FAILED\n         return None\n \n+    # Reject placeholder credentials with a one-shot warning so the\n+    # operator sees the misconfiguration instead of silently shipping a\n+    # broken observability stack (#23823).  The SDK does not validate\n+    # keys at construction time — it queues traces in memory and only\n+    # discovers the auth failure when the background flush thread tries\n+    # to post them, by which point the warning is buried under whatever\n+    # else the process is logging.  Catch it here, surface it once, and\n+    # short-circuit via the same _INIT_FAILED path as the empty case.\n+    placeholder_issues = [\n+        msg\n+        for msg in (\n+            _validate_langfuse_key(\"HERMES_LANGFUSE_PUBLIC_KEY\", public_key),\n+            _validate_langfuse_key(\"HERMES_LANGFUSE_SECRET_KEY\", secret_key),\n+        )\n+        if msg\n+    ]\n+    if placeholder_issues:\n+        logger.warning(\n+            \"Langfuse plugin: credentials look like placeholders, traces will \"\n+            \"NOT be emitted (%s). Set real Langfuse keys (pk-lf-... / sk-lf-...) \"\n+            \"or unset HERMES_LANGFUSE_PUBLIC_KEY / HERMES_LANGFUSE_SECRET_KEY to \"\n+            \"silence this warning.\",\n+            \"; \".join(placeholder_issues),\n+        )\n+        _LANGFUSE_CLIENT = _INIT_FAILED\n+        return None\n+\n     base_url = _env(\"HERMES_LANGFUSE_BASE_URL\") or _env(\"LANGFUSE_BASE_URL\") or \"https://cloud.langfuse.com\"\n     environment = _env(\"HERMES_LANGFUSE_ENV\") or _env(\"LANGFUSE_ENV\")\n     release = _env(\"HERMES_LANGFUSE_RELEASE\") or _env(\"LANGFUSE_RELEASE\")","@@ -328,6 +406,21 @@ def _extract_last_user_message(messages: Any) -> Any:\n     return None\n \n \n+def _coerce_request_messages(\n+    *,\n+    request_messages: Any = None,\n+    messages: Any = None,\n+    conversation_history: Any = None,\n+    user_message: Any = None,\n+) -> list[dict[str, Any]]:\n+    for candidate in (request_messages, messages, conversation_history):\n+        if isinstance(candidate, list):\n+            return candidate\n+    if user_message is None:\n+        return []\n+    return [{\"role\": \"user\", \"content\": user_message}]\n+\n+\n def _serialize_messages(messages: Any) -> list[dict[str, Any]]:\n     if not isinstance(messages, list):\n         return []","@@ -343,8 +436,11 @@ def _serialize_messages(messages: Any) -> list[dict[str, Any]]:\n                 parse_json_strings=(role == \"tool\"),\n             ),\n         }\n-        if role == \"tool\" and message.get(\"tool_call_id\"):\n-            item[\"tool_call_id\"] = message.get(\"tool_call_id\")\n+        if role == \"tool\":\n+            if message.get(\"tool_call_id\"):\n+                item[\"tool_call_id\"] = message.get(\"tool_call_id\")\n+            if message.get(\"name\"):\n+                item[\"name\"] = _safe_value(message.get(\"name\"))\n         if message.get(\"tool_calls\"):\n             item[\"tool_calls\"] = _safe_value(message.get(\"tool_calls\"), parse_json_strings=True)\n         serialized.append(item)","@@ -359,15 +455,16 @@ def _serialize_tool_calls(tool_calls: Any) -> list[dict[str, Any]]:\n         fn = getattr(tool_call, \"function\", None)\n         name = getattr(fn, \"name\", None) if fn else None\n         arguments = getattr(fn, \"arguments\", None) if fn else None\n-        if isinstance(arguments, str):\n-            try:\n-                arguments = json.loads(arguments)\n-            except Exception:\n-                pass\n+        safe_arguments = _safe_value(arguments, parse_json_strings=False)\n         serialized.append({\n             \"id\": getattr(tool_call, \"id\", None),\n+            \"type\": getattr(tool_call, \"type\", None) or \"function\",\n             \"name\": name,\n-            \"arguments\": _safe_value(arguments, parse_json_strings=True),\n+            \"arguments\": safe_arguments,\n+            \"function\": {\n+                \"name\": name,\n+                \"arguments\": safe_arguments,\n+            },\n         })\n     return serialized\n ","@@ -564,6 +661,9 @@ def _finish_trace(task_key: str, *, output: Any = None) -> None:\n             _end_observation(observation)\n         for observation in state.tools.values():\n             _end_observation(observation)\n+        for queue in state.pending_tools_by_name.values():\n+            for observation in queue:\n+                _end_observation(observation)\n         final_output = _merge_trace_output(output, state)\n         if final_output is not None:\n             state.root_span.set_trace_io(output=final_output)","@@ -636,6 +736,7 @@ def on_pre_llm_request(\n     base_url: str = \"\",\n     api_mode: str = \"\",\n     api_call_count: int = 0,\n+    request_messages: Any = None,\n     messages: Any = None,\n     turn_type: str = \"user\",\n     message_count: int = 0,","@@ -643,12 +744,21 @@ def on_pre_llm_request(\n     approx_input_tokens: int = 0,\n     request_char_count: int = 0,\n     max_tokens: Any = None,\n+    conversation_history: Any = None,\n+    user_message: Any = None,\n     **_: Any,\n ) -> None:\n     client = _get_langfuse()\n     if client is None:\n         return\n \n+    input_messages = _coerce_request_messages(\n+        request_messages=request_messages,\n+        messages=messages,\n+        conversation_history=conversation_history,\n+        user_message=user_message,\n+    )\n+\n     task_key = _trace_key(task_id, session_id)\n     req_key = _request_key(api_call_count)\n ","@@ -663,7 +773,7 @@ def on_pre_llm_request(\n                 provider=provider,\n                 model=model,\n                 api_mode=api_mode,\n-                messages=messages,\n+                messages=input_messages,\n                 client=client,\n             )\n             _TRACE_STATE[task_key] = state","@@ -676,7 +786,7 @@ def on_pre_llm_request(\n             client=client,\n             name=f\"LLM call {api_call_count}\",\n             as_type=\"generation\",\n-            input_value=_serialize_messages(messages),\n+            input_value=_serialize_messages(input_messages),\n             metadata={\n                 \"provider\": provider,\n                 \"platform\": platform,","@@ -815,13 +925,12 @@ def on_pre_tool_call(*, tool_name: str = \"\", args: Any = None, task_id: str = \"\"\n         return\n \n     task_key = _trace_key(task_id, session_id)\n-    tool_key = tool_call_id or f\"{tool_name}:{time.time_ns()}\"\n \n     with _STATE_LOCK:\n         state = _TRACE_STATE.get(task_key)\n         if state is None:\n             return\n-        state.tools[tool_key] = _start_child_observation(\n+        observation = _start_child_observation(\n             state,\n             client=client,\n             name=f\"Tool: {tool_name}\",","@@ -829,22 +938,29 @@ def on_pre_tool_call(*, tool_name: str = \"\", args: Any = None, task_id: str = \"\"\n             input_value=_safe_value(args),\n             metadata={\"tool_name\": tool_name, \"tool_call_id\": tool_call_id},\n         )\n+        if tool_call_id:\n+            state.tools[tool_call_id] = observation\n+        else:\n+            state.pending_tools_by_name.setdefault(tool_name, []).append(observation)\n \n \n def on_post_tool_call(*, tool_name: str = \"\", args: Any = None, result: Any = None,\n                       task_id: str = \"\", session_id: str = \"\", tool_call_id: str = \"\", **_: Any) -> None:\n     task_key = _trace_key(task_id, session_id)\n-    tool_key = tool_call_id or \"\"\n     observation = None\n \n     with _STATE_LOCK:\n         state = _TRACE_STATE.get(task_key)\n         if state is None:\n             return\n-        if tool_key:\n-            observation = state.tools.pop(tool_key, None)\n-        elif state.tools:\n-            _, observation = state.tools.popitem()\n+        if tool_call_id:\n+            observation = state.tools.pop(tool_call_id, None)\n+        if observation is None:\n+            queue = state.pending_tools_by_name.get(tool_name)\n+            if queue:\n+                observation = queue.pop(0)\n+                if not queue:\n+                    state.pending_tools_by_name.pop(tool_name, None)\n \n     if observation is None:\n         return","@@ -854,10 +970,24 @@ def on_post_tool_call(*, tool_name: str = \"\", args: Any = None, result: Any = No\n     else:\n         result_value = result\n     result_value = _normalize_payload(result_value, tool_name=tool_name, args=args)\n+    safe_result_value = _safe_value(result_value, parse_json_strings=True)\n+\n+    # Backfill so the generation's tool_call record carries the result alongside arguments.\n+    if tool_call_id:\n+        with _STATE_LOCK:\n+            state = _TRACE_STATE.get(task_key)\n+            if state is not None:\n+                for tool_call in reversed(state.turn_tool_calls):\n+                    if tool_call.get(\"id\") == tool_call_id:\n+                        tool_call[\"output\"] = safe_result_value\n+                        function_payload = tool_call.get(\"function\")\n+                        if isinstance(function_payload, dict):\n+                            function_payload[\"output\"] = safe_result_value\n+                        break\n \n     _end_observation(\n         observation,\n-        output=_safe_value(result_value, parse_json_strings=True),\n+        output=safe_result_value,\n         metadata={\"tool_name\": tool_name, \"args\": _safe_value(args, parse_json_strings=True)},\n     )\n "]},{"file":"run_agent.py","hunks":["@@ -12668,16 +12668,30 @@ def run_conversation(\n \n                     try:\n                         from hermes_cli.plugins import invoke_hook as _invoke_hook\n+                        request_messages = api_kwargs.get(\"messages\")\n+                        if not isinstance(request_messages, list):\n+                            request_messages = api_kwargs.get(\"input\")\n+                        if not isinstance(request_messages, list):\n+                            request_messages = api_messages\n+                        # Shallow-copy the outer list so plugins that retain the\n+                        # reference for async snapshotting don't observe later\n+                        # mutations of api_messages.  The inner dicts are not\n+                        # mutated by the agent loop, so a shallow copy is\n+                        # sufficient; a deepcopy would walk every tool result\n+                        # and base64 image on every API call.\n                         _invoke_hook(\n                             \"pre_api_request\",\n                             task_id=effective_task_id,\n                             session_id=self.session_id or \"\",\n+                            user_message=original_user_message,\n+                            conversation_history=list(messages),\n                             platform=self.platform or \"\",\n                             model=self.model,\n                             provider=self.provider,\n                             base_url=self.base_url,\n                             api_mode=self.api_mode,\n                             api_call_count=api_call_count,\n+                            request_messages=list(request_messages) if isinstance(request_messages, list) else [],\n                             message_count=len(api_messages),\n                             tool_count=len(self.tools or []),\n                             approx_input_tokens=[redacted:secret-assignment]","@@ -14582,7 +14596,9 @@ def _stop_spinner():\n                         finish_reason=finish_reason,\n                         message_count=len(api_messages),\n                         response_model=getattr(response, \"model\", None),\n+                        response=response,\n                         usage=self._usage_summary_for_api_request_hook(response),\n+                        assistant_message=assistant_message,\n                         assistant_content_chars=len(_assistant_text),\n                         assistant_tool_call_count=len(_assistant_tool_calls),\n                     )"]},{"file":"tests/plugins/test_langfuse_plugin.py","hunks":["@@ -2,6 +2,7 @@\n from __future__ import annotations\n \n import importlib\n+import logging\n import sys\n from pathlib import Path\n ","@@ -164,7 +165,542 @@ def test_hooks_noop_without_client(self, monkeypatch):\n \n         # Each hook should just return; no exceptions.\n         mod.on_pre_llm_call(task_id=\"t\", session_id=\"s\", messages=[{\"role\": \"user\", \"content\": \"hi\"}])\n-        mod.on_pre_llm_request(task_id=\"t\", session_id=\"s\", api_call_count=1, messages=[])\n+        mod.on_pre_llm_request(task_id=\"t\", session_id=\"s\", api_call_count=1, request_messages=[])\n         mod.on_post_llm_call(task_id=\"t\", session_id=\"s\", api_call_count=1)\n         mod.on_pre_tool_call(tool_name=\"read_file\", args={}, task_id=\"t\", session_id=\"s\")\n         mod.on_post_tool_call(tool_name=\"read_file\", args={}, result=\"ok\", task_id=\"t\", session_id=\"s\")\n+\n+\n+# ---------------------------------------------------------------------------\n+# Placeholder-credential guard (#23823).\n+#\n+# Regression coverage for the silent-failure bug: when an operator leaves\n+# HERMES_LANGFUSE_PUBLIC_KEY / SECRET_KEY at a template value like\n+# \"placeholder\", \"test-key\", or \"your-langfuse-key\", the SDK accepts the\n+# credentials at construction time (it does no server-side validation\n+# eagerly) but drops every trace at flush time, with no signal in the\n+# Hermes logs.  The fix in `_get_langfuse()` validates the documented\n+# `pk-lf-` / `sk-lf-` prefix Langfuse always issues, surfaces a one-shot\n+# warning naming the offending env var(s), and short-circuits via the\n+# same `_INIT_FAILED` path used for missing credentials so subsequent\n+# hook invocations don't re-log.\n+# ---------------------------------------------------------------------------\n+\n+\n+class _FakeLangfuse:\n+    \"\"\"Stand-in for the real :class:`langfuse.Langfuse` so tests don't\n+    need the optional ``langfuse`` SDK installed.  The plugin's runtime\n+    gate refuses to proceed past ``if Langfuse is None`` when the SDK\n+    is missing, which would short-circuit before the placeholder check\n+    can fire.  Patching ``plugin.Langfuse`` with this class lets the\n+    placeholder validator exercise its full code path.\"\"\"\n+\n+    instances: list[\"_FakeLangfuse\"] = []\n+\n+    def __init__(self, **kwargs):\n+        self.kwargs = kwargs\n+        _FakeLangfuse.instances.append(self)\n+\n+\n+class TestPlaceholderKeyDetection:\n+    LOGGER_NAME = \"plugins.observability.langfuse\"\n+\n+    def _fresh_plugin(self, monkeypatch=None):\n+        mod_name = \"plugins.observability.langfuse\"\n+        sys.modules.pop(mod_name, None)\n+        mod = importlib.import_module(mod_name)\n+        if monkeypatch is not None:\n+            # Pretend the SDK is installed so `_get_langfuse()` actually\n+            # reaches the placeholder check.  Real SDK calls are never\n+            # made because the placeholder/missing-credentials paths\n+            # return before constructing a client.\n+            _FakeLangfuse.instances.clear()\n+            monkeypatch.setattr(mod, \"Langfuse\", _FakeLangfuse, raising=False)\n+        return mod\n+\n+    @staticmethod\n+    def _clear_env(monkeypatch):\n+        for k in (\n+            \"HERMES_LANGFUSE_PUBLIC_KEY\", \"HERMES_LANGFUSE_SECRET_KEY\",\n+            \"LANGFUSE_PUBLIC_KEY\", \"LANGFUSE_SECRET_KEY\",\n+        ):\n+            monkeypatch.delenv(k, raising=False)\n+\n+    # -- helper unit tests (no SDK stub needed: these don't go through\n+    #    _get_langfuse, they exercise the pure-Python helpers directly) ------\n+\n+    def test_redact_key_preview_empty(self, monkeypatch):\n+        self._clear_env(monkeypatch)\n+        plugin = self._fresh_plugin()\n+        assert plugin._redact_key_preview(\"\") == \"<empty>\"\n+\n+    def test_redact_key_preview_short_value_echoed(self, monkeypatch):\n+        \"\"\"Short placeholder strings are echoed in full so the operator\n+        can see exactly which template they forgot to replace.\"\"\"\n+        self._clear_env(monkeypatch)\n+        plugin = self._fresh_plugin()\n+        assert plugin._redact_key_preview(\"placeholder\") == \"'placeholder'\"\n+        assert plugin._redact_key_preview(\"test-key\") == \"'test-key'\"\n+\n+    def test_redact_key_preview_long_value_truncated(self, monkeypatch):\n+        \"\"\"If an operator pasted a real secret into the wrong env var the\n+        preview must NOT echo it in full — only the leading 6 chars.\"\"\"\n+        self._clear_env(monkeypatch)\n+        plugin = self._fresh_plugin()\n+        result = plugin._redact_key_preview(\"[redacted:openai-key]\")\n+        assert \"abcdefghij\" not in result\n+        assert result.startswith(\"'sk-lf-\")\n+        assert result.endswith(\"...'\")\n+\n+    def test_validate_langfuse_key_accepts_documented_prefix(self, monkeypatch):\n+        self._clear_env(monkeypatch)\n+        plugin = self._fresh_plugin()\n+        assert plugin._validate_langfuse_key(\n+            \"HERMES_LANGFUSE_PUBLIC_KEY\", \"pk-lf-real-public-xyz\"\n+        ) is None\n+        assert plugin._validate_langfuse_key(\n+            \"HERMES_LANGFUSE_SECRET_KEY\", \"[redacted:openai-key]\"\n+        ) is None\n+\n+    def test_validate_langfuse_key_rejects_wrong_prefix(self, monkeypatch):\n+        self._clear_env(monkeypatch)\n+        plugin = self._fresh_plugin()\n+        msg = plugin._validate_langfuse_key(\n+            \"HERMES_LANGFUSE_PUBLIC_KEY\", \"placeholder\"\n+        )\n+        assert msg is not None\n+        assert \"HERMES_LANGFUSE_PUBLIC_KEY\" in msg\n+        assert \"pk-lf-\" in msg\n+\n+    def test_validate_langfuse_key_unknown_name_passes(self, monkeypatch):\n+        \"\"\"Defensive: an env var with no registered prefix is trusted.\"\"\"\n+        self._clear_env(monkeypatch)\n+        plugin = self._fresh_plugin()\n+        assert plugin._validate_langfuse_key(\"HERMES_LANGFUSE_BASE_URL\", \"anything\") is None\n+\n+    # -- end-to-end _get_langfuse() behaviour --------------------------------\n+    # These tests pass `monkeypatch` to _fresh_plugin() so the helper can\n+    # stub out `Langfuse` (the optional SDK).  Without that, every call\n+    # short-circuits at `if Langfuse is None` before reaching the\n+    # placeholder validator — masking the very behaviour we're testing.\n+\n+    def test_placeholder_public_key_warns_and_skips(self, monkeypatch, caplog):\n+        self._clear_env(monkeypatch)\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_PUBLIC_KEY\", \"placeholder\")\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_SECRET_KEY\", \"[redacted:openai-key]\")\n+        plugin = self._fresh_plugin(monkeypatch)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            assert plugin._get_langfuse() is None\n+        text = caplog.text\n+        assert \"HERMES_LANGFUSE_PUBLIC_KEY\" in text\n+        assert \"'placeholder'\" in text\n+        assert \"pk-lf-\" in text\n+        # The valid secret value must NOT appear (the var NAME does, in\n+        # the \"or unset ...\" hint, but the value preview shouldn't).\n+        assert \"'sk-lf-\" not in text\n+        # Never constructed the SDK client — short-circuited before that.\n+        assert _FakeLangfuse.instances == []\n+\n+    def test_placeholder_secret_key_warns_and_skips(self, monkeypatch, caplog):\n+        self._clear_env(monkeypatch)\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_PUBLIC_KEY\", \"pk-lf-real-public-xyz\")\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_SECRET_KEY\", \"test-key\")\n+        plugin = self._fresh_plugin(monkeypatch)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            assert plugin._get_langfuse() is None\n+        text = caplog.text\n+        assert \"HERMES_LANGFUSE_SECRET_KEY\" in text\n+        assert \"'test-key'\" in text\n+        assert \"sk-lf-\" in text\n+        # The valid public value must NOT appear.\n+        assert \"'pk-lf-\" not in text\n+        assert _FakeLangfuse.instances == []\n+\n+    def test_both_placeholders_one_warning_with_both_keys(self, monkeypatch, caplog):\n+        self._clear_env(monkeypatch)\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_PUBLIC_KEY\", \"placeholder\")\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_SECRET_KEY\", \"placeholder\")\n+        plugin = self._fresh_plugin(monkeypatch)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            assert plugin._get_langfuse() is None\n+        warnings = [r for r in caplog.records if r.levelname == \"WARNING\"\n+                    and r.name == self.LOGGER_NAME]\n+        assert len(warnings) == 1, (\n+            f\"Expected a single combined warning; got {len(warnings)}:\\n\"\n+            + \"\\n\".join(r.getMessage() for r in warnings)\n+        )\n+        text = warnings[0].getMessage()\n+        assert \"HERMES_LANGFUSE_PUBLIC_KEY\" in text\n+        assert \"HERMES_LANGFUSE_SECRET_KEY\" in text\n+\n+    def test_repeated_calls_do_not_re_warn(self, monkeypatch, caplog):\n+        \"\"\"The cached ``_INIT_FAILED`` sentinel must short-circuit\n+        subsequent calls so each hook invocation isn't a fresh log\n+        line — otherwise a busy gateway will spam the operator's\n+        terminal.\"\"\"\n+        self._clear_env(monkeypatch)\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_PUBLIC_KEY\", \"placeholder\")\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_SECRET_KEY\", \"placeholder\")\n+        plugin = self._fresh_plugin(monkeypatch)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            for _ in range(15):\n+                assert plugin._get_langfuse() is None\n+        warnings = [r for r in caplog.records if r.levelname == \"WARNING\"\n+                    and r.name == self.LOGGER_NAME]\n+        assert len(warnings) == 1, (\n+            f\"Warning fired {len(warnings)} times across 15 calls; \"\n+            \"expected 1 (cached via _INIT_FAILED)\"\n+        )\n+\n+    @pytest.mark.parametrize(\"placeholder\", [\n+        \"placeholder\",\n+        \"test-key\",\n+        \"your-langfuse-key\",\n+        \"change-me\",\n+        \"xxx\",\n+        \"dummy-key-here\",\n+        \"<your-key>\",\n+        \"REPLACE_ME\",\n+    ])\n+    def test_common_placeholders_detected(self, monkeypatch, caplog, placeholder):\n+        \"\"\"A grab-bag of values that real-world ``.env.example`` templates\n+        use as stand-ins.  Any of them in either key must trip the guard.\"\"\"\n+        self._clear_env(monkeypatch)\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_PUBLIC_KEY\", placeholder)\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_SECRET_KEY\", \"[redacted:openai-key]\")\n+        plugin = self._fresh_plugin(monkeypatch)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            assert plugin._get_langfuse() is None\n+        assert \"HERMES_LANGFUSE_PUBLIC_KEY\" in caplog.text\n+\n+    def test_legacy_LANGFUSE_PUBLIC_KEY_also_validated(self, monkeypatch, caplog):\n+        \"\"\"The plugin reads both the canonical HERMES_-prefixed env var and\n+        the legacy bare ``LANGFUSE_PUBLIC_KEY``.  The validator must run on\n+        whichever value ``_get_langfuse()`` actually consumed.\"\"\"\n+        self._clear_env(monkeypatch)\n+        monkeypatch.setenv(\"LANGFUSE_PUBLIC_KEY\", \"placeholder\")\n+        monkeypatch.setenv(\"LANGFUSE_SECRET_KEY\", \"[redacted:openai-key]\")\n+        plugin = self._fresh_plugin(monkeypatch)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            assert plugin._get_langfuse() is None\n+        # Warning names the canonical user-facing env var (the bare\n+        # LANGFUSE_PUBLIC_KEY is a backwards-compat alias for the\n+        # HERMES_-prefixed one — operators set the HERMES_-prefixed one).\n+        assert \"HERMES_LANGFUSE_PUBLIC_KEY\" in caplog.text\n+        assert \"'placeholder'\" in caplog.text\n+\n+    def test_missing_credentials_still_skip_silently(self, monkeypatch, caplog):\n+        \"\"\"Missing-creds is the documented opt-out path (operator hasn't\n+        configured the plugin yet) — it must remain SILENT.  Regression\n+        guard against the placeholder validator accidentally running on\n+        empty values and re-introducing log noise for unconfigured\n+        installs.\"\"\"\n+        self._clear_env(monkeypatch)\n+        plugin = self._fresh_plugin(monkeypatch)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            assert plugin._get_langfuse() is None\n+        warnings = [r for r in caplog.records if r.levelname == \"WARNING\"\n+                    and r.name == self.LOGGER_NAME]\n+        assert warnings == []\n+\n+    def test_sdk_not_installed_still_skips_silently(self, monkeypatch, caplog):\n+        \"\"\"If the langfuse SDK isn't installed at all, the placeholder\n+        check should never run — there's nothing the operator can do\n+        about a credential mismatch when the package is missing, and\n+        re-warning here would dilute the actually-actionable SDK-missing\n+        signal upstream.  The ``Langfuse is None`` guard at the top of\n+        ``_get_langfuse`` already handles this; this test pins that\n+        behaviour.\"\"\"\n+        self._clear_env(monkeypatch)\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_PUBLIC_KEY\", \"placeholder\")\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_SECRET_KEY\", \"placeholder\")\n+        # NO monkeypatch on Langfuse here — falls back to whatever the\n+        # plugin imported at module load (None if SDK absent).\n+        plugin = self._fresh_plugin()\n+        monkeypatch.setattr(plugin, \"Langfuse\", None, raising=False)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            assert plugin._get_langfuse() is None\n+        warnings = [r for r in caplog.records if r.levelname == \"WARNING\"\n+                    and r.name == self.LOGGER_NAME]\n+        assert warnings == []\n+\n+    def test_valid_prefixes_do_not_trigger_placeholder_warning(self, monkeypatch, caplog):\n+        \"\"\"Real Langfuse keys (``pk-lf-…`` / ``sk-lf-…``) must pass the\n+        guard and proceed to SDK init.  We stub the SDK constructor with\n+        a recording fake so the assertion can confirm BOTH that the\n+        placeholder warning didn't fire AND that the client was actually\n+        constructed — the latter is the success signal the bug report\n+        wanted.\"\"\"\n+        self._clear_env(monkeypatch)\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_PUBLIC_KEY\", \"pk-lf-real-public-xyz\")\n+        monkeypatch.setenv(\"HERMES_LANGFUSE_SECRET_KEY\", \"[redacted:openai-key]\")\n+        plugin = self._fresh_plugin(monkeypatch)\n+        with caplog.at_level(logging.WARNING, logger=self.LOGGER_NAME):\n+            client = plugin._get_langfuse()\n+        assert isinstance(client, _FakeLangfuse)\n+        assert client.kwargs[\"public_key\"] == \"pk-lf-real-public-xyz\"\n+        assert client.kwargs[\"secret_key\"] == \"[redacted:openai-key]\"\n+        assert \"placeholders\" not in caplog.text.lower(), (\n+            f\"Valid Langfuse keys tripped the placeholder guard: {caplog.text!r}\"\n+        )\n+\n+\n+class TestRequestMessageCoercion:\n+    def test_prefers_request_messages_then_messages_then_history_then_user_message(self):\n+        sys.modules.pop(\"plugins.observability.langfuse\", None)\n+        mod = importlib.import_module(\"plugins.observability.langfuse\")\n+\n+        assert mod._coerce_request_messages(\n+            request_messages=[{\"role\": \"system\", \"content\": \"s\"}],\n+            messages=[{\"role\": \"user\", \"content\": \"m\"}],\n+            conversation_history=[{\"role\": \"user\", \"content\": \"h\"}],\n+            user_message=\"u\",\n+        ) == [{\"role\": \"system\", \"content\": \"s\"}]\n+        assert mod._coerce_request_messages(\n+            messages=[{\"role\": \"user\", \"content\": \"m\"}],\n+            conversation_history=[{\"role\": \"user\", \"content\": \"h\"}],\n+            user_message=\"u\",\n+        ) == [{\"role\": \"user\", \"content\": \"m\"}]\n+        assert mod._coerce_request_messages(\n+            conversation_history=[{\"role\": \"user\", \"content\": \"h\"}],\n+            user_message=\"u\",\n+        ) == [{\"role\": \"user\", \"content\": \"h\"}]\n+        assert mod._coerce_request_messages(user_message=\"u\") == [{\"role\": \"user\", \"content\": \"u\"}]\n+\n+\n+class TestToolCallOutputBackfill:\n+    def test_post_tool_call_backfills_matching_turn_tool_call_output(self, monkeypatch):\n+        sys.modules.pop(\"plugins.observability.langfuse\", None)\n+        mod = importlib.import_module(\"plugins.observability.langfuse\")\n+\n+        observation = object()\n+        state = mod.TraceState(trace_id=\"trace-1\", root_ctx=None, root_span=None)\n+        state.tools[\"call-1\"] = observation\n+        state.turn_tool_calls.append({\n+            \"id\": \"call-1\",\n+            \"type\": \"function\",\n+            \"name\": \"web_extract\",\n+            \"arguments\": '{\"urls\": [\"https://example.com\"]}',\n+            \"function\": {\n+                \"name\": \"web_extract\",\n+                \"arguments\": '{\"urls\": [\"https://example.com\"]}',\n+            },\n+        })\n+\n+        task_key = mod._trace_key(\"task-1\", \"session-1\")\n+        monkeypatch.setitem(mod._TRACE_STATE, task_key, state)\n+\n+        ended = {}\n+\n+        def fake_end_observation(obs, *, output=None, metadata=None, usage_details=None, cost_details=None):\n+            ended[\"observation\"] = obs\n+            ended[\"output\"] = output\n+            ended[\"metadata\"] = metadata\n+\n+        monkeypatch.setattr(mod, \"_end_observation\", fake_end_observation)\n+\n+        mod.on_post_tool_call(\n+            tool_name=\"web_extract\",\n+            args={\"urls\": [\"https://example.com\"]},\n+            result='{\"results\": [{\"url\": \"https://example.com\", \"content\": \"Example Domain\"}]}',\n+            task_id=\"task-1\",\n+            session_id=\"session-1\",\n+            tool_call_id=\"call-1\",\n+        )\n+\n+        assert ended[\"observation\"] is observation\n+        assert state.turn_tool_calls[0][\"output\"] == ended[\"output\"]\n+        assert state.turn_tool_calls[0][\"function\"][\"output\"] == ended[\"output\"]\n+        assert state.turn_tool_calls[0][\"output\"] == {\n+            \"results\": [{\"url\": \"https://example.com\", \"content\": \"Example Domain\"}]\n+        }\n+\n+    def test_serialize_messages_keeps_tool_name_and_call_id(self):\n+        sys.modules.pop(\"plugins.observability.langfuse\", None)\n+        mod = importlib.import_module(\"plugins.observability.langfuse\")\n+\n+        messages = [{\n+            \"role\": \"tool\",\n+            \"name\": \"web_extract\",\n+            \"tool_call_id\": \"call-1\",\n+            \"content\": '{\"ok\": true}',\n+        }]\n+\n+        assert mod._serialize_messages(messages) == [{\n+            \"role\": \"tool\",\n+            \"name\": \"web_extract\",\n+            \"tool_call_id\": \"call-1\",\n+            \"content\": {\"ok\": True},\n+        }]\n+\n+    def test_serialize_tool_calls_emits_openai_style_function_shape(self):\n+        sys.modules.pop(\"plugins.observability.langfuse\", None)\n+        mod = importlib.import_module(\"plugins.observability.langfuse\")\n+\n+        class _Fn:\n+            name = \"web_extract\"\n+            arguments = '{\"urls\": [\"https://example.com\"]}'\n+\n+        class _ToolCall:\n+            id = \"call-1\"\n+            type = \"function\"\n+            function = _Fn()\n+\n+        assert mod._serialize_tool_calls([_ToolCall()]) == [{\n+            \"id\": \"call-1\",\n+            \"type\": \"function\",\n+            \"name\": \"web_extract\",\n+            \"arguments\": '{\"urls\": [\"https://example.com\"]}',\n+            \"function\": {\n+                \"name\": \"web_extract\",\n+                \"arguments\": '{\"urls\": [\"https://example.com\"]}',\n+            },\n+        }]\n+\n+\n+class TestToolObservationKeying:\n+    \"\"\"Tests for pre/post tool_call observation matching when tool_call_id is absent.\"\"\"\n+\n+    def _make_mod(self):\n+        sys.modules.pop(\"plugins.observability.langfuse\", None)\n+        return importlib.import_module(\"plugins.observability.langfuse\")\n+\n+    def test_empty_tool_call_id_single_tool_sets_output(self, monkeypatch):\n+        mod = self._make_mod()\n+        obs = object()\n+        state = mod.TraceState(trace_id=\"t\", root_ctx=None, root_span=None)\n+        state.pending_tools_by_name.setdefault(\"my_tool\", []).append(obs)\n+\n+        task_key = mod._trace_key(\"task-1\", \"sess-1\")\n+        monkeypatch.setitem(mod._TRACE_STATE, task_key, state)\n+\n+        ended = {}\n+\n+        def fake_end(o, *, output=None, metadata=None, **kw):\n+            ended[\"obs\"] = o\n+            ended[\"output\"] = output\n+\n+        monkeypatch.setattr(mod, \"_end_observation\", fake_end)\n+\n+        mod.on_post_tool_call(\n+            tool_name=\"my_tool\",\n+            args={},\n+            result='{\"ok\": true}',\n+            task_id=\"task-1\",\n+            session_id=\"sess-1\",\n+            tool_call_id=\"\",\n+        )\n+\n+        assert ended[\"obs\"] is obs\n+        assert ended[\"output\"] == {\"ok\": True}\n+        assert state.pending_tools_by_name.get(\"my_tool\") is None\n+\n+    def test_empty_tool_call_id_observations_are_fifo_within_tool_name(self, monkeypatch):\n+        \"\"\"Two queued observations are consumed in FIFO order so the first\n+        post hook gets the first observation's output, not the second.\n+\n+        Sequential-on-one-thread coverage; the real concurrent case is\n+        guarded by ``_STATE_LOCK`` around every read-modify-write on\n+        ``pending_tools_by_name`` and is exercised in\n+        ``test_threaded_post_calls_preserve_fifo_under_lock`` below.\n+        \"\"\"\n+        mod = self._make_mod()\n+        obs_a, obs_b = object(), object()\n+        state = mod.TraceState(trace_id=\"t\", root_ctx=None, root_span=None)\n+        state.pending_tools_by_name[\"web_extract\"] = [obs_a, obs_b]\n+\n+        task_key = mod._trace_key(\"task-1\", \"sess-1\")\n+        monkeypatch.setitem(mod._TRACE_STATE, task_key, state)\n+\n+        calls = []\n+\n+        def fake_end(o, *, output=None, metadata=None, **kw):\n+            calls.append((o, output))\n+\n+        monkeypatch.setattr(mod, \"_end_observation\", fake_end)\n+\n+        mod.on_post_tool_call(\n+            tool_name=\"web_extract\", args={}, result='{\"val\": \"a\"}',\n+            task_id=\"task-1\", session_id=\"sess-1\", tool_call_id=\"\",\n+        )\n+        mod.on_post_tool_call(\n+            tool_name=\"web_extract\", args={}, result='{\"val\": \"b\"}',\n+            task_id=\"task-1\", session_id=\"sess-1\", tool_call_id=\"\",\n+        )\n+\n+        assert calls[0] == (obs_a, {\"val\": \"a\"})\n+        assert calls[1] == (obs_b, {\"val\": \"b\"})\n+        assert state.pending_tools_by_name.get(\"web_extract\") is None\n+\n+    def test_threaded_post_calls_preserve_fifo_under_lock(self, monkeypatch):\n+        \"\"\"The actual concurrency contract: when 8 threads race to drain\n+        the pending queue, no observation is consumed twice and none is\n+        lost.  Validates ``_STATE_LOCK`` discipline, not Python list\n+        semantics.\"\"\"\n+        import threading\n+\n+        mod = self._make_mod()\n+        n = 8\n+        observations = [object() for _ in range(n)]\n+        state = mod.TraceState(trace_id=\"t\", root_ctx=None, root_span=None)\n+        state.pending_tools_by_name[\"web_extract\"] = list(observations)\n+\n+        task_key = mod._trace_key(\"task-thr\", \"sess-thr\")\n+        monkeypatch.setitem(mod._TRACE_STATE, task_key, state)\n+\n+        recorded: list = []\n+        lock = threading.Lock()\n+\n+        def fake_end(o, *, output=None, metadata=None, **kw):\n+            with lock:\n+                recorded.append(o)\n+\n+        monkeypatch.setattr(mod, \"_end_observation\", fake_end)\n+\n+        barrier = threading.Barrier(n)\n+\n+        def worker():\n+            barrier.wait()\n+            mod.on_post_tool_call(\n+                tool_name=\"web_extract\", args={}, result='{\"ok\": true}',\n+                task_id=\"task-thr\", session_id=\"sess-thr\", tool_call_id=\"\",\n+            )\n+\n+        threads = [threading.Thread(target=worker) for _ in range(n)]\n+        for t in threads:\n+            t.start()\n+        for t in threads:\n+            t.join()\n+\n+        # Every observation was consumed exactly once; queue is empty.\n+        assert len(recorded) == n\n+        assert set(map(id, recorded)) == set(map(id, observations))\n+        assert state.pending_tools_by_name.get(\"web_extract\") is None\n+\n+    def test_explicit_tool_call_id_uses_tools_dict(self, monkeypatch):\n+        \"\"\"When tool_call_id is present, pending_tools_by_name is not touched.\"\"\"\n+        mod = self._make_mod()\n+        obs = object()\n+        state = mod.TraceState(trace_id=\"t\", root_ctx=None, root_span=None)\n+        state.tools[\"call-99\"] = obs\n+\n+        task_key = mod._trace_key(\"task-1\", \"sess-1\")\n+        monkeypatch.setitem(mod._TRACE_STATE, task_key, state)\n+\n+        ended = {}\n+\n+        def fake_end(o, *, output=None, metadata=None, **kw):\n+            ended[\"obs\"] = o\n+            ended[\"output\"] = output\n+\n+        monkeypatch.setattr(mod, \"_end_observation\", fake_end)\n+\n+        mod.on_post_tool_call(\n+            tool_name=\"my_tool\", args={}, result='{\"status\": \"done\"}',\n+            task_id=\"task-1\", session_id=\"sess-1\", tool_call_id=\"call-99\",\n+        )\n+\n+        assert ended[\"obs\"] is obs\n+        assert ended[\"output\"] == {\"status\": \"done\"}\n+        assert not state.tools\n+"]},{"file":"tests/run_agent/test_run_agent.py","hunks":["@@ -2524,8 +2524,9 @@ def _record_hook(name, **kwargs):\n         assert [call[\"api_call_count\"] for call in pre_request_calls] == [1, 2]\n         assert [call[\"api_call_count\"] for call in post_request_calls] == [1, 2]\n         assert all(call[\"session_id\"] == agent.session_id for call in pre_request_calls)\n-        assert all(\"message_count\" in c and \"messages\" not in c for c in pre_request_calls)\n-        assert all(\"usage\" in c and \"response\" not in c for c in post_request_calls)\n+        assert all(\"message_count\" in c and isinstance(c.get(\"request_messages\"), list) for c in pre_request_calls)\n+        assert any(msg.get(\"role\") == \"user\" and msg.get(\"content\") == \"search something\" for msg in pre_request_calls[0][\"request_messages\"])\n+        assert all(\"usage\" in c and \"response\" in c and \"assistant_message\" in c for c in post_request_calls)\n \n     def test_content_with_tool_calls_stays_silent_for_non_cli_quiet_mode(self, agent):\n         self._setup_agent(agent)\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}