{"repo":"NousResearch/hermes-agent","public_id":"42306","display_label":"Langfuse missing usage/cost data in spans","summary":"These members concern a bug where Langfuse integration fails to capture or accumulate token usage and cost information in generation spans. The issues and fixes relate to sanitized responses and hook invocation behavior affecting usage data.","subsystem":null,"counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":40560,"merged_at":"2026-06-08T22:04:38+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-08T22:04:38+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":42306,"pr_number":40560,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-06-08T22:04:38+00:00"}],"members":[{"kind":"issue","number":42306,"title":"[Bug]: observability/langfuse: GENERATION spans created but missing usage/token counts/costs (no post-LLM hook activity logged)","url":null,"state":"closed"},{"kind":"pr","number":40560,"title":"fix(langfuse): restore usage/cost when post_api_request sends a sanitized response","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/observability/langfuse/__init__.py","tests/plugins/test_langfuse_plugin.py"],"fix_hunks":[{"file":"plugins/observability/langfuse/__init__.py","hunks":["@@ -837,8 +837,16 @@ def on_post_llm_call(*, task_id: str = \"\", session_id: str = \"\", provider: str =\n     if output.get(\"tool_calls\"):\n         state.turn_tool_calls.extend(output[\"tool_calls\"])\n \n-    # Extract usage: prefer response object, fall back to usage dict from post_api_request\n-    if response is not None:\n+    # Extract usage: prefer a real response object that carries usage, else\n+    # fall back to the usage summary dict from post_api_request.\n+    #\n+    # post_api_request passes `response` as a SANITIZED dict (no ``.usage``\n+    # attribute) alongside a separate `usage` summary dict. Gating on\n+    # ``response is not None`` here took the response-object path on that dict,\n+    # where ``getattr(response, \"usage\", None)`` is always None — so usage and\n+    # cost were silently dropped for every gateway turn. Gate on a real\n+    # ``.usage`` attribute instead so the usage-dict fallback below is reached.\n+    if getattr(response, \"usage\", None) is not None:\n         usage_details, cost_details = _usage_and_cost(\n             response,\n             provider=provider,"]},{"file":"tests/plugins/test_langfuse_plugin.py","hunks":["@@ -704,3 +704,76 @@ def fake_end(o, *, output=None, metadata=None, **kw):\n         assert ended[\"output\"] == {\"status\": \"done\"}\n         assert not state.tools\n \n+\n+class TestUsageFromSanitizedResponse:\n+    \"\"\"Regression: ``post_api_request`` delivers ``response`` as a sanitized\n+    dict (no ``.usage`` attribute) plus a separate ``usage`` summary dict. The\n+    post-call handler must read the ``usage`` dict instead of treating the dict\n+    response as a usage-bearing object and dropping all token/cost data.\"\"\"\n+\n+    def _setup(self, mod, monkeypatch):\n+        # Active client so on_post_llm_call does not early-return.\n+        monkeypatch.setattr(mod, \"_get_langfuse\", lambda: object())\n+        observation = object()\n+        state = mod.TraceState(trace_id=\"trace-1\", root_ctx=None, root_span=None)\n+        state.generations[mod._request_key(1)] = observation\n+        monkeypatch.setitem(mod._TRACE_STATE, mod._trace_key(\"task-1\", \"session-1\"), state)\n+        captured = {}\n+\n+        def fake_end_observation(obs, *, output=None, metadata=None, usage_details=None, cost_details=None):\n+            captured[\"usage_details\"] = usage_details\n+\n+        monkeypatch.setattr(mod, \"_end_observation\", fake_end_observation)\n+        return captured\n+\n+    def test_sanitized_dict_response_uses_usage_dict(self, monkeypatch):\n+        sys.modules.pop(\"plugins.observability.langfuse\", None)\n+        mod = importlib.import_module(\"plugins.observability.langfuse\")\n+        captured = self._setup(mod, monkeypatch)\n+\n+        # A plain dict has no ``.usage`` attribute — mirrors post_api_request.\n+        mod.on_post_llm_call(\n+            task_id=\"task-1\",\n+            session_id=\"session-1\",\n+            api_call_count=1,\n+            model=\"gemini-3-flash-preview\",\n+            response={\"model\": \"gemini-3-flash-preview\", \"usage\": {\"input_tokens\": 100, \"output_tokens\": 20}},\n+            usage={\"input_tokens\": 100, \"output_tokens\": 20},\n+            assistant_content_chars=42,\n+        )\n+\n+        # Before the fix the dict response shadowed the usage dict and tokens\n+        # were lost (usage_details == {}).\n+        assert captured[\"usage_details\"] == {\"input\": 100, \"output\": 20}\n+\n+    def test_real_response_object_with_usage_still_used(self, monkeypatch):\n+        sys.modules.pop(\"plugins.observability.langfuse\", None)\n+        mod = importlib.import_module(\"plugins.observability.langfuse\")\n+        captured = self._setup(mod, monkeypatch)\n+\n+        # A response object that genuinely carries usage must still take the\n+        # response-object path (post_llm_call / legacy behavior).\n+        seen = {}\n+\n+        def fake_usage_and_cost(resp, **_):\n+            seen[\"resp\"] = resp\n+            return {\"input\": 7, \"output\": 3}, {}\n+\n+        monkeypatch.setattr(mod, \"_usage_and_cost\", fake_usage_and_cost)\n+\n+        class _Resp:\n+            usage = {\"prompt_tokens\": 7, \"completion_tokens\": 3}\n+\n+        resp = _Resp()\n+        mod.on_post_llm_call(\n+            task_id=\"task-1\",\n+            session_id=\"session-1\",\n+            api_call_count=1,\n+            model=\"gemini-3-flash-preview\",\n+            response=resp,\n+            usage={\"input_tokens\": 999, \"output_tokens\": 999},\n+            assistant_content_chars=42,\n+        )\n+\n+        assert seen[\"resp\"] is resp\n+        assert captured[\"usage_details\"] == {\"input\": 7, \"output\": 3}\n"]}]},{"kind":"pr","number":42327,"title":"fix(langfuse): accumulate usage across hook invocations for traces (#42306)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#40560","number":40560}],"category":"superseded_by_pr","implication":"superseded"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}