{"repo":"NousResearch/hermes-agent","public_id":"35072","display_label":"Kanban worker iteration-budget exhaustion and retry loop fixes","summary":"These issues and pull requests address bugs where a Kanban worker exhausting its iteration budget could create permanent sticky blocks with no auto-recovery and trigger infinite retry loops. The work focuses on closing gaps in budget-exhaustion handling and aligning recovery guards with configured failure limits.","subsystem":null,"counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":35220,"merged_at":"2026-05-30T08:40:58+00:00"},"resolution":{"state":"resolved","by":"pr-35220"},"freshness":{"state":"verified","last_verified_at":"2026-05-30T08:40:58+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":35072,"pr_number":35220,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-30T08:40:58+00:00"}],"members":[{"kind":"issue","number":35072,"title":"Kanban worker iteration-budget exhaustion creates permanent sticky block (no auto-recovery)","url":null,"state":"closed"},{"kind":"pr","number":35074,"title":"fix(kanban): prevent infinite retry loop when worker exhausts iteration budget","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"6ab71d3bb","number":0},{"kind":"pr","value":"#35220","number":35220}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":35185,"title":"fix(kanban): prevent budget exhaustion from creating permanent sticky blocks","url":null,"state":"closed","close_reason":{"category":"stale_abandoned","implication":"active"},"merged":false},{"kind":"pr","number":35220,"title":"fix(kanban): stop budget-exhaustion retry loop; align recovery guard with configured failure_limit","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/kanban_db.py","tests/hermes_cli/test_kanban_blocked_sticky.py","tests/hermes_cli/test_kanban_db.py"],"fix_hunks":[{"file":"hermes_cli/kanban_db.py","hunks":["@@ -2457,7 +2457,9 @@ def _has_sticky_block(conn: sqlite3.Connection, task_id: str) -> bool:\n     return bool(row) and row[\"kind\"] == \"blocked\"\n \n \n-def recompute_ready(conn: sqlite3.Connection) -> int:\n+def recompute_ready(\n+    conn: sqlite3.Connection, failure_limit: int = None,\n+) -> int:\n     \"\"\"Promote ``todo`` tasks to ``ready`` when all parents are ``done`` or ``archived``.\n \n     Returns the number of tasks promoted.  Safe to call inside or outside","@@ -2465,17 +2467,34 @@ def recompute_ready(conn: sqlite3.Connection) -> int:\n \n     ``blocked`` tasks are also considered for promotion (so a task\n     blocked purely by a parent dependency unblocks itself when the\n-    parent completes), *except* when the most recent block event was a\n-    worker-initiated ``kanban_block`` — those stay blocked until an\n-    explicit ``kanban_unblock`` (#28712).  Without that guard, a\n-    ``review-required`` handoff would auto-respawn, the fresh worker\n-    would find nothing to do, exit cleanly, get recorded as a protocol\n-    violation, and the cycle would repeat indefinitely.\n+    parent completes), *except* in two cases:\n+\n+    1. The most recent block event was a worker-initiated\n+       ``kanban_block`` — those stay blocked until an explicit\n+       ``kanban_unblock`` (#28712).\n+\n+    2. The task's ``consecutive_failures`` has reached the effective\n+       failure limit.  This prevents infinite retry loops when a task\n+       repeatedly exhausts its iteration budget: without this guard the\n+       counter would reset on every recovery cycle and the circuit\n+       breaker could never trip (#35072).\n+\n+    The effective failure limit resolves in the same order as the\n+    circuit breaker in ``_record_task_failure`` so the two never\n+    disagree about when a task is permanently blocked:\n+\n+      1. per-task ``max_retries`` if set\n+      2. caller-supplied ``failure_limit`` (the dispatcher passes the\n+         ``kanban.failure_limit`` config value through ``dispatch_once``)\n+      3. ``DEFAULT_FAILURE_LIMIT``\n     \"\"\"\n+    if failure_limit is None:\n+        failure_limit = DEFAULT_FAILURE_LIMIT\n     promoted = 0\n     with write_txn(conn):\n         todo_rows = conn.execute(\n-            \"SELECT id, status FROM tasks WHERE status IN ('todo', 'blocked')\"\n+            \"SELECT id, status, consecutive_failures, max_retries \"\n+            \"FROM tasks WHERE status IN ('todo', 'blocked')\"\n         ).fetchall()\n         for row in todo_rows:\n             task_id = row[\"id\"]","@@ -2493,13 +2512,25 @@ def recompute_ready(conn: sqlite3.Connection) -> int:\n                 (task_id,),\n             ).fetchall()\n             if all(p[\"status\"] in (\"done\", \"archived\") for p in parents):\n-                # Blocked tasks also get their failure counters reset —\n-                # this is effectively an auto-unblock (circuit-breaker\n-                # recovery; worker-initiated blocks are skipped above).\n                 if cur_status == \"blocked\":\n+                    # Don't auto-recover tasks that have hit the\n+                    # circuit-breaker failure limit.  Without this\n+                    # guard, a task that repeatedly exhausts its\n+                    # iteration budget would cycle forever:\n+                    # block → auto-recover → respawn → budget\n+                    # exhausted → block → …  The counter must also\n+                    # be preserved so the breaker can accumulate\n+                    # across recovery cycles.\n+                    failures = int(row[\"consecutive_failures\"] or 0)\n+                    task_limit = row[\"max_retries\"]\n+                    effective_limit = (\n+                        int(task_limit) if task_limit is not None\n+                        else int(failure_limit)\n+                    )\n+                    if failures >= effective_limit:\n+                        continue\n                     conn.execute(\n-                        \"UPDATE tasks SET status = 'ready', \"\n-                        \"consecutive_failures = 0, last_failure_error = NULL \"\n+                        \"UPDATE tasks SET status = 'ready' \"\n                         \"WHERE id = ? AND status = 'blocked'\",\n                         (task_id,),\n                     )","@@ -5424,7 +5455,7 @@ def dispatch_once(\n     if _crash_auto_blocked:\n         result.auto_blocked.extend(_crash_auto_blocked)\n     result.timed_out = enforce_max_runtime(conn)\n-    result.promoted = recompute_ready(conn)\n+    result.promoted = recompute_ready(conn, failure_limit=failure_limit)\n \n     # Count tasks already running so max_spawn enforces concurrency rather\n     # than a per-tick spawn budget. See the docstring above for the full"]},{"file":"tests/hermes_cli/test_kanban_blocked_sticky.py","hunks":["@@ -106,20 +106,30 @@ def test_worker_block_on_child_with_done_parents_is_still_sticky(kanban_home: Pa\n \n def test_circuit_breaker_block_still_auto_promotes(kanban_home: Path) -> None:\n     \"\"\"A child that was put into ``blocked`` *without* a worker-issued\n-    ``kanban_block`` (e.g. circuit-breaker after repeated spawn\n-    failures, manual DB triage) must still get auto-promoted when its\n-    parents complete — preserves the pre-#28712 recovery semantics.\"\"\"\n+    ``kanban_block`` (e.g. a transient crash, manual DB triage) and whose\n+    ``consecutive_failures`` is still *below* the circuit-breaker limit\n+    must get auto-promoted when its parents complete — preserves the\n+    pre-#28712 recovery semantics for genuinely transient failures.\n+\n+    The complementary case — a block whose failure count has *reached*\n+    the limit must stay blocked — is covered by\n+    ``test_kanban_db.py::test_recompute_ready_skips_tasks_at_failure_limit``\n+    (#35072).  Together they pin the contract: ``recompute_ready`` defers\n+    the give-up decision to the same effective limit the breaker uses, so\n+    the two never disagree.\n+    \"\"\"\n     with kb.connect() as conn:\n         parent = kb.create_task(conn, title=\"parent\")\n         child = kb.create_task(conn, title=\"child\", parents=[parent])\n         kb.complete_task(conn, parent, result=\"ok\")\n \n-        # Simulate a circuit-breaker / direct triage that flips status\n-        # without emitting a ``blocked`` event — exactly what\n-        # ``_record_task_failure`` does after a ``gave_up``.\n+        # Simulate a transient circuit-breaker / direct triage that flips\n+        # status without emitting a ``blocked`` event — exactly what\n+        # ``_record_task_failure`` does below the limit.  One failure is\n+        # under the default limit (2), so recovery is still correct.\n         conn.execute(\n-            \"UPDATE tasks SET status='blocked', consecutive_failures=5, \"\n-            \"last_failure_error='persistent error' WHERE id=?\",\n+            \"UPDATE tasks SET status='blocked', consecutive_failures=1, \"\n+            \"last_failure_error='transient error' WHERE id=?\",\n             (child,),\n         )\n         conn.commit()","@@ -128,8 +138,9 @@ def test_circuit_breaker_block_still_auto_promotes(kanban_home: Path) -> None:\n         assert promoted == 1\n         task = kb.get_task(conn, child)\n         assert task.status == \"ready\"\n-        assert task.consecutive_failures == 0\n-        assert task.last_failure_error is None\n+        # Counter is preserved across recovery (not reset) so the breaker\n+        # can still accumulate if the task keeps failing (#35072).\n+        assert task.consecutive_failures == 1\n \n \n def test_gave_up_event_alone_does_not_make_block_sticky(kanban_home: Path) -> None:"]},{"file":"tests/hermes_cli/test_kanban_db.py","hunks":["@@ -307,7 +307,8 @@ def test_recompute_ready_cascades_through_chain(kanban_home):\n \n \n def test_recompute_ready_promotes_blocked_with_done_parents(kanban_home):\n-    \"\"\"blocked tasks with all parents done should be promoted to ready.\"\"\"\n+    \"\"\"blocked tasks with all parents done should be promoted to ready,\n+    unless the circuit-breaker failure limit has been reached.\"\"\"\n     with kb.connect() as conn:\n         parent = kb.create_task(conn, title=\"parent\", assignee=\"a\")\n         child = kb.create_task(","@@ -316,16 +317,16 @@ def test_recompute_ready_promotes_blocked_with_done_parents(kanban_home):\n         # Complete the parent\n         kb.claim_task(conn, parent)\n         kb.complete_task(conn, parent, result=\"ok\")\n-        # Manually block the child (simulates a worker that failed\n-        # after the parent finished)\n+        # Manually block the child with zero failures (simulates a\n+        # dependency block, not a circuit-breaker block).\n         conn.execute(\n-            \"UPDATE tasks SET status='blocked', consecutive_failures=5, \"\n-            \"last_failure_error='persistent error' WHERE id=?\",\n+            \"UPDATE tasks SET status='blocked', consecutive_failures=0, \"\n+            \"last_failure_error=NULL WHERE id=?\",\n             (child,),\n         )\n         conn.commit()\n         assert kb.get_task(conn, child).status == \"blocked\"\n-        # recompute_ready should promote blocked → ready and reset failures\n+        # recompute_ready should promote blocked → ready\n         promoted = kb.recompute_ready(conn)\n         assert promoted == 1\n         task = kb.get_task(conn, child)","@@ -815,6 +816,149 @@ def test_unblock_resets_failure_counters(kanban_home):\n         assert task.last_failure_error is None\n \n \n+def test_recompute_ready_skips_tasks_at_failure_limit(kanban_home):\n+    \"\"\"recompute_ready must not auto-recover tasks whose consecutive_failures\n+    has reached the circuit-breaker limit (#35072).\n+\n+    Without this guard, a task that repeatedly exhausts its iteration\n+    budget would cycle forever: block → auto-recover (counter reset)\n+    → respawn → budget exhausted → block → …\n+    \"\"\"\n+    with kb.connect() as conn:\n+        parent = kb.create_task(conn, title=\"parent\", assignee=\"a\")\n+        child = kb.create_task(conn, title=\"child\", assignee=\"a\",\n+                               parents=[parent])\n+        # Complete the parent so the child's dependencies are satisfied.\n+        kb.claim_task(conn, parent)\n+        kb.complete_task(conn, parent, summary=\"done\")\n+\n+        # Simulate the child having exhausted its budget twice,\n+        # hitting the default failure limit (2).\n+        kb.claim_task(conn, child)\n+        kb._record_task_failure(\n+            conn, child, error=\"budget exhausted 1\",\n+            outcome=\"timed_out\", release_claim=True, end_run=True,\n+            failure_limit=2,\n+        )\n+        kb._record_task_failure(\n+            conn, child, error=\"budget exhausted 2\",\n+            outcome=\"timed_out\", release_claim=True, end_run=True,\n+            failure_limit=2,\n+        )\n+        task = kb.get_task(conn, child)\n+        assert task.status == \"blocked\"\n+        assert task.consecutive_failures >= 2\n+\n+        # recompute_ready must NOT promote this task — the circuit\n+        # breaker has tripped and it should stay blocked.\n+        promoted = kb.recompute_ready(conn)\n+        assert promoted == 0\n+        assert kb.get_task(conn, child).status == \"blocked\"\n+\n+        # Explicit unblock should still work and reset the counter.\n+        assert kb.unblock_task(conn, child)\n+        task = kb.get_task(conn, child)\n+        assert task.status == \"ready\"\n+        assert task.consecutive_failures == 0\n+\n+\n+def test_recompute_ready_recovers_below_limit(kanban_home):\n+    \"\"\"recompute_ready auto-recovers blocked tasks that haven't hit the\n+    failure limit yet — the counter is preserved across recovery.\"\"\"\n+    with kb.connect() as conn:\n+        t = kb.create_task(conn, title=\"task\", assignee=\"a\")\n+        kb.claim_task(conn, t)\n+        # One failure, below the default limit of 2.\n+        kb._record_task_failure(\n+            conn, t, error=\"budget exhausted 1\",\n+            outcome=\"timed_out\", release_claim=True, end_run=True,\n+            failure_limit=2,\n+        )\n+        task = kb.get_task(conn, t)\n+        assert task.status == \"ready\"\n+        assert task.consecutive_failures == 1\n+\n+        # Simulate being blocked by something else (not circuit breaker).\n+        conn.execute(\n+            \"UPDATE tasks SET status = 'blocked' WHERE id = ?\", (t,),\n+        )\n+        conn.commit()\n+\n+        promoted = kb.recompute_ready(conn)\n+        assert promoted == 1\n+        task = kb.get_task(conn, t)\n+        assert task.status == \"ready\"\n+        # Counter must be preserved, not reset.\n+        assert task.consecutive_failures == 1\n+\n+\n+def test_recompute_ready_honours_dispatcher_failure_limit(kanban_home):\n+    \"\"\"The guard's effective limit must follow the same resolution order\n+    as the circuit breaker (#35072): per-task max_retries → dispatcher\n+    failure_limit → DEFAULT_FAILURE_LIMIT.\n+\n+    Without threading the dispatcher's ``kanban.failure_limit`` through,\n+    the guard falls back to DEFAULT_FAILURE_LIMIT and disagrees with the\n+    breaker — sticking a task prematurely (config limit > default) or\n+    letting a tripped task escape (config limit < default).\n+    \"\"\"\n+    with kb.connect() as conn:\n+        # Config allows MORE retries than the default. A task blocked\n+        # with failures below the configured limit must still recover.\n+        t = kb.create_task(conn, title=\"lenient\", assignee=\"a\")\n+        conn.execute(\n+            \"UPDATE tasks SET status='blocked', consecutive_failures=? \"\n+            \"WHERE id=?\",\n+            (kb.DEFAULT_FAILURE_LIMIT, t),\n+        )\n+        conn.commit()\n+        # Default-limit call would stick it (failures >= default).\n+        assert kb.recompute_ready(conn) == 0\n+        assert kb.get_task(conn, t).status == \"blocked\"\n+        # Dispatcher configured a higher limit → recover, preserve counter.\n+        promoted = kb.recompute_ready(\n+            conn, failure_limit=kb.DEFAULT_FAILURE_LIMIT + 2\n+        )\n+        assert promoted == 1\n+        task = kb.get_task(conn, t)\n+        assert task.status == \"ready\"\n+        assert task.consecutive_failures == kb.DEFAULT_FAILURE_LIMIT\n+\n+        # Config allows FEWER retries than the default. A task at the\n+        # stricter limit must stay blocked even though it's below default.\n+        t2 = kb.create_task(conn, title=\"strict\", assignee=\"a\")\n+        conn.execute(\n+            \"UPDATE tasks SET status='blocked', consecutive_failures=1 \"\n+            \"WHERE id=?\",\n+            (t2,),\n+        )\n+        conn.commit()\n+        # Default-limit (2) would recover it (1 < 2).\n+        # Stricter config limit (1) must keep it blocked (1 >= 1).\n+        assert kb.recompute_ready(conn, failure_limit=1) == 0\n+        assert kb.get_task(conn, t2).status == \"blocked\"\n+\n+\n+def test_recompute_ready_per_task_max_retries_overrides_dispatcher(kanban_home):\n+    \"\"\"A per-task ``max_retries`` wins over the dispatcher failure_limit,\n+    matching ``_record_task_failure``'s resolution order.\"\"\"\n+    with kb.connect() as conn:\n+        t = kb.create_task(conn, title=\"per-task\", assignee=\"a\")\n+        # Per-task allows 4 retries; dispatcher config says 2.\n+        conn.execute(\n+            \"UPDATE tasks SET status='blocked', consecutive_failures=2, \"\n+            \"max_retries=4 WHERE id=?\",\n+            (t,),\n+        )\n+        conn.commit()\n+        # failures(2) < per-task limit(4) → recover, despite dispatcher=2.\n+        promoted = kb.recompute_ready(conn, failure_limit=2)\n+        assert promoted == 1\n+        task = kb.get_task(conn, t)\n+        assert task.status == \"ready\"\n+        assert task.consecutive_failures == 2\n+\n+\n # ---------------------------------------------------------------------------\n # Parent-completion invariant at the claim gate (RCA t_a6acd07d)\n # ---------------------------------------------------------------------------\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}