{"repo":"NousResearch/hermes-agent","public_id":"36910","display_label":"Kanban task/run status desync bug and fixes","summary":"A bug causing desynchronization between task and run statuses in the Kanban board after worker reclaim or restart events. Includes related fixes to flag the desync condition and make the reclaim process aware of claim locks.","subsystem":"cron","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":50366,"merged_at":"2026-06-21T19:49:08+00:00"},"resolution":{"state":"resolved","by":"pr-50366"},"freshness":{"state":"verified","last_verified_at":"2026-06-21T19:49:08+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":36910,"pr_number":50366,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-21T19:49:08+00:00"}],"members":[{"kind":"issue","number":36910,"title":"[Bug]: Title: Kanban Task Status Desynchronization After Worker Reclaim/Restart","url":null,"state":"closed"},{"kind":"pr","number":37092,"title":"fix(kanban): flag run/status desync when a Ready task has an open running run (#36910)","url":null,"state":"closed","close_reason":{"category":"stale_abandoned","implication":"active"},"merged":false},{"kind":"pr","number":50366,"title":"fix(kanban): make reclaim claim-lock-aware to stop task/run status desync","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/kanban_db.py","tests/hermes_cli/test_kanban_reclaim_claim_lock_guard.py"],"fix_hunks":[{"file":"hermes_cli/kanban_db.py","hunks":["@@ -5662,8 +5662,9 @@ def enforce_max_runtime(\n                 \"UPDATE tasks SET status = 'ready', claim_lock = NULL, \"\n                 \"claim_expires = NULL, worker_pid = NULL, \"\n                 \"last_heartbeat_at = NULL \"\n-                \"WHERE id = ? AND status = 'running'\",\n-                (tid,),\n+                \"WHERE id = ? AND status = 'running' \"\n+                \"  AND worker_pid = ? AND claim_lock IS ?\",\n+                (tid, pid, row[\"claim_lock\"]),\n             )\n             if cur.rowcount == 1:\n                 payload = {","@@ -5787,8 +5788,9 @@ def detect_stale_running(\n                 \"UPDATE tasks SET status = 'ready', claim_lock = NULL, \"\n                 \"claim_expires = NULL, worker_pid = NULL, \"\n                 \"last_heartbeat_at = NULL \"\n-                \"WHERE id = ? AND status = 'running'\",\n-                (tid,),\n+                \"WHERE id = ? AND status = 'running' \"\n+                \"  AND claim_lock IS ?\",\n+                (tid, row[\"claim_lock\"]),\n             )\n             if cur.rowcount != 1:\n                 continue","@@ -5960,8 +5962,9 @@ def detect_crashed_workers(conn: sqlite3.Connection) -> list[str]:\n             cur = conn.execute(\n                 \"UPDATE tasks SET status = 'ready', claim_lock = NULL, \"\n                 \"claim_expires = NULL, worker_pid = NULL \"\n-                \"WHERE id = ? AND status = 'running'\",\n-                (row[\"id\"],),\n+                \"WHERE id = ? AND status = 'running' \"\n+                \"  AND worker_pid = ? AND claim_lock IS ?\",\n+                (row[\"id\"], pid, row[\"claim_lock\"]),\n             )\n             if cur.rowcount == 1:\n                 # Rate-limited requeues are a clean release, not a crash —"]},{"file":"tests/hermes_cli/test_kanban_reclaim_claim_lock_guard.py","hunks":["@@ -0,0 +1,113 @@\n+\"\"\"Tests: reclaim paths are claim-lock-aware so they can't desync a re-claimed\n+task (issue #36910).\n+\n+A stale crash/stale-claim/max-runtime reclaim, computed from a snapshot of an\n+OLD worker, used to reset ``tasks.status`` back to ``ready`` with only a\n+``WHERE status='running'`` guard. If the task had since been reclaimed AND\n+re-claimed by a NEW worker (new run, new claim_lock, live pid), that stale\n+UPDATE clobbered the live task: ``tasks.status='ready'`` while the new\n+``task_runs.status='running'`` and the worker kept executing — the board showed\n+the task in the Ready lane and the dispatcher could treat live work as\n+available. The reset is now gated on the snapshot's ``claim_lock`` (and pid),\n+so it only fires when the task is still owned by the worker the reclaim was\n+computed for.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import subprocess\n+from pathlib import Path\n+\n+import pytest\n+\n+from hermes_cli import kanban_db as kb\n+\n+\n[redacted:email]\n+def kanban_home(tmp_path, monkeypatch):\n+    home = tmp_path / \".hermes\"\n+    home.mkdir()\n+    monkeypatch.setenv(\"HERMES_HOME\", str(home))\n+    monkeypatch.setenv(\"HERMES_KANBAN_HOME\", str(home))\n+    monkeypatch.setenv(\"HERMES_KANBAN_CRASH_GRACE_SECONDS\", \"0\")\n+    monkeypatch.setattr(Path, \"home\", lambda: tmp_path)\n+    db_path = kb.kanban_db_path(board=\"default\")\n+    kb._INITIALIZED_PATHS.discard(str(db_path.resolve()))\n+    kb.init_db()\n+    return home\n+\n+\n[redacted:email]\n+def conn(kanban_home):\n+    with kb.connect() as c:\n+        yield c\n+\n+\n+def test_stale_crash_reset_rejected_for_reclaimed_task(conn):\n+    \"\"\"A reset carrying an OLD worker's claim_lock must NOT clobber a task\n+    that has since been re-claimed by a new worker.\"\"\"\n+    host = kb._claimer_id().split(\":\", 1)[0]\n+    tid = kb.create_task(conn, title=\"desync\", assignee=\"w\")\n+\n+    # Worker A claims, then dies.\n+    kb.claim_task(conn, tid, claimer=f\"{host}:A\")\n+    dead = subprocess.Popen([\"true\"])\n+    dead.wait()\n+    kb._set_worker_pid(conn, tid, dead.pid)\n+    old = conn.execute(\n+        \"SELECT claim_lock, worker_pid FROM tasks WHERE id=?\", (tid,)\n+    ).fetchone()\n+\n+    # Reclaim + re-claim by worker B (alive).\n+    conn.execute(\n+        \"UPDATE tasks SET status='ready', claim_lock=NULL, claim_expires=NULL, \"\n+        \"worker_pid=NULL, current_run_id=NULL WHERE id=?\",\n+        (tid,),\n+    )\n+    conn.commit()\n+    kb.claim_task(conn, tid, claimer=f\"{host}:B\")\n+    sleeper = subprocess.Popen([\"sleep\", \"30\"])\n+    try:\n+        kb._set_worker_pid(conn, tid, sleeper.pid)\n+\n+        # The stale reset for worker A — same shape as the guarded UPDATE in\n+        # detect_crashed_workers — must reject (rowcount 0) because B owns it.\n+        cur = conn.execute(\n+            \"UPDATE tasks SET status='ready', claim_lock=NULL, \"\n+            \"claim_expires=NULL, worker_pid=NULL \"\n+            \"WHERE id=? AND status='running' AND worker_pid=? AND claim_lock IS ?\",\n+            (tid, old[\"worker_pid\"], old[\"claim_lock\"]),\n+        )\n+        conn.commit()\n+        assert cur.rowcount == 0, \"stale reclaim wrongly clobbered the re-claimed task\"\n+\n+        final = conn.execute(\n+            \"SELECT status, claim_lock FROM tasks WHERE id=?\", (tid,)\n+        ).fetchone()\n+        assert final[\"status\"] == \"running\"\n+        assert final[\"claim_lock\"] == f\"{host}:B\"\n+    finally:\n+        sleeper.terminate()\n+\n+\n+def test_genuine_crash_still_reclaims(conn):\n+    \"\"\"When the claim_lock still matches the dead worker, the crash reclaim\n+    fires normally — the guard must not break the legitimate path.\"\"\"\n+    host = kb._claimer_id().split(\":\", 1)[0]\n+    tid = kb.create_task(conn, title=\"legit\", assignee=\"w\")\n+    kb.claim_task(conn, tid, claimer=f\"{host}:A\")\n+    dead = subprocess.Popen([\"true\"])\n+    dead.wait()\n+    kb._set_worker_pid(conn, tid, dead.pid)\n+    # Rewind started_at so the launch grace window doesn't skip the check.\n+    conn.execute(\"UPDATE tasks SET started_at = started_at - 9999 WHERE id=?\", (tid,))\n+    conn.execute(\n+        \"UPDATE task_runs SET started_at = started_at - 9999 WHERE task_id=?\", (tid,)\n+    )\n+    conn.commit()\n+    kb._record_worker_exit(dead.pid, 1 << 8)  # nonzero exit → crash\n+\n+    crashed = kb.detect_crashed_workers(conn)\n+    assert tid in crashed\n+    final = conn.execute(\"SELECT status FROM tasks WHERE id=?\", (tid,)).fetchone()\n+    assert final[\"status\"] in (\"ready\", \"blocked\", \"todo\")\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}