{"repo":"NousResearch/hermes-agent","public_id":"62002","display_label":"Cron stale-entry removal deleting live one-shot jobs","summary":"A bug in get_due_jobs causes stale-entry cleanup to incorrectly delete one-shot cron jobs whose runs are still active, losing run outcome and delivery records. The related items are fixes adding liveness checks before any stale-entry removal.","subsystem":null,"counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":62014,"merged_at":"2026-07-10T14:28:34+00:00"},"resolution":{"state":"resolved","by":"pr-62014"},"freshness":{"state":"verified","last_verified_at":"2026-07-10T14:28:34+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":62002,"pr_number":62014,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-07-10T14:28:34+00:00"}],"members":[{"kind":"issue","number":62002,"title":"cron: get_due_jobs stale-entry recovery deletes a live one-shot mid-run when the run outlives its run_claim TTL (run outcome + delivery record lost)","url":null,"state":"closed"},{"kind":"pr","number":62013,"title":"fix(cron): guard stale-entry removal with liveness check","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#62002","number":62002},{"kind":"pr","value":"#62014","number":62014}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":62014,"title":"fix(cron): never stale-remove a one-shot whose run is still alive (#62002)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cron/jobs.py","cron/scheduler.py","tests/cron/test_jobs.py"],"fix_hunks":[{"file":"cron/jobs.py","hunks":["@@ -192,6 +192,26 @@ def _oneshot_run_claim_ttl_seconds() -> float:\n     )\n \n \n+def _job_running_in_this_process(job_id: str) -> bool:\n+    \"\"\"Return True when the scheduler in THIS process is still running ``job_id``.\n+\n+    Direct liveness signal for stale-entry recovery (#62002): the run_claim\n+    TTL alone cannot distinguish \"the claiming tick died\" from \"the run is\n+    alive but slow\" — a run stalled on network I/O (or a laptop that slept\n+    mid-run) legitimately outlives the TTL. The in-process ticker and the run\n+    share this process, so the scheduler's running set settles the common\n+    single-gateway case without any claim-age guesswork.\n+\n+    Imported lazily: the scheduler imports this module at load, so a\n+    module-level import here would be circular.\n+    \"\"\"\n+    try:\n+        from cron.scheduler import get_running_job_ids\n+        return job_id in get_running_job_ids()\n+    except Exception:\n+        return False\n+\n+\n def _jobs_lock_file() -> Path:\n     \"\"\"Return the advisory lock path for the current cron directory.\"\"\"\n     return _current_cron_store().cron_dir / \".jobs.lock\"","@@ -1603,6 +1623,32 @@ def claim_dispatch(job_id: str) -> bool:\n         return True\n \n \n+def heartbeat_run_claim(job_id: str) -> bool:\n+    \"\"\"Refresh a one-shot's ``run_claim`` timestamp while its run is alive.\n+\n+    Called periodically from the scheduler's run monitor (#62002) so a\n+    legitimately long run keeps its claim fresh: an expired claim then really\n+    does mean \"the claiming process died\", and neither another process's tick\n+    nor this process's own next tick will re-dispatch or stale-remove the job\n+    while the run is in flight. mark_job_run() clears the claim on completion.\n+\n+    Returns True if a claim was refreshed; False when the job or its claim is\n+    gone (nothing to refresh — e.g. a manual run that never stamped one).\n+    \"\"\"\n+    with _jobs_lock():\n+        jobs = load_jobs()\n+        for job in jobs:\n+            if job.get(\"id\") != job_id:\n+                continue\n+            claim = job.get(\"run_claim\")\n+            if not isinstance(claim, dict):\n+                return False\n+            claim[\"at\"] = _hermes_now().isoformat()\n+            save_jobs(jobs)\n+            return True\n+    return False\n+\n+\n def advance_next_run(job_id: str) -> bool:\n     \"\"\"Preemptively advance next_run_at for a recurring job before execution.\n ","@@ -1986,6 +2032,25 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:\n                         times = repeat.get(\"times\")\n                         completed = repeat.get(\"completed\", 0)\n                         if times is not None and times > 0 and completed >= times:\n+                            # A live run must never have its job record deleted\n+                            # underneath it (#62002): a run that outlives the\n+                            # run_claim TTL (stream stall, laptop asleep\n+                            # mid-run) satisfies the same completed >= times +\n+                            # expired-claim condition as a dead tick, but\n+                            # mark_job_run() still needs the record to land\n+                            # last_run_at / last_status / last_delivery_error.\n+                            # If this process is still running the job, it is\n+                            # slow, not stale — keep the entry and skip.\n+                            if _job_running_in_this_process(job.get(\"id\", \"\")):\n+                                logger.info(\n+                                    \"Job '%s': dispatch limit reached (%d/%d) \"\n+                                    \"but its run is still in flight in this \"\n+                                    \"process — keeping entry\",\n+                                    job.get(\"name\", job.get(\"id\", \"?\")),\n+                                    completed,\n+                                    times,\n+                                )\n+                                continue\n                             logger.info(\n                                 \"Job '%s': one-shot dispatch limit reached (%d/%d) \"\n                                 \"— removing stale due entry\","]},{"file":"cron/scheduler.py","hunks":["@@ -20,6 +20,7 @@\n import subprocess\n import sys\n import threading\n+import time\n \n # fcntl is Unix-only; on Windows use msvcrt for file locking\n try:","@@ -237,7 +238,7 @@ def _resolve_cron_enabled_toolsets(job: dict, cfg: dict) -> list[str] | None:\n     \"QQBOT_HOME_CHANNEL\": \"QQ_HOME_CHANNEL\",\n }\n \n-from cron.jobs import get_due_jobs, mark_job_run, save_job_output, advance_next_run, claim_dispatch\n+from cron.jobs import get_due_jobs, mark_job_run, save_job_output, advance_next_run, claim_dispatch, heartbeat_run_claim\n \n # Sentinel: when a cron agent has nothing new to report, it can start its\n # response with this marker to suppress delivery.  Output is still saved","@@ -3097,6 +3098,35 @@ def run_job(\n             _cron_timeout = 600.0\n         _cron_inactivity_limit = _cron_timeout if _cron_timeout > 0 else None\n         _POLL_INTERVAL = 5.0\n+        # Keep the one-shot run_claim fresh while the run is alive (#62002):\n+        # the claim TTL is a dead-owner detector, but without a heartbeat a\n+        # run that legitimately outlives it (stream stall, laptop asleep\n+        # mid-run) is indistinguishable from a dead tick — another process\n+        # re-dispatches it and get_due_jobs stale-removes the job record out\n+        # from under the live run. Refreshing the claim from this monitor\n+        # keeps \"expired claim\" meaning \"owner died\".\n+        _job_schedule = job.get(\"schedule\")\n+        _is_oneshot = (\n+            isinstance(_job_schedule, dict) and _job_schedule.get(\"kind\") == \"once\"\n+        )\n+        _CLAIM_HEARTBEAT_SECONDS = 60.0\n+        _last_claim_heartbeat = time.monotonic()\n+\n+        def _heartbeat_run_claim_if_due():\n+            nonlocal _last_claim_heartbeat\n+            if not _is_oneshot:\n+                return\n+            _mono = time.monotonic()\n+            if _mono - _last_claim_heartbeat < _CLAIM_HEARTBEAT_SECONDS:\n+                return\n+            _last_claim_heartbeat = _mono\n+            try:\n+                heartbeat_run_claim(job_id)\n+            except Exception:\n+                logger.debug(\n+                    \"Job '%s': run_claim heartbeat failed\", job_name, exc_info=True\n+                )\n+\n         _cron_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)\n         # Preserve scheduler-scoped ContextVar state (for example skill-declared\n         # env passthrough registrations) when the cron run hops into the worker","@@ -3106,8 +3136,20 @@ def run_job(\n         _inactivity_timeout = False\n         try:\n             if _cron_inactivity_limit is None:\n-                # Unlimited — just wait for the result.\n-                result = _cron_future.result()\n+                # Unlimited — no inactivity watchdog, but a one-shot still\n+                # needs its run_claim heartbeat, so poll instead of blocking.\n+                if _is_oneshot:\n+                    result = None\n+                    while True:\n+                        done, _ = concurrent.futures.wait(\n+                            {_cron_future}, timeout=_POLL_INTERVAL,\n+                        )\n+                        if done:\n+                            result = _cron_future.result()\n+                            break\n+                        _heartbeat_run_claim_if_due()\n+                else:\n+                    result = _cron_future.result()\n             else:\n                 result = None\n                 while True:","@@ -3117,6 +3159,7 @@ def run_job(\n                     if done:\n                         result = _cron_future.result()\n                         break\n+                    _heartbeat_run_claim_if_due()\n                     # Agent still running — check inactivity.\n                     _idle_secs = 0.0\n                     if hasattr(agent, \"get_activity_summary\"):"]},{"file":"tests/cron/test_jobs.py","hunks":["@@ -20,6 +20,7 @@\n     mark_job_run,\n     advance_next_run,\n     claim_dispatch,\n+    heartbeat_run_claim,\n     get_due_jobs,\n     save_job_output,\n )","@@ -1103,6 +1104,102 @@ def test_mark_job_run_clears_one_shot_run_claim(self, tmp_cron_dir, monkeypatch)\n         mark_job_run(\"claimclear\", True)\n         assert get_job(\"claimclear\")[\"run_claim\"] is None\n \n+    def test_stale_maxed_oneshot_kept_while_running_in_this_process(\n+        self, tmp_cron_dir, monkeypatch\n+    ):\n+        \"\"\"#62002: a live run must never have its job record deleted underneath it.\n+\n+        A one-shot whose run outlives the run_claim TTL (stream stall, laptop\n+        asleep mid-run) satisfies the same completed >= times + expired-claim\n+        condition as a dead tick. When the scheduler in this process still has\n+        the job in its running set, the stale-entry recovery must keep the\n+        record so the in-flight run's mark_job_run() can land its outcome —\n+        and remove it only once the run is actually gone.\n+        \"\"\"\n+        import cron.scheduler as scheduler_mod\n+        from cron.jobs import _hermes_now, _oneshot_run_claim_ttl_seconds\n+        monkeypatch.delenv(\"HERMES_CRON_TIMEOUT\", raising=False)\n+        ttl = _oneshot_run_claim_ttl_seconds()\n+        t0 = _hermes_now()\n+        run_at = (t0 - timedelta(seconds=ttl + 300)).isoformat()\n+        # Mid-run store shape: claim_dispatch committed completed=1 and the\n+        # run_claim was stamped at fire time; next_run_at is only resolved by\n+        # mark_job_run, so it still points at the (past) fire time.\n+        save_jobs([{\n+            \"id\": \"inflight\", \"name\": \"flight check\", \"prompt\": \"x\",\n+            \"schedule\": {\"kind\": \"once\", \"run_at\": run_at},\n+            \"next_run_at\": run_at, \"enabled\": True, \"state\": \"scheduled\",\n+            \"repeat\": {\"times\": 1, \"completed\": 1},\n+            \"run_claim\": {\"at\": run_at, \"by\": \"this-machine\"},\n+        }])\n+\n+        # Run still alive in this process → keep the record, dispatch nothing.\n+        monkeypatch.setattr(\n+            scheduler_mod, \"get_running_job_ids\", lambda: frozenset({\"inflight\"})\n+        )\n+        assert get_due_jobs() == []\n+        assert get_job(\"inflight\") is not None  # still visible to list/run\n+\n+        # The claiming tick really died (running set empty) → recovered as before.\n+        monkeypatch.setattr(\n+            scheduler_mod, \"get_running_job_ids\", lambda: frozenset()\n+        )\n+        assert get_due_jobs() == []\n+        assert get_job(\"inflight\") is None  # stale entry cleaned up\n+\n+    def test_run_claim_heartbeat_keeps_long_run_claimed_past_ttl(\n+        self, tmp_cron_dir, monkeypatch\n+    ):\n+        \"\"\"#62002 cross-process leg: a heartbeat-refreshed claim never expires\n+        while the run is alive, so no other tick re-dispatches or stale-removes\n+        the job even when the run outlives the original TTL horizon.\"\"\"\n+        monkeypatch.delenv(\"HERMES_CRON_TIMEOUT\", raising=False)\n+        from cron.jobs import _hermes_now, _oneshot_run_claim_ttl_seconds\n+        ttl = _oneshot_run_claim_ttl_seconds()\n+        t0 = _hermes_now()\n+        run_at = (t0 - timedelta(seconds=5)).isoformat()\n+        save_jobs([{\n+            \"id\": \"slowrun\", \"name\": \"R\", \"prompt\": \"x\",\n+            \"schedule\": {\"kind\": \"once\", \"run_at\": run_at},\n+            \"next_run_at\": run_at, \"enabled\": True, \"state\": \"scheduled\",\n+            \"repeat\": {\"times\": 1, \"completed\": 0},\n+        }])\n+\n+        # Tick claims + dispatches the job.\n+        assert [j[\"id\"] for j in get_due_jobs()] == [\"slowrun\"]\n+        assert claim_dispatch(\"slowrun\") is True\n+\n+        # Mid-run heartbeat before the TTL horizon refreshes the claim.\n+        monkeypatch.setattr(\"cron.jobs._hermes_now\",\n+                            lambda: t0 + timedelta(seconds=ttl - 60))\n+        assert heartbeat_run_claim(\"slowrun\") is True\n+\n+        # Past the ORIGINAL claim's TTL horizon: without the heartbeat this\n+        # tick would stale-remove the maxed one-shot; with it the claim is\n+        # fresh, so the job is skipped and the record survives.\n+        monkeypatch.setattr(\"cron.jobs._hermes_now\",\n+                            lambda: t0 + timedelta(seconds=ttl + 10))\n+        assert get_due_jobs() == []\n+        assert get_job(\"slowrun\") is not None\n+\n+        # Run completes → outcome lands on a record that still exists\n+        # (times=1 reached, so mark_job_run retires the job normally).\n+        mark_job_run(\"slowrun\", True)\n+        assert get_job(\"slowrun\") is None\n+\n+    def test_heartbeat_run_claim_noop_without_claim(self, tmp_cron_dir):\n+        \"\"\"heartbeat_run_claim is a safe no-op when there is nothing to refresh\n+        (manual run that never stamped a claim, or the job is gone).\"\"\"\n+        future = (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat()\n+        save_jobs([{\n+            \"id\": \"noclaim\", \"name\": \"R\", \"prompt\": \"x\",\n+            \"schedule\": {\"kind\": \"once\", \"run_at\": future},\n+            \"next_run_at\": future, \"enabled\": True, \"state\": \"scheduled\",\n+        }])\n+        assert heartbeat_run_claim(\"noclaim\") is False\n+        assert heartbeat_run_claim(\"missing-job\") is False\n+        assert get_job(\"noclaim\").get(\"run_claim\") is None\n+\n \n     def test_broken_cron_without_next_run_is_recovered(self, tmp_cron_dir, monkeypatch):\n         now = datetime(2026, 3, 18, 10, 0, 0, tzinfo=timezone.utc)\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}