{"repo":"NousResearch/hermes-agent","public_id":"38758","display_label":"Cron one-shot jobs re-fire after gateway crash","summary":"These members address a bug where one-shot cron jobs re-trigger indefinitely if the gateway dies during or after execution. The fixes focus on adding a pre-run dispatch guard and committing the dispatch record before side effects occur.","subsystem":"cron","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":56177,"merged_at":"2026-07-01T08:30:37+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-07-01T08:30:37+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":30719,"pr_number":56177,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-07-01T08:30:37+00:00"},{"issue_number":38758,"pr_number":56177,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-07-01T08:30:37+00:00"}],"members":[{"kind":"issue","number":38758,"title":"Cron one-shot whose script kills the gateway re-fires forever (no pre-run dispatch guard)","url":null,"state":"closed"},{"kind":"pr","number":38920,"title":"fix(cron): prevent one-shot jobs from re-firing after gateway death (#38758)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#56177","number":56177}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":56177,"title":"fix(cron): commit one-shot dispatch before side effect to stop crash re-fire loop (#38758)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cron/jobs.py","cron/scheduler.py","tests/cron/test_jobs.py","tests/cron/test_scheduler.py"],"fix_hunks":[{"file":"cron/jobs.py","hunks":["@@ -1249,13 +1249,27 @@ def mark_job_run(job_id: str, success: bool, error: Optional[str] = None,\n                 # be claimed again on its next fire (Phase 4C CAS).\n                 job[\"fire_claim\"] = None\n                 \n-                # Increment completed count\n+                # Increment completed count.  Finite one-shot jobs are\n+                # pre-claimed by claim_dispatch() BEFORE the side effect runs\n+                # (issue #38758), which already incremented completed — do not\n+                # double-count them here.  Recurring jobs and direct callers\n+                # with no pre-run claim still get the legacy increment.\n                 if job.get(\"repeat\"):\n-                    job[\"repeat\"][\"completed\"] = job[\"repeat\"].get(\"completed\", 0) + 1\n-                    \n+                    repeat = job[\"repeat\"]\n+                    times = repeat.get(\"times\")\n+                    completed = repeat.get(\"completed\", 0)\n+                    kind = job.get(\"schedule\", {}).get(\"kind\")\n+                    preclaimed_oneshot = (\n+                        kind == \"once\"\n+                        and times is not None\n+                        and times > 0\n+                        and completed > 0\n+                    )\n+                    if not preclaimed_oneshot:\n+                        completed += 1\n+                        repeat[\"completed\"] = completed\n+\n                     # Check if we've hit the repeat limit\n-                    times = job[\"repeat\"].get(\"times\")\n-                    completed = job[\"repeat\"][\"completed\"]\n                     if times is not None and times > 0 and completed >= times:\n                         # Remove the job (limit reached)\n                         jobs.pop(i)","@@ -1300,6 +1314,69 @@ def mark_job_run(job_id: str, success: bool, error: Optional[str] = None,\n         logger.warning(\"mark_job_run: job_id %s not found, skipping save\", job_id)\n \n \n+def claim_dispatch(job_id: str) -> bool:\n+    \"\"\"Atomically claim a finite one-shot job dispatch BEFORE execution.\n+\n+    Increments ``repeat.completed`` under the cross-process jobs lock and\n+    persists the claim immediately, so that if the tick dies mid-execution\n+    (gateway kill, OOM, segfault, hard-timeout) the dispatch is not lost.\n+    This converts finite one-shot jobs from *at-least-once* to *at-most-times*\n+    semantics — a job that self-destructs fires at most ``repeat.times`` times\n+    instead of infinitely (issue #38758).\n+\n+    Returns ``True`` if the caller may proceed to run the job, ``False`` if the\n+    dispatch limit is already reached (in which case the stale job is removed).\n+\n+    Only claims jobs with ``schedule.kind == \"once\"`` and ``repeat.times > 0``.\n+    Recurring jobs (they use ``advance_next_run``) and infinite-repeat / no-repeat\n+    jobs are left unchanged and always allowed to proceed.\n+    \"\"\"\n+    with _jobs_lock():\n+        jobs = load_jobs()\n+        for i, job in enumerate(jobs):\n+            if job[\"id\"] != job_id:\n+                continue\n+            if job.get(\"schedule\", {}).get(\"kind\") != \"once\":\n+                return True  # recurring jobs use advance_next_run(), not dispatch claims\n+            repeat = job.get(\"repeat\")\n+            if not repeat:\n+                return True  # no repeat limit — always dispatch\n+            times = repeat.get(\"times\")\n+            if times is None or times <= 0:\n+                return True  # infinite — always dispatch\n+            completed = repeat.get(\"completed\", 0)\n+            if completed >= times:\n+                # Already dispatched the max number of times (e.g. a prior\n+                # tick claimed then died before mark_job_run could remove it).\n+                # Clean up so it stops appearing as due on every tick.\n+                jobs.pop(i)\n+                save_jobs(jobs)\n+                logger.info(\n+                    \"Job '%s': dispatch limit reached (%d/%d) — removing\",\n+                    job.get(\"name\", job[\"id\"]),\n+                    completed,\n+                    times,\n+                )\n+                return False\n+            # Claim this dispatch before the side effect runs.\n+            repeat[\"completed\"] = completed + 1\n+            save_jobs(jobs)\n+            logger.debug(\n+                \"Job '%s': claimed dispatch %d/%d\",\n+                job.get(\"name\", job[\"id\"]),\n+                repeat[\"completed\"],\n+                times,\n+            )\n+            return True\n+\n+        logger.debug(\n+            \"claim_dispatch: job_id %s not in store — proceeding without claim \"\n+            \"(handed-in job dict; nothing to persist a claim against)\",\n+            job_id,\n+        )\n+        return True\n+\n+\n def advance_next_run(job_id: str) -> bool:\n     \"\"\"Preemptively advance next_run_at for a recurring job before execution.\n ","@@ -1543,6 +1620,31 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:\n                             break\n                     # Fall through to due.append(job) — execute once now\n \n+            # One-shot dispatch-limit guard (issue #38758): a finite one-shot\n+            # claimed via claim_dispatch() but whose tick died before\n+            # mark_job_run could remove it will have completed >= times while\n+            # still looking due (last_run_at was never written, so the\n+            # recovery helper re-armed it). Remove it instead of re-firing.\n+            if kind == \"once\":\n+                repeat = job.get(\"repeat\")\n+                if repeat:\n+                    times = repeat.get(\"times\")\n+                    completed = repeat.get(\"completed\", 0)\n+                    if times is not None and times > 0 and completed >= times:\n+                        logger.info(\n+                            \"Job '%s': one-shot dispatch limit reached (%d/%d) \"\n+                            \"— removing stale due entry\",\n+                            job.get(\"name\", job[\"id\"]),\n+                            completed,\n+                            times,\n+                        )\n+                        for rj in raw_jobs:\n+                            if rj[\"id\"] == job[\"id\"]:\n+                                raw_jobs.remove(rj)\n+                                needs_save = True\n+                                break\n+                        continue\n+\n             due.append(job)\n \n     if needs_save:"]},{"file":"cron/scheduler.py","hunks":["@@ -236,7 +236,7 @@ def _resolve_cron_enabled_toolsets(job: dict, cfg: dict) -> list[str] | None:\n     \"QQBOT_HOME_CHANNEL\": \"QQ_HOME_CHANNEL\",\n }\n \n-from cron.jobs import get_due_jobs, mark_job_run, save_job_output, advance_next_run\n+from cron.jobs import get_due_jobs, mark_job_run, save_job_output, advance_next_run, claim_dispatch\n \n # Sentinel: when a cron agent has nothing new to report, it can start its\n # response with this marker to suppress delivery.  Output is still saved","@@ -2781,6 +2781,20 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) -\n     failure is recorded via ``mark_job_run``), False only if processing raised.\n     \"\"\"\n     try:\n+        # Pre-run dispatch claim (issue #38758): atomically commit a finite\n+        # one-shot's dispatch BEFORE its side effect runs, so a tick that dies\n+        # mid-execution (gateway kill, OOM, segfault, hard-timeout) cannot\n+        # re-fire the job forever on restart. No-op for recurring jobs (they\n+        # use advance_next_run) and infinite/no-repeat jobs. This lives here in\n+        # the shared body so BOTH the built-in ticker and the external provider\n+        # (Chronos fire_due) get at-most-times semantics.\n+        if not claim_dispatch(job[\"id\"]):\n+            logger.info(\n+                \"Job '%s': one-shot dispatch limit reached — skipping\",\n+                job.get(\"name\", job[\"id\"]),\n+            )\n+            return True  # not an error — already handled/removed\n+\n         success, output, final_response, error = run_job(job)\n \n         output_file = save_job_output(job[\"id\"], output)"]},{"file":"tests/cron/test_jobs.py","hunks":["@@ -19,6 +19,7 @@\n     remove_job,\n     mark_job_run,\n     advance_next_run,\n+    claim_dispatch,\n     get_due_jobs,\n     save_job_output,\n )","@@ -1314,3 +1315,102 @@ def test_cron_output_keep_defaults_on_bad_config(self, monkeypatch):\n             \"hermes_cli.config.load_config\", lambda: {\"cron\": {\"output_retention\": \"oops\"}}\n         )\n         assert jobs._cron_output_keep() == jobs._CRON_OUTPUT_DEFAULT_KEEP\n+\n+\n+# =========================================================================\n+# claim_dispatch — pre-run one-shot crash safety (issue #38758)\n+# =========================================================================\n+\n+class TestClaimDispatch:\n+    \"\"\"One-shot jobs must commit their dispatch BEFORE the side effect runs, so\n+    a tick that dies mid-execution (gateway kill, OOM, hard-timeout) can re-fire\n+    the job at most ``repeat.times`` times instead of infinitely.\"\"\"\n+\n+    def _oneshot(self, times=1, completed=0):\n+        return {\n+            \"id\": \"os1\",\n+            \"name\": \"one-shot\",\n+            \"enabled\": True,\n+            \"schedule\": {\"kind\": \"once\", \"run_at\": \"2026-01-01T00:00:00+00:00\"},\n+            \"repeat\": {\"times\": times, \"completed\": completed},\n+        }\n+\n+    def test_claim_increments_and_persists(self, tmp_cron_dir):\n+        save_jobs([self._oneshot(times=1, completed=0)])\n+        assert claim_dispatch(\"os1\") is True\n+        # Persisted BEFORE any side effect — survives a crash.\n+        assert load_jobs()[0][\"repeat\"][\"completed\"] == 1\n+\n+    def test_already_dispatched_oneshot_is_removed(self, tmp_cron_dir):\n+        # A prior tick claimed (completed==times) then died before mark_job_run\n+        # could remove the job.  The next claim must refuse AND clean up.\n+        save_jobs([self._oneshot(times=1, completed=1)])\n+        assert claim_dispatch(\"os1\") is False\n+        assert load_jobs() == []  # removed, will not re-fire\n+\n+    def test_recurring_job_is_not_claimed(self, tmp_cron_dir):\n+        job = {\n+            \"id\": \"rec\",\n+            \"schedule\": {\"kind\": \"interval\", \"minutes\": 5},\n+            \"repeat\": {\"times\": 3, \"completed\": 0},\n+        }\n+        save_jobs([job])\n+        assert claim_dispatch(\"rec\") is True\n+        # Recurring jobs use advance_next_run(); claim must NOT touch completed.\n+        assert load_jobs()[0][\"repeat\"][\"completed\"] == 0\n+\n+    def test_infinite_oneshot_not_claimed(self, tmp_cron_dir):\n+        job = self._oneshot(times=0, completed=0)  # times<=0 means infinite\n+        save_jobs([job])\n+        assert claim_dispatch(\"os1\") is True\n+        assert load_jobs()[0][\"repeat\"][\"completed\"] == 0\n+\n+    def test_no_repeat_block_not_claimed(self, tmp_cron_dir):\n+        job = {\"id\": \"os1\", \"schedule\": {\"kind\": \"once\", \"run_at\": \"2026-01-01T00:00:00+00:00\"}}\n+        save_jobs([job])\n+        assert claim_dispatch(\"os1\") is True\n+        assert \"repeat\" not in load_jobs()[0]\n+\n+    def test_missing_job_proceeds(self, tmp_cron_dir):\n+        # A handed-in job dict not persisted in the store (external provider /\n+        # direct caller) can't be claimed — proceed rather than suppress it.\n+        save_jobs([])\n+        assert claim_dispatch(\"ghost\") is True\n+\n+    def test_mark_job_run_does_not_double_count_preclaimed_oneshot(self, tmp_cron_dir):\n+        # Full lifecycle: claim bumps completed to times, then mark_job_run must\n+        # NOT increment again — it recognizes the pre-claim and removes the job.\n+        save_jobs([self._oneshot(times=1, completed=0)])\n+        assert claim_dispatch(\"os1\") is True\n+        assert load_jobs()[0][\"repeat\"][\"completed\"] == 1\n+        mark_job_run(\"os1\", success=True)\n+        assert load_jobs() == []  # completed once, removed — not fired twice\n+\n+    def test_mark_job_run_still_increments_recurring(self, tmp_cron_dir):\n+        # The double-count guard is one-shot-specific; recurring jobs keep the\n+        # legacy post-run increment.\n+        job = {\n+            \"id\": \"rec\",\n+            \"schedule\": {\"kind\": \"interval\", \"minutes\": 5},\n+            \"repeat\": {\"times\": 3, \"completed\": 1},\n+        }\n+        save_jobs([job])\n+        mark_job_run(\"rec\", success=True)\n+        assert load_jobs()[0][\"repeat\"][\"completed\"] == 2\n+\n+    def test_get_due_jobs_removes_stale_maxed_oneshot(self, tmp_cron_dir):\n+        # A claimed one-shot whose tick died leaves completed>=times with\n+        # last_run_at still unset, so the recovery helper re-arms it as due.\n+        # get_due_jobs must drop it instead of returning it for another fire.\n+        past = (datetime.now(timezone.utc) - timedelta(seconds=5)).isoformat()\n+        save_jobs([{\n+            \"id\": \"os1\",\n+            \"name\": \"one-shot\",\n+            \"enabled\": True,\n+            \"schedule\": {\"kind\": \"once\", \"run_at\": past},\n+            \"repeat\": {\"times\": 1, \"completed\": 1},\n+            \"next_run_at\": None,\n+        }])\n+        due = get_due_jobs()\n+        assert due == []\n+        assert load_jobs() == []  # cleaned up"]},{"file":"tests/cron/test_scheduler.py","hunks":["@@ -2542,6 +2542,46 @@ def test_whitespace_only_response_is_marked_failed_not_delivered(self):\n         )\n \n \n+class TestOneShotDispatchClaim:\n+    \"\"\"run_one_job must claim a finite one-shot's dispatch BEFORE run_job so a\n+    tick that dies mid-execution can't re-fire it forever (issue #38758).\"\"\"\n+\n+    def _oneshot(self):\n+        return {\n+            \"id\": \"monitor-job\",\n+            \"name\": \"monitor\",\n+            \"deliver\": \"origin\",\n+            \"origin\": {\"platform\": \"telegram\", \"chat_id\": \"123\"},\n+            \"schedule\": {\"kind\": \"once\", \"run_at\": \"2026-01-01T00:00:00+00:00\"},\n+            \"repeat\": {\"times\": 1, \"completed\": 0},\n+        }\n+\n+    def test_claim_runs_before_run_job(self):\n+        order = []\n+        with patch(\"cron.scheduler.get_due_jobs\", return_value=[self._oneshot()]), \\\n+             patch(\"cron.scheduler.claim_dispatch\", side_effect=lambda _id: order.append(\"claim\") or True), \\\n+             patch(\"cron.scheduler.run_job\", side_effect=lambda _j: order.append(\"run\") or (True, \"# out\", \"ok\", None)), \\\n+             patch(\"cron.scheduler.save_job_output\", return_value=\"/tmp/out.md\"), \\\n+             patch(\"cron.scheduler._deliver_result\"), \\\n+             patch(\"cron.scheduler.mark_job_run\"):\n+            from cron.scheduler import tick\n+            tick(verbose=False)\n+        assert order == [\"claim\", \"run\"]  # claim strictly before side effect\n+\n+    def test_refused_claim_skips_run_job(self):\n+        with patch(\"cron.scheduler.get_due_jobs\", return_value=[self._oneshot()]), \\\n+             patch(\"cron.scheduler.claim_dispatch\", return_value=False), \\\n+             patch(\"cron.scheduler.run_job\") as run_mock, \\\n+             patch(\"cron.scheduler.save_job_output\"), \\\n+             patch(\"cron.scheduler._deliver_result\") as deliver_mock, \\\n+             patch(\"cron.scheduler.mark_job_run\") as mark_mock:\n+            from cron.scheduler import tick\n+            tick(verbose=False)\n+        run_mock.assert_not_called()\n+        deliver_mock.assert_not_called()\n+        mark_mock.assert_not_called()\n+\n+\n class TestBuildJobPromptSilentHint:\n     \"\"\"Verify _build_job_prompt always injects [SILENT] guidance.\"\"\"\n \n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}