{"repo":"NousResearch/hermes-agent","public_id":"3752","display_label":"Cron scheduler starvation and missed-job fixes","summary":"These items address cron scheduler issues where holding locks during job execution causes starvation and missed runs, and where jobs fast-forwarded past a missed-grace window are skipped instead of executed. Multiple approaches to decoupling tick dispatch from job completion and releasing locks earlier are represented.","subsystem":"cron","counts":{"members":22,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":6,"merged_prs":3,"attempt_prs":13},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":13021},"resolution":{"state":"partially_resolved","by":"pr-13021,pr-39008,pr-50062"},"freshness":{"state":"verified","last_verified_at":"2026-06-22T10:20:54+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.08173160173160171},"fix_edges":[{"issue_number":9086,"pr_number":13021,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":10769,"pr_number":13021,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":33315,"pr_number":50062,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":37312,"pr_number":39008,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":41037,"pr_number":50062,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":3752,"title":"Long-running cron jobs block the scheduler tick loop","url":null,"state":"closed"},{"kind":"issue","number":9086,"title":"# [Bug]: Serial cron tick execution causes scheduled jobs to be silently skipped when preceding jobs run long","url":null,"state":"closed"},{"kind":"issue","number":27485,"title":"cron: tick lock held for full job duration causes scheduler starvation and missed runs on long-running jobs","url":null,"state":"closed"},{"kind":"issue","number":33315,"title":"bug(cron): recurring jobs perpetually deferred after long execution — missed-grace fast-forward skips instead of running","url":null,"state":"closed"},{"kind":"issue","number":37312,"title":"bug(cron): tick() blocks on as_completed() — decoupling dispatch from completion needed","url":null,"state":"closed"},{"kind":"issue","number":38779,"title":"Cron ticker waits for all due jobs, so one long active job can starve later schedules","url":null,"state":"closed"},{"kind":"pr","number":3764,"title":"fix(cron): release scheduler lock before running jobs","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":7158,"title":"feat(cron): run independent cron jobs in parallel with safe non-overlap","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"eb9cde734","number":0},{"kind":"pr","value":"#3752","number":3752}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":9097,"title":"fix(cron): #9086 differentiate scheduler-blocking from gateway downti…","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"c86915024","number":0},{"kind":"pr","value":"#9086","number":9086},{"kind":"pr","value":"#13021","number":13021},{"kind":"pr","value":"#9086","number":9086},{"kind":"pr","value":"#9086","number":9086}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":9169,"title":"fix(cron): run due jobs in parallel to prevent serial tick starvation","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#13021","number":13021}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":13021,"title":"fix(cron): run due jobs in parallel to prevent serial tick starvation","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cron/jobs.py","cron/scheduler.py","gateway/session_context.py","hermes_cli/config.py","tests/cron/test_scheduler.py","tools/send_message_tool.py"],"fix_hunks":[{"file":"cron/jobs.py","hunks":["@@ -9,6 +9,7 @@\n import json\n import logging\n import tempfile\n+import threading\n import os\n import re\n import uuid","@@ -34,6 +35,11 @@\n HERMES_DIR = get_hermes_home().resolve()\n CRON_DIR = HERMES_DIR / \"cron\"\n JOBS_FILE = CRON_DIR / \"jobs.json\"\n+\n+# In-process lock protecting load_jobs→modify→save_jobs cycles.\n+# Required when tick() runs jobs in parallel threads — without this,\n+# concurrent mark_job_run / advance_next_run calls can clobber each other.\n+_jobs_file_lock = threading.Lock()\n OUTPUT_DIR = CRON_DIR / \"output\"\n ONESHOT_GRACE_SECONDS = 120\n ","@@ -594,43 +600,44 @@ def mark_job_run(job_id: str, success: bool, error: Optional[str] = None,\n     ``delivery_error`` is tracked separately from the agent error — a job\n     can succeed (agent produced output) but fail delivery (platform down).\n     \"\"\"\n-    jobs = load_jobs()\n-    for i, job in enumerate(jobs):\n-        if job[\"id\"] == job_id:\n-            now = _hermes_now().isoformat()\n-            job[\"last_run_at\"] = now\n-            job[\"last_status\"] = \"ok\" if success else \"error\"\n-            job[\"last_error\"] = error if not success else None\n-            # Track delivery failures separately — cleared on successful delivery\n-            job[\"last_delivery_error\"] = delivery_error\n-            \n-            # Increment completed count\n-            if job.get(\"repeat\"):\n-                job[\"repeat\"][\"completed\"] = job[\"repeat\"].get(\"completed\", 0) + 1\n+    with _jobs_file_lock:\n+        jobs = load_jobs()\n+        for i, job in enumerate(jobs):\n+            if job[\"id\"] == job_id:\n+                now = _hermes_now().isoformat()\n+                job[\"last_run_at\"] = now\n+                job[\"last_status\"] = \"ok\" if success else \"error\"\n+                job[\"last_error\"] = error if not success else None\n+                # Track delivery failures separately — cleared on successful delivery\n+                job[\"last_delivery_error\"] = delivery_error\n                 \n-                # Check if we've hit the repeat limit\n-                times = job[\"repeat\"].get(\"times\")\n-                completed = job[\"repeat\"][\"completed\"]\n-                if times is not None and times > 0 and completed >= times:\n-                    # Remove the job (limit reached)\n-                    jobs.pop(i)\n-                    save_jobs(jobs)\n-                    return\n-            \n-            # Compute next run\n-            job[\"next_run_at\"] = compute_next_run(job[\"schedule\"], now)\n+                # Increment completed count\n+                if job.get(\"repeat\"):\n+                    job[\"repeat\"][\"completed\"] = job[\"repeat\"].get(\"completed\", 0) + 1\n+                    \n+                    # Check if we've hit the repeat limit\n+                    times = job[\"repeat\"].get(\"times\")\n+                    completed = job[\"repeat\"][\"completed\"]\n+                    if times is not None and times > 0 and completed >= times:\n+                        # Remove the job (limit reached)\n+                        jobs.pop(i)\n+                        save_jobs(jobs)\n+                        return\n+                \n+                # Compute next run\n+                job[\"next_run_at\"] = compute_next_run(job[\"schedule\"], now)\n \n-            # If no next run (one-shot completed), disable\n-            if job[\"next_run_at\"] is None:\n-                job[\"enabled\"] = False\n-                job[\"state\"] = \"completed\"\n-            elif job.get(\"state\") != \"paused\":\n-                job[\"state\"] = \"scheduled\"\n+                # If no next run (one-shot completed), disable\n+                if job[\"next_run_at\"] is None:\n+                    job[\"enabled\"] = False\n+                    job[\"state\"] = \"completed\"\n+                elif job.get(\"state\") != \"paused\":\n+                    job[\"state\"] = \"scheduled\"\n \n-            save_jobs(jobs)\n-            return\n+                save_jobs(jobs)\n+                return\n \n-    logger.warning(\"mark_job_run: job_id %s not found, skipping save\", job_id)\n+        logger.warning(\"mark_job_run: job_id %s not found, skipping save\", job_id)\n \n \n def advance_next_run(job_id: str) -> bool:","@@ -645,20 +652,21 @@ def advance_next_run(job_id: str) -> bool:\n \n     Returns True if next_run_at was advanced, False otherwise.\n     \"\"\"\n-    jobs = load_jobs()\n-    for job in jobs:\n-        if job[\"id\"] == job_id:\n-            kind = job.get(\"schedule\", {}).get(\"kind\")\n-            if kind not in (\"cron\", \"interval\"):\n+    with _jobs_file_lock:\n+        jobs = load_jobs()\n+        for job in jobs:\n+            if job[\"id\"] == job_id:\n+                kind = job.get(\"schedule\", {}).get(\"kind\")\n+                if kind not in (\"cron\", \"interval\"):\n+                    return False\n+                now = _hermes_now().isoformat()\n+                new_next = compute_next_run(job[\"schedule\"], now)\n+                if new_next and new_next != job.get(\"next_run_at\"):\n+                    job[\"next_run_at\"] = new_next\n+                    save_jobs(jobs)\n+                    return True\n                 return False\n-            now = _hermes_now().isoformat()\n-            new_next = compute_next_run(job[\"schedule\"], now)\n-            if new_next and new_next != job.get(\"next_run_at\"):\n-                job[\"next_run_at\"] = new_next\n-                save_jobs(jobs)\n-                return True\n-            return False\n-    return False\n+        return False\n \n \n def get_due_jobs() -> List[Dict[str, Any]]:"]},{"file":"cron/scheduler.py","hunks":["@@ -747,14 +747,17 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:\n     # scheduler process — every job this process runs is a cron job.\n     os.environ[\"HERMES_CRON_SESSION\"] = \"1\"\n \n+    # Use ContextVars for per-job session/delivery state so parallel jobs\n+    # don't clobber each other's targets (os.environ is process-global).\n+    from gateway.session_context import set_session_vars, clear_session_vars, _VAR_MAP\n+\n+    _ctx_tokens = [redacted:secret-assignment]\n+        platform=origin[\"platform\"] if origin else \"\",\n+        chat_id=str(origin[\"chat_id\"]) if origin else \"\",\n+        chat_name=origin.get(\"chat_name\", \"\") if origin else \"\",\n+    )\n+\n     try:\n-        # Inject origin context so the agent's send_message tool knows the chat.\n-        # Must be INSIDE the try block so the finally cleanup always runs.\n-        if origin:\n-            os.environ[\"HERMES_SESSION_PLATFORM\"] = origin[\"platform\"]\n-            os.environ[\"HERMES_SESSION_CHAT_ID\"] = str(origin[\"chat_id\"])\n-            if origin.get(\"chat_name\"):\n-                os.environ[\"HERMES_SESSION_CHAT_NAME\"] = origin[\"chat_name\"]\n         # Re-read .env and config.yaml fresh every run so provider/key\n         # changes take effect without a gateway restart.\n         from dotenv import load_dotenv","@@ -765,10 +768,10 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:\n \n         delivery_target = _resolve_delivery_target(job)\n         if delivery_target:\n-            os.environ[\"HERMES_CRON_AUTO_DELIVER_PLATFORM\"] = delivery_target[\"platform\"]\n-            os.environ[\"HERMES_CRON_AUTO_DELIVER_CHAT_ID\"] = str(delivery_target[\"chat_id\"])\n+            _VAR_MAP[\"HERMES_CRON_AUTO_DELIVER_PLATFORM\"].set(delivery_target[\"platform\"])\n+            _VAR_MAP[\"HERMES_CRON_AUTO_DELIVER_CHAT_ID\"].set(str(delivery_target[\"chat_id\"]))\n             if delivery_target.get(\"thread_id\") is not None:\n-                os.environ[\"HERMES_CRON_AUTO_DELIVER_THREAD_ID\"] = str(delivery_target[\"thread_id\"])\n+                _VAR_MAP[\"HERMES_CRON_AUTO_DELIVER_THREAD_ID\"].set(str(delivery_target[\"thread_id\"]))\n \n         model = job.get(\"model\") or os.getenv(\"HERMES_MODEL\") or \"\"\n ","@@ -1012,16 +1015,8 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:\n         return False, output, \"\", error_msg\n \n     finally:\n-        # Clean up injected env vars so they don't leak to other jobs\n-        for key in (\n-            \"HERMES_SESSION_PLATFORM\",\n-            \"HERMES_SESSION_CHAT_ID\",\n-            \"HERMES_SESSION_CHAT_NAME\",\n-            \"HERMES_CRON_AUTO_DELIVER_PLATFORM\",\n-            \"HERMES_CRON_AUTO_DELIVER_CHAT_ID\",\n-            \"HERMES_CRON_AUTO_DELIVER_THREAD_ID\",\n-        ):\n-            os.environ.pop(key, None)\n+        # Clean up ContextVar session/delivery state for this job.\n+        clear_session_vars(_ctx_tokens)\n         if _session_db:\n             try:\n                 _session_db.end_session(_cron_session_id, \"cron_complete\")","@@ -1074,15 +1069,42 @@ def tick(verbose: bool = True, adapters=None, loop=None) -> int:\n         if verbose:\n             logger.info(\"%s - %s job(s) due\", _hermes_now().strftime('%H:%M:%S'), len(due_jobs))\n \n-        executed = 0\n+        # Advance next_run_at for all recurring jobs FIRST, under the file lock,\n+        # before any execution begins.  This preserves at-most-once semantics.\n         for job in due_jobs:\n+            advance_next_run(job[\"id\"])\n+\n+        # Resolve max parallel workers: env var > config.yaml > unbounded.\n+        # Set HERMES_CRON_MAX_PARALLEL=1 to restore old serial behaviour.\n+        _max_workers: Optional[int] = None\n+        try:\n+            _env_par = os.getenv(\"HERMES_CRON_MAX_PARALLEL\", \"\").strip()\n+            if _env_par:\n+                _max_workers = int(_env_par) or None\n+        except (ValueError, TypeError):\n+            logger.warning(\"Invalid HERMES_CRON_MAX_PARALLEL value; defaulting to unbounded\")\n+        if _max_workers is None:\n             try:\n-                # For recurring jobs (cron/interval), advance next_run_at to the\n-                # next future occurrence BEFORE execution.  This way, if the\n-                # process crashes mid-run, the job won't re-fire on restart.\n-                # One-shot jobs are left alone so they can retry on restart.\n-                advance_next_run(job[\"id\"])\n+                from hermes_cli.config import load_config\n+                _ucfg = load_config() or {}\n+                _cfg_par = (\n+                    _ucfg.get(\"cron\", {}) if isinstance(_ucfg, dict) else {}\n+                ).get(\"max_parallel_jobs\")\n+                if _cfg_par is not None:\n+                    _max_workers = int(_cfg_par) or None\n+            except Exception:\n+                pass\n \n+        if verbose:\n+            logger.info(\n+                \"Running %d job(s) in parallel (max_workers=%s)\",\n+                len(due_jobs),\n+                _max_workers if _max_workers else \"unbounded\",\n+            )\n+\n+        def _process_job(job: dict) -> bool:\n+            \"\"\"Run one due job end-to-end: execute, save, deliver, mark.\"\"\"\n+            try:\n                 success, output, final_response, error = run_job(job)\n \n                 output_file = save_job_output(job[\"id\"], output)","@@ -1114,13 +1136,23 @@ def tick(verbose: bool = True, adapters=None, loop=None) -> int:\n                     error = \"Agent completed but produced empty response (model error, timeout, or misconfiguration)\"\n \n                 mark_job_run(job[\"id\"], success, error, delivery_error=delivery_error)\n-                executed += 1\n+                return True\n \n             except Exception as e:\n                 logger.error(\"Error processing job %s: %s\", job['id'], e)\n                 mark_job_run(job[\"id\"], False, str(e))\n-\n-        return executed\n+                return False\n+\n+        # Run all due jobs concurrently, each in its own ContextVar copy\n+        # so session/delivery state stays isolated per-thread.\n+        with concurrent.futures.ThreadPoolExecutor(max_workers=_max_workers) as _tick_pool:\n+            _futures = []\n+            for job in due_jobs:\n+                _ctx = contextvars.copy_context()\n+                _futures.append(_tick_pool.submit(_ctx.run, _process_job, job))\n+            _results = [f.result() for f in _futures]\n+\n+        return sum(_results)\n     finally:\n         if fcntl:\n             fcntl.flock(lock_fd, fcntl.LOCK_UN)"]},{"file":"gateway/session_context.py","hunks":["@@ -56,6 +56,12 @@\n _SESSION_USER_NAME: ContextVar = ContextVar(\"HERMES_SESSION_USER_NAME\", default=_UNSET)\n _SESSION_KEY: ContextVar = ContextVar(\"HERMES_SESSION_KEY\", default=_UNSET)\n \n+# Cron auto-delivery vars — set per-job in run_job() so concurrent jobs\n+# don't clobber each other's delivery targets.\n+_CRON_AUTO_DELIVER_PLATFORM: ContextVar = ContextVar(\"HERMES_CRON_AUTO_DELIVER_PLATFORM\", default=_UNSET)\n+_CRON_AUTO_DELIVER_CHAT_ID: ContextVar = ContextVar(\"HERMES_CRON_AUTO_DELIVER_CHAT_ID\", default=_UNSET)\n+_CRON_AUTO_DELIVER_THREAD_ID: ContextVar = ContextVar(\"HERMES_CRON_AUTO_DELIVER_THREAD_ID\", default=_UNSET)\n+\n _VAR_MAP = {\n     \"HERMES_SESSION_PLATFORM\": _SESSION_PLATFORM,\n     \"HERMES_SESSION_CHAT_ID\": _SESSION_CHAT_ID,","@@ -64,6 +70,9 @@\n     \"HERMES_SESSION_USER_ID\": _SESSION_USER_ID,\n     \"HERMES_SESSION_USER_NAME\": _SESSION_USER_NAME,\n     \"HERMES_SESSION_KEY\": _SESSION_KEY,\n+    \"HERMES_CRON_AUTO_DELIVER_PLATFORM\": _CRON_AUTO_DELIVER_PLATFORM,\n+    \"HERMES_CRON_AUTO_DELIVER_CHAT_ID\": _CRON_AUTO_DELIVER_CHAT_ID,\n+    \"HERMES_CRON_AUTO_DELIVER_THREAD_ID\": _CRON_AUTO_DELIVER_THREAD_ID,\n }\n \n "]},{"file":"hermes_cli/config.py","hunks":["@@ -796,6 +796,11 @@ def _ensure_hermes_home_managed(home: Path):\n         # Wrap delivered cron responses with a header (task name) and footer\n         # (\"The agent cannot see this message\").  Set to false for clean output.\n         \"wrap_response\": True,\n+        # Maximum number of due jobs to run in parallel per tick.\n+        # null/0 = unbounded (limited only by thread count).\n+        # 1 = serial (pre-v0.9 behaviour).\n+        # Also overridable via HERMES_CRON_MAX_PARALLEL env var.\n+        \"max_parallel_jobs\": None,\n     },\n \n     # execute_code settings — controls the tool used for programmatic tool calls."]},{"file":"tests/cron/test_scheduler.py","hunks":["@@ -772,9 +772,10 @@ def __init__(self, *args, **kwargs):\n                 pass\n \n             def run_conversation(self, *args, **kwargs):\n-                seen[\"platform\"] = os.getenv(\"HERMES_CRON_AUTO_DELIVER_PLATFORM\")\n-                seen[\"chat_id\"] = os.getenv(\"HERMES_CRON_AUTO_DELIVER_CHAT_ID\")\n-                seen[\"thread_id\"] = os.getenv(\"HERMES_CRON_AUTO_DELIVER_THREAD_ID\")\n+                from gateway.session_context import get_session_env\n+                seen[\"platform\"] = get_session_env(\"HERMES_CRON_AUTO_DELIVER_PLATFORM\") or None\n+                seen[\"chat_id\"] = get_session_env(\"HERMES_CRON_AUTO_DELIVER_CHAT_ID\") or None\n+                seen[\"thread_id\"] = get_session_env(\"HERMES_CRON_AUTO_DELIVER_THREAD_ID\") or None\n                 return {\"final_response\": \"ok\"}\n \n         with patch(\"cron.scheduler._hermes_home\", tmp_path), \\","@@ -1457,3 +1458,125 @@ def test_multiple_media_files_all_delivered(self):\n         self._run_with_loop(adapter, \"123\", media_files, None, {\"id\": \"j3\"})\n         adapter.send_voice.assert_called_once()\n         adapter.send_image_file.assert_called_once()\n+\n+\n+class TestParallelTick:\n+    \"\"\"Verify that tick() runs due jobs concurrently and isolates ContextVars.\"\"\"\n+\n+    @pytest.fixture(autouse=True)\n+    def _isolate_tick_lock(self, tmp_path):\n+        \"\"\"Point the tick file lock at a per-test temp dir to avoid xdist contention.\"\"\"\n+        lock_dir = tmp_path / \"cron\"\n+        lock_dir.mkdir()\n+        with patch(\"cron.scheduler._LOCK_DIR\", lock_dir), \\\n+             patch(\"cron.scheduler._LOCK_FILE\", lock_dir / \".tick.lock\"):\n+            yield\n+\n+    def test_parallel_jobs_run_concurrently(self):\n+        \"\"\"Two jobs launched in the same tick should overlap in time.\"\"\"\n+        import threading\n+        import time\n+\n+        barrier = threading.Barrier(2, timeout=5)\n+        call_order = []\n+\n+        def mock_run_job(job):\n+            \"\"\"Each job hits a barrier — both must be active simultaneously.\"\"\"\n+            call_order.append((\"start\", job[\"id\"]))\n+            barrier.wait()  # blocks until both threads reach here\n+            call_order.append((\"end\", job[\"id\"]))\n+            return (True, \"output\", \"response\", None)\n+\n+        jobs = [\n+            {\"id\": \"job-a\", \"name\": \"a\", \"deliver\": \"local\"},\n+            {\"id\": \"job-b\", \"name\": \"b\", \"deliver\": \"local\"},\n+        ]\n+\n+        with patch(\"cron.scheduler.get_due_jobs\", return_value=jobs), \\\n+             patch(\"cron.scheduler.advance_next_run\"), \\\n+             patch(\"cron.scheduler.run_job\", side_effect=mock_run_job), \\\n+             patch(\"cron.scheduler.save_job_output\", return_value=\"/tmp/out.md\"), \\\n+             patch(\"cron.scheduler._deliver_result\", return_value=None), \\\n+             patch(\"cron.scheduler.mark_job_run\"):\n+            from cron.scheduler import tick\n+            result = tick(verbose=False)\n+\n+        assert result == 2\n+        # Both starts happened before both ends — proof of concurrency\n+        starts = [i for i, (action, _) in enumerate(call_order) if action == \"start\"]\n+        ends = [i for i, (action, _) in enumerate(call_order) if action == \"end\"]\n+        assert len(starts) == 2\n+        assert len(ends) == 2\n+        assert max(starts) < min(ends), f\"Jobs not concurrent: {call_order}\"\n+\n+    def test_parallel_jobs_isolated_contextvars(self):\n+        \"\"\"Each job's ContextVars must be isolated — no cross-contamination.\"\"\"\n+        from gateway.session_context import get_session_env\n+        seen = {}\n+\n+        def mock_run_job(job):\n+            origin = job.get(\"origin\", {})\n+            # run_job sets ContextVars — verify each job sees its own\n+            from gateway.session_context import set_session_vars, clear_session_vars\n+            tokens = [redacted:secret-assignment]\n+                platform=origin.get(\"platform\", \"\"),\n+                chat_id=str(origin.get(\"chat_id\", \"\")),\n+            )\n+            import time\n+            time.sleep(0.05)  # give other thread time to set its vars\n+            platform = get_session_env(\"HERMES_SESSION_PLATFORM\")\n+            chat_id = get_session_env(\"HERMES_SESSION_CHAT_ID\")\n+            seen[job[\"id\"]] = {\"platform\": platform, \"chat_id\": chat_id}\n+            clear_session_vars(tokens)\n+            return (True, \"output\", \"response\", None)\n+\n+        jobs = [\n+            {\"id\": \"tg-job\", \"name\": \"tg\", \"deliver\": \"local\",\n+             \"origin\": {\"platform\": \"telegram\", \"chat_id\": \"111\"}},\n+            {\"id\": \"dc-job\", \"name\": \"dc\", \"deliver\": \"local\",\n+             \"origin\": {\"platform\": \"discord\", \"chat_id\": \"222\"}},\n+        ]\n+\n+        with patch(\"cron.scheduler.get_due_jobs\", return_value=jobs), \\\n+             patch(\"cron.scheduler.advance_next_run\"), \\\n+             patch(\"cron.scheduler.run_job\", side_effect=mock_run_job), \\\n+             patch(\"cron.scheduler.save_job_output\", return_value=\"/tmp/out.md\"), \\\n+             patch(\"cron.scheduler._deliver_result\", return_value=None), \\\n+             patch(\"cron.scheduler.mark_job_run\"):\n+            from cron.scheduler import tick\n+            tick(verbose=False)\n+\n+        assert seen[\"tg-job\"] == {\"platform\": \"telegram\", \"chat_id\": \"111\"}\n+        assert seen[\"dc-job\"] == {\"platform\": \"discord\", \"chat_id\": \"222\"}\n+\n+    def test_max_parallel_env_var(self, monkeypatch):\n+        \"\"\"HERMES_CRON_MAX_PARALLEL=1 should restore serial behaviour.\"\"\"\n+        monkeypatch.setenv(\"HERMES_CRON_MAX_PARALLEL\", \"1\")\n+        call_times = []\n+\n+        def mock_run_job(job):\n+            import time\n+            call_times.append((\"start\", job[\"id\"], time.monotonic()))\n+            time.sleep(0.05)\n+            call_times.append((\"end\", job[\"id\"], time.monotonic()))\n+            return (True, \"output\", \"response\", None)\n+\n+        jobs = [\n+            {\"id\": \"s1\", \"name\": \"s1\", \"deliver\": \"local\"},\n+            {\"id\": \"s2\", \"name\": \"s2\", \"deliver\": \"local\"},\n+        ]\n+\n+        with patch(\"cron.scheduler.get_due_jobs\", return_value=jobs), \\\n+             patch(\"cron.scheduler.advance_next_run\"), \\\n+             patch(\"cron.scheduler.run_job\", side_effect=mock_run_job), \\\n+             patch(\"cron.scheduler.save_job_output\", return_value=\"/tmp/out.md\"), \\\n+             patch(\"cron.scheduler._deliver_result\", return_value=None), \\\n+             patch(\"cron.scheduler.mark_job_run\"):\n+            from cron.scheduler import tick\n+            result = tick(verbose=False)\n+\n+        assert result == 2\n+        # With max_workers=1, second job starts after first ends\n+        end_s1 = [t for action, jid, t in call_times if action == \"end\" and jid == \"s1\"][0]\n+        start_s2 = [t for action, jid, t in call_times if action == \"start\" and jid == \"s2\"][0]\n+        assert start_s2 >= end_s1, \"Jobs ran concurrently despite max_parallel=1\""]},{"file":"tools/send_message_tool.py","hunks":["@@ -359,11 +359,12 @@ def _describe_media_for_mirror(media_files):\n \n def _get_cron_auto_delivery_target():\n     \"\"\"Return the cron scheduler's auto-delivery target for the current run, if any.\"\"\"\n-    platform = os.getenv(\"HERMES_CRON_AUTO_DELIVER_PLATFORM\", \"\").strip().lower()\n-    chat_id = os.getenv(\"HERMES_CRON_AUTO_DELIVER_CHAT_ID\", \"\").strip()\n+    from gateway.session_context import get_session_env\n+    platform = get_session_env(\"HERMES_CRON_AUTO_DELIVER_PLATFORM\", \"\").strip().lower()\n+    chat_id = get_session_env(\"HERMES_CRON_AUTO_DELIVER_CHAT_ID\", \"\").strip()\n     if not platform or not chat_id:\n         return None\n-    thread_id = os.getenv(\"HERMES_CRON_AUTO_DELIVER_THREAD_ID\", \"\").strip() or None\n+    thread_id = get_session_env(\"HERMES_CRON_AUTO_DELIVER_THREAD_ID\", \"\").strip() or None\n     return {\n         \"platform\": platform,\n         \"chat_id\": chat_id,\n"]}]},{"kind":"pr","number":27492,"title":"fix(cron): release tick file lock before executing due jobs","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#27485","number":27485}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":33318,"title":"fix(cron): execute missed-grace jobs once instead of deferring","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#50062","number":50062}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":33361,"title":"fix(cron): execute job once after missed-grace fast-forward instead of skipping","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#50062","number":50062},{"kind":"pr","value":"#50062","number":50062},{"kind":"pr","value":"#33318","number":33318}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":33559,"title":"fix(cron): prevent perpetual-defer loop for slow recurring jobs (fixes #33315)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#33318","number":33318}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":37313,"title":"fix(cron): decouple job dispatch from completion in tick()","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"eb9cde734","number":0},{"kind":"pr","value":"#39008","number":39008}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":37355,"title":"fix(cron): decouple tick() dispatch from job completion #37312","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#37312","number":37312},{"kind":"pr","value":"#39008","number":39008},{"kind":"pr","value":"#37313","number":37313},{"kind":"pr","value":"#37313","number":37313}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":37668,"title":"fix(cron): decouple parallel dispatch from completion so slow jobs can't stall the ticker (fixes #37312)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#37312","number":37312},{"kind":"pr","value":"#39008","number":39008},{"kind":"pr","value":"#37313","number":37313}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":38624,"title":"fix(cron): release tick lock before job execution to prevent scheduler starvation","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"eb9cde734642cc2a8ecd142ba7c63d3964ca835b","number":0},{"kind":"commit","value":"9fbfeb31b9c611c9f06f84ca95ea6b4fbad6c3e1","number":0},{"kind":"pr","value":"#27485","number":27485}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":38718,"title":"fix(cron): NameError get_fallback_chain + silent ticker death + tick() blocks (#36734 #36854 #37312)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#37312","number":37312},{"kind":"pr","value":"#39008","number":39008},{"kind":"pr","value":"#36734","number":36734},{"kind":"pr","value":"#36854","number":36854}],"category":"partial_fix","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":39008,"title":"fix(cron): decouple tick() dispatch from job completion (#37312)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cron/scheduler.py","gateway/run.py","tests/cron/test_cron_profile.py","tests/cron/test_cron_workdir.py","tests/cron/test_parallel_pool.py"],"fix_hunks":[{"file":"cron/scheduler.py","hunks":["@@ -9,6 +9,7 @@\n \"\"\"\n \n import asyncio\n+import atexit\n import concurrent.futures\n import contextvars\n import json","@@ -17,6 +18,7 @@\n import shutil\n import subprocess\n import sys\n+import threading\n from contextlib import contextmanager\n \n # fcntl is Unix-only; on Windows use msvcrt for file locking","@@ -154,6 +156,69 @@ def _resolve_cron_enabled_toolsets(job: dict, cfg: dict) -> list[str] | None:\n # locally for audit.\n SILENT_MARKER = \"[SILENT]\"\n \n+# ---------------------------------------------------------------------------\n+# Persistent thread pool for parallel cron jobs.\n+# The tick function submits jobs here and returns immediately so the ticker\n+# thread is never blocked by long-running jobs (e.g. the fixer running 15+ min).\n+# ---------------------------------------------------------------------------\n+_parallel_pool: Optional[concurrent.futures.ThreadPoolExecutor] = None\n+_parallel_pool_max_workers: Optional[int] = None\n+_running_job_ids: set = set()\n+_running_lock = threading.Lock()\n+\n+# Sequential (env/context-mutating) cron jobs — workdir/profile jobs that touch\n+# process-global runtime state — must run one at a time, but must NOT block the\n+# ticker thread.  A persistent single-thread executor preserves ordering across\n+# ticks while keeping dispatch fire-and-forget, the same as the parallel pool.\n+_sequential_pool: Optional[concurrent.futures.ThreadPoolExecutor] = None\n+\n+\n+def _get_parallel_pool(max_workers: Optional[int]) -> concurrent.futures.ThreadPoolExecutor:\n+    \"\"\"Return (or create) the persistent parallel pool.\"\"\"\n+    global _parallel_pool, _parallel_pool_max_workers\n+    if _parallel_pool is None or _parallel_pool_max_workers != max_workers:\n+        if _parallel_pool is not None:\n+            _parallel_pool.shutdown(wait=False, cancel_futures=False)\n+        _parallel_pool = concurrent.futures.ThreadPoolExecutor(\n+            max_workers=max_workers,\n+            thread_name_prefix=\"cron-parallel\",\n+        )\n+        _parallel_pool_max_workers = max_workers\n+    return _parallel_pool\n+\n+\n+def _get_sequential_pool() -> concurrent.futures.ThreadPoolExecutor:\n+    \"\"\"Return (or create) the persistent single-thread sequential pool.\n+\n+    A single worker guarantees env/context-mutating jobs never overlap, even\n+    across ticks: a job queued by a newer tick waits for the previous tick's\n+    sequential jobs to finish rather than corrupting their os.environ /\n+    profile state.\n+    \"\"\"\n+    global _sequential_pool\n+    if _sequential_pool is None:\n+        _sequential_pool = concurrent.futures.ThreadPoolExecutor(\n+            max_workers=1,\n+            thread_name_prefix=\"cron-seq\",\n+        )\n+    return _sequential_pool\n+\n+\n+def _shutdown_parallel_pool() -> None:\n+    \"\"\"Shut down the persistent pools on process exit.\"\"\"\n+    global _parallel_pool, _parallel_pool_max_workers, _sequential_pool\n+    if _parallel_pool is not None:\n+        _parallel_pool.shutdown(wait=True, cancel_futures=False)\n+        _parallel_pool = None\n+        _parallel_pool_max_workers = None\n+    if _sequential_pool is not None:\n+        _sequential_pool.shutdown(wait=True, cancel_futures=False)\n+        _sequential_pool = None\n+\n+\n+atexit.register(_shutdown_parallel_pool)\n+\n+\n # Backward-compatible module override used by tests and emergency monkeypatches.\n _hermes_home: Path | None = None\n ","@@ -1895,7 +1960,7 @@ def _run_job_impl(job: dict) -> tuple[bool, str, str, Optional[str]]:\n             logger.debug(\"Job '%s': failed to reap stale auxiliary clients: %s\", job_id, e)\n \n \n-def tick(verbose: bool = True, adapters=None, loop=None) -> int:\n+def tick(verbose: bool = True, adapters=None, loop=None, sync: bool = True) -> int:\n     \"\"\"\n     Check and run all due jobs.\n     ","@@ -1939,6 +2004,9 @@ def tick(verbose: bool = True, adapters=None, loop=None) -> int:\n \n         # Advance next_run_at for all recurring jobs FIRST, under the file lock,\n         # before any execution begins.  This preserves at-most-once semantics.\n+        # For parallel jobs that are already running, advance_next_run keeps\n+        # bumping next_run_at forward so the grace window never expires.\n+        # mark_job_run() overwrites next_run_at on completion.\n         for job in due_jobs:\n             advance_next_run(job[\"id\"])\n ","@@ -2030,36 +2098,103 @@ def _process_job(job: dict) -> bool:\n         ]\n \n         _results: list = []\n-\n-        # Sequential pass for env/context-mutating jobs.\n-        for job in sequential_jobs:\n+        _all_futures: list = []\n+\n+        def _submit_with_guard(job: dict, pool: concurrent.futures.ThreadPoolExecutor):\n+            \"\"\"Submit a job fire-and-forget with the in-flight dedup guard.\n+\n+            Returns the future, or None if the job was skipped because a prior\n+            tick's run of the same job is still in flight.  The running-set\n+            membership is released in the worker's finally block.\n+            \"\"\"\n+            job_id = job[\"id\"]\n+            with _running_lock:\n+                if job_id in _running_job_ids:\n+                    logger.info(\"Job '%s' already running — skipping\", job.get(\"name\", job_id))\n+                    return None\n+                _running_job_ids.add(job_id)\n             _ctx = contextvars.copy_context()\n-            _results.append(_ctx.run(_process_job, job))\n \n-        # Parallel pass for the rest — same behaviour as before.\n+            def _run_and_release(j=job, ctx=_ctx):\n+                try:\n+                    return ctx.run(_process_job, j)\n+                finally:\n+                    with _running_lock:\n+                        _running_job_ids.discard(j[\"id\"])\n+\n+            return pool.submit(_run_and_release)\n+\n+        # Sequential pass for env/context-mutating (workdir/profile) jobs.\n+        # Queued to a persistent single-thread pool so they run one at a time\n+        # WITHOUT blocking the ticker thread — a long workdir/profile job no\n+        # longer starves the rest of the schedule (same fix as the parallel\n+        # pass, just serialized).  The in-flight guard prevents a still-running\n+        # job from being re-queued on the next tick.\n+        if sequential_jobs:\n+            seq_pool = _get_sequential_pool()\n+            for job in sequential_jobs:\n+                fut = _submit_with_guard(job, seq_pool)\n+                if fut is None:\n+                    continue\n+                _all_futures.append(fut)\n+                if not sync:\n+                    _results.append(True)  # optimistically counted\n+\n+        # Parallel pass — persistent pool, non-blocking dispatch.\n+        # Jobs that are already running (from a previous tick) are skipped.\n+        # mark_job_run() updates next_run_at on completion, so the next tick\n+        # after completion finds the job due again naturally.  No catch-up\n+        # queue needed.\n         if parallel_jobs:\n-            with concurrent.futures.ThreadPoolExecutor(max_workers=_max_workers) as _tick_pool:\n-                _futures = []\n-                for job in parallel_jobs:\n-                    _ctx = contextvars.copy_context()\n-                    _futures.append(_tick_pool.submit(_ctx.run, _process_job, job))\n-                for f in concurrent.futures.as_completed(_futures, timeout=600):\n-                    try:\n-                        _results.append(f.result())\n-                    except Exception as exc:\n-                        logger.error(\"Parallel cron job future failed: %s\", exc)\n-                        _results.append(False)\n+            pool = _get_parallel_pool(_max_workers)\n+            for job in parallel_jobs:\n+                fut = _submit_with_guard(job, pool)\n+                if fut is None:\n+                    continue\n+                _all_futures.append(fut)\n+                if not sync:\n+                    _results.append(True)  # optimistically counted\n \n         # Best-effort sweep of MCP stdio subprocesses that survived their\n-        # session teardown during this tick.  Runs AFTER every job has\n-        # finished so active sessions (including live user chats) are\n-        # never touched — only PIDs explicitly detected as orphans in\n-        # tools.mcp_tool._run_stdio's finally block are reaped.\n-        try:\n-            from tools.mcp_tool import _kill_orphaned_mcp_children\n-            _kill_orphaned_mcp_children()\n-        except Exception as _e:\n-            logger.debug(\"Post-tick MCP orphan cleanup failed: %s\", _e)\n+        # session teardown.  Must run AFTER jobs finish so active sessions\n+        # (including live user chats) are never touched — only PIDs explicitly\n+        # detected as orphans in tools.mcp_tool._run_stdio's finally block are\n+        # reaped.\n+        def _sweep_mcp_orphans() -> None:\n+            try:\n+                from tools.mcp_tool import _kill_orphaned_mcp_children\n+                _kill_orphaned_mcp_children()\n+            except Exception as _e:\n+                logger.debug(\"Post-tick MCP orphan cleanup failed: %s\", _e)\n+\n+        if sync:\n+            # Sync mode (tests / manual ticks): wait for all dispatched jobs,\n+            # collect results, then sweep once.\n+            for f in concurrent.futures.as_completed(_all_futures):\n+                try:\n+                    _results.append(f.result())\n+                except Exception as exc:\n+                    logger.error(\"Cron job future failed: %s\", exc)\n+                    _results.append(False)\n+            _sweep_mcp_orphans()\n+            return sum(_results)\n+\n+        # Async (gateway ticker) mode: don't block.  Sweep orphans via a\n+        # done-callback fired after the LAST dispatched job completes, so the\n+        # sweep still happens after jobs finish without stalling the tick.\n+        if _all_futures:\n+            _remaining = [len(_all_futures)]\n+\n+            def _on_done(_f: concurrent.futures.Future) -> None:\n+                _remaining[0] -= 1\n+                if _remaining[0] <= 0:\n+                    _sweep_mcp_orphans()\n+\n+            for _f in _all_futures:\n+                _f.add_done_callback(_on_done)\n+        else:\n+            # Nothing dispatched (all skipped / no due jobs) — sweep inline.\n+            _sweep_mcp_orphans()\n \n         return sum(_results)\n     finally:"]},{"file":"gateway/run.py","hunks":["@@ -19215,7 +19215,7 @@ def _start_cron_ticker(stop_event: threading.Event, adapters=None, loop=None, in\n     tick_count = 0\n     while not stop_event.is_set():\n         try:\n-            cron_tick(verbose=False, adapters=adapters, loop=loop)\n+            cron_tick(verbose=False, adapters=adapters, loop=loop, sync=False)\n         except Exception as e:\n             logger.debug(\"Cron tick error: %s\", e)\n "]},{"file":"tests/cron/test_cron_profile.py","hunks":["@@ -410,16 +410,20 @@ def test_profile_jobs_run_sequentially(self, isolated_cron_profile_home, monkeyp\n         import threading\n         import cron.scheduler as sched\n \n-        profile_job = {\"id\": \"a\", \"name\": \"A\", \"profile\": \"default\"}\n-        parallel_job = {\"id\": \"b\", \"name\": \"B\", \"profile\": None}\n+        # Two profile jobs (both sequential) + one parallel job.\n+        profile_a = {\"id\": \"a\", \"name\": \"A\", \"profile\": \"default\"}\n+        profile_b = {\"id\": \"b\", \"name\": \"B\", \"profile\": \"default\"}\n+        parallel_job = {\"id\": \"c\", \"name\": \"C\", \"profile\": None}\n \n-        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: [profile_job, parallel_job])\n+        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: [profile_a, profile_b, parallel_job])\n         monkeypatch.setattr(sched, \"advance_next_run\", lambda *_a, **_kw: None)\n \n         calls: list[tuple[str, str]] = []\n+        order_lock = threading.Lock()\n \n         def fake_run_job(job):\n-            calls.append((job[\"id\"], threading.current_thread().name))\n+            with order_lock:\n+                calls.append((job[\"id\"], threading.current_thread().name))\n             return True, \"output\", \"response\", None\n \n         monkeypatch.setattr(sched, \"run_job\", fake_run_job)","@@ -429,9 +433,17 @@ def fake_run_job(job):\n \n         n = sched.tick(verbose=False)\n \n-        assert n == 2\n+        assert n == 3\n         ids = [job_id for job_id, _thread_name in calls]\n+        # Sequential profile jobs preserve submission order relative to each\n+        # other (single-thread pool).\n         assert ids.index(\"a\") < ids.index(\"b\")\n-        main_thread_name = threading.current_thread().name\n-        profile_thread_name = next(thread for job_id, thread in calls if job_id == \"a\")\n-        assert profile_thread_name == main_thread_name\n+        # Sequential (profile) jobs run on the persistent single-thread\n+        # cron-seq pool — NOT the main thread — so a long profile job never\n+        # blocks the ticker.  Parallel jobs run on the cron-parallel pool.\n+        for jid in (\"a\", \"b\"):\n+            seq_thread = next(t for job_id, t in calls if job_id == jid)\n+            assert seq_thread != threading.current_thread().name\n+            assert seq_thread.startswith(\"cron-seq\"), seq_thread\n+        par_thread = next(t for job_id, t in calls if job_id == \"c\")\n+        assert par_thread.startswith(\"cron-parallel\"), par_thread"]},{"file":"tests/cron/test_cron_workdir.py","hunks":["@@ -207,20 +207,23 @@ class TestTickWorkdirPartition:\n     def test_workdir_jobs_run_sequentially(self, tmp_path, monkeypatch):\n         import cron.scheduler as sched\n \n-        # Two \"jobs\" — one with workdir, one without.  get_due_jobs returns both.\n-        workdir_job = {\"id\": \"a\", \"name\": \"A\", \"workdir\": str(tmp_path)}\n-        parallel_job = {\"id\": \"b\", \"name\": \"B\", \"workdir\": None}\n+        # Two workdir jobs (both sequential) + one parallel job.\n+        workdir_a = {\"id\": \"a\", \"name\": \"A\", \"workdir\": str(tmp_path)}\n+        workdir_b = {\"id\": \"b\", \"name\": \"B\", \"workdir\": str(tmp_path)}\n+        parallel_job = {\"id\": \"c\", \"name\": \"C\", \"workdir\": None}\n \n-        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: [workdir_job, parallel_job])\n+        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: [workdir_a, workdir_b, parallel_job])\n         monkeypatch.setattr(sched, \"advance_next_run\", lambda *_a, **_kw: None)\n \n         # Record call order / thread context.\n         import threading\n-        calls: list[tuple[str, bool]] = []\n+        calls: list[tuple[str, str]] = []\n+        order_lock = threading.Lock()\n \n         def fake_run_job(job):\n             # Return a minimal tuple matching run_job's signature.\n-            calls.append((job[\"id\"], threading.current_thread().name))\n+            with order_lock:\n+                calls.append((job[\"id\"], threading.current_thread().name))\n             return True, \"output\", \"response\", None\n \n         monkeypatch.setattr(sched, \"run_job\", fake_run_job)","@@ -231,16 +234,22 @@ def fake_run_job(job):\n         )\n \n         n = sched.tick(verbose=False)\n-        assert n == 2\n+        assert n == 3\n \n         ids = [c[0] for c in calls]\n-        # Workdir jobs always come before parallel jobs.\n+        # Sequential workdir jobs preserve submission order relative to each\n+        # other (single-thread pool).\n         assert ids.index(\"a\") < ids.index(\"b\")\n \n-        # The workdir job must run on the main thread (sequential pass).\n+        # Workdir jobs run on the persistent single-thread cron-seq pool —\n+        # NOT the main thread — so a long workdir job never blocks the ticker.\n         main_thread_name = threading.current_thread().name\n-        workdir_thread_name = next(t for jid, t in calls if jid == \"a\")\n-        assert workdir_thread_name == main_thread_name\n+        for jid in (\"a\", \"b\"):\n+            workdir_thread_name = next(t for j, t in calls if j == jid)\n+            assert workdir_thread_name != main_thread_name\n+            assert workdir_thread_name.startswith(\"cron-seq\"), workdir_thread_name\n+        par_thread_name = next(t for j, t in calls if j == \"c\")\n+        assert par_thread_name.startswith(\"cron-parallel\"), par_thread_name\n \n \n # ---------------------------------------------------------------------------"]},{"file":"tests/cron/test_parallel_pool.py","hunks":["@@ -0,0 +1,274 @@\n+\"\"\"Tests for the persistent parallel pool and running-job guard in cron/scheduler.py.\n+\n+These verify the fix for the tick-blocking issue where as_completed(timeout=600)\n+prevented the ticker thread from firing, causing all other jobs to be fast-forwarded.\n+\"\"\"\n+\n+import concurrent.futures\n+import threading\n+import time\n+from unittest.mock import patch\n+\n+import pytest\n+\n+\n+class TestPersistentPool:\n+    \"\"\"_get_parallel_pool returns a persistent ThreadPoolExecutor.\"\"\"\n+\n+    def test_pool_is_reused(self, monkeypatch):\n+        \"\"\"Same pool instance returned when max_workers doesn't change.\"\"\"\n+        import cron.scheduler as sched\n+\n+        # Reset module state.\n+        sched._parallel_pool = None\n+        sched._parallel_pool_max_workers = None\n+\n+        pool1 = sched._get_parallel_pool(4)\n+        pool2 = sched._get_parallel_pool(4)\n+        assert pool1 is pool2\n+\n+        # Cleanup.\n+        sched._shutdown_parallel_pool()\n+\n+    def test_pool_is_recreated_on_worker_change(self, monkeypatch):\n+        \"\"\"New pool when max_workers changes.\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._parallel_pool = None\n+        sched._parallel_pool_max_workers = None\n+\n+        pool1 = sched._get_parallel_pool(2)\n+        pool2 = sched._get_parallel_pool(4)\n+        assert pool1 is not pool2\n+\n+        sched._shutdown_parallel_pool()\n+\n+    def test_shutdown_clears_pool(self, monkeypatch):\n+        \"\"\"_shutdown_parallel_pool resets state.\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._parallel_pool = None\n+        sched._parallel_pool_max_workers = None\n+        sched._get_parallel_pool(2)\n+\n+        sched._shutdown_parallel_pool()\n+        assert sched._parallel_pool is None\n+        assert sched._parallel_pool_max_workers is None\n+\n+\n+class TestRunningJobGuard:\n+    \"\"\"_running_job_ids prevents double-dispatch of active jobs.\"\"\"\n+\n+    def test_running_set_prevents_double_dispatch(self, tmp_path, monkeypatch):\n+        \"\"\"A job already in _running_job_ids is skipped on the next tick.\"\"\"\n+        import cron.scheduler as sched\n+\n+        # Reset state.\n+        sched._parallel_pool = None\n+        sched._parallel_pool_max_workers = None\n+        sched._running_job_ids.clear()\n+\n+        job = {\n+            \"id\": \"guard-job\",\n+            \"name\": \"guard-test\",\n+            \"prompt\": \"test\",\n+            \"schedule\": \"every 5m\",\n+            \"enabled\": True,\n+            \"next_run_at\": \"2020-01-01T00:00:00\",\n+            \"deliver\": \"local\",\n+        }\n+\n+        # Simulate the job already running.\n+        sched._running_job_ids.add(\"guard-job\")\n+\n+        dispatched = []\n+        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: [job])\n+        monkeypatch.setattr(sched, \"advance_next_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"run_job\", lambda j: dispatched.append(j[\"id\"]) or (True, \"out\", \"resp\", None))\n+        monkeypatch.setattr(sched, \"save_job_output\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"mark_job_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"_deliver_result\", lambda *_a, **_kw: None)\n+\n+        n = sched.tick(verbose=False)\n+        assert n == 0  # skipped, not dispatched\n+        assert dispatched == []\n+\n+        sched._running_job_ids.discard(\"guard-job\")\n+        sched._shutdown_parallel_pool()\n+\n+\n+class TestSyncMode:\n+    \"\"\"tick() blocks by default (sync=True); tick(sync=False) returns immediately.\"\"\"\n+\n+    def test_sync_true_blocks_and_returns_correct_count(self, tmp_path, monkeypatch):\n+        \"\"\"sync=True waits for jobs and returns actual results.\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._parallel_pool = None\n+        sched._parallel_pool_max_workers = None\n+        sched._running_job_ids.clear()\n+\n+        jobs = [\n+            {\"id\": f\"job-{i}\", \"name\": f\"Job {i}\", \"prompt\": \"test\",\n+             \"schedule\": \"every 5m\", \"enabled\": True,\n+             \"next_run_at\": \"2020-01-01T00:00:00\", \"deliver\": \"local\"}\n+            for i in range(3)\n+        ]\n+\n+        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: jobs)\n+        monkeypatch.setattr(sched, \"advance_next_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"run_job\", lambda j: (True, \"out\", \"resp\", None))\n+        monkeypatch.setattr(sched, \"save_job_output\", lambda *_a, **_kw: \"/tmp/out\")\n+        monkeypatch.setattr(sched, \"mark_job_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"_deliver_result\", lambda *_a, **_kw: None)\n+\n+        n = sched.tick(verbose=False)\n+        assert n == 3\n+\n+        sched._shutdown_parallel_pool()\n+\n+    def test_sync_false_returns_immediately(self, tmp_path, monkeypatch):\n+        \"\"\"sync=False returns before parallel jobs finish (optimistic count).\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._parallel_pool = None\n+        sched._parallel_pool_max_workers = None\n+        sched._running_job_ids.clear()\n+\n+        job = {\n+            \"id\": \"slow-job\",\n+            \"name\": \"slow\",\n+            \"prompt\": \"test\",\n+            \"schedule\": \"every 5m\",\n+            \"enabled\": True,\n+            \"next_run_at\": \"2020-01-01T00:00:00\",\n+            \"deliver\": \"local\",\n+        }\n+\n+        barrier = threading.Barrier(2, timeout=5)\n+\n+        def slow_run(j):\n+            barrier.wait()  # blocks until test thread also waits\n+            return True, \"out\", \"resp\", None\n+\n+        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: [job])\n+        monkeypatch.setattr(sched, \"advance_next_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"run_job\", slow_run)\n+        monkeypatch.setattr(sched, \"save_job_output\", lambda *_a, **_kw: \"/tmp/out\")\n+        monkeypatch.setattr(sched, \"mark_job_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"_deliver_result\", lambda *_a, **_kw: None)\n+\n+        start = time.monotonic()\n+        n = sched.tick(verbose=False, sync=False)  # opt-in: non-blocking\n+        elapsed = time.monotonic() - start\n+\n+        assert n == 1  # optimistic count\n+        assert elapsed < 1.0  # returned immediately, didn't wait for slow_run\n+\n+        # Let the job finish so cleanup works.\n+        barrier.wait()\n+        time.sleep(0.1)\n+        sched._shutdown_parallel_pool()\n+\n+\n+class TestSequentialPool:\n+    \"\"\"Sequential (workdir/profile) jobs use the persistent cron-seq pool.\n+\n+    Verifies the follow-up fix: env/context-mutating jobs no longer run inline\n+    in the ticker thread, so a long workdir/profile job can't starve the\n+    schedule the same way the parallel path used to.\n+    \"\"\"\n+\n+    def test_sequential_job_does_not_block_ticker(self, tmp_path, monkeypatch):\n+        \"\"\"sync=False returns immediately even when a workdir job is slow.\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._parallel_pool = None\n+        sched._parallel_pool_max_workers = None\n+        sched._sequential_pool = None\n+        sched._running_job_ids.clear()\n+\n+        job = {\n+            \"id\": \"slow-workdir\",\n+            \"name\": \"slow-workdir\",\n+            \"prompt\": \"test\",\n+            \"schedule\": \"every 5m\",\n+            \"enabled\": True,\n+            \"next_run_at\": \"2020-01-01T00:00:00\",\n+            \"deliver\": \"local\",\n+            \"workdir\": str(tmp_path),  # makes it sequential\n+        }\n+\n+        barrier = threading.Barrier(2, timeout=5)\n+\n+        def slow_run(j):\n+            barrier.wait()\n+            return True, \"out\", \"resp\", None\n+\n+        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: [job])\n+        monkeypatch.setattr(sched, \"advance_next_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"run_job\", slow_run)\n+        monkeypatch.setattr(sched, \"save_job_output\", lambda *_a, **_kw: \"/tmp/out\")\n+        monkeypatch.setattr(sched, \"mark_job_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"_deliver_result\", lambda *_a, **_kw: None)\n+\n+        start = time.monotonic()\n+        n = sched.tick(verbose=False, sync=False)\n+        elapsed = time.monotonic() - start\n+\n+        assert n == 1  # optimistic count\n+        assert elapsed < 1.0  # did NOT block on the slow workdir job\n+\n+        barrier.wait()\n+        time.sleep(0.1)\n+        sched._shutdown_parallel_pool()\n+\n+    def test_sequential_running_guard_prevents_double_dispatch(self, tmp_path, monkeypatch):\n+        \"\"\"A workdir job already in _running_job_ids is skipped on next tick.\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._parallel_pool = None\n+        sched._parallel_pool_max_workers = None\n+        sched._sequential_pool = None\n+        sched._running_job_ids.clear()\n+\n+        job = {\n+            \"id\": \"guard-seq\",\n+            \"name\": \"guard-seq\",\n+            \"prompt\": \"test\",\n+            \"schedule\": \"every 5m\",\n+            \"enabled\": True,\n+            \"next_run_at\": \"2020-01-01T00:00:00\",\n+            \"deliver\": \"local\",\n+            \"workdir\": str(tmp_path),\n+        }\n+\n+        # Simulate the job already running.\n+        sched._running_job_ids.add(\"guard-seq\")\n+\n+        dispatched = []\n+        monkeypatch.setattr(sched, \"get_due_jobs\", lambda: [job])\n+        monkeypatch.setattr(sched, \"advance_next_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"run_job\", lambda j: dispatched.append(j[\"id\"]) or (True, \"out\", \"resp\", None))\n+        monkeypatch.setattr(sched, \"save_job_output\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"mark_job_run\", lambda *_a, **_kw: None)\n+        monkeypatch.setattr(sched, \"_deliver_result\", lambda *_a, **_kw: None)\n+\n+        n = sched.tick(verbose=False)\n+        assert n == 0  # skipped, not dispatched\n+        assert dispatched == []\n+\n+        sched._running_job_ids.discard(\"guard-seq\")\n+        sched._shutdown_parallel_pool()\n+\n+    def test_get_sequential_pool_is_persistent(self):\n+        \"\"\"_get_sequential_pool returns the same single-thread pool.\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._sequential_pool = None\n+        pool1 = sched._get_sequential_pool()\n+        pool2 = sched._get_sequential_pool()\n+        assert pool1 is pool2\n+\n+        sched._shutdown_parallel_pool()\n+        assert sched._sequential_pool is None\n"]}]},{"kind":"pr","number":50062,"title":"fix(cron): run missed-grace jobs once instead of deferring forever","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cron/jobs.py","tests/cron/test_jobs.py"],"fix_hunks":[{"file":"cron/jobs.py","hunks":["@@ -1240,10 +1240,16 @@ def claim_job_for_fire(job_id: str, *, claim_ttl_seconds: int = 300) -> bool:\n def get_due_jobs() -> List[Dict[str, Any]]:\n     \"\"\"Get all jobs that are due to run now.\n \n-    For recurring jobs (cron/interval), if the scheduled time is stale\n-    (more than one period in the past, e.g. because the gateway was down),\n-    the job is fast-forwarded to the next future run instead of firing\n-    immediately.  This prevents a burst of missed jobs on gateway restart.\n+    For recurring jobs (cron/interval), if the scheduled time is stale (more\n+    than one period in the past, e.g. because the gateway was down OR because a\n+    long-running previous execution overran the interval), the accumulated\n+    missed runs are collapsed — ``next_run_at`` is fast-forwarded to the next\n+    future occurrence so a backlog does NOT burst-fire on restart — but the job\n+    still fires ONCE now. This prevents the perpetual-defer loop (#33315) where\n+    a job whose runtime exceeds ``interval + grace`` would be skipped forever.\n+\n+    Note: firing once on catch-up flows through ``mark_job_run``, so a job with\n+    a ``repeat.times`` limit consumes one of its runs on that catch-up fire.\n     \"\"\"\n     with _jobs_lock():\n         return _get_due_jobs_locked()","@@ -1351,25 +1357,34 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:\n             # the next future occurrence instead of firing a stale run.\n             grace = _compute_grace_seconds(schedule)\n             if kind in {\"cron\", \"interval\"} and (now - next_run_dt).total_seconds() > grace:\n-                # Job is past its catch-up grace window — this is a stale missed run.\n-                # Grace scales with schedule period: daily=2h, hourly=30m, 10min=5m.\n+                # Job is past its catch-up grace window — skip accumulated\n+                # missed runs but still execute once now to avoid deferring\n+                # indefinitely (e.g. a long-running job just finished).\n                 new_next = compute_next_run(schedule, now.isoformat())\n                 if new_next:\n                     logger.info(\n                         \"Job '%s' missed its scheduled time (%s, grace=%ds). \"\n-                        \"Fast-forwarding to next run: %s\",\n+                        \"Running now; next run provisionally set to: %s \"\n+                        \"(re-anchored on completion)\",\n                         job.get(\"name\", job[\"id\"]),\n                         next_run,\n                         grace,\n                         new_next,\n                     )\n-                    # Update the job in storage\n+                    # Persist the fast-forward to storage now (skip accumulated\n+                    # slots). In the built-in ticker path this is shortly\n+                    # overwritten by advance_next_run + mark_job_run, but it is\n+                    # NOT redundant: it (a) protects the crash window between\n+                    # here and mark_job_run, and (b) covers the external\n+                    # fire_due provider path, which does not call\n+                    # advance_next_run. mark_job_run re-anchors next_run_at off\n+                    # the actual completion time, so this value is provisional.\n                     for rj in raw_jobs:\n                         if rj[\"id\"] == job[\"id\"]:\n                             rj[\"next_run_at\"] = new_next\n                             needs_save = True\n                             break\n-                    continue  # Skip this run\n+                    # Fall through to due.append(job) — execute once now\n \n             due.append(job)\n "]},{"file":"tests/cron/test_jobs.py","hunks":["@@ -685,10 +685,11 @@ def test_past_due_within_window_returned(self, tmp_cron_dir):\n         assert len(due) == 1\n         assert due[0][\"id\"] == job[\"id\"]\n \n-    def test_stale_past_due_skipped(self, tmp_cron_dir):\n-        \"\"\"Recurring jobs past their dynamic grace window are fast-forwarded, not fired.\n+    def test_stale_past_due_runs_once_and_fast_forwards(self, tmp_cron_dir):\n+        \"\"\"Recurring jobs past their grace window run once now and fast-forward next_run_at.\n \n         For an hourly job, grace = 30 min. Setting 35 min late exceeds the window.\n+        The job should be returned as due (execute once) with next_run_at in the future.\n         \"\"\"\n         job = create_job(prompt=\"Stale\", schedule=\"every 1h\")\n         # Force next_run_at to 35 minutes ago (beyond the 30-min grace for hourly)","@@ -697,13 +698,62 @@ def test_stale_past_due_skipped(self, tmp_cron_dir):\n         save_jobs(jobs)\n \n         due = get_due_jobs()\n-        assert len(due) == 0\n-        # next_run_at should be fast-forwarded to the future\n+        # Job is returned as due — execute once now instead of skipping\n+        assert len(due) == 1\n+        assert due[0][\"id\"] == job[\"id\"]\n+        # next_run_at should be fast-forwarded to the future (accumulated slots skipped)\n         updated = get_job(job[\"id\"])\n         from cron.jobs import _ensure_aware, _hermes_now\n         next_dt = _ensure_aware(datetime.fromisoformat(updated[\"next_run_at\"]))\n         assert next_dt > _hermes_now()\n \n+\n+    def test_long_execution_does_not_perpetually_defer(self, tmp_cron_dir, monkeypatch):\n+        \"\"\"#33315: a recurring job whose runtime exceeds interval+grace must still\n+        run once when the tick comes back, not skip forever.\n+\n+        Reproduces the production loop: a 5-min interval job whose previous run\n+        overran the interval, leaving next_run_at ~11 min in the past — beyond\n+        the 150s grace for a 5m interval. The job must be returned as due (run\n+        once) AND have next_run_at fast-forwarded (so accumulated missed slots\n+        don't all fire).\"\"\"\n+        from cron.jobs import _ensure_aware, _hermes_now\n+        job = create_job(prompt=\"Long job\", schedule=\"every 5m\")\n+        jobs = load_jobs()\n+        # 11 minutes ago: > grace (150s for a 5m interval) — the \"still running\" miss.\n+        stale = (_hermes_now() - timedelta(minutes=11)).isoformat()\n+        jobs[0][\"next_run_at\"] = stale\n+        jobs[0][\"last_run_at\"] = (_hermes_now() - timedelta(minutes=1)).isoformat()\n+        save_jobs(jobs)\n+\n+        due = get_due_jobs()\n+        assert [j[\"id\"] for j in due] == [job[\"id\"]], \"long-execution job was skipped (perpetual-defer bug)\"\n+        # next_run_at fast-forwarded into the future (no burst of missed slots).\n+        nxt = _ensure_aware(datetime.fromisoformat(get_job(job[\"id\"])[\"next_run_at\"]))\n+        assert nxt > _hermes_now()\n+\n+\n+    def test_stale_repeat_limited_job_consumes_one_run_on_catchup(self, tmp_cron_dir, monkeypatch):\n+        \"\"\"#33315 behavior note: a stale recurring job with a repeat.times limit\n+        fires ONCE on catch-up and consumes one of its runs (it is no longer\n+        silently skipped). Pins the documented repeat-count interaction so it\n+        isn't changed accidentally.\"\"\"\n+        from cron.jobs import _hermes_now\n+        job = create_job(prompt=\"Limited\", schedule=\"every 5m\", repeat=3)\n+        jobs = load_jobs()\n+        jobs[0][\"next_run_at\"] = (_hermes_now() - timedelta(minutes=11)).isoformat()\n+        jobs[0][\"last_run_at\"] = (_hermes_now() - timedelta(minutes=11)).isoformat()\n+        save_jobs(jobs)\n+\n+        # The stale job is returned to fire once (not skipped).\n+        due = get_due_jobs()\n+        assert [j[\"id\"] for j in due] == [job[\"id\"]]\n+        # Simulate the run completing: mark_job_run increments completed.\n+        mark_job_run(job[\"id\"], True)\n+        survived = get_job(job[\"id\"])\n+        assert survived is not None, \"job should survive (3 > 1 completed)\"\n+        assert survived[\"repeat\"][\"completed\"] == 1\n+\n     def test_future_not_returned(self, tmp_cron_dir):\n         create_job(prompt=\"Not yet\", schedule=\"every 1h\")\n         due = get_due_jobs()","@@ -911,10 +961,15 @@ def test_cron_offset_migration_does_not_repair_already_passed_wall_time(self, tm\n             }]\n         )\n \n-        # The wall-clock time has already passed, so this follows the existing\n-        # stale-run fast-forward behavior instead of the timezone-migration\n-        # repair path for future wall-clock runs.\n-        assert get_due_jobs() == []\n+        # The wall-clock time has already passed, so this does NOT take the\n+        # timezone-migration repair path (which is for still-future wall-clock\n+        # runs). It falls through to the stale-grace path, which — since #33315\n+        # — runs the job once now and fast-forwards next_run_at (rather than\n+        # skipping). The key assertion for THIS test is that the repaired\n+        # next_run_at is the normal next cron occurrence, not the migration\n+        # path's same-day rebase.\n+        due = get_due_jobs()\n+        assert [j[\"id\"] for j in due] == [\"cron-tz-missed\"]  # runs once now (#33315)\n         repaired = datetime.fromisoformat(get_job(\"cron-tz-missed\")[\"next_run_at\"])\n         assert repaired == datetime(2026, 5, 26, 9, 0, 0, tzinfo=current_tz)\n \n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}