{"repo":"NousResearch/hermes-agent","public_id":"58818","display_label":"Gateway shutdown draining of in-flight cron work","summary":"These issues and pull requests concern the gateway dropping in-flight cron deliveries during shutdown or restart events, including /update commands, and efforts to ensure active cron work is drained safely before exit.","subsystem":null,"counts":{"members":9,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":2,"attempt_prs":5},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":58992},"resolution":{"state":"resolved","by":"pr-58992,pr-60711"},"freshness":{"state":"verified","last_verified_at":"2026-07-08T05:15:05+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.20444444444444443},"fix_edges":[{"issue_number":58818,"pr_number":58992,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":60432,"pr_number":60711,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":58818,"title":"[Bug]: Planned-restart fires while cron delivery is in-flight — message silently dropped","url":null,"state":"closed"},{"kind":"issue","number":60432,"title":"/update bypasses drain and interrupts in-flight cron tool work","url":null,"state":"closed"},{"kind":"pr","number":58874,"title":"fix(gateway): drain in-flight cron delivery on restart instead of dropping it (#58818)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"dcd70c582","number":0},{"kind":"commit","value":"6b14be018","number":0},{"kind":"pr","value":"#58992","number":58992}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":58992,"title":"fix(gateway): drain in-flight cron delivery on restart instead of dropping it (#58818)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_cron_shutdown_drain.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -19737,6 +19737,48 @@ def _start_cron_ticker(stop_event: threading.Event, adapters=None, loop=None, in\n     InProcessCronScheduler().start(stop_event, adapters=adapters, loop=loop, interval=interval)\n \n \n+# Upper bound for cooperatively draining the cron ticker on shutdown. The cron\n+# thread delivers via ``safe_schedule_threadsafe`` and blocks on\n+# ``future.result(timeout=60)`` (see cron/scheduler.py::_deliver_result), so a\n+# single in-flight delivery unblocks within ~60s. The extra margin covers the\n+# hop back through run_one_job's bookkeeping.\n+_CRON_SHUTDOWN_DRAIN_TIMEOUT = 65.0\n+\n+# Upper bound for cooperatively draining the housekeeping ticker on shutdown.\n+# Housekeeping periodically refreshes the channel directory via\n+# ``safe_schedule_threadsafe(build_channel_directory(...), loop)`` and blocks on\n+# ``fut.result(timeout=30)`` (see ``_start_gateway_housekeeping``) — the same\n+# loop-scheduled-future pattern as cron. So the cooperative bound must cover\n+# that 30s future (plus margin) rather than the old 5s join, otherwise a\n+# channel-directory refresh in flight at shutdown gets abandoned mid-resolve.\n+# Unlike a dropped cron delivery this is not user-facing (it self-heals on the\n+# next tick), but bounding it correctly keeps the drain honest.\n+_HOUSEKEEPING_SHUTDOWN_DRAIN_TIMEOUT = 35.0\n+\n+\n+async def _await_thread_exit(\n+    thread: Optional[threading.Thread], timeout: float, poll: float = 0.1\n+) -> bool:\n+    \"\"\"Wait for a daemon thread to exit WITHOUT blocking the event loop.\n+\n+    A synchronous ``thread.join()`` here would freeze the event loop — fatal\n+    for the cron ticker, whose in-flight delivery is a coroutine scheduled onto\n+    *this* loop via ``safe_schedule_threadsafe``. Blocking the loop deadlocks\n+    that delivery (the loop can never run it), so ``join(timeout=5)`` always\n+    times out and the message is silently dropped on restart (#58818).\n+\n+    Polling ``is_alive()`` with ``await asyncio.sleep`` keeps the loop running\n+    so the pending delivery completes, then the ticker sees ``stop_event`` and\n+    exits. Returns True if the thread exited within ``timeout``.\n+    \"\"\"\n+    if thread is None:\n+        return True\n+    deadline = asyncio.get_running_loop().time() + max(0.0, timeout)\n+    while thread.is_alive() and asyncio.get_running_loop().time() < deadline:\n+        await asyncio.sleep(poll)\n+    return not thread.is_alive()\n+\n+\n async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = False, verbosity: Optional[int] = 0) -> bool:\n     \"\"\"\n     Start the gateway and run until interrupted.","@@ -20231,14 +20273,28 @@ def restart_signal_handler():\n             logger.error(\"Gateway exiting with failure: %s\", runner.exit_reason)\n         return False\n     \n-    # Stop cron scheduler + housekeeping cleanly\n+    # Stop cron scheduler + housekeeping cleanly.\n+    #\n+    # These MUST be awaited cooperatively, not join()ed. A cron delivery in\n+    # flight when the gateway restarts is a coroutine scheduled onto THIS event\n+    # loop (safe_schedule_threadsafe); the ticker thread is blocked on its\n+    # future.result(). A synchronous cron_thread.join() would block the loop,\n+    # so that delivery could never run — it timed out and the message was\n+    # silently dropped (#58818). Awaiting keeps the loop alive so the in-flight\n+    # delivery finishes before we tear down.\n     cron_stop.set()\n     try:\n         cron_provider.stop()\n     except Exception as e:\n         logger.debug(\"Cron provider stop() error: %s\", e)\n-    cron_thread.join(timeout=5)\n-    housekeeping_thread.join(timeout=5)\n+    if not await _await_thread_exit(cron_thread, timeout=_CRON_SHUTDOWN_DRAIN_TIMEOUT):\n+        logger.warning(\n+            \"Cron ticker did not exit within %.0fs of shutdown — an in-flight \"\n+            \"delivery may have been dropped.\", _CRON_SHUTDOWN_DRAIN_TIMEOUT,\n+        )\n+    await _await_thread_exit(\n+        housekeeping_thread, timeout=_HOUSEKEEPING_SHUTDOWN_DRAIN_TIMEOUT\n+    )\n \n     # Stop the planned-stop watcher (daemon=True so this is belt-and-suspenders).\n     _planned_stop_watcher_stop.set()"]},{"file":"tests/gateway/test_cron_shutdown_drain.py","hunks":["@@ -0,0 +1,73 @@\n+\"\"\"Regression tests for #58818.\n+\n+On restart the gateway must drain an in-flight cron delivery instead of\n+dropping it. A cron delivery is a coroutine scheduled onto the gateway event\n+loop (``safe_schedule_threadsafe``) while the ticker thread blocks on its\n+future. The shutdown wait therefore must NOT block the loop with a synchronous\n+``thread.join()`` — doing so deadlocks the delivery (the loop can never run it)\n+and the message is silently lost. ``_await_thread_exit`` waits cooperatively so\n+the pending delivery completes first.\n+\"\"\"\n+import asyncio\n+import threading\n+\n+import pytest\n+\n+import gateway.run as gateway_run\n+\n+\n[redacted:email]\n+async def test_await_thread_exit_lets_loop_scheduled_delivery_complete():\n+    # Reproduces the drop: the worker schedules a coroutine onto THIS loop and\n+    # blocks on its result, exactly like cron/_deliver_result. A blocking join\n+    # would deadlock it; the cooperative wait lets it finish.\n+    loop = asyncio.get_running_loop()\n+    delivered = threading.Event()\n+    worker_done = threading.Event()\n+\n+    async def _delivery():\n+        await asyncio.sleep(0.05)\n+        delivered.set()\n+        return \"ok\"\n+\n+    def _cron_worker():\n+        fut = asyncio.run_coroutine_threadsafe(_delivery(), loop)\n+        fut.result(timeout=10)\n+        worker_done.set()\n+\n+    thread = threading.Thread(target=_cron_worker, daemon=True)\n+    thread.start()\n+\n+    exited = await gateway_run._await_thread_exit(thread, timeout=10)\n+\n+    assert exited is True\n+    assert delivered.is_set(), \"in-flight delivery coroutine never ran (loop was blocked)\"\n+    assert worker_done.is_set()\n+\n+\n[redacted:email]\n+async def test_await_thread_exit_returns_false_on_timeout():\n+    keep_alive = threading.Event()\n+\n+    def _spin():\n+        keep_alive.wait(5)\n+\n+    thread = threading.Thread(target=_spin, daemon=True)\n+    thread.start()\n+    try:\n+        exited = await gateway_run._await_thread_exit(thread, timeout=0.2, poll=0.02)\n+        assert exited is False\n+        assert thread.is_alive()\n+    finally:\n+        keep_alive.set()\n+        thread.join(timeout=2)\n+\n+\n[redacted:email]\n+async def test_await_thread_exit_handles_none_and_dead_thread():\n+    assert await gateway_run._await_thread_exit(None, timeout=1) is True\n+\n+    thread = threading.Thread(target=lambda: None, daemon=True)\n+    thread.start()\n+    thread.join(timeout=2)\n+    assert await gateway_run._await_thread_exit(thread, timeout=1) is True\n"]}]},{"kind":"pr","number":60612,"title":"fix(gateway): drain in-flight cron jobs during /update shutdown (#60432)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#60711","number":60711}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":60631,"title":"fix(gateway,cron): make shutdown drain visible to in-flight cron work","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#60711","number":60711},{"kind":"pr","value":"#60612","number":60612}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":60690,"title":"fix: /update now drains active agents before gateway exit","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"ecc6725855280b4da7181c12bf930d95c7ae9a8d","number":0},{"kind":"pr","value":"#60432","number":60432},{"kind":"pr","value":"#60711","number":60711},{"kind":"pr","value":"#60432","number":60432}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":60708,"title":"fix(gateway,cron): drain in-flight cron work on shutdown and never mark killed runs successful (#60432)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"862aee495","number":0},{"kind":"commit","value":"24e9ed73c","number":0},{"kind":"commit","value":"8a573bb6e","number":0},{"kind":"commit","value":"ecc672585","number":0},{"kind":"pr","value":"#60612","number":60612},{"kind":"pr","value":"#60631","number":60631}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":60711,"title":"fix(gateway,cron): make shutdown drain see in-flight cron work (#60432)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cron/scheduler.py","gateway/run.py","tests/cron/test_shutdown_interrupt.py","tests/gateway/test_cron_active_work_drain.py","tests/gateway/test_shutdown_cache_cleanup.py","tests/gateway/test_update_cron_drain.py"],"fix_hunks":[{"file":"cron/scheduler.py","hunks":["@@ -298,6 +298,102 @@ def _is_token(line: str) -> bool:\n _running_job_ids: set = set()\n _running_lock = threading.Lock()\n \n+# Job IDs the gateway shutdown path force-killed the tool subprocess of\n+# while still in ``_running_job_ids`` (see ``mark_running_jobs_interrupted``\n+# below). ``run_one_job``'s own completion path checks this set before\n+# writing its own ``last_status`` so a cron agent thread that keeps running\n+# in-process after its tool was killed out from under it — and produces a\n+# plausible-looking final response from truncated output — can never\n+# overwrite the interrupted status with a false \"ok\" (#60432).\n+_interrupted_job_ids: set = set()\n+\n+\n+def get_running_job_ids() -> \"frozenset[str]\":\n+    \"\"\"Thread-safe snapshot of cron job IDs currently executing.\n+\n+    A job ID is a member from the moment ``_submit_with_guard`` dispatches\n+    it onto the parallel/sequential pool until ``_process_job`` returns —\n+    i.e. for the job's *entire* run, tool calls included, not just the\n+    ticker's dispatch instant.\n+\n+    The gateway shutdown path (``gateway/run.py::GatewayRunner.\n+    _drain_active_agents``) reads this to treat in-flight cron work as\n+    active the same way it already treats in-flight chat sessions via\n+    ``_running_agents`` — cron jobs run through their own thread pool here,\n+    entirely outside that dict, so without this the drain is structurally\n+    blind to them (#60432).\n+    \"\"\"\n+    with _running_lock:\n+        return frozenset(_running_job_ids)\n+\n+\n+def mark_running_jobs_interrupted(reason: str) -> list:\n+    \"\"\"Best-effort: mark every currently in-flight cron job interrupted.\n+\n+    Called by the gateway shutdown path immediately after it force-kills\n+    tool subprocesses (``process_registry.kill_all()``). A job whose tool\n+    subprocess was just killed out from under it must never be allowed to\n+    report success — even though its agent thread is still alive in this\n+    same process and may go on to produce a plausible-looking final\n+    response from the now-truncated tool output.\n+\n+    Records the job IDs in ``_interrupted_job_ids`` BEFORE writing\n+    ``last_status`` so ``run_one_job``'s own eventual completion for the\n+    same job (racing in its own thread) sees the flag and skips its normal\n+    write instead of clobbering this one — see the check near the end of\n+    ``run_one_job``. This does not attempt to correlate the killed\n+    subprocess PID to a specific job ID (the process registry tracks PIDs,\n+    not cron job IDs); any job still dispatched at the moment of a forced\n+    kill is treated as interrupted, matching the coarser precedent already\n+    set by ``GatewayRunner._interrupt_running_agents``, which interrupts\n+    every entry in ``_running_agents`` on a drain timeout without\n+    per-agent correlation either.\n+\n+    Returns the list of job IDs marked, for the caller to log.\n+    \"\"\"\n+    with _running_lock:\n+        job_ids = list(_running_job_ids)\n+        _interrupted_job_ids.update(job_ids)\n+    marked = []\n+    for job_id in job_ids:\n+        try:\n+            mark_job_run(job_id, False, reason)\n+            marked.append(job_id)\n+        except Exception as e:\n+            logger.warning(\"Failed to mark job %s interrupted: %s\", job_id, e)\n+    return marked\n+\n+\n+def _is_interrupted(job_id: str) -> bool:\n+    \"\"\"Non-destructive peek at whether the shutdown path has marked\n+    ``job_id`` interrupted (see ``mark_running_jobs_interrupted``).\n+\n+    Called by ``run_one_job`` BEFORE it decides what to deliver — a job\n+    whose tool subprocess was killed mid-flight may still produce a\n+    plausible-looking ``final_response`` from the truncated output, and\n+    that must not go out to the user as if it were a normal result.\n+    Unlike ``_consume_interrupted_flag`` below, this does not clear the\n+    flag: the later, authoritative check (right before ``last_status`` is\n+    written) still needs to see it.\"\"\"\n+    with _running_lock:\n+        return job_id in _interrupted_job_ids\n+\n+\n+def _consume_interrupted_flag(job_id: str) -> bool:\n+    \"\"\"Return True and clear the flag if the shutdown path already marked\n+    ``job_id`` interrupted (see ``mark_running_jobs_interrupted``).\n+\n+    Called by ``run_one_job`` right before it would otherwise write its own\n+    ``last_status``. Consuming (discarding) rather than just checking keeps\n+    the flag from leaking across a later, unrelated run of the same job ID\n+    (recurring jobs reuse their ID every fire).\"\"\"\n+    with _running_lock:\n+        if job_id in _interrupted_job_ids:\n+            _interrupted_job_ids.discard(job_id)\n+            return True\n+        return False\n+\n+\n # Sequential (env-mutating) cron jobs — workdir jobs that touch\n # process-global runtime state — must run one at a time, but must NOT block the\n # ticker thread.  A persistent single-thread executor preserves ordering across","@@ -3332,6 +3428,20 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) -\n             if verbose:\n                 logger.info(\"Output saved to: %s\", output_file)\n \n+            # If the gateway shutdown killed this job's tool subprocess\n+            # mid-flight (#60432), the agent may still have produced a\n+            # plausible-looking final_response from the truncated output --\n+            # force the failure path so the delivered message is an honest\n+            # \"this run was interrupted\" summary instead of that response.\n+            # Peek-only: the flag stays set for the authoritative check\n+            # right before mark_job_run below.\n+            if success and _is_interrupted(job[\"id\"]):\n+                success = False\n+                error = (\n+                    \"Interrupted by gateway shutdown before the run finished \"\n+                    \"(tool subprocess was killed mid-flight).\"\n+                )\n+\n             # Deliver the final response to the origin/target chat.\n             # If the agent responded with [SILENT], skip delivery (but\n             # output is already saved above).  Failed jobs always deliver.","@@ -3370,12 +3480,14 @@ def run_one_job(job: dict, *, adapters=None, loop=None, verbose: bool = False) -\n             success = False\n             error = \"Agent completed but produced empty response (model error, timeout, or misconfiguration)\"\n \n-        mark_job_run(job[\"id\"], success, error, delivery_error=delivery_error)\n+        if not _consume_interrupted_flag(job[\"id\"]):\n+            mark_job_run(job[\"id\"], success, error, delivery_error=delivery_error)\n         return True\n \n     except Exception as e:\n         logger.error(\"Error processing job %s: %s\", job['id'], e)\n-        mark_job_run(job[\"id\"], False, str(e))\n+        if not _consume_interrupted_flag(job[\"id\"]):\n+            mark_job_run(job[\"id\"], False, str(e))\n         return False\n \n "]},{"file":"gateway/run.py","hunks":["@@ -4080,6 +4080,26 @@ def _request_clean_exit(self, reason: str) -> None:\n     def _running_agent_count(self) -> int:\n         return len(self._running_agents)\n \n+    def _active_cron_job_count(self) -> int:\n+        \"\"\"Count of cron jobs currently executing, from the cron scheduler's\n+        own in-flight tracking (``cron.scheduler._running_job_ids``).\n+\n+        Cron jobs run through a standalone ``AIAgent`` on the scheduler's own\n+        thread pool (``cron/scheduler.py::run_job``), entirely outside\n+        ``self._running_agents`` — the dict every OTHER active-work check on\n+        this class (``_running_agent_count``, ``_drain_active_agents``) reads.\n+        Without this, the shutdown drain is structurally blind to in-flight\n+        cron work: it can report ``active_at_start=0`` and proceed straight\n+        to killing tool subprocesses while a cron job's terminal command is\n+        still running (#60432). Best-effort: returns 0 if the cron module\n+        can't be imported (e.g. a minimal test double for this class).\n+        \"\"\"\n+        try:\n+            from cron.scheduler import get_running_job_ids\n+            return len(get_running_job_ids())\n+        except Exception:\n+            return 0\n+\n     # ── scale-to-zero idle detection / dormant-quiesce (Phase 0) ──────────────\n     # The gateway-side BEHAVIOUR that consumes the relay scale-to-zero primitives\n     # (gateway-gateway Phase 5). Pure logic lives in gateway/scale_to_zero.py; the","@@ -5550,18 +5570,31 @@ async def _handle_active_session_busy_message(self, event: MessageEvent, session\n     async def _drain_active_agents(self, timeout: float) -> tuple[Dict[str, Any], bool]:\n         snapshot = self._snapshot_running_agents()\n         last_active_count = self._running_agent_count()\n+        last_cron_count = self._active_cron_job_count()\n         last_status_at = 0.0\n \n         def _maybe_update_status(force: bool = False) -> None:\n-            nonlocal last_active_count, last_status_at\n+            nonlocal last_active_count, last_cron_count, last_status_at\n             now = asyncio.get_running_loop().time()\n             active_count = self._running_agent_count()\n-            if force or active_count != last_active_count or (now - last_status_at) >= 1.0:\n+            cron_count = self._active_cron_job_count()\n+            if (\n+                force\n+                or active_count != last_active_count\n+                or cron_count != last_cron_count\n+                or (now - last_status_at) >= 1.0\n+            ):\n                 self._update_runtime_status(\"draining\")\n                 last_active_count = active_count\n+                last_cron_count = cron_count\n                 last_status_at = now\n \n-        if not self._running_agents:\n+        # Cron jobs run on the scheduler's own thread pool, outside\n+        # ``self._running_agents`` — fold their in-flight count into the\n+        # same wait/timeout this method already applies to chat sessions,\n+        # or a cron job's tool work gets killed with zero warning the\n+        # instant it's the only active thing running (#60432).\n+        if not self._running_agents and last_cron_count == 0:\n             _maybe_update_status(force=True)\n             return snapshot, False\n ","@@ -5570,10 +5603,13 @@ def _maybe_update_status(force: bool = False) -> None:\n             return snapshot, True\n \n         deadline = asyncio.get_running_loop().time() + timeout\n-        while self._running_agents and asyncio.get_running_loop().time() < deadline:\n+        while (\n+            (self._running_agents or self._active_cron_job_count())\n+            and asyncio.get_running_loop().time() < deadline\n+        ):\n             _maybe_update_status()\n             await asyncio.sleep(0.1)\n-        timed_out = bool(self._running_agents)\n+        timed_out = bool(self._running_agents) or bool(self._active_cron_job_count())\n         _maybe_update_status(force=True)\n         return snapshot, timed_out\n ","@@ -7944,6 +7980,27 @@ def _kill_tool_subprocesses(phase: str) -> None:\n                         )\n                 except Exception as _e:\n                     logger.debug(\"process_registry.kill_all (%s) error: %s\", phase, _e)\n+                try:\n+                    # Any cron job still dispatched at this instant just had\n+                    # its tool subprocess killed above (kill_all() has no\n+                    # per-job-ID targeting — it's a global sweep). Its agent\n+                    # thread is still alive in this process and may go on to\n+                    # produce a plausible-looking final response from the\n+                    # now-truncated tool output; mark the run interrupted so\n+                    # the scheduler can never report that as success (#60432).\n+                    # No-op when no cron job is in flight.\n+                    from cron.scheduler import mark_running_jobs_interrupted\n+                    _interrupted = mark_running_jobs_interrupted(\n+                        f\"Gateway shutdown ({phase}) killed the job's tool \"\n+                        \"subprocess before the run finished.\"\n+                    )\n+                    if _interrupted:\n+                        logger.warning(\n+                            \"Shutdown (%s): marked %d in-flight cron job(s) interrupted: %s\",\n+                            phase, len(_interrupted), \", \".join(_interrupted),\n+                        )\n+                except Exception as _e:\n+                    logger.debug(\"mark_running_jobs_interrupted (%s) error: %s\", phase, _e)\n                 try:\n                     from tools.async_delegation import interrupt_all as _interrupt_async\n                     _async_n = _interrupt_async(reason=f\"gateway shutdown ({phase})\")","@@ -8004,16 +8061,20 @@ def _phase_elapsed() -> float:\n                 except Exception as _e:\n                     logger.debug(\"pre-drain mark_resume_pending failed for %s: %s\", _sk, _e)\n \n+            _cron_at_start = self._active_cron_job_count()\n             _drain_started_at = time.monotonic()\n             active_agents, timed_out = await self._drain_active_agents(timeout)\n             logger.info(\n                 \"Shutdown phase: drain done at +%.2fs (drain took %.2fs, \"\n-                \"timed_out=%s, active_at_start=%d, active_now=%d)\",\n+                \"timed_out=%s, active_at_start=%d, active_now=%d, \"\n+                \"cron_at_start=%d, cron_now=%d)\",\n                 _phase_elapsed(),\n                 time.monotonic() - _drain_started_at,\n                 timed_out,\n                 len(active_agents),\n                 self._running_agent_count(),\n+                _cron_at_start,\n+                self._active_cron_job_count(),\n             )\n \n             if not timed_out:","@@ -8032,9 +8093,11 @@ def _phase_elapsed() -> float:\n \n             if timed_out:\n                 logger.warning(\n-                    \"Gateway drain timed out after %.1fs with %d active agent(s); interrupting remaining work.\",\n+                    \"Gateway drain timed out after %.1fs with %d active agent(s) \"\n+                    \"and %d in-flight cron job(s); interrupting remaining work.\",\n                     timeout,\n                     self._running_agent_count(),\n+                    self._active_cron_job_count(),\n                 )\n                 # Mark forcibly-interrupted sessions as resume_pending BEFORE\n                 # interrupting the agents.  This preserves each session's"]},{"file":"tests/cron/test_shutdown_interrupt.py","hunks":["@@ -0,0 +1,278 @@\n+\"\"\"Tests for #60432: cron jobs must not be silently invisible to gateway\n+shutdown, and a job whose tool subprocess got killed by shutdown must\n+never be reported as a successful run.\n+\n+Covers the cron/scheduler.py primitives directly:\n+  - get_running_job_ids() -- thread-safe snapshot the gateway drain reads\n+  - mark_running_jobs_interrupted() -- called by the gateway right after\n+    it force-kills tool subprocesses\n+  - the interrupted-flag race guard in run_one_job(), which must win over\n+    the job's own thread finishing normally with a plausible-looking\n+    result AFTER its tool was already killed out from under it\n+\"\"\"\n+\n+from unittest.mock import patch\n+\n+import pytest\n+\n+\n[redacted:email](autouse=True)\n+def _reset_scheduler_state():\n+    \"\"\"Every test starts from a clean slate and leaves one behind, since\n+    these sets are module-level globals shared across the test process.\"\"\"\n+    import cron.scheduler as sched\n+\n+    sched._running_job_ids.clear()\n+    sched._interrupted_job_ids.clear()\n+    yield\n+    sched._running_job_ids.clear()\n+    sched._interrupted_job_ids.clear()\n+\n+\n+class TestGetRunningJobIds:\n+    def test_empty_when_nothing_running(self):\n+        import cron.scheduler as sched\n+\n+        assert sched.get_running_job_ids() == frozenset()\n+\n+    def test_reflects_in_flight_jobs(self):\n+        import cron.scheduler as sched\n+\n+        sched._running_job_ids.add(\"job-1\")\n+        sched._running_job_ids.add(\"job-2\")\n+\n+        result = sched.get_running_job_ids()\n+\n+        assert result == frozenset({\"job-1\", \"job-2\"})\n+\n+    def test_snapshot_is_immutable_and_independent(self):\n+        \"\"\"Mutating _running_job_ids after the call must not change the\n+        already-returned snapshot -- callers (the gateway drain loop) rely\n+        on this to safely count in a tight polling loop.\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._running_job_ids.add(\"job-1\")\n+        snapshot = sched.get_running_job_ids()\n+        sched._running_job_ids.add(\"job-2\")\n+\n+        assert snapshot == frozenset({\"job-1\"})\n+\n+\n+class TestMarkRunningJobsInterrupted:\n+    def test_no_op_when_nothing_running(self):\n+        import cron.scheduler as sched\n+\n+        with patch(\"cron.scheduler.mark_job_run\") as mock_mark:\n+            marked = sched.mark_running_jobs_interrupted(\"shutdown\")\n+\n+        assert marked == []\n+        mock_mark.assert_not_called()\n+\n+    def test_marks_every_in_flight_job(self):\n+        import cron.scheduler as sched\n+\n+        sched._running_job_ids.update({\"job-1\", \"job-2\"})\n+\n+        with patch(\"cron.scheduler.mark_job_run\") as mock_mark:\n+            marked = sched.mark_running_jobs_interrupted(\"gateway shutdown (final-cleanup)\")\n+\n+        assert sorted(marked) == [\"job-1\", \"job-2\"]\n+        assert mock_mark.call_count == 2\n+        called_ids = {c.args[0] for c in mock_mark.call_args_list}\n+        assert called_ids == {\"job-1\", \"job-2\"}\n+        for c in mock_mark.call_args_list:\n+            # success must be False -- an interrupted run is never \"ok\".\n+            assert c.args[1] is False\n+            assert \"gateway shutdown\" in c.args[2]\n+\n+    def test_sets_interrupted_flag_for_consumption_by_run_one_job(self):\n+        import cron.scheduler as sched\n+\n+        sched._running_job_ids.add(\"job-1\")\n+\n+        with patch(\"cron.scheduler.mark_job_run\"):\n+            sched.mark_running_jobs_interrupted(\"shutdown\")\n+\n+        assert \"job-1\" in sched._interrupted_job_ids\n+\n+    def test_one_job_marking_failure_does_not_block_the_others(self):\n+        \"\"\"mark_job_run raising for one job (e.g. a jobs.json write race)\n+        must not prevent the rest from being marked -- this runs during\n+        shutdown, there's no retry window.\"\"\"\n+        import cron.scheduler as sched\n+\n+        sched._running_job_ids.update({\"job-1\", \"job-2\"})\n+\n+        def _side_effect(job_id, success, reason, **kwargs):\n+            if job_id == \"job-1\":\n+                raise OSError(\"disk full\")\n+\n+        with patch(\"cron.scheduler.mark_job_run\", side_effect=_side_effect):\n+            marked = sched.mark_running_jobs_interrupted(\"shutdown\")\n+\n+        assert marked == [\"job-2\"]\n+\n+\n+class TestIsInterrupted:\n+    \"\"\"Peek-only check used at the delivery gate -- must NOT clear the\n+    flag, unlike _consume_interrupted_flag.\"\"\"\n+\n+    def test_false_when_not_marked(self):\n+        import cron.scheduler as sched\n+\n+        assert sched._is_interrupted(\"job-1\") is False\n+\n+    def test_true_when_marked(self):\n+        import cron.scheduler as sched\n+\n+        sched._interrupted_job_ids.add(\"job-1\")\n+\n+        assert sched._is_interrupted(\"job-1\") is True\n+\n+    def test_does_not_clear_the_flag(self):\n+        import cron.scheduler as sched\n+\n+        sched._interrupted_job_ids.add(\"job-1\")\n+\n+        sched._is_interrupted(\"job-1\")\n+\n+        # Still set -- the later, authoritative check before mark_job_run\n+        # must still see it.\n+        assert \"job-1\" in sched._interrupted_job_ids\n+        assert sched._is_interrupted(\"job-1\") is True\n+\n+\n+class TestConsumeInterruptedFlag:\n+    def test_false_when_not_marked(self):\n+        import cron.scheduler as sched\n+\n+        assert sched._consume_interrupted_flag(\"job-1\") is False\n+\n+    def test_true_and_clears_when_marked(self):\n+        import cron.scheduler as sched\n+\n+        sched._interrupted_job_ids.add(\"job-1\")\n+\n+        assert sched._consume_interrupted_flag(\"job-1\") is True\n+        # Consumed -- a second check (e.g. a later, unrelated fire of the\n+        # same recurring job ID) must not still read as interrupted.\n+        assert sched._consume_interrupted_flag(\"job-1\") is False\n+\n+\n+class TestRunOneJobHonoursInterruptedFlag:\n+    \"\"\"run_one_job() must not let a job's own completion overwrite a\n+    status the shutdown path already wrote for the same run.\"\"\"\n+\n+    def _make_job(self, job_id=\"job-1\"):\n+        return {\"id\": job_id, \"name\": \"test job\", \"prompt\": \"do work\"}\n+\n+    def test_success_path_skipped_when_interrupted(self):\n+        import cron.scheduler as sched\n+\n+        job = self._make_job()\n+        sched._interrupted_job_ids.add(job[\"id\"])\n+\n+        with patch(\"cron.scheduler.claim_dispatch\", return_value=True), \\\n+             patch(\"agent.secret_scope.set_secret_scope\", return_value=None), \\\n+             patch(\"agent.secret_scope.build_profile_secret_scope\", return_value=None), \\\n+             patch(\"agent.secret_scope.reset_secret_scope\"), \\\n+             patch(\n+                 \"cron.scheduler.run_job\",\n+                 return_value=(True, \"full output\", \"final response\", None),\n+             ), \\\n+             patch(\"cron.scheduler.save_job_output\", return_value=\"/tmp/out.md\"), \\\n+             patch(\"cron.scheduler._is_cron_silence_response\", return_value=False), \\\n+             patch(\"cron.scheduler._deliver_result\", return_value=None), \\\n+             patch(\"cron.scheduler.mark_job_run\") as mock_mark:\n+            result = sched.run_one_job(job)\n+\n+        assert result is True\n+        # The would-be \"success\" write must NOT happen -- the shutdown\n+        # path already wrote the authoritative interrupted status.\n+        mock_mark.assert_not_called()\n+        # Flag is consumed so a later, unrelated fire of the same job ID\n+        # isn't permanently silenced.\n+        assert job[\"id\"] not in sched._interrupted_job_ids\n+\n+    def test_interrupted_job_delivers_failure_summary_not_raw_response(self):\n+        \"\"\"The status-write guard alone isn't enough: delivery happens\n+        BEFORE mark_job_run in run_one_job's own flow, so a job that kept\n+        running post-kill and produced a plausible-looking final_response\n+        must not have that response sent to the user just because the\n+        eventual status write gets suppressed. Interrupted jobs must route\n+        through the same failure-summary delivery path a real failure\n+        would.\"\"\"\n+        import cron.scheduler as sched\n+\n+        job = self._make_job()\n+        sched._interrupted_job_ids.add(job[\"id\"])\n+\n+        with patch(\"cron.scheduler.claim_dispatch\", return_value=True), \\\n+             patch(\"agent.secret_scope.set_secret_scope\", return_value=None), \\\n+             patch(\"agent.secret_scope.build_profile_secret_scope\", return_value=None), \\\n+             patch(\"agent.secret_scope.reset_secret_scope\"), \\\n+             patch(\n+                 \"cron.scheduler.run_job\",\n+                 return_value=(True, \"full output\", \"a plausible final response\", None),\n+             ), \\\n+             patch(\"cron.scheduler.save_job_output\", return_value=\"/tmp/out.md\"), \\\n+             patch(\n+                 \"cron.scheduler._summarize_cron_failure_for_delivery\",\n+                 return_value=\"This run was interrupted.\",\n+             ) as mock_summarize, \\\n+             patch(\"cron.scheduler._is_cron_silence_response\", return_value=False), \\\n+             patch(\"cron.scheduler._deliver_result\", return_value=None) as mock_deliver, \\\n+             patch(\"cron.scheduler.mark_job_run\"):\n+            result = sched.run_one_job(job)\n+\n+        assert result is True\n+        mock_summarize.assert_called_once()\n+        # The summarizer's error argument must mention the interruption,\n+        # not be silently None / the agent's own (possibly absent) error.\n+        assert \"interrupt\" in mock_summarize.call_args.args[1].lower()\n+        delivered_content = mock_deliver.call_args.args[1]\n+        assert delivered_content == \"This run was interrupted.\"\n+        assert \"plausible final response\" not in delivered_content\n+\n+    def test_success_path_writes_normally_when_not_interrupted(self):\n+        \"\"\"Control case: the guard must not swallow ordinary, un-interrupted\n+        completions -- only ones the shutdown path explicitly flagged.\"\"\"\n+        import cron.scheduler as sched\n+\n+        job = self._make_job()\n+\n+        with patch(\"cron.scheduler.claim_dispatch\", return_value=True), \\\n+             patch(\"agent.secret_scope.set_secret_scope\", return_value=None), \\\n+             patch(\"agent.secret_scope.build_profile_secret_scope\", return_value=None), \\\n+             patch(\"agent.secret_scope.reset_secret_scope\"), \\\n+             patch(\n+                 \"cron.scheduler.run_job\",\n+                 return_value=(True, \"full output\", \"final response\", None),\n+             ), \\\n+             patch(\"cron.scheduler.save_job_output\", return_value=\"/tmp/out.md\"), \\\n+             patch(\"cron.scheduler._is_cron_silence_response\", return_value=False), \\\n+             patch(\"cron.scheduler._deliver_result\", return_value=None), \\\n+             patch(\"cron.scheduler.mark_job_run\") as mock_mark:\n+            result = sched.run_one_job(job)\n+\n+        assert result is True\n+        mock_mark.assert_called_once()\n+        assert mock_mark.call_args.args[0] == job[\"id\"]\n+        assert mock_mark.call_args.args[1] is True  # success\n+\n+    def test_exception_path_also_honours_interrupted_flag(self):\n+        import cron.scheduler as sched\n+\n+        job = self._make_job()\n+        sched._interrupted_job_ids.add(job[\"id\"])\n+\n+        with patch(\"cron.scheduler.claim_dispatch\", return_value=True), \\\n+             patch(\"agent.secret_scope.set_secret_scope\", return_value=None), \\\n+             patch(\"agent.secret_scope.build_profile_secret_scope\", return_value=None), \\\n+             patch(\"agent.secret_scope.reset_secret_scope\"), \\\n+             patch(\"cron.scheduler.run_job\", side_effect=RuntimeError(\"boom\")), \\\n+             patch(\"cron.scheduler.mark_job_run\") as mock_mark:\n+            result = sched.run_one_job(job)\n+\n+        assert result is False\n+        mock_mark.assert_not_called()"]},{"file":"tests/gateway/test_cron_active_work_drain.py","hunks":["@@ -0,0 +1,185 @@\n+\"\"\"Tests for #60432: the gateway shutdown drain was structurally blind to\n+in-flight cron work. Cron jobs run through cron/scheduler.py's own thread\n+pool, entirely outside ``GatewayRunner._running_agents`` -- the dict every\n+other active-work check on this class reads. A shutdown (``/update``,\n+``/restart``, SIGUSR1 -- they all funnel through the same ``stop()``) could\n+report ``active_at_start=0`` and immediately kill tool subprocesses while a\n+cron job's terminal command was still running.\n+\n+These tests cover the gateway side of the fix:\n+  - _active_cron_job_count() reads cron.scheduler's in-flight job set\n+  - _drain_active_agents() waits for cron work the same way it already\n+    waits for chat sessions\n+  - the final tool-subprocess kill marks any still-in-flight cron job\n+    interrupted\n+\n+See tests/cron/test_shutdown_interrupt.py for the cron-side primitives\n+this relies on (get_running_job_ids, mark_running_jobs_interrupted).\n+\"\"\"\n+\n+import asyncio\n+from unittest.mock import MagicMock, patch\n+\n+import pytest\n+\n+from tests.gateway.restart_test_helpers import make_restart_runner\n+\n+\n[redacted:email](autouse=True)\n+def _reset_cron_running_set():\n+    import cron.scheduler as sched\n+\n+    sched._running_job_ids.clear()\n+    sched._interrupted_job_ids.clear()\n+    yield\n+    sched._running_job_ids.clear()\n+    sched._interrupted_job_ids.clear()\n+\n+\n+def _make_async_noop():\n+    async def _noop(*args, **kwargs):\n+        return None\n+\n+    return _noop\n+\n+\n+class TestActiveCronJobCount:\n+    def test_zero_when_no_cron_jobs_running(self):\n+        runner, _adapter = make_restart_runner()\n+        assert runner._active_cron_job_count() == 0\n+\n+    def test_reflects_cron_scheduler_state(self):\n+        import cron.scheduler as sched\n+\n+        runner, _adapter = make_restart_runner()\n+        sched._running_job_ids.add(\"job-1\")\n+\n+        assert runner._active_cron_job_count() == 1\n+\n+    def test_never_raises_if_cron_module_unavailable(self):\n+        \"\"\"Best-effort: a broken/absent import must not take shutdown\n+        counting down with it.\"\"\"\n+        runner, _adapter = make_restart_runner()\n+\n+        with patch(\n+            \"cron.scheduler.get_running_job_ids\", side_effect=ImportError(\"boom\")\n+        ):\n+            assert runner._active_cron_job_count() == 0\n+\n+\n+class TestDrainWaitsForCronWork:\n+    @pytest.mark.asyncio\n+    async def test_drain_returns_immediately_when_nothing_active(self):\n+        runner, _adapter = make_restart_runner()\n+\n+        _snapshot, timed_out = await runner._drain_active_agents(5.0)\n+\n+        assert timed_out is False\n+\n+    @pytest.mark.asyncio\n+    async def test_drain_waits_for_in_flight_cron_job(self):\n+        \"\"\"Before this fix, a cron-only workload made active_at_start=0\n+        and the drain returned instantly -- this is the exact repro from\n+        the issue (a `sleep 1800` cron job in flight during /update).\"\"\"\n+        import cron.scheduler as sched\n+\n+        runner, _adapter = make_restart_runner()\n+        sched._running_job_ids.add(\"job-1\")\n+\n+        async def finish_job():\n+            await asyncio.sleep(0.12)\n+            sched._running_job_ids.discard(\"job-1\")\n+\n+        task = asyncio.create_task(finish_job())\n+        _snapshot, timed_out = await runner._drain_active_agents(2.0)\n+        await task\n+\n+        assert timed_out is False, (\n+            \"drain must wait for the cron job to finish, not report \"\n+            \"active_at_start=0 and return instantly\"\n+        )\n+\n+    @pytest.mark.asyncio\n+    async def test_drain_times_out_if_cron_job_outlives_the_window(self):\n+        import cron.scheduler as sched\n+\n+        runner, _adapter = make_restart_runner()\n+        sched._running_job_ids.add(\"job-1\")  # never removed within the window\n+\n+        _snapshot, timed_out = await runner._drain_active_agents(0.1)\n+\n+        assert timed_out is True\n+\n+    @pytest.mark.asyncio\n+    async def test_drain_still_waits_for_chat_sessions_unchanged(self):\n+        \"\"\"Regression guard: folding cron into the check must not break\n+        the pre-existing chat-session drain behavior.\"\"\"\n+        runner, _adapter = make_restart_runner()\n+        runner._running_agents = {\"session-1\": MagicMock()}\n+\n+        async def finish_agent():\n+            await asyncio.sleep(0.12)\n+            runner._running_agents.clear()\n+\n+        task = asyncio.create_task(finish_agent())\n+        _snapshot, timed_out = await runner._drain_active_agents(2.0)\n+        await task\n+\n+        assert timed_out is False\n+\n+\n+class TestKillToolSubprocessesMarksCronInterrupted:\n+    @pytest.mark.asyncio\n+    async def test_in_flight_cron_job_marked_interrupted_on_forced_kill(self, monkeypatch):\n+        import cron.scheduler as sched\n+        import tools.process_registry as _pr\n+        import tools.terminal_tool as _tt\n+        import tools.browser_tool as _bt\n+\n+        runner, adapter = make_restart_runner()\n+        runner._restart_drain_timeout = 0.01  # force the timeout path\n+        adapter.disconnect = _make_async_noop()\n+\n+        sched._running_job_ids.add(\"job-1\")\n+\n+        monkeypatch.setattr(_pr.process_registry, \"kill_all\", lambda task_id=None: 1)\n+        monkeypatch.setattr(_tt, \"cleanup_all_environments\", lambda: None)\n+        monkeypatch.setattr(_bt, \"cleanup_all_browsers\", lambda: None)\n+\n+        marked_calls = []\n+        real_mark = sched.mark_running_jobs_interrupted\n+\n+        def _spy(reason):\n+            result = real_mark(reason)\n+            marked_calls.append((reason, result))\n+            return result\n+\n+        monkeypatch.setattr(sched, \"mark_running_jobs_interrupted\", _spy)\n+\n+        with patch(\"gateway.status.remove_pid_file\"), patch(\"gateway.status.write_runtime_status\"), \\\n+             patch(\"cron.scheduler.mark_job_run\"):\n+            await runner.stop()\n+\n+        assert marked_calls, \"mark_running_jobs_interrupted was never called during shutdown\"\n+        assert any(result == [\"job-1\"] for _reason, result in marked_calls)\n+\n+    @pytest.mark.asyncio\n+    async def test_no_cron_jobs_running_is_a_silent_no_op(self, monkeypatch):\n+        \"\"\"Graceful shutdown with nothing in flight must not spuriously\n+        mark or log anything cron-related.\"\"\"\n+        import tools.process_registry as _pr\n+        import tools.terminal_tool as _tt\n+        import tools.browser_tool as _bt\n+\n+        runner, adapter = make_restart_runner()\n+        adapter.disconnect = _make_async_noop()\n+\n+        monkeypatch.setattr(_pr.process_registry, \"kill_all\", lambda task_id=None: 0)\n+        monkeypatch.setattr(_tt, \"cleanup_all_environments\", lambda: None)\n+        monkeypatch.setattr(_bt, \"cleanup_all_browsers\", lambda: None)\n+\n+        with patch(\"gateway.status.remove_pid_file\"), patch(\"gateway.status.write_runtime_status\"), \\\n+             patch(\"cron.scheduler.mark_job_run\") as mock_mark:\n+            await runner.stop()\n+\n+        mock_mark.assert_not_called()"]},{"file":"tests/gateway/test_shutdown_cache_cleanup.py","hunks":["@@ -54,6 +54,12 @@ def __init__(self):\n     def _running_agent_count(self):\n         return len(self._running_agents)\n \n+    def _active_cron_job_count(self):\n+        # stop() reads this alongside _running_agent_count when logging the\n+        # drain snapshot (#60432) -- this fake has no cron scheduler, so\n+        # there's never in-flight cron work to report.\n+        return 0\n+\n     def _update_runtime_status(self, *_a, **_kw):\n         pass\n "]},{"file":"tests/gateway/test_update_cron_drain.py","hunks":["@@ -0,0 +1,86 @@\n+\"\"\"Regression tests for #60432.\n+\n+``/update`` (and other gateway shutdown paths) must drain in-flight cron jobs\n+before ``process_registry.kill_all()`` runs in final-cleanup.  Cron work runs on\n+a thread-pool worker and is tracked in ``cron.scheduler._running_job_ids``, not\n+in ``GatewayRunner._running_agents`` — so a zero-agent drain must still wait\n+for cron to finish (or time out and take the interrupt/kill path).\n+\"\"\"\n+import asyncio\n+from unittest.mock import AsyncMock, MagicMock, patch\n+\n+import pytest\n+\n+from tests.gateway.restart_test_helpers import make_restart_runner\n+\n+\n[redacted:email]\n+async def test_drain_active_agents_waits_for_in_flight_cron_jobs():\n+    runner, _adapter = make_restart_runner()\n+    runner._running_agents = {}\n+\n+    cron_count = [1]\n+\n+    def _cron_in_flight():\n+        return frozenset(f\"job-{i}\" for i in range(cron_count[0]))\n+\n+    async def finish_cron():\n+        await asyncio.sleep(0.15)\n+        cron_count[0] = 0\n+\n+    with patch(\"cron.scheduler.get_running_job_ids\", side_effect=_cron_in_flight):\n+        task = asyncio.create_task(finish_cron())\n+        _snapshot, timed_out = await runner._drain_active_agents(1.0)\n+        await task\n+\n+    assert timed_out is False\n+    assert _snapshot == {}\n+\n+\n[redacted:email]\n+async def test_drain_active_agents_times_out_when_cron_still_running():\n+    runner, _adapter = make_restart_runner()\n+    runner._running_agents = {}\n+\n+    with patch(\"cron.scheduler.get_running_job_ids\", return_value=frozenset({\"job-1\"})):\n+        _snapshot, timed_out = await runner._drain_active_agents(0.05)\n+\n+    assert timed_out is True\n+    assert _snapshot == {}\n+\n+\n[redacted:email]\n+async def test_gateway_stop_waits_for_cron_before_final_tool_kill():\n+    \"\"\"Graceful cron completion must finish before final-cleanup kill_all.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    runner._restart_drain_timeout = 1.0\n+\n+    cron_count = [1]\n+    call_order: list[str] = []\n+\n+    def _cron_in_flight():\n+        return frozenset(f\"job-{i}\" for i in range(cron_count[0]))\n+\n+    def _fake_kill_all(task_id=None):\n+        call_order.append(\"kill_all\")\n+        return 0\n+\n+    async def finish_cron():\n+        await asyncio.sleep(0.12)\n+        cron_count[0] = 0\n+\n+    with (\n+        patch(\"cron.scheduler.get_running_job_ids\", side_effect=_cron_in_flight),\n+        patch(\"gateway.status.remove_pid_file\"),\n+        patch(\"gateway.status.write_runtime_status\"),\n+        patch(\"agent.auxiliary_client.shutdown_cached_clients\"),\n+        patch(\"tools.process_registry.process_registry\") as registry_mock,\n+    ):\n+        registry_mock.kill_all.side_effect = _fake_kill_all\n+        adapter.disconnect = AsyncMock()\n+\n+        cron_task = asyncio.create_task(finish_cron())\n+        await runner.stop()\n+        await cron_task\n+\n+    assert call_order == [\"kill_all\"]\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}