{"repo":"NousResearch/hermes-agent","public_id":"63529","display_label":"Gateway shutdown now drains in-flight API runs","summary":"These members address a bug where the gateway's shutdown process did not properly drain or interrupt in-flight API server runs, leaving active sessions without protection during shutdown. The work ensures both API and cron workloads are safely handled when the gateway shuts down.","subsystem":null,"counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":2,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":80917,"merged_at":"2026-08-07T08:41:19+00:00"},"resolution":{"state":"resolved","by":"pr-63891"},"freshness":{"state":"verified","last_verified_at":"2026-08-07T08:41:19+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":63529,"pr_number":63891,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-07-13T17:56:11+00:00"},{"issue_number":63529,"pr_number":80917,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-08-07T08:41:19+00:00"}],"members":[{"kind":"issue","number":63529,"title":"[Bug]: shutdown drain is blind to api_server in-flight runs — desk/API sessions get zero drain protection","url":null,"state":"closed"},{"kind":"pr","number":63648,"title":"fix(gateway): drain in-flight api_server runs on shutdown","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#63891","number":63891}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":63891,"title":"fix(gateway): drain API and cron work safely","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cron/scheduler.py","cron/scheduler_provider.py","gateway/platforms/api_server.py","gateway/run.py","tests/cron/test_scheduler.py","tests/cron/test_scheduler_provider.py","tests/gateway/restart_test_helpers.py","tests/gateway/test_api_server_active_work_drain.py","tests/gateway/test_api_server_jobs.py","tests/gateway/test_cron_fire_webhook.py","tests/gateway/test_external_drain_control.py","tests/gateway/test_shutdown_cache_cleanup.py"],"fix_hunks":[{"file":"cron/scheduler.py","hunks":["@@ -3557,7 +3557,14 @@ def _notify_provider_jobs_changed() -> None:\n         logger.debug(\"on_jobs_changed notify failed: %s\", e)\n \n \n-def tick(verbose: bool = True, adapters=None, loop=None, sync: bool = True) -> int:\n+def tick(\n+    verbose: bool = True,\n+    adapters=None,\n+    loop=None,\n+    sync: bool = True,\n+    *,\n+    can_dispatch=None,\n+):\n     \"\"\"\n     Check and run all due jobs.\n     ","@@ -3568,7 +3575,9 @@ def tick(verbose: bool = True, adapters=None, loop=None, sync: bool = True) -> i\n         verbose: Whether to print status messages\n         adapters: Optional dict mapping Platform → live adapter (from gateway)\n         loop: Optional asyncio event loop (from gateway) for live adapter sends\n-    \n+        can_dispatch: Optional synchronous gate; false leaves due jobs untouched\n+            for the next allowed tick\n+\n     Returns:\n         Number of jobs executed (0 if another tick is already running)\n     \"\"\"","@@ -3590,6 +3599,10 @@ def tick(verbose: bool = True, adapters=None, loop=None, sync: bool = True) -> i\n         return 0\n \n     try:\n+        if can_dispatch is not None and not can_dispatch():\n+            logger.debug(\"Cron dispatch paused while gateway drains existing work\")\n+            return 0\n+\n         due_jobs = get_due_jobs()\n \n         if verbose and not due_jobs:"]},{"file":"cron/scheduler_provider.py","hunks":["@@ -156,14 +156,16 @@ class InProcessCronScheduler(CronScheduler):\n \n     ``start()`` blocks in the tick loop until ``stop_event`` is set, identical\n     to the pre-refactor ``_start_cron_ticker`` core loop. The caller runs it in\n-    a daemon thread.\n+    a daemon thread. ``can_dispatch`` is an optional synchronous gate supplied\n+    by GatewayRunner during external drain; skipped ticks leave due jobs intact\n+    for the next allowed tick.\n     \"\"\"\n \n     @property\n     def name(self) -> str:\n         return \"builtin\"\n \n-    def start(self, stop_event, *, adapters=None, loop=None, interval=60):\n+    def start(self, stop_event, *, adapters=None, loop=None, interval=60, can_dispatch=None):\n         import logging\n         from cron.scheduler import tick as cron_tick\n         from cron.jobs import record_ticker_heartbeat","@@ -176,7 +178,16 @@ def start(self, stop_event, *, adapters=None, loop=None, interval=60):\n         while not stop_event.is_set():\n             ok = False\n             try:\n-                cron_tick(verbose=False, adapters=adapters, loop=loop, sync=False)\n+                if can_dispatch is not None and not can_dispatch():\n+                    logger.debug(\"Cron dispatch paused while gateway drains existing work\")\n+                else:\n+                    cron_tick(\n+                        verbose=False,\n+                        adapters=adapters,\n+                        loop=loop,\n+                        sync=False,\n+                        can_dispatch=can_dispatch,\n+                    )\n                 ok = True\n             except BaseException as e:\n                 # Catch BaseException (not just Exception) so a SystemExit from"]},{"file":"gateway/platforms/api_server.py","hunks":["@@ -35,6 +35,9 @@\n import hashlib\n import hmac\n import json\n+from contextlib import contextmanager\n+from contextvars import ContextVar\n+from functools import wraps\n import logging\n import os\n import socket as _socket","@@ -664,6 +667,66 @@ def _openai_error(message: str, err_type: str = \"invalid_request_error\", param:\n     }\n \n \n+_api_agent_request_reservation: ContextVar[Optional[dict[str, bool]]] = ContextVar(\n+    \"api_agent_request_reservation\", default=None\n+)\n+\n+\n+def _admit_api_agent_request(handler):\n+    \"\"\"Reserve an authenticated API turn before its handler first awaits.\n+\n+    Gateway shutdown and aiohttp requests share an event loop. Keeping the\n+    drain check and reservation in one non-awaiting block prevents a request\n+    admitted immediately before shutdown from becoming invisible while it is\n+    still parsing its body or resolving session state. The mutable reservation\n+    is intentionally shared with child tasks so agent/task bookkeeping releases\n+    this one slot exactly once.\n+    \"\"\"\n+    @wraps(handler)\n+    async def _wrapped(self, request, *args, **kwargs):\n+        auth_err = self._check_auth(request)\n+        if auth_err:\n+            return auth_err\n+        draining = self._draining_response()\n+        if draining is not None:\n+            return draining\n+        reservation = {\"active\": True}\n+        token = [redacted:secret-assignment]\n+        self._pending_agent_requests += 1\n+        try:\n+            return await handler(self, request, *args, **kwargs)\n+        finally:\n+            if reservation[\"active\"]:\n+                reservation[\"active\"] = False\n+                self._pending_agent_requests = max(0, self._pending_agent_requests - 1)\n+            _api_agent_request_reservation.reset(token)\n+\n+    return _wrapped\n+\n+\n+def _release_pending_api_work(adapter, reservation: dict[str, bool]) -> None:\n+    \"\"\"Release a pending-work reservation exactly once.\"\"\"\n+    if reservation[\"active\"]:\n+        reservation[\"active\"] = False\n+        adapter._pending_agent_requests = max(0, adapter._pending_agent_requests - 1)\n+\n+\n+@contextmanager\n+def _reserve_pending_api_work(adapter):\n+    \"\"\"Keep externally-triggered background work visible across awaits.\n+\n+    A handler can detach the reservation to an asyncio task; its done callback\n+    then owns release so shutdown cannot miss the handoff to background work.\n+    \"\"\"\n+    reservation = {\"active\": True, \"detached\": False}\n+    adapter._pending_agent_requests += 1\n+    try:\n+        yield reservation\n+    finally:\n+        if not reservation[\"detached\"]:\n+            _release_pending_api_work(adapter, reservation)\n+\n+\n if AIOHTTP_AVAILABLE:\n     @web.middleware\n     async def body_limit_middleware(request, handler):","@@ -912,6 +975,64 @@ def __init__(self, config: PlatformConfig):\n         # (the /v1/runs path tracks its own in-flight set via\n         # _active_run_tasks).\n         self._inflight_agent_runs: int = 0\n+        # Requests admitted before their handler reaches agent bookkeeping.\n+        # Shutdown counts this reservation so the request cannot slip through\n+        # the drain between its first await and _run_agent()/task registration.\n+        self._pending_agent_requests: int = 0\n+\n+    def active_agent_work_count(self) -> int:\n+        \"\"\"Return all live agent work owned by this API adapter.\n+\n+        ``/v1/runs`` registers an asyncio task before it constructs and stores\n+        its agent, so ``_active_run_agents`` has a real queued-before-agent gap.\n+        Reuse the task-based accounting used by the concurrent-run limit: it\n+        covers that gap and excludes completed tasks retained until cleanup.\n+        \"\"\"\n+        try:\n+            return (\n+                int(getattr(self, \"_pending_agent_requests\", 0))\n+                + int(self._inflight_agent_runs)\n+                + sum(not task.done() for task in self._active_run_tasks.values())\n+            )\n+        except Exception:\n+            return 0\n+\n+    @staticmethod\n+    def _gateway_is_draining() -> bool:\n+        \"\"\"Whether the owning gateway currently refuses new agent turns.\"\"\"\n+        try:\n+            from gateway.run import _gateway_runner_ref\n+\n+            runner = _gateway_runner_ref()\n+            return bool(\n+                runner\n+                and (\n+                    getattr(runner, \"_draining\", False)\n+                    or getattr(runner, \"_external_drain_active\", False)\n+                )\n+            )\n+        except Exception:\n+            return False\n+\n+    def _draining_response(self) -> Optional[\"web.Response\"]:\n+        \"\"\"Return a retryable response while the gateway drains existing work.\"\"\"\n+        if not self._gateway_is_draining():\n+            return None\n+        return web.json_response(\n+            _openai_error(\n+                \"Gateway is draining existing work; retry shortly.\",\n+                code=\"gateway_draining\",\n+            ),\n+            status=503,\n+            headers={\"Retry-After\": \"1\"},\n+        )\n+\n+    def _activate_admitted_request(self) -> None:\n+        \"\"\"Transfer this request's drain reservation to agent bookkeeping.\"\"\"\n+        reservation = _api_agent_request_reservation.get()\n+        if reservation and reservation[\"active\"]:\n+            reservation[\"active\"] = False\n+            self._pending_agent_requests = max(0, self._pending_agent_requests - 1)\n \n     def _readiness_work_counts(self) -> tuple[int, int, int]:\n         \"\"\"Return bounded work counts from each subsystem's public state.\"\"\"","@@ -1919,11 +2040,9 @@ async def _handle_fork_session(self, request: \"web.Request\") -> \"web.Response\":\n         fork = db.get_session(fork_id) or {\"id\": fork_id, \"parent_session_id\": source_id}\n         return web.json_response({\"object\": \"hermes.session\", \"session\": self._session_response(fork)}, status=201)\n \n+    @_admit_api_agent_request\n     async def _handle_session_chat(self, request: \"web.Request\") -> \"web.Response\":\n         \"\"\"POST /api/sessions/{session_id}/chat — one synchronous agent turn.\"\"\"\n-        auth_err = self._check_auth(request)\n-        if auth_err:\n-            return auth_err\n         gateway_session_key, key_err = self._parse_session_key_header(request)\n         if key_err is not None:\n             return key_err","@@ -1963,11 +2082,9 @@ async def _handle_session_chat(self, request: \"web.Request\") -> \"web.Response\":\n             headers=headers,\n         )\n \n+    @_admit_api_agent_request\n     async def _handle_session_chat_stream(self, request: \"web.Request\") -> \"web.StreamResponse\":\n         \"\"\"POST /api/sessions/{session_id}/chat/stream — SSE wrapper over _run_agent.\"\"\"\n-        auth_err = self._check_auth(request)\n-        if auth_err:\n-            return auth_err\n         gateway_session_key, key_err = self._parse_session_key_header(request)\n         if key_err is not None:\n             return key_err","@@ -2104,12 +2221,9 @@ async def _run_and_signal() -> None:\n             logger.debug(\"[api_server] session SSE stream error: %s\", exc)\n         return response\n \n+    @_admit_api_agent_request\n     async def _handle_chat_completions(self, request: \"web.Request\") -> \"web.Response\":\n         \"\"\"POST /v1/chat/completions — OpenAI Chat Completions format.\"\"\"\n-        auth_err = self._check_auth(request)\n-        if auth_err:\n-            return auth_err\n-\n         # Bound total in-flight agent runs (configurable; #7483).\n         limited = self._concurrency_limited_response()\n         if limited is not None:","@@ -2311,8 +2425,8 @@ def _on_tool_complete(tool_call_id, function_name, function_args, function_resul\n             # ``tool_progress_callback`` is intentionally not wired here:\n             # it would duplicate every emit because ``run_agent`` fires it\n             # side-by-side with ``tool_start_callback``/``tool_complete_callback``.\n-            # The structured callbacks are strictly richer (they carry the\n-            # tool_call id), so they own the chat-completions SSE channel.\n+            # The structured callbacks are strictly richer (they carry\n+            # the tool_call id), so they own the chat-completions SSE channel.\n             agent_ref = [None]\n             agent_task = asyncio.ensure_future(self._run_agent(\n                 user_message=user_message,","@@ -3240,12 +3354,9 @@ async def _flush_batch() -> None:\n \n         return response\n \n+    @_admit_api_agent_request\n     async def _handle_responses(self, request: \"web.Request\") -> \"web.Response\":\n         \"\"\"POST /v1/responses — OpenAI Responses API format.\"\"\"\n-        auth_err = self._check_auth(request)\n-        if auth_err:\n-            return auth_err\n-\n         # Bound total in-flight agent runs (configurable; #7483).\n         limited = self._concurrency_limited_response()\n         if limited is not None:","@@ -3788,6 +3899,9 @@ async def _handle_run_job(self, request: \"web.Request\") -> \"web.Response\":\n         auth_err = self._check_auth(request)\n         if auth_err:\n             return auth_err\n+        draining = self._draining_response()\n+        if draining is not None:\n+            return draining\n         cron_err = self._check_jobs_available()\n         if cron_err:\n             return cron_err","@@ -3833,31 +3947,39 @@ async def _handle_cron_fire(self, request: \"web.Request\") -> \"web.Response\":\n                 self._request_audit_log_suffix(request),\n             )\n             return web.json_response({\"error\": \"invalid fire token\"}, status=401)\n+        draining = self._draining_response()\n+        if draining is not None:\n+            return draining\n \n-        try:\n-            body = await request.json()\n-        except Exception:\n-            body = {}\n-        job_id = (body or {}).get(\"job_id\")\n-        if not job_id:\n-            return web.json_response({\"error\": \"missing job_id\"}, status=400)\n+        with _reserve_pending_api_work(self) as reservation:\n+            try:\n+                body = await request.json()\n+            except Exception:\n+                body = {}\n+            job_id = (body or {}).get(\"job_id\")\n+            if not job_id:\n+                return web.json_response({\"error\": \"missing job_id\"}, status=400)\n \n-        from cron.scheduler_provider import resolve_cron_scheduler\n-        provider = resolve_cron_scheduler()\n+            from cron.scheduler_provider import resolve_cron_scheduler\n+            provider = resolve_cron_scheduler()\n \n-        loop = asyncio.get_running_loop()\n-        # Fire in the background (202 immediately). fire_due claims via the\n-        # store CAS, so a retry while this is in flight is de-duped.\n-        task = asyncio.create_task(\n-            asyncio.to_thread(provider.fire_due, job_id, adapters=None, loop=loop)\n-        )\n-        try:\n-            self._background_tasks.add(task)\n-            task.add_done_callback(self._background_tasks.discard)\n-        except (TypeError, AttributeError):\n-            pass\n+            loop = asyncio.get_running_loop()\n+            # Fire in the background (202 immediately). fire_due claims via the\n+            # store CAS, so a retry while this is in flight is de-duped.\n+            task = asyncio.create_task(\n+                asyncio.to_thread(provider.fire_due, job_id, adapters=None, loop=loop)\n+            )\n+            reservation[\"detached\"] = True\n+            task.add_done_callback(\n+                lambda _task: _release_pending_api_work(self, reservation)\n+            )\n+            try:\n+                self._background_tasks.add(task)\n+                task.add_done_callback(self._background_tasks.discard)\n+            except (TypeError, AttributeError):\n+                pass\n \n-        return web.json_response({\"status\": \"accepted\", \"job_id\": job_id}, status=202)\n+            return web.json_response({\"status\": \"accepted\", \"job_id\": job_id}, status=202)\n \n \n     # ------------------------------------------------------------------","@@ -4011,19 +4133,22 @@ def _concurrency_limited_response(self) -> Optional[\"web.Response\"]:\n         \"\"\"Return a 429 response if the concurrent-run cap is reached, else None.\n \n         The cap bounds total in-flight agent activity across every\n-        agent-serving endpoint: the non-streaming chat/responses paths\n-        (tracked by ``_inflight_agent_runs``) plus the ``/v1/runs`` path\n-        (tracked by live entries in ``_active_run_tasks``).  Stream queues are\n-        transport state and may disappear while their underlying run remains\n-        active, so they must not define run concurrency. A configured value of\n-        0 disables the cap entirely.\n+        agent-serving endpoint. Reuse the same adapter-owned work count that\n+        shutdown draining uses, including an admitted request before it reaches\n+        agent/task bookkeeping. Stream queues are transport state and may\n+        disappear while their underlying run remains active, so they must not\n+        define run concurrency. A configured value of 0 disables the cap.\n         \"\"\"\n         limit = self._max_concurrent_runs\n         if limit <= 0:\n             return None\n-        inflight = self._inflight_agent_runs + sum(\n-            not task.done() for task in self._active_run_tasks.values()\n-        )\n+        inflight = self.active_agent_work_count()\n+        # The current request owns one reservation until it hands off to\n+        # _run_agent() or /v1/runs task registration. It must not consume its\n+        # own last available slot; other admitted requests remain counted.\n+        reservation = _api_agent_request_reservation.get()\n+        if reservation and reservation[\"active\"]:\n+            inflight -= 1\n         if inflight >= limit:\n             return web.json_response(\n                 _openai_error(","@@ -4141,6 +4266,7 @@ def _run():\n             finally:\n                 clear_session_vars(tokens)\n \n+        self._activate_admitted_request()\n         self._inflight_agent_runs += 1\n         try:\n             return await loop.run_in_executor(None, _run)","@@ -4215,12 +4341,9 @@ def _callback(event_type: str, tool_name: str = None, preview: str = None, args=\n \n         return _callback\n \n+    @_admit_api_agent_request\n     async def _handle_runs(self, request: \"web.Request\") -> \"web.Response\":\n         \"\"\"POST /v1/runs — start an agent run, return run_id immediately.\"\"\"\n-        auth_err = self._check_auth(request)\n-        if auth_err:\n-            return auth_err\n-\n         # Long-term memory scope header (see chat_completions for details).\n         gateway_session_key, key_err = self._parse_session_key_header(request)\n         if key_err is not None:","@@ -4542,6 +4665,7 @@ def _run_sync():\n                 self._run_approval_sessions.pop(run_id, None)\n                 self._stopping_run_ids.discard(run_id)\n \n+        self._activate_admitted_request()\n         task = asyncio.create_task(_run_and_close())\n         self._active_run_tasks[run_id] = task\n         try:"]},{"file":"gateway/run.py","hunks":["@@ -4122,6 +4122,14 @@ def _request_clean_exit(self, reason: str) -> None:\n     def _running_agent_count(self) -> int:\n         return len(self._running_agents)\n \n+    def _active_work_count(self) -> int:\n+        \"\"\"All agent work the gateway must expose and drain as one total.\"\"\"\n+        return (\n+            self._running_agent_count()\n+            + self._active_cron_job_count()\n+            + self._active_api_run_count()\n+        )\n+\n     def _active_cron_job_count(self) -> int:\n         \"\"\"Count of cron jobs currently executing, from the cron scheduler's\n         own in-flight tracking (``cron.scheduler._running_job_ids``).","@@ -4142,6 +4150,20 @@ def _active_cron_job_count(self) -> int:\n         except Exception:\n             return 0\n \n+    def _active_api_run_count(self) -> int:\n+        \"\"\"Count API-server work that is outside ``_running_agents``.\n+\n+        The primary API server owns the sole HTTP listener. Secondary multiplex\n+        profiles cannot create an ``api_server`` adapter because it binds a port,\n+        so only the primary registry is a supported source of this work.\n+        \"\"\"\n+        try:\n+            adapter = getattr(self, \"adapters\", {}).get(Platform.API_SERVER)\n+            helper = getattr(adapter, \"active_agent_work_count\", None)\n+            return max(0, int(helper())) if callable(helper) else 0\n+        except Exception:\n+            return 0\n+\n     # ── scale-to-zero idle detection / dormant-quiesce (Phase 0) ──────────────\n     # The gateway-side BEHAVIOUR that consumes the relay scale-to-zero primitives\n     # (gateway-gateway Phase 5). Pure logic lives in gateway/scale_to_zero.py; the","@@ -4529,7 +4551,7 @@ def _update_runtime_status(self, gateway_state: Optional[str] = None, exit_reaso\n                 gateway_state=gateway_state,\n                 exit_reason=exit_reason,\n                 restart_requested=self._restart_requested,\n-                active_agents=self._running_agent_count(),\n+                active_agents=self._active_work_count(),\n             )\n         except Exception:\n             pass","@@ -4552,7 +4574,7 @@ def _persist_active_agents(self) -> None:\n         \"\"\"\n         try:\n             from gateway.status import write_runtime_status\n-            write_runtime_status(active_agents=self._running_agent_count())\n+            write_runtime_status(active_agents=self._active_work_count())\n         except Exception:\n             pass\n ","@@ -4579,7 +4601,7 @@ def _enter_external_drain(self) -> None:\n         logger.info(\n             \"External drain ENGAGED (.drain_request.json present) — refusing \"\n             \"new turns; %d in-flight turn(s) will finish. Process stays up.\",\n-            self._running_agent_count(),\n+            self._active_work_count(),\n         )\n         # Flip the persisted lifecycle state so /api/status.gateway_busy /\n         # gateway_drainable track the drain. Preserve active_agents (the","@@ -4630,6 +4652,10 @@ async def _drain_control_watcher(self, interval: float = 1.0) -> None:\n             try:\n                 if drain_requested():\n                     self._enter_external_drain()\n+                    # API and cron work live outside messaging's\n+                    # _running_agents map. Refresh the aggregate while an\n+                    # external caller polls this reversible drain state.\n+                    self._persist_active_agents()\n                 else:\n                     self._exit_external_drain()\n             except asyncio.CancelledError:","@@ -5696,22 +5722,26 @@ async def _drain_active_agents(self, timeout: float) -> tuple[Dict[str, Any], bo\n         snapshot = self._snapshot_running_agents()\n         last_active_count = self._running_agent_count()\n         last_cron_count = self._active_cron_job_count()\n+        last_api_count = self._active_api_run_count()\n         last_status_at = 0.0\n \n         def _maybe_update_status(force: bool = False) -> None:\n-            nonlocal last_active_count, last_cron_count, last_status_at\n+            nonlocal last_active_count, last_cron_count, last_api_count, last_status_at\n             now = asyncio.get_running_loop().time()\n             active_count = self._running_agent_count()\n             cron_count = self._active_cron_job_count()\n+            api_count = self._active_api_run_count()\n             if (\n                 force\n                 or active_count != last_active_count\n                 or cron_count != last_cron_count\n+                or api_count != last_api_count\n                 or (now - last_status_at) >= 1.0\n             ):\n                 self._update_runtime_status(\"draining\")\n                 last_active_count = active_count\n                 last_cron_count = cron_count\n+                last_api_count = api_count\n                 last_status_at = now\n \n         # Cron jobs run on the scheduler's own thread pool, outside","@@ -5719,7 +5749,8 @@ def _maybe_update_status(force: bool = False) -> None:\n         # same wait/timeout this method already applies to chat sessions,\n         # or a cron job's tool work gets killed with zero warning the\n         # instant it's the only active thing running (#60432).\n-        if not self._running_agents and last_cron_count == 0:\n+        # API-server / desk sessions have the same structural gap (#63529).\n+        if not self._running_agents and last_cron_count == 0 and last_api_count == 0:\n             _maybe_update_status(force=True)\n             return snapshot, False\n ","@@ -5729,12 +5760,20 @@ def _maybe_update_status(force: bool = False) -> None:\n \n         deadline = asyncio.get_running_loop().time() + timeout\n         while (\n-            (self._running_agents or self._active_cron_job_count())\n+            (\n+                self._running_agents\n+                or self._active_cron_job_count()\n+                or self._active_api_run_count()\n+            )\n             and asyncio.get_running_loop().time() < deadline\n         ):\n             _maybe_update_status()\n             await asyncio.sleep(0.1)\n-        timed_out = bool(self._running_agents) or bool(self._active_cron_job_count())\n+        timed_out = (\n+            bool(self._running_agents)\n+            or bool(self._active_cron_job_count())\n+            or bool(self._active_api_run_count())\n+        )\n         _maybe_update_status(force=True)\n         return snapshot, timed_out\n ","@@ -8187,12 +8226,14 @@ def _phase_elapsed() -> float:\n                     logger.debug(\"pre-drain mark_resume_pending failed for %s: %s\", _sk, _e)\n \n             _cron_at_start = self._active_cron_job_count()\n+            _api_at_start = self._active_api_run_count()\n             _drain_started_at = time.monotonic()\n             active_agents, timed_out = await self._drain_active_agents(timeout)\n             logger.info(\n                 \"Shutdown phase: drain done at +%.2fs (drain took %.2fs, \"\n                 \"timed_out=%s, active_at_start=%d, active_now=%d, \"\n-                \"cron_at_start=%d, cron_now=%d)\",\n+                \"cron_at_start=%d, cron_now=%d, \"\n+                \"api_at_start=%d, api_now=%d)\",\n                 _phase_elapsed(),\n                 time.monotonic() - _drain_started_at,\n                 timed_out,","@@ -8200,6 +8241,8 @@ def _phase_elapsed() -> float:\n                 self._running_agent_count(),\n                 _cron_at_start,\n                 self._active_cron_job_count(),\n+                _api_at_start,\n+                self._active_api_run_count(),\n             )\n \n             if not timed_out:","@@ -8218,11 +8261,13 @@ def _phase_elapsed() -> float:\n \n             if timed_out:\n                 logger.warning(\n-                    \"Gateway drain timed out after %.1fs with %d active agent(s) \"\n-                    \"and %d in-flight cron job(s); interrupting remaining work.\",\n+                    \"Gateway drain timed out after %.1fs with %d active agent(s), \"\n+                    \"%d in-flight cron job(s), and %d api_server run(s); \"\n+                    \"interrupting remaining work.\",\n                     timeout,\n                     self._running_agent_count(),\n                     self._active_cron_job_count(),\n+                    self._active_api_run_count(),\n                 )\n                 # Mark forcibly-interrupted sessions as resume_pending BEFORE\n                 # interrupting the agents.  This preserves each session's","@@ -20971,13 +21016,21 @@ def restart_signal_handler():\n     # historical in-process 60s ticker; an external provider (e.g. chronos)\n     # may arm a schedule and return. Pass the event loop so cron delivery can\n     # use live adapters (E2EE support).\n-    from cron.scheduler_provider import resolve_cron_scheduler\n+    from cron.scheduler_provider import InProcessCronScheduler, resolve_cron_scheduler\n     cron_stop = threading.Event()\n     cron_provider = resolve_cron_scheduler()\n+    cron_start_kwargs = {\"adapters\": runner.adapters, \"loop\": asyncio.get_running_loop()}\n+    # External cron providers own their remote scheduling contract. Only the\n+    # in-process ticker polls local due jobs, so only it receives the local\n+    # external-drain dispatch gate.\n+    if isinstance(cron_provider, InProcessCronScheduler):\n+        cron_start_kwargs[\"can_dispatch\"] = lambda: not (\n+            runner._draining or runner._external_drain_active\n+        )\n     cron_thread = threading.Thread(\n         target=cron_provider.start,\n         args=(cron_stop,),\n-        kwargs={\"adapters\": runner.adapters, \"loop\": asyncio.get_running_loop()},\n+        kwargs=cron_start_kwargs,\n         daemon=True,\n         name=\"cron-scheduler\",\n     )"]},{"file":"tests/cron/test_scheduler.py","hunks":["@@ -1533,6 +1533,25 @@ def test_run_job_delivers_max_iteration_fallback_summary(self, tmp_path):\n         assert \"final fallback report\" in output\n         assert \"(FAILED)\" not in output\n \n+    def test_tick_skips_due_jobs_while_dispatch_is_paused(self, tmp_path):\n+        \"\"\"The drain gate runs before advancing a due job's schedule.\"\"\"\n+        from cron.scheduler import tick\n+\n+        job = {\n+            \"id\": \"paused-due-job\",\n+            \"name\": \"paused due job\",\n+            \"schedule\": {\"kind\": \"interval\", \"seconds\": 60},\n+            \"next_run_at\": \"2020-01-01T00:00:00+00:00\",\n+            \"enabled\": True,\n+        }\n+        with patch(\"cron.scheduler.get_due_jobs\", return_value=[job]), patch(\n+            \"cron.scheduler.advance_next_run\"\n+        ) as advance, patch(\"cron.scheduler.run_one_job\") as run_one:\n+            assert tick(verbose=False, sync=True, can_dispatch=lambda: False) == 0\n+\n+        advance.assert_not_called()\n+        run_one.assert_not_called()\n+\n     def test_tick_marks_empty_response_as_error(self, tmp_path):\n         \"\"\"When run_job returns success=True but final_response is empty,\n         tick() should mark the job as error so last_status != 'ok'."]},{"file":"tests/cron/test_scheduler_provider.py","hunks":["@@ -129,13 +129,11 @@ def start(self, stop_event, **kw):\n \n \n def test_abc_growth_stays_additive():\n-    \"\"\"Forward-compat guard: the ABC's REQUIRED surface is exactly name+start.\n+    \"\"\"The provider interface stays source-compatible with existing plugins.\n \n-    Any optional hook added later for the external provider\n-    (on_jobs_changed/fire_due/reconcile) must be NON-abstract (carry a default),\n-    so the built-in keeps satisfying the ABC without overriding them. This test\n-    fails loudly if someone makes a future hook abstract (a breaking change that\n-    would force every provider — including the built-in — to implement it).\n+    ``start`` must be the only required implementation hook: future optional\n+    behavior belongs in non-abstract default methods so custom plugins do not\n+    break on import after an upgrade.\n     \"\"\"\n     from cron.scheduler_provider import CronScheduler\n ","@@ -174,6 +172,33 @@ def test_inprocess_provider_ticks_and_stops():\n     assert calls[0].get(\"sync\") is False\n \n \n+def test_inprocess_provider_skips_dispatch_while_draining():\n+    \"\"\"A drain pause keeps due work pending until dispatch is re-enabled.\"\"\"\n+    from cron.scheduler_provider import InProcessCronScheduler\n+\n+    calls = []\n+    stop = threading.Event()\n+    allow_dispatch = threading.Event()\n+    provider = InProcessCronScheduler()\n+\n+    with patch(\"cron.scheduler.tick\", side_effect=lambda *a, **k: calls.append(k) or 0):\n+        thread = threading.Thread(\n+            target=provider.start,\n+            args=(stop,),\n+            kwargs={\"interval\": 0.01, \"can_dispatch\": allow_dispatch.is_set},\n+            daemon=True,\n+        )\n+        thread.start()\n+        time.sleep(0.05)\n+        assert calls == []\n+        allow_dispatch.set()\n+        assert _wait_until(lambda: len(calls) >= 1), \"provider never resumed dispatch\"\n+        stop.set()\n+        thread.join(timeout=5)\n+\n+    assert not thread.is_alive()\n+\n+\n def test_inprocess_provider_stop_is_noop():\n     \"\"\"The default stop() hook is a safe no-op (the stop_event is the real\n     stop signal for the built-in).\"\"\""]},{"file":"tests/gateway/restart_test_helpers.py","hunks":["@@ -115,6 +115,18 @@ def make_restart_runner(\n     runner._running_agent_count = GatewayRunner._running_agent_count.__get__(\n         runner, GatewayRunner\n     )\n+    runner._active_cron_job_count = GatewayRunner._active_cron_job_count.__get__(\n+        runner, GatewayRunner\n+    )\n+    runner._active_api_run_count = GatewayRunner._active_api_run_count.__get__(\n+        runner, GatewayRunner\n+    )\n+    runner._active_work_count = GatewayRunner._active_work_count.__get__(\n+        runner, GatewayRunner\n+    )\n+    runner._persist_active_agents = GatewayRunner._persist_active_agents.__get__(\n+        runner, GatewayRunner\n+    )\n     runner._snapshot_running_agents = GatewayRunner._snapshot_running_agents.__get__(\n         runner, GatewayRunner\n     )"]},{"file":"tests/gateway/test_api_server_active_work_drain.py","hunks":["@@ -0,0 +1,326 @@\n+\"\"\"Regression coverage for #63529 API-server shutdown draining.\n+\n+API-server work is adapter-owned rather than tracked by\n+``GatewayRunner._running_agents``. The shutdown drain must account for the\n+same live state as the API concurrency limiter, including a ``/v1/runs`` task\n+that exists before its agent has been constructed, and it must refuse new API\n+turns once the gateway starts draining.\n+\"\"\"\n+\n+import asyncio\n+from types import SimpleNamespace\n+from unittest.mock import AsyncMock, MagicMock, patch\n+\n+import pytest\n+from aiohttp import web\n+from aiohttp.test_utils import TestClient, TestServer\n+\n+from gateway.config import Platform, PlatformConfig\n+from gateway.platforms.api_server import APIServerAdapter\n+from tests.gateway.restart_test_helpers import make_restart_runner\n+\n+\n+class _RunTask:\n+    def __init__(self, done: bool = False):\n+        self._done = done\n+\n+    def done(self) -> bool:\n+        return self._done\n+\n+\n+def _make_api_adapter(*, inflight: int = 0, queued_ids=()):\n+    tasks = {run_id: _RunTask() for run_id in queued_ids}\n+    adapter = SimpleNamespace(\n+        platform=Platform.API_SERVER,\n+        _inflight_agent_runs=inflight,\n+        _active_run_tasks=tasks,\n+    )\n+\n+    def active_agent_work_count() -> int:\n+        return int(getattr(adapter, \"_pending_agent_requests\", 0)) + int(\n+            adapter._inflight_agent_runs\n+        ) + sum(not task.done() for task in adapter._active_run_tasks.values())\n+\n+    adapter.active_agent_work_count = active_agent_work_count\n+    return adapter\n+\n+\n+def _make_admission_app(adapter: APIServerAdapter) -> web.Application:\n+    app = web.Application()\n+    app.router.add_post(\"/api/sessions/{session_id}/chat\", adapter._handle_session_chat)\n+    app.router.add_post(\n+        \"/api/sessions/{session_id}/chat/stream\", adapter._handle_session_chat_stream\n+    )\n+    app.router.add_post(\"/v1/chat/completions\", adapter._handle_chat_completions)\n+    app.router.add_post(\"/v1/responses\", adapter._handle_responses)\n+    app.router.add_post(\"/v1/runs\", adapter._handle_runs)\n+    return app\n+\n+\n+class TestActiveApiRunCount:\n+    def test_zero_when_no_api_adapter(self):\n+        runner, _adapter = make_restart_runner()\n+        runner.adapters = {}\n+        assert runner._active_api_run_count() == 0\n+\n+    def test_delegates_to_primary_api_adapter(self):\n+        runner, _adapter = make_restart_runner()\n+        runner.adapters = {\n+            Platform.API_SERVER: _make_api_adapter(inflight=2, queued_ids=[\"r1\"])\n+        }\n+        assert runner._active_api_run_count() == 3\n+\n+    def test_ignores_non_api_platforms(self):\n+        runner, _adapter = make_restart_runner()\n+        other = SimpleNamespace(\n+            platform=Platform.DISCORD,\n+            active_agent_work_count=lambda: 99,\n+        )\n+        runner.adapters = {Platform.DISCORD: other}\n+        assert runner._active_api_run_count() == 0\n+\n+    def test_never_raises_on_broken_adapter(self):\n+        runner, _adapter = make_restart_runner()\n+\n+        class Bad:\n+            platform = Platform.API_SERVER\n+\n+            @staticmethod\n+            def active_agent_work_count() -> int:\n+                raise RuntimeError(\"boom\")\n+\n+        runner.adapters = {Platform.API_SERVER: Bad()}\n+        assert runner._active_api_run_count() == 0\n+\n+\n+class TestAPIServerAdapterWorkCount:\n+    def test_concurrency_limit_counts_other_pending_admissions(self):\n+        adapter = APIServerAdapter(PlatformConfig(enabled=True))\n+        adapter._max_concurrent_runs = 1\n+        adapter._pending_agent_requests = 1\n+\n+        response = adapter._concurrency_limited_response()\n+\n+        assert response is not None\n+        assert response.status == 429\n+\n+    @pytest.mark.asyncio\n+    async def test_concurrency_limit_excludes_current_pending_admission(self):\n+        adapter = APIServerAdapter(PlatformConfig(enabled=True))\n+        adapter._max_concurrent_runs = 1\n+        app = _make_admission_app(adapter)\n+\n+        async with TestClient(TestServer(app)) as client:\n+            with patch.object(adapter, \"_run_agent\", new=AsyncMock(return_value=({}, {}))):\n+                response = await client.post(\n+                    \"/api/sessions/s/chat\",\n+                    json={\"message\": \"hello\"},\n+                )\n+\n+        assert response.status == 404\n+\n+    def test_counts_pending_admission_before_agent_bookkeeping(self):\n+        adapter = APIServerAdapter(PlatformConfig(enabled=True))\n+        adapter._pending_agent_requests = 1\n+\n+        assert adapter.active_agent_work_count() == 1\n+\n+    def test_counts_live_run_task_before_agent_creation(self):\n+        adapter = APIServerAdapter(PlatformConfig(enabled=True))\n+        adapter._inflight_agent_runs = 2\n+        adapter._active_run_tasks = {\n+            \"queued\": _RunTask(),\n+            \"finished\": _RunTask(done=True),\n+        }\n+        adapter._active_run_agents = {}\n+\n+        assert adapter.active_agent_work_count() == 3\n+\n+    def test_does_not_double_count_started_run_agent(self):\n+        adapter = APIServerAdapter(PlatformConfig(enabled=True))\n+        adapter._inflight_agent_runs = 0\n+        adapter._active_run_tasks = {\"run-1\": _RunTask()}\n+        adapter._active_run_agents = {\"run-1\": object()}\n+\n+        assert adapter.active_agent_work_count() == 1\n+\n+\n+class TestDrainWaitsForApiWork:\n+    @pytest.mark.asyncio\n+    async def test_drain_returns_immediately_when_nothing_active(self):\n+        runner, _adapter = make_restart_runner()\n+        runner.adapters = {}\n+\n+        _snapshot, timed_out = await runner._drain_active_agents(5.0)\n+\n+        assert timed_out is False\n+\n+    @pytest.mark.asyncio\n+    async def test_drain_waits_for_real_queued_run_before_agent_creation(self):\n+        \"\"\"A live /v1/runs task must block drain before it has an agent.\"\"\"\n+        runner, _adapter = make_restart_runner()\n+        api = APIServerAdapter(PlatformConfig(enabled=True))\n+        runner.adapters = {Platform.API_SERVER: api}\n+        app = _make_admission_app(api)\n+        original_create_task = asyncio.create_task\n+        task_started = asyncio.Event()\n+        allow_task = asyncio.Event()\n+\n+        def delayed_create_task(coro):\n+            async def delayed():\n+                task_started.set()\n+                await allow_task.wait()\n+                return await coro\n+\n+            return original_create_task(delayed())\n+\n+        mock_agent = MagicMock()\n+        mock_agent.run_conversation.return_value = {\"final_response\": \"done\"}\n+        mock_agent.session_prompt_tokens = 0\n+        mock_agent.session_completion_tokens = 0\n+        mock_agent.session_total_tokens = 0\n+\n+        with patch(\n+            \"gateway.platforms.api_server.asyncio.create_task\",\n+            side_effect=delayed_create_task,\n+        ), patch.object(api, \"_create_agent\", return_value=mock_agent):\n+            async with TestClient(TestServer(app)) as client:\n+                response = await client.post(\"/v1/runs\", json={\"input\": \"hello\"})\n+                assert response.status == 202\n+                await task_started.wait()\n+\n+                assert api._active_run_agents == {}\n+                assert runner._active_api_run_count() == 1\n+                drain_task = original_create_task(runner._drain_active_agents(2.0))\n+                await asyncio.sleep(0.1)\n+                assert not drain_task.done()\n+\n+                allow_task.set()\n+                _snapshot, timed_out = await drain_task\n+\n+        assert timed_out is False\n+\n+    @pytest.mark.asyncio\n+    async def test_drain_times_out_if_api_run_outlives_the_window(self):\n+        runner, _adapter = make_restart_runner()\n+        runner.adapters = {Platform.API_SERVER: _make_api_adapter(queued_ids=[\"run-1\"])}\n+\n+        _snapshot, timed_out = await runner._drain_active_agents(0.1)\n+\n+        assert timed_out is True\n+\n+    @pytest.mark.asyncio\n+    async def test_drain_still_waits_for_chat_cron_and_api_work(self):\n+        import cron.scheduler as sched\n+\n+        runner, _adapter = make_restart_runner()\n+        runner._running_agents = {\"session-1\": MagicMock()}\n+        sched._running_job_ids.add(\"job-1\")\n+        runner.adapters = {Platform.API_SERVER: _make_api_adapter(queued_ids=[\"run-1\"])}\n+\n+        async def finish_all():\n+            await asyncio.sleep(0.12)\n+            runner._running_agents.clear()\n+            sched._running_job_ids.discard(\"job-1\")\n+            runner.adapters[Platform.API_SERVER]._active_run_tasks.clear()\n+\n+        task = asyncio.create_task(finish_all())\n+        try:\n+            _snapshot, timed_out = await runner._drain_active_agents(2.0)\n+        finally:\n+            await task\n+            sched._running_job_ids.discard(\"job-1\")\n+\n+        assert timed_out is False\n+\n+\n+class TestDrainAdmission:\n+    @pytest.mark.asyncio\n+    async def test_drain_refuses_every_agent_start_endpoint(self):\n+        adapter = APIServerAdapter(PlatformConfig(enabled=True))\n+        runner = SimpleNamespace(_draining=True, _external_drain_active=False)\n+        app = _make_admission_app(adapter)\n+        paths = (\n+            \"/api/sessions/missing/chat\",\n+            \"/api/sessions/missing/chat/stream\",\n+            \"/v1/chat/completions\",\n+            \"/v1/responses\",\n+            \"/v1/runs\",\n+        )\n+\n+        with patch(\"gateway.run._gateway_runner_ref\", lambda: runner):\n+            async with TestClient(TestServer(app)) as client:\n+                for path in paths:\n+                    response = await client.post(path, json={})\n+                    payload = await response.json()\n+\n+                    assert response.status == 503\n+                    assert response.headers[\"Retry-After\"] == \"1\"\n+                    assert payload[\"error\"][\"code\"] == \"gateway_draining\"\n+\n+    @pytest.mark.asyncio\n+    async def test_external_drain_refuses_every_agent_start_endpoint(self):\n+        adapter = APIServerAdapter(PlatformConfig(enabled=True))\n+        runner = SimpleNamespace(_draining=False, _external_drain_active=True)\n+        app = _make_admission_app(adapter)\n+        paths = (\n+            \"/api/sessions/missing/chat\",\n+            \"/api/sessions/missing/chat/stream\",\n+            \"/v1/chat/completions\",\n+            \"/v1/responses\",\n+            \"/v1/runs\",\n+        )\n+\n+        with patch(\"gateway.run._gateway_runner_ref\", lambda: runner):\n+            async with TestClient(TestServer(app)) as client:\n+                for path in paths:\n+                    response = await client.post(path, json={})\n+                    payload = await response.json()\n+\n+                    assert response.status == 503\n+                    assert payload[\"error\"][\"code\"] == \"gateway_draining\"\n+\n+    @pytest.mark.asyncio\n+    async def test_admitted_request_blocks_drain_before_agent_bookkeeping(self):\n+        adapter = APIServerAdapter(PlatformConfig(enabled=True))\n+        runner, _adapter = make_restart_runner()\n+        runner.adapters = {Platform.API_SERVER: adapter}\n+        app = _make_admission_app(adapter)\n+        body_read_started = asyncio.Event()\n+        allow_body_read = asyncio.Event()\n+\n+        async def delayed_read_json(_request):\n+            body_read_started.set()\n+            await allow_body_read.wait()\n+            return {\"message\": \"hello\"}, None\n+\n+        with patch.object(\n+            adapter,\n+            \"_get_existing_session_or_404\",\n+            return_value=({}, None),\n+        ), patch.object(\n+            adapter,\n+            \"_read_json_body\",\n+            side_effect=delayed_read_json,\n+        ), patch.object(\n+            adapter,\n+            \"_run_agent\",\n+            new=AsyncMock(return_value=({\"final_response\": \"done\"}, {})),\n+        ):\n+            async with TestClient(TestServer(app)) as client:\n+                request_task = asyncio.create_task(\n+                    client.post(\"/api/sessions/missing/chat\", json={})\n+                )\n+                await body_read_started.wait()\n+\n+                assert adapter._pending_agent_requests == 1\n+                drain_task = asyncio.create_task(runner._drain_active_agents(2.0))\n+                await asyncio.sleep(0.1)\n+                assert not drain_task.done()\n+\n+                allow_body_read.set()\n+                response = await request_task\n+                assert response.status == 200\n+                _snapshot, timed_out = await drain_task\n+\n+        assert timed_out is False"]},{"file":"tests/gateway/test_api_server_jobs.py","hunks":["@@ -11,6 +11,7 @@\n \"\"\"\n \n import logging\n+from types import SimpleNamespace\n from unittest.mock import MagicMock, patch\n \n import pytest","@@ -494,6 +495,19 @@ async def test_run_job(self, adapter):\n                 assert data[\"job\"] == triggered_job\n                 mock_trigger.assert_called_once_with(VALID_JOB_ID)\n \n+    @pytest.mark.asyncio\n+    async def test_run_job_refuses_during_gateway_drain(self, adapter):\n+        app = _create_app(adapter)\n+        runner = SimpleNamespace(_draining=False, _external_drain_active=True)\n+\n+        with patch(\"gateway.run._gateway_runner_ref\", lambda: runner):\n+            async with TestClient(TestServer(app)) as cli:\n+                resp = await cli.post(f\"/api/jobs/{VALID_JOB_ID}/run\")\n+                payload = await resp.json()\n+\n+        assert resp.status == 503\n+        assert payload[\"error\"][\"code\"] == \"gateway_draining\"\n+\n \n # ---------------------------------------------------------------------------\n # 17. test_auth_required"]},{"file":"tests/gateway/test_cron_fire_webhook.py","hunks":["@@ -8,6 +8,9 @@\n \"\"\"\n \n import asyncio\n+import threading\n+from types import SimpleNamespace\n+from unittest.mock import patch\n \n import pytest\n from aiohttp import web","@@ -108,6 +111,86 @@ async def test_missing_token_401(adapter, monkeypatch):\n     assert spy.fired == []\n \n \n[redacted:email]\n+async def test_valid_token_refuses_during_gateway_drain(adapter, monkeypatch):\n+    spy = _SpyProvider()\n+    runner = SimpleNamespace(_draining=False, _external_drain_active=True)\n+    monkeypatch.setattr(\"cron.scheduler_provider.resolve_cron_scheduler\", lambda: spy)\n+    monkeypatch.setattr(\n+        \"plugins.cron_providers.chronos.verify.get_fire_verifier\",\n+        lambda: (lambda **kw: {\"purpose\": \"cron_fire\"}),\n+    )\n+\n+    app = _create_app(adapter)\n+    with patch(\"gateway.run._gateway_runner_ref\", lambda: runner):\n+        async with TestClient(TestServer(app)) as cli:\n+            response = await cli.post(\n+                \"/api/cron/fire\",\n+                headers={\"Authorization\": \"Bearer good\"},\n+                json={\"job_id\": \"abc123\"},\n+            )\n+            payload = await response.json()\n+\n+    assert response.status == 503\n+    assert payload[\"error\"][\"code\"] == \"gateway_draining\"\n+    assert spy.fired == []\n+\n+\n[redacted:email]\n+async def test_valid_fire_reservation_blocks_drain_before_body_and_task(adapter, monkeypatch):\n+    runner = SimpleNamespace(_draining=False, _external_drain_active=False)\n+    body_started = asyncio.Event()\n+    release_body = asyncio.Event()\n+    fired = threading.Event()\n+    release_fire = threading.Event()\n+\n+    class BlockingProvider:\n+        def fire_due(self, job_id, *, adapters=None, loop=None):\n+            fired.set()\n+            release_fire.wait(timeout=2)\n+            return True\n+\n+    original_json = web.Request.json\n+\n+    async def delayed_json(request):\n+        body_started.set()\n+        await release_body.wait()\n+        return await original_json(request)\n+\n+    monkeypatch.setattr(\"cron.scheduler_provider.resolve_cron_scheduler\", BlockingProvider)\n+    monkeypatch.setattr(\n+        \"plugins.cron_providers.chronos.verify.get_fire_verifier\",\n+        lambda: (lambda **kw: {\"purpose\": \"cron_fire\"}),\n+    )\n+    app = _create_app(adapter)\n+    with patch(\"gateway.run._gateway_runner_ref\", lambda: runner), patch.object(\n+        web.Request, \"json\", delayed_json\n+    ):\n+        async with TestClient(TestServer(app)) as cli:\n+            request_task = asyncio.create_task(\n+                cli.post(\n+                    \"/api/cron/fire\",\n+                    headers={\"Authorization\": \"Bearer good\"},\n+                    json={\"job_id\": \"abc123\"},\n+                )\n+            )\n+            await body_started.wait()\n+            assert adapter.active_agent_work_count() == 1\n+\n+            release_body.set()\n+            response = await request_task\n+            assert response.status == 202\n+            await asyncio.to_thread(fired.wait, 2)\n+            assert adapter.active_agent_work_count() == 1\n+            release_fire.set()\n+            for _ in range(50):\n+                if adapter.active_agent_work_count() == 0:\n+                    break\n+                await asyncio.sleep(0.01)\n+\n+    assert adapter.active_agent_work_count() == 0\n+\n+\n @pytest.mark.asyncio\n async def test_missing_job_id_400(adapter, monkeypatch):\n     \"\"\"Valid token but no job_id → 400, no fire.\"\"\""]},{"file":"tests/gateway/test_external_drain_control.py","hunks":["@@ -19,6 +19,7 @@\n \n import gateway.drain_control as dc\n from gateway.run import GatewayRunner\n+from gateway.config import Platform\n from gateway.platforms.base import MessageEvent, MessageType\n from tests.gateway.restart_test_helpers import make_restart_runner, make_restart_source\n ","@@ -238,6 +239,16 @@ def _drain_runner():\n \n \n class TestDrainStateMachine:\n+    def test_active_work_count_includes_api_and_cron_work(self, monkeypatch):\n+        runner, _ = _drain_runner()\n+        runner.adapters = {\n+            Platform.API_SERVER: MagicMock(active_agent_work_count=MagicMock(return_value=2))\n+        }\n+        runner._running_agents = {\"session\": MagicMock()}\n+        monkeypatch.setattr(\"cron.scheduler.get_running_job_ids\", lambda: {\"job-1\"})\n+\n+        assert runner._active_work_count() == 4\n+\n     def test_enter_sets_flag_and_flips_state(self):\n         runner, _ = _drain_runner()\n         runner._enter_external_drain()","@@ -289,6 +300,26 @@ def test_exit_when_loop_stopped_does_not_revert(self):\n \n \n class TestDrainWatcher:\n+    @pytest.mark.asyncio\n+    async def test_watcher_persists_aggregate_work_during_external_drain(self, home, monkeypatch):\n+        runner, _ = _drain_runner()\n+        runner._drain_control_watcher = GatewayRunner._drain_control_watcher.__get__(\n+            runner, GatewayRunner\n+        )\n+        runner._persist_active_agents = MagicMock()\n+        dc.write_drain_request()\n+        task = asyncio.create_task(runner._drain_control_watcher(interval=0.01))\n+        await asyncio.sleep(0.03)\n+        runner._running = False\n+        await asyncio.sleep(0.02)\n+        task.cancel()\n+        try:\n+            await task\n+        except asyncio.CancelledError:\n+            pass\n+\n+        runner._persist_active_agents.assert_called()\n+\n     @pytest.mark.asyncio\n     async def test_watcher_enters_then_exits_with_marker(self, home):\n         runner, _ = _drain_runner()"]},{"file":"tests/gateway/test_shutdown_cache_cleanup.py","hunks":["@@ -60,6 +60,11 @@ def _active_cron_job_count(self):\n         # there's never in-flight cron work to report.\n         return 0\n \n+    def _active_api_run_count(self):\n+        # The shutdown log also reports adapter-owned API work (#63529).\n+        # This fake has no API server adapter, so it is always idle.\n+        return 0\n+\n     def _update_runtime_status(self, *_a, **_kw):\n         pass\n \n"]}]},{"kind":"pr","number":80917,"title":"fix(gateway): interrupt every in-flight API turn on shutdown, not just /v1/runs (#63529)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/platforms/api_server.py","gateway/run.py","tests/gateway/test_api_server_active_work_drain.py"]}],"trust":{"default_tier":"verified","includes_suggested":false}}