{"repo":"NousResearch/hermes-agent","public_id":"19471","display_label":"Gateway replace and event loop fixes after SIGKILL","summary":"These issues and PRs address bugs in the gateway's --replace behavior when an old process survives SIGKILL, and a related event loop loss problem during restart. A Telegram polling retry reschedule fix using get_running_loop is also included.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":41864,"merged_at":"2026-06-08T06:57:33+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-08T06:57:33+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":19471,"pr_number":41864,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-06-08T06:57:33+00:00"}],"members":[{"kind":"issue","number":19471,"title":"[BUG] `--profile` gateway enters crash loop after SIGTERM→SIGKILL — event loop lost on restart","url":null,"state":"closed"},{"kind":"pr","number":19997,"title":"fix(gateway): abort replace if old PID survives SIGKILL","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#41864","number":41864}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":41864,"title":"fix(gateway): abort --replace when old PID survives SIGKILL (#19997, #19471)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_runner_startup_failures.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -18765,8 +18765,10 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =\n             # ``os.kill(pid, 0)`` on Windows is NOT a no-op — use the\n             # handle-based existence check instead.\n             from gateway.status import _pid_exists\n+            old_gateway_exited = False\n             for _ in range(20):\n                 if not _pid_exists(existing_pid):\n+                    old_gateway_exited = True\n                     break  # Process is gone\n                 time.sleep(0.5)\n             else:","@@ -18777,9 +18779,34 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =\n                 )\n                 try:\n                     terminate_pid(existing_pid, force=True)\n-                    time.sleep(0.5)\n-                except (ProcessLookupError, PermissionError, OSError):\n+                except ProcessLookupError:\n+                    old_gateway_exited = True\n+                except (PermissionError, OSError):\n                     pass\n+                # Confirm the force-kill actually reaped the process before we\n+                # clear its PID file / scoped locks. SIGKILL can fail to take\n+                # (e.g. an uninterruptible-sleep or zombie-reaping parent), and\n+                # if we blindly clear the metadata and start a fresh instance\n+                # we end up with two live gateways fighting over the same\n+                # token — the duplicate-gateway failure in #19471.\n+                if not old_gateway_exited:\n+                    for _ in range(20):\n+                        if not _pid_exists(existing_pid):\n+                            old_gateway_exited = True\n+                            break\n+                        time.sleep(0.25)\n+                if not old_gateway_exited:\n+                    logger.error(\n+                        \"Old gateway (PID %d) still appears alive after SIGKILL; \"\n+                        \"aborting replacement to avoid a duplicate gateway.\",\n+                        existing_pid,\n+                    )\n+                    try:\n+                        from gateway.status import clear_takeover_marker\n+                        clear_takeover_marker()\n+                    except Exception:\n+                        pass\n+                    return False\n             remove_pid_file()\n             # remove_pid_file() is a no-op when the PID doesn't match.\n             # Force-unlink to cover the old-process-crashed case."]},{"file":"tests/gateway/test_runner_startup_failures.py","hunks":["@@ -206,8 +206,17 @@ def _mock_remove_pid_file():\n         \"gateway.status.release_all_scoped_locks\",\n         lambda **kwargs: 0,\n     )\n-    monkeypatch.setattr(\"gateway.status.terminate_pid\", lambda pid, force=False: calls.append((pid, force)))\n-    monkeypatch.setattr(\"gateway.status._pid_exists\", lambda pid: True)\n+    # force-kill reaps the process: terminate_pid(force=True) flips it dead,\n+    # and the post-kill re-poll via _pid_exists then sees it gone so the\n+    # replacement proceeds.\n+    def _mock_terminate_pid(pid, force=False):\n+        calls.append((pid, force))\n+        if force:\n+            _pid_state[\"alive\"] = False\n+    monkeypatch.setattr(\"gateway.status.terminate_pid\", _mock_terminate_pid)\n+    monkeypatch.setattr(\n+        \"gateway.status._pid_exists\", lambda pid: _pid_state[\"alive\"]\n+    )\n     monkeypatch.setattr(\"gateway.run.os.getpid\", lambda: 100)\n     monkeypatch.setattr(\"gateway.run.os.kill\", lambda pid, sig: None)\n     monkeypatch.setattr(\"time.sleep\", lambda _: None)","@@ -224,6 +233,65 @@ def _mock_remove_pid_file():\n     assert calls == [(42, False), (42, True)]\n \n \n[redacted:email]\n+async def test_start_gateway_replace_aborts_when_force_killed_pid_still_alive(\n+    monkeypatch, tmp_path\n+):\n+    \"\"\"Regression for #19471 (duplicate-gateway half).\n+\n+    If SIGKILL fails to reap the old gateway, --replace must NOT clear the PID\n+    file / scoped locks and start a fresh instance — that leaves two live\n+    gateways fighting over the same token. It should abort instead.\n+    \"\"\"\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+\n+    calls = []\n+    removed_pid = False\n+    released_locks = False\n+\n+    class _RunnerShouldNotStart:\n+        def __init__(self, config):\n+            raise AssertionError(\"replacement must not start while old PID is alive\")\n+\n+    def _mock_remove_pid_file():\n+        nonlocal removed_pid\n+        removed_pid = True\n+\n+    def _mock_release_all_scoped_locks(**kwargs):\n+        nonlocal released_locks\n+        released_locks = True\n+        return 0\n+\n+    monkeypatch.setattr(\"gateway.status.get_running_pid\", lambda: 42)\n+    monkeypatch.setattr(\"gateway.status.remove_pid_file\", _mock_remove_pid_file)\n+    monkeypatch.setattr(\n+        \"gateway.status.release_all_scoped_locks\",\n+        _mock_release_all_scoped_locks,\n+    )\n+    monkeypatch.setattr(\n+        \"gateway.status.terminate_pid\",\n+        lambda pid, force=False: calls.append((pid, force)),\n+    )\n+    # _pid_exists never goes False — the force-kill did not take.\n+    monkeypatch.setattr(\"gateway.status._pid_exists\", lambda pid: True)\n+    monkeypatch.setattr(\"gateway.run.os.getpid\", lambda: 100)\n+    monkeypatch.setattr(\"gateway.run.os.kill\", lambda pid, sig: None)\n+    monkeypatch.setattr(\"time.sleep\", lambda _: None)\n+    monkeypatch.setattr(\"tools.skills_sync.sync_skills\", lambda quiet=True: None)\n+    monkeypatch.setattr(\"hermes_logging.setup_logging\", lambda hermes_home, mode: tmp_path)\n+    monkeypatch.setattr(\"hermes_logging._add_rotating_handler\", lambda *args, **kwargs: None)\n+    monkeypatch.setattr(\"gateway.run.GatewayRunner\", _RunnerShouldNotStart)\n+\n+    from gateway.run import start_gateway\n+\n+    ok = await start_gateway(config=GatewayConfig(), replace=True, verbosity=None)\n+\n+    assert ok is False\n+    assert calls == [(42, False), (42, True)]\n+    assert removed_pid is False\n+    assert released_locks is False\n+\n+\n @pytest.mark.asyncio\n async def test_start_gateway_replace_writes_takeover_marker_before_sigterm(\n     monkeypatch, tmp_path\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}