{"repo":"NousResearch/hermes-agent","public_id":"35393","display_label":"Gateway s6-overlay sleep and restart issues in Docker","summary":"These issues and fixes address gateway process crashes and container startup failures related to s6-overlay's sleep binary and signal handling in Docker environments. Fixes include fallback heartbeat logic and HOME directory alignment for affected services.","subsystem":null,"counts":{"members":6,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":3,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":37120},"resolution":{"state":"partially_resolved","by":"pr-37120"},"freshness":{"state":"verified","last_verified_at":"2026-06-02T01:59:28+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.24799999999999997},"fix_edges":[{"issue_number":36208,"pr_number":37120,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":35393,"title":"[Bug] Gateway repeatedly restarts with signal=UNKNOWN under s6-overlay on Docker 28.5.2","url":null,"state":"closed"},{"kind":"issue","number":35394,"title":"[Bug] Gateway repeatedly restarts with signal=UNKNOWN under s6-overlay on Docker 28.5.2","url":null,"state":"closed"},{"kind":"issue","number":36208,"title":"[Bug]: docker container does not start from version 2026.5.28","url":null,"state":"closed"},{"kind":"pr","number":36221,"title":"fix(gateway): fall back when s6 sleep is missing","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#37120","number":37120},{"kind":"pr","value":"#36208","number":36208}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":36267,"title":"fix(gateway): replace os.execvp(\"sleep\", ...) with signal.pause()","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#37120","number":37120},{"kind":"pr","value":"#36208","number":36208}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":37120,"title":"fix(gateway): fall back to in-process heartbeat when s6 sleep is missing (#36208)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/gateway.py","tests/hermes_cli/test_gateway_s6_dispatch.py"],"fix_hunks":[{"file":"hermes_cli/gateway.py","hunks":["@@ -5877,15 +5877,60 @@ def _maybe_redirect_run_to_s6_supervision(args) -> bool:\n         file=sys.stderr,\n         flush=True,\n     )\n-    # Block until the container is signalled. The supervised gateway's\n-    # lifetime is independent of this process — s6-supervise restarts\n-    # it on crash, and we don't want the container to exit when the\n-    # gateway flaps. `sleep infinity` matches the static main-hermes\n-    # service's pattern (see docker/s6-rc.d/main-hermes/run): the CMD\n-    # process is a no-op heartbeat that keeps /init alive until\n+    # Keep the CMD process alive as a no-op heartbeat. The supervised\n+    # gateway's lifetime is independent of this process — s6-supervise\n+    # restarts it on crash, and we don't want the container to exit when\n+    # the gateway flaps. The CMD process keeps /init alive until\n     # `docker stop` sends SIGTERM, at which point /init runs stage 3\n     # shutdown (which tears down the supervised gateway cleanly).\n-    os.execvp(\"sleep\", [\"sleep\", \"infinity\"])\n+    #\n+    # Prefer `sleep infinity` (matches the static main-hermes service's\n+    # pattern in docker/s6-rc.d/main-hermes/run, and frees the Python\n+    # interpreter — the heartbeat is a tiny `sleep` process, not a\n+    # resident interpreter). But `os.execvp` does a PATH lookup for the\n+    # `sleep` binary and historically crashed the whole container with\n+    # FileNotFoundError when PATH was empty/truncated/clobbered at this\n+    # point — e.g. after user customizations rewrote PATH, or on minimal\n+    # images without `sleep` on PATH (issue #36208). Fall back to an\n+    # in-process block (no external binary, can't fail on PATH) so the\n+    # container keeps running instead of dying during boot.\n+    try:\n+        os.execvp(\"sleep\", [\"sleep\", \"infinity\"])\n+    except OSError:\n+        # execvp only returns by raising; on success it replaces this\n+        # process. ENOENT (no `sleep` on PATH) and any other exec error\n+        # land here.\n+        print(\n+            \"→ `sleep` is unavailable; keeping the s6 CMD process alive \"\n+            \"in-process until the container is stopped.\",\n+            file=sys.stderr,\n+            flush=True,\n+        )\n+        _block_until_terminated()\n+    return True  # unreachable on the execvp success path\n+\n+\n+def _block_until_terminated() -> None:\n+    \"\"\"Keep the s6 CMD process alive until the container is stopped.\n+\n+    Fallback heartbeat for when ``os.execvp(\"sleep\", ...)`` can't run\n+    (``sleep`` missing from PATH — issue #36208). Installs a SIGTERM\n+    handler that exits with the conventional 128+signum code so\n+    ``docker stop`` produces a clean, expected exit, then blocks on\n+    ``signal.pause()``. Falls back to ``threading.Event().wait()`` on\n+    platforms without ``signal.pause()`` (e.g. Windows) — although this\n+    path only runs inside the s6 Linux container image, the fallback\n+    keeps the helper safe to import and unit-test anywhere.\n+    \"\"\"\n+    signal.signal(signal.SIGTERM, lambda signum, _frame: sys.exit(128 + signum))\n+    pause = getattr(signal, \"pause\", None)\n+    if pause is not None:\n+        while True:\n+            pause()\n+    else:  # pragma: no cover - non-Unix fallback, not exercised in the s6 image\n+        import threading\n+\n+        threading.Event().wait()\n \n \n def _gateway_command_inner(args):"]},{"file":"tests/hermes_cli/test_gateway_s6_dispatch.py","hunks":["@@ -361,28 +361,6 @@ def _stub_s6(monkeypatch: pytest.MonkeyPatch, *, on_s6: bool) -> _CallRecorder:\n     return rec\n \n \n-class _ExecvpCalled(BaseException):\n-    \"\"\"Sentinel raised by the os.execvp stub so tests can assert on it\n-    without actually replacing the test runner process. Inherits from\n-    BaseException so it bypasses generic ``except Exception`` blocks in\n-    the code under test (just like a real exec would).\"\"\"\n-\n-    def __init__(self, argv: list[str]) -> None:\n-        self.argv = argv\n-\n-\n-def _stub_execvp(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]:\n-    \"\"\"Replace os.execvp with a recorder that raises _ExecvpCalled.\"\"\"\n-    calls: list[list[str]] = []\n-\n-    def fake_execvp(file: str, args: list[str]) -> None:  # noqa: ANN401\n-        calls.append([file, *args])\n-        raise _ExecvpCalled([file, *args])\n-\n-    monkeypatch.setattr(\"hermes_cli.gateway.os.execvp\", fake_execvp)\n-    return calls\n-\n-\n def test_redirect_noop_on_host(monkeypatch: pytest.MonkeyPatch) -> None:\n     \"\"\"Host runs (non-s6) must not redirect. Returns False; caller\n     continues to the foreground gateway code path unchanged.\"\"\"","@@ -407,14 +385,31 @@ def test_redirect_fires_inside_s6_container(\n \n     1. Dispatch `start` to the service manager.\n     2. Print the loud breadcrumb to stderr.\n-    3. exec `sleep infinity` to keep the CMD alive without binding\n-       container lifetime to gateway PID lifetime.\n+    3. exec `sleep infinity` to keep the CMD alive (the cheap heartbeat;\n+       no resident Python interpreter) without binding container\n+       lifetime to gateway PID lifetime.\n     \"\"\"\n     from hermes_cli import gateway as gw\n \n     rec = _stub_s6(monkeypatch, on_s6=True)\n     monkeypatch.setattr(\"hermes_cli.gateway._profile_suffix\", lambda: \"\")\n-    execvp_calls = _stub_execvp(monkeypatch)\n+\n+    class _ExecvpCalled(BaseException):\n+        def __init__(self, argv: list[str]) -> None:\n+            self.argv = argv\n+\n+    execvp_calls: list[list[str]] = []\n+\n+    def fake_execvp(file: str, args: list[str]) -> None:\n+        execvp_calls.append([file, *args])\n+        raise _ExecvpCalled([file, *args])\n+\n+    monkeypatch.setattr(\"hermes_cli.gateway.os.execvp\", fake_execvp)\n+    # If the fallback ran, the normal sleep path was wrongly skipped.\n+    monkeypatch.setattr(\n+        \"hermes_cli.gateway._block_until_terminated\",\n+        lambda: pytest.fail(\"fallback should not run when sleep is available\"),\n+    )\n     monkeypatch.delenv(\"HERMES_S6_SUPERVISED_CHILD\", raising=False)\n     monkeypatch.delenv(\"HERMES_GATEWAY_NO_SUPERVISE\", raising=False)\n ","@@ -428,11 +423,90 @@ def test_redirect_fires_inside_s6_container(\n     assert \"s6 supervision\" in err\n     assert \"--no-supervise\" in err\n     assert \"HERMES_GATEWAY_NO_SUPERVISE\" in err\n-    # 3. exec'd `sleep infinity`.\n+    # 3. exec'd `sleep infinity` (the preferred cheap heartbeat).\n     assert execvp_calls == [[\"sleep\", \"sleep\", \"infinity\"]]\n     assert excinfo.value.argv == [\"sleep\", \"sleep\", \"infinity\"]\n \n \n+def test_redirect_falls_back_when_sleep_missing(\n+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],\n+) -> None:\n+    \"\"\"Regression guard for issue #36208: when ``os.execvp(\"sleep\", ...)``\n+    raises (no `sleep` on a clobbered/empty PATH, or a minimal image\n+    without it), the redirect must NOT crash the container — it falls\n+    back to the in-process ``_block_until_terminated`` heartbeat so the\n+    container keeps running.\n+    \"\"\"\n+    from hermes_cli import gateway as gw\n+\n+    rec = _stub_s6(monkeypatch, on_s6=True)\n+    monkeypatch.setattr(\"hermes_cli.gateway._profile_suffix\", lambda: \"\")\n+\n+    def missing_sleep(file: str, args: list[str]) -> None:\n+        raise FileNotFoundError(2, \"No such file or directory\", file)\n+\n+    monkeypatch.setattr(\"hermes_cli.gateway.os.execvp\", missing_sleep)\n+    block_calls: list[bool] = []\n+    monkeypatch.setattr(\n+        \"hermes_cli.gateway._block_until_terminated\",\n+        lambda: block_calls.append(True),\n+    )\n+    monkeypatch.delenv(\"HERMES_S6_SUPERVISED_CHILD\", raising=False)\n+    monkeypatch.delenv(\"HERMES_GATEWAY_NO_SUPERVISE\", raising=False)\n+\n+    # Must not raise FileNotFoundError — that was the #36208 crash.\n+    result = gw._maybe_redirect_run_to_s6_supervision(_Args())\n+\n+    assert result is True\n+    assert rec.calls == [(\"start\", \"gateway-default\")]\n+    # Fell back to the in-process heartbeat instead of crashing.\n+    assert block_calls == [True]\n+    err = capsys.readouterr().err\n+    assert \"`sleep` is unavailable\" in err\n+\n+\n+def test_block_until_terminated_installs_sigterm_handler_and_blocks(\n+    monkeypatch: pytest.MonkeyPatch,\n+) -> None:\n+    \"\"\"``_block_until_terminated`` must register a SIGTERM handler (so\n+    `docker stop` exits cleanly) and then block on signal.pause() — never\n+    touching an external binary. Regression guard for issue #36208, where\n+    os.execvp(\"sleep\", ...) crashed the container with FileNotFoundError\n+    when PATH lacked a directory containing `sleep`.\n+    \"\"\"\n+    import signal as _signal\n+    from hermes_cli import gateway as gw\n+\n+    registered: dict[int, object] = {}\n+    monkeypatch.setattr(\n+        \"hermes_cli.gateway.signal.signal\",\n+        lambda signum, handler: registered.__setitem__(signum, handler),\n+    )\n+\n+    # Make signal.pause() raise after the first call so the infinite loop\n+    # terminates deterministically instead of hanging the test.\n+    pause_calls = {\"n\": 0}\n+\n+    def fake_pause() -> None:\n+        pause_calls[\"n\"] += 1\n+        raise KeyboardInterrupt  # break out of the `while True: pause()` loop\n+\n+    monkeypatch.setattr(\"hermes_cli.gateway.signal.pause\", fake_pause)\n+\n+    with pytest.raises(KeyboardInterrupt):\n+        gw._block_until_terminated()\n+\n+    # A SIGTERM handler was installed...\n+    assert _signal.SIGTERM in registered\n+    # ...and it exits with the conventional 128+signum code.\n+    handler = registered[_signal.SIGTERM]\n+    with pytest.raises(SystemExit) as exc:\n+        handler(_signal.SIGTERM, None)  # type: ignore[operator]\n+    assert exc.value.code == 128 + _signal.SIGTERM\n+    # ...and we actually blocked on pause().\n+    assert pause_calls[\"n\"] == 1\n+\n+\n def test_redirect_short_circuits_supervised_child(\n     monkeypatch: pytest.MonkeyPatch,\n ) -> None:","@@ -516,10 +590,25 @@ def test_redirect_no_supervise_env_falsy_values_dont_opt_out(\n \n     _stub_s6(monkeypatch, on_s6=True)\n     monkeypatch.setattr(\"hermes_cli.gateway._profile_suffix\", lambda: \"\")\n-    _stub_execvp(monkeypatch)\n+\n+    # The redirect reaching its `sleep` heartbeat means it did NOT opt\n+    # out. Stub execvp to record + raise (so it doesn't replace the test\n+    # process) rather than actually exec.\n+    class _ExecvpCalled(BaseException):\n+        pass\n+\n+    execvp_calls: list[str] = []\n+\n+    def fake_execvp(file: str, args: list[str]) -> None:\n+        execvp_calls.append(file)\n+        raise _ExecvpCalled\n+\n+    monkeypatch.setattr(\"hermes_cli.gateway.os.execvp\", fake_execvp)\n     monkeypatch.delenv(\"HERMES_S6_SUPERVISED_CHILD\", raising=False)\n \n     for falsy in (\"\", \"0\", \"false\", \"no\", \"off\", \"garbage\"):\n+        execvp_calls.clear()\n         monkeypatch.setenv(\"HERMES_GATEWAY_NO_SUPERVISE\", falsy)\n         with pytest.raises(_ExecvpCalled):\n             gw._maybe_redirect_run_to_s6_supervision(_Args())\n+        assert execvp_calls == [\"sleep\"], f\"redirect should fire for {falsy!r}\"\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}