{"repo":"NousResearch/hermes-agent","public_id":"45963","display_label":"prevent gateway auto-start on profile creation","summary":"A bug causes gateway services to auto-start when creating new profiles in s6 containers, leading to token-conflict zombie processes. These members address registering profile gateways without triggering automatic startup.","subsystem":null,"counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":2,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":46266},"resolution":{"state":"resolved","by":"pr-46266"},"freshness":{"state":"verified","last_verified_at":"2026-06-15T10:51:49+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.4},"fix_edges":[{"issue_number":45963,"pr_number":46266,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":45963,"pr_number":46293,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":45963,"title":"[Bug]: `profile create` auto-starts a doomed resident gateway for every new profile in the s6 container (token-conflict zombies that resurrect on restart)","url":null,"state":"closed"},{"kind":"pr","number":45968,"title":"fix(s6): prevent profile create from auto-starting gateway service","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"40d7c264f","number":0},{"kind":"pr","value":"#46266","number":46266}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":46020,"title":"fix(profiles): register s6 gateway without auto-starting (#45963)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"40d7c264f","number":0},{"kind":"pr","value":"#46266","number":46266},{"kind":"pr","value":"#45963","number":45963}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":46266,"title":"fix(s6): register profile gateways without auto-starting","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/profiles.py","hermes_cli/service_manager.py","tests/docker/test_container_restart.py","tests/hermes_cli/test_profiles_s6_hooks.py","tests/hermes_cli/test_service_manager.py"],"fix_hunks":[{"file":"hermes_cli/profiles.py","hunks":["@@ -1229,7 +1229,7 @@ def _maybe_register_gateway_service(profile_name: str) -> None:\n     if not mgr.supports_runtime_registration():\n         return  # host backend; no-op\n     try:\n-        mgr.register_profile_gateway(profile_name)\n+        mgr.register_profile_gateway(profile_name, start_now=False)\n     except ValueError:\n         # Already registered (e.g. the container-boot reconciler ran\n         # first and brought up a stale slot). That's fine."]},{"file":"hermes_cli/service_manager.py","hunks":["@@ -77,6 +77,7 @@ def register_profile_gateway(\n         profile: str,\n         *,\n         extra_env: dict[str, str] | None = None,\n+        start_now: bool = True,\n     ) -> None: ...\n     def unregister_profile_gateway(self, profile: str) -> None: ...\n     def list_profile_gateways(self) -> list[str]: ...","@@ -175,6 +176,7 @@ def register_profile_gateway(\n         profile: str,\n         *,\n         extra_env: dict[str, str] | None = None,\n+        start_now: bool = True,\n     ) -> None:\n         raise NotImplementedError(\n             f\"{type(self).__name__} does not support runtime profile \"","@@ -823,15 +825,15 @@ def register_profile_gateway(\n         profile: str,\n         *,\n         extra_env: dict[str, str] | None = None,\n+        start_now: bool = True,\n     ) -> None:\n         \"\"\"Create the s6 service directory for a profile gateway.\n \n         Triggers ``s6-svscanctl -a`` so s6-svscan picks the new directory\n-        up immediately. The service is created in the *up* state — to\n-        register without auto-starting, follow up with ``stop(profile)``\n-        (or pass the start flag via the future ``start_now=False`` arg,\n-        which the Phase 4 reconciliation path uses via a ``down``\n-        marker file written directly).\n+        up immediately.  When *start_now* is ``True`` (the default) the\n+        service starts immediately; when ``False`` a ``down`` marker file\n+        is written so s6-supervise leaves the service stopped until the\n+        user explicitly runs ``hermes -p <profile> gateway start``.\n \n         Raises:\n             ValueError: if the profile name is invalid or the service","@@ -879,6 +881,13 @@ def register_profile_gateway(\n             # rationale.\n             _seed_supervise_skeleton(tmp_dir)\n \n+            # When start_now is False, write a `down` marker so\n+            # s6-supervise does not auto-start the service on rescan.\n+            # Mirrors the same pattern in container_boot.py\n+            # _register_gateway_slot when start=False.\n+            if not start_now:\n+                (tmp_dir / \"down\").touch()\n+\n             tmp_dir.rename(svc_dir)\n         except Exception:\n             shutil.rmtree(tmp_dir, ignore_errors=True)"]},{"file":"tests/docker/test_container_restart.py","hunks":["@@ -296,7 +296,8 @@ def test_live_gateway_autostarts_after_real_restart_without_manual_state_stamp(\n         )\n         if r.returncode == 0 and '\"gateway_state\"' in r.stdout:\n             state = r.stdout\n-            break\n+            if '\"running\"' in state:\n+                break\n         time.sleep(0.5)\n     assert '\"running\"' in state, (\n         f\"gateway never persisted running state pre-restart: {state!r}\""]},{"file":"tests/hermes_cli/test_profiles_s6_hooks.py","hunks":["@@ -54,10 +54,12 @@ def supports_runtime_registration(self) -> bool:\n     def register_profile_gateway(\n         self, profile: str, *,\n         extra_env: dict[str, str] | None = None,\n+        start_now: bool = True,\n     ) -> None:\n         if self.raise_on_register is not None:\n             raise self.raise_on_register\n         self.registered.append(profile)\n+        self.last_start_now = start_now\n \n     def unregister_profile_gateway(self, profile: str) -> None:\n         if self.raise_on_unregister is not None:","@@ -107,6 +109,21 @@ def test_register_calls_through_on_s6(monkeypatch: pytest.MonkeyPatch) -> None:\n     assert mgr.registered == [\"coder\"]\n \n \n+def test_register_passes_start_now_false(monkeypatch: pytest.MonkeyPatch) -> None:\n+    \"\"\"_maybe_register_gateway_service must register with start_now=False\n+    so that profile creation does not auto-start a gateway that may\n+    conflict with the main gateway's bot-token lock.\"\"\"\n+    _patch_detect_s6(monkeypatch)\n+    mgr = _S6Manager()\n+    monkeypatch.setattr(\n+        \"hermes_cli.service_manager.get_service_manager\", lambda: mgr,\n+    )\n+    _maybe_register_gateway_service(\"coder\")\n+    assert mgr.last_start_now is False, (\n+        \"profile creation must not auto-start the gateway service\"\n+    )\n+\n+\n def test_register_swallows_duplicate_value_error(\n     monkeypatch: pytest.MonkeyPatch,\n ) -> None:"]},{"file":"tests/hermes_cli/test_service_manager.py","hunks":["@@ -572,6 +572,35 @@ def test_s6_register_creates_service_dir_and_triggers_scan(\n     ), f\"s6-svscanctl -a not invoked; saw: {fake_subprocess_run}\"\n \n \n+def test_s6_register_start_now_false_writes_down_marker(\n+    s6_scandir, fake_subprocess_run,\n+) -> None:\n+    \"\"\"When start_now=False, a `down` marker must be written so\n+    s6-supervise does not auto-start the service on rescan.\"\"\"\n+    mgr = S6ServiceManager(scandir=s6_scandir)\n+    mgr.register_profile_gateway(\"coder\", start_now=False)\n+\n+    svc_dir = s6_scandir / \"gateway-coder\"\n+    assert svc_dir.is_dir()\n+    assert (svc_dir / \"down\").is_file(), (\n+        \"start_now=False must write a `down` marker file\"\n+    )\n+\n+\n+def test_s6_register_start_now_true_no_down_marker(\n+    s6_scandir, fake_subprocess_run,\n+) -> None:\n+    \"\"\"When start_now=True (default), no `down` marker should exist.\"\"\"\n+    mgr = S6ServiceManager(scandir=s6_scandir)\n+    mgr.register_profile_gateway(\"coder\")\n+\n+    svc_dir = s6_scandir / \"gateway-coder\"\n+    assert svc_dir.is_dir()\n+    assert not (svc_dir / \"down\").exists(), (\n+        \"start_now=True must NOT write a `down` marker file\"\n+    )\n+\n+\n def test_s6_register_extra_env_is_quoted(s6_scandir, fake_subprocess_run) -> None:\n     mgr = S6ServiceManager(scandir=s6_scandir)\n     mgr.register_profile_gateway(\n"]}]},{"kind":"pr","number":46293,"title":"fix(docker): skip gateway reconciliation in dashboard container (autodetect)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/container_boot.py","scripts/release.py","tests/hermes_cli/test_container_boot.py"],"fix_hunks":[{"file":"hermes_cli/container_boot.py","hunks":["@@ -207,8 +207,15 @@ def _read_container_argv() -> tuple[str, ...]:\n     return tuple(part.decode(\"utf-8\", \"replace\") for part in raw.split(b\"\\0\") if part)\n \n \n-def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool:\n-    \"\"\"Return True for Docker commands equivalent to `gateway run`.\"\"\"\n+def _strip_container_argv_prefix(argv: Sequence[str]) -> list[str]:\n+    \"\"\"Strip the s6/wrapper prefix off PID 1 argv, leaving the hermes args.\n+\n+    The container PID 1 argv looks like\n+    ``/init /opt/hermes/docker/main-wrapper.sh <subcommand> [args...]`` and\n+    the wrapper re-execs ``hermes <subcommand>``. Peel ``init`` →\n+    ``main-wrapper.sh`` → ``hermes`` so callers can match on the bare\n+    subcommand. Shared by the legacy-gateway and dashboard role detectors.\n+    \"\"\"\n     args = list(argv)\n     if args and Path(args[0]).name == \"init\":\n         args = args[1:]","@@ -216,11 +223,38 @@ def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool:\n         args = args[1:]\n     if args and Path(args[0]).name == \"hermes\":\n         args = args[1:]\n+    return args\n+\n+\n+def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool:\n+    \"\"\"Return True for Docker commands equivalent to `gateway run`.\"\"\"\n+    args = _strip_container_argv_prefix(argv)\n     if \"--no-supervise\" in args:\n         return False\n     return len(args) >= 2 and args[0] == \"gateway\" and args[1] == \"run\"\n \n \n+def _is_dashboard_container(argv: Sequence[str]) -> bool:\n+    \"\"\"Return True when the container's command is the dashboard.\n+\n+    A dashboard-only container (``hermes dashboard ...``) never spawns or\n+    supervises per-profile gateways — that is the gateway container's job.\n+    Reconciling profile gateway s6 slots there is not just wasted work: when\n+    the gateway and dashboard containers share a bind-mounted HERMES_HOME,\n+    both race to ``flock()`` the same ``logs/gateways/<profile>/lock`` files,\n+    producing \"Resource busy\" failures and an s6-log restart storm. So the\n+    dashboard container skips reconciliation entirely.\n+\n+    Detected from PID 1 argv (``/proc/1/cmdline``) rather than an operator\n+    flag: the role is a fact about the container's command, not a tunable,\n+    and a flag can be forgotten in a hand-written compose/k8s manifest —\n+    reintroducing the exact storm this prevents. Mirrors the argv handling\n+    in :func:`_is_legacy_gateway_run_request`.\n+    \"\"\"\n+    args = _strip_container_argv_prefix(argv)\n+    return bool(args) and args[0] == \"dashboard\"\n+\n+\n def _read_desired_state(profile_dir: Path) -> str | None:\n     \"\"\"Read the persisted gateway desired state for reconciliation.\n ","@@ -393,6 +427,22 @@ def _write_reconcile_log(\n \n def main() -> int:\n     \"\"\"Entry point invoked from /etc/cont-init.d/02-reconcile-profiles.\"\"\"\n+    # A dashboard-only container never spawns or supervises per-profile\n+    # gateways, so reconciling their s6 slots here is pure waste — and\n+    # actively harmful: when the gateway and dashboard containers share a\n+    # bind-mounted HERMES_HOME, both race to flock() the same s6-log lock\n+    # files under logs/gateways/<profile>/lock, producing \"Resource busy\"\n+    # failures and a restart storm. Detect the role from PID 1 argv and\n+    # skip reconciliation in the dashboard container. No operator flag:\n+    # the role is a fact about the container's command, and a flag can be\n+    # forgotten in a hand-written manifest, reintroducing the storm.\n+    if _is_dashboard_container(_read_container_argv()):\n+        print(\n+            \"reconcile: skipping (dashboard container — does not need \"\n+            \"per-profile gateways)\"\n+        )\n+        return 0\n+\n     hermes_home = Path(os.environ.get(\"HERMES_HOME\", \"/opt/data\"))\n     scandir = Path(os.environ.get(\"S6_PROFILE_GATEWAY_SCANDIR\", \"/run/service\"))\n     actions = reconcile_profile_gateways("]},{"file":"scripts/release.py","hunks":["@@ -84,6 +84,7 @@\n     \"[redacted:email]\": \"synapsesx\",\n     \"[redacted:email]\": \"itsflownium\",\n     \"[redacted:email]\": \"dirtyren\",\n+    \"[redacted:email]\": \"895252509\",\n     \"[redacted:email]\": \"zxcasongs\",\n     \"[redacted:email]\": \"alfred-smith-0\",\n     \"[redacted:email]\": \"tangtaizong666\","]},{"file":"tests/hermes_cli/test_container_boot.py","hunks":["@@ -708,3 +708,144 @@ def test_profiles_default_subdir_is_skipped_with_warning(\n     assert any(\n         \"profiles/default/\" in record.message for record in caplog.records\n     )\n+\n+\n+# ---------------------------------------------------------------------------\n+# Dashboard-container role detection (skip reconcile on the dashboard)\n+# ---------------------------------------------------------------------------\n+\n+\n[redacted:email](\n+    \"container_argv\",\n+    [\n+        # Bare subcommand (docker run ... dashboard ...).\n+        (\"dashboard\",),\n+        (\"dashboard\", \"--host\", \"127.0.0.1\", \"--no-open\"),\n+        # Through s6 /init + the main-wrapper that re-execs `hermes`.\n+        (\"/init\", \"/opt/hermes/docker/main-wrapper.sh\", \"dashboard\"),\n+        (\n+            \"/init\",\n+            \"/opt/hermes/docker/main-wrapper.sh\",\n+            \"dashboard\",\n+            \"--host\",\n+            \"127.0.0.1\",\n+            \"--no-open\",\n+        ),\n+        # Wrapper that kept the explicit `hermes` argv0.\n+        (\"/init\", \"/opt/hermes/docker/main-wrapper.sh\", \"hermes\", \"dashboard\"),\n+    ],\n+)\n+def test_is_dashboard_container_true_for_dashboard_argv(\n+    container_argv: tuple[str, ...],\n+) -> None:\n+    \"\"\"A dashboard command is detected across every wrapper prefix shape.\"\"\"\n+    from hermes_cli.container_boot import _is_dashboard_container\n+\n+    assert _is_dashboard_container(container_argv) is True\n+\n+\n[redacted:email](\n+    \"container_argv\",\n+    [\n+        (),  # empty (/proc/1/cmdline unreadable) — not the dashboard\n+        (\"gateway\", \"run\"),\n+        (\"/init\", \"/opt/hermes/docker/main-wrapper.sh\", \"gateway\", \"run\"),\n+        (\"/init\", \"/opt/hermes/docker/main-wrapper.sh\", \"hermes\", \"gateway\", \"run\"),\n+        (\"chat\",),\n+        # A profile literally named \"dashboard\" must NOT match — the token\n+        # we key on is the SUBCOMMAND, and `gateway run -p dashboard` is a\n+        # gateway container.\n+        (\"gateway\", \"run\", \"-p\", \"dashboard\"),\n+    ],\n+)\n+def test_is_dashboard_container_false_for_non_dashboard_argv(\n+    container_argv: tuple[str, ...],\n+) -> None:\n+    \"\"\"Gateway / other commands (and empty argv) are not the dashboard.\"\"\"\n+    from hermes_cli.container_boot import _is_dashboard_container\n+\n+    assert _is_dashboard_container(container_argv) is False\n+\n+\n+def test_main_skips_reconcile_in_dashboard_container(\n+    tmp_path: Path,\n+    monkeypatch: pytest.MonkeyPatch,\n+    capsys: pytest.CaptureFixture[str],\n+) -> None:\n+    \"\"\"main() must NOT reconcile when PID 1 argv is the dashboard command.\n+\n+    A running profile is seeded so that, if reconcile ran, it would create\n+    the gateway-<profile> slot. Asserting the slot is absent proves the\n+    skip is real, not just a log line.\n+    \"\"\"\n+    from hermes_cli import container_boot\n+\n+    scandir = tmp_path / \"run-service\"; scandir.mkdir()\n+    _make_profile(tmp_path, \"worker\", state=\"running\")\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+    monkeypatch.setenv(\"S6_PROFILE_GATEWAY_SCANDIR\", str(scandir))\n+    monkeypatch.setattr(\n+        container_boot,\n+        \"_read_container_argv\",\n+        lambda: (\"/init\", \"/opt/hermes/docker/main-wrapper.sh\", \"dashboard\"),\n+    )\n+\n+    rc = container_boot.main()\n+\n+    assert rc == 0\n+    assert not (scandir / \"gateway-worker\").exists()\n+    assert not (scandir / \"gateway-default\").exists()\n+    assert \"skipping (dashboard container\" in capsys.readouterr().out\n+\n+\n+def test_main_reconciles_in_gateway_container(\n+    tmp_path: Path,\n+    monkeypatch: pytest.MonkeyPatch,\n+) -> None:\n+    \"\"\"main() reconciles normally when PID 1 argv is the gateway command —\n+    the dashboard skip is scoped strictly to the dashboard role.\"\"\"\n+    from hermes_cli import container_boot\n+\n+    scandir = tmp_path / \"run-service\"; scandir.mkdir()\n+    _make_profile(tmp_path, \"worker\", state=\"running\")\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+    monkeypatch.setenv(\"S6_PROFILE_GATEWAY_SCANDIR\", str(scandir))\n+    monkeypatch.setattr(\n+        container_boot,\n+        \"_read_container_argv\",\n+        lambda: (\"/init\", \"/opt/hermes/docker/main-wrapper.sh\", \"gateway\", \"run\"),\n+    )\n+\n+    rc = container_boot.main()\n+\n+    assert rc == 0\n+    # The worker slot was registered + started (prior_state running).\n+    assert (scandir / \"gateway-worker\").exists()\n+    assert not (scandir / \"gateway-worker\" / \"down\").exists()\n+\n+\n+def test_main_ignores_removed_skip_reconcile_env_var(\n+    tmp_path: Path,\n+    monkeypatch: pytest.MonkeyPatch,\n+) -> None:\n+    \"\"\"The legacy HERMES_SKIP_PROFILE_RECONCILE flag is gone: setting it on a\n+    gateway container must NOT suppress reconciliation. Role is decided by\n+    PID 1 argv alone, so a stale flag in someone's manifest is inert.\"\"\"\n+    from hermes_cli import container_boot\n+\n+    scandir = tmp_path / \"run-service\"; scandir.mkdir()\n+    _make_profile(tmp_path, \"worker\", state=\"running\")\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+    monkeypatch.setenv(\"S6_PROFILE_GATEWAY_SCANDIR\", str(scandir))\n+    monkeypatch.setenv(\"HERMES_SKIP_PROFILE_RECONCILE\", \"1\")\n+    monkeypatch.setattr(\n+        container_boot,\n+        \"_read_container_argv\",\n+        lambda: (\"/init\", \"/opt/hermes/docker/main-wrapper.sh\", \"gateway\", \"run\"),\n+    )\n+\n+    rc = container_boot.main()\n+\n+    assert rc == 0\n+    # Reconcile still ran despite the stale env var.\n+    assert (scandir / \"gateway-worker\").exists()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}