# NousResearch/hermes-agent · prevent gateway auto-start on profile creation ## Open now - Nothing open on GitHub. A bug causes gateway services to auto-start when creating new profiles in s6 containers, leading to token-conflict zombie processes. These members address registering profile gateways without triggering automatic startup. ## TL;DR - 5 member(s): 0 open issue(s), 1 closed issue(s), 0 open PR(s), 2 merged PR(s), 2 PR(s) closed without merging. - A fix was merged to main in PR #46266. Verify it addresses your case. - **Recommended:** read PR #46266 (and any competing PRs below) before filing. Confirm the merged change covers your symptom. ## Maintainer resolution verdict - Internal-tracker verdict over this cluster's issue members: `resolved` · deciding PR(s): `pr-46266`. This is distinct from the fix status above. Weigh it yourself. ## How this cluster was grouped - Link evidence (`STRUCTURAL_SPANNING`): every member is connected by structural evidence: shared changed files, or a closing/superseded reference. - Pairwise link density `0.40`: the share of member pairs that carry a link, counting a structural link in full and a relevance-only link at 0.36. `1.00` means every pair is linked. ## Closed / duplicate issues - #45963: \[Bug\]: \`profile create\` auto-starts a doomed resident gateway for every new profile in the s6 container (token-conflict zombies that resurrect on restart) ## Prior & competing PRs - **#46266: fix(s6): register profile gateways without auto-starting**. merged to main · fix tier `verified` - maintainer direction: category `merged`; implication `superseded` - files: `hermes_cli/profiles.py`, `hermes_cli/service_manager.py`, `tests/docker/test_container_restart.py`, `tests/hermes_cli/test_profiles_s6_hooks.py`, `tests/hermes_cli/test_service_manager.py` - fix hunks: - `hermes_cli/profiles.py` ```diff @@ -1229,7 +1229,7 @@ def _maybe_register_gateway_service(profile_name: str) -> None: if not mgr.supports_runtime_registration(): return # host backend; no-op try: - mgr.register_profile_gateway(profile_name) + mgr.register_profile_gateway(profile_name, start_now=False) except ValueError: # Already registered (e.g. the container-boot reconciler ran # first and brought up a stale slot). That's fine. ``` - `hermes_cli/service_manager.py` ```diff @@ -77,6 +77,7 @@ def register_profile_gateway( profile: str, *, extra_env: dict[str, str] | None = None, + start_now: bool = True, ) -> None: ... def unregister_profile_gateway(self, profile: str) -> None: ... def list_profile_gateways(self) -> list[str]: ... ``` ```diff @@ -175,6 +176,7 @@ def register_profile_gateway( profile: str, *, extra_env: dict[str, str] | None = None, + start_now: bool = True, ) -> None: raise NotImplementedError( f"{type(self).__name__} does not support runtime profile " ``` ```diff @@ -823,15 +825,15 @@ def register_profile_gateway( profile: str, *, extra_env: dict[str, str] | None = None, + start_now: bool = True, ) -> None: """Create the s6 service directory for a profile gateway. Triggers ``s6-svscanctl -a`` so s6-svscan picks the new directory - up immediately. The service is created in the *up* state — to - register without auto-starting, follow up with ``stop(profile)`` - (or pass the start flag via the future ``start_now=False`` arg, - which the Phase 4 reconciliation path uses via a ``down`` - marker file written directly). + up immediately. When *start_now* is ``True`` (the default) the + service starts immediately; when ``False`` a ``down`` marker file + is written so s6-supervise leaves the service stopped until the + user explicitly runs ``hermes -p gateway start``. Raises: ValueError: if the profile name is invalid or the service ``` ```diff @@ -879,6 +881,13 @@ def register_profile_gateway( # rationale. _seed_supervise_skeleton(tmp_dir) + # When start_now is False, write a `down` marker so + # s6-supervise does not auto-start the service on rescan. + # Mirrors the same pattern in container_boot.py + # _register_gateway_slot when start=False. + if not start_now: + (tmp_dir / "down").touch() + tmp_dir.rename(svc_dir) except Exception: shutil.rmtree(tmp_dir, ignore_errors=True) ``` - `tests/docker/test_container_restart.py` ```diff @@ -296,7 +296,8 @@ def test_live_gateway_autostarts_after_real_restart_without_manual_state_stamp( ) if r.returncode == 0 and '"gateway_state"' in r.stdout: state = r.stdout - break + if '"running"' in state: + break time.sleep(0.5) assert '"running"' in state, ( f"gateway never persisted running state pre-restart: {state!r}" ``` - `tests/hermes_cli/test_profiles_s6_hooks.py` ```diff @@ -54,10 +54,12 @@ def supports_runtime_registration(self) -> bool: def register_profile_gateway( self, profile: str, *, extra_env: dict[str, str] | None = None, + start_now: bool = True, ) -> None: if self.raise_on_register is not None: raise self.raise_on_register self.registered.append(profile) + self.last_start_now = start_now def unregister_profile_gateway(self, profile: str) -> None: if self.raise_on_unregister is not None: ``` ```diff @@ -107,6 +109,21 @@ def test_register_calls_through_on_s6(monkeypatch: pytest.MonkeyPatch) -> None: assert mgr.registered == ["coder"] +def test_register_passes_start_now_false(monkeypatch: pytest.MonkeyPatch) -> None: + """_maybe_register_gateway_service must register with start_now=False + so that profile creation does not auto-start a gateway that may + conflict with the main gateway's bot-token lock.""" + _patch_detect_s6(monkeypatch) + mgr = _S6Manager() + monkeypatch.setattr( + "hermes_cli.service_manager.get_service_manager", lambda: mgr, + ) + _maybe_register_gateway_service("coder") + assert mgr.last_start_now is False, ( + "profile creation must not auto-start the gateway service" + ) + + def test_register_swallows_duplicate_value_error( monkeypatch: pytest.MonkeyPatch, ) -> None: ``` - `tests/hermes_cli/test_service_manager.py` ```diff @@ -572,6 +572,35 @@ def test_s6_register_creates_service_dir_and_triggers_scan( ), f"s6-svscanctl -a not invoked; saw: {fake_subprocess_run}" +def test_s6_register_start_now_false_writes_down_marker( + s6_scandir, fake_subprocess_run, +) -> None: + """When start_now=False, a `down` marker must be written so + s6-supervise does not auto-start the service on rescan.""" + mgr = S6ServiceManager(scandir=s6_scandir) + mgr.register_profile_gateway("coder", start_now=False) + + svc_dir = s6_scandir / "gateway-coder" + assert svc_dir.is_dir() + assert (svc_dir / "down").is_file(), ( + "start_now=False must write a `down` marker file" + ) + + +def test_s6_register_start_now_true_no_down_marker( + s6_scandir, fake_subprocess_run, +) -> None: + """When start_now=True (default), no `down` marker should exist.""" + mgr = S6ServiceManager(scandir=s6_scandir) + mgr.register_profile_gateway("coder") + + svc_dir = s6_scandir / "gateway-coder" + assert svc_dir.is_dir() + assert not (svc_dir / "down").exists(), ( + "start_now=True must NOT write a `down` marker file" + ) + + def test_s6_register_extra_env_is_quoted(s6_scandir, fake_subprocess_run) -> None: mgr = S6ServiceManager(scandir=s6_scandir) mgr.register_profile_gateway( ``` - **#46293: fix(docker): skip gateway reconciliation in dashboard container (autodetect)**. merged to main · fix tier `verified` - maintainer direction: category `merged`; implication `superseded` - files: `hermes_cli/container_boot.py`, `scripts/release.py`, `tests/hermes_cli/test_container_boot.py` - fix hunks: - `hermes_cli/container_boot.py` ```diff @@ -207,8 +207,15 @@ def _read_container_argv() -> tuple[str, ...]: return tuple(part.decode("utf-8", "replace") for part in raw.split(b"\0") if part) -def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool: - """Return True for Docker commands equivalent to `gateway run`.""" +def _strip_container_argv_prefix(argv: Sequence[str]) -> list[str]: + """Strip the s6/wrapper prefix off PID 1 argv, leaving the hermes args. + + The container PID 1 argv looks like + ``/init /opt/hermes/docker/main-wrapper.sh [args...]`` and + the wrapper re-execs ``hermes ``. Peel ``init`` → + ``main-wrapper.sh`` → ``hermes`` so callers can match on the bare + subcommand. Shared by the legacy-gateway and dashboard role detectors. + """ args = list(argv) if args and Path(args[0]).name == "init": args = args[1:] ``` ```diff @@ -216,11 +223,38 @@ def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool: args = args[1:] if args and Path(args[0]).name == "hermes": args = args[1:] + return args + + +def _is_legacy_gateway_run_request(argv: Sequence[str]) -> bool: + """Return True for Docker commands equivalent to `gateway run`.""" + args = _strip_container_argv_prefix(argv) if "--no-supervise" in args: return False return len(args) >= 2 and args[0] == "gateway" and args[1] == "run" +def _is_dashboard_container(argv: Sequence[str]) -> bool: + """Return True when the container's command is the dashboard. + + A dashboard-only container (``hermes dashboard ...``) never spawns or + supervises per-profile gateways — that is the gateway container's job. + Reconciling profile gateway s6 slots there is not just wasted work: when + the gateway and dashboard containers share a bind-mounted HERMES_HOME, + both race to ``flock()`` the same ``logs/gateways//lock`` files, + producing "Resource busy" failures and an s6-log restart storm. So the + dashboard container skips reconciliation entirely. + + Detected from PID 1 argv (``/proc/1/cmdline``) rather than an operator + flag: the role is a fact about the container's command, not a tunable, + and a flag can be forgotten in a hand-written compose/k8s manifest — + reintroducing the exact storm this prevents. Mirrors the argv handling + in :func:`_is_legacy_gateway_run_request`. + """ + args = _strip_container_argv_prefix(argv) + return bool(args) and args[0] == "dashboard" + + def _read_desired_state(profile_dir: Path) -> str | None: """Read the persisted gateway desired state for reconciliation. ``` ```diff @@ -393,6 +427,22 @@ def _write_reconcile_log( def main() -> int: """Entry point invoked from /etc/cont-init.d/02-reconcile-profiles.""" + # A dashboard-only container never spawns or supervises per-profile + # gateways, so reconciling their s6 slots here is pure waste — and + # actively harmful: when the gateway and dashboard containers share a + # bind-mounted HERMES_HOME, both race to flock() the same s6-log lock + # files under logs/gateways//lock, producing "Resource busy" + # failures and a restart storm. Detect the role from PID 1 argv and + # skip reconciliation in the dashboard container. No operator flag: + # the role is a fact about the container's command, and a flag can be + # forgotten in a hand-written manifest, reintroducing the storm. + if _is_dashboard_container(_read_container_argv()): + print( + "reconcile: skipping (dashboard container — does not need " + "per-profile gateways)" + ) + return 0 + hermes_home = Path(os.environ.get("HERMES_HOME", "/opt/data")) scandir = Path(os.environ.get("S6_PROFILE_GATEWAY_SCANDIR", "/run/service")) actions = reconcile_profile_gateways( ``` - `scripts/release.py` ```diff @@ -84,6 +84,7 @@ "[redacted:email]": "synapsesx", "[redacted:email]": "itsflownium", "[redacted:email]": "dirtyren", + "[redacted:email]": "895252509", "[redacted:email]": "zxcasongs", "[redacted:email]": "alfred-smith-0", "[redacted:email]": "tangtaizong666", ``` - `tests/hermes_cli/test_container_boot.py` ```diff @@ -708,3 +708,144 @@ def test_profiles_default_subdir_is_skipped_with_warning( assert any( "profiles/default/" in record.message for record in caplog.records ) + + +# --------------------------------------------------------------------------- +# Dashboard-container role detection (skip reconcile on the dashboard) +# --------------------------------------------------------------------------- + + [redacted:email]( + "container_argv", + [ + # Bare subcommand (docker run ... dashboard ...). + ("dashboard",), + ("dashboard", "--host", "127.0.0.1", "--no-open"), + # Through s6 /init + the main-wrapper that re-execs `hermes`. + ("/init", "/opt/hermes/docker/main-wrapper.sh", "dashboard"), + ( + "/init", + "/opt/hermes/docker/main-wrapper.sh", + "dashboard", + "--host", + "127.0.0.1", + "--no-open", + ), + # Wrapper that kept the explicit `hermes` argv0. + ("/init", "/opt/hermes/docker/main-wrapper.sh", "hermes", "dashboard"), + ], +) +def test_is_dashboard_container_true_for_dashboard_argv( + container_argv: tuple[str, ...], +) -> None: + """A dashboard command is detected across every wrapper prefix shape.""" + from hermes_cli.container_boot import _is_dashboard_container + + assert _is_dashboard_container(container_argv) is True + + [redacted:email]( + "container_argv", + [ + (), # empty (/proc/1/cmdline unreadable) — not the dashboard + ("gateway", "run"), + ("/init", "/opt/hermes/docker/main-wrapper.sh", "gateway", "run"), + ("/init", "/opt/hermes/docker/main-wrapper.sh", "hermes", "gateway", "run"), + ("chat",), + # A profile literally named "dashboard" must NOT match — the token + # we key on is the SUBCOMMAND, and `gateway run -p dashboard` is a + # gateway container. + ("gateway", "run", "-p", "dashboard"), + ], +) +def test_is_dashboard_container_false_for_non_dashboard_argv( + container_argv: tuple[str, ...], +) -> None: + """Gateway / other commands (and empty argv) are not the dashboard.""" + from hermes_cli.container_boot import _is_dashboard_container + + assert _is_dashboard_container(container_argv) is False + + +def test_main_skips_reconcile_in_dashboard_container( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """main() must NOT reconcile when PID 1 argv is the dashboard command. + + A running profile is seeded so that, if reconcile ran, it would create + the gateway- slot. Asserting the slot is absent proves the + skip is real, not just a log line. + """ + from hermes_cli import container_boot + + scandir = tmp_path / "run-service"; scandir.mkdir() + _make_profile(tmp_path, "worker", state="running") + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("S6_PROFILE_GATEWAY_SCANDIR", str(scandir)) + monkeypatch.setattr( + container_boot, + "_read_container_argv", + lambda: ("/init", "/opt/hermes/docker/main-wrapper.sh", "dashboard"), + ) + + rc = container_boot.main() + + assert rc == 0 + assert not (scandir / "gateway-worker").exists() + assert not (scandir / "gateway-default").exists() + assert "skipping (dashboard container" in capsys.readouterr().out + + +def test_main_reconciles_in_gateway_container( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """main() reconciles normally when PID 1 argv is the gateway command — + the dashboard skip is scoped strictly to the dashboard role.""" + from hermes_cli import container_boot + + scandir = tmp_path / "run-service"; scandir.mkdir() + _make_profile(tmp_path, "worker", state="running") + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("S6_PROFILE_GATEWAY_SCANDIR", str(scandir)) + monkeypatch.setattr( + container_boot, + "_read_container_argv", + lambda: ("/init", "/opt/hermes/docker/main-wrapper.sh", "gateway", "run"), + ) + + rc = container_boot.main() + + assert rc == 0 + # The worker slot was registered + started (prior_state running). + assert (scandir / "gateway-worker").exists() + assert not (scandir / "gateway-worker" / "down").exists() + + +def test_main_ignores_removed_skip_reconcile_env_var( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The legacy HERMES_SKIP_PROFILE_RECONCILE flag is gone: setting it on a + gateway container must NOT suppress reconciliation. Role is decided by + PID 1 argv alone, so a stale flag in someone's manifest is inert.""" + from hermes_cli import container_boot + + scandir = tmp_path / "run-service"; scandir.mkdir() + _make_profile(tmp_path, "worker", state="running") + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setenv("S6_PROFILE_GATEWAY_SCANDIR", str(scandir)) + monkeypatch.setenv("HERMES_SKIP_PROFILE_RECONCILE", "1") + monkeypatch.setattr( + container_boot, + "_read_container_argv", + lambda: ("/init", "/opt/hermes/docker/main-wrapper.sh", "gateway", "run"), + ) + + rc = container_boot.main() + + assert rc == 0 + # Reconcile still ran despite the stale env var. + assert (scandir / "gateway-worker").exists() ``` - **#45968: fix(s6): prevent profile create from auto-starting gateway service**. closed without merge - maintainer direction: category `superseded_by_pr`; implication `superseded`; refs `40d7c264f`, `#46266` - **#46020: fix(profiles): register s6 gateway without auto-starting (#45963)**. closed without merge - maintainer direction: category `superseded_by_pr`; implication `superseded`; refs `40d7c264f`, `#46266`, `#45963` ## Fix status - Merged to main in PR #46266 (cited above). No release data is tracked. Verify downstream. - Fix last verified (merged) `2026-06-15`.