# NousResearch/hermes-agent · Tokenless profile gateway s6 crash-loop and disk fill ## Open now - Nothing open on GitHub. A bug where tokenless profile gateways crash-loop under s6 fills /tmp with tirith directories, exhausting disk space. The associated fixes address exit codes, finish scripts, restart suppression, env override reading, and s6 registration to stop the loop. ## TL;DR - 3 member(s): 0 open issue(s), 1 closed issue(s), 0 open PR(s), 1 merged PR(s), 1 PR(s) closed without merging. - A fix was merged to main in PR #51698. Verify it addresses your case. - **Recommended:** read PR #51698 (and any competing PRs below) before filing. Confirm the merged change covers your symptom. ## Maintainer resolution verdict - Internal-tracker verdict over this cluster's issue members: `resolved` · deciding PR(s): `pr-51698`. This is distinct from the fix status above. Weigh it yourself. ## How this cluster was grouped - Link evidence (`STRUCTURAL_SPANNING`): every member is connected by structural evidence: shared changed files, or a closing/superseded reference. - Pairwise link density `0.67`: the share of member pairs that carry a link, counting a structural link in full and a relevance-only link at 0.36. `1.00` means every pair is linked. ## Closed / duplicate issues - #51228: \[Bug\]: Tokenless profile gateways crash-loop under s6, filling /tmp with tirith dirs (23GB / disk full in hours) ## Prior & competing PRs - **#51357: fix(gateway): exit 78 (EX_CONFIG) on fatal startup errors, s6 finish stops restart loop**. closed without merge · fix tier `suggested` - maintainer direction: category `uncategorized`; implication `needs_llm` - _suggested-tier fix detail withheld; re-request with `?tier=all` to include it._ - **#51698: fix(gateway): stop tokenless-profile s6 crash-loop (exit 78 + finish script + propagation)**. merged to main · fix tier `verified` - maintainer direction: category `merged` - files: `gateway/restart.py`, `gateway/run.py`, `hermes_cli/container_boot.py`, `hermes_cli/service_manager.py`, `scripts/release.py`, `tests/gateway/test_runner_startup_failures.py`, `tests/hermes_cli/test_container_boot.py`, `tests/hermes_cli/test_service_manager.py` - fix hunks: - `gateway/restart.py` ```diff @@ -6,6 +6,12 @@ # the gateway after a graceful drain/reload path completes. GATEWAY_SERVICE_RESTART_EXIT_CODE = 75 +# EX_CONFIG from sysexits.h — fatal configuration error (e.g. token +# collision, no messaging platforms). The s6 finish script translates +# this into exit 125 (permanent failure) so the supervisor stops +# restarting the gateway. See #51228. +GATEWAY_FATAL_CONFIG_EXIT_CODE = 78 + DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT = float( DEFAULT_CONFIG["agent"]["restart_drain_timeout"] ) ``` - `gateway/run.py` ```diff @@ -1701,6 +1701,7 @@ def _profile_runtime_scope(profile_home: "Path"): ) from gateway.restart import ( DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, + GATEWAY_FATAL_CONFIG_EXIT_CODE, GATEWAY_SERVICE_RESTART_EXIT_CODE, parse_restart_drain_timeout, ) ``` ```diff @@ -5747,6 +5748,7 @@ async def start(self) -> bool: write_runtime_status(gateway_state="startup_failed", exit_reason=reason) except Exception: pass + self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE self._request_clean_exit(reason) self._startup_restore_in_progress = False return True ``` ```diff @@ -5762,6 +5764,7 @@ async def start(self) -> bool: write_runtime_status(gateway_state="startup_failed", exit_reason=reason) except Exception: pass + self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE self._request_clean_exit(reason) self._startup_restore_in_progress = False return True ``` ```diff @@ -17780,6 +17783,15 @@ def restart_signal_handler(): if runner.should_exit_cleanly: if runner.exit_reason: logger.error("Gateway exiting cleanly: %s", runner.exit_reason) + # A clean exit that carries an explicit exit code (e.g. a fatal + # config error stamped with GATEWAY_FATAL_CONFIG_EXIT_CODE) must + # propagate that code to the process so the s6 finish script can + # translate it (78 → 125) and stop the supervisor restart loop. + # Without this, the early `return True` below makes main() exit 0, + # the finish script's `[ "$1" = "78" ]` check never matches, and + # s6 crash-loops the gateway anyway (#51228). + if runner.exit_code is not None: + raise SystemExit(runner.exit_code) return True # Start the background cron scheduler via the resolved provider so ``` - `hermes_cli/container_boot.py` ```diff @@ -398,6 +398,10 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None: run.write_text(S6ServiceManager._render_run_script(profile, extra_env={})) run.chmod(0o755) + finish = tmp_dir / "finish" + finish.write_text(S6ServiceManager._render_finish_script()) + finish.chmod(0o755) + # Persistent log rotation (OQ8-C). log_subdir = tmp_dir / "log" log_subdir.mkdir() ``` - `hermes_cli/service_manager.py` ```diff @@ -709,6 +709,30 @@ def _render_run_script( lines.append(f"exec s6-setuidgid hermes {gateway_cmd}") return "\n".join(lines) + "\n" + @staticmethod + def _render_finish_script() -> str: + """Generate the finish script for a profile-gateway s6 service. + + When the gateway exits with EX_CONFIG (78) — a fatal + configuration error such as a token collision or no messaging + platforms — we tell s6-supervise to stop restarting by exiting + 125 (permanent failure). Any other exit code lets s6 restart + normally. See #51228. + """ + from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE + + code = GATEWAY_FATAL_CONFIG_EXIT_CODE + return ( + "#!/command/with-contenv sh\n" + "# shellcheck shell=sh\n" + "# $1 = exit code from the run script.\n" + f"# Exit {code} (EX_CONFIG) = fatal config error — don't restart.\n" + f'if [ "$1" = "{code}" ]; then\n' + " exit 125\n" + "fi\n" + "exit 0\n" + ) + @staticmethod def _render_log_run(profile: str) -> str: """Generate the log/run script for a profile-gateway service. ``` ```diff @@ -956,6 +980,10 @@ def register_profile_gateway( run_path.write_text(run_script) run_path.chmod(0o755) + finish_path = tmp_dir / "finish" + finish_path.write_text(self._render_finish_script()) + finish_path.chmod(0o755) + # Persistent log rotation (OQ8-C). log_subdir = tmp_dir / "log" log_subdir.mkdir() ``` - `scripts/release.py` ```diff @@ -130,6 +130,7 @@ "290859878+synapsesx@users.noreply.github.com": "synapsesx", "157689911+itsflownium@users.noreply.github.com": "itsflownium", "dirtyren@users.noreply.github.com": "dirtyren", + "[redacted:email]": "francescomucio", "[redacted:email]": "tkwong", "[redacted:email]": "buihongduc132", "[redacted:email]": "EtherAura", # PR #45205 salvage (Linux in-app update relaunch / GUI-skew terminal state) ``` - `tests/gateway/test_runner_startup_failures.py` ```diff @@ -3,6 +3,7 @@ from gateway.config import GatewayConfig, Platform, PlatformConfig from gateway.platforms.base import BasePlatformAdapter +from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE from gateway.run import GatewayRunner from gateway.status import read_runtime_status ``` ```diff @@ -151,6 +152,7 @@ def __init__(self, config): self.config = config self.should_exit_cleanly = True self.exit_reason = None + self.exit_code = None self.adapters = {} async def start(self): ``` ```diff @@ -185,6 +187,7 @@ def __init__(self, config): self.config = config self.should_exit_cleanly = True self.exit_reason = None + self.exit_code = None self.adapters = {} async def start(self): ``` ```diff @@ -333,6 +336,7 @@ def __init__(self, config): self.config = config self.should_exit_cleanly = True self.exit_reason = None + self.exit_code = None self.adapters = {} async def start(self): ``` ```diff @@ -458,6 +462,94 @@ async def test_runner_degrades_gracefully_when_all_adapters_missing(monkeypatch, ), "Expected degraded-mode warning when all adapters are missing" +class _NonRetryableFailureAdapter(BasePlatformAdapter): + """Simulates a fatal config error like token collision.""" + def __init__(self): + super().__init__(PlatformConfig(enabled=True, token="***"), Platform.DISCORD) + + async def connect(self) -> bool: + self._set_fatal_error( + "discord-bot-token_lock", + "Discord bot token already in use (PID 999). Stop the other gateway first.", + retryable=False, + ) + return False + + async def disconnect(self) -> None: + self._mark_disconnected() + + async def send(self, chat_id, content, reply_to=None, metadata=None): + raise NotImplementedError + + async def get_chat_info(self, chat_id): + return {"id": chat_id} + + [redacted:email] +async def test_runner_exits_with_ex_config_on_nonretryable_startup_error(monkeypatch, tmp_path): + """Non-retryable startup errors (token collision, no platforms) must + set exit_code to 78 (EX_CONFIG) so the s6 finish script can translate + it to exit 125 (permanent failure). See #51228.""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + config = GatewayConfig( + platforms={ + Platform.DISCORD: PlatformConfig(enabled=True, token="***") + }, + sessions_dir=tmp_path / "sessions", + ) + runner = GatewayRunner(config) + + monkeypatch.setattr(runner, "_create_adapter", lambda platform, platform_config: _NonRetryableFailureAdapter()) + + ok = await runner.start() + + assert ok is True # start() returns True (clean exit requested) + assert runner.should_exit_cleanly is True + assert runner.exit_code == GATEWAY_FATAL_CONFIG_EXIT_CODE + state = read_runtime_status() + assert state["gateway_state"] == "startup_failed" + + [redacted:email] +async def test_start_gateway_propagates_fatal_config_exit_code(monkeypatch, tmp_path): + """A clean exit carrying GATEWAY_FATAL_CONFIG_EXIT_CODE must surface as a + process-level SystemExit(78) — NOT a truthy return — so main() exits 78 + and the s6 finish script can translate it to 125 (no restart). + + This guards the propagation gap: runner.start() stamps exit_code=78 and + requests a clean exit, but start_gateway()'s clean-exit branch used to + `return True` before the SystemExit(exit_code) site, so main() exited 0 + and s6 crash-looped anyway (#51228).""" + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + + class _FatalConfigRunner: + def __init__(self, config): + self.config = config + self.should_exit_cleanly = True + self.exit_reason = "discord: Discord bot token already in use" + self.exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE + self.adapters = {} + + async def start(self): + return True + + async def stop(self): + return None + + monkeypatch.setattr("gateway.status.get_running_pid", lambda: None) + monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None) + monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path) + monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None) + monkeypatch.setattr("gateway.run.GatewayRunner", _FatalConfigRunner) + + from gateway.run import start_gateway + + with pytest.raises(SystemExit) as exc_info: + await start_gateway(config=GatewayConfig(), replace=False, verbosity=0) + + assert exc_info.value.code == GATEWAY_FATAL_CONFIG_EXIT_CODE + + def test_runner_warns_when_docker_gateway_lacks_explicit_output_mount(monkeypatch, tmp_path, caplog): monkeypatch.setenv("HERMES_HOME", str(tmp_path)) monkeypatch.setenv("TERMINAL_ENV", "docker") ``` - `tests/hermes_cli/test_container_boot.py` ```diff @@ -128,6 +128,24 @@ def test_running_profile_is_registered_and_autostarted(tmp_path: Path) -> None: assert not (svc / "down").exists() +def test_registered_profile_has_finish_script(tmp_path: Path) -> None: + """The finish script must be written so s6 stops restarting on + fatal config errors (exit 78 → exit 125). See #51228.""" + scandir = tmp_path / "run-service"; scandir.mkdir() + _make_profile(tmp_path, "coder", state="running") + + reconcile_profile_gateways( + hermes_home=tmp_path, scandir=scandir, dry_run=False, + ) + + finish = scandir / "gateway-coder" / "finish" + assert finish.exists() + assert finish.stat().st_mode & 0o111 # executable + text = finish.read_text() + assert "78" in text + assert "125" in text + + def test_stopped_profile_is_registered_but_not_started(tmp_path: Path) -> None: scandir = tmp_path / "run-service"; scandir.mkdir() _make_profile(tmp_path, "writer", state="stopped") ``` - `tests/hermes_cli/test_service_manager.py` ```diff @@ -673,6 +673,30 @@ def test_render_run_script_uses_replace_to_take_over_stale_holder() -> None: ) +def test_render_finish_script_exits_125_on_ex_config() -> None: + """The finish script must translate exit 78 (EX_CONFIG) into exit 125 + (permanent failure) so s6 stops restarting on fatal config errors. + See #51228.""" + text = S6ServiceManager._render_finish_script() + assert '[ "$1" = "78" ]' in text + assert "exit 125" in text + assert "exit 0" in text + + +def test_s6_register_writes_finish_script( + s6_scandir, fake_subprocess_run, +) -> None: + """The finish script must be written alongside the run script.""" + mgr = S6ServiceManager(scandir=s6_scandir) + mgr.register_profile_gateway("coder") + + finish_path = s6_scandir / "gateway-coder" / "finish" + assert finish_path.is_file() + assert finish_path.stat().st_mode & 0o111 # executable + assert "78" in finish_path.read_text() + assert "125" in finish_path.read_text() + + def test_s6_register_rejects_invalid_profile_name(s6_scandir) -> None: mgr = S6ServiceManager(scandir=s6_scandir) with pytest.raises(ValueError): ``` ## Fix status - Merged to main in PR #51698 (cited above). No release data is tracked. Verify downstream. - Fix last verified (merged) `2026-06-24`.