{"repo":"NousResearch/hermes-agent","public_id":"51228","display_label":"Tokenless profile gateway s6 crash-loop and disk fill","summary":"A bug where tokenless profile gateways crash-loop under s6 fills /tmp with tirith directories, exhausting disk space. The associated fixes address exit codes, finish scripts, restart suppression, env override reading, and s6 registration to stop the loop.","subsystem":null,"counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":51698,"merged_at":"2026-06-24T06:34:52+00:00"},"resolution":{"state":"resolved","by":"pr-51698"},"freshness":{"state":"verified","last_verified_at":"2026-06-24T06:34:52+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":51228,"pr_number":51698,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-24T06:34:52+00:00"}],"members":[{"kind":"issue","number":51228,"title":"[Bug]: Tokenless profile gateways crash-loop under s6, filling /tmp with tirith dirs (23GB / disk full in hours)","url":null,"state":"closed"},{"kind":"pr","number":51357,"title":"fix(gateway): exit 78 (EX_CONFIG) on fatal startup errors, s6 finish stops restart loop","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":51698,"title":"fix(gateway): stop tokenless-profile s6 crash-loop (exit 78 + finish script + propagation)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/restart.py","gateway/run.py","hermes_cli/container_boot.py","hermes_cli/service_manager.py","scripts/release.py","tests/gateway/test_runner_startup_failures.py","tests/hermes_cli/test_container_boot.py","tests/hermes_cli/test_service_manager.py"],"fix_hunks":[{"file":"gateway/restart.py","hunks":["@@ -6,6 +6,12 @@\n # the gateway after a graceful drain/reload path completes.\n GATEWAY_SERVICE_RESTART_EXIT_CODE = 75\n \n+# EX_CONFIG from sysexits.h — fatal configuration error (e.g. token\n+# collision, no messaging platforms).  The s6 finish script translates\n+# this into exit 125 (permanent failure) so the supervisor stops\n+# restarting the gateway.  See #51228.\n+GATEWAY_FATAL_CONFIG_EXIT_CODE = 78\n+\n DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT = float(\n     DEFAULT_CONFIG[\"agent\"][\"restart_drain_timeout\"]\n )"]},{"file":"gateway/run.py","hunks":["@@ -1701,6 +1701,7 @@ def _profile_runtime_scope(profile_home: \"Path\"):\n )\n from gateway.restart import (\n     DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,\n+    GATEWAY_FATAL_CONFIG_EXIT_CODE,\n     GATEWAY_SERVICE_RESTART_EXIT_CODE,\n     parse_restart_drain_timeout,\n )","@@ -5747,6 +5748,7 @@ async def start(self) -> bool:\n                 write_runtime_status(gateway_state=\"startup_failed\", exit_reason=reason)\n             except Exception:\n                 pass\n+            self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE\n             self._request_clean_exit(reason)\n             self._startup_restore_in_progress = False\n             return True","@@ -5762,6 +5764,7 @@ async def start(self) -> bool:\n                     write_runtime_status(gateway_state=\"startup_failed\", exit_reason=reason)\n                 except Exception:\n                     pass\n+                self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE\n                 self._request_clean_exit(reason)\n                 self._startup_restore_in_progress = False\n                 return True","@@ -17780,6 +17783,15 @@ def restart_signal_handler():\n     if runner.should_exit_cleanly:\n         if runner.exit_reason:\n             logger.error(\"Gateway exiting cleanly: %s\", runner.exit_reason)\n+        # A clean exit that carries an explicit exit code (e.g. a fatal\n+        # config error stamped with GATEWAY_FATAL_CONFIG_EXIT_CODE) must\n+        # propagate that code to the process so the s6 finish script can\n+        # translate it (78 → 125) and stop the supervisor restart loop.\n+        # Without this, the early `return True` below makes main() exit 0,\n+        # the finish script's `[ \"$1\" = \"78\" ]` check never matches, and\n+        # s6 crash-loops the gateway anyway (#51228).\n+        if runner.exit_code is not None:\n+            raise SystemExit(runner.exit_code)\n         return True\n     \n     # Start the background cron scheduler via the resolved provider so"]},{"file":"hermes_cli/container_boot.py","hunks":["@@ -398,6 +398,10 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None:\n         run.write_text(S6ServiceManager._render_run_script(profile, extra_env={}))\n         run.chmod(0o755)\n \n+        finish = tmp_dir / \"finish\"\n+        finish.write_text(S6ServiceManager._render_finish_script())\n+        finish.chmod(0o755)\n+\n         # Persistent log rotation (OQ8-C).\n         log_subdir = tmp_dir / \"log\"\n         log_subdir.mkdir()"]},{"file":"hermes_cli/service_manager.py","hunks":["@@ -709,6 +709,30 @@ def _render_run_script(\n         lines.append(f\"exec s6-setuidgid hermes {gateway_cmd}\")\n         return \"\\n\".join(lines) + \"\\n\"\n \n+    @staticmethod\n+    def _render_finish_script() -> str:\n+        \"\"\"Generate the finish script for a profile-gateway s6 service.\n+\n+        When the gateway exits with EX_CONFIG (78) — a fatal\n+        configuration error such as a token collision or no messaging\n+        platforms — we tell s6-supervise to stop restarting by exiting\n+        125 (permanent failure).  Any other exit code lets s6 restart\n+        normally.  See #51228.\n+        \"\"\"\n+        from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE\n+\n+        code = GATEWAY_FATAL_CONFIG_EXIT_CODE\n+        return (\n+            \"#!/command/with-contenv sh\\n\"\n+            \"# shellcheck shell=sh\\n\"\n+            \"# $1 = exit code from the run script.\\n\"\n+            f\"# Exit {code} (EX_CONFIG) = fatal config error — don't restart.\\n\"\n+            f'if [ \"$1\" = \"{code}\" ]; then\\n'\n+            \"  exit 125\\n\"\n+            \"fi\\n\"\n+            \"exit 0\\n\"\n+        )\n+\n     @staticmethod\n     def _render_log_run(profile: str) -> str:\n         \"\"\"Generate the log/run script for a profile-gateway service.","@@ -956,6 +980,10 @@ def register_profile_gateway(\n             run_path.write_text(run_script)\n             run_path.chmod(0o755)\n \n+            finish_path = tmp_dir / \"finish\"\n+            finish_path.write_text(self._render_finish_script())\n+            finish_path.chmod(0o755)\n+\n             # Persistent log rotation (OQ8-C).\n             log_subdir = tmp_dir / \"log\"\n             log_subdir.mkdir()"]},{"file":"scripts/release.py","hunks":["@@ -130,6 +130,7 @@\n     \"290859878+synapsesx@users.noreply.github.com\": \"synapsesx\",\n     \"157689911+itsflownium@users.noreply.github.com\": \"itsflownium\",\n     \"dirtyren@users.noreply.github.com\": \"dirtyren\",\n+    \"[redacted:email]\": \"francescomucio\",\n     \"[redacted:email]\": \"tkwong\",\n     \"[redacted:email]\": \"buihongduc132\",\n     \"[redacted:email]\": \"EtherAura\",  # PR #45205 salvage (Linux in-app update relaunch / GUI-skew terminal state)"]},{"file":"tests/gateway/test_runner_startup_failures.py","hunks":["@@ -3,6 +3,7 @@\n \n from gateway.config import GatewayConfig, Platform, PlatformConfig\n from gateway.platforms.base import BasePlatformAdapter\n+from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE\n from gateway.run import GatewayRunner\n from gateway.status import read_runtime_status\n ","@@ -151,6 +152,7 @@ def __init__(self, config):\n             self.config = config\n             self.should_exit_cleanly = True\n             self.exit_reason = None\n+            self.exit_code = None\n             self.adapters = {}\n \n         async def start(self):","@@ -185,6 +187,7 @@ def __init__(self, config):\n             self.config = config\n             self.should_exit_cleanly = True\n             self.exit_reason = None\n+            self.exit_code = None\n             self.adapters = {}\n \n         async def start(self):","@@ -333,6 +336,7 @@ def __init__(self, config):\n             self.config = config\n             self.should_exit_cleanly = True\n             self.exit_reason = None\n+            self.exit_code = None\n             self.adapters = {}\n \n         async def start(self):","@@ -458,6 +462,94 @@ async def test_runner_degrades_gracefully_when_all_adapters_missing(monkeypatch,\n     ), \"Expected degraded-mode warning when all adapters are missing\"\n \n \n+class _NonRetryableFailureAdapter(BasePlatformAdapter):\n+    \"\"\"Simulates a fatal config error like token collision.\"\"\"\n+    def __init__(self):\n+        super().__init__(PlatformConfig(enabled=True, token=\"***\"), Platform.DISCORD)\n+\n+    async def connect(self) -> bool:\n+        self._set_fatal_error(\n+            \"discord-bot-token_lock\",\n+            \"Discord bot token already in use (PID 999). Stop the other gateway first.\",\n+            retryable=False,\n+        )\n+        return False\n+\n+    async def disconnect(self) -> None:\n+        self._mark_disconnected()\n+\n+    async def send(self, chat_id, content, reply_to=None, metadata=None):\n+        raise NotImplementedError\n+\n+    async def get_chat_info(self, chat_id):\n+        return {\"id\": chat_id}\n+\n+\n[redacted:email]\n+async def test_runner_exits_with_ex_config_on_nonretryable_startup_error(monkeypatch, tmp_path):\n+    \"\"\"Non-retryable startup errors (token collision, no platforms) must\n+    set exit_code to 78 (EX_CONFIG) so the s6 finish script can translate\n+    it to exit 125 (permanent failure).  See #51228.\"\"\"\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+    config = GatewayConfig(\n+        platforms={\n+            Platform.DISCORD: PlatformConfig(enabled=True, token=\"***\")\n+        },\n+        sessions_dir=tmp_path / \"sessions\",\n+    )\n+    runner = GatewayRunner(config)\n+\n+    monkeypatch.setattr(runner, \"_create_adapter\", lambda platform, platform_config: _NonRetryableFailureAdapter())\n+\n+    ok = await runner.start()\n+\n+    assert ok is True  # start() returns True (clean exit requested)\n+    assert runner.should_exit_cleanly is True\n+    assert runner.exit_code == GATEWAY_FATAL_CONFIG_EXIT_CODE\n+    state = read_runtime_status()\n+    assert state[\"gateway_state\"] == \"startup_failed\"\n+\n+\n[redacted:email]\n+async def test_start_gateway_propagates_fatal_config_exit_code(monkeypatch, tmp_path):\n+    \"\"\"A clean exit carrying GATEWAY_FATAL_CONFIG_EXIT_CODE must surface as a\n+    process-level SystemExit(78) — NOT a truthy return — so main() exits 78\n+    and the s6 finish script can translate it to 125 (no restart).\n+\n+    This guards the propagation gap: runner.start() stamps exit_code=78 and\n+    requests a clean exit, but start_gateway()'s clean-exit branch used to\n+    `return True` before the SystemExit(exit_code) site, so main() exited 0\n+    and s6 crash-looped anyway (#51228).\"\"\"\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+\n+    class _FatalConfigRunner:\n+        def __init__(self, config):\n+            self.config = config\n+            self.should_exit_cleanly = True\n+            self.exit_reason = \"discord: Discord bot token already in use\"\n+            self.exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE\n+            self.adapters = {}\n+\n+        async def start(self):\n+            return True\n+\n+        async def stop(self):\n+            return None\n+\n+    monkeypatch.setattr(\"gateway.status.get_running_pid\", lambda: None)\n+    monkeypatch.setattr(\"tools.skills_sync.sync_skills\", lambda quiet=True: None)\n+    monkeypatch.setattr(\"hermes_logging.setup_logging\", lambda hermes_home, mode: tmp_path)\n+    monkeypatch.setattr(\"hermes_logging._add_rotating_handler\", lambda *args, **kwargs: None)\n+    monkeypatch.setattr(\"gateway.run.GatewayRunner\", _FatalConfigRunner)\n+\n+    from gateway.run import start_gateway\n+\n+    with pytest.raises(SystemExit) as exc_info:\n+        await start_gateway(config=GatewayConfig(), replace=False, verbosity=0)\n+\n+    assert exc_info.value.code == GATEWAY_FATAL_CONFIG_EXIT_CODE\n+\n+\n def test_runner_warns_when_docker_gateway_lacks_explicit_output_mount(monkeypatch, tmp_path, caplog):\n     monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n     monkeypatch.setenv(\"TERMINAL_ENV\", \"docker\")"]},{"file":"tests/hermes_cli/test_container_boot.py","hunks":["@@ -128,6 +128,24 @@ def test_running_profile_is_registered_and_autostarted(tmp_path: Path) -> None:\n     assert not (svc / \"down\").exists()\n \n \n+def test_registered_profile_has_finish_script(tmp_path: Path) -> None:\n+    \"\"\"The finish script must be written so s6 stops restarting on\n+    fatal config errors (exit 78 → exit 125).  See #51228.\"\"\"\n+    scandir = tmp_path / \"run-service\"; scandir.mkdir()\n+    _make_profile(tmp_path, \"coder\", state=\"running\")\n+\n+    reconcile_profile_gateways(\n+        hermes_home=tmp_path, scandir=scandir, dry_run=False,\n+    )\n+\n+    finish = scandir / \"gateway-coder\" / \"finish\"\n+    assert finish.exists()\n+    assert finish.stat().st_mode & 0o111  # executable\n+    text = finish.read_text()\n+    assert \"78\" in text\n+    assert \"125\" in text\n+\n+\n def test_stopped_profile_is_registered_but_not_started(tmp_path: Path) -> None:\n     scandir = tmp_path / \"run-service\"; scandir.mkdir()\n     _make_profile(tmp_path, \"writer\", state=\"stopped\")"]},{"file":"tests/hermes_cli/test_service_manager.py","hunks":["@@ -673,6 +673,30 @@ def test_render_run_script_uses_replace_to_take_over_stale_holder() -> None:\n     )\n \n \n+def test_render_finish_script_exits_125_on_ex_config() -> None:\n+    \"\"\"The finish script must translate exit 78 (EX_CONFIG) into exit 125\n+    (permanent failure) so s6 stops restarting on fatal config errors.\n+    See #51228.\"\"\"\n+    text = S6ServiceManager._render_finish_script()\n+    assert '[ \"$1\" = \"78\" ]' in text\n+    assert \"exit 125\" in text\n+    assert \"exit 0\" in text\n+\n+\n+def test_s6_register_writes_finish_script(\n+    s6_scandir, fake_subprocess_run,\n+) -> None:\n+    \"\"\"The finish script must be written alongside the run script.\"\"\"\n+    mgr = S6ServiceManager(scandir=s6_scandir)\n+    mgr.register_profile_gateway(\"coder\")\n+\n+    finish_path = s6_scandir / \"gateway-coder\" / \"finish\"\n+    assert finish_path.is_file()\n+    assert finish_path.stat().st_mode & 0o111  # executable\n+    assert \"78\" in finish_path.read_text()\n+    assert \"125\" in finish_path.read_text()\n+\n+\n def test_s6_register_rejects_invalid_profile_name(s6_scandir) -> None:\n     mgr = S6ServiceManager(scandir=s6_scandir)\n     with pytest.raises(ValueError):\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}