Tokenless profile gateway s6 crash-loop and disk fill

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #51698 fix(gateway): stop tokenless-profile s6 crash-loop (exit 78 + finish script + propagation)

  merged to main

  files: gateway/restart.py, gateway/run.py, hermes_cli/container_boot.py, hermes_cli/service_manager.py, scripts/release.py, tests/gateway/test_runner_startup_failures.py, tests/hermes_cli/test_container_boot.py, tests/hermes_cli/test_service_manager.py

  Show the diff

  • gateway/restart.py

    @@ -6,6 +6,12 @@
     # the gateway after a graceful drain/reload path completes.
     GATEWAY_SERVICE_RESTART_EXIT_CODE = 75
     
    +# EX_CONFIG from sysexits.h — fatal configuration error (e.g. token
    +# collision, no messaging platforms).  The s6 finish script translates
    +# this into exit 125 (permanent failure) so the supervisor stops
    +# restarting the gateway.  See #51228.
    +GATEWAY_FATAL_CONFIG_EXIT_CODE = 78
    +
     DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT = float(
         DEFAULT_CONFIG["agent"]["restart_drain_timeout"]
     )

  • gateway/run.py

    @@ -1701,6 +1701,7 @@ def _profile_runtime_scope(profile_home: "Path"):
     )
     from gateway.restart import (
         DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,
    +    GATEWAY_FATAL_CONFIG_EXIT_CODE,
         GATEWAY_SERVICE_RESTART_EXIT_CODE,
         parse_restart_drain_timeout,
     )
    @@ -5747,6 +5748,7 @@ async def start(self) -> bool:
                     write_runtime_status(gateway_state="startup_failed", exit_reason=reason)
                 except Exception:
                     pass
    +            self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE
                 self._request_clean_exit(reason)
                 self._startup_restore_in_progress = False
                 return True
    @@ -5762,6 +5764,7 @@ async def start(self) -> bool:
                         write_runtime_status(gateway_state="startup_failed", exit_reason=reason)
                     except Exception:
                         pass
    +                self._exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE
                     self._request_clean_exit(reason)
                     self._startup_restore_in_progress = False
                     return True
    @@ -17780,6 +17783,15 @@ def restart_signal_handler():
         if runner.should_exit_cleanly:
             if runner.exit_reason:
                 logger.error("Gateway exiting cleanly: %s", runner.exit_reason)
    +        # A clean exit that carries an explicit exit code (e.g. a fatal
    +        # config error stamped with GATEWAY_FATAL_CONFIG_EXIT_CODE) must
    +        # propagate that code to the process so the s6 finish script can
    +        # translate it (78 → 125) and stop the supervisor restart loop.
    +        # Without this, the early `return True` below makes main() exit 0,
    +        # the finish script's `[ "$1" = "78" ]` check never matches, and
    +        # s6 crash-loops the gateway anyway (#51228).
    +        if runner.exit_code is not None:
    +            raise SystemExit(runner.exit_code)
             return True
         
         # Start the background cron scheduler via the resolved provider so

  • hermes_cli/container_boot.py

    @@ -398,6 +398,10 @@ def _register_service(scandir: Path, profile: str, *, start: bool) -> None:
             run.write_text(S6ServiceManager._render_run_script(profile, extra_env={}))
             run.chmod(0o755)
     
    +        finish = tmp_dir / "finish"
    +        finish.write_text(S6ServiceManager._render_finish_script())
    +        finish.chmod(0o755)
    +
             # Persistent log rotation (OQ8-C).
             log_subdir = tmp_dir / "log"
             log_subdir.mkdir()

  • hermes_cli/service_manager.py

    @@ -709,6 +709,30 @@ def _render_run_script(
             lines.append(f"exec s6-setuidgid hermes {gateway_cmd}")
             return "\n".join(lines) + "\n"
     
    +    @staticmethod
    +    def _render_finish_script() -> str:
    +        """Generate the finish script for a profile-gateway s6 service.
    +
    +        When the gateway exits with EX_CONFIG (78) — a fatal
    +        configuration error such as a token collision or no messaging
    +        platforms — we tell s6-supervise to stop restarting by exiting
    +        125 (permanent failure).  Any other exit code lets s6 restart
    +        normally.  See #51228.
    +        """
    +        from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
    +
    +        code = GATEWAY_FATAL_CONFIG_EXIT_CODE
    +        return (
    +            "#!/command/with-contenv sh\n"
    +            "# shellcheck shell=sh\n"
    +            "# $1 = exit code from the run script.\n"
    +            f"# Exit {code} (EX_CONFIG) = fatal config error — don't restart.\n"
    +            f'if [ "$1" = "{code}" ]; then\n'
    +            "  exit 125\n"
    +            "fi\n"
    +            "exit 0\n"
    +        )
    +
         @staticmethod
         def _render_log_run(profile: str) -> str:
             """Generate the log/run script for a profile-gateway service.
    @@ -956,6 +980,10 @@ def register_profile_gateway(
                 run_path.write_text(run_script)
                 run_path.chmod(0o755)
     
    +            finish_path = tmp_dir / "finish"
    +            finish_path.write_text(self._render_finish_script())
    +            finish_path.chmod(0o755)
    +
                 # Persistent log rotation (OQ8-C).
                 log_subdir = tmp_dir / "log"
                 log_subdir.mkdir()

  • scripts/release.py

    @@ -130,6 +130,7 @@
         "[email protected]": "synapsesx",
         "[email protected]": "itsflownium",
         "[email protected]": "dirtyren",
    +    "[redacted:email]": "francescomucio",
         "[redacted:email]": "tkwong",
         "[redacted:email]": "buihongduc132",
         "[redacted:email]": "EtherAura",  # PR #45205 salvage (Linux in-app update relaunch / GUI-skew terminal state)

  • tests/gateway/test_runner_startup_failures.py

    @@ -3,6 +3,7 @@
     
     from gateway.config import GatewayConfig, Platform, PlatformConfig
     from gateway.platforms.base import BasePlatformAdapter
    +from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE
     from gateway.run import GatewayRunner
     from gateway.status import read_runtime_status
     
    @@ -151,6 +152,7 @@ def __init__(self, config):
                 self.config = config
                 self.should_exit_cleanly = True
                 self.exit_reason = None
    +            self.exit_code = None
                 self.adapters = {}
     
             async def start(self):
    @@ -185,6 +187,7 @@ def __init__(self, config):
                 self.config = config
                 self.should_exit_cleanly = True
                 self.exit_reason = None
    +            self.exit_code = None
                 self.adapters = {}
     
             async def start(self):
    @@ -333,6 +336,7 @@ def __init__(self, config):
                 self.config = config
                 self.should_exit_cleanly = True
                 self.exit_reason = None
    +            self.exit_code = None
                 self.adapters = {}
     
             async def start(self):
    @@ -458,6 +462,94 @@ async def test_runner_degrades_gracefully_when_all_adapters_missing(monkeypatch,
         ), "Expected degraded-mode warning when all adapters are missing"
     
     
    +class _NonRetryableFailureAdapter(BasePlatformAdapter):
    +    """Simulates a fatal config error like token collision."""
    +    def __init__(self):
    +        super().__init__(PlatformConfig(enabled=True, token="***"), Platform.DISCORD)
    +
    +    async def connect(self) -> bool:
    +        self._set_fatal_error(
    +            "discord-bot-token_lock",
    +            "Discord bot token already in use (PID 999). Stop the other gateway first.",
    +            retryable=False,
    +        )
    +        return False
    +
    +    async def disconnect(self) -> None:
    +        self._mark_disconnected()
    +
    +    async def send(self, chat_id, content, reply_to=None, metadata=None):
    +        raise NotImplementedError
    +
    +    async def get_chat_info(self, chat_id):
    +        return {"id": chat_id}
    +
    +
    [redacted:email]
    +async def test_runner_exits_with_ex_config_on_nonretryable_startup_error(monkeypatch, tmp_path):
    +    """Non-retryable startup errors (token collision, no platforms) must
    +    set exit_code to 78 (EX_CONFIG) so the s6 finish script can translate
    +    it to exit 125 (permanent failure).  See #51228."""
    +    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
    +    config = GatewayConfig(
    +        platforms={
    +            Platform.DISCORD: PlatformConfig(enabled=True, token="***")
    +        },
    +        sessions_dir=tmp_path / "sessions",
    +    )
    +    runner = GatewayRunner(config)
    +
    +    monkeypatch.setattr(runner, "_create_adapter", lambda platform, platform_config: _NonRetryableFailureAdapter())
    +
    +    ok = await runner.start()
    +
    +    assert ok is True  # start() returns True (clean exit requested)
    +    assert runner.should_exit_cleanly is True
    +    assert runner.exit_code == GATEWAY_FATAL_CONFIG_EXIT_CODE
    +    state = read_runtime_status()
    +    assert state["gateway_state"] == "startup_failed"
    +
    +
    [redacted:email]
    +async def test_start_gateway_propagates_fatal_config_exit_code(monkeypatch, tmp_path):
    +    """A clean exit carrying GATEWAY_FATAL_CONFIG_EXIT_CODE must surface as a
    +    process-level SystemExit(78) — NOT a truthy return — so main() exits 78
    +    and the s6 finish script can translate it to 125 (no restart).
    +
    +    This guards the propagation gap: runner.start() stamps exit_code=78 and
    +    requests a clean exit, but start_gateway()'s clean-exit branch used to
    +    `return True` before the SystemExit(exit_code) site, so main() exited 0
    +    and s6 crash-looped anyway (#51228)."""
    +    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
    +
    +    class _FatalConfigRunner:
    +        def __init__(self, config):
    +            self.config = config
    +            self.should_exit_cleanly = True
    +            self.exit_reason = "discord: Discord bot token already in use"
    +            self.exit_code = GATEWAY_FATAL_CONFIG_EXIT_CODE
    +            self.adapters = {}
    +
    +        async def start(self):
    +            return True
    +
    +        async def stop(self):
    +            return None
    +
    +    monkeypatch.setattr("gateway.status.get_running_pid", lambda: None)
    +    monkeypatch.setattr("tools.skills_sync.sync_skills", lambda quiet=True: None)
    +    monkeypatch.setattr("hermes_logging.setup_logging", lambda hermes_home, mode: tmp_path)
    +    monkeypatch.setattr("hermes_logging._add_rotating_handler", lambda *args, **kwargs: None)
    +    monkeypatch.setattr("gateway.run.GatewayRunner", _FatalConfigRunner)
    +
    +    from gateway.run import start_gateway
    +
    +    with pytest.raises(SystemExit) as exc_info:
    +        await start_gateway(config=GatewayConfig(), replace=False, verbosity=0)
    +
    +    assert exc_info.value.code == GATEWAY_FATAL_CONFIG_EXIT_CODE
    +
    +
     def test_runner_warns_when_docker_gateway_lacks_explicit_output_mount(monkeypatch, tmp_path, caplog):
         monkeypatch.setenv("HERMES_HOME", str(tmp_path))
         monkeypatch.setenv("TERMINAL_ENV", "docker")

  • tests/hermes_cli/test_container_boot.py

    @@ -128,6 +128,24 @@ def test_running_profile_is_registered_and_autostarted(tmp_path: Path) -> None:
         assert not (svc / "down").exists()
     
     
    +def test_registered_profile_has_finish_script(tmp_path: Path) -> None:
    +    """The finish script must be written so s6 stops restarting on
    +    fatal config errors (exit 78 → exit 125).  See #51228."""
    +    scandir = tmp_path / "run-service"; scandir.mkdir()
    +    _make_profile(tmp_path, "coder", state="running")
    +
    +    reconcile_profile_gateways(
    +        hermes_home=tmp_path, scandir=scandir, dry_run=False,
    +    )
    +
    +    finish = scandir / "gateway-coder" / "finish"
    +    assert finish.exists()
    +    assert finish.stat().st_mode & 0o111  # executable
    +    text = finish.read_text()
    +    assert "78" in text
    +    assert "125" in text
    +
    +
     def test_stopped_profile_is_registered_but_not_started(tmp_path: Path) -> None:
         scandir = tmp_path / "run-service"; scandir.mkdir()
         _make_profile(tmp_path, "writer", state="stopped")

  • tests/hermes_cli/test_service_manager.py

    @@ -673,6 +673,30 @@ def test_render_run_script_uses_replace_to_take_over_stale_holder() -> None:
         )
     
     
    +def test_render_finish_script_exits_125_on_ex_config() -> None:
    +    """The finish script must translate exit 78 (EX_CONFIG) into exit 125
    +    (permanent failure) so s6 stops restarting on fatal config errors.
    +    See #51228."""
    +    text = S6ServiceManager._render_finish_script()
    +    assert '[ "$1" = "78" ]' in text
    +    assert "exit 125" in text
    +    assert "exit 0" in text
    +
    +
    +def test_s6_register_writes_finish_script(
    +    s6_scandir, fake_subprocess_run,
    +) -> None:
    +    """The finish script must be written alongside the run script."""
    +    mgr = S6ServiceManager(scandir=s6_scandir)
    +    mgr.register_profile_gateway("coder")
    +
    +    finish_path = s6_scandir / "gateway-coder" / "finish"
    +    assert finish_path.is_file()
    +    assert finish_path.stat().st_mode & 0o111  # executable
    +    assert "78" in finish_path.read_text()
    +    assert "125" in finish_path.read_text()
    +
    +
     def test_s6_register_rejects_invalid_profile_name(s6_scandir) -> None:
         mgr = S6ServiceManager(scandir=s6_scandir)
         with pytest.raises(ValueError):