macOS launchd gateway integration bugs and fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #40581 fix(gateway): user launchd domain + Background session, detached fallback (macOS 26)

  merged to main

  files: hermes_cli/gateway.py, tests/hermes_cli/test_gateway_service.py

  Show the diff

  • hermes_cli/gateway.py

    @@ -3000,7 +3000,112 @@ def get_launchd_label() -> str:
     
     
     def _launchd_domain() -> str:
    -    return f"gui/{os.getuid()}"  # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
    +    # The `user/<uid>` domain (vs the older `gui/<uid>`) is reachable from
    +    # non-Aqua/background sessions (SSH, headless, login items) and is the only
    +    # one that supports service management on macOS 26+. `gui/<uid>` returns
    +    # error 125 ("Domain does not support specified action") there. See #23387.
    +    return f"user/{os.getuid()}"  # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
    +
    +
    +# On macOS, exit code 125 ("Domain does not support specified action") and
    +# 3/113 ("Could not find service") all mean the job isn't currently loaded in
    +# the target domain, so start/restart should re-bootstrap the plist and retry.
    +_LAUNCHD_JOB_UNLOADED_EXIT_CODES = frozenset({3, 113, 125})
    +
    +# When even a fresh bootstrap can't manage the domain, launchctl returns 5
    +# ("Input/output error") or a persistent 125. On those hosts launchd cannot
    +# supervise the gateway at all, so we degrade to a detached background process
    +# (the documented `nohup hermes gateway run` workaround). See #23387.
    +_LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES = frozenset({5, 125})
    +
    +
    +def _launchd_error_indicates_unloaded(exc: subprocess.CalledProcessError) -> bool:
    +    """True when launchctl failed because the job isn't loaded (retry bootstrap)."""
    +    return exc.returncode in _LAUNCHD_JOB_UNLOADED_EXIT_CODES
    +
    +
    +def _launchctl_domain_unsupported(returncode: int) -> bool:
    +    """True when launchctl can't manage the domain even after a fresh bootstrap.
    +
    +    Codes 5 and 125 persist on macOS hosts where neither `gui/<uid>` nor
    +    `user/<uid>` supports service management; treat these as "launchd
    +    unavailable" and degrade gracefully to a detached process.
    +    """
    +    return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES
    +
    +
    +def _gateway_run_command() -> list[str]:
    +    """Build the `python -m hermes_cli.main [--profile X] gateway run --replace` argv.
    +
    +    Profile-aware: honors the active HERMES_HOME via `_profile_arg()` so the
    +    detached fallback launches into the same profile as the CLI invocation.
    +    """
    +    cmd = [get_python_path(), "-m", "hermes_cli.main"]
    +    profile_arg = _profile_arg()
    +    if profile_arg:
    +        cmd.extend(profile_arg.split())
    +    cmd.extend(["gateway", "run", "--replace"])
    +    return cmd
    +
    +
    +def _spawn_detached_gateway() -> bool:
    +    """Launch the gateway as a detached background process (launchd fallback).
    +
    +    Used when launchctl can no longer bootstrap/kickstart the gateway on
    +    macOS 26+ (issue #23387). Mirrors the `nohup hermes gateway run --replace`
    +    workaround but keeps it CLI-managed: stdout/stderr go to the profile's
    +    gateway logs and the PID is tracked via the gateway.pid file that
    +    `run_gateway` writes, so stop/status/restart keep working.
    +    """
    +    from hermes_cli._subprocess_compat import windows_detach_popen_kwargs
    +
    +    log_dir = get_hermes_home() / "logs"
    +    log_dir.mkdir(parents=True, exist_ok=True)
    +    out_path = log_dir / "gateway.log"
    +    err_path = log_dir / "gateway.error.log"
    +    try:
    +        out = open(out_path, "ab")
    +        err = open(err_path, "ab")
    +    except OSError:
    +        return False
    +    try:
    +        with out, err:
    +            subprocess.Popen(
    +                _gateway_run_command(),
    +                stdin=subprocess.DEVNULL,
    +                stdout=out,
    +                stderr=err,
    +                **windows_detach_popen_kwargs(),
    +            )
    +    except OSError:
    +        return False
    +    return True
    +
    +
    +def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True) -> bool:
    +    """Start the gateway detached when launchd can't manage it, with guidance.
    +
    +    Returns True if the detached gateway was launched. When it can't be
    +    launched, prints the manual workaround and (by default) exits non-zero so
    +    the failure surfaces instead of silently doing nothing.
    +    """
    +    from hermes_constants import display_hermes_home as _dhh
    +
    +    print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).")
    +    if _spawn_detached_gateway():
    +        print("✓ Started gateway as a background process instead")
    +        print("  It will NOT auto-start at login or auto-restart on crash.")
    +        print(f"  Logs: {_dhh()}/logs/gateway.log")
    +        print("  Stop it with: hermes gateway stop")
    +        return True
    +    print_error("Failed to start the gateway as a background process.")
    +    print(
    +        f"  Try manually: nohup hermes gateway run --replace "
    +        f"> {_dhh()}/logs/gateway.log 2>&1 &"
    +    )
    +    if exit_on_failure:
    +        sys.exit(1)
    +    return False
     
     
     def generate_launchd_plist() -> str:
    @@ -3077,6 +3182,12 @@ def generate_launchd_plist() -> str:
             <key>HERMES_HOME</key>
             <string>{hermes_home}</string>
         </dict>
    +
    +    <key>LimitLoadToSessionType</key>
    +    <array>
    +        <string>Aqua</string>
    +        <string>Background</string>
    +    </array>
         
         <key>RunAtLoad</key>
         <true/>
    @@ -3154,11 +3265,17 @@ def launchd_install(force: bool = False):
         print(f"Installing launchd service to: {plist_path}")
         plist_path.write_text(generate_launchd_plist())
     
    -    subprocess.run(
    -        ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
    -        check=True,
    -        timeout=30,
    -    )
    +    try:
    +        subprocess.run(
    +            ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
    +            check=True,
    +            timeout=30,
    +        )
    +    except subprocess.CalledProcessError as e:
    +        if not _launchctl_domain_unsupported(e.returncode):
    +            raise
    +        _launchd_fallback_to_detached(f"launchctl bootstrap exit {e.returncode}")
    +        return
     
         print()
         print("✓ Service installed and loaded!")
    @@ -3195,16 +3312,22 @@ def launchd_start():
             print("↻ launchd plist missing; regenerating service definition")
             plist_path.parent.mkdir(parents=True, exist_ok=True)
             plist_path.write_text(generate_launchd_plist(), encoding="utf-8")
    -        subprocess.run(
    -            ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
    -            check=True,
    -            timeout=30,
    -        )
    -        subprocess.run(
    -            ["launchctl", "kickstart", f"{_launchd_domain()}/{label}"],
    -            check=True,
    -            timeout=30,
    -        )
    +        try:
    +            subprocess.run(
    +                ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
    +                check=True,
    +                timeout=30,
    +            )
    +            subprocess.run(
    +                ["launchctl", "kickstart", f"{_launchd_domain()}/{label}"],
    +                check=True,
    +                timeout=30,
    +            )
    +        except subprocess.CalledProcessError as e:
    +            if not _launchctl_domain_unsupported(e.returncode):
    +                raise
    +            _launchd_fallback_to_detached(f"launchctl exit {e.returncode}")
    +            return
             print("✓ Service started")
             return
     
    @@ -3216,19 +3339,28 @@ def launchd_start():
                 timeout=30,
             )
         except subprocess.CalledProcessError as e:
    -        if e.returncode not in {3, 113}:
    +        if not _launchd_error_indicates_unloaded(e):
                 raise
    +        # Job not loaded in this domain — re-bootstrap the plist and retry.
             print("↻ launchd job was unloaded; reloading service definition")
    -        subprocess.run(
    -            ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
    -            check=True,
    -            timeout=30,
    -        )
    -        subprocess.run(
    -            ["launchctl", "kickstart", f"{_launchd_domain()}/{label}"],
    -            check=True,
    -            timeout=30,
    -        )
    +        try:
    +            subprocess.run(
    +                ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
    +                check=True,
    +                timeout=30,
    +            )
    +            subprocess.run(
    +                ["launchctl", "kickstart", f"{_launchd_domain()}/{label}"],
    +                check=True,
    +                timeout=30,
    +            )
    +        except subprocess.CalledProcessError as e2:
    +            # Even a fresh bootstrap can't manage the domain on this host —
    +            # degrade to a detached background process (issue #23387).
    +            if not _launchctl_domain_unsupported(e2.returncode):
    +                raise
    +            _launchd_fallback_to_detached(f"launchctl exit {e2.returncode}")
    +            return
         print("✓ Service started")
     
     
    @@ -3250,8 +3382,13 @@ def launchd_stop():
         try:
             subprocess.run(["launchctl", "bootout", target], check=True, timeout=90)
         except subprocess.CalledProcessError as e:
    -        if e.returncode in {3, 113}:
    -            pass  # Already unloaded — nothing to stop.
    +        # Job already unloaded (3/113/125), or the domain can't be managed at
    +        # all (5/125, macOS 26+ detached-fallback process, issue #23387) — in
    +        # both cases just fall through to the PID-based kill below.
    +        if _launchd_error_indicates_unloaded(e) or _launchctl_domain_unsupported(
    +            e.returncode
    +        ):
    +            pass
             else:
                 raise
         _wait_for_gateway_exit(timeout=10.0, force_after=5.0)
    @@ -3335,17 +3472,29 @@ def launchd_restart():
             subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90)
             print("✓ Service restarted")
         except subprocess.CalledProcessError as e:
    -        if e.returncode not in {3, 113}:
    +        if not _launchd_error_indicates_unloaded(e):
    +            # Not a "job unloaded" code. If the domain is fundamentally
    +            # unmanageable (error 5), degrade to detached; the old process was
    +            # already drained/terminated above. Otherwise re-raise.
    +            if _launchctl_domain_unsupported(e.returncode):
    +                _launchd_fallback_to_detached(f"launchctl kickstart exit {e.returncode}")
    +                return
                 raise
             # Job not loaded — bootstrap and start fresh
             print("↻ launchd job was unloaded; reloading")
             plist_path = get_launchd_plist_path()
    -        subprocess.run(
    -            ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
    -            check=True,
    -            timeout=30,
    -        )
    -        subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
    +        try:
    +            subprocess.run(
    +                ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
    +                check=True,
    +                timeout=30,
    +            )
    +            subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
    +        except subprocess.CalledProcessError as e2:
    +            if not _launchctl_domain_unsupported(e2.returncode):
    +                raise
    +            _launchd_fallback_to_detached(f"launchctl exit {e2.returncode}")
    +            return
             print("✓ Service restarted")
     
     

  • tests/hermes_cli/test_gateway_service.py

    @@ -679,6 +679,162 @@ def test_launchd_status_reports_local_stale_plist_when_unloaded(self, tmp_path,
             assert "stale" in output.lower()
             assert "not loaded" in output.lower()
     
    +    def test_launchd_domain_uses_user_domain(self):
    +        # The user/<uid> domain (not gui/<uid>) is the one reachable from
    +        # non-Aqua/background sessions on macOS 26+ (issue #23387).
    +        assert gateway_cli._launchd_domain() == f"user/{os.getuid()}"
    +
    +    def test_launchctl_domain_unsupported_recognizes_macos26_codes(self):
    +        # Codes that persist after a fresh bootstrap → launchd truly unavailable.
    +        assert gateway_cli._launchctl_domain_unsupported(5) is True
    +        assert gateway_cli._launchctl_domain_unsupported(125) is True
    +        assert gateway_cli._launchctl_domain_unsupported(3) is False
    +        assert gateway_cli._launchctl_domain_unsupported(113) is False
    +        assert gateway_cli._launchctl_domain_unsupported(0) is False
    +
    +    def test_launchd_start_reloads_on_kickstart_exit_code_125(self, tmp_path, monkeypatch):
    +        """Exit code 125 means the job is absent from the domain → bootstrap recovery."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
    +        label = gateway_cli.get_launchd_label()
    +
    +        calls = []
    +        domain = gateway_cli._launchd_domain()
    +        target = f"{domain}/{label}"
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            if cmd and cmd[0] == "launchctl":
    +                calls.append(cmd)
    +            if cmd == ["launchctl", "kickstart", target] and calls.count(cmd) == 1:
    +                raise gateway_cli.subprocess.CalledProcessError(
    +                    125, cmd, stderr="Domain does not support specified action"
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        gateway_cli.launchd_start()
    +
    +        assert calls == [
    +            ["launchctl", "kickstart", target],
    +            ["launchctl", "bootstrap", domain, str(plist_path)],
    +            ["launchctl", "kickstart", target],
    +        ]
    +
    +    def test_launchd_start_falls_back_to_detached_when_rebootstrap_fails(self, tmp_path, monkeypatch, capsys):
    +        """If even a fresh bootstrap can't manage the domain, spawn detached."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
    +        label = gateway_cli.get_launchd_label()
    +        target = f"{gateway_cli._launchd_domain()}/{label}"
    +
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +        monkeypatch.setattr(gateway_cli, "refresh_launchd_plist_if_needed", lambda: False)
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            if cmd == ["launchctl", "kickstart", target]:
    +                # First kickstart: job not loaded (125). After bootstrap also
    +                # fails, this won't be reached again.
    +                raise gateway_cli.subprocess.CalledProcessError(
    +                    125, cmd, stderr="Domain does not support specified action"
    +                )
    +            if cmd[:2] == ["launchctl", "bootstrap"]:
    +                raise gateway_cli.subprocess.CalledProcessError(
    +                    5, cmd, stderr="Input/output error"
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        spawned = []
    +        monkeypatch.setattr(
    +            gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True
    +        )
    +
    +        gateway_cli.launchd_start()
    +
    +        assert spawned == [True]
    +        assert "background process" in capsys.readouterr().out.lower()
    +
    +    def test_launchd_install_falls_back_to_detached_on_bootstrap_5(self, tmp_path, monkeypatch, capsys):
    +        """macOS bootstrap error 5 should spawn a detached gateway, not crash."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            if cmd[:2] == ["launchctl", "bootstrap"]:
    +                raise gateway_cli.subprocess.CalledProcessError(
    +                    5, cmd, stderr="Input/output error"
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        spawned = []
    +        monkeypatch.setattr(
    +            gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True
    +        )
    +
    +        gateway_cli.launchd_install(force=True)
    +
    +        assert spawned == [True]
    +        assert "Service installed and loaded" not in capsys.readouterr().out
    +
    +    def test_launchd_restart_falls_back_to_detached_on_error_5(self, monkeypatch, capsys):
    +        """kickstart -k error 5 (domain unmanageable) should relaunch detached."""
    +        target = f"{gateway_cli._launchd_domain()}/{gateway_cli.get_launchd_label()}"
    +
    +        monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 5.0)
    +        monkeypatch.setattr(gateway_cli, "_request_gateway_self_restart", lambda pid: False)
    +        monkeypatch.setattr(gateway_cli, "_wait_for_gateway_exit", lambda timeout, force_after=None: True)
    +        monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False: None)
    +        monkeypatch.setattr("gateway.status.get_running_pid", lambda: 321)
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            if cmd == ["launchctl", "kickstart", "-k", target]:
    +                raise gateway_cli.subprocess.CalledProcessError(
    +                    5, cmd, stderr="Input/output error"
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        spawned = []
    +        monkeypatch.setattr(
    +            gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True
    +        )
    +
    +        gateway_cli.launchd_restart()
    +
    +        assert spawned == [True]
    +
    +    def test_launchd_stop_tolerates_domain_unsupported_bootout(self, monkeypatch, capsys):
    +        """bootout exit 125 (macOS 26) must fall through to PID-based kill, not raise."""
    +        def fake_run(cmd, check=False, **kwargs):
    +            if "bootout" in cmd:
    +                raise gateway_cli.subprocess.CalledProcessError(
    +                    125, cmd, stderr="Domain does not support specified action"
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +        monkeypatch.setattr(gateway_cli, "_wait_for_gateway_exit", lambda **kw: None)
    +
    +        gateway_cli.launchd_stop()
    +
    +        assert "stopped" in capsys.readouterr().out.lower()
    +
    +    def test_launchd_fallback_exits_when_spawn_fails(self, monkeypatch, capsys):
    +        """If the detached spawn fails, surface the manual workaround and exit 1."""
    +        monkeypatch.setattr(gateway_cli, "_spawn_detached_gateway", lambda: False)
    +
    +        with pytest.raises(SystemExit) as exc:
    +            gateway_cli._launchd_fallback_to_detached("test reason")
    +        assert exc.value.code == 1
    +        out = capsys.readouterr().out
    +        assert "nohup hermes gateway run" in out
    +
     
     class TestGatewayServiceDetection:
         def test_supports_systemd_services_requires_systemctl_binary(self, monkeypatch):
    @@ -1629,6 +1785,14 @@ def test_launchd_plist_includes_profile(self, tmp_path, monkeypatch):
             assert "<string>--profile</string>" in plist
             assert "<string>mybot</string>" in plist
     
    +    def test_launchd_plist_supports_aqua_and_background_sessions(self):
    +        # macOS 26+ only loads the agent in non-Aqua sessions when the plist
    +        # opts into Background as well (issue #23387).
    +        plist = gateway_cli.generate_launchd_plist()
    +        assert "<key>LimitLoadToSessionType</key>" in plist
    +        assert "<string>Aqua</string>" in plist
    +        assert "<string>Background</string>" in plist
    +
         def test_launchd_plist_path_uses_real_user_home_not_profile_home(self, tmp_path, monkeypatch):
             profile_dir = tmp_path / ".hermes" / "profiles" / "orcha"
             profile_dir.mkdir(parents=True)

• #43663 fix(gateway): probe macOS launchd domain instead of hardcoding user/<uid> (#40831)

  merged to main

  files: hermes_cli/gateway.py, tests/hermes_cli/test_gateway_service.py

  Show the diff

  • hermes_cli/gateway.py

    @@ -3067,12 +3067,77 @@ def get_launchd_label() -> str:
         return f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway"
     
     
    +# Cached launchd domain result — probing is cheap but should only run once per
    +# process invocation (each ``hermes gateway start/stop/status`` call).
    +_resolved_launchd_domain: str | None = None
    +
    +
     def _launchd_domain() -> str:
    -    # The `user/<uid>` domain (vs the older `gui/<uid>`) is reachable from
    -    # non-Aqua/background sessions (SSH, headless, login items) and is the only
    -    # one that supports service management on macOS 26+. `gui/<uid>` returns
    -    # error 125 ("Domain does not support specified action") there. See #23387.
    -    return f"user/{os.getuid()}"  # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
    +    """Return the launchd domain that actually manages the gateway service.
    +
    +    Probes ``gui/<uid>`` first (Aqua sessions), then ``user/<uid>``
    +    (Background/SSH sessions).  When neither domain contains a loaded
    +    service, falls back to ``launchctl managername`` as a heuristic.
    +
    +    The result is cached for the lifetime of the process so that repeated
    +    calls (``start``, ``stop``, ``restart``) use a consistent domain.
    +
    +    See #40831, #23387.
    +    """
    +    global _resolved_launchd_domain
    +    if _resolved_launchd_domain is not None:
    +        return _resolved_launchd_domain
    +
    +    uid = os.getuid()  # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
    +    label = get_launchd_label()
    +    gui_domain = f"gui/{uid}"
    +    user_domain = f"user/{uid}"
    +
    +    # 1. Probe gui/<uid> first — in Aqua sessions the service is loaded here.
    +    try:
    +        subprocess.run(
    +            ["launchctl", "print", f"{gui_domain}/{label}"],
    +            check=True,
    +            timeout=5,
    +            capture_output=True,
    +        )
    +        _resolved_launchd_domain = gui_domain
    +        return gui_domain
    +    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
    +        pass
    +
    +    # 2. Probe user/<uid> — in Background/SSH sessions this is the working domain.
    +    try:
    +        subprocess.run(
    +            ["launchctl", "print", f"{user_domain}/{label}"],
    +            check=True,
    +            timeout=5,
    +            capture_output=True,
    +        )
    +        _resolved_launchd_domain = user_domain
    +        return user_domain
    +    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
    +        pass
    +
    +    # 3. Neither domain has the service loaded — use managername as heuristic.
    +    #    Aqua → gui/<uid>, anything else (Background, loginwindow) → user/<uid>.
    +    try:
    +        result = subprocess.run(
    +            ["launchctl", "managername"],
    +            capture_output=True,
    +            text=True,
    +            timeout=5,
    +        )
    +        if "Aqua" in (result.stdout or ""):
    +            _resolved_launchd_domain = gui_domain
    +            return gui_domain
    +    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
    +        pass
    +
    +    # 4. Default to user/<uid> (matches the pre-probing behavior for
    +    #    Background/SSH sessions and is the recommended domain on macOS 26+).
    +    _resolved_launchd_domain = user_domain
    +    return user_domain
     
     
     # On macOS, exit code 125 ("Domain does not support specified action") and

  • tests/hermes_cli/test_gateway_service.py

    @@ -495,7 +495,10 @@ def fake_run(cmd, check=False, **kwargs):
             label = gateway_cli.get_launchd_label()
             domain = gateway_cli._launchd_domain()
             assert "--replace" in plist_path.read_text(encoding="utf-8")
    -        assert calls[:2] == [
    +        # The calls list includes launchctl print probes from _launchd_domain()
    +        # before the bootout/bootstrap calls. Filter to only bootout/bootstrap.
    +        service_calls = [c for c in calls if "bootout" in c or "bootstrap" in c]
    +        assert service_calls[:2] == [
                 ["launchctl", "bootout", f"{domain}/{label}"],
                 ["launchctl", "bootstrap", domain, str(plist_path)],
             ]
    @@ -679,10 +682,22 @@ def test_launchd_status_reports_local_stale_plist_when_unloaded(self, tmp_path,
             assert "stale" in output.lower()
             assert "not loaded" in output.lower()
     
    -    def test_launchd_domain_uses_user_domain(self):
    +    def test_launchd_domain_uses_user_domain(self, monkeypatch):
             # The user/<uid> domain (not gui/<uid>) is the one reachable from
             # non-Aqua/background sessions on macOS 26+ (issue #23387).
    -        assert gateway_cli._launchd_domain() == f"user/{os.getuid()}"
    +        # When gui/<uid> fails to probe and user/<uid> succeeds,
    +        # _launchd_domain() must return user/<uid>.
    +        gateway_cli._resolved_launchd_domain = None
    +        monkeypatch.setattr(os, "getuid", lambda: 501)
    +        label = gateway_cli.get_launchd_label()
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            if "print" in cmd and "gui/" in " ".join(cmd):
    +                raise subprocess.CalledProcessError(1, cmd, stderr="Domain error")
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +        assert gateway_cli._launchd_domain() == "user/501"
     
         def test_launchctl_domain_unsupported_recognizes_macos26_codes(self):
             # Codes that persist after a fresh bootstrap → launchd truly unavailable.
    @@ -836,6 +851,114 @@ def test_launchd_fallback_exits_when_spawn_fails(self, monkeypatch, capsys):
             assert "nohup hermes gateway run" in out
     
     
    +class TestLaunchdDomainDetection:
    +    """Regression tests for _launchd_domain() probing (#40831).
    +
    +    The function must detect which launchd domain actually contains (or can
    +    manage) the service, rather than hardcoding ``user/<uid>`` or ``gui/<uid>``.
    +    """
    +
    +    def _reset_domain_cache(self):
    +        """Clear any cached domain result between tests."""
    +        gateway_cli._resolved_launchd_domain = None
    +
    +    def test_prefers_gui_domain_when_service_loaded_there(self, monkeypatch):
    +        """In an Aqua session where the service is loaded under gui/<uid>,
    +        _launchd_domain() must return ``gui/<uid>`` — not ``user/<uid>``."""
    +        self._reset_domain_cache()
    +        monkeypatch.setattr(os, "getuid", lambda: 501)
    +        label = gateway_cli.get_launchd_label()
    +
    +        run_calls = []
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            run_calls.append(cmd)
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        domain = gateway_cli._launchd_domain()
    +        assert domain == f"gui/501"
    +        # Should have probed gui first
    +        assert run_calls[0] == ["launchctl", "print", f"gui/501/{label}"]
    +
    +    def test_falls_back_to_user_domain_when_gui_fails(self, monkeypatch):
    +        """In a Background/SSH session where gui/<uid> fails but user/<uid>
    +        works, _launchd_domain() must return ``user/<uid>``."""
    +        self._reset_domain_cache()
    +        monkeypatch.setattr(os, "getuid", lambda: 501)
    +        label = gateway_cli.get_launchd_label()
    +
    +        run_calls = []
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            run_calls.append(cmd)
    +            if "print" in cmd and "gui/" in " ".join(cmd):
    +                raise subprocess.CalledProcessError(1, cmd, stderr="Domain error")
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        domain = gateway_cli._launchd_domain()
    +        assert domain == f"user/501"
    +        # Should have tried gui first, then user
    +        assert len(run_calls) >= 2
    +
    +    def test_uses_managername_heuristic_when_both_probe_fail(self, monkeypatch):
    +        """When neither domain contains a loaded service, use
    +        ``launchctl managername`` as a tiebreaker: Aqua -> gui, else -> user."""
    +        self._reset_domain_cache()
    +        monkeypatch.setattr(os, "getuid", lambda: 501)
    +        label = gateway_cli.get_launchd_label()
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            if "print" in cmd:
    +                raise subprocess.CalledProcessError(1, cmd, stderr="not found")
    +            if "managername" in cmd:
    +                return SimpleNamespace(returncode=0, stdout="Aqua\n", stderr="")
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        domain = gateway_cli._launchd_domain()
    +        assert domain == f"gui/501"
    +
    +    def test_managername_background_selects_user_domain(self, monkeypatch):
    +        """When managername is Background (non-Aqua), use user/<uid>."""
    +        self._reset_domain_cache()
    +        monkeypatch.setattr(os, "getuid", lambda: 501)
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            if "print" in cmd:
    +                raise subprocess.CalledProcessError(1, cmd, stderr="not found")
    +            if "managername" in cmd:
    +                return SimpleNamespace(returncode=0, stdout="Background\n", stderr="")
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        domain = gateway_cli._launchd_domain()
    +        assert domain == f"user/501"
    +
    +    def test_caches_result_across_calls(self, monkeypatch):
    +        """Domain detection should run once and cache the result."""
    +        self._reset_domain_cache()
    +        monkeypatch.setattr(os, "getuid", lambda: 501)
    +
    +        run_count = [0]
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            run_count[0] += 1
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        d1 = gateway_cli._launchd_domain()
    +        d2 = gateway_cli._launchd_domain()
    +        assert d1 == d2
    +        assert run_count[0] == 1  # Only probed once
    +
    +
     class TestGatewayServiceDetection:
         def test_supports_systemd_services_requires_systemctl_binary(self, monkeypatch):
             monkeypatch.setattr(gateway_cli, "is_linux", lambda: True)

• #52984 fix(macos): distinguish launchd supervision from detached fallback in gateway status (#42524)

  merged to main

  files: hermes_cli/gateway.py, tests/hermes_cli/test_gateway_service.py

  Show the diff

  • hermes_cli/gateway.py

    @@ -138,16 +138,22 @@ def _get_service_pids() -> set:
                     timeout=5,
                 )
                 if result.returncode == 0:
    -                # Output: "PID\tStatus\tLabel" header, then one data line
    -                for line in result.stdout.strip().splitlines():
    -                    parts = line.split()
    -                    if len(parts) >= 3 and parts[2] == label:
    -                        try:
    -                            pid = int(parts[0])
    -                            if pid > 0:
    -                                pids.add(pid)
    -                        except ValueError:
    -                            pass
    +                # Try plist format first (macOS 26+): "PID" = <N>;
    +                pid = _parse_launchd_pid_from_list_output(result.stdout)
    +                if pid is not None and pid > 0:
    +                    pids.add(pid)
    +                else:
    +                    # Fall back to legacy tab-separated format:
    +                    # "PID\tStatus\tLabel"
    +                    for line in result.stdout.strip().splitlines():
    +                        parts = line.split()
    +                        if len(parts) >= 3 and parts[2] == label:
    +                            try:
    +                                pid = int(parts[0])
    +                                if pid > 0:
    +                                    pids.add(pid)
    +                            except ValueError:
    +                                pass
             except (FileNotFoundError, subprocess.TimeoutExpired):
                 pass
     
    @@ -1129,7 +1135,37 @@ def _recover_pending_systemd_restart(
         return False
     
     
    +def _parse_launchd_pid_from_list_output(output: str) -> int | None:
    +    """Extract the PID from ``launchctl list <label>`` output.
    +
    +    When launchd is actively supervising a process, the output includes a
    +    ``"PID" = <number>;`` line.  When the service definition is only *registered*
    +    but not running (macOS 26+ with an unmanageable domain, fallback active),
    +    the output lacks a PID field entirely.  Returns ``None`` when no PID is
    +    found or the PID is non-positive (e.g. ``-1`` for a recently-crashed service).
    +    """
    +    for line in output.splitlines():
    +        stripped = line.strip()
    +        if stripped.startswith('"PID"') or stripped.startswith("PID"):
    +            parts = stripped.split("=", 1)
    +            if len(parts) == 2:
    +                val = parts[1].strip().rstrip(";").strip('"')
    +                try:
    +                    pid = int(val)
    +                    return pid if pid > 0 else None
    +                except ValueError:
    +                    return None
    +    return None
    +
    +
     def _probe_launchd_service_running() -> bool:
    +    """Return True when launchd is actively supervising the gateway process.
    +
    +    ``launchctl list <label>`` returns exit 0 whenever the service definition is
    +    registered with launchd — even when ``state = not running`` (macOS 26+).
    +    We additionally require a PID in the output to confirm launchd is actually
    +    managing a live process, not just holding a static definition.
    +    """
         if not get_launchd_plist_path().exists():
             return False
         try:
    @@ -1141,7 +1177,9 @@ def _probe_launchd_service_running() -> bool:
             )
         except subprocess.TimeoutExpired:
             return False
    -    return result.returncode == 0
    +    if result.returncode != 0:
    +        return False
    +    return _parse_launchd_pid_from_list_output(result.stdout) is not None
     
     
     def get_gateway_runtime_snapshot(system: bool = False) -> GatewayRuntimeSnapshot:
    @@ -1235,12 +1273,23 @@ def _print_gateway_process_mismatch(snapshot: GatewayRuntimeSnapshot) -> None:
         if not snapshot.has_process_service_mismatch:
             return
         print()
    -    print(
    -        "⚠ Gateway process is running for this profile, but the service is not active"
    -    )
    -    print(f"  PID(s): {_format_gateway_pids(snapshot.gateway_pids, limit=None)}")
    -    print("  This is usually a manual foreground/tmux/nohup run, so `hermes gateway`")
    -    print("  can refuse to start another copy until this process stops.")
    +    # Distinguish the managed detached fallback (macOS launchd exit-5 path)
    +    # from a genuinely manual foreground/tmux/nohup run.
    +    if _launchd_unsupported_marker_exists():
    +        print(
    +            "⚠ Gateway is running as a detached fallback process — "
    +            "launchd cannot supervise it"
    +        )
    +        print(f"  PID(s): {_format_gateway_pids(snapshot.gateway_pids, limit=None)}")
    +        print("  Auto-start at login and auto-restart on crash are NOT available.")
    +        print("  Stop it with: hermes gateway stop")
    +    else:
    +        print(
    +            "⚠ Gateway process is running for this profile, but the service is not active"
    +        )
    +        print(f"  PID(s): {_format_gateway_pids(snapshot.gateway_pids, limit=None)}")
    +        print("  This is usually a manual foreground/tmux/nohup run, so `hermes gateway`")
    +        print("  can refuse to start another copy until this process stops.")
     
     
     def _print_other_profiles_gateway_status() -> None:
    @@ -3404,6 +3453,47 @@ def _launchctl_domain_unsupported(returncode: int) -> bool:
         return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES
     
     
    +# ── launchd unsupported marker ─────────────────────────────────────────────
    +# When launchd can't manage the domain on this host (error 5/125, macOS 26+),
    +# we write a persistent marker so `launchd_status()` can explain that launchd
    +# supervision is unavailable regardless of whether a fallback process is
    +# currently running.  The marker is cleared when bootstrap/kickstart succeeds,
    +# so an OS update that fixes the underlying issue allows automatic recovery.
    +
    +
    +def _launchd_unsupported_marker_path() -> Path:
    +    return get_hermes_home() / ".gateway-launchd-unsupported"
    +
    +
    +def _write_launchd_unsupported_marker() -> None:
    +    """Persist that launchd cannot supervise the gateway on this host."""
    +    import json
    +    from datetime import datetime, timezone
    +
    +    try:
    +        _launchd_unsupported_marker_path().write_text(
    +            json.dumps({
    +                "written_at": datetime.now(timezone.utc).isoformat(),
    +                "reason": "launchd domain unsupported (exit 5/125)",
    +            }),
    +            encoding="utf-8",
    +        )
    +    except OSError:
    +        pass
    +
    +
    +def _clear_launchd_unsupported_marker() -> None:
    +    """Clear the unsupported marker when launchd bootstrap succeeds."""
    +    try:
    +        _launchd_unsupported_marker_path().unlink(missing_ok=True)
    +    except OSError:
    +        pass
    +
    +
    +def _launchd_unsupported_marker_exists() -> bool:
    +    return _launchd_unsupported_marker_path().exists()
    +
    +
     def _gateway_run_command() -> list[str]:
         """Build the `python -m hermes_cli.main [--profile X] gateway run --replace` argv.
     
    @@ -3461,6 +3551,7 @@ def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True)
         """
         from hermes_constants import display_hermes_home as _dhh
     
    +    _write_launchd_unsupported_marker()
         print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).")
         if _spawn_detached_gateway():
             print("✓ Started gateway as a background process instead")
    @@ -3702,6 +3793,7 @@ def launchd_install(force: bool = False):
     
         print()
         print("✓ Service installed and loaded!")
    +    _clear_launchd_unsupported_marker()
         print()
         print("Next steps:")
         print("  hermes gateway status             # Check status")
    @@ -3755,6 +3847,7 @@ def launchd_start():
                 _launchd_fallback_to_detached(f"launchctl exit {e.returncode}")
                 return
             print("✓ Service started")
    +        _clear_launchd_unsupported_marker()
             return
     
         refresh_launchd_plist_if_needed()
    @@ -3788,6 +3881,7 @@ def launchd_start():
                 _launchd_fallback_to_detached(f"launchctl exit {e2.returncode}")
                 return
         print("✓ Service started")
    +    _clear_launchd_unsupported_marker()
     
     
     def launchd_stop():
    @@ -3883,6 +3977,7 @@ def launchd_restart():
             pid = get_running_pid()
             if pid is not None and _request_gateway_self_restart(pid):
                 print("✓ Service restart requested")
    +            _clear_launchd_unsupported_marker()
                 return
             if pid is not None:
                 # Announce the drain BEFORE waiting on it. This wait can run for
    @@ -3907,6 +4002,7 @@ def launchd_restart():
                         )
             subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90)
             print("✓ Service restarted")
    +        _clear_launchd_unsupported_marker()
         except subprocess.CalledProcessError as e:
             if not _launchd_error_indicates_unloaded(e):
                 # Not a "job unloaded" code. If the domain is fundamentally
    @@ -3932,6 +4028,7 @@ def launchd_restart():
                 _launchd_fallback_to_detached(f"launchctl exit {e2.returncode}")
                 return
             print("✓ Service restarted")
    +        _clear_launchd_unsupported_marker()
     
     
     def launchd_status(deep: bool = False):
    @@ -3944,12 +4041,35 @@ def launchd_status(deep: bool = False):
                 text=True,
                 timeout=10,
             )
    -        loaded = result.returncode == 0
    -        loaded_output = result.stdout
    +        service_listed = result.returncode == 0
    +        list_output = result.stdout
         except subprocess.TimeoutExpired:
    -        loaded = False
    -        loaded_output = ""
    +        service_listed = False
    +        list_output = ""
    +
    +    # Determine whether launchd is actively supervising a process.
    +    # ``launchctl list`` returns exit 0 whenever the service definition is
    +    # registered — even when ``state = not running`` (macOS 26+ with an
    +    # unmanageable domain).  A PID in the output confirms a live process.
    +    launchd_pid = _parse_launchd_pid_from_list_output(list_output) if service_listed else None
     
    +    # Hermes PID tracking — may be a detached fallback process spawned when
    +    # launchd cannot manage the domain on this host.
    +    from gateway.status import get_running_pid
    +    fallback_pid = get_running_pid(cleanup_stale=False)
    +
    +    # Avoid double-counting: when launchd IS supervising, fallback_pid and
    +    # launchd_pid point at the same process (the gateway writes both the
    +    # launchd PID and the Hermes PID file).
    +    if launchd_pid is not None and fallback_pid == launchd_pid:
    +        fallback_pid = None
    +
    +    # Persistent marker written when launchd bootstrap/kickstart fails with
    +    # exit 5/125 on this host.  Lets us explain *why* launchd can't supervise
    +    # even when no fallback process is currently running.
    +    launchd_unsupported = _launchd_unsupported_marker_exists()
    +
    +    # ── Report ──
         print(f"Launchd plist: {plist_path}")
         if launchd_plist_is_current():
             print("✓ Service definition matches the current Hermes install")
    @@ -3957,13 +4077,33 @@ def launchd_status(deep: bool = False):
             print("⚠ Service definition is stale relative to the current Hermes install")
             print("  Run: hermes gateway start")
     
    -    if loaded:
    -        print("✓ Gateway service is loaded")
    -        print(loaded_output)
    +    if service_listed:
    +        if launchd_pid is not None:
    +            print(f"✓ Gateway is supervised by launchd (PID {launchd_pid})")
    +            print("  Auto-start at login and auto-restart on crash are available.")
    +            if launchd_unsupported:
    +                print("  (launchd domain was previously unavailable but is now working)")
    +        elif launchd_unsupported:
    +            print("⚠ Gateway service is registered but launchd is not supervising it")
    +            print("  launchd cannot manage the gateway on this macOS version.")
    +            if fallback_pid:
    +                print(f"✓ Detached fallback process is running (PID {fallback_pid})")
    +                print("  Cron jobs will fire. Stop with: hermes gateway stop")
    +            else:
    +                print("✗ No fallback process is running")
    +                print("  Run: hermes gateway start")
    +            print("  ⚠ Auto-start at login and auto-restart on crash are NOT available.")
    +        else:
    +            print("✓ Gateway service is registered with launchd")
    +            print(list_output)
    +            if fallback_pid:
    +                print(f"  Detached gateway process is running (PID {fallback_pid})")
         else:
             print("✗ Gateway service is not loaded")
             print("  Service definition exists locally but launchd has not loaded it.")
             print("  Run: hermes gateway start")
    +        if fallback_pid:
    +            print(f"  Note: a detached gateway process is running (PID {fallback_pid})")
     
         if deep:
             log_file = get_hermes_home() / "logs" / "gateway.log"

  • tests/hermes_cli/test_gateway_service.py

    @@ -993,6 +993,8 @@ def fake_run(cmd, check=False, **kwargs):
     
             assert spawned == [True]
             assert "background process" in capsys.readouterr().out.lower()
    +        # Verify the unsupported marker was written so status can explain why
    +        assert gateway_cli._launchd_unsupported_marker_exists()
     
         def test_launchd_install_falls_back_to_detached_on_bootstrap_5(self, tmp_path, monkeypatch, capsys):
             """macOS bootstrap error 5 should spawn a detached gateway, not crash."""
    @@ -1028,6 +1030,7 @@ def fake_run(cmd, check=False, **kwargs):
     
             assert spawned == [True]
             assert "Service installed and loaded" not in capsys.readouterr().out
    +        assert gateway_cli._launchd_unsupported_marker_exists()
     
         def test_launchd_restart_falls_back_to_detached_on_error_5(self, monkeypatch, capsys):
             """kickstart -k error 5 (domain unmanageable) should relaunch detached."""
    @@ -1056,6 +1059,7 @@ def fake_run(cmd, check=False, **kwargs):
             gateway_cli.launchd_restart()
     
             assert spawned == [True]
    +        assert gateway_cli._launchd_unsupported_marker_exists()
     
         def test_launchd_stop_tolerates_domain_unsupported_bootout(self, monkeypatch, capsys):
             """bootout exit 125 (macOS 26) must fall through to PID-based kill, not raise."""
    @@ -1082,6 +1086,177 @@ def test_launchd_fallback_exits_when_spawn_fails(self, monkeypatch, capsys):
             assert exc.value.code == 1
             out = capsys.readouterr().out
             assert "nohup hermes gateway run" in out
    +        # Marker is still written so status knows launchd is unavailable
    +        assert gateway_cli._launchd_unsupported_marker_exists()
    +
    +    # ── PID parsing ──────────────────────────────────────────────────────
    +
    +    def test_parse_launchd_pid_from_list_output_with_pid(self):
    +        output = '{\n    "PID" = 12345;\n    "Label" = "ai.hermes.gateway";\n}'
    +        assert gateway_cli._parse_launchd_pid_from_list_output(output) == 12345
    +
    +    def test_parse_launchd_pid_from_list_output_without_pid(self):
    +        output = '{\n    "Label" = "ai.hermes.gateway";\n    "OnDemand" = true;\n}'
    +        assert gateway_cli._parse_launchd_pid_from_list_output(output) is None
    +
    +    def test_parse_launchd_pid_from_list_output_empty(self):
    +        assert gateway_cli._parse_launchd_pid_from_list_output("") is None
    +
    +    def test_parse_launchd_pid_from_list_output_unquoted_pid(self):
    +        """Older macOS versions may output PID without quotes."""
    +        output = "{\n    PID = 99999;\n}"
    +        assert gateway_cli._parse_launchd_pid_from_list_output(output) == 99999
    +
    +    def test_parse_launchd_pid_from_list_output_negative_pid_returns_none(self):
    +        """PID = -1 (recently-crashed service sentinel) must return None."""
    +        output = '{\n    "PID" = -1;\n    "Label" = "ai.hermes.gateway";\n}'
    +        assert gateway_cli._parse_launchd_pid_from_list_output(output) is None
    +
    +    # ── Probe requires PID ───────────────────────────────────────────────
    +
    +    def test_probe_launchd_service_running_false_without_pid_in_output(self, tmp_path, monkeypatch):
    +        """launchctl list returns 0 but no PID → not actually running."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +        monkeypatch.setattr(
    +            gateway_cli.subprocess,
    +            "run",
    +            lambda *args, **kwargs: SimpleNamespace(
    +                returncode=0,
    +                stdout='{\n    "Label" = "ai.hermes.gateway";\n}',
    +                stderr="",
    +            ),
    +        )
    +        assert gateway_cli._probe_launchd_service_running() is False
    +
    +    def test_probe_launchd_service_running_true_with_pid_in_output(self, tmp_path, monkeypatch):
    +        """launchctl list returns 0 with PID → actually running."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +        monkeypatch.setattr(
    +            gateway_cli.subprocess,
    +            "run",
    +            lambda *args, **kwargs: SimpleNamespace(
    +                returncode=0,
    +                stdout='{\n    "PID" = 55555;\n    "Label" = "ai.hermes.gateway";\n}',
    +                stderr="",
    +            ),
    +        )
    +        assert gateway_cli._probe_launchd_service_running() is True
    +
    +    # ── Unsupport marker lifecycle ───────────────────────────────────────
    +
    +    def test_launchd_unsupported_marker_write_and_clear(self, tmp_path, monkeypatch):
    +        monkeypatch.setattr(gateway_cli, "get_hermes_home", lambda: tmp_path)
    +        assert not gateway_cli._launchd_unsupported_marker_exists()
    +        gateway_cli._write_launchd_unsupported_marker()
    +        assert gateway_cli._launchd_unsupported_marker_exists()
    +        gateway_cli._clear_launchd_unsupported_marker()
    +        assert not gateway_cli._launchd_unsupported_marker_exists()
    +
    +    def test_launchd_start_clears_unsupported_marker_on_bootstrap_success(self, tmp_path, monkeypatch, capsys):
    +        """When bootstrap succeeds (OS update fixes the issue), clear the marker."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +        # Pre-seed the marker as if a previous fallback wrote it
    +        monkeypatch.setattr(gateway_cli, "get_hermes_home", lambda: tmp_path)
    +        # Bypass the temp-home service write guard (added on main after PR #42567)
    +        monkeypatch.setattr(gateway_cli, "_refuse_temp_home_service_write", lambda d, k: False)
    +        gateway_cli._write_launchd_unsupported_marker()
    +        assert gateway_cli._launchd_unsupported_marker_exists()
    +
    +        # Simulate a bootstrap that succeeds
    +        def fake_run(cmd, check=False, **kwargs):
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        gateway_cli.launchd_install(force=True)
    +
    +        assert "Service installed and loaded" in capsys.readouterr().out
    +        assert not gateway_cli._launchd_unsupported_marker_exists()
    +
    +    # ── launchd_status with active supervision ───────────────────────────
    +
    +    def test_launchd_status_reports_supervised_when_pid_present(self, tmp_path, monkeypatch, capsys):
    +        """When launchd is actively supervising, report it clearly."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +
    +        def fake_run(cmd, capture_output=False, text=False, timeout=None, check=False, **kwargs):
    +            if isinstance(cmd, list) and cmd[:2] == ["launchctl", "list"]:
    +                return SimpleNamespace(
    +                    returncode=0,
    +                    stdout='{\n    "PID" = 77777;\n    "Label" = "ai.hermes.gateway";\n}',
    +                    stderr="",
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +        # No fallback PID — when launchd supervises, get_running_pid returns
    +        # the same PID; launchd_status deduplicates it.
    +        monkeypatch.setattr("gateway.status.get_running_pid", lambda cleanup_stale=False: 77777)
    +
    +        gateway_cli.launchd_status()
    +
    +        out = capsys.readouterr().out
    +        assert "supervised by launchd" in out
    +        assert "Auto-start at login" in out
    +
    +    def test_launchd_status_reports_fallback_when_unsupported_and_pid_running(self, tmp_path, monkeypatch, capsys):
    +        """When the unsupported marker exists and a fallback PID is running."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +
    +        def fake_run(cmd, capture_output=False, text=False, timeout=None, check=False, **kwargs):
    +            if isinstance(cmd, list) and cmd[:2] == ["launchctl", "list"]:
    +                return SimpleNamespace(
    +                    returncode=0,
    +                    stdout='{\n    "Label" = "ai.hermes.gateway";\n    "OnDemand" = true;\n}',
    +                    stderr="",
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +        monkeypatch.setattr("gateway.status.get_running_pid", lambda cleanup_stale=False: 88888)
    +        # Pre-seed the unsupported marker
    +        monkeypatch.setattr(gateway_cli, "get_hermes_home", lambda: tmp_path)
    +        gateway_cli._write_launchd_unsupported_marker()
    +
    +        gateway_cli.launchd_status()
    +
    +        out = capsys.readouterr().out
    +        assert "cannot manage the gateway on this macos version" in out.lower()
    +        assert "Detached fallback process is running" in out
    +        assert "PID 88888" in out
    +        assert "NOT available" in out
    +
    +    def test_launchd_status_reports_fallback_unavailable_when_unsupported_no_pid(self, tmp_path, monkeypatch, capsys):
    +        """Unsupported marker exists but no fallback process is running."""
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +
    +        def fake_run(cmd, capture_output=False, text=False, timeout=None, check=False, **kwargs):
    +            if isinstance(cmd, list) and cmd[:2] == ["launchctl", "list"]:
    +                return SimpleNamespace(
    +                    returncode=0,
    +                    stdout='{\n    "Label" = "ai.hermes.gateway";\n    "OnDemand" = true;\n}',
    +                    stderr="",
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +        monkeypatch.setattr("gateway.status.get_running_pid", lambda cleanup_stale=False: None)
    +        monkeypatch.setattr(gateway_cli, "get_hermes_home", lambda: tmp_path)
    +        gateway_cli._write_launchd_unsupported_marker()
    +
    +        gateway_cli.launchd_status()
    +
    +        out = capsys.readouterr().out
    +        assert "cannot manage the gateway on this macos version" in out.lower()
    +        assert "No fallback process is running" in out
    +        assert "NOT available" in out
     
     
     class TestLaunchdDomainDetection:

• #54159 fix(gateway): bootout stale launchd registration before bootstrap on restart (#42006)

  merged to main

  files: hermes_cli/gateway.py, infographic/launchd-bootout-42006/infographic.png, tests/hermes_cli/test_gateway_service.py

  Show the diff

  • hermes_cli/gateway.py

    @@ -4082,6 +4082,11 @@ def launchd_restart():
             print("↻ launchd job was unloaded; reloading")
             plist_path = get_launchd_plist_path()
             try:
    +            subprocess.run(
    +                ["launchctl", "bootout", target],
    +                check=False,
    +                timeout=90,
    +            )
                 subprocess.run(
                     ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
                     check=True,

  • tests/hermes_cli/test_gateway_service.py

    @@ -1077,6 +1077,46 @@ def fake_run(cmd, check=False, **kwargs):
             assert spawned == [True]
             assert gateway_cli._launchd_unsupported_marker_exists()
     
    +    def test_launchd_restart_boots_out_stale_registration_before_bootstrap(
    +        self, tmp_path, monkeypatch
    +    ):
    +        plist_path = tmp_path / "ai.hermes.gateway.plist"
    +        plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
    +        label = gateway_cli.get_launchd_label()
    +        domain = gateway_cli._launchd_domain()
    +        target = f"{domain}/{label}"
    +
    +        monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
    +        monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 5.0)
    +        monkeypatch.setattr(gateway_cli, "_request_gateway_self_restart", lambda pid: False)
    +        monkeypatch.setattr(
    +            gateway_cli, "_wait_for_gateway_exit", lambda timeout, force_after=None: True
    +        )
    +        monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False: None)
    +        monkeypatch.setattr("gateway.status.get_running_pid", lambda: 321)
    +
    +        calls = []
    +
    +        def fake_run(cmd, check=False, **kwargs):
    +            if cmd and cmd[0] == "launchctl":
    +                calls.append(cmd)
    +            if cmd == ["launchctl", "kickstart", "-k", target]:
    +                raise gateway_cli.subprocess.CalledProcessError(
    +                    3, cmd, stderr="Could not find service"
    +                )
    +            return SimpleNamespace(returncode=0, stdout="", stderr="")
    +
    +        monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
    +
    +        gateway_cli.launchd_restart()
    +
    +        assert calls == [
    +            ["launchctl", "kickstart", "-k", target],
    +            ["launchctl", "bootout", target],
    +            ["launchctl", "bootstrap", domain, str(plist_path)],
    +            ["launchctl", "kickstart", target],
    +        ]
    +
         def test_launchd_stop_tolerates_domain_unsupported_bootout(self, monkeypatch, capsys):
             """bootout exit 125 (macOS 26) must fall through to PID-based kill, not raise."""
             def fake_run(cmd, check=False, **kwargs):

• #65502 fix(gateway): preserve external supervisor ownership (salvage #65105)

  merged to main

  files: gateway/restart.py, gateway/slash_commands.py, hermes_cli/gateway.py, hermes_cli/main.py, hermes_cli/subcommands/gateway.py, scripts/release.py, tests/gateway/test_restart_service_detection.py, tests/hermes_cli/test_gateway_external_supervisor.py, tests/hermes_cli/test_subcommands_profile_gateway.py, website/docs/reference/cli-commands.md