Retry backoff, rate-limit handling, and local endpoint timeouts

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #3809 fix(agent): user-friendly 429 rate limit messages with Retry-After support

  merged to main

  files: run_agent.py

  Show the diff

  • run_agent.py

    @@ -7229,7 +7229,10 @@ def _stop_spinner():
                                 retry_count = 0
                                 continue
                             _final_summary = self._summarize_api_error(api_error)
    -                        self._vprint(f"{self.log_prefix}❌ Max retries ({max_retries}) exceeded. Giving up.", force=True)
    +                        if is_rate_limited:
    +                            self._vprint(f"{self.log_prefix}❌ Rate limit persisted after {max_retries} retries. Please try again later.", force=True)
    +                        else:
    +                            self._vprint(f"{self.log_prefix}❌ Max retries ({max_retries}) exceeded. Giving up.", force=True)
                             self._vprint(f"{self.log_prefix}   💀 Final error: {_final_summary}", force=True)
     
                             # Detect SSE stream-drop pattern (e.g. "Network
    @@ -7289,8 +7292,22 @@ def _stop_spinner():
                                 "error": _final_summary,
                             }
     
    -                    wait_time = min(2 ** retry_count, 60)  # Exponential backoff: 2s, 4s, 8s, 16s, 32s, 60s, 60s
    -                    self._emit_status(f"⏳ Retrying in {wait_time}s (attempt {retry_count}/{max_retries})...")
    +                    # For rate limits, respect the Retry-After header if present
    +                    _retry_after = None
    +                    if is_rate_limited:
    +                        _resp_headers = getattr(getattr(api_error, "response", None), "headers", None)
    +                        if _resp_headers and hasattr(_resp_headers, "get"):
    +                            _ra_raw = _resp_headers.get("retry-after") or _resp_headers.get("Retry-After")
    +                            if _ra_raw:
    +                                try:
    +                                    _retry_after = min(int(_ra_raw), 120)  # Cap at 2 minutes
    +                                except (TypeError, ValueError):
    +                                    pass
    +                    wait_time = _retry_after if _retry_after else min(2 ** retry_count, 60)
    +                    if is_rate_limited:
    +                        self._emit_status(f"⏱️ Rate limit reached. Waiting {wait_time}s before retry (attempt {retry_count + 1}/{max_retries})...")
    +                    else:
    +                        self._emit_status(f"⏳ Retrying in {wait_time}s (attempt {retry_count}/{max_retries})...")
                         logger.warning(
                             "Retrying API call in %ss (attempt %s/%s) %s error=%s",
                             wait_time,

• #6368 fix(agent): disable stale stream timeout for local providers

  merged to main

  files: run_agent.py

• #7950 fix: is_local_endpoint misses Docker/Podman DNS names

  merged to main

  files: agent/model_metadata.py, tests/agent/test_local_stream_timeout.py, tests/tools/test_tool_result_storage.py, tools/tool_result_storage.py

  Show the diff

  • agent/model_metadata.py

    @@ -179,6 +179,12 @@ def _strip_provider_prefix(model: str) -> str:
     
     # Local server hostnames / address patterns
     _LOCAL_HOSTS = ("localhost", "127.0.0.1", "::1", "0.0.0.0")
    +# Docker / Podman / Lima DNS names that resolve to the host machine
    +_CONTAINER_LOCAL_SUFFIXES = (
    +    ".docker.internal",
    +    ".containers.internal",
    +    ".lima.internal",
    +)
     
     
     def _normalize_base_url(base_url: str) -> str:
    @@ -254,6 +260,9 @@ def is_local_endpoint(base_url: str) -> bool:
             return False
         if host in _LOCAL_HOSTS:
             return True
    +    # Docker / Podman / Lima internal DNS names (e.g. host.docker.internal)
    +    if any(host.endswith(suffix) for suffix in _CONTAINER_LOCAL_SUFFIXES):
    +        return True
         # RFC-1918 private ranges and link-local
         import ipaddress
         try:

  • tests/agent/test_local_stream_timeout.py

    @@ -22,6 +22,9 @@ class TestLocalStreamReadTimeout:
             "http://0.0.0.0:5000",
             "http://192.168.1.100:8000",
             "http://10.0.0.5:1234",
    +        "http://host.docker.internal:11434",
    +        "http://host.containers.internal:11434",
    +        "http://host.lima.internal:11434",
         ])
         def test_local_endpoint_bumps_read_timeout(self, base_url):
             """Local endpoint + default timeout -> bumps to base_timeout."""
    @@ -68,3 +71,38 @@ def test_empty_base_url_keeps_default(self):
                 if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
                     _stream_read_timeout = _base_timeout
                 assert _stream_read_timeout == 120.0
    +
    +
    +class TestIsLocalEndpoint:
    +    """Direct unit tests for is_local_endpoint."""
    +
    +    @pytest.mark.parametrize("url", [
    +        "http://localhost:11434",
    +        "http://127.0.0.1:8080",
    +        "http://0.0.0.0:5000",
    +        "http://[::1]:11434",
    +        "http://192.168.1.100:8000",
    +        "http://10.0.0.5:1234",
    +        "http://172.17.0.1:11434",
    +    ])
    +    def test_classic_local_addresses(self, url):
    +        assert is_local_endpoint(url) is True
    +
    +    @pytest.mark.parametrize("url", [
    +        "http://host.docker.internal:11434",
    +        "http://host.docker.internal:8080/v1",
    +        "http://gateway.docker.internal:11434",
    +        "http://host.containers.internal:11434",
    +        "http://host.lima.internal:11434",
    +    ])
    +    def test_container_dns_names(self, url):
    +        assert is_local_endpoint(url) is True
    +
    +    @pytest.mark.parametrize("url", [
    +        "https://api.openai.com",
    +        "https://openrouter.ai/api",
    +        "https://api.anthropic.com",
    +        "https://evil.docker.internal.example.com",
    +    ])
    +    def test_remote_endpoints(self, url):
    +        assert is_local_endpoint(url) is False

  • tests/tools/test_tool_result_storage.py

    @@ -124,6 +124,34 @@ def test_uses_parent_dir_of_remote_path(self):
             cmd = env.execute.call_args[0][0]
             assert "mkdir -p /data/data/com.termux/files/usr/tmp/hermes-results" in cmd
     
    +    def test_path_with_spaces_is_quoted(self):
    +        env = MagicMock()
    +        env.execute.return_value = {"output": "", "returncode": 0}
    +        remote_path = "/tmp/hermes results/abc file.txt"
    +        _write_to_sandbox("content", remote_path, env)
    +        cmd = env.execute.call_args[0][0]
    +        assert "'/tmp/hermes results'" in cmd
    +        assert "'/tmp/hermes results/abc file.txt'" in cmd
    +
    +    def test_shell_metacharacters_neutralized(self):
    +        """Paths with shell metacharacters must be quoted to prevent injection."""
    +        env = MagicMock()
    +        env.execute.return_value = {"output": "", "returncode": 0}
    +        malicious_path = "/tmp/hermes-results/$(whoami).txt"
    +        _write_to_sandbox("content", malicious_path, env)
    +        cmd = env.execute.call_args[0][0]
    +        # The $() must not appear unquoted — shlex.quote wraps it
    +        assert "'/tmp/hermes-results/$(whoami).txt'" in cmd
    +
    +    def test_semicolon_injection_neutralized(self):
    +        env = MagicMock()
    +        env.execute.return_value = {"output": "", "returncode": 0}
    +        malicious_path = "/tmp/x; rm -rf /; echo .txt"
    +        _write_to_sandbox("content", malicious_path, env)
    +        cmd = env.execute.call_args[0][0]
    +        # The semicolons must be inside quotes, not acting as command separators
    +        assert "'/tmp/x; rm -rf /; echo .txt'" in cmd
    +
     
     class TestResolveStorageDir:
         def test_defaults_to_storage_dir_without_env(self):

  • tools/tool_result_storage.py

    @@ -24,6 +24,7 @@
     
     import logging
     import os
    +import shlex
     import uuid
     
     from tools.budget_config import (
    @@ -79,7 +80,7 @@ def _write_to_sandbox(content: str, remote_path: str, env) -> bool:
         marker = _heredoc_marker(content)
         storage_dir = os.path.dirname(remote_path)
         cmd = (
    -        f"mkdir -p {storage_dir} && cat > {remote_path} << '{marker}'\n"
    +        f"mkdir -p {shlex.quote(storage_dir)} && cat > {shlex.quote(remote_path)} << '{marker}'\n"
             f"{content}\n"
             f"{marker}"
         )

• #9248 fix(local): recognize unqualified hostnames as local endpoints

  merged to main

  files: agent/model_metadata.py, tests/agent/test_local_stream_timeout.py

• #77607 fix(agent): exponential backoff for rate-limit fallback cooldown (#30223 salvage)

  merged to main

  files: agent/agent_runtime_helpers.py, agent/chat_completion_helpers.py, tests/run_agent/test_24996_fallback_exhaustion_cooldown.py

• #116353 Transient provider outages no longer end the turn: bounded auto-recovery ladder after retries and fallback, visible on CLI/TUI/gateway/API/cron (#85426, #107307)

  merged to main

  files: agent/AGENTS.md, agent/agent_init.py, agent/turn_api_error.py, agent/turn_recovery_autorecover.py, agent/turn_retry_state.py, cli-config.yaml.example, gateway/platforms/api_server.py, gateway/platforms/api_server_openai_routes.py, hermes_cli/config_defaults.py, tests/agent/test_turn_recovery_autorecover.py, tests/agent/test_turn_retry_state.py, tests/gateway/test_api_server_status_stream.py, website/docs/user-guide/configuration.md