{"repo":"NousResearch/hermes-agent","public_id":"1826","display_label":"Retry backoff, rate-limit handling, and local endpoint timeouts","summary":"These items address HTTP 429/503/529 retry logic with configurable backoff and user-friendly messaging, Discord chunk-drop bugs on rate limits, and stale-stream timeout behavior for local/Docker/mDNS provider endpoints.","subsystem":"agent","counts":{"members":33,"open_issues":6,"open_issues_distinct":6,"open_prs":5,"closed_issues":6,"merged_prs":6,"attempt_prs":15},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":3809},"resolution":{"state":"partially_resolved","by":"pr-6368,pr-9248"},"freshness":{"state":"verified","last_verified_at":"2026-09-19T21:33:22+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.047954545454545444},"fix_edges":[{"issue_number":1826,"pr_number":3809,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":2642,"pr_number":3809,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":5889,"pr_number":6368,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":7905,"pr_number":7950,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":7905,"pr_number":9248,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":29702,"pr_number":77607,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":33693,"pr_number":116353,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":55540,"pr_number":116353,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":85426,"pr_number":116353,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":94978,"pr_number":116353,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":107307,"pr_number":116353,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":1826,"title":"[UX] Rate limit errors (429) should show user-friendly message with configurable retry instead of traceback","url":null,"state":"closed"},{"kind":"issue","number":4657,"title":"[Feature]: Configurable server retry interval","url":null,"state":"closed"},{"kind":"issue","number":5570,"title":"feat: configurable max API retries + stream retries with smarter backoff","url":null,"state":"open"},{"kind":"issue","number":5889,"title":"[Bug]:Hermes reconnects after 180s of provider silence even though oMLX is still processing","url":null,"state":"closed"},{"kind":"issue","number":7069,"title":"[Bug]: Infinite retry loop caused by stream stale timeout during local LLM prefill phase","url":null,"state":"closed"},{"kind":"issue","number":7905,"title":"is_local_endpoint misses Docker/Podman DNS names — stale timeout fires on local LLM proxies","url":null,"state":"closed"},{"kind":"issue","number":20346,"title":"[Bug]: using DNS for local provider is not addressed in recent fix","url":null,"state":"open"},{"kind":"issue","number":29702,"title":"Feature Request: Configurable model provider retry interval / backoff","url":null,"state":"closed"},{"kind":"issue","number":44468,"title":"[Bug]: `hermes send` to Discord drops remaining chunks on a 429 — no retry in the standalone send path (Telegram has one)","url":null,"state":"open"},{"kind":"issue","number":49031,"title":"[Feature Request] Configurable retry backoff for 429 rate-limit errors","url":null,"state":"open"},{"kind":"issue","number":54875,"title":"Configurable retry backoff parameters (base_delay, max_delay) for HTTP 429","url":null,"state":"open"},{"kind":"issue","number":55540,"title":"[Feature]: Longer/configurable backoff for HTTP 503/529 provider overload (parity with Z.AI path)","url":null,"state":"open"},{"kind":"pr","number":1835,"title":"fix(agent): user-friendly 429 rate limit messages with countdown and Retry-After support","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#3809","number":3809}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":3809,"title":"fix(agent): user-friendly 429 rate limit messages with Retry-After support","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["run_agent.py"],"fix_hunks":[{"file":"run_agent.py","hunks":["@@ -7229,7 +7229,10 @@ def _stop_spinner():\n                             retry_count = 0\n                             continue\n                         _final_summary = self._summarize_api_error(api_error)\n-                        self._vprint(f\"{self.log_prefix}❌ Max retries ({max_retries}) exceeded. Giving up.\", force=True)\n+                        if is_rate_limited:\n+                            self._vprint(f\"{self.log_prefix}❌ Rate limit persisted after {max_retries} retries. Please try again later.\", force=True)\n+                        else:\n+                            self._vprint(f\"{self.log_prefix}❌ Max retries ({max_retries}) exceeded. Giving up.\", force=True)\n                         self._vprint(f\"{self.log_prefix}   💀 Final error: {_final_summary}\", force=True)\n \n                         # Detect SSE stream-drop pattern (e.g. \"Network","@@ -7289,8 +7292,22 @@ def _stop_spinner():\n                             \"error\": _final_summary,\n                         }\n \n-                    wait_time = min(2 ** retry_count, 60)  # Exponential backoff: 2s, 4s, 8s, 16s, 32s, 60s, 60s\n-                    self._emit_status(f\"⏳ Retrying in {wait_time}s (attempt {retry_count}/{max_retries})...\")\n+                    # For rate limits, respect the Retry-After header if present\n+                    _retry_after = None\n+                    if is_rate_limited:\n+                        _resp_headers = getattr(getattr(api_error, \"response\", None), \"headers\", None)\n+                        if _resp_headers and hasattr(_resp_headers, \"get\"):\n+                            _ra_raw = _resp_headers.get(\"retry-after\") or _resp_headers.get(\"Retry-After\")\n+                            if _ra_raw:\n+                                try:\n+                                    _retry_after = min(int(_ra_raw), 120)  # Cap at 2 minutes\n+                                except (TypeError, ValueError):\n+                                    pass\n+                    wait_time = _retry_after if _retry_after else min(2 ** retry_count, 60)\n+                    if is_rate_limited:\n+                        self._emit_status(f\"⏱️ Rate limit reached. Waiting {wait_time}s before retry (attempt {retry_count + 1}/{max_retries})...\")\n+                    else:\n+                        self._emit_status(f\"⏳ Retrying in {wait_time}s (attempt {retry_count}/{max_retries})...\")\n                     logger.warning(\n                         \"Retrying API call in %ss (attempt %s/%s) %s error=%s\",\n                         wait_time,\n"]}]},{"kind":"pr","number":6123,"title":"fix(run_agent): disable stale timeout for local providers (#5889)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#6368","number":6368},{"kind":"pr","value":"#5889","number":5889}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":6368,"title":"fix(agent): disable stale stream timeout for local providers","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["run_agent.py"]},{"kind":"pr","number":7431,"title":"fix(agent): add exponential backoff to inner streaming retry loop","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"985e19c110f19e1131b52f78c6505e2a06837127","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":7906,"title":"fix: is_local_endpoint misses Docker/Podman DNS names","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#7950","number":7950},{"kind":"pr","value":"#9248","number":9248}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":7950,"title":"fix: is_local_endpoint misses Docker/Podman DNS names","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/model_metadata.py","tests/agent/test_local_stream_timeout.py","tests/tools/test_tool_result_storage.py","tools/tool_result_storage.py"],"fix_hunks":[{"file":"agent/model_metadata.py","hunks":["@@ -179,6 +179,12 @@ def _strip_provider_prefix(model: str) -> str:\n \n # Local server hostnames / address patterns\n _LOCAL_HOSTS = (\"localhost\", \"127.0.0.1\", \"::1\", \"0.0.0.0\")\n+# Docker / Podman / Lima DNS names that resolve to the host machine\n+_CONTAINER_LOCAL_SUFFIXES = (\n+    \".docker.internal\",\n+    \".containers.internal\",\n+    \".lima.internal\",\n+)\n \n \n def _normalize_base_url(base_url: str) -> str:","@@ -254,6 +260,9 @@ def is_local_endpoint(base_url: str) -> bool:\n         return False\n     if host in _LOCAL_HOSTS:\n         return True\n+    # Docker / Podman / Lima internal DNS names (e.g. host.docker.internal)\n+    if any(host.endswith(suffix) for suffix in _CONTAINER_LOCAL_SUFFIXES):\n+        return True\n     # RFC-1918 private ranges and link-local\n     import ipaddress\n     try:"]},{"file":"tests/agent/test_local_stream_timeout.py","hunks":["@@ -22,6 +22,9 @@ class TestLocalStreamReadTimeout:\n         \"http://0.0.0.0:5000\",\n         \"http://192.168.1.100:8000\",\n         \"http://10.0.0.5:1234\",\n+        \"http://host.docker.internal:11434\",\n+        \"http://host.containers.internal:11434\",\n+        \"http://host.lima.internal:11434\",\n     ])\n     def test_local_endpoint_bumps_read_timeout(self, base_url):\n         \"\"\"Local endpoint + default timeout -> bumps to base_timeout.\"\"\"","@@ -68,3 +71,38 @@ def test_empty_base_url_keeps_default(self):\n             if _stream_read_timeout == 120.0 and base_url and is_local_endpoint(base_url):\n                 _stream_read_timeout = _base_timeout\n             assert _stream_read_timeout == 120.0\n+\n+\n+class TestIsLocalEndpoint:\n+    \"\"\"Direct unit tests for is_local_endpoint.\"\"\"\n+\n+    @pytest.mark.parametrize(\"url\", [\n+        \"http://localhost:11434\",\n+        \"http://127.0.0.1:8080\",\n+        \"http://0.0.0.0:5000\",\n+        \"http://[::1]:11434\",\n+        \"http://192.168.1.100:8000\",\n+        \"http://10.0.0.5:1234\",\n+        \"http://172.17.0.1:11434\",\n+    ])\n+    def test_classic_local_addresses(self, url):\n+        assert is_local_endpoint(url) is True\n+\n+    @pytest.mark.parametrize(\"url\", [\n+        \"http://host.docker.internal:11434\",\n+        \"http://host.docker.internal:8080/v1\",\n+        \"http://gateway.docker.internal:11434\",\n+        \"http://host.containers.internal:11434\",\n+        \"http://host.lima.internal:11434\",\n+    ])\n+    def test_container_dns_names(self, url):\n+        assert is_local_endpoint(url) is True\n+\n+    @pytest.mark.parametrize(\"url\", [\n+        \"https://api.openai.com\",\n+        \"https://openrouter.ai/api\",\n+        \"https://api.anthropic.com\",\n+        \"https://evil.docker.internal.example.com\",\n+    ])\n+    def test_remote_endpoints(self, url):\n+        assert is_local_endpoint(url) is False"]},{"file":"tests/tools/test_tool_result_storage.py","hunks":["@@ -124,6 +124,34 @@ def test_uses_parent_dir_of_remote_path(self):\n         cmd = env.execute.call_args[0][0]\n         assert \"mkdir -p /data/data/com.termux/files/usr/tmp/hermes-results\" in cmd\n \n+    def test_path_with_spaces_is_quoted(self):\n+        env = MagicMock()\n+        env.execute.return_value = {\"output\": \"\", \"returncode\": 0}\n+        remote_path = \"/tmp/hermes results/abc file.txt\"\n+        _write_to_sandbox(\"content\", remote_path, env)\n+        cmd = env.execute.call_args[0][0]\n+        assert \"'/tmp/hermes results'\" in cmd\n+        assert \"'/tmp/hermes results/abc file.txt'\" in cmd\n+\n+    def test_shell_metacharacters_neutralized(self):\n+        \"\"\"Paths with shell metacharacters must be quoted to prevent injection.\"\"\"\n+        env = MagicMock()\n+        env.execute.return_value = {\"output\": \"\", \"returncode\": 0}\n+        malicious_path = \"/tmp/hermes-results/$(whoami).txt\"\n+        _write_to_sandbox(\"content\", malicious_path, env)\n+        cmd = env.execute.call_args[0][0]\n+        # The $() must not appear unquoted — shlex.quote wraps it\n+        assert \"'/tmp/hermes-results/$(whoami).txt'\" in cmd\n+\n+    def test_semicolon_injection_neutralized(self):\n+        env = MagicMock()\n+        env.execute.return_value = {\"output\": \"\", \"returncode\": 0}\n+        malicious_path = \"/tmp/x; rm -rf /; echo .txt\"\n+        _write_to_sandbox(\"content\", malicious_path, env)\n+        cmd = env.execute.call_args[0][0]\n+        # The semicolons must be inside quotes, not acting as command separators\n+        assert \"'/tmp/x; rm -rf /; echo .txt'\" in cmd\n+\n \n class TestResolveStorageDir:\n     def test_defaults_to_storage_dir_without_env(self):"]},{"file":"tools/tool_result_storage.py","hunks":["@@ -24,6 +24,7 @@\n \n import logging\n import os\n+import shlex\n import uuid\n \n from tools.budget_config import (","@@ -79,7 +80,7 @@ def _write_to_sandbox(content: str, remote_path: str, env) -> bool:\n     marker = _heredoc_marker(content)\n     storage_dir = os.path.dirname(remote_path)\n     cmd = (\n-        f\"mkdir -p {storage_dir} && cat > {remote_path} << '{marker}'\\n\"\n+        f\"mkdir -p {shlex.quote(storage_dir)} && cat > {shlex.quote(remote_path)} << '{marker}'\\n\"\n         f\"{content}\\n\"\n         f\"{marker}\"\n     )\n"]}]},{"kind":"pr","number":8021,"title":"fix(agent): detect Docker/Podman DNS names in is_local_endpoint","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#7906","number":7906}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":8486,"title":"feat: configurable max API retries + stream retries with smarter backoff","url":null,"state":"open","close_reason":{"refs":[{"kind":"commit","value":"165b2e481afa1cd7385c5c9f9ebf33eb4a524131","number":0},{"kind":"pr","value":"#14730","number":14730}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":9248,"title":"fix(local): recognize unqualified hostnames as local endpoints","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/model_metadata.py","tests/agent/test_local_stream_timeout.py"]},{"kind":"pr","number":13519,"title":"Amindadgar PR#8486 retry ci fix","url":null,"state":"open","close_reason":{"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":20359,"title":"fix(agent): treat mDNS/.lan/unqualified hostnames as local endpoints (#20346)","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#9248","number":9248}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":30223,"title":"fix: exponential backoff for rate-limit fallback cooldown","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#77607","number":77607}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":44488,"title":"fix(discord): retry 429s in the standalone send path, honoring retry_after","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#79564","number":79564}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":44542,"title":"fix(discord): retry standalone send on 429 rate-limit instead of dropping chunks","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#44488","number":44488}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":44558,"title":"fix(tools): retry Discord send on 429 rate-limit (#44468)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#44488","number":44488},{"kind":"pr","value":"#44468","number":44468}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":56037,"title":"feat(agent): make overload (503/529) backoff configurable","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"059b3d416f","number":0},{"kind":"pr","value":"#116353","number":116353}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":77607,"title":"fix(agent): exponential backoff for rate-limit fallback cooldown (#30223 salvage)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/agent_runtime_helpers.py","agent/chat_completion_helpers.py","tests/run_agent/test_24996_fallback_exhaustion_cooldown.py"]},{"kind":"pr","number":102551,"title":"feat(agent): configurable rate-limit retry policy for 429s","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#46912","number":46912},{"kind":"pr","value":"#49031","number":49031},{"kind":"pr","value":"#70738","number":70738}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":109329,"title":"fix(discord): retry standalone sends after rate limits","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#44468","number":44468},{"kind":"pr","value":"#44488","number":44488},{"kind":"pr","value":"#44542","number":44542},{"kind":"pr","value":"#79564","number":79564},{"kind":"pr","value":"#109384","number":109384},{"kind":"pr","value":"#35647","number":35647}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":116353,"title":"Transient provider outages no longer end the turn: bounded auto-recovery ladder after retries and fallback, visible on CLI/TUI/gateway/API/cron (#85426, #107307)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/AGENTS.md","agent/agent_init.py","agent/turn_api_error.py","agent/turn_recovery_autorecover.py","agent/turn_retry_state.py","cli-config.yaml.example","gateway/platforms/api_server.py","gateway/platforms/api_server_openai_routes.py","hermes_cli/config_defaults.py","tests/agent/test_turn_recovery_autorecover.py","tests/agent/test_turn_retry_state.py","tests/gateway/test_api_server_status_stream.py","website/docs/user-guide/configuration.md"]}],"trust":{"default_tier":"verified","includes_suggested":false}}