{"repo":"NousResearch/hermes-agent","public_id":"42483","display_label":"Web extraction summarization truncation and config changes","summary":"These items address LLM summarization truncation behavior in web extraction and introduce configurable thresholds, alongside auto-setting the extract backend when Tavily is selected.","subsystem":"agent","counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":54843},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-29T17:00:50+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.4},"fix_edges":[{"issue_number":42483,"pr_number":54843,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":42483,"title":"feat(web_extract): fix LLM summarization truncation + make it configurable","url":null,"state":"closed"},{"kind":"pr","number":42495,"title":"feat(setup): auto-set web.extract_backend when Tavily is selected as web backend","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":42599,"title":"feat(web): make web_extract LLM summarization thresholds configurable","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"ee8cbfdc03eb9b7cdd486165486f2e3cad0d8645","number":0},{"kind":"pr","value":"#54843","number":54843},{"kind":"pr","value":"#42483","number":42483}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":42671,"title":"fix(web): fix LLM summarization truncation and make configurable","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"ee8cbfdc03eb9b7cdd486165486f2e3cad0d8645","number":0},{"kind":"pr","value":"#42483","number":42483}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":54843,"title":"feat(web_extract): truncate-and-store instead of LLM summarization","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/context_references.py","hermes_cli/config.py","tests/integration/test_web_tools.py","tests/tools/test_browser_secret_exfil.py","tests/tools/test_web_providers.py","tests/tools/test_web_tools_config.py","tests/tools/test_web_tools_tavily.py","tests/tools/test_web_tools_truncate.py","tests/tools/test_website_policy.py","tools/code_execution_tool.py","tools/credential_files.py","tools/web_tools.py"],"fix_hunks":[{"file":"agent/context_references.py","hunks":["@@ -328,9 +328,9 @@ async def _fetch_url_content(\n async def _default_url_fetcher(url: str) -> str:\n     from tools.web_tools import web_extract_tool\n \n-    raw = await web_extract_tool([url], format=\"markdown\", use_llm_processing=True)\n+    raw = await web_extract_tool([url], format=\"markdown\")\n     payload = json.loads(raw)\n-    docs = payload.get(\"data\", {}).get(\"documents\", [])\n+    docs = payload.get(\"results\", [])\n     if not docs:\n         return \"\"\n     doc = docs[0]"]},{"file":"hermes_cli/config.py","hunks":["@@ -1161,6 +1161,7 @@ def _ensure_hermes_home_managed(home: Path):\n         \"backend\": \"\",           # shared fallback — applies to both search and extract\n         \"search_backend\": \"\",    # per-capability override for web_search (e.g. \"searxng\")\n         \"extract_backend\": \"\",   # per-capability override for web_extract (e.g. \"native\")\n+        \"extract_char_limit\": 15000,  # per-page char budget for web_extract; larger pages truncate + store full text in cache/web\n     },\n \n     \"browser\": {"]},{"file":"tests/integration/test_web_tools.py","hunks":["@@ -32,7 +32,6 @@\n     web_extract_tool,\n     check_firecrawl_api_key,\n     check_web_api_key,\n-    check_auxiliary_model,\n     _get_backend,\n )\n ","@@ -129,12 +128,11 @@ def test_environment(self) -> bool:\n             backend = _get_backend()\n             self.log_result(\"Web Backend API Key\", \"passed\", f\"Using {backend} backend\")\n         \n-        # Check auxiliary LLM provider (optional)\n-        if not check_auxiliary_model():\n-            self.log_result(\"Auxiliary LLM\", \"skipped\", \"No auxiliary LLM provider available (LLM tests will be skipped)\")\n-            self.test_llm = False\n-        else:\n-            self.log_result(\"Auxiliary LLM\", \"passed\", \"Found\")\n+        # Auxiliary LLM summarization was removed — web_extract is now\n+        # truncate-and-store (no LLM). Keep the flag off so any residual\n+        # LLM-path assertions stay skipped.\n+        self.log_result(\"Auxiliary LLM\", \"skipped\", \"web_extract no longer uses an LLM (truncate-and-store)\")\n+        self.test_llm = False\n         \n         return True\n     ","@@ -261,12 +259,11 @@ async def test_web_extract(self, urls: List[str] = None):\n                     print(f\"    - {url}\")\n                 \n                 if self.verbose:\n-                    print(f\"  Calling web_extract_tool(urls={test_urls}, format='markdown', use_llm_processing=False)\")\n+                    print(f\"  Calling web_extract_tool(urls={test_urls}, format='markdown')\")\n                 \n                 result = await web_extract_tool(\n                     test_urls,\n                     format=\"markdown\",\n-                    use_llm_processing=False\n                 )\n                 \n                 # Parse result","@@ -360,8 +357,7 @@ async def test_web_extract_with_llm(self, urls: List[str] = None):\n             result = await web_extract_tool(\n                 [test_url],\n                 format=\"markdown\",\n-                use_llm_processing=True,\n-                min_length=1000  # Lower threshold for testing\n+                char_limit=1000,  # small budget to force truncation in the test\n             )\n             \n             data = json.loads(result)","@@ -466,7 +462,7 @@ def save_results(self):\n                 \"web_backend\": _get_backend() if check_web_api_key() else None,\n                 \"firecrawl_api_key\": check_firecrawl_api_key(),\n                 \"parallel_api_key\": bool(os.getenv(\"PARALLEL_API_KEY\")),\n-                \"auxiliary_model\": check_auxiliary_model(),\n+                \"auxiliary_model\": False,\n             }\n         }\n         "]},{"file":"tests/tools/test_browser_secret_exfil.py","hunks":["@@ -126,7 +126,6 @@ async def allow_url(_url: str) -> bool:\n         try:\n             result = await web_tools.web_extract_tool(\n                 urls=[\"https://wttr.in/Köln\"],\n-                use_llm_processing=False,\n             )\n         finally:\n             web_search_registry._reset_for_tests()"]},{"file":"tests/tools/test_web_providers.py","hunks":["@@ -418,7 +418,6 @@ def _register_fake() -> None:\n             result = json.loads(asyncio.run(\n                 web_tools.web_extract_tool(\n                     [\"https://example.com\"],\n-                    use_llm_processing=False,\n                 )\n             ))\n "]},{"file":"tests/tools/test_web_tools_config.py","hunks":["@@ -160,47 +160,6 @@ def test_nous_auth_token_respects_hermes_home_override(self, tmp_path):\n             importlib.reload(tools.web_tools)\n             assert tools.web_tools._read_nous_access_token() == \"nous-token\"\n \n-    def test_check_auxiliary_model_re_resolves_backend_each_call(self):\n-        \"\"\"Availability checks should not be pinned to module import state.\"\"\"\n-        import tools.web_tools\n-\n-        # Simulate the pre-fix import-time cache slot for regression coverage.\n-        tools.web_tools.__dict__[\"_aux_async_client\"] = None\n-\n-        with patch(\n-            \"tools.web_tools.get_async_text_auxiliary_client\",\n-            side_effect=[(None, None), (MagicMock(base_url=\"https://api.openrouter.ai/v1\"), \"test-model\")],\n-        ):\n-            assert tools.web_tools.check_auxiliary_model() is False\n-            assert tools.web_tools.check_auxiliary_model() is True\n-\n-    @pytest.mark.asyncio\n-    async def test_summarizer_re_resolves_backend_after_initial_unavailable_state(self):\n-        \"\"\"Summarization should pick up a backend that becomes available later in-process.\"\"\"\n-        import tools.web_tools\n-\n-        tools.web_tools.__dict__[\"_aux_async_client\"] = None\n-\n-        response = MagicMock()\n-        response.choices = [MagicMock(message=MagicMock(content=\"summary text\"))]\n-\n-        with patch(\n-            \"tools.web_tools._resolve_web_extract_auxiliary\",\n-            side_effect=[(None, None, {}), (MagicMock(base_url=\"https://api.openrouter.ai/v1\"), \"test-model\", {})],\n-        ), patch(\n-            \"tools.web_tools.async_call_llm\",\n-            new=AsyncMock(return_value=response),\n-        ) as mock_async_call:\n-            assert tools.web_tools.check_auxiliary_model() is False\n-            result = await tools.web_tools._call_summarizer_llm(\n-                \"Some content worth summarizing\",\n-                \"Source: https://example.com\\n\\n\",\n-                None,\n-            )\n-\n-        assert result == \"summary text\"\n-        mock_async_call.assert_awaited_once()\n-\n     # ── Singleton caching ────────────────────────────────────────────\n \n     def test_singleton_returns_same_instance(self):"]},{"file":"tests/tools/test_web_tools_tavily.py","hunks":["@@ -215,13 +215,13 @@ def test_extract_dispatches_to_tavily(self):\n \n         with patch(\"tools.web_tools._get_backend\", return_value=\"tavily\"), \\\n              patch.dict(os.environ, {\"TAVILY_API_KEY\": \"tvly-test\"}), \\\n-             patch(\"tools.web_tools.httpx.post\", return_value=mock_response), \\\n-             patch(\"tools.web_tools.process_content_with_llm\", return_value=None):\n+             patch(\"tools.web_tools.httpx.post\", return_value=mock_response):\n             from tools.web_tools import web_extract_tool\n             result = json.loads(asyncio.get_event_loop().run_until_complete(\n-                web_extract_tool([\"https://example.com\"], use_llm_processing=False)\n+                web_extract_tool([\"https://example.com\"])\n             ))\n             assert \"results\" in result\n             assert len(result[\"results\"]) == 1\n             assert result[\"results\"][0][\"url\"] == \"https://example.com\"\n+            assert \"Extracted content\" in result[\"results\"][0][\"content\"]\n "]},{"file":"tests/tools/test_web_tools_truncate.py","hunks":["@@ -0,0 +1,142 @@\n+\"\"\"Unit tests for the truncate-and-store web_extract path (no LLM).\n+\n+Covers convert_base64_images_to_links, _truncate_with_footer, _store_full_text,\n+_get_extract_char_limit, and the end-to-end web_extract_tool truncation behavior.\n+\"\"\"\n+import asyncio\n+import json\n+import os\n+from unittest.mock import patch\n+\n+import pytest\n+\n+import tools.web_tools as wt\n+\n+\n+class TestImageConversion:\n+    def test_markdown_base64_image_keeps_alt_drops_blob(self):\n+        blob = \"A\" * 5000\n+        text = f\"before ![a cat]( data:image/png;base64,{blob}) after\"\n+        out = wt.convert_base64_images_to_links(text)\n+        assert \"[IMAGE: a cat]\" in out\n+        assert \"base64\" not in out\n+        assert blob not in out\n+        assert \"before\" in out and \"after\" in out\n+\n+    def test_markdown_base64_image_no_alt(self):\n+        out = wt.convert_base64_images_to_links(\"x ![](data:image/jpeg;base64,QQ==) y\")\n+        assert \"[IMAGE]\" in out\n+        assert \"base64\" not in out\n+\n+    def test_real_http_image_links_preserved(self):\n+        text = \"see ![logo](https://example.com/logo.png) here\"\n+        out = wt.convert_base64_images_to_links(text)\n+        # Real image URLs must survive so the agent can inspect them.\n+        assert \"![logo](https://example.com/logo.png)\" in out\n+\n+    def test_bare_and_parenthesised_base64_become_placeholder(self):\n+        blob = \"Z\" * 3000\n+        bare = wt.convert_base64_images_to_links(f\"data:image/gif;base64,{blob}\")\n+        assert bare == \"[IMAGE]\"\n+        paren = wt.convert_base64_images_to_links(f\"(data:image/gif;base64,{blob})\")\n+        assert paren == \"[IMAGE]\"\n+\n+\n+class TestTruncation:\n+    def test_short_content_returned_whole(self):\n+        content = \"# Title\\n\\nshort body\\n\"\n+        out, truncated = wt._truncate_with_footer(content, \"https://e.com\", 15000)\n+        assert out == content\n+        assert truncated is False\n+\n+    def test_long_content_truncated_with_footer(self, tmp_path, monkeypatch):\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path / \".hermes\"))\n+        body = \"\\n\".join(f\"line {i} \" + \"x\" * 50 for i in range(2000))\n+        out, truncated = wt._truncate_with_footer(body, \"https://example.com/page\", 4000)\n+        assert truncated is True\n+        assert \"[TRUNCATED]\" in out\n+        assert \"Full text saved to:\" in out\n+        assert \"read_file\" in out\n+        # Head and tail are both present (first and last lines survive).\n+        assert \"line 0 \" in out\n+        assert \"line 1999 \" in out\n+        # The omitted middle is gone.\n+        assert \"line 1000 \" not in out\n+        # Sent text is bounded near the budget (+ footer overhead).\n+        assert len(out) < 4000 + 2000\n+\n+    def test_truncation_stores_full_text_readable(self, tmp_path, monkeypatch):\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path / \".hermes\"))\n+        body = \"UNIQUE_MIDDLE_MARKER\\n\" + (\"\\n\".join(f\"row {i}\" for i in range(5000)))\n+        out, truncated = wt._truncate_with_footer(body, \"https://example.com/doc\", 3000)\n+        assert truncated is True\n+        # Extract the stored path from the footer and confirm full text is there.\n+        path_line = next(ln for ln in out.splitlines() if \"Full text saved to:\" in ln)\n+        stored_path = path_line.split(\"Full text saved to:\", 1)[1].strip()\n+        assert os.path.exists(stored_path)\n+        full = open(stored_path).read()\n+        assert \"UNIQUE_MIDDLE_MARKER\" in full\n+        assert \"row 2500\" in full  # the omitted-middle row is in the stored file\n+\n+\n+class TestCharLimitConfig:\n+    def test_default_when_unset(self):\n+        with patch(\"tools.web_tools._load_web_config\", return_value={}):\n+            assert wt._get_extract_char_limit() == wt.DEFAULT_EXTRACT_CHAR_LIMIT\n+\n+    def test_config_override(self):\n+        with patch(\"tools.web_tools._load_web_config\", return_value={\"extract_char_limit\": 40000}):\n+            assert wt._get_extract_char_limit() == 40000\n+\n+    def test_clamps_floor(self):\n+        with patch(\"tools.web_tools._load_web_config\", return_value={\"extract_char_limit\": 100}):\n+            assert wt._get_extract_char_limit() == 2000\n+\n+    def test_bad_value_falls_back(self):\n+        with patch(\"tools.web_tools._load_web_config\", return_value={\"extract_char_limit\": \"nope\"}):\n+            assert wt._get_extract_char_limit() == wt.DEFAULT_EXTRACT_CHAR_LIMIT\n+\n+\n+class TestEndToEnd:\n+    def test_web_extract_truncates_large_page_no_llm(self, tmp_path, monkeypatch):\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path / \".hermes\"))\n+        big = \"\\n\".join(f\"para {i} \" + \"y\" * 80 for i in range(3000))\n+\n+        class FakeProvider:\n+            name = \"fake\"\n+            display_name = \"Fake\"\n+\n+            def supports_extract(self):\n+                return True\n+\n+            async def extract(self, urls, **kwargs):\n+                return [{\"url\": urls[0], \"title\": \"Big Page\", \"content\": big,\n+                         \"raw_content\": big, \"metadata\": {}}]\n+\n+        with patch(\"tools.web_tools._ensure_web_plugins_loaded\"), \\\n+             patch(\"tools.web_tools._get_extract_backend\", return_value=\"fake\"), \\\n+             patch(\"tools.web_tools.async_is_safe_url\", new=_AsyncTrue()), \\\n+             patch(\"agent.web_search_registry.get_provider\", return_value=FakeProvider()):\n+            result = json.loads(asyncio.new_event_loop().run_until_complete(\n+                wt.web_extract_tool([\"https://example.com/big\"], char_limit=5000)\n+            ))\n+\n+        assert \"results\" in result\n+        content = result[\"results\"][0][\"content\"]\n+        assert \"[TRUNCATED]\" in content\n+        assert \"Full text saved to:\" in content\n+        # No LLM was involved: para 0 (head) and the last para (tail) are verbatim.\n+        assert \"para 0 \" in content\n+        assert \"para 2999 \" in content\n+\n+\n+def _make_awaitable(value):\n+    async def _coro(*a, **k):\n+        return value\n+    return _coro()\n+\n+\n+class _AsyncTrue:\n+    \"\"\"Async callable that always returns True (re-awaitable per call).\"\"\"\n+    async def __call__(self, *a, **k):\n+        return True"]},{"file":"tests/tools/test_website_policy.py","hunks":["@@ -398,7 +398,7 @@ async def _allow_ssrf(_url: str) -> bool:\n         # Force the firecrawl plugin to be the active extract provider.\n         monkeypatch.setenv(\"FIRECRAWL_API_KEY\", \"fake-key\")\n \n-        result = json.loads(await web_tools.web_extract_tool([\"https://blocked.test\"], use_llm_processing=False))\n+        result = json.loads(await web_tools.web_extract_tool([\"https://blocked.test\"]))\n \n         assert result[\"results\"][0][\"url\"] == \"https://blocked.test\"\n         assert \"Blocked by website policy\" in result[\"results\"][0][\"error\"]","@@ -443,7 +443,7 @@ def scrape(self, url, formats):\n         monkeypatch.setattr(\"tools.interrupt.is_interrupted\", lambda: False)\n         monkeypatch.setenv(\"FIRECRAWL_API_KEY\", \"fake-key\")\n \n-        result = json.loads(await web_tools.web_extract_tool([\"https://allowed.test\"], use_llm_processing=False))\n+        result = json.loads(await web_tools.web_extract_tool([\"https://allowed.test\"]))\n \n         assert result[\"results\"][0][\"url\"] == \"https://blocked.test/final\"\n         assert result[\"results\"][0][\"content\"] == \"\""]},{"file":"tools/code_execution_tool.py","hunks":["@@ -219,9 +219,9 @@ def check_sandbox_requirements() -> bool:\n     ),\n     \"web_extract\": (\n         \"web_extract\",\n-        \"urls: list\",\n-        '\"\"\"Extract content from URLs. Returns dict with results list of {url, title, content, error}.\"\"\"',\n-        '{\"urls\": urls}',\n+        \"urls: list, char_limit: int = None\",\n+        '\"\"\"Extract content from URLs (no LLM summarization). Returns dict with results list of {url, title, content, error}. Pages over char_limit (default 15000) are head+tail truncated with the full text stored on disk; the content footer gives the path. content is markdown.\"\"\"',\n+        '{\"urls\": urls, \"char_limit\": char_limit}',\n     ),\n     \"read_file\": (\n         \"read_file\",","@@ -1727,8 +1727,9 @@ def _resolve_child_cwd(mode: str, staging_dir: str) -> str:\n      \"  web_search(query: str, limit: int = 5) -> dict\\n\"\n      \"    Returns {\\\"data\\\": {\\\"web\\\": [{\\\"url\\\", \\\"title\\\", \\\"description\\\"}, ...]}}\"),\n     (\"web_extract\",\n-     \"  web_extract(urls: list[str]) -> dict\\n\"\n-     \"    Returns {\\\"results\\\": [{\\\"url\\\", \\\"title\\\", \\\"content\\\", \\\"error\\\"}, ...]} where content is markdown\"),\n+     \"  web_extract(urls: list[str], char_limit: int = None) -> dict\\n\"\n+     \"    Returns {\\\"results\\\": [{\\\"url\\\", \\\"title\\\", \\\"content\\\", \\\"error\\\"}, ...]} where content is markdown.\\n\"\n+     \"    No LLM summarization. Pages over char_limit (default 15000) are head+tail truncated; full text stored on disk (path in the content footer).\"),\n     (\"read_file\",\n      \"  read_file(path: str, offset: int = 1, limit: int = 500) -> dict\\n\"\n      \"    Lines are 1-indexed. Returns {\\\"content\\\": \\\"...\\\", \\\"total_lines\\\": N}\"),"]},{"file":"tools/credential_files.py","hunks":["@@ -349,6 +349,7 @@ def iter_skills_files(\n     (\"cache/audio\", \"audio_cache\"),\n     (\"cache/videos\", \"video_cache\"),\n     (\"cache/screenshots\", \"browser_screenshots\"),\n+    (\"cache/web\", \"web_cache\"),\n ]\n \n "]},{"file":"tools/web_tools.py","hunks":["@@ -83,11 +83,6 @@\n _async_parallel_client: Optional[Any] = None\n _exa_client: Optional[Any] = None\n \n-from agent.auxiliary_client import (\n-    async_call_llm,\n-    extract_content_or_reasoning,\n-    get_async_text_auxiliary_client,\n-)\n from tools.debug_helpers import DebugSession\n # Imported solely so unit tests can monkeypatch these names on\n # tools.web_tools (the firecrawl plugin reads them via its own import chain).","@@ -305,445 +300,144 @@ def _web_requires_env() -> list[str]:\n # unit-test patches.\n \n \n-DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION = 5000\n-\n-def _is_nous_auxiliary_client(client: Any) -> bool:\n-    \"\"\"Return True when the resolved auxiliary backend is Nous Portal.\"\"\"\n-    from urllib.parse import urlparse\n-\n-    base_url = str(getattr(client, \"base_url\", \"\") or \"\")\n-    host = (urlparse(base_url).hostname or \"\").lower()\n-    return host == \"nousresearch.com\" or host.endswith(\".nousresearch.com\")\n-\n-\n-def _resolve_web_extract_auxiliary(model: Optional[str] = None) -> tuple[Optional[Any], Optional[str], Dict[str, Any]]:\n-    \"\"\"Resolve the current web-extract auxiliary client, model, and extra body.\"\"\"\n-    client, default_model = get_async_text_auxiliary_client(\"web_extract\")\n-    configured_model = os.getenv(\"AUXILIARY_WEB_EXTRACT_MODEL\", \"\").strip()\n-    effective_model = model or configured_model or default_model\n-\n-    extra_body: Dict[str, Any] = {}\n-    if client is not None and _is_nous_auxiliary_client(client):\n-        from agent.auxiliary_client import get_auxiliary_extra_body\n-        from agent.portal_tags import nous_portal_tags\n-        extra_body = get_auxiliary_extra_body() or {\"tags\": nous_portal_tags()}\n-\n-    return client, effective_model, extra_body\n-\n-\n-def _get_default_summarizer_model() -> Optional[str]:\n-    \"\"\"Return the current default model for web extraction summarization.\"\"\"\n-    _, model, _ = _resolve_web_extract_auxiliary()\n-    return model\n+# Default budget (characters) of clean page text sent to the model. Pages at\n+# or under this size are returned whole; larger pages are head+tail truncated\n+# and the full text is stored on disk (see _store_full_text). Spending context,\n+# not API dollars — so this is generous relative to the old 5k summary cap.\n+# Override via web.extract_char_limit in config.yaml.\n+DEFAULT_EXTRACT_CHAR_LIMIT = 15000\n \n _debug = DebugSession(\"web_tools\", env_var=\"WEB_TOOLS_DEBUG\")\n \n \n-async def process_content_with_llm(\n-    content: str, \n-    url: str = \"\", \n-    title: str = \"\",\n-    model: Optional[str] = None,\n-    min_length: int = DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION\n-) -> Optional[str]:\n-    \"\"\"\n-    Process web content using LLM to create intelligent summaries with key excerpts.\n-    \n-    This function uses Gemini 3 Flash Preview (or specified model) via OpenRouter API \n-    to intelligently extract key information and create markdown summaries,\n-    significantly reducing token usage while preserving all important information.\n-    \n-    For very large content (>500k chars), uses chunked processing with synthesis.\n-    For extremely large content (>2M chars), refuses to process entirely.\n-    \n-    Args:\n-        content (str): The raw content to process\n-        url (str): The source URL (for context, optional)\n-        title (str): The page title (for context, optional)\n-        model (str): The model to use for processing (default: google/gemini-3-flash-preview)\n-        min_length (int): Minimum content length to trigger processing (default: 5000)\n-        \n-    Returns:\n-        Optional[str]: Processed markdown content, or None if content too short or processing fails\n-    \"\"\"\n-    # Size thresholds\n-    MAX_CONTENT_SIZE = 2_000_000  # 2M chars - refuse entirely above this\n-    CHUNK_THRESHOLD = 500_000     # 500k chars - use chunked processing above this\n-    CHUNK_SIZE = 100_000          # 100k chars per chunk\n-    MAX_OUTPUT_SIZE = 5000        # Hard cap on final output size\n-    \n+def _get_extract_char_limit() -> int:\n+    \"\"\"Resolve the per-page char budget from config, clamped to a sane range.\"\"\"\n     try:\n-        content_len = len(content)\n-        \n-        # Refuse if content is absurdly large\n-        if content_len > MAX_CONTENT_SIZE:\n-            size_mb = content_len / 1_000_000\n-            logger.warning(\"Content too large (%.1fMB > 2MB limit). Refusing to process.\", size_mb)\n-            return f\"[Content too large to process: {size_mb:.1f}MB. Try a more focused source URL.]\"\n-        \n-        # Skip processing if content is too short\n-        if content_len < min_length:\n-            logger.debug(\"Content too short (%d < %d chars), skipping LLM processing\", content_len, min_length)\n-            return None\n-        \n-        # Create context information\n-        context_info = []\n-        if title:\n-            context_info.append(f\"Title: {title}\")\n-        if url:\n-            context_info.append(f\"Source: {url}\")\n-        context_str = \"\\n\".join(context_info) + \"\\n\\n\" if context_info else \"\"\n-        \n-        # Check if we need chunked processing\n-        if content_len > CHUNK_THRESHOLD:\n-            logger.info(\"Content large (%d chars). Using chunked processing...\", content_len)\n-            return await _process_large_content_chunked(\n-                content, context_str, model, CHUNK_SIZE, MAX_OUTPUT_SIZE\n-            )\n-        \n-        # Standard single-pass processing for normal content\n-        logger.info(\"Processing content with LLM (%d characters)\", content_len)\n-        \n-        processed_content = await _call_summarizer_llm(content, context_str, model)\n-        \n-        if processed_content:\n-            # Enforce output cap\n-            if len(processed_content) > MAX_OUTPUT_SIZE:\n-                processed_content = processed_content[:MAX_OUTPUT_SIZE] + \"\\n\\n[... summary truncated for context management ...]\"\n-            \n-            # Log compression metrics\n-            processed_length = len(processed_content)\n-            compression_ratio = processed_length / content_len if content_len > 0 else 1.0\n-            logger.info(\"Content processed: %d -> %d chars (%.1f%%)\", content_len, processed_length, compression_ratio * 100)\n-        \n-        return processed_content\n-        \n-    except Exception as e:\n-        logger.warning(\n-            \"web_extract LLM summarization failed (%s). \"\n-            \"Tip: increase auxiliary.web_extract.timeout in config.yaml \"\n-            \"or switch to a faster auxiliary model.\",\n-            str(e)[:120],\n-        )\n-        # Fall back to truncated raw content instead of returning a useless\n-        # error message.  The first ~5000 chars are almost always more useful\n-        # to the model than \"[Failed to process content: ...]\".\n-        truncated = content[:MAX_OUTPUT_SIZE]\n-        if len(content) > MAX_OUTPUT_SIZE:\n-            truncated += (\n-                f\"\\n\\n[Content truncated — showing first {MAX_OUTPUT_SIZE:,} of \"\n-                f\"{len(content):,} chars. LLM summarization timed out. \"\n-                f\"To fix: increase auxiliary.web_extract.timeout in config.yaml, \"\n-                f\"or use a faster auxiliary model. Use browser_navigate for the full page.]\"\n-            )\n-        return truncated\n-\n-\n-async def _call_summarizer_llm(\n-    content: str, \n-    context_str: str, \n-    model: Optional[str], \n-    max_tokens: int = 20000,\n-    is_chunk: bool = False,\n-    chunk_info: str = \"\"\n-) -> Optional[str]:\n-    \"\"\"\n-    Make a single LLM call to summarize content.\n-    \n-    Args:\n-        content: The content to summarize\n-        context_str: Context information (title, URL)\n-        model: Model to use\n-        max_tokens: Maximum output tokens\n-        is_chunk: Whether this is a chunk of a larger document\n-        chunk_info: Information about chunk position (e.g., \"Chunk 2/5\")\n-        \n-    Returns:\n-        Summarized content or None on failure\n-    \"\"\"\n-    if is_chunk:\n-        # Chunk-specific prompt - aware that this is partial content\n-        system_prompt = \"\"\"You are an expert content analyst processing a SECTION of a larger document. Your job is to extract and summarize the key information from THIS SECTION ONLY.\n-\n-Important guidelines for chunk processing:\n-1. Do NOT write introductions or conclusions - this is a partial document\n-2. Focus on extracting ALL key facts, figures, data points, and insights from this section\n-3. Preserve important quotes, code snippets, and specific details verbatim\n-4. Use bullet points and structured formatting for easy synthesis later\n-5. Note any references to other sections (e.g., \"as mentioned earlier\", \"see below\") without trying to resolve them\n-\n-Your output will be combined with summaries of other sections, so focus on thorough extraction rather than narrative flow.\"\"\"\n-\n-        user_prompt = f\"\"\"Extract key information from this SECTION of a larger document:\n-\n-{context_str}{chunk_info}\n-\n-SECTION CONTENT:\n-{content}\n-\n-Extract all important information from this section in a structured format. Focus on facts, data, insights, and key details. Do not add introductions or conclusions.\"\"\"\n-\n-    else:\n-        # Standard full-document prompt\n-        system_prompt = \"\"\"You are an expert content analyst. Your job is to process web content and create a comprehensive yet concise summary that preserves all important information while dramatically reducing bulk.\n+        configured = _load_web_config().get(\"extract_char_limit\")\n+        if configured is not None:\n+            value = int(configured)\n+            # Floor at 2k (below that the footer dominates), no hard ceiling\n+            # beyond a generous guard so a typo can't blow up context.\n+            return max(2000, min(value, 500_000))\n+    except (TypeError, ValueError):\n+        pass\n+    return DEFAULT_EXTRACT_CHAR_LIMIT\n \n-Create a well-structured markdown summary that includes:\n-1. Key excerpts (quotes, code snippets, important facts) in their original format\n-2. Comprehensive summary of all other important information\n-3. Proper markdown formatting with headers, bullets, and emphasis\n \n-Your goal is to preserve ALL important information while reducing length. Never lose key facts, figures, insights, or actionable information. Make it scannable and well-organized.\"\"\"\n+def convert_base64_images_to_links(text: str) -> str:\n+    \"\"\"Replace inline base64 image blobs with labeled markdown links.\n \n-        user_prompt = f\"\"\"Please process this web content and create a comprehensive markdown summary:\n+    base64 image payloads are token bombs (a single inline PNG can be tens of\n+    thousands of characters), so we never send the raw bytes to the model. But\n+    we preserve the fact that an image was there, and its alt text, as an\n+    inspectable placeholder. Real (http/https) markdown image links are left\n+    untouched so the agent can ``web_extract`` / ``vision_analyze`` them.\n \n-{context_str}CONTENT TO PROCESS:\n-{content}\n+    Transformations:\n+      ``![alt](data:image/png;base64,AAAA...)``  -> ``[IMAGE: alt](base64 image omitted)``\n+      ``(data:image/png;base64,AAAA...)``        -> ``[IMAGE]``\n+      bare ``data:image/...;base64,AAAA...``     -> ``[IMAGE]``\n+    \"\"\"\n+    # 1. Markdown image with base64 source -> keep alt text, drop the blob.\n+    def _md_repl(m: \"re.Match[str]\") -> str:\n+        alt = (m.group(\"alt\") or \"\").strip()\n+        return f\"[IMAGE: {alt}]\" if alt else \"[IMAGE]\"\n \n-Create a markdown summary that captures all key information in a well-organized, scannable format. Include important quotes and code snippets in their original formatting. Focus on actionable information, specific details, and unique insights.\"\"\"\n+    md_b64 = re.compile(\n+        r\"!\\[(?P<alt>[^\\]]*)\\]\\(\\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\\s]+\\)\"\n+    )\n+    out = md_b64.sub(_md_repl, text)\n \n-    # Call the LLM with retry logic — keep retries low since summarization\n-    # is a nice-to-have; the caller falls back to truncated content on failure.\n-    max_retries = 2\n-    retry_delay = 2\n-    last_error = None\n+    # 2. Parenthesised base64 (non-markdown) and 3. bare base64 -> [IMAGE].\n+    out = re.sub(r\"\\(\\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\\s]+\\)\", \"[IMAGE]\", out)\n+    out = re.sub(r\"data:image/[^;]+;base64,[A-Za-z0-9+/=]+\", \"[IMAGE]\", out)\n+    return out\n \n-    for attempt in range(max_retries):\n-        try:\n-            aux_client, effective_model, extra_body = _resolve_web_extract_auxiliary(model)\n-            if aux_client is None or not effective_model:\n-                logger.warning(\"No auxiliary model available for web content processing\")\n-                return None\n-            call_kwargs = {\n-                \"task\": \"web_extract\",\n-                \"model\": effective_model,\n-                \"messages\": [\n-                    {\"role\": \"system\", \"content\": system_prompt},\n-                    {\"role\": \"user\", \"content\": user_prompt},\n-                ],\n-                \"temperature\": 0.1,\n-                \"max_tokens\": max_tokens,\n-                # No explicit timeout — async_call_llm reads auxiliary.web_extract.timeout\n-                # from config.yaml. Fresh configs ship with 360s; if the key is absent\n-                # the runtime default is 30s (_DEFAULT_AUX_TIMEOUT in\n-                # agent/auxiliary_client.py). Users with slow local models should set\n-                # or increase auxiliary.web_extract.timeout in config.yaml.\n-            }\n-            if extra_body:\n-                call_kwargs[\"extra_body\"] = extra_body\n-            response = await async_call_llm(**call_kwargs)\n-            content = extract_content_or_reasoning(response)\n-            if content:\n-                return content\n-            # Reasoning-only / empty response — let the retry loop handle it\n-            logger.warning(\"LLM returned empty content (attempt %d/%d), retrying\", attempt + 1, max_retries)\n-            if attempt < max_retries - 1:\n-                await asyncio.sleep(retry_delay)\n-                retry_delay = min(retry_delay * 2, 60)\n-                continue\n-            return content  # Return whatever we got after exhausting retries\n-        except RuntimeError:\n-            logger.warning(\"No auxiliary model available for web content processing\")\n-            return None\n-        except Exception as api_error:\n-            last_error = api_error\n-            if attempt < max_retries - 1:\n-                logger.warning(\"LLM API call failed (attempt %d/%d): %s\", attempt + 1, max_retries, str(api_error)[:100])\n-                logger.warning(\"Retrying in %ds...\", retry_delay)\n-                await asyncio.sleep(retry_delay)\n-                retry_delay = min(retry_delay * 2, 60)\n-            else:\n-                raise last_error\n-    \n-    return None\n \n+def _store_full_text(url: str, content: str) -> Optional[str]:\n+    \"\"\"Write the full extracted page to cache/web and return its absolute path.\n \n-async def _process_large_content_chunked(\n-    content: str, \n-    context_str: str, \n-    model: Optional[str], \n-    chunk_size: int,\n-    max_output_size: int\n-) -> Optional[str]:\n-    \"\"\"\n-    Process large content by chunking, summarizing each chunk in parallel,\n-    then synthesizing the summaries.\n-    \n-    Args:\n-        content: The large content to process\n-        context_str: Context information\n-        model: Model to use\n-        chunk_size: Size of each chunk in characters\n-        max_output_size: Maximum final output size\n-        \n-    Returns:\n-        Synthesized summary or None on failure\n+    The file is mounted read-only into remote backends (Docker/Modal/SSH) via\n+    credential_files._CACHE_DIRS, so the agent's terminal/read_file tools can\n+    page through the complete text on any backend. Returns None on failure\n+    (storage is best-effort; truncated content is still returned to the model).\n     \"\"\"\n-    # Split content into chunks\n-    chunks = []\n-    for i in range(0, len(content), chunk_size):\n-        chunk = content[i:i + chunk_size]\n-        chunks.append(chunk)\n-    \n-    logger.info(\"Split into %d chunks of ~%d chars each\", len(chunks), chunk_size)\n-    \n-    # Summarize each chunk in parallel\n-    async def summarize_chunk(chunk_idx: int, chunk_content: str) -> tuple[int, Optional[str]]:\n-        \"\"\"Summarize a single chunk.\"\"\"\n-        try:\n-            chunk_info = f\"[Processing chunk {chunk_idx + 1} of {len(chunks)}]\"\n-            summary = await _call_summarizer_llm(\n-                chunk_content, \n-                context_str, \n-                model, \n-                max_tokens=[redacted:secret-assignment]\n-                is_chunk=True,\n-                chunk_info=chunk_info\n-            )\n-            if summary:\n-                logger.info(\"Chunk %d/%d summarized: %d -> %d chars\", chunk_idx + 1, len(chunks), len(chunk_content), len(summary))\n-            return chunk_idx, summary\n-        except Exception as e:\n-            logger.warning(\"Chunk %d/%d failed: %s\", chunk_idx + 1, len(chunks), str(e)[:50])\n-            return chunk_idx, None\n-    \n-    # Run all chunk summarizations in parallel\n-    tasks = [summarize_chunk(i, chunk) for i, chunk in enumerate(chunks)]\n-    # Use return_exceptions=True so a single task failure does not discard\n-    # all other successfully summarized chunks.\n-    results = await asyncio.gather(*tasks, return_exceptions=True)\n-\n-    # Filter out exceptions, then collect successful summaries in order\n-    successful_results = []\n-    for result_item in results:\n-        if isinstance(result_item, BaseException):\n-            logger.warning(\"Chunk summarization task failed: %s\", result_item)\n-            continue\n-        successful_results.append(result_item)\n-\n-    summaries = []\n-    for chunk_idx, summary in sorted(successful_results, key=lambda x: x[0]):\n-        if summary:\n-            summaries.append(f\"## Section {chunk_idx + 1}\\n{summary}\")\n-    \n-    if not summaries:\n-        logger.debug(\"All chunk summarizations failed\")\n-        return \"[Failed to process large content: all chunk summarizations failed]\"\n-    \n-    logger.info(\"Got %d/%d chunk summaries\", len(summaries), len(chunks))\n-    \n-    # If only one chunk succeeded, just return it (with cap)\n-    if len(summaries) == 1:\n-        result = summaries[0]\n-        if len(result) > max_output_size:\n-            result = result[:max_output_size] + \"\\n\\n[... truncated ...]\"\n-        return result\n-    \n-    # Synthesize the summaries into a final summary\n-    logger.info(\"Synthesizing %d summaries...\", len(summaries))\n-    \n-    combined_summaries = \"\\n\\n---\\n\\n\".join(summaries)\n-    \n-    synthesis_prompt = f\"\"\"You have been given summaries of different sections of a large document. \n-Synthesize these into ONE cohesive, comprehensive summary that:\n-1. Removes redundancy between sections\n-2. Preserves all key facts, figures, and actionable information\n-3. Is well-organized with clear structure\n-4. Is under {max_output_size} characters\n-\n-{context_str}SECTION SUMMARIES:\n-{combined_summaries}\n-\n-Create a single, unified markdown summary.\"\"\"\n-\n     try:\n-        aux_client, effective_model, extra_body = _resolve_web_extract_auxiliary(model)\n-        if aux_client is None or not effective_model:\n-            logger.warning(\"No auxiliary model for synthesis, concatenating summaries\")\n-            fallback = \"\\n\\n\".join(summaries)\n-            if len(fallback) > max_output_size:\n-                fallback = fallback[:max_output_size] + \"\\n\\n[... truncated ...]\"\n-            return fallback\n-\n-        call_kwargs = {\n-            \"task\": \"web_extract\",\n-            \"model\": effective_model,\n-            \"messages\": [\n-                {\"role\": \"system\", \"content\": \"You synthesize multiple summaries into one cohesive, comprehensive summary. Be thorough but concise.\"},\n-                {\"role\": \"user\", \"content\": synthesis_prompt},\n-            ],\n-            \"temperature\": 0.1,\n-            \"max_tokens\": 20000,\n-        }\n-        if extra_body:\n-            call_kwargs[\"extra_body\"] = extra_body\n-        response = await async_call_llm(**call_kwargs)\n-        final_summary = extract_content_or_reasoning(response)\n-\n-        # Retry once on empty content (reasoning-only response)\n-        if not final_summary:\n-            logger.warning(\"Synthesis LLM returned empty content, retrying once\")\n-            response = await async_call_llm(**call_kwargs)\n-            final_summary = extract_content_or_reasoning(response)\n-\n-        # If still None after retry, fall back to concatenated summaries\n-        if not final_summary:\n-            logger.warning(\"Synthesis failed after retry — concatenating chunk summaries\")\n-            fallback = \"\\n\\n\".join(summaries)\n-            if len(fallback) > max_output_size:\n-                fallback = fallback[:max_output_size] + \"\\n\\n[... truncated ...]\"\n-            return fallback\n-\n-        # Enforce hard cap\n-        if len(final_summary) > max_output_size:\n-            final_summary = final_summary[:max_output_size] + \"\\n\\n[... summary truncated for context management ...]\"\n-        \n-        original_len = len(content)\n-        final_len = len(final_summary)\n-        compression = final_len / original_len if original_len > 0 else 1.0\n-        \n-        logger.info(\"Synthesis complete: %d -> %d chars (%.2f%%)\", original_len, final_len, compression * 100)\n-        return final_summary\n-        \n-    except Exception as e:\n-        logger.warning(\"Synthesis failed: %s\", str(e)[:100])\n-        # Fall back to concatenated summaries with truncation\n-        fallback = \"\\n\\n\".join(summaries)\n-        if len(fallback) > max_output_size:\n-            fallback = fallback[:max_output_size] + \"\\n\\n[... truncated due to synthesis failure ...]\"\n-        return fallback\n+        import hashlib\n+        from urllib.parse import urlparse\n+        from hermes_constants import get_hermes_dir\n+\n+        cache_dir = get_hermes_dir(\"cache/web\", \"web_cache\")\n+        cache_dir.mkdir(parents=True, exist_ok=True)\n+\n+        host = (urlparse(url).hostname or \"page\").replace(\":\", \"_\")\n+        slug = re.sub(r\"[^A-Za-z0-9._-]\", \"-\", host)[:60].strip(\"-\") or \"page\"\n+        digest = hashlib.sha256(url.encode(\"utf-8\")).hexdigest()[:10]\n+        path = cache_dir / f\"{slug}-{digest}.md\"\n+        path.write_text(content, encoding=\"utf-8\")\n+        return str(path)\n+    except Exception as exc:  # noqa: BLE001\n+        logger.debug(\"Failed to store full web_extract text for %s: %s\", url, exc)\n+        return None\n+\n+\n+def _truncate_with_footer(\n+    content: str,\n+    url: str,\n+    char_limit: int,\n+) -> tuple[str, bool]:\n+    \"\"\"Return (model_text, was_truncated) for one page's clean content.\n+\n+    Pages at or under ``char_limit`` are returned whole. Larger pages get a\n+    head+tail window (~75% head / ~25% tail) cut on a markdown line boundary\n+    where possible, plus an explicit footer telling the model exactly how much\n+    it is seeing, where the full text is stored, and which read_file call pages\n+    in the omitted middle. Deterministic — no model involvement.\n+    \"\"\"\n+    if len(content) <= char_limit:\n+        return content, False\n+\n+    head_budget = int(char_limit * 0.75)\n+    tail_budget = char_limit - head_budget\n+\n+    head = content[:head_budget]\n+    tail = content[-tail_budget:]\n+    # Snap the head cut back to the last newline so we don't slice mid-line.\n+    nl = head.rfind(\"\\n\")\n+    if nl > head_budget * 0.5:\n+        head = head[:nl]\n+    # Snap the tail cut forward to the next newline for the same reason.\n+    nl = tail.find(\"\\n\")\n+    if 0 <= nl < tail_budget * 0.5:\n+        tail = tail[nl + 1:]\n+\n+    total = len(content)\n+    stored_path = _store_full_text(url, content)\n+    shown = len(head) + len(tail)\n+\n+    footer_lines = [\n+        \"\",\n+        \"─\" * 8 + \" [TRUNCATED] \" + \"─\" * 8,\n+        f\"Showing {len(head):,} chars (head) + {len(tail):,} chars (tail) \"\n+        f\"of {total:,} total clean characters.\",\n+    ]\n+    if stored_path:\n+        footer_lines.append(f\"Full text saved to: {stored_path}\")\n+        footer_lines.append(\n+            f'To read the omitted middle: read_file path=\"{stored_path}\" '\n+            f\"offset=<line> limit=<n>  (the file is the complete page).\"\n+        )\n+    else:\n+        footer_lines.append(\n+            \"Full text could not be stored; re-run web_extract on a more \"\n+            \"specific URL or use browser_navigate for the complete page.\"\n+        )\n+    footer_lines.append(\"─\" * 29)\n \n+    model_text = head + \"\\n\\n[... middle omitted — see footer ...]\\n\\n\" + tail\n+    model_text += \"\\n\" + \"\\n\".join(footer_lines)\n+    return model_text, True\n \n-def clean_base64_images(text: str) -> str:\n-    \"\"\"\n-    Remove base64 encoded images from text to reduce token count and clutter.\n-    \n-    This function finds and removes base64 encoded images in various formats:\n-    - (data:image/png;base64,...)\n-    - (data:image/jpeg;base64,...)\n-    - (data:image/svg+xml;base64,...)\n-    - data:image/[type];base64,... (without parentheses)\n-    \n-    Args:\n-        text: The text content to clean\n-        \n-    Returns:\n-        Cleaned text with base64 images replaced with placeholders\n-    \"\"\"\n-    # Pattern to match base64 encoded images wrapped in parentheses\n-    # Matches: (data:image/[type];base64,[base64-string])\n-    base64_with_parens_pattern = r'\\(data:image/[^;]+;base64,[A-Za-z0-9+/=]+\\)'\n-    \n-    # Pattern to match base64 encoded images without parentheses\n-    # Matches: data:image/[type];base64,[base64-string]\n-    base64_pattern = r'data:image/[^;]+;base64,[A-Za-z0-9+/=]+'\n-    \n-    # Replace parentheses-wrapped images first\n-    cleaned_text = re.sub(base64_with_parens_pattern, '[BASE64_IMAGE_REMOVED]', text)\n-    \n-    # Then replace any remaining non-parentheses images\n-    cleaned_text = re.sub(base64_pattern, '[BASE64_IMAGE_REMOVED]', cleaned_text)\n-    \n-    return cleaned_text\n \n \n # ─── Exa / Parallel inline helpers — moved into plugins ──────────────────────","@@ -894,29 +588,32 @@ def web_search_tool(query: str, limit: int = 5) -> str:\n async def web_extract_tool(\n     urls: List[str],\n     format: str = None,\n-    use_llm_processing: bool = True,\n-    model: Optional[str] = None,\n-    min_length: int = DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION\n+    char_limit: Optional[int] = None,\n ) -> str:\n     \"\"\"\n     Extract content from specific web pages using available extraction API backend.\n \n-    This function provides a generic interface for web content extraction that\n-    can work with multiple backends. Currently uses Firecrawl.\n+    Returns clean page content (markdown/text) with NO LLM summarization. The\n+    extract backends (Firecrawl, Tavily, Exa, Parallel) already return clean,\n+    boilerplate-stripped content, so we return it directly and fast. Pages over\n+    ``char_limit`` are head+tail truncated with an explicit footer; the full\n+    text is stored under cache/web and the footer tells the model how to\n+    read_file the omitted middle. Inline base64 images are replaced with\n+    ``[IMAGE: alt]`` placeholders (real image URLs are preserved as links).\n \n     Args:\n         urls (List[str]): List of URLs to extract content from\n         format (str): Desired output format (\"markdown\" or \"html\", optional)\n-        use_llm_processing (bool): Whether to process content with LLM for summarization (default: True)\n-        model (Optional[str]): The model to use for LLM processing (defaults to current auxiliary backend model)\n-        min_length (int): Minimum content length to trigger LLM processing (default: 5000)\n+        char_limit (Optional[int]): Per-page char budget sent to the model\n+            (default: web.extract_char_limit or 15000). Larger pages truncate.\n \n     Security: URLs are checked for embedded secrets before fetching.\n-    \n+\n     Returns:\n-        str: JSON string containing extracted content. If LLM processing is enabled and successful,\n-             the 'content' field will contain the processed markdown summary instead of raw content.\n-    \n+        str: JSON string with a ``results`` list; each entry has\n+             ``url``, ``title``, ``content``, ``error``. ``content`` is the\n+             (possibly truncated) clean page text.\n+\n     Raises:\n         Exception: If extraction fails or API key is not set\n     \"\"\"","@@ -944,16 +641,14 @@ async def web_extract_tool(\n         \"parameters\": {\n             \"urls\": normalized_urls,\n             \"format\": format,\n-            \"use_llm_processing\": use_llm_processing,\n-            \"model\": model,\n-            \"min_length\": min_length\n+            \"char_limit\": char_limit,\n         },\n         \"error\": None,\n         \"pages_extracted\": 0,\n-        \"pages_processed_with_llm\": 0,\n+        \"pages_truncated\": 0,\n         \"original_response_size\": 0,\n         \"final_response_size\": 0,\n-        \"compression_metrics\": [],\n+        \"truncation_metrics\": [],\n         \"processing_applied\": []\n     }\n     ","@@ -1053,91 +748,39 @@ async def web_extract_tool(\n         \n         debug_call_data[\"pages_extracted\"] = pages_extracted\n         debug_call_data[\"original_response_size\"] = len(json.dumps(response))\n-        effective_model = model or _get_default_summarizer_model()\n-        auxiliary_available = check_auxiliary_model()\n-        \n-        # Process each result with LLM if enabled\n-        if use_llm_processing and auxiliary_available:\n-            logger.info(\"Processing extracted content with LLM (parallel)...\")\n-            debug_call_data[\"processing_applied\"].append(\"llm_processing\")\n-            \n-            # Prepare tasks for parallel processing\n-            async def process_single_result(result):\n-                \"\"\"Process a single result with LLM and return updated result with metrics.\"\"\"\n-                url = result.get('url', 'Unknown URL')\n-                title = result.get('title', '')\n-                raw_content = result.get('raw_content', '') or result.get('content', '')\n-                \n-                if not raw_content:\n-                    return result, None, \"no_content\"\n-                \n-                original_size = len(raw_content)\n-                \n-                # Process content with LLM\n-                processed = await process_content_with_llm(\n-                    raw_content, url, title, effective_model, min_length\n-                )\n-                \n-                if processed:\n-                    processed_size = len(processed)\n-                    compression_ratio = processed_size / original_size if original_size > 0 else 1.0\n-                    \n-                    # Update result with processed content\n-                    result['content'] = processed\n-                    result['raw_content'] = raw_content\n-                    \n-                    metrics = {\n-                        \"url\": url,\n-                        \"original_size\": original_size,\n-                        \"processed_size\": processed_size,\n-                        \"compression_ratio\": compression_ratio,\n-                        \"model_used\": effective_model\n-                    }\n-                    return result, metrics, \"processed\"\n-                else:\n-                    metrics = {\n-                        \"url\": url,\n-                        \"original_size\": original_size,\n-                        \"processed_size\": original_size,\n-                        \"compression_ratio\": 1.0,\n-                        \"model_used\": None,\n-                        \"reason\": \"content_too_short\"\n-                    }\n-                    return result, metrics, \"too_short\"\n-            \n-            # Run all LLM processing in parallel\n-            results_list = response.get('results', [])\n-            tasks = [process_single_result(result) for result in results_list]\n-            # Use return_exceptions=True so a single task failure does not\n-            # discard all other successfully processed results.\n-            processed_results = await asyncio.gather(*tasks, return_exceptions=True)\n-\n-            # Collect metrics and print results\n-            for result_item in processed_results:\n-                if isinstance(result_item, BaseException):\n-                    logger.warning(\"Web result processing task failed: %s\", result_item)\n-                    continue\n-                result, metrics, status = result_item\n-                url = result.get('url', 'Unknown URL')\n-                if status == \"processed\":\n-                    debug_call_data[\"compression_metrics\"].append(metrics)\n-                    debug_call_data[\"pages_processed_with_llm\"] += 1\n-                    logger.info(\"%s (processed)\", url)\n-                elif status == \"too_short\":\n-                    debug_call_data[\"compression_metrics\"].append(metrics)\n-                    logger.info(\"%s (no processing - content too short)\", url)\n-                else:\n-                    logger.warning(\"%s (no content to process)\", url)\n-        else:\n-            if use_llm_processing and not auxiliary_available:\n-                logger.warning(\"LLM processing requested but no auxiliary model available, returning raw content\")\n-                debug_call_data[\"processing_applied\"].append(\"llm_processing_unavailable\")\n-            # Print summary of extracted pages for debugging (original behavior)\n-            for result in response.get('results', []):\n-                url = result.get('url', 'Unknown URL')\n-                content_length = len(result.get('raw_content', ''))\n-                logger.info(\"%s (%d characters)\", url, content_length)\n-        \n+\n+        effective_char_limit = char_limit if char_limit is not None else _get_extract_char_limit()\n+        try:\n+            effective_char_limit = max(2000, min(int(effective_char_limit), 500_000))\n+        except (TypeError, ValueError):\n+            effective_char_limit = DEFAULT_EXTRACT_CHAR_LIMIT\n+\n+        # Truncate-and-store: no LLM. For each result, convert inline base64\n+        # images to labeled placeholders (keeping alt text + real image URLs),\n+        # then return the clean content directly if within budget, or a\n+        # head+tail window plus a footer pointing at the stored full text.\n+        debug_call_data[\"processing_applied\"].append(\"truncate_and_store\")\n+        for result in response.get(\"results\", []):\n+            if result.get(\"error\"):\n+                continue\n+            url = result.get(\"url\", \"\")\n+            raw_content = result.get(\"raw_content\", \"\") or result.get(\"content\", \"\")\n+            if not raw_content:\n+                continue\n+            clean = convert_base64_images_to_links(raw_content)\n+            model_text, truncated = _truncate_with_footer(clean, url, effective_char_limit)\n+            result[\"content\"] = model_text\n+            if truncated:\n+                debug_call_data[\"pages_truncated\"] += 1\n+                debug_call_data[\"truncation_metrics\"].append({\n+                    \"url\": url,\n+                    \"original_size\": len(clean),\n+                    \"sent_size\": len(model_text),\n+                })\n+                logger.info(\"%s (truncated %d -> %d chars)\", url, len(clean), len(model_text))\n+            else:\n+                logger.info(\"%s (%d chars, whole)\", url, len(clean))\n+\n         # Trim output to minimal fields per entry: title, content, error\n         trimmed_results = [\n             {","@@ -1153,16 +796,16 @@ async def process_single_result(result):\n \n         if trimmed_response.get(\"results\") == []:\n             result_json = tool_error(\"Content was inaccessible or not found\")\n-\n-            cleaned_result = clean_base64_images(result_json)\n-        \n         else:\n             result_json = json.dumps(trimmed_response, indent=2, ensure_ascii=False)\n-            \n-            cleaned_result = clean_base64_images(result_json)\n-        \n+\n+        # base64 images were already converted to placeholders per-result above;\n+        # this is a belt-and-suspenders sweep over the serialized JSON in case a\n+        # provider tucked a blob somewhere unexpected (e.g. metadata).\n+        cleaned_result = convert_base64_images_to_links(result_json)\n+\n         debug_call_data[\"final_response_size\"] = len(cleaned_result)\n-        debug_call_data[\"processing_applied\"].append(\"base64_image_removal\")\n+        debug_call_data[\"processing_applied\"].append(\"base64_image_conversion\")\n         \n         # Log debug information\n         _debug.log_call(\"web_extract_tool\", debug_call_data)","@@ -1193,28 +836,18 @@ def check_web_api_key() -> bool:\n     )\n \n \n-def check_auxiliary_model() -> bool:\n-    \"\"\"Check if an auxiliary text model is available for LLM content processing.\"\"\"\n-    client, _, _ = _resolve_web_extract_auxiliary()\n-    return client is not None\n-\n-\n-\n-\n if __name__ == \"__main__\":\n     \"\"\"\n     Simple test/demo when run directly\n     \"\"\"\n     print(\"🌐 Standalone Web Tools Module\")\n     print(\"=\" * 40)\n-    \n+\n     # Check if API keys are available\n     web_available = check_web_api_key()\n     tool_gateway_available = _is_tool_gateway_ready()\n     firecrawl_key_available = bool(os.getenv(\"FIRECRAWL_API_KEY\", \"\").strip())\n     firecrawl_url_available = bool(os.getenv(\"FIRECRAWL_API_URL\", \"\").strip())\n-    nous_available = check_auxiliary_model()\n-    default_summarizer_model = _get_default_summarizer_model()\n \n     if web_available:\n         backend = _get_backend()","@@ -1246,29 +879,20 @@ def check_auxiliary_model() -> bool:\n             f\"{_firecrawl_backend_help_suffix()}\"\n         )\n \n-    if not nous_available:\n-        print(\"❌ No auxiliary model available for LLM content processing\")\n-        print(\"Set OPENROUTER_API_KEY, configure Nous Portal, or set OPENAI_BASE_URL + OPENAI_API_KEY\")\n-        print(\"⚠️  Without an auxiliary model, LLM content processing will be disabled\")\n-    else:\n-        print(f\"✅ Auxiliary model available: {default_summarizer_model}\")\n-\n     if not web_available:\n         sys.exit(1)\n \n     print(\"🛠️  Web tools ready for use!\")\n-    \n-    if nous_available:\n-        print(f\"🧠 LLM content processing available with {default_summarizer_model}\")\n-        print(f\"   Default min length for processing: {DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION} chars\")\n-    \n+    print(f\"   Extract char limit: {_get_extract_char_limit()} chars \"\n+          \"(pages over this are truncated; full text stored in cache/web)\")\n+\n     # Show debug mode status\n     if _debug.active:\n         print(f\"🐛 Debug mode ENABLED - Session ID: {_debug.session_id}\")\n         print(f\"   Debug logs will be saved to: {_debug.log_dir}/web_tools_debug_{_debug.session_id}.json\")\n     else:\n         print(\"🐛 Debug mode disabled (set WEB_TOOLS_DEBUG=true to enable)\")\n-    \n+\n     print(\"\\nBasic usage:\")\n     print(\"  from web_tools import web_search_tool, web_extract_tool\")\n     print(\"  import asyncio\")","@@ -1276,37 +900,16 @@ def check_auxiliary_model() -> bool:\n     print(\"  # Search (synchronous)\")\n     print(\"  results = web_search_tool('Python tutorials')\")\n     print(\"\")\n-    print(\"  # Extract (asynchronous)\")\n+    print(\"  # Extract (asynchronous, no LLM — truncate-and-store)\")\n     print(\"  async def main():\")\n     print(\"      content = await web_extract_tool(['https://example.com'])\")\n+    print(\"      # bigger budget for one call:\")\n+    print(\"      content = await web_extract_tool(['https://docs.python.org'], char_limit=40000)\")\n     print(\"  asyncio.run(main())\")\n-    \n-    if nous_available:\n-        print(\"\\nLLM-enhanced usage:\")\n-        print(\"  # Content automatically processed for pages >5000 chars (default)\")\n-        print(\"  content = await web_extract_tool(['https://python.org/about/'])\")\n-        print(\"\")\n-        print(\"  # Customize processing parameters\")\n-        print(\"  content = await web_extract_tool(\")\n-        print(\"      ['https://docs.python.org'],\")\n-        print(\"      model='google/gemini-3-flash-preview',\")\n-        print(\"      min_length=3000\")\n-        print(\"  )\")\n-        print(\"\")\n-        print(\"  # Disable LLM processing\")\n-        print(\"  raw_content = await web_extract_tool(['https://example.com'], use_llm_processing=False)\")\n-    \n+\n     print(\"\\nDebug mode:\")\n-    print(\"  # Enable debug logging\")\n     print(\"  export WEB_TOOLS_DEBUG=true\")\n-    print(\"  # Debug logs capture:\")\n-    print(\"  # - All tool calls with parameters\")\n-    print(\"  # - Original API responses\")\n-    print(\"  # - LLM compression metrics\")\n-    print(\"  # - Final processed results\")\n     print(\"  # Logs saved to: ./logs/web_tools_debug_UUID.json\")\n-    \n-    print(\"\\n📝 Run 'python test_web_tools_llm.py' to test LLM processing capabilities\")\n \n \n # ---------------------------------------------------------------------------","@@ -1338,7 +941,7 @@ def check_auxiliary_model() -> bool:\n \n WEB_EXTRACT_SCHEMA = {\n     \"name\": \"web_extract\",\n-    \"description\": \"Extract content from web page URLs. Returns page content in markdown format. Also works with PDF URLs (arxiv papers, documents, etc.) — pass the PDF link directly and it converts to markdown text. Pages under 5000 chars return full markdown; larger pages are LLM-summarized and capped at ~5000 chars per page. Pages over 2M chars are refused. If a URL fails or times out, use the browser tool to access it instead.\",\n+    \"description\": \"Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer telling you the full text's saved file path and the read_file call to page through the omitted middle. Inline images appear as [IMAGE: alt] placeholders; real image URLs are kept as links. If a URL fails or times out, use the browser tool instead.\",\n     \"parameters\": {\n         \"type\": \"object\",\n         \"properties\": {","@@ -1347,6 +950,11 @@ def check_auxiliary_model() -> bool:\n                 \"items\": {\"type\": \"string\"},\n                 \"description\": \"List of URLs to extract content from (max 5 URLs per call)\",\n                 \"maxItems\": 5\n+            },\n+            \"char_limit\": {\n+                \"type\": \"integer\",\n+                \"description\": \"Optional per-page character budget sent back (default 15000). Pages larger than this are head+tail truncated with the full text stored to disk. Raise it when you need more of a long page inline.\",\n+                \"minimum\": 2000\n             }\n         },\n         \"required\": [\"urls\"]","@@ -1368,7 +976,10 @@ def check_auxiliary_model() -> bool:\n     toolset=\"web\",\n     schema=WEB_EXTRACT_SCHEMA,\n     handler=lambda args, **kw: web_extract_tool(\n-        args.get(\"urls\", [])[:5] if isinstance(args.get(\"urls\"), list) else [], \"markdown\"),\n+        args.get(\"urls\", [])[:5] if isinstance(args.get(\"urls\"), list) else [],\n+        \"markdown\",\n+        char_limit=args.get(\"char_limit\"),\n+    ),\n     check_fn=check_web_api_key,\n     requires_env=_web_requires_env(),\n     is_async=True,\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}