Firecrawl config/env resolution bug and fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #59518 fix(web): config-aware env resolution for all web providers (salvage of #40192 by @liuhao1024)

  merged to main

  files: agent/web_search_provider.py, plugins/web/brave_free/provider.py, plugins/web/exa/provider.py, plugins/web/firecrawl/provider.py, plugins/web/parallel/provider.py, plugins/web/tavily/provider.py, tests/tools/test_web_tools_config.py, tools/web_tools.py

  Show the diff

  • agent/web_search_provider.py

    @@ -52,7 +52,33 @@
     from __future__ import annotations
     
     import abc
    -from typing import Any, Dict, List
    +import os
    +from typing import Any, Dict, List, Optional
    +
    +
    +def get_provider_env(name: str) -> str:
    +    """Config-aware env lookup for web providers.
    +
    +    Resolves *name* via :func:`hermes_cli.config.get_env_value` (checks
    +    ``os.environ`` first, then ``~/.hermes/.env``) so credentials set
    +    through Hermes' config layer are visible even when they were never
    +    exported into the process environment — gateway sessions, delegate
    +    children, and subprocess agent runs (issue #40190). Falls back to a
    +    bare ``os.getenv`` when the config module is unavailable (stripped
    +    installs, early import contexts).
    +
    +    Returns the stripped value, or ``""`` when unset.
    +    """
    +    val: Optional[str] = None
    +    try:
    +        from hermes_cli.config import get_env_value
    +
    +        val = get_env_value(name)
    +    except Exception:  # noqa: BLE001 — config layer optional here
    +        val = None
    +    if val is None:
    +        val = os.getenv(name, "")
    +    return (val or "").strip()
     
     
     # ---------------------------------------------------------------------------

  • plugins/web/brave_free/provider.py

    @@ -49,7 +49,9 @@ def display_name(self) -> str:
     
         def is_available(self) -> bool:
             """Return True when ``BRAVE_SEARCH_API_KEY`` is set to a non-empty value."""
    -        return bool(os.getenv("BRAVE_SEARCH_API_KEY", "").strip())
    +        from agent.web_search_provider import get_provider_env
    +
    +        return bool(get_provider_env("BRAVE_SEARCH_API_KEY"))
     
         def supports_search(self) -> bool:
             return True
    @@ -65,7 +67,9 @@ def search(self, query: str, limit: int = 5) -> Dict[str, Any]:
             """
             import httpx
     
    -        api_key = [redacted:secret-assignment]"BRAVE_SEARCH_API_KEY", "").strip()
    +        from agent.web_search_provider import get_provider_env
    +
    +        api_key = [redacted:secret-assignment]"BRAVE_SEARCH_API_KEY")
             if not api_key:
                 return {"success": False, "error": "BRAVE_SEARCH_API_KEY is not set"}
     

  • plugins/web/exa/provider.py

    @@ -51,7 +51,9 @@ def _get_exa_client() -> Any:
         if cached is not None:
             return cached
     
    -    api_key = [redacted:secret-assignment]"EXA_API_KEY")
    +    from agent.web_search_provider import get_provider_env
    +
    +    api_key = [redacted:secret-assignment]"EXA_API_KEY")
         if not api_key:
             raise ValueError(
                 "EXA_API_KEY environment variable not set. "
    @@ -100,7 +102,9 @@ def display_name(self) -> str:
     
         def is_available(self) -> bool:
             """Return True when ``EXA_API_KEY`` is set to a non-empty value."""
    -        return bool(os.getenv("EXA_API_KEY", "").strip())
    +        from agent.web_search_provider import get_provider_env
    +
    +        return bool(get_provider_env("EXA_API_KEY"))
     
         def supports_search(self) -> bool:
             return True

  • plugins/web/firecrawl/provider.py

    @@ -122,8 +122,10 @@ def __repr__(self) -> str:
     
     def _get_direct_firecrawl_config() -> Optional[tuple]:
         """Return explicit direct Firecrawl kwargs + cache key, or None when unset."""
    -    api_key = [redacted:secret-assignment]"FIRECRAWL_API_KEY", "").strip()
    -    api_url = os.getenv("FIRECRAWL_API_URL", "").strip().rstrip("/")
    +    from hermes_cli.config import get_env_value
    +
    +    api_key = [redacted:secret-assignment]"FIRECRAWL_API_KEY") or "").strip()
    +    api_url = (get_env_value("FIRECRAWL_API_URL") or "").strip().rstrip("/")
     
         if not api_key and not api_url:
             return None

  • plugins/web/parallel/provider.py

    @@ -73,7 +73,9 @@ def _get_sync_client() -> Any:
         if cached is not None:
             return cached
     
    -    api_key = [redacted:secret-assignment]"PARALLEL_API_KEY")
    +    from agent.web_search_provider import get_provider_env
    +
    +    api_key = [redacted:secret-assignment]"PARALLEL_API_KEY")
         if not api_key:
             raise ValueError(
                 "PARALLEL_API_KEY environment variable not set. "
    @@ -99,7 +101,9 @@ def _get_async_client() -> Any:
         if cached is not None:
             return cached
     
    -    api_key = [redacted:secret-assignment]"PARALLEL_API_KEY")
    +    from agent.web_search_provider import get_provider_env
    +
    +    api_key = [redacted:secret-assignment]"PARALLEL_API_KEY")
         if not api_key:
             raise ValueError(
                 "PARALLEL_API_KEY environment variable not set. "
    @@ -153,7 +157,9 @@ def display_name(self) -> str:
     
         def is_available(self) -> bool:
             """Return True when ``PARALLEL_API_KEY`` is set to a non-empty value."""
    -        return bool(os.getenv("PARALLEL_API_KEY", "").strip())
    +        from agent.web_search_provider import get_provider_env
    +
    +        return bool(get_provider_env("PARALLEL_API_KEY"))
     
         def supports_search(self) -> bool:
             return True

  • plugins/web/tavily/provider.py

    @@ -41,14 +41,16 @@ def _tavily_request(endpoint: str, payload: Dict[str, Any]) -> Dict[str, Any]:
         """
         import httpx
     
    -    api_key = [redacted:secret-assignment]"TAVILY_API_KEY")
    +    from agent.web_search_provider import get_provider_env
    +
    +    api_key = [redacted:secret-assignment]"TAVILY_API_KEY")
         if not api_key:
             raise ValueError(
                 "TAVILY_API_KEY environment variable not set. "
                 "Get your API key at https://app.tavily.com/home"
             )
     
    -    base_url = os.getenv("TAVILY_BASE_URL", "https://api.tavily.com")
    +    base_url = get_provider_env("TAVILY_BASE_URL") or "https://api.tavily.com"
         payload = dict(payload)  # don't mutate caller's dict
         payload["api_key"] = api_key
         url = f"{base_url}/{endpoint.lstrip('/')}"
    @@ -138,7 +140,9 @@ def display_name(self) -> str:
     
         def is_available(self) -> bool:
             """Return True when ``TAVILY_API_KEY`` is set to a non-empty value."""
    -        return bool(os.getenv("TAVILY_API_KEY", "").strip())
    +        from agent.web_search_provider import get_provider_env
    +
    +        return bool(get_provider_env("TAVILY_API_KEY"))
     
         def supports_search(self) -> bool:
             return True

  • tests/tools/test_web_tools_config.py

    @@ -787,3 +787,95 @@ def test_tool_registry_entries_not_filtered_out(self):
                     "web_search tool was filtered out despite custom provider being available"
                 assert web_extract_entry is not None, \
                     "web_extract tool was filtered out despite custom provider being available"
    +
    +
    +class TestFirecrawlEnvResolution:
    +    """Verify Firecrawl reads env values from hermes_cli.config.get_env_value,
    +    not just os.getenv.  This catches the regression reported in #40190 where
    +    values stored in ~/.hermes/.env were invisible to the provider."""
    +
    +    def test_direct_config_reads_via_get_env_value(self, monkeypatch: pytest.MonkeyPatch) -> None:
    +        """_get_direct_firecrawl_config() must use get_env_value, not os.getenv."""
    +        # Ensure os.environ does NOT carry the key
    +        monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
    +        monkeypatch.delenv("FIRECRAWL_API_URL", raising=False)
    +
    +        fake_key = "fc-test-key-from-dotenv"
    +        with patch(
    +            "hermes_cli.config.get_env_value",
    +            side_effect=lambda k: fake_key if k == "FIRECRAWL_API_KEY" else None,
    +        ):
    +            from plugins.web.firecrawl.provider import _get_direct_firecrawl_config
    +
    +            result = _get_direct_firecrawl_config()
    +            assert result is not None, "get_env_value fallback should find the key"
    +            kwargs, _cache_key = result
    +            assert kwargs["api_key"] == fake_key
    +
    +    def test_direct_config_reads_url_via_get_env_value(self, monkeypatch: pytest.MonkeyPatch) -> None:
    +        """Self-hosted URL from .env must be picked up."""
    +        monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
    +        monkeypatch.delenv("FIRECRAWL_API_URL", raising=False)
    +
    +        fake_url = "https://firecrawl.internal.example.com"
    +        with patch(
    +            "hermes_cli.config.get_env_value",
    +            side_effect=lambda k: fake_url if k == "FIRECRAWL_API_URL" else None,
    +        ):
    +            from plugins.web.firecrawl.provider import _get_direct_firecrawl_config
    +
    +            result = _get_direct_firecrawl_config()
    +            assert result is not None
    +            kwargs, _cache_key = result
    +            assert kwargs["api_url"] == fake_url.rstrip("/")
    +
    +
    +class TestSiblingProvidersEnvResolution:
    +    """The same #40190 bug class widened: every keyed web provider must
    +    resolve its credential through the config-aware lookup (os.environ OR
    +    ~/.hermes/.env), not bare os.getenv. Parametrized over the four
    +    providers that previously read only the process environment."""
    +
    +    _CASES = [
    +        ("plugins.web.exa.provider", "ExaWebSearchProvider", "EXA_API_KEY"),
    +        ("plugins.web.parallel.provider", "ParallelWebSearchProvider", "PARALLEL_API_KEY"),
    +        ("plugins.web.tavily.provider", "TavilyWebSearchProvider", "TAVILY_API_KEY"),
    +        ("plugins.web.brave_free.provider", "BraveFreeWebSearchProvider", "BRAVE_SEARCH_API_KEY"),
    +    ]
    +
    +    @pytest.mark.parametrize("module_path,cls_name,env_key", _CASES)
    +    def test_is_available_reads_via_get_env_value(
    +        self, monkeypatch, module_path, cls_name, env_key
    +    ):
    +        """is_available() must see a key that lives only in the .env layer."""
    +        monkeypatch.delenv(env_key, raising=False)
    +
    +        import importlib
    +        module = importlib.import_module(module_path)
    +        provider = getattr(module, cls_name)()
    +
    +        assert provider.is_available() is False
    +
    +        with patch(
    +            "hermes_cli.config.get_env_value",
    +            side_effect=lambda k: "test-key-from-dotenv" if k == env_key else None,
    +        ):
    +            assert provider.is_available() is True, (
    +                f"{cls_name}.is_available() ignored {env_key} from the "
    +                "config-aware env layer (get_env_value)"
    +            )
    +
    +    def test_get_provider_env_falls_back_to_os_environ(self, monkeypatch):
    +        """When the config layer has no value, process env still wins."""
    +        from agent.web_search_provider import get_provider_env
    +
    +        monkeypatch.setenv("WSP_TEST_FALLBACK_KEY", "  from-process-env  ")
    +        with patch("hermes_cli.config.get_env_value", return_value=None):
    +            assert get_provider_env("WSP_TEST_FALLBACK_KEY") == "from-process-env"
    +
    +    def test_get_provider_env_unset_returns_empty(self, monkeypatch):
    +        monkeypatch.delenv("WSP_TEST_UNSET_KEY", raising=False)
    +        with patch("hermes_cli.config.get_env_value", return_value=None):
    +            from agent.web_search_provider import get_provider_env
    +
    +            assert get_provider_env("WSP_TEST_UNSET_KEY") == ""

  • tools/web_tools.py

    @@ -1001,8 +1001,9 @@ def check_web_api_key() -> bool:
         # Check if API keys are available
         web_available = check_web_api_key()
         tool_gateway_available = _is_tool_gateway_ready()
    -    firecrawl_key_available = bool(os.getenv("FIRECRAWL_API_KEY", "").strip())
    -    firecrawl_url_available = bool(os.getenv("FIRECRAWL_API_URL", "").strip())
    +    from hermes_cli.config import get_env_value as _gev
    +    firecrawl_key_available = bool((_gev("FIRECRAWL_API_KEY") or "").strip())
    +    firecrawl_url_available = bool((_gev("FIRECRAWL_API_URL") or "").strip())
     
         if web_available:
             backend = _get_backend()
    @@ -1020,7 +1021,7 @@ def check_web_api_key() -> bool:
             elif backend == "ddgs":
                 print("   Using DuckDuckGo via ddgs package (search only)")
             elif firecrawl_url_available:
    -            print(f"   Using self-hosted Firecrawl: {os.getenv('FIRECRAWL_API_URL').strip().rstrip('/')}")
    +            print(f"   Using self-hosted Firecrawl: {(_gev('FIRECRAWL_API_URL') or '').strip().rstrip('/')}")
             elif firecrawl_key_available:
                 print("   Using direct Firecrawl cloud API")
             elif tool_gateway_available: