Bedrock bearer-token routing and region picker fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #64628 fix(bedrock): bearer-token Converse routing + region-scoped picker (#28156)

  merged to main

  files: agent/bedrock_adapter.py, agent/conversation_loop.py, hermes_cli/model_setup_flows.py, hermes_cli/runtime_provider.py, scripts/release.py, tests/agent/test_bedrock_adapter.py, tests/hermes_cli/test_bedrock_region_scoped_picker.py

  Show the diff

  • agent/bedrock_adapter.py

    @@ -528,10 +528,19 @@ def _convert_content_to_converse(content) -> List[Dict]:
                             mime_part = header[5:].split(";")[0]
                             if mime_part:
                                 media_type = mime_part
    +                    # Decode base64 to raw bytes — boto3 re-encodes at the
    +                    # wire layer, so passing the base64 string directly
    +                    # results in double-encoding and Bedrock rejects it with
    +                    # "Failed to sanitize image".  Ref: #33317.
    +                    import base64
    +                    try:
    +                        raw_bytes = base64.b64decode(data)
    +                    except Exception:
    +                        raw_bytes = data.encode("utf-8")
                         blocks.append({
                             "image": {
                                 "format": media_type.split("/")[-1] if "/" in media_type else "jpeg",
    -                            "source": {"bytes": data},
    +                            "source": {"bytes": raw_bytes},
                             }
                         })
                     else:

  • agent/conversation_loop.py

    @@ -2619,6 +2619,31 @@ def _perform_api_call(next_api_kwargs):
                         )
                         continue
     
    +                # ── Bedrock AnthropicBedrock SDK streaming failure ──
    +                # The Anthropic SDK's stream accumulator raises RuntimeError
    +                # "Unexpected event order" when Bedrock returns an error event
    +                # before message_start (throttling, overload, validation).
    +                # Fall back to the native Converse API path for the rest of
    +                # this session — it handles these errors gracefully.  Ref: #28156.
    +                if (
    +                    isinstance(api_error, RuntimeError)
    +                    and "unexpected event order" in str(api_error).lower()
    +                    and getattr(agent, "provider", "") == "bedrock"
    +                    and agent.api_mode == "anthropic_messages"
    +                    and not getattr(agent, "_bedrock_converse_fallback_attempted", False)
    +                ):
    +                    agent._bedrock_converse_fallback_attempted = True
    +                    agent.api_mode = "bedrock_converse"
    +                    agent._bedrock_region = getattr(agent, "_bedrock_region", None) or "us-east-1"
    +                    agent.client = None  # Drop the AnthropicBedrock client
    +                    agent._client_kwargs = {}
    +                    agent._vprint(
    +                        f"{agent.log_prefix}⚠️  AnthropicBedrock SDK streaming failed — "
    +                        f"falling back to native Converse API for this session.",
    +                        force=True,
    +                    )
    +                    continue
    +
                     status_code = getattr(api_error, "status_code", None)
                     error_context = agent._extract_api_error_context(api_error)
     

  • hermes_cli/model_setup_flows.py

    @@ -27,6 +27,54 @@
     from hermes_cli.config import clear_model_endpoint_credentials
     
     
    +# AWS cross-region inference profile prefixes. Any geo-prefixed profile only
    +# routes from endpoints in its own geography, so the Bedrock picker must not
    +# offer (e.g.) us.* profiles to an eu-central-2 endpoint — selecting one
    +# produces a config AWS rejects regardless of credentials (#28156).
    +# global.* routes from everywhere. Full set per the AWS cross-region
    +# inference docs.
    +BEDROCK_GEO_PREFIXES = (
    +    "us.", "eu.", "ap.", "apac.", "jp.", "ca.", "sa.", "me.", "af.",
    +)
    +
    +
    +def bedrock_region_geo_prefix(region_name: str) -> str:
    +    """Map an AWS region name to its inference-profile geo prefix ('' = unknown)."""
    +    r = (region_name or "").lower()
    +    for geo, region_prefixes in (
    +        ("us.", ("us-", "us_gov")),
    +        ("eu.", ("eu-",)),
    +        ("ap.", ("ap-",)),
    +        ("ca.", ("ca-",)),
    +        ("sa.", ("sa-",)),
    +        ("me.", ("me-",)),
    +        ("af.", ("af-",)),
    +    ):
    +        if r.startswith(region_prefixes):
    +            return geo
    +    return ""
    +
    +
    +def bedrock_model_routable_from_region(model_id: str, region_name: str) -> bool:
    +    """True when *model_id* can be invoked from *region_name*'s endpoint.
    +
    +    Bare foundation-model ids and ``global.*`` profiles route from anywhere.
    +    Geo-prefixed inference profiles (``us.*``, ``eu.*``, ...) only route from
    +    endpoints in their own geography. Unknown region shapes hide nothing.
    +    """
    +    mid = (model_id or "").lower()
    +    matched_geo = next((p for p in BEDROCK_GEO_PREFIXES if mid.startswith(p)), None)
    +    if matched_geo is None or mid.startswith("global."):
    +        return True
    +    geo = bedrock_region_geo_prefix(region_name)
    +    if not geo:
    +        return True
    +    if geo == "ap.":
    +        # Asia-Pacific regions can carry ap./apac./jp. profile spellings.
    +        return matched_geo in ("ap.", "apac.", "jp.")
    +    return matched_geo == geo
    +
    +
     def _prune_replaced_custom_model_config_credentials(
         base_url: str,
         *,
    @@ -2265,6 +2313,8 @@ def _model_flow_bedrock(config, current_model=""):
                 "global.twelvelabs.",
             )
             _EXCLUDE_SUBSTRINGS = ("safeguard", "voxtral", "palmyra-vision")
    +
    +
             filtered = []
             for m in live_models:
                 mid = m["id"]
    @@ -2272,44 +2322,59 @@ def _model_flow_bedrock(config, current_model=""):
                     continue
                 if any(s in mid.lower() for s in _EXCLUDE_SUBSTRINGS):
                     continue
    +            if not bedrock_model_routable_from_region(mid, region):
    +                continue
                 filtered.append(m)
     
    -        # Deduplicate: prefer inference profiles (us.*, global.*) over bare
    -        # foundation model IDs.
    +        # Deduplicate: prefer inference profiles (geo-prefixed or global.*)
    +        # over bare foundation model IDs.
    +        _PROFILE_PREFIXES = BEDROCK_GEO_PREFIXES + ("global.",)
             profile_base_ids = set()
             for m in filtered:
                 mid = m["id"]
    -            if mid.startswith(("us.", "global.")):
    -                base = mid.split(".", 1)[1] if "." in mid[3:] else mid
    -                profile_base_ids.add(base)
    +            _pp = next((p for p in _PROFILE_PREFIXES if mid.startswith(p)), None)
    +            if _pp:
    +                profile_base_ids.add(mid[len(_pp):])
     
             deduped = []
             for m in filtered:
                 mid = m["id"]
    -            if not mid.startswith(("us.", "global.")) and mid in profile_base_ids:
    +            if (
    +                not mid.startswith(_PROFILE_PREFIXES)
    +                and mid in profile_base_ids
    +            ):
                     continue
                 deduped.append(m)
     
    -        _RECOMMENDED = [
    -            "us.anthropic.claude-sonnet-4-6",
    -            "us.anthropic.claude-opus-4-6",
    -            "us.anthropic.claude-haiku-4-5",
    -            "us.amazon.nova-pro",
    -            "us.amazon.nova-lite",
    -            "us.amazon.nova-micro",
    +        # Recommended models, matched geo-agnostically so an EU (eu.*) or
    +        # APAC (apac.*) picker pins its own region's profile of the same
    +        # model rather than a us.* one it can't route to (#28156).
    +        _RECOMMENDED_BASES = [
    +            "anthropic.claude-sonnet-4-6",
    +            "anthropic.claude-opus-4-6",
    +            "anthropic.claude-haiku-4-5",
    +            "amazon.nova-pro",
    +            "amazon.nova-lite",
    +            "amazon.nova-micro",
                 "deepseek.v3",
    -            "us.meta.llama4-maverick",
    -            "us.meta.llama4-scout",
    +            "meta.llama4-maverick",
    +            "meta.llama4-scout",
             ]
     
    +        def _base_id(mid: str) -> str:
    +            _pp = next((p for p in _PROFILE_PREFIXES if mid.startswith(p)), None)
    +            return mid[len(_pp):] if _pp else mid
    +
             def _sort_key(m):
                 mid = m["id"]
    -            for i, rec in enumerate(_RECOMMENDED):
    -                if mid.startswith(rec):
    -                    return (0, i, mid)
    +            base = _base_id(mid)
    +            for i, rec in enumerate(_RECOMMENDED_BASES):
    +                if base.startswith(rec):
    +                    # In-region geo profile beats global.* for the same model
    +                    return (0, i, 0 if not mid.startswith("global.") else 1, mid)
                 if mid.startswith("global."):
    -                return (1, 0, mid)
    -            return (2, 0, mid)
    +                return (1, 0, 0, mid)
    +            return (2, 0, 0, mid)
     
             deduped.sort(key=_sort_key)
             model_list = [m["id"] for m in deduped]

  • hermes_cli/runtime_provider.py

    @@ -1975,8 +1975,15 @@ def resolve_runtime_provider(
             # Dual-path routing: Claude models use AnthropicBedrock SDK for full
             # feature parity (prompt caching, thinking budgets, adaptive thinking).
             # Non-Claude models use the Converse API for multi-model support.
    +        #
    +        # Exception: Bearer Token auth (AWS_BEARER_TOKEN_BEDROCK) is NOT
    +        # supported by the AnthropicBedrock SDK (it only does SigV4 signing —
    +        # a bearer-only setup fails at runtime with "could not resolve
    +        # credentials from session"). Route these users through the Converse
    +        # API regardless of model. Ref: #28156.
             _current_model = str(target_model or model_cfg.get("default") or "").strip()
    -        if is_anthropic_bedrock_model(_current_model):
    +        _has_bearer_token = [redacted:secret-assignment]"AWS_BEARER_TOKEN_BEDROCK", "").strip())
    +        if is_anthropic_bedrock_model(_current_model) and not _has_bearer_token:
                 # Claude on Bedrock → AnthropicBedrock SDK → anthropic_messages path
                 runtime = {
                     "provider": "bedrock",

  • scripts/release.py

    @@ -47,6 +47,7 @@
     AUTHOR_MAP = {
         "[email protected]": "webtecnica",  # PR #63360 salvage (nous: restore inference-api base_url)
         "[redacted:email]": "Epoxidex",  # PR #29820 salvage (ollama: top-level reasoning_effort=none; #25758)
    +    "[redacted:email]": "JiaDe-Wu",  # PR #34742 salvage (bedrock: bearer routing + streaming fallback + image decode; #28156)
         "[redacted:email]": "minchang",  # PR #42231 salvage (providers: add Upstage Solar)
         "[redacted:email]": "neo-claw-bot",  # PR #58465 salvage (moa: drop empty user turns from advisory view)
         "[redacted:email]": "pixel4039",  # PR #64420 salvage (streaming: retry zero-chunk streams)

  • tests/agent/test_bedrock_adapter.py

    @@ -1712,3 +1712,85 @@ def test_accepts_boto3_with_unparseable_version(self):
             with patch.dict("sys.modules", {"boto3": fake_boto3}):
                 result = _require_boto3()
                 assert result is fake_boto3
    +
    +class TestImageBase64Decoding:
    +    """Image data URLs must be decoded to raw bytes before passing to Converse API.
    +
    +    boto3 re-encodes at the wire layer, so passing the base64 string directly
    +    results in double-encoding. Bedrock rejects with 'Failed to sanitize image'.
    +    Ref: #33317.
    +    """
    +
    +    def test_data_url_decoded_to_bytes(self):
    +        from agent.bedrock_adapter import _convert_content_to_converse
    +        import base64
    +
    +        # A tiny 1x1 red PNG
    +        raw_png = base64.b64decode(
    +            "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
    +        )
    +        data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
    +
    +        content = [{"type": "image_url", "image_url": {"url": data_url}}]
    +        blocks = _convert_content_to_converse(content)
    +
    +        assert len(blocks) == 1
    +        img_block = blocks[0]["image"]
    +        assert img_block["format"] == "png"
    +        # Must be raw bytes, not a base64 string
    +        assert isinstance(img_block["source"]["bytes"], bytes)
    +        assert img_block["source"]["bytes"] == raw_png
    +
    +    def test_invalid_base64_falls_back_to_encode(self):
    +        from agent.bedrock_adapter import _convert_content_to_converse
    +
    +        data_url = "data:image/jpeg;base64,NOT_VALID_BASE64!!!"
    +        content = [{"type": "image_url", "image_url": {"url": data_url}}]
    +        blocks = _convert_content_to_converse(content)
    +
    +        # Should not crash — falls back to encoding the string as bytes
    +        assert len(blocks) == 1
    +        assert isinstance(blocks[0]["image"]["source"]["bytes"], bytes)
    +
    +
    +class TestBearerTokenRoutesToConverse:
    +    """Bearer Token users must go through Converse API, not AnthropicBedrock SDK.
    +
    +    The AnthropicBedrock SDK only supports SigV4 signing — it cannot use
    +    AWS_BEARER_TOKEN_BEDROCK. Ref: #28156.
    +    """
    +
    +    def _resolve(self, monkeypatch, *, bearer: bool):
    +        import os
    +
    +        from hermes_cli import runtime_provider as rp
    +
    +        if bearer:
    +            monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "test-bearer-token-123")
    +        else:
    +            monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
    +        monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
    +        assert "AWS_BEARER_TOKEN_BEDROCK" in os.environ or not bearer
    +
    +        monkeypatch.setattr(
    +            rp,
    +            "_get_model_config",
    +            lambda: {
    +                "default": "us.anthropic.claude-sonnet-4-6",
    +                "provider": "bedrock",
    +            },
    +        )
    +        monkeypatch.setattr(rp, "load_config", lambda: {"bedrock": {}})
    +        return rp.resolve_runtime_provider(requested="bedrock")
    +
    +    def test_bearer_token_forces_converse_for_claude(self, monkeypatch):
    +        """Claude model + Bearer Token → bedrock_converse, not anthropic_messages."""
    +        runtime = self._resolve(monkeypatch, bearer=True)
    +        assert runtime["api_mode"] == "bedrock_converse"
    +        assert "bedrock_anthropic" not in runtime
    +
    +    def test_sigv4_claude_still_uses_anthropic_bedrock_sdk(self, monkeypatch):
    +        """Without a bearer token, Claude keeps the AnthropicBedrock SDK path."""
    +        runtime = self._resolve(monkeypatch, bearer=False)
    +        assert runtime["api_mode"] == "anthropic_messages"
    +        assert runtime.get("bedrock_anthropic") is True

  • tests/hermes_cli/test_bedrock_region_scoped_picker.py

    @@ -0,0 +1,82 @@
    +"""Regression tests for #28156 — Bedrock picker must be region-scoped.
    +
    +Geo-prefixed cross-region inference profiles (us.*, eu.*, apac.*, ...) only
    +route from endpoints in their own geography. Offering us.* profiles to an
    +eu-central-2 picker produces configs AWS rejects regardless of credentials.
    +"""
    +
    +from hermes_cli.model_setup_flows import (
    +    BEDROCK_GEO_PREFIXES,
    +    bedrock_model_routable_from_region,
    +    bedrock_region_geo_prefix,
    +)
    +
    +
    +class TestRegionGeoPrefix:
    +    def test_known_geographies(self):
    +        assert bedrock_region_geo_prefix("us-east-1") == "us."
    +        assert bedrock_region_geo_prefix("eu-central-2") == "eu."
    +        assert bedrock_region_geo_prefix("ap-southeast-1") == "ap."
    +        assert bedrock_region_geo_prefix("ca-central-1") == "ca."
    +        assert bedrock_region_geo_prefix("sa-east-1") == "sa."
    +        assert bedrock_region_geo_prefix("me-south-1") == "me."
    +        assert bedrock_region_geo_prefix("af-south-1") == "af."
    +
    +    def test_unknown_region_is_empty(self):
    +        assert bedrock_region_geo_prefix("") == ""
    +        assert bedrock_region_geo_prefix("moon-base-1") == ""
    +
    +
    +class TestRoutableFromRegion:
    +    def test_us_profile_not_offered_in_eu(self):
    +        assert not bedrock_model_routable_from_region(
    +            "us.anthropic.claude-sonnet-4-6", "eu-central-2"
    +        )
    +
    +    def test_eu_profile_offered_in_eu(self):
    +        assert bedrock_model_routable_from_region(
    +            "eu.anthropic.claude-sonnet-4-6", "eu-central-2"
    +        )
    +
    +    def test_global_profile_offered_everywhere(self):
    +        for region in ("eu-central-2", "us-east-1", "ap-southeast-1"):
    +            assert bedrock_model_routable_from_region(
    +                "global.anthropic.claude-sonnet-4-6", region
    +            )
    +
    +    def test_bare_foundation_id_offered_everywhere(self):
    +        assert bedrock_model_routable_from_region(
    +            "anthropic.claude-3-sonnet-20240229-v1:0", "eu-central-2"
    +        )
    +
    +    def test_apac_spellings_route_in_ap_regions(self):
    +        for prefix in ("ap.", "apac.", "jp."):
    +            assert bedrock_model_routable_from_region(
    +                f"{prefix}anthropic.claude-sonnet-4-6", "ap-northeast-1"
    +            )
    +
    +    def test_eu_profile_not_offered_in_us(self):
    +        assert not bedrock_model_routable_from_region(
    +            "eu.anthropic.claude-sonnet-4-6", "us-east-1"
    +        )
    +
    +    def test_unknown_region_hides_nothing(self):
    +        assert bedrock_model_routable_from_region(
    +            "us.anthropic.claude-sonnet-4-6", ""
    +        )
    +
    +
    +class TestGeoPrefixContract:
    +    def test_every_geo_prefix_maps_to_a_routable_region_or_is_alias(self):
    +        """Invariant: each geo prefix is either reachable from some region's
    +        geo mapping or an ap-family alias — no dead entries."""
    +        mapped = {
    +            bedrock_region_geo_prefix(r)
    +            for r in (
    +                "us-east-1", "eu-central-2", "ap-southeast-1",
    +                "ca-central-1", "sa-east-1", "me-south-1", "af-south-1",
    +            )
    +        }
    +        ap_aliases = {"apac.", "jp."}
    +        for prefix in BEDROCK_GEO_PREFIXES:
    +            assert prefix in mapped or prefix in ap_aliases