{"repo":"NousResearch/hermes-agent","public_id":"8147","display_label":"Vision image format bugs across multiple providers","summary":"Multiple issues and fixes address incorrect image format handling when routing vision requests, particularly OpenAI-style image_url payloads being sent to incompatible endpoints like MiniMax and Claude. Fixes also touch custom provider API mode and context length configuration.","subsystem":"agent","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":1,"attempt_prs":0},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":8243,"merged_at":"2026-04-12T08:44:18+00:00"},"resolution":{"state":"partially_resolved","by":"pr-8243"},"freshness":{"state":"stale","last_verified_at":"2026-04-12T08:44:18+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.4533333333333333},"fix_edges":[{"issue_number":8147,"pr_number":8243,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-12T08:44:18+00:00"},{"issue_number":8161,"pr_number":8243,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-12T08:44:18+00:00"},{"issue_number":8181,"pr_number":8243,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-12T08:44:18+00:00"}],"members":[{"kind":"issue","number":8147,"title":"vision_analyze fails with MiniMax — OpenAI image_url format sent to Anthropic-compatible endpoint","url":null,"state":"closed"},{"kind":"issue","number":8911,"title":"Vision tool fails with MiniMax provider: image not detected","url":null,"state":"closed"},{"kind":"pr","number":8243,"title":"fix: openai-codex context length, custom provider api_mode, vision image format (#8161, #8181, #8147)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/auxiliary_client.py","agent/models_dev.py","hermes_cli/main.py","tests/agent/test_auxiliary_client.py","tests/agent/test_models_dev.py","tests/hermes_cli/test_custom_provider_model_switch.py"],"fix_hunks":[{"file":"agent/auxiliary_client.py","hunks":["@@ -2065,6 +2065,75 @@ def _get_task_timeout(task: str, default: float = _DEFAULT_AUX_TIMEOUT) -> float\n     return default\n \n \n+# ---------------------------------------------------------------------------\n+# Anthropic-compatible endpoint detection + image block conversion\n+# ---------------------------------------------------------------------------\n+\n+# Providers that use Anthropic-compatible endpoints (via OpenAI SDK wrapper).\n+# Their image content blocks must use Anthropic format, not OpenAI format.\n+_ANTHROPIC_COMPAT_PROVIDERS = frozenset({\"minimax\", \"minimax-cn\"})\n+\n+\n+def _is_anthropic_compat_endpoint(provider: str, base_url: str) -> bool:\n+    \"\"\"Detect if an endpoint expects Anthropic-format content blocks.\n+\n+    Returns True for known Anthropic-compatible providers (MiniMax) and\n+    any endpoint whose URL contains ``/anthropic`` in the path.\n+    \"\"\"\n+    if provider in _ANTHROPIC_COMPAT_PROVIDERS:\n+        return True\n+    url_lower = (base_url or \"\").lower()\n+    return \"/anthropic\" in url_lower\n+\n+\n+def _convert_openai_images_to_anthropic(messages: list) -> list:\n+    \"\"\"Convert OpenAI ``image_url`` content blocks to Anthropic ``image`` blocks.\n+\n+    Only touches messages that have list-type content with ``image_url`` blocks;\n+    plain text messages pass through unchanged.\n+    \"\"\"\n+    converted = []\n+    for msg in messages:\n+        content = msg.get(\"content\")\n+        if not isinstance(content, list):\n+            converted.append(msg)\n+            continue\n+        new_content = []\n+        changed = False\n+        for block in content:\n+            if block.get(\"type\") == \"image_url\":\n+                image_url_val = (block.get(\"image_url\") or {}).get(\"url\", \"\")\n+                if image_url_val.startswith(\"data:\"):\n+                    # Parse data URI: data:<media_type>;base64,<data>\n+                    header, _, b64data = image_url_val.partition(\",\")\n+                    media_type = \"image/png\"\n+                    if \":\" in header and \";\" in header:\n+                        media_type = header.split(\":\", 1)[1].split(\";\", 1)[0]\n+                    new_content.append({\n+                        \"type\": \"image\",\n+                        \"source\": {\n+                            \"type\": \"base64\",\n+                            \"media_type\": media_type,\n+                            \"data\": b64data,\n+                        },\n+                    })\n+                else:\n+                    # URL-based image\n+                    new_content.append({\n+                        \"type\": \"image\",\n+                        \"source\": {\n+                            \"type\": \"url\",\n+                            \"url\": image_url_val,\n+                        },\n+                    })\n+                changed = True\n+            else:\n+                new_content.append(block)\n+        converted.append({**msg, \"content\": new_content} if changed else msg)\n+    return converted\n+\n+\n+\n def _build_call_kwargs(\n     provider: str,\n     model: str,","@@ -2255,6 +2324,11 @@ def call_llm(\n         tools=tools, timeout=effective_timeout, extra_body=extra_body,\n         base_url=resolved_base_url)\n \n+    # Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)\n+    _client_base = str(getattr(client, \"base_url\", \"\") or \"\")\n+    if _is_anthropic_compat_endpoint(resolved_provider, _client_base):\n+        kwargs[\"messages\"] = _convert_openai_images_to_anthropic(kwargs[\"messages\"])\n+\n     # Handle max_tokens vs max_completion_tokens retry, then payment fallback.\n     try:\n         return _validate_llm_response(","@@ -2443,6 +2517,11 @@ async def async_call_llm(\n         tools=tools, timeout=effective_timeout, extra_body=extra_body,\n         base_url=resolved_base_url)\n \n+    # Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)\n+    _client_base = str(getattr(client, \"base_url\", \"\") or \"\")\n+    if _is_anthropic_compat_endpoint(resolved_provider, _client_base):\n+        kwargs[\"messages\"] = _convert_openai_images_to_anthropic(kwargs[\"messages\"])\n+\n     try:\n         return _validate_llm_response(\n             await client.chat.completions.create(**kwargs), task)"]},{"file":"agent/models_dev.py","hunks":["@@ -144,6 +144,8 @@ class ProviderInfo:\n PROVIDER_TO_MODELS_DEV: Dict[str, str] = {\n     \"openrouter\": \"openrouter\",\n     \"anthropic\": \"anthropic\",\n+    \"openai\": \"openai\",\n+    \"openai-codex\": \"openai\",\n     \"zai\": \"zai\",\n     \"kimi-coding\": \"kimi-for-coding\",\n     \"minimax\": \"minimax\","]},{"file":"hermes_cli/main.py","hunks":["@@ -1107,6 +1107,7 @@ def _named_custom_provider_map(cfg) -> dict[str, dict[str, str]]:\n                 \"base_url\": base_url,\n                 \"api_key\": entry.get(\"api_key\", \"\"),\n                 \"model\": entry.get(\"model\", \"\"),\n+                \"api_mode\": entry.get(\"api_mode\", \"\"),\n             }\n         return custom_provider_map\n ","@@ -1955,6 +1956,12 @@ def _model_flow_named_custom(config, provider_info):\n     model[\"base_url\"] = base_url\n     if api_key:\n         [redacted:secret-assignment]\"api_key\"] = api_key\n+    # Apply api_mode from custom_providers entry, or clear stale value\n+    custom_api_mode = provider_info.get(\"api_mode\", \"\")\n+    if custom_api_mode:\n+        model[\"api_mode\"] = custom_api_mode\n+    else:\n+        model.pop(\"api_mode\", None)  # let runtime auto-detect from URL\n     save_config(cfg)\n     deactivate_provider()\n "]},{"file":"tests/agent/test_auxiliary_client.py","hunks":["@@ -1560,3 +1560,74 @@ def test_warning_only_fires_once(self, monkeypatch, caplog):\n \n         assert not any(\"OPENAI_BASE_URL is set\" in rec.message for rec in caplog.records), \\\n             \"Warning should not fire a second time\"\n+\n+\n+# ---------------------------------------------------------------------------\n+# Anthropic-compatible image block conversion\n+# ---------------------------------------------------------------------------\n+\n+class TestAnthropicCompatImageConversion:\n+    \"\"\"Tests for _is_anthropic_compat_endpoint and _convert_openai_images_to_anthropic.\"\"\"\n+\n+    def test_known_providers_detected(self):\n+        from agent.auxiliary_client import _is_anthropic_compat_endpoint\n+        assert _is_anthropic_compat_endpoint(\"minimax\", \"\")\n+        assert _is_anthropic_compat_endpoint(\"minimax-cn\", \"\")\n+\n+    def test_openrouter_not_detected(self):\n+        from agent.auxiliary_client import _is_anthropic_compat_endpoint\n+        assert not _is_anthropic_compat_endpoint(\"openrouter\", \"\")\n+        assert not _is_anthropic_compat_endpoint(\"anthropic\", \"\")\n+\n+    def test_url_based_detection(self):\n+        from agent.auxiliary_client import _is_anthropic_compat_endpoint\n+        assert _is_anthropic_compat_endpoint(\"custom\", \"https://api.minimax.io/anthropic\")\n+        assert _is_anthropic_compat_endpoint(\"custom\", \"https://example.com/anthropic/v1\")\n+        assert not _is_anthropic_compat_endpoint(\"custom\", \"https://api.openai.com/v1\")\n+\n+    def test_base64_image_converted(self):\n+        from agent.auxiliary_client import _convert_openai_images_to_anthropic\n+        messages = [{\n+            \"role\": \"user\",\n+            \"content\": [\n+                {\"type\": \"text\", \"text\": \"describe\"},\n+                {\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/png;base64,iVBOR=\"}}\n+            ]\n+        }]\n+        result = _convert_openai_images_to_anthropic(messages)\n+        img_block = result[0][\"content\"][1]\n+        assert img_block[\"type\"] == \"image\"\n+        assert img_block[\"source\"][\"type\"] == \"base64\"\n+        assert img_block[\"source\"][\"media_type\"] == \"image/png\"\n+        assert img_block[\"source\"][\"data\"] == \"iVBOR=\"\n+\n+    def test_url_image_converted(self):\n+        from agent.auxiliary_client import _convert_openai_images_to_anthropic\n+        messages = [{\n+            \"role\": \"user\",\n+            \"content\": [\n+                {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://example.com/img.jpg\"}}\n+            ]\n+        }]\n+        result = _convert_openai_images_to_anthropic(messages)\n+        img_block = result[0][\"content\"][0]\n+        assert img_block[\"type\"] == \"image\"\n+        assert img_block[\"source\"][\"type\"] == \"url\"\n+        assert img_block[\"source\"][\"url\"] == \"https://example.com/img.jpg\"\n+\n+    def test_text_only_messages_unchanged(self):\n+        from agent.auxiliary_client import _convert_openai_images_to_anthropic\n+        messages = [{\"role\": \"user\", \"content\": \"Hello\"}]\n+        result = _convert_openai_images_to_anthropic(messages)\n+        assert result[0] is messages[0]  # same object, not copied\n+\n+    def test_jpeg_media_type_parsed(self):\n+        from agent.auxiliary_client import _convert_openai_images_to_anthropic\n+        messages = [{\n+            \"role\": \"user\",\n+            \"content\": [\n+                {\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/jpeg;base64,/9j/=\"}}\n+            ]\n+        }]\n+        result = _convert_openai_images_to_anthropic(messages)\n+        assert result[0][\"content\"][0][\"source\"][\"media_type\"] == \"image/jpeg\""]},{"file":"tests/agent/test_models_dev.py","hunks":["@@ -87,7 +87,10 @@ def test_known_providers_mapped(self):\n \n     def test_unmapped_provider_not_in_dict(self):\n         assert \"nous\" not in PROVIDER_TO_MODELS_DEV\n-        assert \"openai-codex\" not in PROVIDER_TO_MODELS_DEV\n+\n+    def test_openai_codex_mapped_to_openai(self):\n+        assert PROVIDER_TO_MODELS_DEV[\"openai\"] == \"openai\"\n+        assert PROVIDER_TO_MODELS_DEV[\"openai-codex\"] == \"openai\"\n \n \n class TestExtractContext:"]},{"file":"tests/hermes_cli/test_custom_provider_model_switch.py","hunks":["@@ -122,3 +122,54 @@ def test_no_saved_model_still_works(self, config_home):\n         model = config.get(\"model\")\n         assert isinstance(model, dict)\n         assert model[\"default\"] == \"model-X\"\n+\n+    def test_api_mode_set_from_provider_info(self, config_home):\n+        \"\"\"When custom_providers entry has api_mode, it should be applied.\"\"\"\n+        import yaml\n+        from hermes_cli.main import _model_flow_named_custom\n+\n+        provider_info = {\n+            \"name\": \"Anthropic Proxy\",\n+            \"base_url\": \"https://proxy.example.com/anthropic\",\n+            \"api_key\": \"***\",\n+            \"model\": \"claude-3\",\n+            \"api_mode\": \"anthropic_messages\",\n+        }\n+\n+        with patch(\"hermes_cli.models.fetch_api_models\", return_value=[\"claude-3\"]), \\\n+             patch.dict(\"sys.modules\", {\"simple_term_menu\": None}), \\\n+             patch(\"builtins.input\", return_value=\"1\"), \\\n+             patch(\"builtins.print\"):\n+            _model_flow_named_custom({}, provider_info)\n+\n+        config = yaml.safe_load((config_home / \"config.yaml\").read_text()) or {}\n+        model = config.get(\"model\")\n+        assert isinstance(model, dict)\n+        assert model.get(\"api_mode\") == \"anthropic_messages\"\n+\n+    def test_api_mode_cleared_when_not_specified(self, config_home):\n+        \"\"\"When custom_providers entry has no api_mode, stale api_mode is removed.\"\"\"\n+        import yaml\n+        from hermes_cli.main import _model_flow_named_custom\n+\n+        # Pre-seed a stale api_mode in config\n+        config_path = config_home / \"config.yaml\"\n+        config_path.write_text(yaml.dump({\"model\": {\"api_mode\": \"anthropic_messages\"}}))\n+\n+        provider_info = {\n+            \"name\": \"My vLLM\",\n+            \"base_url\": \"https://vllm.example.com/v1\",\n+            \"api_key\": \"***\",\n+            \"model\": \"llama-3\",\n+        }\n+\n+        with patch(\"hermes_cli.models.fetch_api_models\", return_value=[\"llama-3\"]), \\\n+             patch.dict(\"sys.modules\", {\"simple_term_menu\": None}), \\\n+             patch(\"builtins.input\", return_value=\"1\"), \\\n+             patch(\"builtins.print\"):\n+            _model_flow_named_custom({}, provider_info)\n+\n+        config = yaml.safe_load((config_home / \"config.yaml\").read_text()) or {}\n+        model = config.get(\"model\")\n+        assert isinstance(model, dict)\n+        assert \"api_mode\" not in model, \"Stale api_mode should be removed\"\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}