feat: allow custom endpoints to use responses API via api_mode override (#1651)

Add HERMES_API_MODE env var and model.api_mode config field to let
custom OpenAI-compatible endpoints opt into codex_responses mode
without requiring the OpenAI Codex OAuth provider path.

- _get_configured_api_mode() reads HERMES_API_MODE env (precedence)
  then model.api_mode from config.yaml; validates against whitelist
- Applied in both _resolve_openrouter_runtime() and
  _resolve_named_custom_runtime() (original PR only covered openrouter)
- Fix _dump_api_request_debug() to show /responses URL when in
  codex_responses mode instead of always showing /chat/completions
- Tests for config override, env override, invalid values, named
  custom providers, and debug dump URL for both API modes

Inspired by PR #1041 by @mxyhi.

Co-authored-by: mxyhi <mxyhi@users.noreply.github.com>

tests/test_run_agent_codex_responses.py

@@ -750,3 +750,40 @@ def test_run_conversation_codex_continues_after_ack_for_directory_listing_prompt
         for msg in result["messages"]
     )
     assert any(msg.get("role") == "tool" and msg.get("tool_call_id") == "call_1" for msg in result["messages"])
+
+
+def test_dump_api_request_debug_uses_responses_url(monkeypatch, tmp_path):
+    """Debug dumps should show /responses URL when in codex_responses mode."""
+    import json
+    agent = _build_agent(monkeypatch)
+    agent.base_url = "http://127.0.0.1:9208/v1"
+    agent.logs_dir = tmp_path
+
+    dump_file = agent._dump_api_request_debug(_codex_request_kwargs(), reason="preflight")
+
+    payload = json.loads(dump_file.read_text())
+    assert payload["request"]["url"] == "http://127.0.0.1:9208/v1/responses"
+
+
+def test_dump_api_request_debug_uses_chat_completions_url(monkeypatch, tmp_path):
+    """Debug dumps should show /chat/completions URL for chat_completions mode."""
+    import json
+    _patch_agent_bootstrap(monkeypatch)
+    agent = run_agent.AIAgent(
+        model="gpt-4o",
+        base_url="http://127.0.0.1:9208/v1",
+        api_key="test-key",
+        quiet_mode=True,
+        max_iterations=1,
+        skip_context_files=True,
+        skip_memory=True,
+    )
+    agent.logs_dir = tmp_path
+
+    dump_file = agent._dump_api_request_debug(
+        {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]},
+        reason="preflight",
+    )
+
+    payload = json.loads(dump_file.read_text())
+    assert payload["request"]["url"] == "http://127.0.0.1:9208/v1/chat/completions"

tests/test_runtime_provider_resolution.py

@@ -326,3 +326,74 @@ def test_resolve_requested_provider_precedence(monkeypatch):
 
     monkeypatch.delenv("HERMES_INFERENCE_PROVIDER", raising=False)
     assert rp.resolve_requested_provider() == "auto"
+
+
+# ── api_mode override tests ─────────────────────────────────────────────
+
+
+def test_custom_endpoint_api_mode_from_config(monkeypatch):
+    """model.api_mode in config.yaml should override the default chat_completions."""
+    monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter")
+    monkeypatch.setattr(
+        rp, "_get_model_config",
+        lambda: {
+            "provider": "custom",
+            "base_url": "http://127.0.0.1:9208/v1",
+            "api_mode": "codex_responses",
+        },
+    )
+    monkeypatch.setenv("OPENAI_BASE_URL", "http://127.0.0.1:9208/v1")
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+    monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False)
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+    monkeypatch.delenv("HERMES_API_MODE", raising=False)
+
+    resolved = rp.resolve_runtime_provider(requested="custom")
+
+    assert resolved["api_mode"] == "codex_responses"
+    assert resolved["base_url"] == "http://127.0.0.1:9208/v1"
+
+
+def test_env_api_mode_overrides_config(monkeypatch):
+    """HERMES_API_MODE env var takes precedence over config."""
+    monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter")
+    monkeypatch.setattr(rp, "_get_model_config", lambda: {"api_mode": "chat_completions"})
+    monkeypatch.setenv("OPENAI_BASE_URL", "http://127.0.0.1:9208/v1")
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+    monkeypatch.setenv("HERMES_API_MODE", "codex_responses")
+    monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False)
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+
+    resolved = rp.resolve_runtime_provider(requested="custom")
+
+    assert resolved["api_mode"] == "codex_responses"
+
+
+def test_invalid_api_mode_ignored(monkeypatch):
+    """Invalid api_mode values should fall back to chat_completions."""
+    monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "openrouter")
+    monkeypatch.setattr(rp, "_get_model_config", lambda: {"api_mode": "bogus_mode"})
+    monkeypatch.setenv("OPENAI_BASE_URL", "http://127.0.0.1:9208/v1")
+    monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+    monkeypatch.delenv("HERMES_API_MODE", raising=False)
+    monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False)
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+
+    resolved = rp.resolve_runtime_provider(requested="custom")
+
+    assert resolved["api_mode"] == "chat_completions"
+
+
+def test_named_custom_provider_respects_api_mode(monkeypatch):
+    """Named custom providers should also pick up api_mode overrides."""
+    monkeypatch.setattr(rp, "resolve_provider", lambda *a, **k: "my-server")
+    monkeypatch.setattr(
+        rp, "_get_named_custom_provider",
+        lambda p: {"name": "my-server", "base_url": "http://localhost:8000/v1", "api_key": "sk-test"},
+    )
+    monkeypatch.setenv("HERMES_API_MODE", "codex_responses")
+
+    resolved = rp.resolve_runtime_provider(requested="my-server")
+
+    assert resolved["api_mode"] == "codex_responses"
+    assert resolved["base_url"] == "http://localhost:8000/v1"