From a05bda6d996021fb3568dc96f91995447ef8fc57 Mon Sep 17 00:00:00 2001 From: Simon Rosenberg Date: Thu, 18 Jun 2026 10:51:06 +0200 Subject: [PATCH] [AgentProfile][agent-server] materialize endpoint (resolve dry-run) POST /api/agent-profiles/{name}/materialize calls resolve_agent_profile_dry_run and returns AgentProfileDiagnostics (200 always; dangling LLM/MCP refs surface in the body, not as 4xx). Tests: valid OpenHands + ACP profiles, dangling LLM/MCP refs, 404 on unknown name, and raw-secret redaction assertion. Co-Authored-By: Claude Sonnet 4.6 --- .../agent_server/agent_profiles_router.py | 50 +++++- .../test_agent_profiles_router.py | 144 +++++++++++++++++- 2 files changed, 191 insertions(+), 3 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/agent_profiles_router.py b/openhands-agent-server/openhands/agent_server/agent_profiles_router.py index 9776020fe7..c9680b2c8a 100644 --- a/openhands-agent-server/openhands/agent_server/agent_profiles_router.py +++ b/openhands-agent-server/openhands/agent_server/agent_profiles_router.py @@ -6,8 +6,9 @@ pointer-only — unlike the LLM ``/activate`` it must **not** write ``agent_settings`` (the creation-time-only contract). -``POST /{id}/materialize`` is a fast-follow once the resolver (#3717) lands; it -is deliberately not implemented here so this router ships independently. +``POST /{name}/materialize`` performs a dry-run resolve of a profile's LLM and +MCP references and returns :class:`~openhands.sdk.profiles.AgentProfileDiagnostics` +(never raises on dangling refs — those appear in the body). """ import copy @@ -31,13 +32,16 @@ PersistedSettings, get_settings_store, ) +from openhands.sdk.llm.llm_profile_store import LLMProfileStore from openhands.sdk.logger import get_logger from openhands.sdk.profiles import ( ACPAgentProfile, + AgentProfileDiagnostics, AgentProfileStore, OpenHandsAgentProfile, ProfileLimitExceeded, ProfileVerificationSettings, + resolve_agent_profile_dry_run, validate_agent_profile, ) from openhands.sdk.profiles.agent_profile_store import PROFILE_NAME_PATTERN @@ -553,3 +557,45 @@ def set_pointer(settings: PersistedSettings) -> PersistedSettings: id=profile_id, message=f"Agent profile '{profile_id}' activated", ) + + +@agent_profiles_router.post( + "/{name}/materialize", + response_model=AgentProfileDiagnostics, +) +async def materialize_agent_profile( + request: Request, name: ProfileName +) -> AgentProfileDiagnostics: + """Dry-run resolve a profile's LLM/MCP references; return a diagnostics report. + + Dangling LLM/MCP references are reported in the body (valid=False) rather + than raising — the only error status is 404 (unknown profile name). + resolved_settings is redacted (api_key_set booleans; no raw secrets). + """ + cipher = get_cipher(request) + + store = AgentProfileStore() + try: + with _store_errors(): + profile = store.load(name, cipher=cipher) + except FileNotFoundError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Agent profile '{name}' not found", + ) + + # The store leaves skills[].mcp_tools encrypted on load; decrypt so the + # resolver builds settings from plaintext (not ciphertext) values. + profile = _decrypt_profile_mcp_tools(profile, cipher) + + config = get_config(request) + settings = get_settings_store(config).load() or PersistedSettings() + mcp_config = settings.agent_settings.mcp_config + + llm_store = LLMProfileStore() + return resolve_agent_profile_dry_run( + profile, + llm_store=llm_store, + mcp_config=mcp_config, + cipher=cipher, + ) diff --git a/tests/agent_server/test_agent_profiles_router.py b/tests/agent_server/test_agent_profiles_router.py index fcd76971c2..993656d90b 100644 --- a/tests/agent_server/test_agent_profiles_router.py +++ b/tests/agent_server/test_agent_profiles_router.py @@ -17,7 +17,13 @@ from openhands.agent_server.api import create_app from openhands.agent_server.config import Config from openhands.agent_server.persistence import reset_stores -from openhands.sdk.profiles import AgentProfileStore, OpenHandsAgentProfile +from openhands.sdk.llm import LLM +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.profiles import ( + ACPAgentProfile, + AgentProfileStore, + OpenHandsAgentProfile, +) @pytest.fixture @@ -676,3 +682,139 @@ def test_save_at_limit_returns_409(client, store, monkeypatch): response = client.post("/api/agent-profiles/second", json={"llm_profile_ref": "y"}) assert response.status_code == 409 assert "limit" in response.json()["detail"].lower() + + +# ── Materialize (resolve dry-run) ──────────────────────────────────────────── + + +@pytest.fixture +def temp_llm_profiles_dir(): + with tempfile.TemporaryDirectory() as tmpdir: + llm_dir = Path(tmpdir) / "llm-profiles" + llm_dir.mkdir(parents=True, exist_ok=True) + yield llm_dir + + +@pytest.fixture +def client_with_llm_store( + temp_agent_profiles_dir, temp_settings_dir, temp_llm_profiles_dir, monkeypatch +): + """Test client with isolated agent-profile/settings/llm-profile dirs, no cipher.""" + reset_stores() + monkeypatch.setenv("OH_PERSISTENCE_DIR", str(temp_settings_dir)) + config = Config(static_files_path=None, session_api_keys=[], secret_key=None) + app = create_app(config) + with ( + patch( + "openhands.agent_server.agent_profiles_router.AgentProfileStore", + lambda: AgentProfileStore(base_dir=temp_agent_profiles_dir), + ), + patch( + "openhands.agent_server.agent_profiles_router.LLMProfileStore", + lambda: LLMProfileStore(base_dir=temp_llm_profiles_dir), + ), + ): + yield TestClient(app) + reset_stores() + + +@pytest.fixture +def llm_store(temp_llm_profiles_dir): + return LLMProfileStore(base_dir=temp_llm_profiles_dir) + + +def test_materialize_valid_openhands_profile(client_with_llm_store, store, llm_store): + """Valid OpenHands profile with a resolved LLM returns 200 + valid=True.""" + llm_store.save("base-llm", LLM(model="gpt-4o"), include_secrets=True) + store.save(OpenHandsAgentProfile(name="p", llm_profile_ref="base-llm")) + + response = client_with_llm_store.post("/api/agent-profiles/p/materialize") + + assert response.status_code == 200 + body = response.json() + assert body["valid"] is True + assert body["agent_kind"] == "openhands" + assert body["llm_profile_ref"] == "base-llm" + assert body["llm_profile_resolved"] is True + assert body["errors"] == [] + assert body["resolved_settings"] is not None + assert body["dangling_mcp_server_refs"] == [] + + +def test_materialize_valid_acp_profile(client_with_llm_store, store): + """Valid ACP profile returns 200 + valid=True (no LLM ref needed).""" + store.save(ACPAgentProfile(name="acp-p", acp_server="codex", acp_model="gpt-5.5")) + + response = client_with_llm_store.post("/api/agent-profiles/acp-p/materialize") + + assert response.status_code == 200 + body = response.json() + assert body["valid"] is True + assert body["agent_kind"] == "acp" + assert body["errors"] == [] + assert body["resolved_settings"] is not None + + +def test_materialize_dangling_llm_ref(client_with_llm_store, store): + """A profile referencing a missing LLM profile returns 200, valid=False.""" + store.save(OpenHandsAgentProfile(name="p", llm_profile_ref="nonexistent")) + + response = client_with_llm_store.post("/api/agent-profiles/p/materialize") + + assert response.status_code == 200 + body = response.json() + assert body["valid"] is False + assert body["llm_profile_ref"] == "nonexistent" + assert body["llm_profile_resolved"] is False + assert body["resolved_settings"] is None + assert any("nonexistent" in e for e in body["errors"]) + + +def test_materialize_dangling_mcp_ref(client_with_llm_store, store, llm_store): + """A profile with a missing MCP server ref returns 200, valid=False.""" + llm_store.save("base-llm", LLM(model="gpt-4o"), include_secrets=True) + store.save( + OpenHandsAgentProfile( + name="p", + llm_profile_ref="base-llm", + mcp_server_refs=["missing-server"], + ) + ) + + response = client_with_llm_store.post("/api/agent-profiles/p/materialize") + + assert response.status_code == 200 + body = response.json() + assert body["valid"] is False + assert body["dangling_mcp_server_refs"] == ["missing-server"] + assert body["resolved_settings"] is None + + +def test_materialize_unknown_name_returns_404(client_with_llm_store): + """Materializing an unknown profile name returns 404.""" + response = client_with_llm_store.post("/api/agent-profiles/ghost/materialize") + + assert response.status_code == 404 + assert "not found" in response.json()["detail"].lower() + + +def test_materialize_no_raw_secrets_in_resolved_settings( + client_with_llm_store, store, llm_store +): + """resolved_settings must not contain raw API key values.""" + raw_key = "sk-secret-key-should-not-appear" + from pydantic import SecretStr + + llm_store.save( + "base-llm", + LLM(model="gpt-4o", api_key=SecretStr(raw_key)), + include_secrets=True, + ) + store.save(OpenHandsAgentProfile(name="p", llm_profile_ref="base-llm")) + + response = client_with_llm_store.post("/api/agent-profiles/p/materialize") + + assert response.status_code == 200 + body = response.json() + assert body["valid"] is True + assert raw_key not in response.text