diff --git a/inferedgelab/studio/routes.py b/inferedgelab/studio/routes.py index 3299f3d..6cdf126 100644 --- a/inferedgelab/studio/routes.py +++ b/inferedgelab/studio/routes.py @@ -213,12 +213,16 @@ def _build_imported_compare_response(base: dict[str, Any], new: dict[str, Any]) def _with_compare_keys(result: dict[str, Any]) -> dict[str, Any]: enriched = dict(result) if not enriched.get("backend_key"): - engine = enriched.get("engine") or enriched.get("backend") - device = enriched.get("device") or enriched.get("device_name") + engine = _first_display_value( + enriched.get("engine_backend"), + enriched.get("engine"), + enriched.get("backend"), + ) + device = _first_display_value(enriched.get("device_name"), enriched.get("device")) if engine and device: enriched["backend_key"] = f"{engine}__{device}" if not enriched.get("compare_key"): - model = enriched.get("model") + model = _first_display_value(enriched.get("model_name"), enriched.get("model")) batch = enriched.get("batch") height = enriched.get("height") width = enriched.get("width") @@ -226,3 +230,25 @@ def _with_compare_keys(result: dict[str, Any]) -> dict[str, Any]: if model and batch and height and width and precision: enriched["compare_key"] = f"{model}__b{batch}__h{height}w{width}__{precision}" return enriched + + +def _first_display_value(*values: Any) -> str: + for value in values: + display_value = _display_value(value) + if display_value: + return display_value + return "" + + +def _display_value(value: Any) -> str: + if value is None or value == "": + return "" + if isinstance(value, dict): + return _first_display_value( + value.get("name"), + value.get("backend"), + value.get("path"), + value.get("status"), + value.get("id"), + ) + return str(value) diff --git a/inferedgelab/studio/static/app.js b/inferedgelab/studio/static/app.js index b14c15d..87abdea 100644 --- a/inferedgelab/studio/static/app.js +++ b/inferedgelab/studio/static/app.js @@ -76,27 +76,41 @@ function setEmpty(selector, label = "No data available") { async function fetchJson(url) { assertHttpStudio(); - const response = await fetch(url, { headers: { Accept: "application/json" } }); - if (!response.ok) { - throw new Error(await responseErrorMessage(response)); + try { + const response = await fetch(url, { headers: { Accept: "application/json" } }); + if (!response.ok) { + throw new Error(await responseErrorMessage(response)); + } + return await parseJsonResponse(response); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Invalid JSON response from ${url}.`); + } + throw error; } - return response.json(); } async function postJson(url, payload) { assertHttpStudio(); - const response = await fetch(url, { - method: "POST", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - }); - if (!response.ok) { - throw new Error(await responseErrorMessage(response)); + try { + const response = await fetch(url, { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + }); + if (!response.ok) { + throw new Error(await responseErrorMessage(response)); + } + return await parseJsonResponse(response); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Invalid JSON response from ${url}.`); + } + throw error; } - return response.json(); } function assertHttpStudio() { @@ -122,6 +136,14 @@ async function responseErrorMessage(response) { return fallback; } +async function parseJsonResponse(response) { + try { + return await response.json(); + } catch (error) { + throw new SyntaxError("Response body is not valid JSON."); + } +} + async function loadJobs(preferredJobId = selectedJobId) { setLoading("#job-list"); try { @@ -143,8 +165,8 @@ async function loadJobs(preferredJobId = selectedJobId) { currentJobs = []; selectedJob = null; selectedJobId = null; - setEmpty("#job-list"); - renderJobDetail(); + setEmpty("#job-list", `Error: ${formatError(error)}`); + renderJobDetail(`Unable to load jobs: ${formatError(error)}`); } renderPipeline(); } @@ -168,7 +190,7 @@ async function loadJobDetail(jobId) { updateDecision(extractDecision(selectedJob)); } catch (error) { selectedJob = null; - renderJobDetail(`Unable to load ${jobId}.`); + renderJobDetail(`Unable to load ${jobId}: ${formatError(error)}`); } renderPipeline(); } @@ -182,11 +204,11 @@ async function loadCompare() { if (compareData.status === "empty") { activeDecision = null; renderDecision(decision); - } else if (!activeDecision) { + } else { updateDecision(decision); } } catch (error) { - compareData = null; + compareData = { status: "error", error: formatError(error) }; renderCompare(); if (!activeDecision) { updateDecision(null); @@ -226,8 +248,9 @@ async function runModel() { async function importResult() { const button = document.querySelector("#import-button"); const jsonPath = document.querySelector("#import-json-path").value.trim(); - if (!jsonPath) { - setStatus("#import-status", "Error: enter a result path.", "error"); + const jsonPayload = document.querySelector("#import-json-payload").value.trim(); + if (!jsonPath && !jsonPayload) { + setStatus("#import-status", "Error: enter a result path or JSON payload.", "error"); return; } @@ -236,7 +259,7 @@ async function importResult() { setStatus("#import-status", "Loading: importing Runtime result...", "loading"); renderPipeline(); try { - const payload = await postJson("/studio/api/import", { path: jsonPath }); + const payload = await postJson("/studio/api/import", buildImportPayload(jsonPath, jsonPayload)); importedResult = payload.result; setStatus( "#import-status", @@ -246,8 +269,10 @@ async function importResult() { "success", ); setState("#import-state", "completed"); + renderImportEvidence(payload); renderImportedResult(); await loadCompare(); + renderPipeline(); } catch (error) { setStatus("#import-status", `Error: ${formatError(error)}`, "error"); setState("#import-state", "idle"); @@ -257,6 +282,17 @@ async function importResult() { } } +function buildImportPayload(path, jsonPayload) { + if (jsonPayload) { + try { + return { result: JSON.parse(jsonPayload) }; + } catch (error) { + throw new Error("JSON payload is not valid JSON."); + } + } + return { path }; +} + async function loadJetsonCommand() { const textarea = document.querySelector("#jetson-command"); setState("#jetson-state", "running"); @@ -307,6 +343,34 @@ function renderPipeline() { }); } +function renderImportEvidence(payload) { + const target = document.querySelector("#import-evidence"); + if (!target) { + return; + } + target.replaceChildren(); + + const result = payload?.result || {}; + const missing = missingResultKeys(result); + target.append( + evidenceItem("model", runtimeModelName(result)), + evidenceItem("backend", normalizedBackendKey(result) || runtimeBackendName(result)), + evidenceItem("compare", result.compare_key || fallbackLabel("compare_key")), + evidenceItem("mean", result.mean_ms === undefined ? "-" : `${formatNumber(result.mean_ms)} ms`), + ); + if (missing.length > 0) { + target.append( + createElement("p", "evidence-warning", `Fallback metadata: missing ${missing.join(", ")}.`), + ); + } +} + +function evidenceItem(label, value) { + const item = createElement("div", "evidence-item"); + item.append(createElement("span", "metric-name", label), createElement("strong", "", formatValue(value))); + return item; +} + function renderRunPanel() { document.querySelector("#run-button").onclick = runModel; document.querySelector("#import-button").onclick = importResult; @@ -422,6 +486,11 @@ function renderCompare() { const target = document.querySelector("#compare-panel"); target.replaceChildren(); + if (compareData?.status === "error") { + target.append(errorCompareCard(compareData.error)); + return; + } + if (!compareData || compareData.status === "empty") { target.append(emptyCompareCard()); return; @@ -430,6 +499,7 @@ function renderCompare() { const base = compareData.base || compareData.data?.base || {}; const newer = compareData.new || compareData.data?.new || {}; const result = compareData.result || compareData.data?.result || {}; + const judgement = compareData.judgement || compareData.data?.judgement || {}; const meanMetric = result.metrics?.mean_ms || {}; const speedup = result.speedup || result.backend_comparison?.speedup || calculateSpeedup(base, newer); const tensorRt = findResultByBackend([base, newer], "tensorrt"); @@ -439,7 +509,7 @@ function renderCompare() { target.append( compareMetricCard("TensorRT", tensorRt?.mean_ms, normalizedBackendKey(tensorRt) || "tensorrt"), compareMetricCard("ONNX Runtime", onnx?.mean_ms, normalizedBackendKey(onnx) || "onnxruntime"), - compareSummaryCard(meanMetric, speedup, base, newer, sameBackend), + compareSummaryCard(meanMetric, speedup, base, newer, sameBackend, judgement.overall), ); } @@ -462,7 +532,7 @@ function renderDecision(decision) { target.append( createElement("p", "caption", "Decision"), createElement("h3", "", decisionName.toUpperCase()), - createElement("p", "body-text", decision.reason || "-"), + createElement("p", "body-text", decisionReason(decision)), createElement("p", "caption", decision.notes || decision.recommended_action || ""), ); } @@ -494,13 +564,13 @@ function compareMetricCard(label, meanMs, backendKey) { return card; } -function compareSummaryCard(metric, speedup, base, newer, sameBackend = false) { - const card = createElement("article", "compare-card highlight"); +function compareSummaryCard(metric, speedup, base, newer, sameBackend = false, overall = "unknown") { + const card = createElement("article", `compare-card highlight ${compareTone(overall)}`); const diff = formatLatencyDiff(metric); const faster = sameBackend ? "Same backend" : speedup ? `${formatNumber(speedup)}x faster` : "speedup unavailable"; const note = sameBackend ? "Import a TensorRT and an ONNX Runtime result to compare backend speedup." : `Latency diff: ${diff}`; card.append( - createElement("p", "caption", "Latency comparison"), + createElement("p", "caption", `Latency comparison / ${overall || "unknown"}`), createElement("h3", "", faster), createElement("p", "body-text", note), createElement("p", "caption", `${normalizedBackendKey(base) || "-"} -> ${normalizedBackendKey(newer) || "-"}`), @@ -508,6 +578,16 @@ function compareSummaryCard(metric, speedup, base, newer, sameBackend = false) { return card; } +function errorCompareCard(message) { + const card = createElement("article", "compare-card empty error-card"); + card.append( + createElement("p", "caption", "Compare error"), + createElement("h3", "", "Unable to load compare data"), + createElement("p", "body-text", message || "No error detail was provided."), + ); + return card; +} + function emptyCompareCard() { const card = createElement("article", "compare-card empty"); card.append( @@ -525,6 +605,14 @@ function extractDecision(payload) { return payload.deployment_decision || payload.result?.deployment_decision || payload.data?.deployment_decision || null; } +function decisionReason(decision) { + const decisionName = String(decision?.decision || "unknown").toLowerCase(); + if (decisionName === "unknown" && !decision?.guard_status) { + return "AIGuard evidence not provided."; + } + return decision?.reason || "-"; +} + function extractRuntimeResult(job) { const result = job?.result; if (!result) { @@ -641,6 +729,35 @@ function displayValue(value) { return String(value); } +function missingResultKeys(result = {}) { + const missing = []; + if (!result.compare_key) { + missing.push("compare_key"); + } + if (!normalizedBackendKey(result)) { + missing.push("backend_key"); + } + return missing; +} + +function fallbackLabel(key) { + return `${key} unavailable`; +} + +function compareTone(overall) { + const value = String(overall || "").toLowerCase(); + if (value.includes("improvement")) { + return "improvement"; + } + if (value.includes("regression")) { + return "regression"; + } + if (value.includes("neutral")) { + return "neutral"; + } + return "unknown"; +} + function formatLatencyDiff(metric) { if (!metric || Object.keys(metric).length === 0) { return "-"; diff --git a/inferedgelab/studio/static/index.html b/inferedgelab/studio/static/index.html index 14f525d..882bd4f 100644 --- a/inferedgelab/studio/static/index.html +++ b/inferedgelab/studio/static/index.html @@ -132,7 +132,7 @@ } } - +
@@ -192,8 +192,15 @@

Runtime result JSON

+ +

+
@@ -275,6 +282,6 @@

Future Work

- + diff --git a/inferedgelab/studio/static/style.css b/inferedgelab/studio/static/style.css index 30f3712..a602fb1 100644 --- a/inferedgelab/studio/static/style.css +++ b/inferedgelab/studio/static/style.css @@ -417,6 +417,37 @@ body { margin: 0; } +.evidence-summary { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 8px; +} + +.evidence-item { + border: 1px solid var(--line); + border-radius: 10px; + background: rgba(15, 23, 42, 0.78); + padding: 10px; +} + +.evidence-item strong { + display: block; + margin-top: 6px; + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", + monospace; + font-size: 0.82rem; + overflow-wrap: anywhere; +} + +.evidence-warning { + grid-column: 1 / -1; + margin: 0; + color: var(--yellow); + font-size: 0.82rem; + line-height: 1.45; +} + .metric-name, .metric-value { display: block; @@ -446,6 +477,19 @@ body { background: linear-gradient(180deg, rgba(34, 211, 238, 0.09), var(--panel)); } +.compare-card.improvement { + border-color: rgba(34, 197, 94, 0.32); +} + +.compare-card.regression, +.error-card { + border-color: rgba(239, 68, 68, 0.32); +} + +.compare-card.neutral { + border-color: rgba(234, 179, 8, 0.32); +} + .compare-card.empty { grid-column: 1 / -1; } @@ -517,4 +561,8 @@ body { .metric-grid { grid-template-columns: 1fr; } + + .evidence-summary { + grid-template-columns: 1fr; + } } diff --git a/tests/test_studio_routes.py b/tests/test_studio_routes.py index 9b84154..cf65750 100644 --- a/tests/test_studio_routes.py +++ b/tests/test_studio_routes.py @@ -15,6 +15,33 @@ def _get_route(app, path: str): raise AssertionError(f"route not found: {path}") +def _runtime_result( + *, + engine: str = "onnxruntime", + device: str = "cpu", + mean_ms: float = 45.0, + p99_ms: float = 50.0, + backend_key: str | None = "onnxruntime__cpu", +) -> dict: + result = { + "runtime_role": "runtime-result", + "model": "yolov8n", + "engine": engine, + "device": device, + "precision": "fp32", + "batch": 1, + "height": 640, + "width": 640, + "mean_ms": mean_ms, + "p99_ms": p99_ms, + "timestamp": "2026-04-29T12:00:00Z", + "compare_key": "yolov8n__b1__h640w640__fp32", + } + if backend_key is not None: + result["backend_key"] = backend_key + return result + + def test_studio_route_returns_local_studio_html(): app = api.create_app() route = _get_route(app, "/studio") @@ -32,9 +59,10 @@ def test_studio_route_returns_local_studio_html(): assert "Import" in html assert "Jetson Helper" in html assert 'data-critical="studio-dark"' in html - assert 'href="/studio/static/style.css?v=' in html - assert 'src="/studio/static/app.js?v=' in html + assert 'href="/studio/static/style.css?v=9"' in html + assert 'src="/studio/static/app.js?v=9"' in html assert 'value="results/latest.json"' in html + assert 'id="import-json-payload"' in html def test_studio_static_assets_are_served(): @@ -69,6 +97,10 @@ def test_studio_static_assets_include_redesigned_ui_contracts(): assert "DOMContentLoaded" in app_text assert "Open Studio from http://127.0.0.1:8000/studio" in app_text assert "responseErrorMessage" in app_text + assert "parseJsonResponse" in app_text + assert "renderImportEvidence" in app_text + assert "AIGuard evidence not provided" in app_text + assert "compareTone" in app_text assert "runtimeModelName" in app_text assert "Same backend" in app_text assert "hasImportedEvidence" in app_text @@ -78,6 +110,8 @@ def test_studio_static_assets_include_redesigned_ui_contracts(): assert ".form-stack button" in style_text assert ".tool-card" in style_text assert ".state-pill.optional" in style_text + assert ".evidence-summary" in style_text + assert ".compare-card.improvement" in style_text def test_studio_app_preserves_selected_job_detail_contract(): @@ -117,9 +151,11 @@ def test_studio_compare_latest_api_returns_json_structure(): response = route.endpoint(request=request) assert route.status_code is None + assert response["status"] == "empty" assert "deployment_decision" in response assert "data" in response assert response.get("source") in {None, "/api/compare-latest"} + assert response["deployment_decision"]["decision"] == "unknown" def test_studio_run_api_creates_analyze_job(): @@ -158,21 +194,7 @@ def test_studio_import_api_accepts_runtime_result_json(): app = api.create_app() route = _get_route(app, "/studio/api/import") request = SimpleNamespace(app=app) - result = { - "runtime_role": "runtime-result", - "model": "yolov8n", - "engine": "onnxruntime", - "device": "cpu", - "precision": "fp32", - "batch": 1, - "height": 640, - "width": 640, - "mean_ms": 45.0, - "p99_ms": 50.0, - "timestamp": "2026-04-29T12:00:00Z", - "compare_key": "yolov8n__b1__h640w640__fp32", - "backend_key": "onnxruntime__cpu", - } + result = _runtime_result() response = route.endpoint(request=request, payload={"result": result}) @@ -182,6 +204,20 @@ def test_studio_import_api_accepts_runtime_result_json(): assert response["compare_ready"] is False +def test_studio_import_api_generates_missing_compare_keys(): + app = api.create_app() + route = _get_route(app, "/studio/api/import") + request = SimpleNamespace(app=app) + result = _runtime_result(backend_key=None) + result.pop("compare_key") + + response = route.endpoint(request=request, payload={"result": result}) + + assert response["status"] == "imported" + assert response["result"]["backend_key"] == "onnxruntime__cpu" + assert response["result"]["compare_key"] == "yolov8n__b1__h640w640__fp32" + + def test_studio_import_api_accepts_existing_result_path(): app = api.create_app() route = _get_route(app, "/studio/api/import") @@ -206,3 +242,37 @@ def test_studio_jetson_command_api_returns_command(): assert "--engine tensorrt" in response["command"] assert "--device jetson" in response["command"] assert "--output results/jetson/" in response["command"] + + +def test_studio_importing_two_compatible_results_returns_compare_data(): + app = api.create_app() + request = SimpleNamespace(app=app) + import_route = _get_route(app, "/studio/api/import") + compare_route = _get_route(app, "/studio/api/compare/latest") + + first = import_route.endpoint( + request=request, + payload={"result": _runtime_result(engine="onnxruntime", device="cpu", mean_ms=45.0, p99_ms=50.0)}, + ) + second = import_route.endpoint( + request=request, + payload={ + "result": _runtime_result( + engine="tensorrt", + device="jetson", + mean_ms=9.9, + p99_ms=12.0, + backend_key="tensorrt__jetson", + ) + }, + ) + compare = compare_route.endpoint(request=request) + + assert first["compare_ready"] is False + assert second["compare_ready"] is True + assert compare["status"] == "ok" + assert compare["base"]["backend_key"] == "onnxruntime__cpu" + assert compare["new"]["backend_key"] == "tensorrt__jetson" + assert compare["result"]["metrics"]["mean_ms"]["new"] == 9.9 + assert compare["judgement"]["overall"] == "improvement" + assert compare["deployment_decision"]["decision"] == "unknown"