diff --git a/devtools/grafana/dashboards/forge-issue-detail.json b/devtools/grafana/dashboards/forge-issue-detail.json index dd02972f..7190e75d 100644 --- a/devtools/grafana/dashboards/forge-issue-detail.json +++ b/devtools/grafana/dashboards/forge-issue-detail.json @@ -74,7 +74,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT t.metadata['workflow_step'] as step, round(sum(o.total_cost), 4) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY cost DESC" + "rawSql": "SELECT CASE t.metadata['workflow_step']\n WHEN 'entry' THEN 'Entry'\n WHEN 'prd_approval_gate' THEN 'PRD'\n WHEN 'spec_approval_gate' THEN 'Spec'\n WHEN 'plan_approval_gate' THEN 'Epic Plan'\n WHEN 'task_approval_gate' THEN 'Tasks'\n WHEN 'create_pr' THEN 'PR Creation'\n WHEN 'human_review_gate' THEN 'Human Review'\n WHEN 'ci_evaluator' THEN 'CI'\n WHEN 'attempt_ci_fix' THEN 'CI Fix'\n ELSE t.metadata['workflow_step']\nEND as step, round(sum(o.total_cost), 4) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY cost DESC" } }, "refId": "A", @@ -128,7 +128,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT t.metadata['workflow_step'] as step, sum(o.usage_details['input']) as input_tokens, sum(o.usage_details['output']) as output_tokens FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY input_tokens DESC" + "rawSql": "SELECT CASE t.metadata['workflow_step']\n WHEN 'entry' THEN 'Entry'\n WHEN 'prd_approval_gate' THEN 'PRD'\n WHEN 'spec_approval_gate' THEN 'Spec'\n WHEN 'plan_approval_gate' THEN 'Epic Plan'\n WHEN 'task_approval_gate' THEN 'Tasks'\n WHEN 'create_pr' THEN 'PR Creation'\n WHEN 'human_review_gate' THEN 'Human Review'\n WHEN 'ci_evaluator' THEN 'CI'\n WHEN 'attempt_ci_fix' THEN 'CI Fix'\n ELSE t.metadata['workflow_step']\nEND as step, sum(o.usage_details['input']) as input_tokens, sum(o.usage_details['output']) as output_tokens FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY input_tokens DESC" } }, "refId": "A", @@ -250,7 +250,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT t.id as id, o_agg.model, round(o_agg.latency_s, 1) as latency_s, round(o_agg.cost, 4) as cost, o_agg.total_tokens as tokens FROM default.traces t FINAL JOIN (SELECT trace_id, any(provided_model_name) as model, dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) as latency_s, sum(total_cost) as cost, sum(usage_details['total']) as total_tokens FROM default.observations FINAL GROUP BY trace_id) o_agg ON t.id = o_agg.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' ORDER BY t.timestamp" + "rawSql": "SELECT CASE t.metadata['workflow_step'] WHEN 'entry' THEN 'Entry' WHEN 'prd_approval_gate' THEN 'PRD' WHEN 'spec_approval_gate' THEN 'Spec' WHEN 'plan_approval_gate' THEN 'Epic Plan' WHEN 'task_approval_gate' THEN 'Tasks' WHEN 'create_pr' THEN 'PR Creation' WHEN 'human_review_gate' THEN 'Human Review' WHEN 'ci_evaluator' THEN 'CI' WHEN 'attempt_ci_fix' THEN 'CI Fix' ELSE t.metadata['workflow_step'] END AS step, row_number() OVER (PARTITION BY t.metadata['workflow_step'] ORDER BY t.timestamp ASC) AS iteration, t.id as id, o_agg.model, round(o_agg.latency_s, 1) as latency_s, round(o_agg.cost, 4) as cost, o_agg.total_tokens as tokens FROM default.traces t FINAL JOIN (SELECT trace_id, any(provided_model_name) as model, dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) as latency_s, sum(total_cost) as cost, sum(usage_details['total']) as total_tokens FROM default.observations o FINAL GROUP BY trace_id) o_agg ON t.id = o_agg.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' ORDER BY t.timestamp" } }, "refId": "A", @@ -275,12 +275,62 @@ } }, "overrides": [ + { + "matcher": { + "id": "byName", + "options": "step" + }, + "properties": [ + { + "id": "displayName", + "value": "Step" + }, + { + "id": "custom", + "value": { + "align": "auto", + "displayMode": "auto" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "iteration" + }, + "properties": [ + { + "id": "displayName", + "value": "Iteration" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom", + "value": { + "align": "auto", + "displayMode": "auto" + } + } + ] + }, { "matcher": { "id": "byName", "options": "cost" }, "properties": [ + { + "id": "displayName", + "value": "Cost" + }, { "id": "unit", "value": "currencyUSD" @@ -293,6 +343,10 @@ "options": "latency_s" }, "properties": [ + { + "id": "displayName", + "value": "Latency (s)" + }, { "id": "unit", "value": "s" @@ -305,6 +359,10 @@ "options": "id" }, "properties": [ + { + "id": "displayName", + "value": "Id" + }, { "id": "links", "value": [ @@ -316,6 +374,34 @@ ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "tokens" + }, + "properties": [ + { + "id": "displayName", + "value": "Tokens" + }, + { + "id": "unit", + "value": "locale" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "model" + }, + "properties": [ + { + "id": "displayName", + "value": "Model" + } + ] } ] } @@ -594,6 +680,201 @@ } } }, + "panel-39": { + "kind": "Panel", + "spec": { + "id": 39, + "title": "Total Duration", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT sum(duration_s) as value FROM (SELECT dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as duration_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' GROUP BY t.id)" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 600, + "color": "red" + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-40": { + "kind": "Panel", + "spec": { + "id": 40, + "title": "Total Tokens", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT sum(usage_details['total']) as value FROM default.observations FINAL WHERE trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id = '${jira_issue}')" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "locale", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1500000, + "color": "red" + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-32": { + "kind": "Panel", + "spec": { + "id": 32, + "title": "CI Fix Attempts", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT count() as value FROM default.traces FINAL WHERE session_id = '${jira_issue}' AND metadata['workflow_step'] = 'attempt_ci_fix'" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, "panel-8": { "kind": "Panel", "spec": { @@ -617,7 +898,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT t.metadata['workflow_step'] as step, dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as duration_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY t.session_id, t.id, step ORDER BY min(o.start_time)" + "rawSql": "SELECT CASE t.metadata['workflow_step']\n WHEN 'entry' THEN 'Entry'\n WHEN 'prd_approval_gate' THEN 'PRD'\n WHEN 'spec_approval_gate' THEN 'Spec'\n WHEN 'plan_approval_gate' THEN 'Epic Plan'\n WHEN 'task_approval_gate' THEN 'Tasks'\n WHEN 'create_pr' THEN 'PR Creation'\n WHEN 'human_review_gate' THEN 'Human Review'\n WHEN 'ci_evaluator' THEN 'CI'\n WHEN 'attempt_ci_fix' THEN 'CI Fix'\n ELSE t.metadata['workflow_step']\nEND as step, dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as duration_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY t.session_id, t.id, step ORDER BY min(o.start_time)" } }, "refId": "A", @@ -646,6 +927,145 @@ } } } + }, + "panel-28": { + "kind": "Panel", + "spec": { + "id": 28, + "title": "Iteration Count per Stage", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT CASE metadata['workflow_step']\n WHEN 'entry' THEN 'Entry'\n WHEN 'prd_approval_gate' THEN 'PRD'\n WHEN 'spec_approval_gate' THEN 'Spec'\n WHEN 'plan_approval_gate' THEN 'Epic Plan'\n WHEN 'task_approval_gate' THEN 'Tasks'\n WHEN 'create_pr' THEN 'PR Creation'\n WHEN 'human_review_gate' THEN 'Human Review'\n WHEN 'ci_evaluator' THEN 'CI'\n WHEN 'attempt_ci_fix' THEN 'CI Fix'\n ELSE metadata['workflow_step']\nEND AS workflow_step, count() AS iteration_count FROM default.traces FINAL WHERE session_id = '${jira_issue}' AND metadata['workflow_step'] != '' GROUP BY workflow_step ORDER BY iteration_count DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "workflow_step" + }, + "fieldConfig": { + "defaults": { + "noDataText": "No Data" + }, + "overrides": [] + } + } + } + } + }, + "panel-35": { + "kind": "Panel", + "spec": { + "id": 35, + "title": "Machine Time vs Idle Time per Stage", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT step, machine_s as machine_time, greatest(wall_s - machine_s, 0) as idle_time FROM (SELECT CASE t.metadata['workflow_step']\n WHEN 'entry' THEN 'Entry'\n WHEN 'prd_approval_gate' THEN 'PRD'\n WHEN 'spec_approval_gate' THEN 'Spec'\n WHEN 'plan_approval_gate' THEN 'Epic Plan'\n WHEN 'task_approval_gate' THEN 'Tasks'\n WHEN 'create_pr' THEN 'PR Creation'\n WHEN 'human_review_gate' THEN 'Human Review'\n WHEN 'ci_evaluator' THEN 'CI'\n WHEN 'attempt_ci_fix' THEN 'CI Fix'\n ELSE t.metadata['workflow_step']\nEND AS step, sum(dateDiff('second', o.start_time, coalesce(o.end_time, o.start_time))) as machine_s, dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as wall_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step) ORDER BY machine_time + idle_time DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "stacking": "normal" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "machine_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + } + } + } + } } }, "layout": { @@ -666,7 +1086,7 @@ "spec": { "x": 0, "y": 0, - "width": 5, + "width": 4, "height": 4, "element": { "kind": "ElementReference", @@ -677,9 +1097,9 @@ { "kind": "GridLayoutItem", "spec": { - "x": 5, + "x": 4, "y": 0, - "width": 4, + "width": 3, "height": 4, "element": { "kind": "ElementReference", @@ -690,9 +1110,9 @@ { "kind": "GridLayoutItem", "spec": { - "x": 9, + "x": 7, "y": 0, - "width": 4, + "width": 3, "height": 4, "element": { "kind": "ElementReference", @@ -703,28 +1123,54 @@ { "kind": "GridLayoutItem", "spec": { - "x": 13, + "x": 10, "y": 0, - "width": 5, + "width": 4, "height": 4, "element": { "kind": "ElementReference", - "name": "panel-4" + "name": "panel-39" } } }, { "kind": "GridLayoutItem", "spec": { - "x": 18, + "x": 14, "y": 0, - "width": 6, + "width": 3, "height": 4, "element": { "kind": "ElementReference", "name": "panel-5" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 17, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 21, + "y": 0, + "width": 3, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-40" + } + } } ] } @@ -758,6 +1204,59 @@ } } }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Iterations & Timing", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-28" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 8, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-32" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 0, + "width": 8, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-35" + } + } + } + ] + } + } + } + }, { "kind": "RowsLayoutRow", "spec": {