diff --git a/src/main/java/dev/braintrust/devserver/Devserver.java b/src/main/java/dev/braintrust/devserver/Devserver.java index c861fad7..d787d279 100644 --- a/src/main/java/dev/braintrust/devserver/Devserver.java +++ b/src/main/java/dev/braintrust/devserver/Devserver.java @@ -607,6 +607,7 @@ private void setScoreSpanAttributes( Map scoreSpanAttrs = new LinkedHashMap<>(); scoreSpanAttrs.put("type", "score"); scoreSpanAttrs.put("name", scorerName); + scoreSpanAttrs.put("purpose", "scorer"); if (braintrustGeneration != null) { scoreSpanAttrs.put("generation", braintrustGeneration); } diff --git a/src/main/java/dev/braintrust/eval/Eval.java b/src/main/java/dev/braintrust/eval/Eval.java index 51b89249..fa7660b2 100644 --- a/src/main/java/dev/braintrust/eval/Eval.java +++ b/src/main/java/dev/braintrust/eval/Eval.java @@ -230,6 +230,7 @@ private void recordScores( Map spanAttrs = new LinkedHashMap<>(); spanAttrs.put("type", "score"); spanAttrs.put("name", scorer.getName()); + spanAttrs.put("purpose", "scorer"); scoreSpan.setAttribute("braintrust.span_attributes", toJson(spanAttrs)); var scoresJson = toJson(scorerScores); scoreSpan.setAttribute("braintrust.output_json", scoresJson); diff --git a/src/test/java/dev/braintrust/devserver/DevserverTest.java b/src/test/java/dev/braintrust/devserver/DevserverTest.java index 6eade953..49034c33 100644 --- a/src/test/java/dev/braintrust/devserver/DevserverTest.java +++ b/src/test/java/dev/braintrust/devserver/DevserverTest.java @@ -433,6 +433,7 @@ void testStreamingEval() throws Exception { assertNotNull(spanAttrsJson, "Score span should have span_attributes"); JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson); assertEquals("score", spanAttrs.get("type").asText()); + assertEquals("scorer", spanAttrs.get("purpose").asText()); assertEquals("test-gen-1", spanAttrs.get("generation").asText()); // Scorer name should be either simple_scorer or the remote scorer diff --git a/src/test/java/dev/braintrust/eval/EvalTest.java b/src/test/java/dev/braintrust/eval/EvalTest.java index 5f234887..a74699d2 100644 --- a/src/test/java/dev/braintrust/eval/EvalTest.java +++ b/src/test/java/dev/braintrust/eval/EvalTest.java @@ -104,6 +104,30 @@ public void evalOtelTraceWithProperAttributes() { numRootSpans.get() * 4, spans.size(), "each eval case should make four spans (one per scorer)"); + + // All score spans should have purpose=scorer in span_attributes + var scoreSpans = + spans.stream() + .filter( + s -> { + var attrs = + s.getAttributes() + .get( + AttributeKey.stringKey( + "braintrust.span_attributes")); + return attrs != null && attrs.contains("\"type\":\"score\""); + }) + .toList(); + assertFalse(scoreSpans.isEmpty(), "should have score spans"); + for (var scoreSpan : scoreSpans) { + var spanAttrsJson = + scoreSpan + .getAttributes() + .get(AttributeKey.stringKey("braintrust.span_attributes")); + assertTrue( + spanAttrsJson.contains("\"purpose\":\"scorer\""), + "score span should have purpose=scorer in span_attributes: " + spanAttrsJson); + } } boolean isFruitOrVegetable(String str) {