Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/main/java/dev/braintrust/devserver/Devserver.java
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,7 @@ private void setScoreSpanAttributes(
Map<String, Object> scoreSpanAttrs = new LinkedHashMap<>();
scoreSpanAttrs.put("type", "score");
scoreSpanAttrs.put("name", scorerName);
scoreSpanAttrs.put("purpose", "scorer");
if (braintrustGeneration != null) {
scoreSpanAttrs.put("generation", braintrustGeneration);
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/dev/braintrust/eval/Eval.java
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ private void recordScores(
Map<String, Object> spanAttrs = new LinkedHashMap<>();
spanAttrs.put("type", "score");
spanAttrs.put("name", scorer.getName());
spanAttrs.put("purpose", "scorer");
scoreSpan.setAttribute("braintrust.span_attributes", toJson(spanAttrs));
var scoresJson = toJson(scorerScores);
scoreSpan.setAttribute("braintrust.output_json", scoresJson);
Expand Down
1 change: 1 addition & 0 deletions src/test/java/dev/braintrust/devserver/DevserverTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ void testStreamingEval() throws Exception {
assertNotNull(spanAttrsJson, "Score span should have span_attributes");
JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson);
assertEquals("score", spanAttrs.get("type").asText());
assertEquals("scorer", spanAttrs.get("purpose").asText());
assertEquals("test-gen-1", spanAttrs.get("generation").asText());

// Scorer name should be either simple_scorer or the remote scorer
Expand Down
24 changes: 24 additions & 0 deletions src/test/java/dev/braintrust/eval/EvalTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,30 @@ public void evalOtelTraceWithProperAttributes() {
numRootSpans.get() * 4,
spans.size(),
"each eval case should make four spans (one per scorer)");

// All score spans should have purpose=scorer in span_attributes
var scoreSpans =
spans.stream()
.filter(
s -> {
var attrs =
s.getAttributes()
.get(
AttributeKey.stringKey(
"braintrust.span_attributes"));
return attrs != null && attrs.contains("\"type\":\"score\"");
})
.toList();
assertFalse(scoreSpans.isEmpty(), "should have score spans");
for (var scoreSpan : scoreSpans) {
var spanAttrsJson =
scoreSpan
.getAttributes()
.get(AttributeKey.stringKey("braintrust.span_attributes"));
assertTrue(
spanAttrsJson.contains("\"purpose\":\"scorer\""),
"score span should have purpose=scorer in span_attributes: " + spanAttrsJson);
}
}

boolean isFruitOrVegetable(String str) {
Expand Down
Loading