Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,20 @@
<artifactId>jsoup</artifactId>
<version>1.17.2</version>
</dependency>

<!-- JTokkit — CL100K_BASE token estimation for MCP tool metrics -->
<dependency>
<groupId>com.knuddels</groupId>
<artifactId>jtokkit</artifactId>
<version>1.1.0</version>
</dependency>

<!-- Lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
Expand Down
82 changes: 82 additions & 0 deletions src/main/java/com/jobtracker/controller/ToolMetricsController.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package com.jobtracker.controller;

import com.jobtracker.dto.metrics.AvgExecutionTimeProjection;
import com.jobtracker.dto.metrics.MostExpensiveExecutionResponse;
import com.jobtracker.dto.metrics.ToolUsageByDayProjection;
import com.jobtracker.dto.metrics.TopExpensiveToolProjection;
import com.jobtracker.entity.ToolExecutionMetric;
import com.jobtracker.repository.ToolExecutionMetricRepository;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.data.domain.Pageable;
import org.springframework.http.ResponseEntity;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import java.util.List;

@Tag(name = "Tool Metrics", description = "MCP tool usage analytics — token costs, execution times, and call volumes")
@RestController
@RequestMapping("/api/v1/internal/metrics/tools")
public class ToolMetricsController {

private final ToolExecutionMetricRepository repository;

public ToolMetricsController(ToolExecutionMetricRepository repository) {
this.repository = repository;
}

@Operation(
summary = "Top expensive tools",
description = "Tools ranked by average total tokens (request + response). Identifies habitual heavy callers.")
@GetMapping("/top-expensive")
@Transactional(readOnly = true)
public ResponseEntity<List<TopExpensiveToolProjection>> getTopExpensiveTools() {
return ResponseEntity.ok(repository.findTopExpensiveTools());
}

@Operation(
summary = "Tool call volume by day",
description = "Total number of tool invocations per calendar day, most-recent first.")
@GetMapping("/usage-by-day")
@Transactional(readOnly = true)
public ResponseEntity<List<ToolUsageByDayProjection>> getUsageByDay() {
return ResponseEntity.ok(repository.findUsageByDay());
}

@Operation(
summary = "Most expensive individual executions",
description = "The N executions with the highest total token count. Default limit is 20.")
@GetMapping("/most-expensive")
@Transactional(readOnly = true)
public ResponseEntity<List<MostExpensiveExecutionResponse>> getMostExpensiveExecutions(
@RequestParam(defaultValue = "20") int limit) {
List<ToolExecutionMetric> rows = repository.findMostExpensiveExecutions(Pageable.ofSize(limit));
List<MostExpensiveExecutionResponse> body = rows.stream()
.map(m -> new MostExpensiveExecutionResponse(
m.getId(),
m.getToolName(),
m.getExecutionTimeMs(),
m.getRequestBytes(),
m.getResponseBytes(),
m.getRequestTokens(),
m.getResponseTokens(),
m.getTotalTokens(),
m.isExpensive(),
m.getCreatedAt()))
.toList();
return ResponseEntity.ok(body);
}

@Operation(
summary = "Average execution time per tool",
description = "Wall-clock average in milliseconds per tool, slowest-first. Useful for latency profiling.")
@GetMapping("/avg-execution-time")
@Transactional(readOnly = true)
public ResponseEntity<List<AvgExecutionTimeProjection>> getAvgExecutionTime() {
return ResponseEntity.ok(repository.findAvgExecutionTimePerTool());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package com.jobtracker.dto.metrics;

public interface AvgExecutionTimeProjection {
String getToolName();
Double getAvgExecutionTimeMs();
Long getCalls();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package com.jobtracker.dto.metrics;

import java.time.LocalDateTime;
import java.util.UUID;

public record MostExpensiveExecutionResponse(
UUID id,
String toolName,
long executionTimeMs,
int requestBytes,
int responseBytes,
int requestTokens,
int responseTokens,
int totalTokens,
boolean expensive,
LocalDateTime createdAt
) {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package com.jobtracker.dto.metrics;

public interface ToolUsageByDayProjection {
String getDay();
Long getCalls();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package com.jobtracker.dto.metrics;

public interface TopExpensiveToolProjection {
String getToolName();
Long getCalls();
Double getAvgTokens();
Long getMaxTokens();
}
61 changes: 61 additions & 0 deletions src/main/java/com/jobtracker/entity/ToolExecutionMetric.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package com.jobtracker.entity;

import jakarta.persistence.*;
import lombok.*;
import org.hibernate.annotations.UuidGenerator;

import java.time.LocalDateTime;
import java.util.UUID;

@Entity
@Table(name = "tool_execution_metrics", indexes = {
@Index(name = "idx_tool_metrics_tool_name", columnList = "tool_name"),
@Index(name = "idx_tool_metrics_created_at", columnList = "created_at"),
@Index(name = "idx_tool_metrics_expensive", columnList = "expensive")
})
@Getter
@Setter
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ToolExecutionMetric {

@Id
@UuidGenerator(style = UuidGenerator.Style.TIME)
@Column(name = "id", columnDefinition = "BINARY(16)", updatable = false, nullable = false)
private UUID id;

@Column(name = "tool_name", nullable = false, length = 255)
private String toolName;

@Column(name = "execution_time_ms", nullable = false)
private long executionTimeMs;

@Column(name = "request_bytes", nullable = false)
private int requestBytes;

@Column(name = "response_bytes", nullable = false)
private int responseBytes;

@Column(name = "request_tokens", nullable = false)
private int requestTokens;

@Column(name = "response_tokens", nullable = false)
private int responseTokens;

@Column(name = "total_tokens", nullable = false)
private int totalTokens;

@Column(name = "expensive", nullable = false)
private boolean expensive;

@Column(name = "created_at", nullable = false, updatable = false)
private LocalDateTime createdAt;

@PrePersist
protected void onCreate() {
if (createdAt == null) {
createdAt = LocalDateTime.now();
}
}
}
Loading
Loading