From 732d8366d78fc75ac01a91da228eeb15da115de4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 04:17:30 +0000 Subject: [PATCH] feat: add MCP tool usage metrics system (token cost tracking) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds infrastructure to automatically measure every MCP tool execution and store token estimates, byte sizes, and wall-clock timings for observability and cost-ranking dashboards. Components added: - ToolExecutionMetric entity + V26 Flyway migration (tool_execution_metrics table) - TokenEstimatorService — CL100K_BASE estimation via JTokkit (cl100k_base) - ToolMetricsCollector — generic measure(toolName, request, Supplier) wrapper; metrics failures never break business execution (logged only) - ToolExecutionMetricRepository — JPQL + native queries for top-expensive, usage-by-day, most-expensive-executions, and avg-execution-time-per-tool - ToolMetricsController — GET /api/v1/internal/metrics/tools/{top-expensive,usage-by-day, most-expensive,avg-execution-time} - Interface projections + MostExpensiveExecutionResponse record in dto/metrics/ - McpApplicationTools fully integrated with ToolMetricsCollector on all 9 methods Alerts: WARN log emitted when responseTokens > 5 000 (expensive threshold). https://claude.ai/code/session_01EFuRv2jTHxB7PLXwQTAPbz --- pom.xml | 14 +++ .../controller/ToolMetricsController.java | 82 +++++++++++++ .../metrics/AvgExecutionTimeProjection.java | 7 ++ .../MostExpensiveExecutionResponse.java | 17 +++ .../dto/metrics/ToolUsageByDayProjection.java | 6 + .../metrics/TopExpensiveToolProjection.java | 8 ++ .../entity/ToolExecutionMetric.java | 61 ++++++++++ .../mcp/tools/McpApplicationTools.java | 110 +++++++++++------ .../ToolExecutionMetricRepository.java | 62 ++++++++++ .../service/TokenEstimatorService.java | 65 ++++++++++ .../service/ToolMetricsCollector.java | 111 ++++++++++++++++++ .../V26__create_tool_execution_metrics.sql | 17 +++ 12 files changed, 524 insertions(+), 36 deletions(-) create mode 100644 src/main/java/com/jobtracker/controller/ToolMetricsController.java create mode 100644 src/main/java/com/jobtracker/dto/metrics/AvgExecutionTimeProjection.java create mode 100644 src/main/java/com/jobtracker/dto/metrics/MostExpensiveExecutionResponse.java create mode 100644 src/main/java/com/jobtracker/dto/metrics/ToolUsageByDayProjection.java create mode 100644 src/main/java/com/jobtracker/dto/metrics/TopExpensiveToolProjection.java create mode 100644 src/main/java/com/jobtracker/entity/ToolExecutionMetric.java create mode 100644 src/main/java/com/jobtracker/repository/ToolExecutionMetricRepository.java create mode 100644 src/main/java/com/jobtracker/service/TokenEstimatorService.java create mode 100644 src/main/java/com/jobtracker/service/ToolMetricsCollector.java create mode 100644 src/main/resources/db/migration/V26__create_tool_execution_metrics.sql diff --git a/pom.xml b/pom.xml index 33e09e75..cf7093f3 100644 --- a/pom.xml +++ b/pom.xml @@ -168,6 +168,20 @@ jsoup 1.17.2 + + + + com.knuddels + jtokkit + 1.1.0 + + + + + org.projectlombok + lombok + true + commons-codec commons-codec diff --git a/src/main/java/com/jobtracker/controller/ToolMetricsController.java b/src/main/java/com/jobtracker/controller/ToolMetricsController.java new file mode 100644 index 00000000..5e77d892 --- /dev/null +++ b/src/main/java/com/jobtracker/controller/ToolMetricsController.java @@ -0,0 +1,82 @@ +package com.jobtracker.controller; + +import com.jobtracker.dto.metrics.AvgExecutionTimeProjection; +import com.jobtracker.dto.metrics.MostExpensiveExecutionResponse; +import com.jobtracker.dto.metrics.ToolUsageByDayProjection; +import com.jobtracker.dto.metrics.TopExpensiveToolProjection; +import com.jobtracker.entity.ToolExecutionMetric; +import com.jobtracker.repository.ToolExecutionMetricRepository; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.data.domain.Pageable; +import org.springframework.http.ResponseEntity; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@Tag(name = "Tool Metrics", description = "MCP tool usage analytics — token costs, execution times, and call volumes") +@RestController +@RequestMapping("/api/v1/internal/metrics/tools") +public class ToolMetricsController { + + private final ToolExecutionMetricRepository repository; + + public ToolMetricsController(ToolExecutionMetricRepository repository) { + this.repository = repository; + } + + @Operation( + summary = "Top expensive tools", + description = "Tools ranked by average total tokens (request + response). Identifies habitual heavy callers.") + @GetMapping("/top-expensive") + @Transactional(readOnly = true) + public ResponseEntity> getTopExpensiveTools() { + return ResponseEntity.ok(repository.findTopExpensiveTools()); + } + + @Operation( + summary = "Tool call volume by day", + description = "Total number of tool invocations per calendar day, most-recent first.") + @GetMapping("/usage-by-day") + @Transactional(readOnly = true) + public ResponseEntity> getUsageByDay() { + return ResponseEntity.ok(repository.findUsageByDay()); + } + + @Operation( + summary = "Most expensive individual executions", + description = "The N executions with the highest total token count. Default limit is 20.") + @GetMapping("/most-expensive") + @Transactional(readOnly = true) + public ResponseEntity> getMostExpensiveExecutions( + @RequestParam(defaultValue = "20") int limit) { + List rows = repository.findMostExpensiveExecutions(Pageable.ofSize(limit)); + List body = rows.stream() + .map(m -> new MostExpensiveExecutionResponse( + m.getId(), + m.getToolName(), + m.getExecutionTimeMs(), + m.getRequestBytes(), + m.getResponseBytes(), + m.getRequestTokens(), + m.getResponseTokens(), + m.getTotalTokens(), + m.isExpensive(), + m.getCreatedAt())) + .toList(); + return ResponseEntity.ok(body); + } + + @Operation( + summary = "Average execution time per tool", + description = "Wall-clock average in milliseconds per tool, slowest-first. Useful for latency profiling.") + @GetMapping("/avg-execution-time") + @Transactional(readOnly = true) + public ResponseEntity> getAvgExecutionTime() { + return ResponseEntity.ok(repository.findAvgExecutionTimePerTool()); + } +} diff --git a/src/main/java/com/jobtracker/dto/metrics/AvgExecutionTimeProjection.java b/src/main/java/com/jobtracker/dto/metrics/AvgExecutionTimeProjection.java new file mode 100644 index 00000000..7422ed20 --- /dev/null +++ b/src/main/java/com/jobtracker/dto/metrics/AvgExecutionTimeProjection.java @@ -0,0 +1,7 @@ +package com.jobtracker.dto.metrics; + +public interface AvgExecutionTimeProjection { + String getToolName(); + Double getAvgExecutionTimeMs(); + Long getCalls(); +} diff --git a/src/main/java/com/jobtracker/dto/metrics/MostExpensiveExecutionResponse.java b/src/main/java/com/jobtracker/dto/metrics/MostExpensiveExecutionResponse.java new file mode 100644 index 00000000..295428b5 --- /dev/null +++ b/src/main/java/com/jobtracker/dto/metrics/MostExpensiveExecutionResponse.java @@ -0,0 +1,17 @@ +package com.jobtracker.dto.metrics; + +import java.time.LocalDateTime; +import java.util.UUID; + +public record MostExpensiveExecutionResponse( + UUID id, + String toolName, + long executionTimeMs, + int requestBytes, + int responseBytes, + int requestTokens, + int responseTokens, + int totalTokens, + boolean expensive, + LocalDateTime createdAt +) {} diff --git a/src/main/java/com/jobtracker/dto/metrics/ToolUsageByDayProjection.java b/src/main/java/com/jobtracker/dto/metrics/ToolUsageByDayProjection.java new file mode 100644 index 00000000..69cb8825 --- /dev/null +++ b/src/main/java/com/jobtracker/dto/metrics/ToolUsageByDayProjection.java @@ -0,0 +1,6 @@ +package com.jobtracker.dto.metrics; + +public interface ToolUsageByDayProjection { + String getDay(); + Long getCalls(); +} diff --git a/src/main/java/com/jobtracker/dto/metrics/TopExpensiveToolProjection.java b/src/main/java/com/jobtracker/dto/metrics/TopExpensiveToolProjection.java new file mode 100644 index 00000000..86ccdf93 --- /dev/null +++ b/src/main/java/com/jobtracker/dto/metrics/TopExpensiveToolProjection.java @@ -0,0 +1,8 @@ +package com.jobtracker.dto.metrics; + +public interface TopExpensiveToolProjection { + String getToolName(); + Long getCalls(); + Double getAvgTokens(); + Long getMaxTokens(); +} diff --git a/src/main/java/com/jobtracker/entity/ToolExecutionMetric.java b/src/main/java/com/jobtracker/entity/ToolExecutionMetric.java new file mode 100644 index 00000000..eb5c1ded --- /dev/null +++ b/src/main/java/com/jobtracker/entity/ToolExecutionMetric.java @@ -0,0 +1,61 @@ +package com.jobtracker.entity; + +import jakarta.persistence.*; +import lombok.*; +import org.hibernate.annotations.UuidGenerator; + +import java.time.LocalDateTime; +import java.util.UUID; + +@Entity +@Table(name = "tool_execution_metrics", indexes = { + @Index(name = "idx_tool_metrics_tool_name", columnList = "tool_name"), + @Index(name = "idx_tool_metrics_created_at", columnList = "created_at"), + @Index(name = "idx_tool_metrics_expensive", columnList = "expensive") +}) +@Getter +@Setter +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ToolExecutionMetric { + + @Id + @UuidGenerator(style = UuidGenerator.Style.TIME) + @Column(name = "id", columnDefinition = "BINARY(16)", updatable = false, nullable = false) + private UUID id; + + @Column(name = "tool_name", nullable = false, length = 255) + private String toolName; + + @Column(name = "execution_time_ms", nullable = false) + private long executionTimeMs; + + @Column(name = "request_bytes", nullable = false) + private int requestBytes; + + @Column(name = "response_bytes", nullable = false) + private int responseBytes; + + @Column(name = "request_tokens", nullable = false) + private int requestTokens; + + @Column(name = "response_tokens", nullable = false) + private int responseTokens; + + @Column(name = "total_tokens", nullable = false) + private int totalTokens; + + @Column(name = "expensive", nullable = false) + private boolean expensive; + + @Column(name = "created_at", nullable = false, updatable = false) + private LocalDateTime createdAt; + + @PrePersist + protected void onCreate() { + if (createdAt == null) { + createdAt = LocalDateTime.now(); + } + } +} diff --git a/src/main/java/com/jobtracker/mcp/tools/McpApplicationTools.java b/src/main/java/com/jobtracker/mcp/tools/McpApplicationTools.java index 565d2549..88655907 100644 --- a/src/main/java/com/jobtracker/mcp/tools/McpApplicationTools.java +++ b/src/main/java/com/jobtracker/mcp/tools/McpApplicationTools.java @@ -7,6 +7,7 @@ import com.jobtracker.dto.application.UpdateReminderRequest; import com.jobtracker.dto.application.UpdateStatusRequest; import com.jobtracker.service.ApplicationService; +import com.jobtracker.service.ToolMetricsCollector; import org.springaicommunity.mcp.annotation.McpTool; import org.springaicommunity.mcp.annotation.McpTool.McpAnnotations; import org.springaicommunity.mcp.annotation.McpToolParam; @@ -14,16 +15,21 @@ import java.time.LocalDate; import java.time.LocalDateTime; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.UUID; @Component public class McpApplicationTools { private final ApplicationService applicationService; + private final ToolMetricsCollector metricsCollector; - public McpApplicationTools(ApplicationService applicationService) { + public McpApplicationTools(ApplicationService applicationService, + ToolMetricsCollector metricsCollector) { this.applicationService = applicationService; + this.metricsCollector = metricsCollector; } // --- Read tools --- @@ -49,17 +55,17 @@ public ApplicationPageResponse listApplications( @McpToolParam(required = false, description = "Page size (default 20)") Integer size, @McpToolParam(required = false, description = "Sort field,direction e.g. createdAt,desc") String sort) { LocalDate from = applicationDateFrom != null ? LocalDate.parse(applicationDateFrom) : null; - LocalDate to = applicationDateTo != null ? LocalDate.parse(applicationDateTo) : null; - return applicationService.getAll( - status, - recruiterName, - from, to, - interviewScheduled, - null, - archived, - page != null ? page : 0, - size != null ? size : 20, - sort != null ? sort : "createdAt,desc"); + LocalDate to = applicationDateTo != null ? LocalDate.parse(applicationDateTo) : null; + int p = page != null ? page : 0; + int s = size != null ? size : 20; + String so = sort != null ? sort : "createdAt,desc"; + + return metricsCollector.measure( + "List-Applications", + params("status", status, "recruiterName", recruiterName, + "from", applicationDateFrom, "to", applicationDateTo, + "page", p, "size", s, "sort", so), + () -> applicationService.getAll(status, recruiterName, from, to, interviewScheduled, null, archived, p, s, so)); } @McpTool( @@ -74,7 +80,10 @@ public ApplicationPageResponse listApplications( openWorldHint = false)) public ApplicationResponse getApplication( @McpToolParam(required = true, description = "Application UUID") String id) { - return applicationService.getById(UUID.fromString(id)); + return metricsCollector.measure( + "Get-Application", + params("id", id), + () -> applicationService.getById(UUID.fromString(id))); } @McpTool( @@ -88,7 +97,10 @@ public ApplicationResponse getApplication( idempotentHint = true, openWorldHint = false)) public List getUpcomingApplications() { - return applicationService.getUpcoming(); + return metricsCollector.measure( + "Get-Upcoming-Applications", + null, + applicationService::getUpcoming); } @McpTool( @@ -102,7 +114,10 @@ public List getUpcomingApplications() { idempotentHint = true, openWorldHint = false)) public List getOverdueApplications() { - return applicationService.getOverdue(); + return metricsCollector.measure( + "Get-Overdue-Applications", + null, + applicationService::getOverdue); } // --- Write tools --- @@ -130,20 +145,16 @@ public ApplicationResponse createApplication( @McpToolParam(required = true, description = "Whether a DM reminder to the recruiter is enabled") Boolean recruiterDmReminderEnabled, @McpToolParam(required = false, description = "Personal notes about this application") String note, @McpToolParam(required = false, description = "Platform or job board where the vacancy was found, e.g. LinkedIn, Gupy, Indeed, Catho") String platform) { - return applicationService.create(new ApplicationRequest( - vacancyName, - recruiterName, - organization, - vacancyLink, + ApplicationRequest request = new ApplicationRequest( + vacancyName, recruiterName, organization, vacancyLink, applicationDate != null ? LocalDate.parse(applicationDate) : null, rhAcceptedConnection != null ? rhAcceptedConnection : Boolean.FALSE, interviewScheduled != null ? interviewScheduled : Boolean.FALSE, nextStepDateTime != null ? LocalDateTime.parse(nextStepDateTime) : null, status, recruiterDmReminderEnabled != null ? recruiterDmReminderEnabled : Boolean.FALSE, - note, - platform, - null)); + note, platform, null); + return metricsCollector.measure("Create-Application", request, () -> applicationService.create(request)); } @McpTool( @@ -170,20 +181,19 @@ public ApplicationResponse updateApplication( @McpToolParam(required = true, description = "Whether a DM reminder to the recruiter is enabled") Boolean recruiterDmReminderEnabled, @McpToolParam(required = false, description = "Personal notes about this application") String note, @McpToolParam(required = false, description = "Platform or job board where the vacancy was found, e.g. LinkedIn, Gupy, Indeed, Catho") String platform) { - return applicationService.update(UUID.fromString(id), new ApplicationRequest( - vacancyName, - recruiterName, - organization, - vacancyLink, + ApplicationRequest request = new ApplicationRequest( + vacancyName, recruiterName, organization, vacancyLink, applicationDate != null ? LocalDate.parse(applicationDate) : null, rhAcceptedConnection != null ? rhAcceptedConnection : Boolean.FALSE, interviewScheduled != null ? interviewScheduled : Boolean.FALSE, nextStepDateTime != null ? LocalDateTime.parse(nextStepDateTime) : null, status, recruiterDmReminderEnabled != null ? recruiterDmReminderEnabled : Boolean.FALSE, - note, - platform, - null)); + note, platform, null); + return metricsCollector.measure( + "Update-Application", + params("id", id, "request", request), + () -> applicationService.update(UUID.fromString(id), request)); } @McpTool( @@ -199,7 +209,10 @@ public ApplicationResponse updateApplication( public ApplicationResponse updateApplicationStatus( @McpToolParam(required = true, description = "Application UUID") String id, @McpToolParam(required = true, description = "New status display name") String status) { - return applicationService.updateStatus(UUID.fromString(id), new UpdateStatusRequest(status)); + return metricsCollector.measure( + "Update-Application-Status", + params("id", id, "status", status), + () -> applicationService.updateStatus(UUID.fromString(id), new UpdateStatusRequest(status))); } @McpTool( @@ -215,7 +228,10 @@ public ApplicationResponse updateApplicationStatus( public void updateApplicationReminder( @McpToolParam(required = true, description = "Application UUID") String id, @McpToolParam(required = true, description = "true to enable the DM reminder, false to disable it") boolean enabled) { - applicationService.updateReminder(UUID.fromString(id), new UpdateReminderRequest(enabled)); + metricsCollector.measure( + "Update-Application-Reminder", + params("id", id, "enabled", enabled), + () -> { applicationService.updateReminder(UUID.fromString(id), new UpdateReminderRequest(enabled)); return null; }); } @McpTool( @@ -230,7 +246,10 @@ public void updateApplicationReminder( openWorldHint = false)) public void markRecruiterDmSent( @McpToolParam(required = true, description = "Application UUID") String id) { - applicationService.markDmSent(UUID.fromString(id), new MarkDmSentRequest()); + metricsCollector.measure( + "Mark-Recruiter-DM-Sent", + params("id", id), + () -> { applicationService.markDmSent(UUID.fromString(id), new MarkDmSentRequest()); return null; }); } @McpTool( @@ -245,7 +264,10 @@ public void markRecruiterDmSent( openWorldHint = false)) public void archiveApplication( @McpToolParam(required = true, description = "Application UUID") String id) { - applicationService.archive(UUID.fromString(id)); + metricsCollector.measure( + "Archive-Application", + params("id", id), + () -> { applicationService.archive(UUID.fromString(id)); return null; }); } @McpTool( @@ -260,6 +282,22 @@ public void archiveApplication( openWorldHint = false)) public void deleteApplication( @McpToolParam(required = true, description = "Application UUID") String id) { - applicationService.delete(UUID.fromString(id)); + metricsCollector.measure( + "Delete-Application", + params("id", id), + () -> { applicationService.delete(UUID.fromString(id)); return null; }); + } + + // --- helpers --- + + /** Builds a null-safe parameter map for use as the request descriptor in measure(). */ + private static Map params(Object... kvPairs) { + var map = new LinkedHashMap(); + for (int i = 0; i + 1 < kvPairs.length; i += 2) { + if (kvPairs[i + 1] != null) { + map.put(kvPairs[i].toString(), kvPairs[i + 1]); + } + } + return map; } } diff --git a/src/main/java/com/jobtracker/repository/ToolExecutionMetricRepository.java b/src/main/java/com/jobtracker/repository/ToolExecutionMetricRepository.java new file mode 100644 index 00000000..eccdfca3 --- /dev/null +++ b/src/main/java/com/jobtracker/repository/ToolExecutionMetricRepository.java @@ -0,0 +1,62 @@ +package com.jobtracker.repository; + +import com.jobtracker.dto.metrics.AvgExecutionTimeProjection; +import com.jobtracker.dto.metrics.ToolUsageByDayProjection; +import com.jobtracker.dto.metrics.TopExpensiveToolProjection; +import com.jobtracker.entity.ToolExecutionMetric; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.UUID; + +@Repository +public interface ToolExecutionMetricRepository extends JpaRepository { + + /** + * Top tools ranked by average total tokens (identifies habitually expensive callers). + */ + @Query(""" + SELECT m.toolName AS toolName, + COUNT(m) AS calls, + AVG(m.totalTokens) AS avgTokens, + MAX(m.totalTokens) AS maxTokens + FROM ToolExecutionMetric m + GROUP BY m.toolName + ORDER BY avgTokens DESC + """) + List findTopExpensiveTools(); + + /** + * Daily call volume — native SQL so DATE() is resolved by the DB engine. + */ + @Query(value = """ + SELECT DATE_FORMAT(created_at, '%Y-%m-%d') AS day, + COUNT(*) AS calls + FROM tool_execution_metrics + GROUP BY DATE(created_at) + ORDER BY day DESC + """, nativeQuery = true) + List findUsageByDay(); + + /** + * Most token-heavy individual executions, useful for spotting outliers. + */ + @Query("SELECT m FROM ToolExecutionMetric m ORDER BY m.totalTokens DESC") + List findMostExpensiveExecutions(Pageable pageable); + + /** + * Average wall-clock execution time per tool, ordered slowest-first. + */ + @Query(""" + SELECT m.toolName AS toolName, + AVG(m.executionTimeMs) AS avgExecutionTimeMs, + COUNT(m) AS calls + FROM ToolExecutionMetric m + GROUP BY m.toolName + ORDER BY avgExecutionTimeMs DESC + """) + List findAvgExecutionTimePerTool(); +} diff --git a/src/main/java/com/jobtracker/service/TokenEstimatorService.java b/src/main/java/com/jobtracker/service/TokenEstimatorService.java new file mode 100644 index 00000000..711710eb --- /dev/null +++ b/src/main/java/com/jobtracker/service/TokenEstimatorService.java @@ -0,0 +1,65 @@ +package com.jobtracker.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.knuddels.jtokkit.Encodings; +import com.knuddels.jtokkit.api.Encoding; +import com.knuddels.jtokkit.api.EncodingType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +/** + * Estimates token counts and serialized byte sizes for arbitrary objects. + * Uses cl100k_base (GPT-4 / Claude-compatible) encoding via JTokkit. + * This is a cost-ranking heuristic — not a billing-accurate counter. + */ +@Service +public class TokenEstimatorService { + + private static final Logger log = LoggerFactory.getLogger(TokenEstimatorService.class); + + private final ObjectMapper objectMapper; + private final Encoding encoding; + + public TokenEstimatorService(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + this.encoding = Encodings.newDefaultEncodingRegistry().getEncoding(EncodingType.CL100K_BASE); + } + + /** + * Returns an estimated token count for {@code object} after JSON serialization. + * Returns 0 on any failure so callers are never blocked. + */ + public int countTokens(Object object) { + if (object == null) { + return 0; + } + try { + String json = objectMapper.writeValueAsString(object); + return encoding.countTokens(json); + } catch (Exception e) { + log.warn("[TOKEN-ESTIMATOR] countTokens failed for {}: {}", objectClass(object), e.getMessage()); + return 0; + } + } + + /** + * Returns the UTF-8 byte size of {@code object} after JSON serialization. + * Returns 0 on any failure so callers are never blocked. + */ + public int countBytes(Object object) { + if (object == null) { + return 0; + } + try { + return objectMapper.writeValueAsBytes(object).length; + } catch (Exception e) { + log.warn("[TOKEN-ESTIMATOR] countBytes failed for {}: {}", objectClass(object), e.getMessage()); + return 0; + } + } + + private static String objectClass(Object obj) { + return obj == null ? "null" : obj.getClass().getSimpleName(); + } +} diff --git a/src/main/java/com/jobtracker/service/ToolMetricsCollector.java b/src/main/java/com/jobtracker/service/ToolMetricsCollector.java new file mode 100644 index 00000000..7e2fd0ba --- /dev/null +++ b/src/main/java/com/jobtracker/service/ToolMetricsCollector.java @@ -0,0 +1,111 @@ +package com.jobtracker.service; + +import com.jobtracker.entity.ToolExecutionMetric; +import com.jobtracker.repository.ToolExecutionMetricRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.function.Supplier; + +/** + * Generic wrapper that measures MCP tool executions and stores the resulting metrics. + * + *

Metrics collection is fire-and-forget: any persistence failure is logged and + * swallowed so that a metrics error can never interrupt business execution. + * + *

Usage: + * 

{@code
+ * return metricsCollector.measure("List-Applications", requestParams, () ->
+ *         applicationService.getAll(...));
+ * }
+ */ +@Service +public class ToolMetricsCollector { + + private static final Logger log = LoggerFactory.getLogger(ToolMetricsCollector.class); + private static final int EXPENSIVE_TOKEN_THRESHOLD = 5_000; + + private final ToolExecutionMetricRepository repository; + private final TokenEstimatorService tokenEstimator; + + public ToolMetricsCollector(ToolExecutionMetricRepository repository, + TokenEstimatorService tokenEstimator) { + this.repository = repository; + this.tokenEstimator = tokenEstimator; + } + + /** + * Measures a single tool execution end-to-end. + * + * @param toolName the MCP tool name as registered (e.g. "List-Applications") + * @param request the request parameters sent to the tool (used for token estimation) + * @param execution the actual tool logic to run; exceptions propagate to the caller + * @param the tool's return type + * @return the unmodified value returned by {@code execution} + */ + public T measure(String toolName, Object request, Supplier execution) { + int requestTokens = safeCountTokens(request); + int requestBytes = safeCountBytes(request); + + long start = System.currentTimeMillis(); + T response = execution.get(); // business exceptions propagate + long elapsed = System.currentTimeMillis() - start; + + persistMetrics(toolName, requestTokens, requestBytes, response, elapsed); + return response; + } + + // --- private helpers --- + + private void persistMetrics(String toolName, int requestTokens, int requestBytes, + Object response, long executionTimeMs) { + try { + int responseTokens = safeCountTokens(response); + int responseBytes = safeCountBytes(response); + int totalTokens = requestTokens + responseTokens; + boolean expensive = totalTokens > EXPENSIVE_TOKEN_THRESHOLD; + + if (responseTokens > EXPENSIVE_TOKEN_THRESHOLD) { + log.warn("[MCP-METRICS] High-token response — tool={} responseTokens={} responseBytes={}", + toolName, responseTokens, responseBytes); + } + + ToolExecutionMetric metric = ToolExecutionMetric.builder() + .toolName(toolName) + .executionTimeMs(executionTimeMs) + .requestBytes(requestBytes) + .responseBytes(responseBytes) + .requestTokens(requestTokens) + .responseTokens(responseTokens) + .totalTokens(totalTokens) + .expensive(expensive) + .createdAt(LocalDateTime.now()) + .build(); + + repository.save(metric); + + } catch (Exception e) { + log.error("[MCP-METRICS] Failed to persist metrics for tool={}: {}", toolName, e.getMessage()); + } + } + + private int safeCountTokens(Object obj) { + try { + return tokenEstimator.countTokens(obj); + } catch (Exception e) { + log.debug("[MCP-METRICS] Token count error: {}", e.getMessage()); + return 0; + } + } + + private int safeCountBytes(Object obj) { + try { + return tokenEstimator.countBytes(obj); + } catch (Exception e) { + log.debug("[MCP-METRICS] Byte count error: {}", e.getMessage()); + return 0; + } + } +} diff --git a/src/main/resources/db/migration/V26__create_tool_execution_metrics.sql b/src/main/resources/db/migration/V26__create_tool_execution_metrics.sql new file mode 100644 index 00000000..67abe33a --- /dev/null +++ b/src/main/resources/db/migration/V26__create_tool_execution_metrics.sql @@ -0,0 +1,17 @@ +CREATE TABLE tool_execution_metrics ( + id BINARY(16) NOT NULL, + tool_name VARCHAR(255) NOT NULL, + execution_time_ms BIGINT NOT NULL, + request_bytes INT NOT NULL, + response_bytes INT NOT NULL, + request_tokens INT NOT NULL, + response_tokens INT NOT NULL, + total_tokens INT NOT NULL, + expensive BOOLEAN NOT NULL DEFAULT FALSE, + created_at DATETIME(6) NOT NULL, + CONSTRAINT pk_tool_execution_metrics PRIMARY KEY (id) +); + +CREATE INDEX idx_tool_metrics_tool_name ON tool_execution_metrics (tool_name); +CREATE INDEX idx_tool_metrics_created_at ON tool_execution_metrics (created_at); +CREATE INDEX idx_tool_metrics_expensive ON tool_execution_metrics (expensive);