From 23d156236048687ba6d03bd4d945267bfb1d912a Mon Sep 17 00:00:00 2001 From: Marcus Lee Date: Sat, 25 Oct 2025 18:05:45 -0700 Subject: [PATCH] (Untested) Enhance entity handling and statistics in AI analysis - Introduced new Entity and EntityType definitions for standardized entity detection in video analysis. - Updated existing interfaces to utilize the new Entity type for improved type safety. - Added functions to retrieve entity statistics from analysis results and events, enabling better insights into detected entities. - Created a new SQL migration to define the entity type enum in the database for consistency across the application. This commit significantly improves the structure and functionality of entity management within the AI analysis framework. --- lib/ai-analysis-queries.ts | 109 +++++++++++++++++- lib/supabase.ts | 36 +++++- lib/types/elasticsearch.ts | 40 ++++++- .../20251026005911_structure_entities.sql | 35 ++++++ worker/analysis-worker.ts | 10 +- worker/gemini-client.ts | 37 +++++- worker/types.ts | 40 ++++++- 7 files changed, 287 insertions(+), 20 deletions(-) create mode 100644 supabase/migrations/20251026005911_structure_entities.sql diff --git a/lib/ai-analysis-queries.ts b/lib/ai-analysis-queries.ts index b344016..bca934a 100644 --- a/lib/ai-analysis-queries.ts +++ b/lib/ai-analysis-queries.ts @@ -3,7 +3,7 @@ * Use these in your Next.js API routes or components */ -import { supabase } from "./supabase"; +import { supabase, type Entity, type EntityType } from "./supabase"; export interface AnalysisJobSummary { total: number; @@ -18,7 +18,7 @@ export interface AnalysisResult { job_id: number; summary: string; tags: string[]; - entities: any[]; + entities: Entity[]; raw: any; created_at: string; // Joined from job @@ -37,7 +37,7 @@ export interface AnalysisEvent { severity: "Minor" | "Medium" | "High"; type: "Crime" | "Medical Emergency" | "Traffic Incident" | "Property Damage" | "Safety Hazard" | "Suspicious Activity" | "Normal Activity" | "Camera Interference"; timestamp_seconds: number; - affected_entities: any[]; + affected_entities: Entity[]; created_at: string; } @@ -389,3 +389,106 @@ export async function getCriticalEvents(limit = 20): Promise { return data || []; } +/** + * Get entity statistics - count of each entity type across all analysis results + */ +export async function getEntityStatistics(): Promise> { + const { data, error } = await supabase + .from("ai_analysis_results") + .select("entities"); + + if (error) { + throw error; + } + + const entityCounts = new Map(); + + for (const row of data || []) { + const entities = (row.entities || []) as Entity[]; + for (const entity of entities) { + entityCounts.set(entity.type, (entityCounts.get(entity.type) || 0) + 1); + } + } + + return Array.from(entityCounts.entries()) + .map(([type, count]) => ({ type, count })) + .sort((a, b) => b.count - a.count); +} + +/** + * Get entity statistics for a specific asset + */ +export async function getEntityStatisticsForAsset( + assetId: string +): Promise> { + // Get all analysis results for this asset + const { data: jobs, error: jobsError } = await supabase + .from("ai_analysis_jobs") + .select("id") + .eq("source_id", assetId); + + if (jobsError) { + throw jobsError; + } + + const jobIds = (jobs || []).map((j) => j.id); + + if (jobIds.length === 0) { + return []; + } + + const { data: results, error: resultsError } = await supabase + .from("ai_analysis_results") + .select("entities") + .in("job_id", jobIds); + + if (resultsError) { + throw resultsError; + } + + const entityByType = new Map(); + + for (const row of results || []) { + const entities = (row.entities || []) as Entity[]; + for (const entity of entities) { + const existing = entityByType.get(entity.type) || []; + existing.push(entity); + entityByType.set(entity.type, existing); + } + } + + return Array.from(entityByType.entries()) + .map(([type, entities]) => ({ + type, + count: entities.length, + entities, + })) + .sort((a, b) => b.count - a.count); +} + +/** + * Get entity statistics from events (affected entities) + */ +export async function getEntityStatisticsFromEvents(): Promise> { + const { data, error } = await supabase + .from("ai_analysis_events") + .select("affected_entities"); + + if (error) { + throw error; + } + + const entityCounts = new Map(); + + for (const row of data || []) { + const entities = (row.affected_entities || []) as Entity[]; + for (const entity of entities) { + entityCounts.set(entity.type, (entityCounts.get(entity.type) || 0) + 1); + } + } + + return Array.from(entityCounts.entries()) + .map(([type, count]) => ({ type, count })) + .sort((a, b) => b.count - a.count); +} + diff --git a/lib/supabase.ts b/lib/supabase.ts index 81868af..ee75d27 100644 --- a/lib/supabase.ts +++ b/lib/supabase.ts @@ -12,6 +12,38 @@ if (!supabaseUrl || !supabaseAnonKey) { // Client for both frontend and backend (no RLS in demo project) export const supabase = createClient(supabaseUrl, supabaseAnonKey); +// Entity types detected in video analysis +// Matches the public.entity_type enum in the database +export type EntityType = + | 'person' + | 'vehicle' + | 'car' + | 'truck' + | 'bus' + | 'motorcycle' + | 'bicycle' + | 'animal' + | 'pet' + | 'dog' + | 'cat' + | 'package' + | 'bag' + | 'backpack' + | 'weapon' + | 'phone' + | 'laptop' + | 'object' + | 'location' + | 'activity' + | 'other'; + +// Entity detected in video analysis +export interface Entity { + type: EntityType; + name: string; + confidence: number; +} + // Database types // Camera is now based on mux.live_streams table with camera-specific fields export interface Camera { @@ -105,7 +137,7 @@ export interface AIAnalysisResult { job_id: number; summary: string | null; tags: string[] | any; - entities: any[] | any; + entities: Entity[] | any; transcript_ref: string | null; embeddings_ref: string | null; raw: any; @@ -121,7 +153,7 @@ export interface AIAnalysisEvent { severity: "Minor" | "Medium" | "High"; type: "Crime" | "Medical Emergency" | "Traffic Incident" | "Property Damage" | "Safety Hazard" | "Suspicious Activity" | "Normal Activity" | "Camera Interference"; timestamp_seconds: number; - affected_entities: any[] | any; + affected_entities: Entity[] | any; created_at: string; } diff --git a/lib/types/elasticsearch.ts b/lib/types/elasticsearch.ts index 341909f..3c3797c 100644 --- a/lib/types/elasticsearch.ts +++ b/lib/types/elasticsearch.ts @@ -15,6 +15,42 @@ export type DocumentType = 'event' | 'analysis' */ export type AssetType = 'live' | 'vod' +/** + * Entity types detected in video analysis + * Matches the public.entity_type enum in the database + */ +export type EntityType = + | 'person' + | 'vehicle' + | 'car' + | 'truck' + | 'bus' + | 'motorcycle' + | 'bicycle' + | 'animal' + | 'pet' + | 'dog' + | 'cat' + | 'package' + | 'bag' + | 'backpack' + | 'weapon' + | 'phone' + | 'laptop' + | 'object' + | 'location' + | 'activity' + | 'other'; + +/** + * Entity detected in video analysis + */ +export interface Entity { + type: EntityType; + name: string; + confidence: number; +} + /** * Event severity levels */ @@ -86,7 +122,7 @@ export interface EventDocument extends BaseDocument { /** Timestamp in seconds from asset start */ timestamp_seconds: number /** Entities involved in the event */ - affected_entities: any[] + affected_entities: Entity[] /** Optional tags */ tags?: string[] } @@ -101,7 +137,7 @@ export interface AnalysisDocument extends BaseDocument { /** Tags from analysis */ tags: string[] /** Entities detected in segment */ - entities: any[] + entities: Entity[] /** Start time in seconds from asset start */ asset_start_seconds: number /** End time in seconds from asset start */ diff --git a/supabase/migrations/20251026005911_structure_entities.sql b/supabase/migrations/20251026005911_structure_entities.sql new file mode 100644 index 0000000..7085771 --- /dev/null +++ b/supabase/migrations/20251026005911_structure_entities.sql @@ -0,0 +1,35 @@ +-- Create Entity Type Enum +-- Defines standardized entity types for AI analysis to enable proper statistics tracking + +-- Create enum type for entity types +create type public.entity_type as enum ( + 'person', + 'vehicle', + 'car', + 'truck', + 'bus', + 'motorcycle', + 'bicycle', + 'animal', + 'pet', + 'dog', + 'cat', + 'package', + 'bag', + 'backpack', + 'weapon', + 'phone', + 'laptop', + 'object', + 'location', + 'activity', + 'other' +); + +-- Add comment for documentation +comment on type public.entity_type is 'Standardized entity types detected in video analysis for statistics and filtering'; + +-- Note: Entities are stored as JSONB in ai_analysis_results.entities and ai_analysis_events.affected_entities +-- The enum provides type validation for the application layer and enables future query optimization +-- Each entity object in JSONB should have structure: { type: entity_type, name: text, confidence: float } + diff --git a/worker/analysis-worker.ts b/worker/analysis-worker.ts index ac3b19e..0c9a898 100644 --- a/worker/analysis-worker.ts +++ b/worker/analysis-worker.ts @@ -10,7 +10,7 @@ import { fetchAndTransmuxSegment } from "./ffmpeg-transmux.js"; import { analyzeVideoWithGemini } from "./gemini-client.js"; import { analyzeVideoWithRoboflow } from "./roboflow-detector.js"; import { startSegmentScheduler } from "./segment-scheduler.js"; -import type { AnalysisJob, RoboflowAnalysisResponse } from "./types.js"; +import type { AnalysisJob, RoboflowAnalysisResponse, Entity, Event as AnalysisEvent } from "./types.js"; // Configuration const SUPABASE_URL = process.env.SUPABASE_URL!; @@ -105,13 +105,7 @@ async function dequeueJob(): Promise { */ async function markJobSucceeded( job: AnalysisJob, - result: { - summary: string; - tags: string[]; - entities: any[]; - events: any[]; - raw: any; - }, + result: import("./types.js").GeminiAnalysisResponse, detections?: RoboflowAnalysisResponse, ) { // Insert Gemini result diff --git a/worker/gemini-client.ts b/worker/gemini-client.ts index 7c45cbb..6377bd1 100644 --- a/worker/gemini-client.ts +++ b/worker/gemini-client.ts @@ -11,7 +11,32 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { randomBytes } from "node:crypto"; import { z } from "zod"; -import type { GeminiAnalysisResponse } from "./types.js"; +import type { GeminiAnalysisResponse, EntityType } from "./types.js"; + +// Define entity types enum for Zod schema +const entityTypeEnum: [EntityType, ...EntityType[]] = [ + 'person', + 'vehicle', + 'car', + 'truck', + 'bus', + 'motorcycle', + 'bicycle', + 'animal', + 'pet', + 'dog', + 'cat', + 'package', + 'bag', + 'backpack', + 'weapon', + 'phone', + 'laptop', + 'object', + 'location', + 'activity', + 'other', +]; const GEMINI_API_KEY = process.env.GEMINI_API_KEY!; const GEMINI_MODEL = process.env.GEMINI_MODEL || "gemini-2.5-flash"; @@ -32,7 +57,7 @@ const analysisSchema = z.object({ tags: z.array(z.string()).max(10).describe("Relevant tags and keywords"), entities: z.array( z.object({ - type: z.string().describe("Entity type: person, object, location, activity, etc."), + type: z.enum(entityTypeEnum).describe("Entity type - must be one of: person, vehicle, car, truck, bus, motorcycle, bicycle, animal, pet, dog, cat, package, bag, backpack, weapon, phone, laptop, object, location, activity, or other"), name: z.string().describe("Name or description of the entity"), confidence: z.number().min(0).max(1).describe("Confidence score 0-1"), }), @@ -137,7 +162,13 @@ export async function analyzeVideoWithGemini( text: `Analyze this video segment and provide: 1. A concise summary (2-3 sentences) of what is happening. 2. Relevant tags or keywords (up to 10). -3. Detected people, objects, locations, and activities with confidence scores. +3. Detected entities with confidence scores. For entity types, use one of these standardized categories: + - People: 'person' + - Vehicles: 'vehicle' (generic), 'car', 'truck', 'bus', 'motorcycle', 'bicycle' + - Animals: 'animal' (generic), 'pet', 'dog', 'cat' + - Objects: 'package', 'bag', 'backpack', 'weapon', 'phone', 'laptop', 'object' (generic) + - Other: 'location', 'activity', 'other' + Use the most specific category that applies (e.g., 'car' instead of 'vehicle', 'dog' instead of 'pet'). 4. Notable events with detailed information: IMPORTANT: Only describe events that are genuinely noteworthy and would matter to a human reviewer. If nothing significant happens, simply return no events. Avoid flagging trivial or routine actions. diff --git a/worker/types.ts b/worker/types.ts index 86917e4..6a548bb 100644 --- a/worker/types.ts +++ b/worker/types.ts @@ -2,6 +2,42 @@ * Type definitions for AI Analysis Worker */ +/** + * Entity types detected in video analysis + * Matches the public.entity_type enum in the database + */ +export type EntityType = + | 'person' + | 'vehicle' + | 'car' + | 'truck' + | 'bus' + | 'motorcycle' + | 'bicycle' + | 'animal' + | 'pet' + | 'dog' + | 'cat' + | 'package' + | 'bag' + | 'backpack' + | 'weapon' + | 'phone' + | 'laptop' + | 'object' + | 'location' + | 'activity' + | 'other'; + +/** + * Entity detected in video analysis + */ +export interface Entity { + type: EntityType; + name: string; + confidence: number; +} + export interface AnalysisJob { id: number; source_type: "vod" | "live"; @@ -23,7 +59,7 @@ export interface AnalysisResult { job_id: number; summary: string; tags: string[]; - entities: any[]; + entities: Entity[]; transcript_ref: string | null; embeddings_ref: string | null; raw: any; @@ -42,7 +78,7 @@ export interface Event { export interface GeminiAnalysisResponse { summary: string; tags: string[]; - entities: any[]; + entities: Entity[]; events: Event[]; raw: any; }