diff --git a/app/src/features/human/emotionInference.test.ts b/app/src/features/human/emotionInference.test.ts new file mode 100644 index 000000000..16e178cce --- /dev/null +++ b/app/src/features/human/emotionInference.test.ts @@ -0,0 +1,295 @@ +import { describe, expect, it } from 'vitest'; + +import { + emotionToFace, + inferEmotionFromAssistantText, + inferEmotionFromOutcome, + inferEmotionFromReactionEmoji, + resolveEmotion, +} from './emotionInference'; + +// --------------------------------------------------------------------------- +// inferEmotionFromAssistantText +// --------------------------------------------------------------------------- + +describe('inferEmotionFromAssistantText', () => { + it('detects "sorry" as apologetic', () => { + const signal = inferEmotionFromAssistantText("I'm sorry, I couldn't complete that."); + expect(signal.emotion).toBe('apologetic'); + expect(signal.intensity).toBeGreaterThan(0); + }); + + it('detects "unfortunately" as apologetic', () => { + const signal = inferEmotionFromAssistantText('Unfortunately, the request failed.'); + expect(signal.emotion).toBe('apologetic'); + }); + + it('detects "i apologize" as apologetic', () => { + const signal = inferEmotionFromAssistantText('I apologize for the confusion.'); + expect(signal.emotion).toBe('apologetic'); + }); + + it('detects "i can\'t" as apologetic', () => { + const signal = inferEmotionFromAssistantText("I can't do that right now."); + expect(signal.emotion).toBe('apologetic'); + }); + + it('detects "unable to" as apologetic', () => { + const signal = inferEmotionFromAssistantText('I am unable to process that.'); + expect(signal.emotion).toBe('apologetic'); + }); + + it('detects "great news" as excited', () => { + const signal = inferEmotionFromAssistantText('Great news! The task completed.'); + expect(signal.emotion).toBe('excited'); + expect(signal.intensity).toBeGreaterThan(0); + }); + + it('detects "successfully" as excited', () => { + const signal = inferEmotionFromAssistantText('The file was successfully created.'); + expect(signal.emotion).toBe('excited'); + }); + + it('detects "done!" as excited', () => { + const signal = inferEmotionFromAssistantText('Done! Everything is set up.'); + expect(signal.emotion).toBe('excited'); + }); + + it('detects "congratulations" as excited', () => { + const signal = inferEmotionFromAssistantText('Congratulations on your achievement!'); + expect(signal.emotion).toBe('excited'); + }); + + it('detects "perfect" as excited', () => { + const signal = inferEmotionFromAssistantText('That is perfect, exactly what was needed.'); + expect(signal.emotion).toBe('excited'); + }); + + it('detects "be careful" as cautious', () => { + const signal = inferEmotionFromAssistantText('Be careful when running this command.'); + expect(signal.emotion).toBe('cautious'); + expect(signal.intensity).toBeGreaterThan(0); + }); + + it('detects "warning" as cautious', () => { + const signal = inferEmotionFromAssistantText('Warning: this operation is irreversible.'); + expect(signal.emotion).toBe('cautious'); + }); + + it('detects "note that" as cautious', () => { + const signal = inferEmotionFromAssistantText('Note that this may take a while.'); + expect(signal.emotion).toBe('cautious'); + }); + + it('detects "important:" as cautious', () => { + const signal = inferEmotionFromAssistantText('Important: back up your data first.'); + expect(signal.emotion).toBe('cautious'); + }); + + it('detects "caution" as cautious', () => { + const signal = inferEmotionFromAssistantText('Caution should be taken here.'); + expect(signal.emotion).toBe('cautious'); + }); + + it('returns neutral for generic text', () => { + const signal = inferEmotionFromAssistantText('Here is the result of your query.'); + expect(signal.emotion).toBe('neutral'); + expect(signal.intensity).toBe(0); + }); + + it('returns neutral for empty string', () => { + const signal = inferEmotionFromAssistantText(''); + expect(signal.emotion).toBe('neutral'); + }); + + it('is case-insensitive', () => { + expect(inferEmotionFromAssistantText('SORRY about that.').emotion).toBe('apologetic'); + expect(inferEmotionFromAssistantText('GREAT NEWS everyone!').emotion).toBe('excited'); + expect(inferEmotionFromAssistantText('WARNING: check this.').emotion).toBe('cautious'); + }); + + it('prioritises apology patterns over later patterns in the same text', () => { + // "sorry" appears before "done!" — apology wins + const signal = inferEmotionFromAssistantText("Sorry, but it's done!"); + expect(signal.emotion).toBe('apologetic'); + }); +}); + +// --------------------------------------------------------------------------- +// inferEmotionFromOutcome +// --------------------------------------------------------------------------- + +describe('inferEmotionFromOutcome', () => { + it('returns delighted for single-round success', () => { + const signal = inferEmotionFromOutcome({ rounds_used: 1, hadToolFailures: false }); + expect(signal.emotion).toBe('delighted'); + expect(signal.intensity).toBe(0.8); + }); + + it('returns proud for multi-round success', () => { + const signal = inferEmotionFromOutcome({ rounds_used: 3, hadToolFailures: false }); + expect(signal.emotion).toBe('proud'); + expect(signal.intensity).toBe(0.7); + }); + + it('returns concerned when there were tool failures', () => { + const signal = inferEmotionFromOutcome({ rounds_used: 1, hadToolFailures: true }); + expect(signal.emotion).toBe('concerned'); + expect(signal.intensity).toBe(0.6); + }); + + it('tool failures override multi-round optimism', () => { + const signal = inferEmotionFromOutcome({ rounds_used: 5, hadToolFailures: true }); + expect(signal.emotion).toBe('concerned'); + }); + + it('returns neutral for zero rounds (edge case)', () => { + const signal = inferEmotionFromOutcome({ rounds_used: 0, hadToolFailures: false }); + expect(signal.emotion).toBe('neutral'); + }); +}); + +// --------------------------------------------------------------------------- +// inferEmotionFromReactionEmoji +// --------------------------------------------------------------------------- + +describe('inferEmotionFromReactionEmoji', () => { + it('returns delighted for positive emoji 😊', () => { + const signal = inferEmotionFromReactionEmoji('😊'); + expect(signal.emotion).toBe('delighted'); + expect(signal.intensity).toBe(0.9); + }); + + it('returns delighted for positive emoji 🎉', () => { + expect(inferEmotionFromReactionEmoji('🎉').emotion).toBe('delighted'); + }); + + it('returns delighted for positive emoji ✅', () => { + expect(inferEmotionFromReactionEmoji('✅').emotion).toBe('delighted'); + }); + + it('returns delighted for positive emoji 👍', () => { + expect(inferEmotionFromReactionEmoji('👍').emotion).toBe('delighted'); + }); + + it('returns delighted for positive emoji 🙌', () => { + expect(inferEmotionFromReactionEmoji('🙌').emotion).toBe('delighted'); + }); + + it('returns concerned for negative emoji 😔', () => { + const signal = inferEmotionFromReactionEmoji('😔'); + expect(signal.emotion).toBe('concerned'); + expect(signal.intensity).toBe(0.7); + }); + + it('returns concerned for negative emoji ❌', () => { + expect(inferEmotionFromReactionEmoji('❌').emotion).toBe('concerned'); + }); + + it('returns concerned for negative emoji ⚠️', () => { + expect(inferEmotionFromReactionEmoji('⚠️').emotion).toBe('concerned'); + }); + + it('returns neutral for null', () => { + expect(inferEmotionFromReactionEmoji(null).emotion).toBe('neutral'); + }); + + it('returns neutral for undefined', () => { + expect(inferEmotionFromReactionEmoji(undefined).emotion).toBe('neutral'); + }); + + it('returns neutral for unrecognized emoji', () => { + expect(inferEmotionFromReactionEmoji('🐶').emotion).toBe('neutral'); + }); + + it('returns neutral for empty string', () => { + expect(inferEmotionFromReactionEmoji('').emotion).toBe('neutral'); + }); +}); + +// --------------------------------------------------------------------------- +// resolveEmotion +// --------------------------------------------------------------------------- + +describe('resolveEmotion', () => { + it('picks the highest intensity non-neutral signal', () => { + const result = resolveEmotion([ + { emotion: 'cautious', intensity: 0.6, source: 'a' }, + { emotion: 'delighted', intensity: 0.9, source: 'b' }, + { emotion: 'proud', intensity: 0.7, source: 'c' }, + ]); + expect(result).toBe('delighted'); + }); + + it('returns neutral when all signals are neutral', () => { + const result = resolveEmotion([ + { emotion: 'neutral', intensity: 0, source: 'a' }, + { emotion: 'neutral', intensity: 0, source: 'b' }, + ]); + expect(result).toBe('neutral'); + }); + + it('returns neutral for an empty array', () => { + expect(resolveEmotion([])).toBe('neutral'); + }); + + it('breaks ties by returning the first matching signal', () => { + const result = resolveEmotion([ + { emotion: 'excited', intensity: 0.7, source: 'first' }, + { emotion: 'proud', intensity: 0.7, source: 'second' }, + ]); + // excited comes first and shares the top intensity — it should win + expect(result).toBe('excited'); + }); + + it('ignores neutral signals even when they appear between non-neutral ones', () => { + const result = resolveEmotion([ + { emotion: 'neutral', intensity: 0, source: 'a' }, + { emotion: 'apologetic', intensity: 0.7, source: 'b' }, + { emotion: 'neutral', intensity: 0, source: 'c' }, + ]); + expect(result).toBe('apologetic'); + }); + + it('handles a single non-neutral signal', () => { + expect(resolveEmotion([{ emotion: 'confused', intensity: 0.5, source: 'x' }])).toBe('confused'); + }); +}); + +// --------------------------------------------------------------------------- +// emotionToFace +// --------------------------------------------------------------------------- + +describe('emotionToFace', () => { + it('maps neutral to null (no override)', () => { + expect(emotionToFace('neutral')).toBeNull(); + }); + + it('maps delighted to happy', () => { + expect(emotionToFace('delighted')).toBe('happy'); + }); + + it('maps proud to happy', () => { + expect(emotionToFace('proud')).toBe('happy'); + }); + + it('maps excited to happy', () => { + expect(emotionToFace('excited')).toBe('happy'); + }); + + it('maps concerned to concerned', () => { + expect(emotionToFace('concerned')).toBe('concerned'); + }); + + it('maps apologetic to concerned', () => { + expect(emotionToFace('apologetic')).toBe('concerned'); + }); + + it('maps confused to confused', () => { + expect(emotionToFace('confused')).toBe('confused'); + }); + + it('maps cautious to confused', () => { + expect(emotionToFace('cautious')).toBe('confused'); + }); +}); diff --git a/app/src/features/human/emotionInference.ts b/app/src/features/human/emotionInference.ts new file mode 100644 index 000000000..c37c83718 --- /dev/null +++ b/app/src/features/human/emotionInference.ts @@ -0,0 +1,154 @@ +import type { MascotFace } from './Mascot'; + +export type MascotEmotion = + | 'neutral' + | 'delighted' + | 'proud' + | 'concerned' + | 'confused' + | 'apologetic' + | 'excited' + | 'cautious'; + +export interface EmotionSignal { + emotion: MascotEmotion; + intensity: number; // 0-1 + source: string; +} + +/** + * How long (ms) to hold a face driven by each emotion before decaying to idle. + * Tuned so lighter emotions feel like a soft beat and heavier ones linger. + */ +export const EMOTION_HOLD_MS: Record = { + neutral: 700, + delighted: 900, + proud: 1200, + concerned: 900, + confused: 800, + apologetic: 700, + excited: 1000, + cautious: 600, +}; + +/** + * How many characters of accumulated text to skip between successive scans. + * Exported so tests can wire up the threshold without magic numbers. + */ +export const TEXT_SCAN_INTERVAL = 200; + +const APOLOGY_PATTERNS = ['sorry', 'unfortunately', 'i apologize', "i can't", 'unable to']; +const EXCITEMENT_PATTERNS = ['great news', 'successfully', 'done!', 'congratulations', 'perfect']; +const CAUTION_PATTERNS = ['be careful', 'warning', 'note that', 'important:', 'caution']; + +/** + * Scans assistant text for emotional keywords/patterns. + * Pure — no side effects, no state. + */ +export function inferEmotionFromAssistantText(text: string): EmotionSignal { + const lower = text.toLowerCase(); + + for (const pattern of APOLOGY_PATTERNS) { + if (lower.includes(pattern)) { + return { emotion: 'apologetic', intensity: 0.7, source: 'text:apology' }; + } + } + + for (const pattern of EXCITEMENT_PATTERNS) { + if (lower.includes(pattern)) { + return { emotion: 'excited', intensity: 0.7, source: 'text:excitement' }; + } + } + + for (const pattern of CAUTION_PATTERNS) { + if (lower.includes(pattern)) { + return { emotion: 'cautious', intensity: 0.6, source: 'text:caution' }; + } + } + + return { emotion: 'neutral', intensity: 0, source: 'text:none' }; +} + +/** + * Infers an emotion from an inference outcome (rounds used, tool failures). + * Pure — no side effects, no state. + */ +export function inferEmotionFromOutcome(outcome: { + rounds_used: number; + hadToolFailures: boolean; +}): EmotionSignal { + const { rounds_used, hadToolFailures } = outcome; + + if (hadToolFailures) { + return { emotion: 'concerned', intensity: 0.6, source: 'outcome:tool_failures' }; + } + if (rounds_used === 1) { + return { emotion: 'delighted', intensity: 0.8, source: 'outcome:single_round' }; + } + if (rounds_used > 1) { + return { emotion: 'proud', intensity: 0.7, source: 'outcome:multi_round' }; + } + + return { emotion: 'neutral', intensity: 0, source: 'outcome:none' }; +} + +const POSITIVE_EMOJIS = new Set(['😊', '😄', '🎉', '✅', '👍', '🙌', '💪', '🎊']); +const NEGATIVE_EMOJIS = new Set(['😔', '😢', '❌', '⚠️', '😕']); + +/** + * Infers an emotion from a reaction emoji attached to a response. + * Pure — no side effects, no state. + */ +export function inferEmotionFromReactionEmoji(emoji: string | null | undefined): EmotionSignal { + if (!emoji) { + return { emotion: 'neutral', intensity: 0, source: 'emoji:none' }; + } + if (POSITIVE_EMOJIS.has(emoji)) { + return { emotion: 'delighted', intensity: 0.9, source: 'emoji:positive' }; + } + if (NEGATIVE_EMOJIS.has(emoji)) { + return { emotion: 'concerned', intensity: 0.7, source: 'emoji:negative' }; + } + return { emotion: 'neutral', intensity: 0, source: 'emoji:unrecognized' }; +} + +/** + * Picks the dominant emotion from a set of signals. + * Highest intensity non-neutral signal wins; ties break in favour of the + * first signal in the array. If all signals are neutral, returns 'neutral'. + * + * Pure — no side effects, no state. + */ +export function resolveEmotion(signals: EmotionSignal[]): MascotEmotion { + let best: EmotionSignal | null = null; + for (const signal of signals) { + if (signal.emotion === 'neutral') continue; + if (best === null || signal.intensity > best.intensity) { + best = signal; + } + } + return best?.emotion ?? 'neutral'; +} + +/** + * Maps a resolved emotion to the closest MascotFace, or null when the emotion + * does not override the activity-driven face (i.e. neutral). + * + * Pure — no side effects, no state. + */ +export function emotionToFace(emotion: MascotEmotion): MascotFace | null { + switch (emotion) { + case 'neutral': + return null; + case 'delighted': + case 'proud': + case 'excited': + return 'happy'; + case 'concerned': + case 'apologetic': + return 'concerned'; + case 'confused': + case 'cautious': + return 'confused'; + } +} diff --git a/app/src/features/human/useHumanMascot.test.ts b/app/src/features/human/useHumanMascot.test.ts index 5936b9d13..afbeaae72 100644 --- a/app/src/features/human/useHumanMascot.test.ts +++ b/app/src/features/human/useHumanMascot.test.ts @@ -2,6 +2,7 @@ import { act, renderHook } from '@testing-library/react'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { ChatEventListeners } from '../../services/chatService'; +import { EMOTION_HOLD_MS } from './emotionInference'; import { VISEMES } from './Mascot/visemes'; import { ACK_FACE_HOLD_MS, pickViseme, useHumanMascot } from './useHumanMascot'; import { playBase64Audio } from './voice/audioPlayer'; @@ -184,8 +185,9 @@ describe('useHumanMascot state machine', () => { ); }); expect(result.current.face).toBe('happy'); + // single-round success → delighted emotion → EMOTION_HOLD_MS.delighted hold act(() => { - vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1); + vi.advanceTimersByTime(EMOTION_HOLD_MS.delighted + 1); }); expect(result.current.face).toBe('idle'); }); @@ -311,8 +313,9 @@ describe('useHumanMascot TTS playback', () => { }); expect(result.current.face).toBe('happy'); + // single-round success → delighted emotion → EMOTION_HOLD_MS.delighted hold act(() => { - vi.advanceTimersByTime(ACK_FACE_HOLD_MS + 1); + vi.advanceTimersByTime(EMOTION_HOLD_MS.delighted + 1); }); expect(result.current.face).toBe('idle'); }); @@ -419,3 +422,129 @@ describe('useHumanMascot TTS playback', () => { expect(result.current.face).toBe('idle'); }); }); + +describe('useHumanMascot emotion layer', () => { + beforeEach(() => { + capturedListeners = null; + vi.useFakeTimers(); + (synthesizeSpeech as ReturnType).mockReset(); + (playBase64Audio as ReturnType).mockReset(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + function fakeEvent(extra: T): T & { thread_id: string; request_id: string } { + return { thread_id: 't', request_id: 'r', ...extra }; + } + + function fakeDoneEmotion(text: string, rounds_used = 1, reaction_emoji?: string | null) { + return { + thread_id: 't', + request_id: 'r', + full_response: text, + rounds_used, + total_input_tokens: 1, + total_output_tokens: 1, + reaction_emoji, + }; + } + + it('emotion starts as neutral', () => { + const { result } = renderHook(() => useHumanMascot()); + expect(result.current.emotion).toBe('neutral'); + }); + + it('apology text triggers concerned face on done', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + // Stream the apology text via text_delta so it accumulates into accumulatedTextRef, + // then fire onDone. The text signal (apologetic, 0.7) vs outcome (delighted, 0.8) — + // outcome wins unless we also add a tool failure. Instead, use rounds_used=2 so + // outcome becomes proud (0.7), and text apologetic (0.7) ties — first-in-array wins. + // To make apologetic clearly dominate: fire a tool failure too so concerned (0.6) + // from outcome. Actually apologetic (text) at 0.7 > concerned (outcome) at 0.6. + // Simplest path: send the apology text via text_delta then onDone with a failed tool. + const apologyText = 'sorry, I could not complete the task as requested'; + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onTextDelta?.(fakeEvent({ round: 1, delta: apologyText })); + capturedListeners?.onToolResult?.( + fakeEvent({ tool_name: 'lookup', skill_id: 's', output: 'err', success: false, round: 1 }) + ); + capturedListeners?.onDone?.(fakeDoneEmotion(apologyText, 1)); + }); + // apologetic (text, 0.7) > concerned (outcome, 0.6) → apologetic → concerned face + expect(result.current.face).toBe('concerned'); + }); + + it('excitement text triggers happy face on done', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onDone?.(fakeDoneEmotion('successfully completed the task', 1)); + }); + // excited → happy face + expect(result.current.face).toBe('happy'); + }); + + it('single-round success triggers delighted emotion', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onDone?.(fakeDoneEmotion('Here is the answer.', 1)); + }); + expect(result.current.emotion).toBe('delighted'); + expect(result.current.face).toBe('happy'); + }); + + it('multi-round success triggers proud emotion', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onDone?.(fakeDoneEmotion('Here is the answer.', 3)); + }); + expect(result.current.emotion).toBe('proud'); + expect(result.current.face).toBe('happy'); + }); + + it('tool failure triggers concerned emotion', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onToolResult?.( + fakeEvent({ tool_name: 'lookup', skill_id: 's', output: 'err', success: false, round: 1 }) + ); + capturedListeners?.onDone?.(fakeDoneEmotion('Here is the answer.', 1)); + }); + expect(result.current.emotion).toBe('concerned'); + expect(result.current.face).toBe('concerned'); + }); + + it('emotion resets on new inference_start', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + // First turn — get a non-neutral emotion. + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onDone?.(fakeDoneEmotion('Great news, done!', 1)); + }); + expect(result.current.emotion).not.toBe('neutral'); + // Second turn begins — emotion should reset to neutral. + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + }); + expect(result.current.emotion).toBe('neutral'); + }); + + it('positive reaction_emoji triggers delighted emotion', () => { + const { result } = renderHook(() => useHumanMascot({ speakReplies: false })); + act(() => { + capturedListeners?.onInferenceStart?.(fakeEvent({})); + capturedListeners?.onDone?.(fakeDoneEmotion('Here is the answer.', 1, '🎉')); + }); + // emoji delighted (intensity 0.9) beats outcome delighted (intensity 0.8) + // but both resolve to 'delighted' — verify the face and emotion + expect(result.current.emotion).toBe('delighted'); + expect(result.current.face).toBe('happy'); + }); +}); diff --git a/app/src/features/human/useHumanMascot.ts b/app/src/features/human/useHumanMascot.ts index a11f9f579..3812b61e1 100644 --- a/app/src/features/human/useHumanMascot.ts +++ b/app/src/features/human/useHumanMascot.ts @@ -2,6 +2,16 @@ import debug from 'debug'; import { useEffect, useRef, useState } from 'react'; import { subscribeChatEvents } from '../../services/chatService'; +import { + EMOTION_HOLD_MS, + emotionToFace, + inferEmotionFromAssistantText, + inferEmotionFromOutcome, + inferEmotionFromReactionEmoji, + type MascotEmotion, + resolveEmotion, + TEXT_SCAN_INTERVAL, +} from './emotionInference'; import type { MascotFace } from './Mascot'; import { lerpViseme, VISEMES, type VisemeShape } from './Mascot/visemes'; import { type PlaybackHandle, playBase64Audio } from './voice/audioPlayer'; @@ -86,6 +96,7 @@ export interface UseHumanMascotOptions { export interface UseHumanMascotResult { face: MascotFace; viseme: VisemeShape; + emotion: MascotEmotion; } /** @@ -126,6 +137,13 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas const [, force] = useState(0); + // Emotion layer — inferred from conversation content and outcomes. + const [emotion, setEmotion] = useState('neutral'); + const emotionRef = useRef('neutral'); + const accumulatedTextRef = useRef(''); + const hadToolFailureRef = useRef(false); + const lastScanLenRef = useRef(0); + function clearAckTimer() { if (ackTimerRef.current != null) { window.clearTimeout(ackTimerRef.current); @@ -133,6 +151,12 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas } } + /** Keeps emotionRef in sync so async TTS callbacks read a fresh value. */ + function updateEmotion(e: MascotEmotion) { + emotionRef.current = e; + setEmotion(e); + } + function holdThenIdle(ackFace: MascotFace, ms = ACK_FACE_HOLD_MS) { clearAckTimer(); setFace(ackFace); @@ -147,6 +171,10 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas onInferenceStart: () => { clearAckTimer(); setFace('thinking'); + updateEmotion('neutral'); + accumulatedTextRef.current = ''; + hadToolFailureRef.current = false; + lastScanLenRef.current = 0; }, onIterationStart: e => { // Subsequent iterations mean the agent is grinding through tool rounds. @@ -161,6 +189,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas }, onToolResult: e => { if (!e.success) { + hadToolFailureRef.current = true; // Don't fully derail — let the next inference step take over. setFace('concerned'); } else { @@ -174,11 +203,44 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas setFace('speaking'); targetRef.current = pickViseme(e.delta); lastDeltaAtRef.current = window.performance.now(); + accumulatedTextRef.current += e.delta; + const len = accumulatedTextRef.current.length; + if (len - lastScanLenRef.current >= TEXT_SCAN_INTERVAL) { + lastScanLenRef.current = len; + const signal = inferEmotionFromAssistantText(accumulatedTextRef.current); + if (signal.emotion !== 'neutral') { + mascotLog( + '[emotion] mid-stream scan: %s (intensity %.2f)', + signal.emotion, + signal.intensity + ); + updateEmotion(signal.emotion); + } + } }, onDone: e => { + // Final emotion inference from all signals. + const textSignal = inferEmotionFromAssistantText(accumulatedTextRef.current); + const outcomeSignal = inferEmotionFromOutcome({ + rounds_used: e.rounds_used, + hadToolFailures: hadToolFailureRef.current, + }); + const emojiSignal = inferEmotionFromReactionEmoji(e.reaction_emoji); + const resolved = resolveEmotion([emojiSignal, outcomeSignal, textSignal]); + mascotLog( + '[emotion] resolved: %s (emoji=%s outcome=%s text=%s)', + resolved, + emojiSignal.emotion, + outcomeSignal.emotion, + textSignal.emotion + ); + updateEmotion(resolved); + if (!speakRef.current || !e.full_response?.trim()) { - // Soft acknowledgement beat instead of snapping back to idle. - holdThenIdle('happy'); + const emotionFace = emotionToFace(resolved); + const ackFace = emotionFace ?? 'happy'; + const holdMs = EMOTION_HOLD_MS[resolved]; + holdThenIdle(ackFace, holdMs); return; } // Fire-and-forget — startTtsPlayback owns its cleanup via finally. @@ -190,6 +252,7 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas playbackRef.current?.stop(); playbackRef.current = null; visemeFramesRef.current = []; + updateEmotion('concerned'); holdThenIdle('concerned'); }, }); @@ -284,11 +347,17 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas if (isStillCurrent()) { playbackRef.current = null; visemeFramesRef.current = []; - if (degraded) { - holdThenIdle('concerned'); - } else { - holdThenIdle('happy'); - } + const currentEmotion = emotionRef.current; + const emotionFace = emotionToFace(currentEmotion); + const ackFace = degraded ? 'concerned' : (emotionFace ?? 'happy'); + const holdMs = degraded ? ACK_FACE_HOLD_MS : EMOTION_HOLD_MS[currentEmotion]; + mascotLog( + '[emotion] tts finally — emotion=%s face=%s holdMs=%d', + currentEmotion, + ackFace, + holdMs + ); + holdThenIdle(ackFace, holdMs); } } } @@ -330,5 +399,5 @@ export function useHumanMascot(options: UseHumanMascotOptions = {}): UseHumanMas // can reflect mic-on without racing the chat event subscription. const effectiveFace: MascotFace = listening && face !== 'speaking' ? 'listening' : face; - return { face: effectiveFace, viseme }; + return { face: effectiveFace, viseme, emotion }; }