From cd17478a6e3fb7738675e6a15ff5c3d4ae24d3ba Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 1/9] fix missing import --- .../components/long-running-response/long-running-response.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/components/long-running-response/long-running-response.ts b/src/app/components/long-running-response/long-running-response.ts index 8eab06cc..7591d588 100644 --- a/src/app/components/long-running-response/long-running-response.ts +++ b/src/app/components/long-running-response/long-running-response.ts @@ -39,6 +39,7 @@ import {MatIcon} from '@angular/material/icon'; MatButton, MatIcon, NgxJsonViewerModule, + MarkdownComponent, ], }) export class LongRunningResponseComponent implements OnChanges { From 05ac75d40e4f7043f9334da32e23a623d8baca3c Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 2/9] extend types for transcriptions and streaming --- src/app/core/models/types.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/core/models/types.ts b/src/app/core/models/types.ts index fd4f538e..1d795976 100644 --- a/src/app/core/models/types.ts +++ b/src/app/core/models/types.ts @@ -128,11 +128,12 @@ export declare interface Event extends LlmResponse { nodeInfo?: { path?: string;[key: string]: any; }; data?: any; output?: { result?: any; }; - inputTranscription?: { text: string; }; - outputTranscription?: { text: string; }; + inputTranscription?: { text: string; finished?: boolean; }; + outputTranscription?: { text: string; finished?: boolean; }; usageMetadata?: any; interrupted?: boolean; turnComplete?: boolean; + partial?: boolean; } export interface ComputerUsePayload { From 47cdb7cc2515ee74c5a516f1c6f27d4da6293984 Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 3/9] handle optionality of the blob --- src/app/core/services/websocket.service.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app/core/services/websocket.service.ts b/src/app/core/services/websocket.service.ts index 4aa984e3..20c80864 100644 --- a/src/app/core/services/websocket.service.ts +++ b/src/app/core/services/websocket.service.ts @@ -66,7 +66,9 @@ export class WebSocketService implements WebSocketServiceInterface { } sendMessage(data: LiveRequest) { - data.blob.data = this.arrayBufferToBase64(data.blob.data.buffer); + if (data.blob?.data) { + data.blob.data = this.arrayBufferToBase64(data.blob.data.buffer); + } if (!this.socket$ || this.socket$.closed) { console.error('WebSocket is not open.'); return; From 584e77700bd7904da8ca36d38a9c52a0c6470248 Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 4/9] filter incoming audio based on pcm --- src/app/core/services/websocket.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app/core/services/websocket.service.ts b/src/app/core/services/websocket.service.ts index 20c80864..8c667e2a 100644 --- a/src/app/core/services/websocket.service.ts +++ b/src/app/core/services/websocket.service.ts @@ -103,7 +103,8 @@ export class WebSocketService implements WebSocketServiceInterface { if ( msg['content'] && msg['content']['parts'] && - msg['content']['parts'][0]['inlineData'] + msg['content']['parts'][0]['inlineData'] && + msg['content']['parts'][0]['inlineData']['mimeType']?.startsWith('audio/pcm') ) { const pcmBytes = this.base64ToUint8Array( msg['content']['parts'][0]['inlineData']['data'], From 710e5bc13efed32825b4b2e73b6ac482969d1c63 Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 5/9] extend stream chat service to support reusing live connection and send text too --- .../core/services/interfaces/stream-chat.ts | 3 +++ src/app/core/services/stream-chat.service.ts | 24 +++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/app/core/services/interfaces/stream-chat.ts b/src/app/core/services/interfaces/stream-chat.ts index c893ffca..978f1ac7 100644 --- a/src/app/core/services/interfaces/stream-chat.ts +++ b/src/app/core/services/interfaces/stream-chat.ts @@ -17,6 +17,8 @@ import {ElementRef, InjectionToken} from '@angular/core'; import {Observable} from 'rxjs'; +import {Event} from '../../models/types'; +import {AgentRunRequest} from '../../models/AgentRunRequest'; export const STREAM_CHAT_SERVICE = new InjectionToken('StreamChatService'); @@ -51,4 +53,5 @@ export declare abstract class StreamChatService { abstract stopVideoStreaming(videoContainer: ElementRef): void; abstract onStreamClose(): Observable; abstract closeStream(): void; + abstract sendMessage(req: AgentRunRequest, flags?: LiveFlags): void; } diff --git a/src/app/core/services/stream-chat.service.ts b/src/app/core/services/stream-chat.service.ts index a80babd4..aa20a00b 100644 --- a/src/app/core/services/stream-chat.service.ts +++ b/src/app/core/services/stream-chat.service.ts @@ -26,6 +26,8 @@ import {VIDEO_SERVICE} from './interfaces/video'; import {WEBSOCKET_SERVICE} from './interfaces/websocket'; import {VideoService} from './video.service'; import {WebSocketService} from './websocket.service'; +import {map,filter} from 'rxjs/operators'; +import {AgentRunRequest} from '../models/AgentRunRequest'; /** * Service for supporting live streaming with audio/video. @@ -39,9 +41,25 @@ export class StreamChatService implements StreamChatServiceInterface { private readonly webSocketService = inject(WEBSOCKET_SERVICE); private audioIntervalId: number|undefined = undefined; private videoIntervalId: number|undefined = undefined; + private currentUrl: string|undefined = undefined; constructor() {} + connect(serverUrl: string) { + if (this.currentUrl !== serverUrl) { + this.webSocketService.connect(serverUrl); + this.currentUrl = serverUrl; + } + } + + sendMessage({ appName, userId, sessionId, newMessage }: AgentRunRequest, flags?: LiveFlags): void { + this.connect(this.getWsUrl(appName, userId, sessionId!!, flags)); + const request: LiveRequest = { + content: newMessage, + }; + this.webSocketService.sendMessage(request); + } + private getWsUrl(appName: string, userId: string, sessionId: string, flags?: LiveFlags): string { const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws'; let url = `${protocol}://${URLUtil.getWSServerUrl()}/run_live?app_name=${appName}&user_id=${userId}&session_id=${sessionId}`; @@ -68,14 +86,13 @@ export class StreamChatService implements StreamChatServiceInterface { sessionId, flags, }: {appName: string; userId: string; sessionId: string; flags?: LiveFlags;}) { - this.webSocketService.connect(this.getWsUrl(appName, userId, sessionId, flags)); + this.connect(this.getWsUrl(appName, userId, sessionId, flags)); await this.startAudioStreaming(); } stopAudioChat() { this.stopAudioStreaming(); - this.webSocketService.closeConnection(); } private async startAudioStreaming() { @@ -118,7 +135,7 @@ export class StreamChatService implements StreamChatServiceInterface { videoContainer: ElementRef; flags?: LiveFlags; }) { - this.webSocketService.connect(this.getWsUrl(appName, userId, sessionId, flags)); + this.connect(this.getWsUrl(appName, userId, sessionId, flags)); await this.startAudioStreaming(); await this.startVideoStreaming(videoContainer); @@ -127,7 +144,6 @@ export class StreamChatService implements StreamChatServiceInterface { stopVideoChat(videoContainer: ElementRef) { this.stopAudioStreaming(); this.stopVideoStreaming(videoContainer); - this.webSocketService.closeConnection(); } async startVideoStreaming(videoContainer: ElementRef) { From 9c94caa0e1bc2696f6bba3b6a01aeead23d94944 Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 6/9] add live toggle --- .../call-controls/call-controls.component.html | 6 +++--- .../call-controls/call-controls.component.scss | 4 ++++ .../call-controls/call-controls.component.ts | 1 + .../chat-panel/chat-panel.component.html | 1 + .../chat-panel/chat-panel.component.ts | 1 + src/app/components/chat/chat.component.html | 16 ++++++++++++++++ src/app/components/chat/chat.component.ts | 8 ++++++++ 7 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/app/components/call-controls/call-controls.component.html b/src/app/components/call-controls/call-controls.component.html index 698e519e..70847c48 100644 --- a/src/app/components/call-controls/call-controls.component.html +++ b/src/app/components/call-controls/call-controls.component.html @@ -5,7 +5,7 @@ class="video-rec-btn" [class.recording]="isVideoRecording" [matTooltip]="isVideoRecording ? i18n.turnOffCamTooltip : i18n.useCamTooltip" - [disabled]="!isBidiStreamingEnabled" + [disabled]="!isBidiStreamingEnabled || !useLive" > videocam @@ -22,12 +22,12 @@ (click)="onCallClick()" class="audio-rec-btn" [class.recording]="isAudioRecording" - [disabled]="!isBidiStreamingEnabled" + [disabled]="!isBidiStreamingEnabled || !useLive" > {{ isAudioRecording ? 'call_end' : 'call' }} - @if (showFlags && !isAudioRecording) { + @if (useLive && showFlags && !isAudioRecording) {
Live Flags
diff --git a/src/app/components/call-controls/call-controls.component.scss b/src/app/components/call-controls/call-controls.component.scss index ff6f5975..f8d44ad8 100644 --- a/src/app/components/call-controls/call-controls.component.scss +++ b/src/app/components/call-controls/call-controls.component.scss @@ -19,6 +19,10 @@ button { } } +button.audio-rec-btn:not(.recording):disabled { + color: gray !important; +} + button.audio-rec-btn:not(.recording) { color: #34a853 !important; } diff --git a/src/app/components/call-controls/call-controls.component.ts b/src/app/components/call-controls/call-controls.component.ts index ff1f6c59..e3e7682c 100644 --- a/src/app/components/call-controls/call-controls.component.ts +++ b/src/app/components/call-controls/call-controls.component.ts @@ -40,6 +40,7 @@ export class CallControlsComponent { @Input() isVideoRecording = false; @Input() micVolume = 0; @Input() isBidiStreamingEnabled: boolean | null = false; + @Input() useLive: boolean | null = false; @Output() readonly toggleAudioRecording = new EventEmitter(); @Output() readonly toggleVideoRecording = new EventEmitter(); diff --git a/src/app/components/chat-panel/chat-panel.component.html b/src/app/components/chat-panel/chat-panel.component.html index af82ecc6..cc89a458 100644 --- a/src/app/components/chat-panel/chat-panel.component.html +++ b/src/app/components/chat-panel/chat-panel.component.html @@ -207,6 +207,7 @@

Evaluation Result

diff --git a/src/app/components/chat-panel/chat-panel.component.ts b/src/app/components/chat-panel/chat-panel.component.ts index 00e6acb6..73aaa441 100644 --- a/src/app/components/chat-panel/chat-panel.component.ts +++ b/src/app/components/chat-panel/chat-panel.component.ts @@ -135,6 +135,7 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit { @Input() isEditFunctionArgsEnabled: boolean = false; @Input() isTokenStreamingEnabled: boolean = false; @Input() useSse: boolean = false; + @Input() useLive: boolean = false; @Input() userInput: string = ''; @Input() userEditEvalCaseMessage: string = ''; @Input() selectedFiles: { file: File; url: string }[] = []; diff --git a/src/app/components/chat/chat.component.html b/src/app/components/chat/chat.component.html index cab97598..668e2d41 100644 --- a/src/app/components/chat/chat.component.html +++ b/src/app/components/chat/chat.component.html @@ -565,6 +565,17 @@ Streaming } + + @if ((isBidiStreamingEnabledObs | async) && canEditSession()) { + + }
@switch (chatType()) { @@ -609,6 +620,7 @@ [invocationDisplayMap]="invocationDisplayMap()" [viewMode]="viewMode()" [shouldShowEvent]="shouldShowEventFn" + [useLive]="useLive()" > } @case ('eval-case') { @@ -642,6 +654,7 @@ [invocationDisplayMap]="invocationDisplayMap()" [viewMode]="viewMode()" [shouldShowEvent]="shouldShowEventFn" + [useLive]="useLive()" > } @case ('eval-result') { @@ -723,6 +736,7 @@ [invocationDisplayMap]="invocationDisplayMap()" [viewMode]="viewMode()" [shouldShowEvent]="shouldShowEventFn" + [useLive]="useLive()" >
@@ -775,6 +789,7 @@ [invocationDisplayMap]="invocationDisplayMap()" [viewMode]="viewMode()" [shouldShowEvent]="shouldShowEventFn" + [useLive]="useLive()" >
@@ -796,6 +811,7 @@ [invocationDisplayMap]="invocationDisplayMap()" [viewMode]="viewMode()" [shouldShowEvent]="shouldShowEventFn" + [useLive]="useLive()" > } } diff --git a/src/app/components/chat/chat.component.ts b/src/app/components/chat/chat.component.ts index e7625387..203abc44 100644 --- a/src/app/components/chat/chat.component.ts +++ b/src/app/components/chat/chat.component.ts @@ -290,6 +290,7 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { showAppSelectorDrawer = false; showSessionSelectorDrawer = false; useSse = signal(window.localStorage.getItem('adk-use-sse') === 'true'); + useLive = signal(window.localStorage.getItem('adk-use-live') === 'true'); currentSessionState: SessionState | undefined = {}; root_agent = ROOT_AGENT; updatedSessionState: WritableSignal = signal(null); @@ -630,6 +631,8 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { this.featureFlagService.isApplicationSelectorEnabled(); readonly isTokenStreamingEnabledObs: Observable = this.featureFlagService.isTokenStreamingEnabled(); + readonly isBidiStreamingEnabledObs: Observable = + this.featureFlagService.isBidiStreamingEnabled(); readonly isExportSessionEnabledObs: Observable = this.featureFlagService.isExportSessionEnabled(); readonly isNewSessionButtonEnabledObs: Observable = @@ -2728,6 +2731,11 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { window.localStorage.setItem('adk-use-sse', String(this.useSse())); } + toggleLive() { + this.useLive.set(!this.useLive()); + window.localStorage.setItem('adk-use-live', String(this.useLive())); + } + enterBuilderMode() { const url = this.router .createUrlTree([], { From 2643ddd407bd2741ffebc6aa50fc1c0db86ba2ce Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 7/9] rename sse to streaming --- .../components/chat-panel/chat-panel.component.ts | 4 ++-- src/app/components/chat/chat.component.html | 14 +++++++------- src/app/components/chat/chat.component.ts | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/app/components/chat-panel/chat-panel.component.ts b/src/app/components/chat-panel/chat-panel.component.ts index 73aaa441..074e05cb 100644 --- a/src/app/components/chat-panel/chat-panel.component.ts +++ b/src/app/components/chat-panel/chat-panel.component.ts @@ -134,7 +134,7 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit { @Input() agentGraphData: any = null; @Input() isEditFunctionArgsEnabled: boolean = false; @Input() isTokenStreamingEnabled: boolean = false; - @Input() useSse: boolean = false; + @Input() useStreaming: boolean = false; @Input() useLive: boolean = false; @Input() userInput: string = ''; @Input() userEditEvalCaseMessage: string = ''; @@ -179,7 +179,7 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit { @Output() readonly toggleVideoRecording = new EventEmitter(); @Output() readonly longRunningResponseComplete = new EventEmitter(); @Output() readonly toggleHideIntermediateEvents = new EventEmitter(); - @Output() readonly toggleSse = new EventEmitter(); + @Output() readonly toggleStreaming = new EventEmitter(); @ViewChild('videoContainer', { read: ElementRef }) videoContainer!: ElementRef; @ViewChild('autoScroll') scrollContainer!: ElementRef; diff --git a/src/app/components/chat/chat.component.html b/src/app/components/chat/chat.component.html index 668e2d41..d067c15c 100644 --- a/src/app/components/chat/chat.component.html +++ b/src/app/components/chat/chat.component.html @@ -556,12 +556,12 @@ } @if ((isTokenStreamingEnabledObs | async) && canEditSession()) { - } @@ -590,8 +590,8 @@ (toggleHideIntermediateEvents)="toggleHideIntermediateEvents()" [traceData]="traceData" [isTokenStreamingEnabled]="(isTokenStreamingEnabledObs | async) ?? false" - [useSse]="useSse()" - (toggleSse)="toggleSse()" + [useStreaming]="useStreaming()" + (toggleStreaming)="toggleStreaming()" [isChatMode]="true" [selectedFiles]="selectedFiles" [updatedSessionState]="updatedSessionState()" @@ -750,8 +750,8 @@ (toggleHideIntermediateEvents)="toggleHideIntermediateEvents()" [traceData]="traceData" [isTokenStreamingEnabled]="(isTokenStreamingEnabledObs | async) ?? false" - [useSse]="useSse()" - (toggleSse)="toggleSse()" + [useStreaming]="useStreaming()" + (toggleStreaming)="toggleStreaming()" [isChatMode]="false" [evalCase]="evalCase" [isEvalEditMode]="isEvalEditMode()" diff --git a/src/app/components/chat/chat.component.ts b/src/app/components/chat/chat.component.ts index 203abc44..99bd1ec9 100644 --- a/src/app/components/chat/chat.component.ts +++ b/src/app/components/chat/chat.component.ts @@ -289,7 +289,7 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { showBuilderAssistant = true; showAppSelectorDrawer = false; showSessionSelectorDrawer = false; - useSse = signal(window.localStorage.getItem('adk-use-sse') === 'true'); + useStreaming = signal(window.localStorage.getItem('adk-use-streaming') === 'true'); useLive = signal(window.localStorage.getItem('adk-use-live') === 'true'); currentSessionState: SessionState | undefined = {}; root_agent = ROOT_AGENT; @@ -1100,7 +1100,7 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { userId: this.userId, sessionId: this.sessionId, newMessage: content, - streaming: this.useSse(), + streaming: this.useStreaming(), stateDelta: this.updatedSessionState(), }; if (functionCallEventId) { @@ -2726,9 +2726,9 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { this.selectedFiles.splice(index, 1); } - toggleSse() { - this.useSse.set(!this.useSse()); - window.localStorage.setItem('adk-use-sse', String(this.useSse())); + toggleStreaming() { + this.useStreaming.set(!this.useStreaming()); + window.localStorage.setItem('adk-use-streaming', String(this.useStreaming())); } toggleLive() { From 2c2ecd9a350fa00e16199837f4c5cf9f45ea2195 Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 8/9] remove bidi streaming warning --- src/app/components/chat/chat.component.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/app/components/chat/chat.component.ts b/src/app/components/chat/chat.component.ts index 99bd1ec9..eb884c9b 100644 --- a/src/app/components/chat/chat.component.ts +++ b/src/app/components/chat/chat.component.ts @@ -529,9 +529,6 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { window.localStorage.setItem('adk-hide-intermediate-events', String(newVal)); } - // TODO: Remove this once backend supports restarting bidi streaming. - sessionHasUsedBidi = new Set(); - eventData = new Map(); traceData: any[] = []; renderedEventGraph: SafeHtml | undefined; @@ -1818,11 +1815,6 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { } async startAudioRecording(flags?: LiveFlags) { - if (this.sessionId && this.sessionHasUsedBidi.has(this.sessionId)) { - this.openSnackBar(BIDI_STREAMING_RESTART_WARNING, 'OK'); - return; - } - // Lazily create a real session if it does not exist const isSessionActive = await this.ensureSessionActive(); if (!isSessionActive) { @@ -1836,7 +1828,6 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { sessionId: this.sessionId, flags: flags, }); - this.sessionHasUsedBidi.add(this.sessionId); } stopAudioRecording() { From 30b6ad89d26c15eba8347806a33536323eb66e6f Mon Sep 17 00:00:00 2001 From: Robert Schuh Date: Sat, 16 May 2026 01:01:11 +0200 Subject: [PATCH 9/9] add support for sending live messages --- src/app/components/chat/chat.component.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/app/components/chat/chat.component.ts b/src/app/components/chat/chat.component.ts index eb884c9b..aac0ffd2 100644 --- a/src/app/components/chat/chat.component.ts +++ b/src/app/components/chat/chat.component.ts @@ -1110,6 +1110,10 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { submitAgentRunRequest(req: AgentRunRequest) { this.autoSelectLatestEvent = true; + if (this.useLive()) { + this.streamChatService.sendMessage(req, {}); + return; + } this.agentService.runSse(req).subscribe({ next: async (chunkJson: any) => { if (chunkJson.error) { @@ -1150,7 +1154,7 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { }); } - private appendEventRow(apiEvent: any, reverseOrder: boolean = false) { + private appendEventRow(apiEvent: AdkEvent, reverseOrder: boolean = false) { if (apiEvent.inputTranscription !== undefined) { apiEvent.author = 'user'; } else if (apiEvent.outputTranscription !== undefined) { @@ -1173,7 +1177,7 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { if (apiEvent?.longRunningToolIds && apiEvent.longRunningToolIds.length > 0) { const startIndex = this.longRunningEvents.length; this.getAsyncFunctionsFromParts( - apiEvent.longRunningToolIds, apiEvent.content.parts, apiEvent.invocationId); + apiEvent.longRunningToolIds, apiEvent.content.parts, apiEvent.invocationId!!); // Store event ID for later reference this.functionCallEventId = apiEvent.id;