Added Screensharing capabilities to ADK Web UI

src/app/app.component.spec.ts

@@ -49,10 +49,12 @@ import {MockStreamChatService} from './core/services/testing/mock-stream-chat.se
 import {MockStringToColorService} from './core/services/testing/mock-string-to-color.service';
 import {MockTraceService} from './core/services/testing/mock-trace.service';
 import {MockVideoService} from './core/services/testing/mock-video.service';
+import {MockScreenSharingService} from './core/services/testing/mock-screensharing.service';
 import {MockWebSocketService} from './core/services/testing/mock-websocket.service';
 import {TRACE_SERVICE, TraceService} from './core/services/trace.service';
 import {VIDEO_SERVICE, VideoService} from './core/services/video.service';
 import {WEBSOCKET_SERVICE, WebSocketService,} from './core/services/websocket.service';
+import { SCREEN_SHARING_SERVICE } from './core/services/screensharing.service';
 
 describe('AppComponent', () => {
   beforeEach(async () => {
@@ -64,6 +66,7 @@ describe('AppComponent', () => {
     const audioService = new MockAudioService();
     const webSocketService = new MockWebSocketService();
     const videoService = new MockVideoService();
+    const screenSharingService = new MockScreenSharingService();
     const streamChatService = new MockStreamChatService();
     const eventService = new MockEventService();
     const downloadService = new MockDownloadService();
@@ -114,6 +117,10 @@ describe('AppComponent', () => {
               provide: VIDEO_SERVICE,
               useValue: videoService,
             },
+            {
+              provide: SCREEN_SHARING_SERVICE,
+              useValue: screenSharingService,
+            },
             {
               provide: STREAM_CHAT_SERVICE,
               useValue: streamChatService,

src/app/components/chat-panel/chat-panel.component.html

@@ -17,6 +17,7 @@
 @if (appName != "") {
 <div #autoScroll class="chat-messages">
   <div #videoContainer></div>
+  <div #screenSharingContainer></div>
   @for (message of messages; track message; let i = $index) {
   <div
     [ngClass]="{
@@ -380,6 +381,18 @@
         >
           <mat-icon>videocam</mat-icon>
         </button>
+        <button
+          mat-icon-button
+          matSuffix
+          (click)="toggleScreenSharing.emit()"
+          [ngStyle]="{
+              'background-color': isScreenSharing ? 'rgb(234, 67, 53)' : 'rgb(51, 53, 55)'
+            }"
+          [matTooltip]="isScreenSharing ? i18n.stopScreenShareTooltip : i18n.startScreenShareTooltip"
+        >
+          <mat-icon>screen_share</mat-icon>
+        </button>
+
       </div>
     </div>
   </mat-form-field>

src/app/components/chat-panel/chat-panel.component.i18n.ts

@@ -44,6 +44,8 @@ export const CHAT_PANEL_MESSAGES = {
   turnOffMicTooltip: 'Turn off microphone',
   useMicTooltip: 'Use microphone',
   turnOffCamTooltip: 'Turn off camera',
+  stopScreenShareTooltip: 'Turn off screensharing',
+  startScreenShareTooltip: 'Turn on screensharing',
   useCamTooltip: 'Use camera',
   updatedSessionStateChipLabel: 'Updated session state',
 };

src/app/components/chat-panel/chat-panel.component.ts

@@ -68,6 +68,7 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit {
   @Input() eventData = new Map<string, any>();
   @Input() isAudioRecording: boolean = false;
   @Input() isVideoRecording: boolean = false;
+  @Input() isScreenSharing: boolean = false;
   @Input() hoveredEventMessageIndices: number[] = [];
 
   @Output() readonly userInputChange = new EventEmitter<string>();
@@ -94,8 +95,10 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit {
   @Output() readonly updateState = new EventEmitter<void>();
   @Output() readonly toggleAudioRecording = new EventEmitter<void>();
   @Output() readonly toggleVideoRecording = new EventEmitter<void>();
+  @Output() readonly toggleScreenSharing = new EventEmitter<void>();
 
   @ViewChild('videoContainer', {read: ElementRef}) videoContainer!: ElementRef;
+  @ViewChild('screenSharingContainer', {read: ElementRef}) screenSharingContainer!: ElementRef;
   @ViewChild('autoScroll') scrollContainer!: ElementRef;
   @ViewChild('messageTextarea') public textarea: ElementRef|undefined;
   scrollInterrupted = false;

src/app/components/chat/chat.component.html

@@ -223,6 +223,7 @@
             [eventData]="eventData"
             [isAudioRecording]="isAudioRecording"
             [isVideoRecording]="isVideoRecording"
+            [isScreenSharing]="isScreenSharing"
             [hoveredEventMessageIndices]="hoveredEventMessageIndices"
             (clickEvent)="clickEvent($event)"
             (handleKeydown)="handleKeydown($event.event, $event.message)"
@@ -240,6 +241,7 @@
             (updateState)="updateState()"
             (toggleAudioRecording)="toggleAudioRecording()"
             (toggleVideoRecording)="toggleVideoRecording()"
+            (toggleScreenSharing)="toggleScreenSharing()"
         ></app-chat-panel>
       }
     </mat-card>

src/app/components/chat/chat.component.spec.ts

@@ -55,9 +55,11 @@ import {MockStreamChatService} from '../../core/services/testing/mock-stream-cha
 import {MockStringToColorService} from '../../core/services/testing/mock-string-to-color.service';
 import {MockTraceService} from '../../core/services/testing/mock-trace.service';
 import {MockVideoService} from '../../core/services/testing/mock-video.service';
+import {MockScreenSharingService} from '../../core/services/testing/mock-screensharing.service';
 import {MockWebSocketService} from '../../core/services/testing/mock-websocket.service';
 import {TRACE_SERVICE, TraceService} from '../../core/services/trace.service';
 import {VIDEO_SERVICE, VideoService} from '../../core/services/video.service';
+import {SCREEN_SHARING_SERVICE, ScreenSharingService,} from '../../core/services/screensharing.service';
 import {WEBSOCKET_SERVICE, WebSocketService,} from '../../core/services/websocket.service';
 import {fakeAsync,
         tick} from '../../testing/utils';
@@ -110,6 +112,7 @@ describe('ChatComponent', () => {
   let mockAudioService: MockAudioService;
   let mockWebSocketService: MockWebSocketService;
   let mockVideoService: MockVideoService;
+  let mockScreenSharingService: MockScreenSharingService;
   let mockStreamChatService: MockStreamChatService;
   let mockEventService: MockEventService;
   let mockDownloadService: MockDownloadService;
@@ -132,6 +135,7 @@ describe('ChatComponent', () => {
     mockAudioService = new MockAudioService();
     mockWebSocketService = new MockWebSocketService();
     mockVideoService = new MockVideoService();
+    mockScreenSharingService = new MockScreenSharingService();
     mockStreamChatService = new MockStreamChatService();
     mockEventService = new MockEventService();
     mockDownloadService = new MockDownloadService();
@@ -200,6 +204,7 @@ describe('ChatComponent', () => {
             {provide: AUDIO_SERVICE, useValue: mockAudioService},
             {provide: WEBSOCKET_SERVICE, useValue: mockWebSocketService},
             {provide: VIDEO_SERVICE, useValue: mockVideoService},
+            {provide: SCREEN_SHARING_SERVICE, useValue: mockScreenSharingService},
             {provide: EVENT_SERVICE, useValue: mockEventService},
             {provide: STREAM_CHAT_SERVICE, useValue: mockStreamChatService},
             {provide: DOWNLOAD_SERVICE, useValue: mockDownloadService},

src/app/components/chat/chat.component.ts

@@ -169,6 +169,7 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy {
   evalSetId = '';
   isAudioRecording = false;
   isVideoRecording = false;
+  isScreenSharing = false;
   longRunningEvents: any[] = [];
   functionCallEventId = '';
   redirectUri = URLUtil.getBaseUrlWithoutPath();
@@ -958,6 +959,11 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy {
       this.stopVideoRecording();
       this.isVideoRecording = false;
     }
+    if (this.isScreenSharing) {
+      this.stopScreenSharing();
+      this.isScreenSharing = false;
+    }
+
     this.evalTab()?.resetEvalResults();
     this.traceData = [];
     this.bottomPanelVisible = false;
@@ -1029,6 +1035,41 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy {
     this.isVideoRecording = false;
   }
 
+  toggleScreenSharing() {
+    this.isScreenSharing ? this.stopScreenSharing() : this.startScreenSharing();
+  }
+
+  startScreenSharing() {
+    if (this.sessionHasUsedBidi.has(this.sessionId)) {
+      this.openSnackBar(BIDI_STREAMING_RESTART_WARNING, 'OK')
+      return;
+    }
+    const screenSharingContainer = this.chatPanel()?.screenSharingContainer;
+    if (!screenSharingContainer) {
+      return;
+    }
+    this.isScreenSharing = true;
+    this.streamChatService.startScreenSharingChat({
+      appName: this.appName,
+      userId: this.userId,
+      sessionId: this.sessionId,
+      screenSharingContainer,
+    });
+    this.messages.update(
+        messages => [...messages, {role: 'user', text: 'Sharing Screen...'}]);
+    this.sessionHasUsedBidi.add(this.sessionId);
+  }
+
+  stopScreenSharing() {
+    const screenSharingContainer = this.chatPanel()?.screenSharingContainer;
+    if (!screenSharingContainer) {
+      return;
+    }
+
+    this.streamChatService.stopScreenSharingChat(screenSharingContainer);
+    this.isScreenSharing = false;
+  }
+
   private getAsyncFunctionsFromParts(pendingIds: any[], parts: any[], invocationId: string) {
     for (const part of parts) {
       if (part.functionCall && pendingIds.includes(part.functionCall.id)) {

src/app/components/side-panel/side-panel.component.spec.ts

@@ -41,6 +41,7 @@ import {MockFeatureFlagService} from '../../core/services/testing/mock-feature-f
 import {MockSafeValuesService} from '../../core/services/testing/mock-safevalues.service';
 import {TRACE_SERVICE, TraceService} from '../../core/services/trace.service';
 import {VIDEO_SERVICE, VideoService} from '../../core/services/video.service';
+import {SCREEN_SHARING_SERVICE, ScreenSharingService} from '../../core/services/screensharing.service';
 import {WEBSOCKET_SERVICE, WebSocketService,} from '../../core/services/websocket.service';
 
 import {SidePanelComponent} from './side-panel.component';
@@ -70,6 +71,7 @@ describe('SidePanelComponent', () => {
   let mockAudioService: jasmine.SpyObj<AudioService>;
   let mockWebSocketService: jasmine.SpyObj<WebSocketService>;
   let mockVideoService: jasmine.SpyObj<VideoService>;
+  let mockScreenSharingService: jasmine.SpyObj<ScreenSharingService>;
   let mockEventService: jasmine.SpyObj<EventService>;
   let mockDownloadService: jasmine.SpyObj<DownloadService>;
   let mockEvalService: jasmine.SpyObj<EvalService>;
@@ -106,6 +108,10 @@ describe('SidePanelComponent', () => {
         'VideoService',
         ['startRecording', 'stopRecording'],
     );
+    mockScreenSharingService = jasmine.createSpyObj(
+        'ScreenSharingService',
+        ['startScreenSharing', 'stopScreenSharing'],
+    );
     mockEventService = jasmine.createSpyObj('EventService', ['getTrace']);
     mockDownloadService = jasmine.createSpyObj(
         'DownloadService',
@@ -175,6 +181,7 @@ describe('SidePanelComponent', () => {
             {provide: AUDIO_SERVICE, useValue: mockAudioService},
             {provide: WEBSOCKET_SERVICE, useValue: mockWebSocketService},
             {provide: VIDEO_SERVICE, useValue: mockVideoService},
+            {provide: SCREEN_SHARING_SERVICE, useValue: mockScreenSharingService},
             {provide: EVENT_SERVICE, useValue: mockEventService},
             {provide: DOWNLOAD_SERVICE, useValue: mockDownloadService},
             {provide: EVAL_SERVICE, useValue: mockEvalService},