Add support for chat streaming (#18)

### Motivation and Context

<!-- Thank you for your contribution to the copilot-chat repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->
responses will now be streamed directly from the Kernel to all users in
the chat.
### Description
- `ChatSkill.cs`:
- added `StreamResponseToClient()` to broadcast the streamed response to
all clients.
- changed `GetChatResponseAsync()` to use `Task.WhenAll` when extracting
audience/user intent and semantic/document memories.
- changed `GetChatContextTokenLimit()` to account for the tokens used in
extracting the audience

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Preview
**regular messages:**

![gif](https://github.com/microsoft/chat-copilot/assets/52973358/b5ba0b1f-3ddc-4648-8eff-587735770b7a)

**planner messages:**

![gif2](https://github.com/microsoft/chat-copilot/assets/52973358/79bfa394-7857-4e06-9302-174a4e6d1247)

**multiuser messages:**

![gif](https://github.com/microsoft/chat-copilot/assets/52973358/8013ce33-3eff-4bc2-bf74-dafac378e9b0)

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [Contribution
Guidelines](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md#dev-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

webapi/CopilotChat/Controllers/ChatController.cs

@@ -41,7 +41,6 @@ public class ChatController : ControllerBase, IDisposable
     private readonly ITelemetryService _telemetryService;
     private const string ChatSkillName = "ChatSkill";
     private const string ChatFunctionName = "Chat";
-    private const string ReceiveResponseClientCall = "ReceiveResponse";
     private const string GeneratingResponseClientCall = "ReceiveBotResponseStatus";
 
     public ChatController(ILogger<ChatController> logger, ITelemetryService telemetryService)
@@ -130,7 +129,6 @@ public async Task<IActionResult> ChatAsync(
         if (ask.Variables.Where(v => v.Key == "chatId").Any())
         {
             var chatId = ask.Variables.Where(v => v.Key == "chatId").First().Value;
-            await messageRelayHubContext.Clients.Group(chatId).SendAsync(ReceiveResponseClientCall, chatSkillAskResult, chatId);
             await messageRelayHubContext.Clients.Group(chatId).SendAsync(GeneratingResponseClientCall, chatId, null);
         }
 

webapi/CopilotChat/Controllers/DocumentImportController.cs

@@ -70,7 +70,7 @@ private enum SupportedFileType
     private readonly ChatMessageRepository _messageRepository;
     private readonly ChatParticipantRepository _participantRepository;
     private const string GlobalDocumentUploadedClientCall = "GlobalDocumentUploaded";
-    private const string ChatDocumentUploadedClientCall = "ChatDocumentUploaded";
+    private const string ReceiveMessageClientCall = "ReceiveMessage";
     private readonly ITesseractEngine _tesseractEngine;
 
     /// <summary>
@@ -148,7 +148,7 @@ public async Task<IActionResult> ImportDocumentsAsync(
 
             var chatId = documentImportForm.ChatId.ToString();
             await messageRelayHubContext.Clients.Group(chatId)
-                .SendAsync(ChatDocumentUploadedClientCall, chatMessage, chatId);
+                .SendAsync(ReceiveMessageClientCall, chatMessage, chatId);
 
             return this.Ok(chatMessage);
         }

webapi/CopilotChat/Hubs/MessageRelayHub.cs

@@ -30,7 +30,7 @@ public MessageRelayHub(ILogger<MessageRelayHub> logger)
     /// TODO: Retrieve the user ID from the claims and call this method
     /// from the OnConnectedAsync method instead of the frontend.
     /// </summary>
-    /// <param name="chatId">The ChatID used as group id for SignalR.</param>
+    /// <param name="chatId">The chat ID used as group id for SignalR.</param>
     public async Task AddClientToGroupAsync(string chatId)
     {
         await this.Groups.AddToGroupAsync(this.Context.ConnectionId, chatId);
@@ -39,17 +39,18 @@ public async Task AddClientToGroupAsync(string chatId)
     /// <summary>
     /// Sends a message to all users except the sender.
     /// </summary>
-    /// <param name="chatId">The ChatID used as group id for SignalR.</param>
+    /// <param name="chatId">The chat ID used as group id for SignalR.</param>
+    /// <param name="senderId">The user ID of the user that sent the message.</param>
     /// <param name="message">The message to send.</param>
-    public async Task SendMessageAsync(string chatId, object message)
+    public async Task SendMessageAsync(string chatId, string senderId, object message)
     {
-        await this.Clients.OthersInGroup(chatId).SendAsync(ReceiveMessageClientCall, message, chatId);
+        await this.Clients.OthersInGroup(chatId).SendAsync(ReceiveMessageClientCall, chatId, senderId, message);
     }
 
     /// <summary>
     /// Sends the typing state to all users except the sender.
     /// </summary>
-    /// <param name="chatId">The ChatID used as group id for SignalR.</param>
+    /// <param name="chatId">The chat ID used as group id for SignalR.</param>
     /// <param name="userId">The user ID of the user who is typing.</param>
     /// <param name="isTyping">Whether the user is typing.</param>
     /// <returns>A task that represents the asynchronous operation.</returns>

webapp/src/components/chat/ChatInput.tsx

@@ -14,10 +14,7 @@ import { GetResponseOptions, useChat } from '../../libs/useChat';
 import { useAppDispatch, useAppSelector } from '../../redux/app/hooks';
 import { RootState } from '../../redux/app/store';
 import { addAlert } from '../../redux/features/app/appSlice';
-import {
-    editConversationInput,
-    updateBotResponseStatusFromServer,
-} from '../../redux/features/conversations/conversationsSlice';
+import { editConversationInput, updateBotResponseStatus } from '../../redux/features/conversations/conversationsSlice';
 import { SpeechService } from './../../libs/services/SpeechService';
 import { updateUserIsTyping } from './../../redux/features/conversations/conversationsSlice';
 import { ChatStatus } from './ChatStatus';
@@ -153,7 +150,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({ isDraggingOver, onDragLeav
 
         setValue('');
         dispatch(editConversationInput({ id: selectedId, newInput: '' }));
-        dispatch(updateBotResponseStatusFromServer({ chatId: selectedId, status: 'Calling the kernel' }));
+        dispatch(updateBotResponseStatus({ chatId: selectedId, status: 'Calling the kernel' }));
         onSubmit({ value, messageType, chatId: selectedId }).catch((error) => {
             const message = `Error submitting chat input: ${(error as Error).message}`;
             log(message);

webapp/src/components/chat/ChatRoom.tsx

@@ -8,7 +8,7 @@ import { AuthorRoles, IChatMessage } from '../../libs/models/ChatMessage';
 import { GetResponseOptions, useChat } from '../../libs/useChat';
 import { useAppDispatch, useAppSelector } from '../../redux/app/hooks';
 import { RootState } from '../../redux/app/store';
-import { updateConversationFromUser } from '../../redux/features/conversations/conversationsSlice';
+import { addMessageToConversationFromUser } from '../../redux/features/conversations/conversationsSlice';
 import { SharedStyles } from '../../styles';
 import { ChatInput } from './ChatInput';
 import { ChatHistory } from './chat-history/ChatHistory';
@@ -48,7 +48,6 @@ export const ChatRoom: React.FC = () => {
 
     const dispatch = useAppDispatch();
     const scrollViewTargetRef = React.useRef<HTMLDivElement>(null);
-    const scrollTargetRef = React.useRef<HTMLDivElement>(null);
     const [shouldAutoScroll, setShouldAutoScroll] = React.useState(true);
 
     const [isDraggingOver, setIsDraggingOver] = React.useState(false);
@@ -65,7 +64,7 @@ export const ChatRoom: React.FC = () => {
 
     React.useEffect(() => {
         if (!shouldAutoScroll) return;
-        scrollToTarget(scrollTargetRef.current);
+        scrollViewTargetRef.current?.scrollTo(0, scrollViewTargetRef.current.scrollHeight);
     }, [messages, shouldAutoScroll]);
 
     React.useEffect(() => {
@@ -98,7 +97,7 @@ export const ChatRoom: React.FC = () => {
             authorRole: AuthorRoles.User,
         };
 
-        dispatch(updateConversationFromUser({ message: chatInput }));
+        dispatch(addMessageToConversationFromUser({ message: chatInput, chatId: selectedId }));
 
         await chat.getResponse(options);
 
@@ -108,21 +107,13 @@ export const ChatRoom: React.FC = () => {
     return (
         <div className={classes.root} onDragEnter={onDragEnter} onDragOver={onDragEnter} onDragLeave={onDragLeave}>
             <div ref={scrollViewTargetRef} className={classes.scroll}>
-                <div ref={scrollViewTargetRef} className={classes.history}>
+                <div className={classes.history}>
                     <ChatHistory messages={messages} onGetResponse={handleSubmit} />
                 </div>
-                <div>
-                    <div ref={scrollTargetRef} />
-                </div>
             </div>
             <div className={classes.input}>
                 <ChatInput isDraggingOver={isDraggingOver} onDragLeave={onDragLeave} onSubmit={handleSubmit} />
             </div>
         </div>
     );
-};
-
-const scrollToTarget = (element: HTMLElement | null) => {
-    if (!element) return;
-    element.scrollIntoView({ block: 'start', behavior: 'smooth' });
-};
+};

webapp/src/components/chat/chat-history/UserFeedbackActions.tsx

@@ -3,7 +3,7 @@ import { useCallback } from 'react';
 import { UserFeedback } from '../../../libs/models/ChatMessage';
 import { useAppDispatch, useAppSelector } from '../../../redux/app/hooks';
 import { RootState } from '../../../redux/app/store';
-import { setUserFeedback } from '../../../redux/features/conversations/conversationsSlice';
+import { updateMessageProperty } from '../../../redux/features/conversations/conversationsSlice';
 import { ThumbDislike16, ThumbLike16 } from '../../shared/BundledIcons';
 
 const useClasses = makeStyles({
@@ -27,10 +27,12 @@ export const UserFeedbackActions: React.FC<IUserFeedbackProps> = ({ messageIndex
     const onUserFeedbackProvided = useCallback(
         (positive: boolean) => {
             dispatch(
-                setUserFeedback({
-                    userFeedback: positive ? UserFeedback.Positive : UserFeedback.Negative,
-                    messageIndex,
+                updateMessageProperty({
                     chatId: selectedId,
+                    messageIdOrIndex: messageIndex,
+                    property: 'userFeedback',
+                    value: positive ? UserFeedback.Positive : UserFeedback.Negative,
+                    frontLoad: true,
                 }),
             );
         },

webapp/src/components/chat/plan-viewer/PlanViewer.tsx

@@ -7,7 +7,7 @@ import { IAskVariables } from '../../../libs/semantic-kernel/model/Ask';
 import { GetResponseOptions } from '../../../libs/useChat';
 import { useAppDispatch, useAppSelector } from '../../../redux/app/hooks';
 import { RootState } from '../../../redux/app/store';
-import { updateMessageState } from '../../../redux/features/conversations/conversationsSlice';
+import { updateMessageProperty } from '../../../redux/features/conversations/conversationsSlice';
 import { PlanStepCard } from './PlanStepCard';
 
 const useClasses = makeStyles({
@@ -61,7 +61,7 @@ export const PlanViewer: React.FC<PlanViewerProps> = ({ message, messageIndex, g
     const parsedContent: Plan = JSON.parse(message.content);
     const originalPlan = parsedContent.proposedPlan;
 
-    const planState = message.state ?? parsedContent.state;
+    const planState = message.planState ?? parsedContent.state;
 
     // If plan came from ActionPlanner, use parameters from top-level of plan
     if (parsedContent.type === PlanType.Action) {
@@ -79,10 +79,12 @@ export const PlanViewer: React.FC<PlanViewerProps> = ({ message, messageIndex, g
 
     const onPlanAction = async (planState: PlanState.PlanApproved | PlanState.PlanRejected) => {
         dispatch(
-            updateMessageState({
-                newMessageState: planState,
-                messageIndex,
+            updateMessageProperty({
+                messageIdOrIndex: messageIndex,
                 chatId: selectedId,
+                property: 'planState',
+                value: planState,
+                frontLoad: true,
             }),
         );
 

webapp/src/libs/models/ChatMessage.ts

@@ -50,7 +50,7 @@ export interface IChatMessage {
     prompt?: string;
     authorRole: AuthorRoles;
     debug?: string;
-    state?: PlanState;
+    planState?: PlanState;
     // TODO: Persistent RLHF, view only right now
     userFeedback?: UserFeedback;
 }

webapp/src/libs/services/ChatService.ts

@@ -7,7 +7,6 @@ import { IChatParticipant } from '../models/ChatParticipant';
 import { IChatSession } from '../models/ChatSession';
 import { IChatUser } from '../models/ChatUser';
 import { IAsk, IAskVariables } from '../semantic-kernel/model/Ask';
-import { IAskResult } from '../semantic-kernel/model/AskResult';
 import { BaseService } from './BaseService';
 
 export class ChatService extends BaseService {
@@ -93,7 +92,7 @@ export class ChatService extends BaseService {
         ask: IAsk,
         accessToken: string,
         enabledPlugins?: Plugin[],
-    ): Promise<IAskResult> => {
+    ): Promise<IChatMessage> => {
         // If skill requires any additional api properties, append to context
         if (enabledPlugins && enabledPlugins.length > 0) {
             const openApiSkillVariables: IAskVariables[] = [];
@@ -122,7 +121,7 @@ export class ChatService extends BaseService {
             ask.variables = ask.variables ? ask.variables.concat(openApiSkillVariables) : openApiSkillVariables;
         }
 
-        const result = await this.getResponseAsync<IAskResult>(
+        const result = await this.getResponseAsync<IChatMessage>(
             {
                 commandPath: 'chat',
                 method: 'POST',

webapp/src/redux/features/conversations/conversationsSlice.ts

@@ -3,7 +3,6 @@
 import { createSlice, PayloadAction, Slice } from '@reduxjs/toolkit';
 import { ChatMessageType, IChatMessage, UserFeedback } from '../../../libs/models/ChatMessage';
 import { IChatUser } from '../../../libs/models/ChatUser';
-import { PlanState } from '../../../libs/models/Plan';
 import { ChatState } from './ChatState';
 import {
     ConversationInputChange,
@@ -51,37 +50,27 @@ export const conversationsSlice: Slice<ConversationsState> = createSlice({
             state.conversations[action.payload].userDataLoaded = true;
         },
         /*
-         * updateConversationFromUser() and updateConversationFromServer() both update the conversations state.
+         * addMessageToConversationFromUser() and addMessageToConversationFromServer() both update the conversations state.
          * However they are for different purposes. The former action is for updating the conversation from the
          * webapp and will be captured by the SignalR middleware and the payload will be broadcasted to all clients
          * in the same group.
-         * The updateConversationFromServer() action is triggered by the SignalR middleware when a response is received
+         * The addMessageToConversationFromServer() action is triggered by the SignalR middleware when a response is received
          * from the webapi.
          */
-        updateConversationFromUser: (
+        addMessageToConversationFromUser: (
             state: ConversationsState,
-            action: PayloadAction<{ message: IChatMessage; chatId?: string }>,
+            action: PayloadAction<{ message: IChatMessage; chatId: string }>,
         ) => {
             const { message, chatId } = action.payload;
-            const id = chatId ?? state.selectedId;
-            updateConversation(state, id, message);
+            updateConversation(state, chatId, message);
         },
-        updateConversationFromServer: (
+        addMessageToConversationFromServer: (
             state: ConversationsState,
             action: PayloadAction<{ message: IChatMessage; chatId: string }>,
         ) => {
             const { message, chatId } = action.payload;
             updateConversation(state, chatId, message);
         },
-        updateMessageState: (
-            state: ConversationsState,
-            action: PayloadAction<{ newMessageState: PlanState; messageIndex: number; chatId?: string }>,
-        ) => {
-            const { newMessageState, messageIndex, chatId } = action.payload;
-            const id = chatId ?? state.selectedId;
-            state.conversations[id].messages[messageIndex].state = newMessageState;
-            frontLoadChat(state, id);
-        },
         /*
          * updateUserIsTyping() and updateUserIsTypingFromServer() both update a user's typing state.
          * However they are for different purposes. The former action is for updating an user's typing state from
@@ -104,44 +93,41 @@ export const conversationsSlice: Slice<ConversationsState> = createSlice({
             const { userId, chatId, isTyping } = action.payload;
             updateUserTypingState(state, userId, chatId, isTyping);
         },
-        updateBotResponseStatusFromServer: (
+        updateBotResponseStatus: (
             state: ConversationsState,
             action: PayloadAction<{ chatId: string; status: string }>,
         ) => {
             const { chatId, status } = action.payload;
             const conversation = state.conversations[chatId];
             conversation.botResponseStatus = status;
         },
-        setUserFeedback: (
+        updateMessageProperty: <K extends keyof IChatMessage, V extends IChatMessage[K]>(
             state: ConversationsState,
-            action: PayloadAction<{ userFeedback: UserFeedback; messageIndex: number; chatId?: string }>,
+            action: PayloadAction<{
+                property: K;
+                value: V;
+                chatId: string;
+                messageIdOrIndex: string | number;
+                frontLoad?: boolean;
+            }>,
         ) => {
-            const { userFeedback, messageIndex, chatId } = action.payload;
-            const id = chatId ?? state.selectedId;
-            state.conversations[id].messages[messageIndex].userFeedback = userFeedback;
-            frontLoadChat(state, id);
+            const { property, value, messageIdOrIndex, chatId, frontLoad } = action.payload;
+            const conversation = state.conversations[chatId];
+            const conversationMessage =
+                typeof messageIdOrIndex === 'number'
+                    ? conversation.messages[messageIdOrIndex]
+                    : conversation.messages.find((m) => m.id === messageIdOrIndex);
+
+            if (conversationMessage) {
+                conversationMessage[property] = value;
+            }
+            if (frontLoad) {
+                frontLoadChat(state, chatId);
+            }
         },
     },
 });
 
-export const {
-    setConversations,
-    editConversationTitle,
-    editConversationInput,
-    setSelectedConversation,
-    addConversation,
-    updateConversationFromUser,
-    updateConversationFromServer,
-    updateMessageState,
-    updateUserIsTyping,
-    updateUserIsTypingFromServer,
-    updateBotResponseStatusFromServer,
-    setUsersLoaded,
-    setUserFeedback,
-} = conversationsSlice.actions;
-
-export default conversationsSlice.reducer;
-
 const frontLoadChat = (state: ConversationsState, id: string) => {
     const conversation = state.conversations[id];
     const { [id]: _, ...rest } = state.conversations;
@@ -164,3 +150,20 @@ const updateUserTypingState = (state: ConversationsState, userId: string, chatId
         user.isTyping = isTyping;
     }
 };
+
+export const {
+    setConversations,
+    editConversationTitle,
+    editConversationInput,
+    setSelectedConversation,
+    addConversation,
+    addMessageToConversationFromUser,
+    addMessageToConversationFromServer,
+    updateMessageProperty,
+    updateUserIsTyping,
+    updateUserIsTypingFromServer,
+    updateBotResponseStatus,
+    setUsersLoaded,
+} = conversationsSlice.actions;
+
+export default conversationsSlice.reducer;