[Feature] Add streaming ability in chat page using Azure SignalR service

app/Directory.Packages.props

@@ -52,5 +52,7 @@
     <PackageVersion Include="xunit" Version="2.9.0" />
     <PackageVersion Include="FluentAssertions" Version="6.12.0" />
     <PackageVersion Include="NSubstitute" Version="5.1.0" />
+    <PackageVersion Include="Microsoft.Azure.SignalR" Version="1.27.0" />
+    <PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.8" />
   </ItemGroup>
 </Project>

app/SharedWebComponents/Components/Answer.razor

@@ -28,11 +28,11 @@
                                 }
                             </div>
                         }
-                        @if (answer is { FollowupQuestions.Count: > 0 })
+                        @if (Retort?.Context?.FollowupQuestions is { Length: > 0 })
                         {
                             <div class="pt-4">
                                 <MudText Typo="Typo.subtitle2" Class="pb-2">Follow-up questions:</MudText>
-                                @foreach (var followup in answer.FollowupQuestions)
+                                @foreach (var followup in Retort.Context.FollowupQuestions)
                                 {
                                     <MudChip Variant="Variant.Text" Color="Color.Tertiary"
                                         OnClick="@(_ => OnAskFollowupAsync(followup))">

app/SharedWebComponents/Components/SettingsPanel.razor

@@ -41,6 +41,10 @@
                      Color="Color.Primary" 
                      Label="Use query-contextual summaries instead of whole documents" />
 
+        <MudCheckBox @bind-Checked="@Settings.Overrides.UseStreaming" Size="Size.Large"
+                     Color="Color.Primary" 
+                     Label="Use streaming responses" />
+
         <MudCheckBox @bind-Checked="@Settings.Overrides.SuggestFollowupQuestions" Size="Size.Large"
                      Color="Color.Primary" Label="Suggest follow-up questions"
                      aria-label="Suggest follow-up questions checkbox." />

app/SharedWebComponents/Models/StreamingMessage.cs

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace SharedWebComponents.Models;
+
+internal class StreamingMessage
+{
+    public string Type { get; set; } = "";
+    public object? Content { get; set; }
+} 

app/SharedWebComponents/Pages/Chat.razor

@@ -88,6 +88,10 @@
                                     OnClick="@OnClearChat" Disabled=@(_isReceivingResponse || _questionAndAnswerMap is { Count: 0 }) />
                         </MudTooltip>
                     </MudItem>
+                    <MudItem xs="12" Class="pa-2">
+                        <MudCheckBox @bind-Checked="_useStreaming" Label="Use streaming mode" Color="Color.Secondary"
+                            Disabled="@_isReceivingResponse" />
+                    </MudItem>
                 </MudGrid>
             </MudItem>
         </MudGrid>

app/SharedWebComponents/Pages/Chat.razor.cs

@@ -1,30 +1,173 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 namespace SharedWebComponents.Pages;
+using Microsoft.AspNetCore.SignalR.Client;
+using System.Text.Json;
+using SharedWebComponents.Models;
 
-public sealed partial class Chat
+public sealed partial class Chat : IAsyncDisposable
 {
     private string _userQuestion = "";
     private UserQuestion _currentQuestion;
     private string _lastReferenceQuestion = "";
     private bool _isReceivingResponse = false;
+    private bool _useStreaming = true;
+    private HubConnection? _hubConnection;
+    private string _streamingResponse = "";
 
     private readonly Dictionary<UserQuestion, ChatAppResponseOrError?> _questionAndAnswerMap = [];
 
     [Inject] public required ISessionStorageService SessionStorage { get; set; }
 
     [Inject] public required ApiClient ApiClient { get; set; }
 
+    [Inject] public required NavigationManager NavigationManager { get; set; }
+
     [CascadingParameter(Name = nameof(Settings))]
     public required RequestSettingsOverrides Settings { get; set; }
 
     [CascadingParameter(Name = nameof(IsReversed))]
     public required bool IsReversed { get; set; }
 
-    private Task OnAskQuestionAsync(string question)
+    protected override async Task OnInitializedAsync()
     {
-        _userQuestion = question;
-        return OnAskClickedAsync();
+        await ConnectToHub();
+    }
+
+    private async Task ConnectToHub()
+    {
+        if (_hubConnection?.State == HubConnectionState.Connected)
+        {
+            return;
+        }
+
+        _hubConnection = new HubConnectionBuilder()
+            .WithUrl(NavigationManager.ToAbsoluteUri("/chat-hub"))
+            .WithAutomaticReconnect(new[] { TimeSpan.Zero, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(30) })
+            .Build();
+
+        _hubConnection.On<string>("ReceiveMessage", (message) =>
+        {
+            try
+            {
+                if (_currentQuestion != default)
+                {
+                    var streamingMessage = JsonSerializer.Deserialize<StreamingMessage>(message);
+                    if (streamingMessage?.Content == null) return;
+
+                    switch (streamingMessage.Type.ToLowerInvariant())
+                    {
+                        case "content":
+                            _streamingResponse += streamingMessage.Content;
+                            UpdateAnswerInMap(_streamingResponse);
+                            break;
+
+                        case "answer":
+                            if (streamingMessage.Content is JsonElement answerElement)
+                            {
+                                var answer = answerElement.GetString();
+                                if (answer != null)
+                                {
+                                    _streamingResponse = answer;
+                                    UpdateAnswerInMap(answer);
+                                }
+                            }
+                            else if (streamingMessage.Content is string answerString)
+                            {
+                                _streamingResponse = answerString;
+                                UpdateAnswerInMap(answerString);
+                            }
+                            break;
+
+                        case "thoughts":
+                            if (streamingMessage.Content is JsonElement thoughtsElement)
+                            {
+                                var thoughts = thoughtsElement.EnumerateArray()
+                                    .Select(t => new Thoughts(
+                                        t.GetProperty("Title").GetString()!,
+                                        t.GetProperty("Description").GetString()!))
+                                    .ToArray();
+                                UpdateThoughtsInMap(thoughts);
+                            }
+                            break;
+
+                        case "followup":
+                            if (streamingMessage.Content is JsonElement followupElement)
+                            {
+                                var followupQuestions = followupElement.ValueKind == JsonValueKind.Array
+                                    ? followupElement.EnumerateArray()
+                                        .Select(q => q.GetString() ?? string.Empty)
+                                        .Where(q => !string.IsNullOrEmpty(q))
+                                        .ToArray()
+                                    : new[] { followupElement.GetString() ?? string.Empty };
+
+                                if (followupQuestions.Any())
+                                {
+                                    UpdateFollowupQuestionsInMap(followupQuestions);
+                                }
+                            }
+                            break;
+
+                        case "supporting":
+                            if (streamingMessage.Content is JsonElement supportingElement)
+                            {
+                                var supportingContent = supportingElement.EnumerateArray()
+                                    .Select(s => new SupportingContentRecord(
+                                        s.GetProperty("Title").GetString()!,
+                                        s.GetProperty("Description").GetString()!
+                                    ))
+                                    .ToArray();
+
+                                if (supportingContent.Any())
+                                {
+                                    UpdateSupportingContentInMap(supportingContent);
+                                }
+                            }
+                            break;
+
+                        case "images":
+                            if (streamingMessage.Content is JsonElement imagesElement)
+                            {
+                                var images = imagesElement.EnumerateArray()
+                                    .Select(i => new SupportingImageRecord(
+                                        i.GetProperty("Title").GetString()!,
+                                        i.GetProperty("Url").GetString()!))
+                                    .ToArray();
+
+                                if (images.Any())
+                                {
+                                    UpdateImagesInMap(images);
+                                }
+                            }
+                            break;
+
+                        case "complete":
+                            if (streamingMessage.Content is JsonElement completeElement)
+                            {
+                                var citationBaseUrl = completeElement.GetProperty("citationBaseUrl").GetString();
+                                UpdateAnswerInMap(_streamingResponse, citationBaseUrl);
+                                _userQuestion = "";
+                                _currentQuestion = default;
+                            }
+                            break;
+                    }
+                    StateHasChanged();
+                }
+            }
+            catch (JsonException ex)
+            {
+                Console.WriteLine($"Error deserializing response: {ex.Message}");
+            }
+        });
+
+        try
+        {
+            await _hubConnection.StartAsync();
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error starting SignalR connection: {ex.Message}");
+        }
     }
 
     private async Task OnAskClickedAsync()
@@ -38,24 +181,47 @@ private async Task OnAskClickedAsync()
         _lastReferenceQuestion = _userQuestion;
         _currentQuestion = new(_userQuestion, DateTime.Now);
         _questionAndAnswerMap[_currentQuestion] = null;
+        _streamingResponse = "";
 
         try
         {
             var history = _questionAndAnswerMap
-                .Where(x => x.Value?.Choices is { Length: > 0})
-                .SelectMany(x => new ChatMessage[] { new ChatMessage("user", x.Key.Question), new ChatMessage("assistant", x.Value!.Choices[0].Message.Content) })
+                .Where(x => x.Value?.Choices is { Length: > 0 })
+                .SelectMany(x => new ChatMessage[] {
+                    new ChatMessage("user", x.Key.Question),
+                    new ChatMessage("assistant", x.Value!.Choices[0].Message.Content)
+                })
                 .ToList();
 
             history.Add(new ChatMessage("user", _userQuestion));
 
             var request = new ChatRequest([.. history], Settings.Overrides);
-            var result = await ApiClient.ChatConversationAsync(request);
 
-            _questionAndAnswerMap[_currentQuestion] = result.Response;
-            if (result.IsSuccessful)
+            if (_useStreaming && _hubConnection?.State == HubConnectionState.Connected)
             {
-                _userQuestion = "";
-                _currentQuestion = default;
+                try
+                {
+                    await _hubConnection.InvokeAsync("SendChatRequest", request);
+                }
+                catch (Exception ex)
+                {
+                    _questionAndAnswerMap[_currentQuestion] = new ChatAppResponseOrError(
+                        Array.Empty<ResponseChoice>(),
+                        $"Error: {ex.Message}");
+                    _userQuestion = "";
+                    _currentQuestion = default;
+                }
+            }
+            else
+            {
+                var result = await ApiClient.ChatConversationAsync(request);
+                _questionAndAnswerMap[_currentQuestion] = result.Response;
+
+                if (_questionAndAnswerMap[_currentQuestion]?.Error == null)
+                {
+                    _userQuestion = "";
+                    _currentQuestion = default;
+                }
             }
         }
         finally
@@ -70,4 +236,101 @@ private void OnClearChat()
         _currentQuestion = default;
         _questionAndAnswerMap.Clear();
     }
-}
+
+    public async ValueTask DisposeAsync()
+    {
+        if (_hubConnection is not null)
+        {
+            await _hubConnection.DisposeAsync();
+        }
+    }
+
+    private async Task OnAskQuestionAsync(string question)
+    {
+        _userQuestion = question;
+        await OnAskClickedAsync();
+    }
+
+    private void UpdateAnswerInMap(string answer, string? citationBaseUrl = null)
+    {
+        var currentResponse = _questionAndAnswerMap[_currentQuestion];
+        var choice = currentResponse?.Choices.FirstOrDefault();
+        var context = choice?.Context ?? new ResponseContext(null, null, Array.Empty<string>(), Array.Empty<Thoughts>());
+
+        _questionAndAnswerMap[_currentQuestion] = new ChatAppResponseOrError(new[] {
+            new ResponseChoice(
+                Index: 0,
+                Message: new ResponseMessage("assistant", answer),
+                Context: context,
+                CitationBaseUrl: citationBaseUrl ?? choice?.CitationBaseUrl ?? "")
+        });
+    }
+
+    private void UpdateThoughtsInMap(Thoughts[] thoughts)
+    {
+        var currentResponse = _questionAndAnswerMap[_currentQuestion];
+        if (currentResponse?.Choices.FirstOrDefault() is { } choice)
+        {
+            var context = new ResponseContext(
+                choice.Context.DataPointsContent,
+                choice.Context.DataPointsImages,
+                choice.Context.FollowupQuestions,
+                thoughts);
+
+            _questionAndAnswerMap[_currentQuestion] = new ChatAppResponseOrError(new[] {
+                choice with { Context = context }
+            });
+        }
+    }
+
+    private void UpdateFollowupQuestionsInMap(string[] followupQuestions)
+    {
+        var currentResponse = _questionAndAnswerMap[_currentQuestion];
+        if (currentResponse?.Choices.FirstOrDefault() is { } choice)
+        {
+            var context = new ResponseContext(
+                choice.Context.DataPointsContent,
+                choice.Context.DataPointsImages,
+                followupQuestions,
+                choice.Context.Thoughts);
+
+            _questionAndAnswerMap[_currentQuestion] = new ChatAppResponseOrError(new[] {
+                choice with { Context = context }
+            });
+        }
+    }
+
+    private void UpdateSupportingContentInMap(SupportingContentRecord[] supportingContent)
+    {
+        var currentResponse = _questionAndAnswerMap[_currentQuestion];
+        if (currentResponse?.Choices.FirstOrDefault() is { } choice)
+        {
+            var context = new ResponseContext(
+                supportingContent,
+                choice.Context.DataPointsImages,
+                choice.Context.FollowupQuestions,
+                choice.Context.Thoughts);
+
+            _questionAndAnswerMap[_currentQuestion] = new ChatAppResponseOrError(new[] {
+                choice with { Context = context }
+            });
+        }
+    }
+
+    private void UpdateImagesInMap(SupportingImageRecord[] images)
+    {
+        var currentResponse = _questionAndAnswerMap[_currentQuestion];
+        if (currentResponse?.Choices.FirstOrDefault() is { } choice)
+        {
+            var context = new ResponseContext(
+                choice.Context.DataPointsContent,
+                images,
+                choice.Context.FollowupQuestions,
+                choice.Context.Thoughts);
+
+            _questionAndAnswerMap[_currentQuestion] = new ChatAppResponseOrError(new[] {
+                choice with { Context = context }
+            });
+        }
+    }
+}