Extend OpenTelemetry capabilities for Local Development

app/Directory.Packages.props

@@ -8,6 +8,7 @@
     <PackageVersion Include="Azure.AI.OpenAI" Version="1.0.0-beta.17" />
     <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.0" />
     <PackageVersion Include="Azure.Identity" Version="1.11.4" />
+    <PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.3.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.5.1" />
     <PackageVersion Include="Azure.Storage.Blobs" Version="12.19.1" />
     <PackageVersion Include="Azure.Storage.Files.Shares" Version="12.17.1" />
@@ -43,6 +44,9 @@
     <PackageVersion Include="Microsoft.SemanticKernel" Version="1.15.0" />
     <PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
     <PackageVersion Include="MudBlazor" Version="6.11.1" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
     <PackageVersion Include="PdfSharpCore" Version="1.3.62" />
     <PackageVersion Include="Pinecone.NET" Version="1.3.2" />
     <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.5.0" />

app/backend/MinimalApi.csproj

@@ -17,6 +17,7 @@
     <PackageReference Include="Azure.AI.OpenAI" />
     <PackageReference Include="Azure.Extensions.AspNetCore.Configuration.Secrets" />
     <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" />
     <PackageReference Include="Azure.Search.Documents" />
     <PackageReference Include="Azure.Storage.Blobs" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
@@ -25,6 +26,9 @@
     <PackageReference Include="Microsoft.ML" />
     <PackageReference Include="Microsoft.SemanticKernel" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" />
     <PackageReference Include="PdfSharpCore" />
     <PackageReference Include="Swashbuckle.AspNetCore" />
   </ItemGroup>

app/backend/Program.cs

@@ -1,6 +1,13 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using Azure.Monitor.OpenTelemetry.Exporter;
 using Microsoft.AspNetCore.Antiforgery;
+using OpenTelemetry;
+using OpenTelemetry.Exporter;
+using OpenTelemetry.Logs;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -17,14 +24,49 @@
 builder.Services.AddAntiforgery(options => { options.HeaderName = "X-CSRF-TOKEN-HEADER"; options.FormFieldName = "X-CSRF-TOKEN-FORM"; });
 builder.Services.AddHttpClient();
 
+List<IDisposable> disposables = [];
+static string? GetEnvVar(string key) => Environment.GetEnvironmentVariable(key);
+
 if (builder.Environment.IsDevelopment())
 {
     builder.Services.AddDistributedMemoryCache();
+    var defaultEndpoint = GetEnvVar("OTEL_EXPORTER_OTLP_ENDPOINT") ?? "http://localhost:4317";
+    builder.Logging.AddOpenTelemetry(
+        options =>
+        {
+            options.AddOtlpExporter(config => {
+                config.Protocol = OtlpExportProtocol.Grpc;
+                config.Endpoint = new Uri(defaultEndpoint);
+            });
+        }
+    );
+    var meterProvider = Sdk.CreateMeterProviderBuilder()
+        .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("SearchDemo"))
+        .AddMeter("Microsoft.SemanticKernel*")
+        .AddMeter("Azure.*")
+        .AddOtlpExporter(config =>
+        {
+            config.Protocol = OtlpExportProtocol.Grpc;
+            config.Endpoint = new Uri(defaultEndpoint);
+        })
+        .Build();
+    disposables.Add(meterProvider);
+    var traceProvider = Sdk.CreateTracerProviderBuilder()
+        .AddSource("Microsoft.SemanticKernel*")
+        .AddSource("Azure.*")
+        .AddSource("Microsoft.ML.*")
+        .AddAspNetCoreInstrumentation()
+        .AddHttpClientInstrumentation()
+        .AddOtlpExporter(config =>
+        {
+            config.Protocol = OtlpExportProtocol.Grpc;
+            config.Endpoint = new Uri(defaultEndpoint);
+        })
+        .Build();
+    disposables.Add(traceProvider);
 }
 else
 {
-    static string? GetEnvVar(string key) => Environment.GetEnvironmentVariable(key);
-
     builder.Services.AddStackExchangeRedisCache(options =>
     {
         var name = builder.Configuration["AzureRedisCacheName"] +
@@ -51,8 +93,6 @@
             {name},abortConnect=false,ssl={ssl},allowAdmin=true,password={key}
             """;
         options.InstanceName = "content";
-
-
     });
 
     // set application telemetry
@@ -62,6 +102,11 @@
         {
             option.ConnectionString = appInsightsConnectionString;
         });
+        var meterProvider = Sdk.CreateMeterProviderBuilder()
+            .AddMeter("Microsoft.SemanticKernel*")
+            .AddAzureMonitorMetricExporter(options => options.ConnectionString = appInsightsConnectionString)
+            .Build();
+        disposables.Add(meterProvider);
     }
 }
 
@@ -100,3 +145,8 @@
 app.MapApi();
 
 app.Run();
+
+foreach (var d in disposables)
+{
+    d.Dispose();
+}

app/shared/Shared/Services/AzureSearchEmbedService.cs

@@ -322,8 +322,8 @@ public IEnumerable<Section> CreateSections(
         const int SentenceSearchLimit = 100;
         const int SectionOverlap = 100;
 
-        var sentenceEndings = new[] { '.', '!', '?' };
-        var wordBreaks = new[] { ',', ';', ':', ' ', '(', ')', '[', ']', '{', '}', '\t', '\n' };
+        var sentenceEndings = new[] { '.', '。', '．', '!', '?', '‼', '⁇', '⁈', '⁉' };
+        var wordBreaks = new[] { ',', '、', ';', ':', ' ', '(', ')', '[', ']', '{', '}', '\t', '\n' };
         var allText = string.Concat(pageMap.Select(p => p.Text));
         var length = allText.Length;
         var start = 0;