From a7e7ff21cea61532a64e95072ce0f121415d1d0b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 14 Mar 2024 13:08:29 +1100 Subject: [PATCH 1/4] Add full-width and ideographic punctuation --- app/shared/Shared/Services/AzureSearchEmbedService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/shared/Shared/Services/AzureSearchEmbedService.cs b/app/shared/Shared/Services/AzureSearchEmbedService.cs index ab31d2b8..b1feb683 100644 --- a/app/shared/Shared/Services/AzureSearchEmbedService.cs +++ b/app/shared/Shared/Services/AzureSearchEmbedService.cs @@ -322,8 +322,8 @@ public IEnumerable
CreateSections( const int SentenceSearchLimit = 100; const int SectionOverlap = 100; - var sentenceEndings = new[] { '.', '!', '?' }; - var wordBreaks = new[] { ',', ';', ':', ' ', '(', ')', '[', ']', '{', '}', '\t', '\n' }; + var sentenceEndings = new[] { '.', '。', '．', '!', '?', '‼', '⁇', '⁈', '⁉' }; + var wordBreaks = new[] { ',', '、', ';', ':', ' ', '(', ')', '[', ']', '{', '}', '\t', '\n' }; var allText = string.Concat(pageMap.Select(p => p.Text)); var length = allText.Length; var start = 0; From df75879fdabcca525a0eb5db096cd75d47a1b8c6 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 5 Aug 2024 17:56:30 +1000 Subject: [PATCH 2/4] add otel for OLTP exports --- app/backend/MinimalApi.csproj | 2 ++ app/backend/Program.cs | 29 +++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/app/backend/MinimalApi.csproj b/app/backend/MinimalApi.csproj index 04783d07..7b617016 100644 --- a/app/backend/MinimalApi.csproj +++ b/app/backend/MinimalApi.csproj @@ -17,6 +17,7 @@ + @@ -25,6 +26,7 @@ + diff --git a/app/backend/Program.cs b/app/backend/Program.cs index ecada14c..09e1094d 100644 --- a/app/backend/Program.cs +++ b/app/backend/Program.cs @@ -1,6 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. +using Azure.Monitor.OpenTelemetry.Exporter; using Microsoft.AspNetCore.Antiforgery; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; var builder = WebApplication.CreateBuilder(args); @@ -17,14 +22,30 @@ builder.Services.AddAntiforgery(options => { options.HeaderName = "X-CSRF-TOKEN-HEADER"; options.FormFieldName = "X-CSRF-TOKEN-FORM"; }); builder.Services.AddHttpClient(); +static string? GetEnvVar(string key) => Environment.GetEnvironmentVariable(key); + if (builder.Environment.IsDevelopment()) { builder.Services.AddDistributedMemoryCache(); + + builder.Logging.AddOpenTelemetry( + options => + { + options.AddOtlpExporter(); + } + ); + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("Microsoft.SemanticKernel*") + .AddOtlpExporter() + .Build(); + + using var traceProvider = Sdk.CreateTracerProviderBuilder() + .AddSource("Microsoft.SemanticKernel*") + .AddOtlpExporter() + .Build(); } else { - static string? GetEnvVar(string key) => Environment.GetEnvironmentVariable(key); - builder.Services.AddStackExchangeRedisCache(options => { var name = builder.Configuration["AzureRedisCacheName"] + @@ -62,6 +83,10 @@ { option.ConnectionString = appInsightsConnectionString; }); + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("Microsoft.SemanticKernel*") + .AddAzureMonitorMetricExporter(options => options.ConnectionString = appInsightsConnectionString) + .Build(); } } From 90653acfe49d277619e9e33768d1182d573f9813 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 5 Aug 2024 19:25:36 +1000 Subject: [PATCH 3/4] Setup default GRPC/OTLP tracing, metrics and logs for local development --- app/Directory.Packages.props | 4 ++++ app/backend/MinimalApi.csproj | 2 ++ app/backend/Program.cs | 28 ++++++++++++++++++++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/app/Directory.Packages.props b/app/Directory.Packages.props index 602cb7e7..12f59475 100644 --- a/app/Directory.Packages.props +++ b/app/Directory.Packages.props @@ -8,6 +8,7 @@ + @@ -43,6 +44,9 @@ + + + diff --git a/app/backend/MinimalApi.csproj b/app/backend/MinimalApi.csproj index 7b617016..3bed56e7 100644 --- a/app/backend/MinimalApi.csproj +++ b/app/backend/MinimalApi.csproj @@ -27,6 +27,8 @@ + + diff --git a/app/backend/Program.cs b/app/backend/Program.cs index 09e1094d..f31d0d20 100644 --- a/app/backend/Program.cs +++ b/app/backend/Program.cs @@ -3,8 +3,10 @@ using Azure.Monitor.OpenTelemetry.Exporter; using Microsoft.AspNetCore.Antiforgery; using OpenTelemetry; +using OpenTelemetry.Exporter; using OpenTelemetry.Logs; using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; using OpenTelemetry.Trace; var builder = WebApplication.CreateBuilder(args); @@ -27,21 +29,39 @@ if (builder.Environment.IsDevelopment()) { builder.Services.AddDistributedMemoryCache(); - + var defaultEndpoint = GetEnvVar("OTEL_EXPORTER_OTLP_ENDPOINT") ?? "http://localhost:4317"; builder.Logging.AddOpenTelemetry( options => { - options.AddOtlpExporter(); + options.AddOtlpExporter(config => { + config.Protocol = OtlpExportProtocol.Grpc; + config.Endpoint = new Uri(defaultEndpoint); + }); } ); using var meterProvider = Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("SearchDemo")) .AddMeter("Microsoft.SemanticKernel*") - .AddOtlpExporter() + .AddMeter("Azure.*") + .AddOtlpExporter(config => + { + config.Protocol = OtlpExportProtocol.Grpc; + config.Endpoint = new Uri(defaultEndpoint); + }) .Build(); using var traceProvider = Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("SearchDemo")) .AddSource("Microsoft.SemanticKernel*") - .AddOtlpExporter() + .AddSource("Azure.*") + .AddSource("Microsoft.ML.*") + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddOtlpExporter(config => + { + config.Protocol = OtlpExportProtocol.Grpc; + config.Endpoint = new Uri(defaultEndpoint); + }) .Build(); } else From 4aac46f8ab9f4dbdbcd5cb67fc91355cc1c19c5c Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 5 Aug 2024 19:36:55 +1000 Subject: [PATCH 4/4] Cleanup OTEL services after app host has finished. --- app/backend/Program.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/app/backend/Program.cs b/app/backend/Program.cs index f31d0d20..580f6af5 100644 --- a/app/backend/Program.cs +++ b/app/backend/Program.cs @@ -24,6 +24,7 @@ builder.Services.AddAntiforgery(options => { options.HeaderName = "X-CSRF-TOKEN-HEADER"; options.FormFieldName = "X-CSRF-TOKEN-FORM"; }); builder.Services.AddHttpClient(); +List disposables = []; static string? GetEnvVar(string key) => Environment.GetEnvironmentVariable(key); if (builder.Environment.IsDevelopment()) @@ -39,7 +40,7 @@ }); } ); - using var meterProvider = Sdk.CreateMeterProviderBuilder() + var meterProvider = Sdk.CreateMeterProviderBuilder() .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("SearchDemo")) .AddMeter("Microsoft.SemanticKernel*") .AddMeter("Azure.*") @@ -49,9 +50,8 @@ config.Endpoint = new Uri(defaultEndpoint); }) .Build(); - - using var traceProvider = Sdk.CreateTracerProviderBuilder() - .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("SearchDemo")) + disposables.Add(meterProvider); + var traceProvider = Sdk.CreateTracerProviderBuilder() .AddSource("Microsoft.SemanticKernel*") .AddSource("Azure.*") .AddSource("Microsoft.ML.*") @@ -63,6 +63,7 @@ config.Endpoint = new Uri(defaultEndpoint); }) .Build(); + disposables.Add(traceProvider); } else { @@ -92,8 +93,6 @@ {name},abortConnect=false,ssl={ssl},allowAdmin=true,password={key} """; options.InstanceName = "content"; - - }); // set application telemetry @@ -103,10 +102,11 @@ { option.ConnectionString = appInsightsConnectionString; }); - using var meterProvider = Sdk.CreateMeterProviderBuilder() + var meterProvider = Sdk.CreateMeterProviderBuilder() .AddMeter("Microsoft.SemanticKernel*") .AddAzureMonitorMetricExporter(options => options.ConnectionString = appInsightsConnectionString) .Build(); + disposables.Add(meterProvider); } } @@ -145,3 +145,8 @@ app.MapApi(); app.Run(); + +foreach (var d in disposables) +{ + d.Dispose(); +}