com.openai.unity 8.2.0 (#277)

- Added structured output support - Added support for Azure OpenAI assistants - Fixed Azure OpenAI Id parsing for events - Fixed Assistant.CreateThreadAndRunAsync to properly copy assistant parameters - Removed stream from CreateThreadAndRunRequest and CreateRunRequest - They were overridden by the presence of IStreamEventHandler anyway - com.utilities.rest -> 3.2.3 - com.utilities.encoder.wav -> 1.2.2
RageAgainstThePixel · Aug 18, 2024 · b087171 · b087171
1 parent 75ad592
commit b087171
Show file tree

Hide file tree

Showing 30 changed files with 817 additions and 249 deletions.
diff --git a/Documentation~/README.md b/Documentation~/README.md
@@ -10,8 +10,6 @@ An OpenAI API account is required.
 
 ***All copyrights, trademarks, logos, and assets are the property of their respective owners.***
 
-> This repository is available to transfer to the OpenAI organization if they so choose to accept it.
-
 ## Installing
 
 Requires Unity 2021.3 LTS or higher.
@@ -64,17 +62,17 @@ The recommended installation method is though the unity package manager and [Ope
   - [List Models](#list-models)
   - [Retrieve Models](#retrieve-model)
   - [Delete Fine Tuned Model](#delete-fine-tuned-model)
-- [Assistants](#assistants) :warning: :construction:
+- [Assistants](#assistants)
   - [List Assistants](#list-assistants)
   - [Create Assistant](#create-assistant)
   - [Retrieve Assistant](#retrieve-assistant)
   - [Modify Assistant](#modify-assistant)
   - [Delete Assistant](#delete-assistant)
-  - [Assistant Streaming](#assistant-streaming) :warning: :construction:
-  - [Threads](#threads) :warning: :construction:
+  - [Assistant Streaming](#assistant-streaming)
+  - [Threads](#threads)
     - [Create Thread](#create-thread)
     - [Create Thread and Run](#create-thread-and-run)
-      - [Streaming](#create-thread-and-run-streaming) :warning: :construction:
+      - [Streaming](#create-thread-and-run-streaming)
     - [Retrieve Thread](#retrieve-thread)
     - [Modify Thread](#modify-thread)
     - [Delete Thread](#delete-thread)
@@ -86,10 +84,11 @@ The recommended installation method is though the unity package manager and [Ope
     - [Thread Runs](#thread-runs)
       - [List Runs](#list-thread-runs)
       - [Create Run](#create-thread-run)
-        - [Streaming](#create-thread-run-streaming) :warning: :construction:
+        - [Streaming](#create-thread-run-streaming)
       - [Retrieve Run](#retrieve-thread-run)
       - [Modify Run](#modify-thread-run)
       - [Submit Tool Outputs to Run](#thread-submit-tool-outputs-to-run)
+      - [Structured Outputs](#thread-structured-outputs) :new:
       - [List Run Steps](#list-thread-run-steps)
       - [Retrieve Run Step](#retrieve-thread-run-step)
       - [Cancel Run](#cancel-thread-run)
@@ -114,13 +113,14 @@ The recommended installation method is though the unity package manager and [Ope
   - [Streaming](#chat-streaming)
   - [Tools](#chat-tools)
   - [Vision](#chat-vision)
+  - [Json Schema](#chat-json-schema) :new:
   - [Json Mode](#chat-json-mode)
 - [Audio](#audio)
   - [Create Speech](#create-speech)
     - [Stream Speech](#stream-speech)
   - [Create Transcription](#create-transcription)
   - [Create Translation](#create-translation)
-- [Images](#images) :warning: :construction:
+- [Images](#images)
   - [Create Image](#create-image)
   - [Edit Image](#edit-image)
   - [Create Image Variation](#create-image-variation)
@@ -306,31 +306,22 @@ This setup allows your front end application to securely communicate with your b
 
 #### Back End Example
 
-In this example, we demonstrate how to set up and use `OpenAIProxyStartup` in a new ASP.NET Core web app. The proxy server will handle authentication and forward requests to the OpenAI API, ensuring that your API keys and other sensitive information remain secure.
+In this example, we demonstrate how to set up and use `OpenAIProxy` in a new ASP.NET Core web app. The proxy server will handle authentication and forward requests to the OpenAI API, ensuring that your API keys and other sensitive information remain secure.
 
 1. Create a new [ASP.NET Core minimal web API](https://learn.microsoft.com/en-us/aspnet/core/tutorials/min-web-api?view=aspnetcore-6.0) project.
 2. Add the OpenAI-DotNet nuget package to your project.
     - Powershell install: `Install-Package OpenAI-DotNet-Proxy`
+    - Dotnet install: `dotnet add package OpenAI-DotNet-Proxy`
     - Manually editing .csproj: `<PackageReference Include="OpenAI-DotNet-Proxy" />`
 3. Create a new class that inherits from `AbstractAuthenticationFilter` and override the `ValidateAuthentication` method. This will implement the `IAuthenticationFilter` that you will use to check user session token against your internal server.
-4. In `Program.cs`, create a new proxy web application by calling `OpenAIProxyStartup.CreateDefaultHost` method, passing your custom `AuthenticationFilter` as a type argument.
+4. In `Program.cs`, create a new proxy web application by calling `OpenAIProxy.CreateWebApplication` method, passing your custom `AuthenticationFilter` as a type argument.
 5. Create `OpenAIAuthentication` and `OpenAIClientSettings` as you would normally with your API keys, org id, or Azure settings.
 
 ```csharp
 public partial class Program
 {
     private class AuthenticationFilter : AbstractAuthenticationFilter
     {
-        public override void ValidateAuthentication(IHeaderDictionary request)
-        {
-            // You will need to implement your own class to properly test
-            // custom issued tokens you've setup for your end users.
-            if (!request.Authorization.ToString().Contains(TestUserToken))
-            {
-                throw new AuthenticationException("User is not authorized");
-            }
-        }
-
         public override async Task ValidateAuthenticationAsync(IHeaderDictionary request)
         {
             await Task.CompletedTask; // remote resource call
@@ -349,7 +340,7 @@ public partial class Program
         var auth = OpenAIAuthentication.LoadFromEnv();
         var settings = new OpenAIClientSettings(/* your custom settings if using Azure OpenAI */);
         using var openAIClient = new OpenAIClient(auth, settings);
-        OpenAIProxyStartup.CreateWebApplication<AuthenticationFilter>(args, openAIClient).Run();
+        OpenAIProxy.CreateWebApplication<AuthenticationFilter>(args, openAIClient).Run();
     }
 }
 ```
@@ -814,6 +805,87 @@ foreach (var message in messages.Items.OrderBy(response => response.CreatedAt))
 }
 ```
 
+##### [Thread Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
+
+Structured Outputs is the evolution of JSON mode. While both ensure valid JSON is produced, only Structured Outputs ensure schema adherence.
+
+> [!IMPORTANT]
+>
+> - When using JSON mode, always instruct the model to produce JSON via some message in the conversation, for example via your system message. If you don't include an explicit instruction to generate JSON, the model may generate an unending stream of whitespace and the request may run continually until it reaches the token limit. To help ensure you don't forget, the API will throw an error if the string "JSON" does not appear somewhere in the context.
+> - The JSON in the message the model returns may be partial (i.e. cut off) if `finish_reason` is length, which indicates the generation exceeded max_tokens or the conversation exceeded the token limit. To guard against this, check `finish_reason` before parsing the response.
+
+```csharp
+var mathSchema = new JsonSchema("math_response", @"
+{
+  ""type"": ""object"",
+  ""properties"": {
+    ""steps"": {
+      ""type"": ""array"",
+      ""items"": {
+        ""type"": ""object"",
+        ""properties"": {
+          ""explanation"": {
+            ""type"": ""string""
+          },
+          ""output"": {
+            ""type"": ""string""
+          }
+        },
+        ""required"": [
+          ""explanation"",
+          ""output""
+        ],
+        ""additionalProperties"": false
+      }
+    },
+    ""final_answer"": {
+      ""type"": ""string""
+    }
+  },
+  ""required"": [
+    ""steps"",
+    ""final_answer""
+  ],
+  ""additionalProperties"": false
+}");
+var assistant = await OpenAIClient.AssistantsEndpoint.CreateAssistantAsync(
+    new CreateAssistantRequest(
+        name: "Math Tutor",
+        instructions: "You are a helpful math tutor. Guide the user through the solution step by step.",
+        model: "gpt-4o-2024-08-06",
+        jsonSchema: mathSchema));
+ThreadResponse thread = null;
+
+try
+{
+    var run = await assistant.CreateThreadAndRunAsync("how can I solve 8x + 7 = -23",
+        async @event =>
+        {
+            Debug.Log(@event.ToJsonString());
+            await Task.CompletedTask;
+        });
+    thread = await run.GetThreadAsync();
+    run = await run.WaitForStatusChangeAsync();
+    Debug.Log($"Created thread and run: {run.ThreadId} -> {run.Id} -> {run.CreatedAt}");
+    var messages = await thread.ListMessagesAsync();
+
+    foreach (var response in messages.Items)
+    {
+        Debug.Log($"{response.Role}: {response.PrintContent()}");
+    }
+}
+finally
+{
+    await assistant.DeleteAsync(deleteToolResources: thread == null);
+
+    if (thread != null)
+    {
+        var isDeleted = await thread.DeleteAsync(deleteToolResources: true);
+        Assert.IsTrue(isDeleted);
+    }
+}
+```
+
 ###### [List Thread Run Steps](https://platform.openai.com/docs/api-reference/runs/listRunSteps)
 
 Returns a list of run steps belonging to a run.
@@ -1061,7 +1133,7 @@ var chatRequest = new ChatRequest(messages);
 var response = await api.ChatEndpoint.StreamCompletionAsync(chatRequest, async partialResponse =>
 {
     Debug.Log(partialResponse.FirstChoice.Delta.ToString());
-    await Task.Completed;
+    await Task.CompletedTask;
 });
 var choice = response.FirstChoice;
 Debug.Log($"[{choice.Index}] {choice.Message.Role}: {choice.Message} | Finish Reason: {choice.FinishReason}");
@@ -1179,10 +1251,67 @@ var result = await api.ChatEndpoint.GetCompletionAsync(chatRequest);
 Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice} | Finish Reason: {result.FirstChoice.FinishDetails}");
 ```
 
-#### [Chat Json Mode](https://platform.openai.com/docs/guides/text-generation/json-mode)
+#### [Chat Json Schema](https://platform.openai.com/docs/guides/structured-outputs)
 
-> [!WARNING]
-> Beta Feature. API subject to breaking changes.
+The evolution of  [Json Mode](#chat-json-mode). While both ensure valid JSON is produced, only Structured Outputs ensure schema adherence.
+
+> [!IMPORTANT]
+>
+> - When using JSON mode, always instruct the model to produce JSON via some message in the conversation, for example via your system message. If you don't include an explicit instruction to generate JSON, the model may generate an unending stream of whitespace and the request may run continually until it reaches the token limit. To help ensure you don't forget, the API will throw an error if the string "JSON" does not appear somewhere in the context.
+> - The JSON in the message the model returns may be partial (i.e. cut off) if `finish_reason` is length, which indicates the generation exceeded max_tokens or the conversation exceeded the token limit. To guard against this, check `finish_reason` before parsing the response.
+
+```csharp
+var messages = new List<Message>
+{
+    new(Role.System, "You are a helpful math tutor. Guide the user through the solution step by step."),
+    new(Role.User, "how can I solve 8x + 7 = -23")
+};
+
+var mathSchema = new JsonSchema("math_response", @"
+{
+  ""type"": ""object"",
+  ""properties"": {
+    ""steps"": {
+      ""type"": ""array"",
+      ""items"": {
+        ""type"": ""object"",
+        ""properties"": {
+          ""explanation"": {
+            ""type"": ""string""
+          },
+          ""output"": {
+            ""type"": ""string""
+          }
+        },
+        ""required"": [
+          ""explanation"",
+          ""output""
+        ],
+        ""additionalProperties"": false
+      }
+    },
+    ""final_answer"": {
+      ""type"": ""string""
+    }
+  },
+  ""required"": [
+    ""steps"",
+    ""final_answer""
+  ],
+  ""additionalProperties"": false
+}");
+var chatRequest = new ChatRequest(messages, model: new("gpt-4o-2024-08-06"), jsonSchema: mathSchema);
+var response = await OpenAIClient.ChatEndpoint.GetCompletionAsync(chatRequest);
+
+foreach (var choice in response.Choices)
+{
+    Debug.Log($"[{choice.Index}] {choice.Message.Role}: {choice} | Finish Reason: {choice.FinishReason}");
+}
+
+response.GetUsage();
+```
+
+#### [Chat Json Mode](https://platform.openai.com/docs/guides/text-generation/json-mode)
 
 > [!IMPORTANT]
 >

diff --git a/Runtime/Assistants/AssistantExtensions.cs b/Runtime/Assistants/AssistantExtensions.cs
@@ -81,7 +81,21 @@ public static async Task<RunResponse> CreateThreadAndRunAsync(this AssistantResp
         /// <param name="cancellationToken">Optional, <see cref="CancellationToken"/>.</param>
         /// <returns><see cref="RunResponse"/>.</returns>
         public static async Task<RunResponse> CreateThreadAndRunAsync(this AssistantResponse assistant, CreateThreadRequest request = null, Func<IServerSentEvent, Task> streamEventHandler = null, CancellationToken cancellationToken = default)
-            => await assistant.Client.ThreadsEndpoint.CreateThreadAndRunAsync(new CreateThreadAndRunRequest(assistant.Id, createThreadRequest: request), streamEventHandler, cancellationToken);
+        {
+            var threadRunRequest = new CreateThreadAndRunRequest(
+                assistant.Id,
+                assistant.Model,
+                assistant.Instructions,
+                assistant.Tools,
+                assistant.ToolResources,
+                assistant.Metadata,
+                assistant.Temperature,
+                assistant.TopP,
+                jsonSchema: assistant.ResponseFormatObject?.JsonSchema,
+                responseFormat: assistant.ResponseFormat,
+                createThreadRequest: request);
+            return await assistant.Client.ThreadsEndpoint.CreateThreadAndRunAsync(threadRunRequest, streamEventHandler, cancellationToken);
+        }
 
         #region Tools
 

diff --git a/Runtime/Assistants/AssistantResponse.cs b/Runtime/Assistants/AssistantResponse.cs
@@ -1,7 +1,6 @@
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
 using Newtonsoft.Json;
-using OpenAI.Extensions;
 using System;
 using System.Collections.Generic;
 using UnityEngine.Scripting;
@@ -29,7 +28,7 @@ internal AssistantResponse(
             [JsonProperty("metadata")] Dictionary<string, string> metadata,
             [JsonProperty("temperature")] double temperature,
             [JsonProperty("top_p")] double topP,
-            [JsonProperty("response_format")][JsonConverter(typeof(ResponseFormatConverter))] ChatResponseFormat responseFormat)
+            [JsonProperty("response_format")] ResponseFormatObject responseFormat)
         {
             Id = id;
             Object = @object;
@@ -43,7 +42,7 @@ internal AssistantResponse(
             Metadata = metadata;
             Temperature = temperature;
             TopP = topP;
-            ResponseFormat = responseFormat;
+            ResponseFormatObject = responseFormat;
         }
 
         /// <summary>
@@ -173,8 +172,10 @@ internal AssistantResponse(
         /// </remarks>
         [Preserve]
         [JsonProperty("response_format")]
-        [JsonConverter(typeof(ResponseFormatConverter))]
-        public ChatResponseFormat ResponseFormat { get; }
+        public ResponseFormatObject ResponseFormatObject { get; }
+
+        [JsonIgnore]
+        public ChatResponseFormat ResponseFormat => ResponseFormatObject ?? ChatResponseFormat.Auto;
 
         [Preserve]
         public static implicit operator string(AssistantResponse assistant) => assistant?.Id;