From afeeec40b53f9b0454a6aae79a0e04c2ecf6852b Mon Sep 17 00:00:00 2001 From: dantelmomsft Date: Mon, 20 May 2024 12:03:07 +0200 Subject: [PATCH] support for image attachments --- .github/workflows/app-ci.yaml | 5 - .github/workflows/nightly-jobs.yaml | 6 - app/backend/manifest.yml | 11 - app/backend/pom.xml | 2 +- .../samples/assistant/agent/AgentContext.java | 18 +- .../agent/HistoryReportingAgent.java | 6 +- .../samples/assistant/agent/IntentAgent.java | 65 ++-- .../assistant/agent/IntentResponse.java | 10 +- .../samples/assistant/agent/PaymentAgent.java | 11 +- .../assistant/controller/ChatAppRequest.java | 2 + .../assistant/controller/ChatController.java | 42 ++- .../assistant/controller/ChatResponse.java | 17 +- .../assistant/controller/ResponseMessage.java | 4 +- .../controller/content/ContentController.java | 26 +- .../assistant/proxy/BlobStorageProxy.java | 9 +- app/frontend/.env.dev | 2 +- app/frontend/.env.local | 2 +- app/frontend/src/api/api.ts | 20 ++ app/frontend/src/api/models.ts | 1 + app/frontend/src/components/AttachmentType.ts | 4 + .../QuestionInput/QuestionContext.ts | 7 + .../QuestionInput/QuestionInput.module.css | 15 +- .../QuestionInput/QuestionInput.tsx | 61 +++- .../UserChatMessage.module.css | 10 + .../UserChatMessage/UserChatMessage.tsx | 22 +- app/frontend/src/index.tsx | 4 - app/frontend/src/pages/chat/Chat.tsx | 55 +-- app/frontend/src/pages/layout/Layout.tsx | 4 +- .../src/pages/oneshot/OneShot.module.css | 66 ---- app/frontend/src/pages/oneshot/OneShot.tsx | 324 ------------------ deploy/app-service/infra/main.bicep | 6 +- 31 files changed, 308 insertions(+), 529 deletions(-) delete mode 100644 app/backend/manifest.yml create mode 100644 app/frontend/src/components/AttachmentType.ts create mode 100644 app/frontend/src/components/QuestionInput/QuestionContext.ts delete mode 100644 app/frontend/src/pages/oneshot/OneShot.module.css delete mode 100644 app/frontend/src/pages/oneshot/OneShot.tsx diff --git a/.github/workflows/app-ci.yaml b/.github/workflows/app-ci.yaml index 9f56928..d72b8d5 100644 --- a/.github/workflows/app-ci.yaml +++ b/.github/workflows/app-ci.yaml @@ -51,11 +51,6 @@ jobs: mkdir -p ../backend/src/main/resources/static cp -r ./build/* ../backend/src/main/resources/static - - name: Verify Indexer project - run: | - echo "Testing indexer project." - cd ./app/indexer - mvn test - name: Build Spring Boot App run: | diff --git a/.github/workflows/nightly-jobs.yaml b/.github/workflows/nightly-jobs.yaml index 9350a1f..72345d5 100644 --- a/.github/workflows/nightly-jobs.yaml +++ b/.github/workflows/nightly-jobs.yaml @@ -91,12 +91,6 @@ jobs: java-version: '17' cache: 'maven' - - name: Verify Indexer project - run: | - echo "Testing indexer project." - cd ./app/indexer - mvn test - - name: Build Spring Boot App run: | echo "Building Spring Boot app." diff --git a/app/backend/manifest.yml b/app/backend/manifest.yml deleted file mode 100644 index f48c3b0..0000000 --- a/app/backend/manifest.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -applications: -- name: rest-service-guides - memory: 256M - instances: 1 - random-route: true - domain: cfapps.io - timeout: 180 - buildpack: java_buildpack -# For Maven target/gs-rest-service-cors-0.1.0.jar - path: build/libs/gs-rest-service-cors-0.1.0.jar diff --git a/app/backend/pom.xml b/app/backend/pom.xml index 6c82c3e..30103ba 100644 --- a/app/backend/pom.xml +++ b/app/backend/pom.xml @@ -18,7 +18,7 @@ 4.9.0 11.6.0-beta.8 - 1.0.1 + 1.0.2 4.5.1 3.11.0 diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/AgentContext.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/AgentContext.java index 27191c1..1afd566 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/AgentContext.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/AgentContext.java @@ -1,21 +1,25 @@ package com.microsoft.openai.samples.assistant.agent; -public class AgentContext { - private String result; +import java.util.HashMap; + +public class AgentContext extends HashMap{ + private String result; - public void AgentContext() { + public AgentContext() { + super(); } - public void AgentContext(String result) { - this.result = result; + public AgentContext(String result) { + super(); + this.put("result", result); } public String getResult() { - return result; + return (String)this.get("result"); } public void setResult(String result) { - this.result = result; + this.put("result", result); } } diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/HistoryReportingAgent.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/HistoryReportingAgent.java index 0a62215..2c682e4 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/HistoryReportingAgent.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/HistoryReportingAgent.java @@ -78,11 +78,9 @@ public HistoryReportingAgent(String azureClientKey, String clientEndpoint, Strin .build(); } - public AgentContext run (ChatHistory userChatHistory) { + public AgentContext run (ChatHistory userChatHistory, AgentContext agentContext){ LOGGER.info("======== HistoryAndTransaction Agent: Starting ========"); - AgentContext agentContext = new AgentContext(); - var agentChatHistory = new ChatHistory(HISTORY_AGENT_SYSTEM_MESSAGE); userChatHistory.forEach( chatMessageContent -> { if(chatMessageContent.getAuthorRole() != AuthorRole.SYSTEM) @@ -155,7 +153,7 @@ public static void main(String[] args) throws NoSuchMethodException { HistoryReportingAgent agent = new HistoryReportingAgent(AZURE_CLIENT_KEY, CLIENT_ENDPOINT, MODEL_ID); - agent.run(new ChatHistory()); + agent.run(new ChatHistory(), new AgentContext()); diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentAgent.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentAgent.java index 51c5e7b..94fd1bc 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentAgent.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentAgent.java @@ -11,6 +11,7 @@ import com.microsoft.semantickernel.orchestration.*; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; +import org.json.JSONException; import org.json.JSONObject; public class IntentAgent { @@ -22,26 +23,23 @@ public class IntentAgent { private ChatCompletionService chat; private String INTENT_SYSTEM_MESSAGE = """ - You are a personal financial advisor who help the user with their recurrent bill payments. - The user may want to pay the bill uploading a photo of the bill, or it may start the payment checking payments history for a specific payee. - In other cases it may want to just review the payments history. - Based on the conversation you need to identify the user intent and ask the user for the missing information. - The available intents are: - "BillPayment", "RepeatTransaction","TransactionHistory" - If none of the intents are identified provide the user with the list of the available intents. - - If an intent is identified return the output as json format as below - { - "intent": "BillPayment" - } - - If none of the intents are identified ask the user for more clarity and list of the available intents. Use always a json format as output - { - "intent": "None" - "clarify_sentence": "" - } - - Don't add any comments in the output or other characters, just the json format. +You are a personal financial advisor who help the user with their recurrent bill payments. +The user may want to pay the bill uploading a photo of the bill, or it may start the payment checking payments history for a specific payee. +In other cases it may want to just review the payments history. +Based on the conversation you need to identify the user intent. +The available intents are: +"BillPayment", "RepeatTransaction","TransactionHistory" +If none of the intents are identified provide the user with the list of the available intents. + +If an intent is identified return the output as json format as below +{ +"intent": "BillPayment" + } + +If you don't understand or if an intent is not identified be polite with the user, ask clarifying question also using the list the available intents. + +Don't add any comments in the output or other characters, just the use a json format. + """; public IntentAgent(OpenAIAsyncClient client, String modelId){ @@ -73,10 +71,8 @@ public IntentAgent(String azureClientKey, String clientEndpoint, String modelId) } public IntentResponse run(ChatHistory userChatHistory){ - var agentChatHistory = new ChatHistory(); + var agentChatHistory = new ChatHistory(INTENT_SYSTEM_MESSAGE); agentChatHistory.addAll(userChatHistory); - agentChatHistory.addSystemMessage(INTENT_SYSTEM_MESSAGE); - var messages = chat.getChatMessageContentsAsync( agentChatHistory, @@ -92,10 +88,27 @@ public IntentResponse run(ChatHistory userChatHistory){ var message = messages.get(0); - JSONObject json = new JSONObject(message.getContent()); - IntentType intentType = IntentType.valueOf(json.get("intent").toString()); + JSONObject jsonData = new JSONObject(); + + /** + * Try to see if the model answered with a formatted json. If not it is just trying to move the conversation forward to understand the user intent + * but without answering with a formatted output. In this case the intent is None and the clarifying sentence is not used. + */ + try{ + jsonData = new JSONObject(message.getContent()); + }catch (JSONException e){ + return new IntentResponse(IntentType.None,message.getContent()); + } + + IntentType intentType = IntentType.valueOf(jsonData.get("intent").toString()); + String clarifySentence = ""; + try { + clarifySentence = jsonData.get("clarify_sentence").toString(); + } catch(Exception e){ + // this is the case where the intent has been identified and the clarifying sentence is not present in the json outpu + } - return new IntentResponse(intentType,json); + return new IntentResponse(intentType, clarifySentence != null ? clarifySentence.toString() : ""); } public static void main(String[] args) throws NoSuchMethodException { diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentResponse.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentResponse.java index a478c8b..e87c2c7 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentResponse.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentResponse.java @@ -8,20 +8,20 @@ public class IntentResponse { private IntentType intentType; - private JSONObject jsonData; + private String message; - public IntentResponse(IntentType intentType, JSONObject jsonData) { + public IntentResponse(IntentType intentType, String message) { this.intentType = intentType; - this.jsonData = jsonData; + this.message = message; } public IntentType getIntentType() { return intentType; } - public JSONObject getJsonData() { - return jsonData; + public String getMessage() { + return this.message; } } diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/PaymentAgent.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/PaymentAgent.java index 21cf6cd..b5d694d 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/PaymentAgent.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/PaymentAgent.java @@ -46,10 +46,6 @@ public class PaymentAgent { you are a personal financial advisor who help the user with their recurrent bill payments. The user may want to pay the bill uploading a photo of the bill, or it may start the payment checking transactions history for a specific payee. For the bill payment you need to know the: bill id or invoice number, payee name, the total amount and the bill expiration date. if you don't have enough information to pay the bill ask the user to provide the missing information. - you have the below functions available: - - paymentHistory: returns the list of the last payments based on the payee name - - payBill: it pays the bill based on the bill id or invoice number, payee name, total amount - - invoiceScan: it scans the invoice or bill photo to extract data Always check if the bill has been paid already based on payment history before asking to execute the bill payment. Always ask for the payment method to use: direct debit, credit card, or bank transfer @@ -83,18 +79,16 @@ public PaymentAgent(OpenAIAsyncClient client, String modelId, DocumentIntelligen } - public AgentContext run (ChatHistory userChatHistory) { + public void run (ChatHistory userChatHistory, AgentContext agentContext){ LOGGER.info("======== Payment Agent: Starting ========"); - AgentContext agentContext = new AgentContext(); - var agentChatHistory = new ChatHistory(PAYMENT_AGENT_SYSTEM_MESSAGE); userChatHistory.forEach( chatMessageContent -> { if(chatMessageContent.getAuthorRole() != AuthorRole.SYSTEM) agentChatHistory.addMessage(chatMessageContent); - }); + }); while (true) { @@ -149,7 +143,6 @@ public AgentContext run (ChatHistory userChatHistory) { })); } } - return agentContext; } diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java index 0a50218..89a7feb 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java @@ -5,6 +5,8 @@ public record ChatAppRequest( List messages, + + List attachments, ChatAppRequestContext context, boolean stream, String approach) {} diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java index 3707a0a..1367f45 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java @@ -7,8 +7,10 @@ import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy; +import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; +import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -61,7 +63,7 @@ public ResponseEntity openAIAsk(@RequestBody ChatAppRequest chatRe return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(null); } - ChatHistory chatHistory = convertSKChatHistory(chatRequest.messages()); + ChatHistory chatHistory = convertSKChatHistory(chatRequest); LOGGER.info("Processing chat conversation..", chatHistory.getLastMessage().get().getContent()); @@ -69,34 +71,54 @@ public ResponseEntity openAIAsk(@RequestBody ChatAppRequest chatRe LOGGER.info("Intent Type for chat conversation: {}", response.getIntentType()); if (response.getIntentType() == IntentType.None) { - chatHistory.addAssistantMessage(response.getJsonData().get("clarify_sentence").toString()); + chatHistory.addAssistantMessage(response.getMessage()); } + var agentContext = new AgentContext(); + agentContext.put("requestContext", chatRequest.context()); + agentContext.put("attachments", chatRequest.attachments()); + agentContext.put("approach", chatRequest.approach()); if (response.getIntentType() == IntentType.BillPayment || response.getIntentType() == IntentType.RepeatTransaction) { - var agentContext = paymentAgent.run(chatHistory); + paymentAgent.run(chatHistory,agentContext); chatHistory.addAssistantMessage(agentContext.getResult()); } if (response.getIntentType() == IntentType.TransactionHistory) { - var agentContext = historyReportingAgent.run(chatHistory); + historyReportingAgent.run(chatHistory,agentContext); chatHistory.addAssistantMessage(agentContext.getResult()); } return ResponseEntity.ok( - ChatResponse.buildChatResponse(chatHistory)); + ChatResponse.buildChatResponse(chatHistory, agentContext)); } - private ChatHistory convertSKChatHistory(List protocolChatHistory) { + private ChatHistory convertSKChatHistory(ChatAppRequest chatAppRequest) { ChatHistory chatHistory = new ChatHistory(false); - protocolChatHistory.forEach( - historyChat -> { - if("user".equals(historyChat.role())) - chatHistory.addUserMessage(historyChat.content()); + /* + ChatMessageContent lastUserMessage = new ChatMessageContent(AuthorRole.USER, + chatAppRequest.messages().remove(chatAppRequest.messages().size()-1).content()); + + if(chatAppRequest.attachments() != null && !chatAppRequest.attachments().isEmpty()) { + // prepare last user message including attachments. Append list of attachments to the last user message content + lastUserMessage = new ChatMessageContent(AuthorRole.USER, + chatAppRequest.messages().remove(chatAppRequest.messages().size()-1).content() + " " +chatAppRequest.attachments().toString()); + } + */ + + chatAppRequest.messages().forEach( + historyChat -> { + if("user".equals(historyChat.role())) { + if(historyChat.attachments() == null || historyChat.attachments().isEmpty()) + chatHistory.addUserMessage(historyChat.content()); + else + chatHistory.addUserMessage(historyChat.content() + " " + historyChat.attachments().toString()); + } if("assistant".equals(historyChat.role())) chatHistory.addAssistantMessage(historyChat.content()); }); + //-chatHistory.addUserMessage(lastUserMessage.getContent()); return chatHistory; diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java index ec1ee2a..60fb144 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java @@ -2,6 +2,7 @@ package com.microsoft.openai.samples.assistant.controller; +import com.microsoft.openai.samples.assistant.agent.AgentContext; import com.microsoft.openai.samples.assistant.common.ChatGPTMessage; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; @@ -10,13 +11,16 @@ public record ChatResponse(List choices) { - public static ChatResponse buildChatResponse(ChatHistory chatHistory) { + public static ChatResponse buildChatResponse(ChatHistory chatHistory, AgentContext agentContext) { List dataPoints = Collections.emptyList(); + String thoughts = ""; + List attachments = Collections.emptyList(); + if(agentContext.get("dataPoints") != null) dataPoints.addAll((List) agentContext.get("dataPoints")); + if(agentContext.get("thoughts") != null) thoughts = (String)agentContext.get("thoughts"); + if(agentContext.get("attachments") != null) attachments.addAll((List) agentContext.get("attachments")); - String thoughts = ""; - return new ChatResponse( List.of( @@ -24,11 +28,14 @@ public static ChatResponse buildChatResponse(ChatHistory chatHistory) { 0, new ResponseMessage( chatHistory.getLastMessage().get().getContent(), - ChatGPTMessage.ChatRole.ASSISTANT.toString()), + ChatGPTMessage.ChatRole.ASSISTANT.toString(), + attachments + ), new ResponseContext(thoughts, dataPoints), new ResponseMessage( chatHistory.getLastMessage().get().getContent(), - ChatGPTMessage.ChatRole.ASSISTANT.toString())))); + ChatGPTMessage.ChatRole.ASSISTANT.toString(), + attachments)))); } } diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ResponseMessage.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ResponseMessage.java index 074cef1..78ff80c 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ResponseMessage.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ResponseMessage.java @@ -1,4 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.openai.samples.assistant.controller; -public record ResponseMessage(String content, String role) {} +import java.util.List; + +public record ResponseMessage(String content, String role, List attachments) {} diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/content/ContentController.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/content/ContentController.java index 572d5cc..a5b505b 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/content/ContentController.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/content/ContentController.java @@ -16,9 +16,8 @@ import org.springframework.http.ResponseEntity; import org.springframework.util.MimeTypeUtils; import org.springframework.util.StringUtils; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; @RestController public class ContentController { @@ -57,4 +56,25 @@ public ResponseEntity getContent(@PathVariable String fileN .contentType(contentType) .body(new InputStreamResource(fileInputStream)); } + + @PostMapping("/api/content") + public ResponseEntity uploadContent(@RequestParam("file") MultipartFile file) { + LOGGER.info("Received request to upload a file [{}}", file.getOriginalFilename()); + + if (file.isEmpty()) { + LOGGER.warn("Uploaded file is empty"); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Uploaded file is empty"); + } + + try { + byte[] bytes = file.getBytes(); + blobStorageProxy.storeFile(bytes, file.getOriginalFilename()); + } catch (IOException ex) { + LOGGER.error("Cannot store file [{}] to blob.{}", file.getOriginalFilename(), ex.getMessage()); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error occurred while storing file"); + } + + return ResponseEntity.ok(file.getOriginalFilename()); + } + } diff --git a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/proxy/BlobStorageProxy.java b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/proxy/BlobStorageProxy.java index dbf601e..7cdc9fe 100644 --- a/app/backend/src/main/java/com/microsoft/openai/samples/assistant/proxy/BlobStorageProxy.java +++ b/app/backend/src/main/java/com/microsoft/openai/samples/assistant/proxy/BlobStorageProxy.java @@ -2,13 +2,13 @@ package com.microsoft.openai.samples.assistant.proxy; import com.azure.core.credential.TokenCredential; +import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.io.ByteArrayOutputStream; -import java.io.IOException; +import java.io.*; /** * This class is a proxy to the Blob storage API. It is responsible for: - calling the API - @@ -45,4 +45,9 @@ public byte[] getFileAsBytes(String fileName) throws IOException { return outputStream.toByteArray(); } + + public void storeFile(byte[] bytes, String originalFilename) { + BlobClient blobClient = client.getBlobClient(originalFilename); + blobClient.upload(new ByteArrayInputStream(bytes), bytes.length, true); + } } diff --git a/app/frontend/.env.dev b/app/frontend/.env.dev index e34f261..5fce15e 100644 --- a/app/frontend/.env.dev +++ b/app/frontend/.env.dev @@ -1 +1 @@ -VITE_BACKEND_URI=http://127.0.0.1:8081/api +VITE_BACKEND_URI=http://localhost:8081/api diff --git a/app/frontend/.env.local b/app/frontend/.env.local index e34f261..5fce15e 100644 --- a/app/frontend/.env.local +++ b/app/frontend/.env.local @@ -1 +1 @@ -VITE_BACKEND_URI=http://127.0.0.1:8081/api +VITE_BACKEND_URI=http://localhost:8081/api diff --git a/app/frontend/src/api/api.ts b/app/frontend/src/api/api.ts index fb8f511..e9f7c2f 100644 --- a/app/frontend/src/api/api.ts +++ b/app/frontend/src/api/api.ts @@ -49,3 +49,23 @@ export async function chatApi(request: ChatAppRequest, idToken: string | undefin export function getCitationFilePath(citation: string): string { return `${BACKEND_URI}/content/${citation}`; } + +export function uploadAttachment(file: File): Promise { + const formData = new FormData(); + formData.append("file", file); + + return fetch(`${BACKEND_URI}/content`, { + method: "POST", + body: formData + }).then(response => { + if (response.status > 299 || !response.ok) { + throw Error("Failed to upload attachment"); + } + return response.text(); + }); +} + +export function getImage(name: string): string { + return `${BACKEND_URI}/content/${name}`; + } + diff --git a/app/frontend/src/api/models.ts b/app/frontend/src/api/models.ts index c468488..badfda0 100644 --- a/app/frontend/src/api/models.ts +++ b/app/frontend/src/api/models.ts @@ -34,6 +34,7 @@ export type ChatAppRequestOverrides = { export type ResponseMessage = { content: string; role: string; + attachments?: string[]; }; export type ResponseContext = { diff --git a/app/frontend/src/components/AttachmentType.ts b/app/frontend/src/components/AttachmentType.ts new file mode 100644 index 0000000..83d8ed0 --- /dev/null +++ b/app/frontend/src/components/AttachmentType.ts @@ -0,0 +1,4 @@ +export type AttachmentType = { + name: string; + file: File; //Reference to the javascript File object. +}; \ No newline at end of file diff --git a/app/frontend/src/components/QuestionInput/QuestionContext.ts b/app/frontend/src/components/QuestionInput/QuestionContext.ts new file mode 100644 index 0000000..11f1aa9 --- /dev/null +++ b/app/frontend/src/components/QuestionInput/QuestionContext.ts @@ -0,0 +1,7 @@ +import { AttachmentType } from "../AttachmentType"; + +export type QuestionContextType = { + question: string; + attachments?: string[]; + +}; \ No newline at end of file diff --git a/app/frontend/src/components/QuestionInput/QuestionInput.module.css b/app/frontend/src/components/QuestionInput/QuestionInput.module.css index 419523e..18b1c52 100644 --- a/app/frontend/src/components/QuestionInput/QuestionInput.module.css +++ b/app/frontend/src/components/QuestionInput/QuestionInput.module.css @@ -3,7 +3,7 @@ box-shadow: 0px 8px 16px rgba(0, 0, 0, 0.14), 0px 0px 2px rgba(0, 0, 0, 0.12); - height: 90px; + height: 150px; width: 100%; padding: 15px; background: white; @@ -19,3 +19,16 @@ flex-direction: column; justify-content: flex-end; } + +.attachmentContainer { + display: flex; + justify-content: space-between; + align-items: center; +} + +.imagePreview { + border: 1px solid #ddd; /* Gray border */ + border-radius: 4px; /* Rounded border */ + padding: 5px; /* Some padding */ + height: 130px; /* Set a small width */ +} \ No newline at end of file diff --git a/app/frontend/src/components/QuestionInput/QuestionInput.tsx b/app/frontend/src/components/QuestionInput/QuestionInput.tsx index 1bc69cc..3954a6d 100644 --- a/app/frontend/src/components/QuestionInput/QuestionInput.tsx +++ b/app/frontend/src/components/QuestionInput/QuestionInput.tsx @@ -1,30 +1,49 @@ -import { useState } from "react"; +import { useState, useRef } from "react"; import { Stack, TextField } from "@fluentui/react"; import { Button, Tooltip, Field, Textarea } from "@fluentui/react-components"; -import { Send28Filled } from "@fluentui/react-icons"; +import { Send28Filled, Attach24Filled, Delete16Filled } from "@fluentui/react-icons"; +import { QuestionContextType } from "./QuestionContext"; +import { uploadAttachment } from "../../api"; import styles from "./QuestionInput.module.css"; interface Props { - onSend: (question: string) => void; + onSend: (questionContext: QuestionContextType) => void; disabled: boolean; placeholder?: string; clearOnSend?: boolean; } -export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend }: Props) => { +export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend }: Props) => { const [question, setQuestion] = useState(""); + const inputFile = useRef(null); + const [attachmentRef, setAttachmentRef] = useState(null); + const [previewImage, setPreviewImage] = useState(null); - const sendQuestion = () => { + const sendQuestion = async() => { if (disabled || !question.trim()) { return; } - onSend(question); + if( attachmentRef != null){ + + console.log("Uploading file... "+ attachmentRef.name); + await uploadAttachment(attachmentRef); + } + + const questionContext = { + question: question, + attachments: attachmentRef != null ? [attachmentRef.name] : [] + }; + + onSend(questionContext); if (clearOnSend) { setQuestion(""); } + + setAttachmentRef(null); + setPreviewImage(null); }; const onEnterPress = (ev: React.KeyboardEvent) => { @@ -42,10 +61,33 @@ export const QuestionInput = ({ onSend, disabled, placeholder, clearOnSend }: Pr } }; + const onAttach = (_ev : React.MouseEvent) => { + inputFile.current?.click(); + } + + const onFileSelected = (_ev : React.ChangeEvent) => { + if (_ev.target.files) { + setAttachmentRef(_ev.target.files[0]); + setPreviewImage(URL.createObjectURL(_ev.target.files[0])); + } + + } + + const onAttachDelete = (_ev : React.MouseEvent) => { + setAttachmentRef(null); + setPreviewImage(null); + } const sendQuestionDisabled = disabled || !question.trim(); return ( + {previewImage && ( + +
+ +
+ )} +
+ +
+ ); }; diff --git a/app/frontend/src/components/UserChatMessage/UserChatMessage.module.css b/app/frontend/src/components/UserChatMessage/UserChatMessage.module.css index 591f314..a34fa31 100644 --- a/app/frontend/src/components/UserChatMessage/UserChatMessage.module.css +++ b/app/frontend/src/components/UserChatMessage/UserChatMessage.module.css @@ -15,3 +15,13 @@ 0px 0px 2px rgba(0, 0, 0, 0.12); outline: transparent solid 1px; } + +.attachementPreview { + display: flex; + justify-content: flex-end; + margin-left: auto; + border: 1px solid #ddd; /* Gray border */ + border-radius: 4px; /* Rounded border */ + padding: 5px; /* Some padding */ + height: 500px; /* Set a small width */ +} \ No newline at end of file diff --git a/app/frontend/src/components/UserChatMessage/UserChatMessage.tsx b/app/frontend/src/components/UserChatMessage/UserChatMessage.tsx index 2f5e8d3..5d0ee35 100644 --- a/app/frontend/src/components/UserChatMessage/UserChatMessage.tsx +++ b/app/frontend/src/components/UserChatMessage/UserChatMessage.tsx @@ -1,13 +1,31 @@ import styles from "./UserChatMessage.module.css"; +import { Stack} from "@fluentui/react"; +import { AttachmentType } from "../AttachmentType"; +import { getImage} from "../../api"; interface Props { message: string; + attachments?: string[]; } -export const UserChatMessage = ({ message }: Props) => { +// {attachment.name} + +export const UserChatMessage = ({message, attachments}: Props) => { return ( + <> + {attachments && ( + <> + {attachments.map((attachment, index) => ( +
+ {attachment} + +
+ ))} + + )}
-
{message}
+
{message}
+ ); }; diff --git a/app/frontend/src/index.tsx b/app/frontend/src/index.tsx index d0c06c3..688c31b 100644 --- a/app/frontend/src/index.tsx +++ b/app/frontend/src/index.tsx @@ -49,10 +49,6 @@ const router = createHashRouter([ index: true, element: }, - { - path: "qa", - lazy: () => import("./pages/oneshot/OneShot") - }, { path: "*", lazy: () => import("./pages/NoPage") diff --git a/app/frontend/src/pages/chat/Chat.tsx b/app/frontend/src/pages/chat/Chat.tsx index 6116c38..2fe8d08 100644 --- a/app/frontend/src/pages/chat/Chat.tsx +++ b/app/frontend/src/pages/chat/Chat.tsx @@ -3,6 +3,7 @@ import { Checkbox, ChoiceGroup, Panel, DefaultButton, TextField, SpinButton, Dro import { SparkleFilled } from "@fluentui/react-icons"; import readNDJSONStream from "ndjson-readablestream"; + import styles from "./Chat.module.css"; import { @@ -17,6 +18,7 @@ import { } from "../../api"; import { Answer, AnswerError, AnswerLoading } from "../../components/Answer"; import { QuestionInput } from "../../components/QuestionInput"; +import { QuestionContextType } from "../../components/QuestionInput/QuestionContext"; import { ExampleList } from "../../components/Example"; import { UserChatMessage } from "../../components/UserChatMessage"; import { AnalysisPanel, AnalysisPanelTabs } from "../../components/AnalysisPanel"; @@ -25,6 +27,8 @@ import { ClearChatButton } from "../../components/ClearChatButton"; import { useLogin, getToken } from "../../authConfig"; import { useMsal } from "@azure/msal-react"; import { TokenClaimsDisplay } from "../../components/TokenClaimsDisplay"; +import { AttachmentType } from "../../components/AttachmentType"; + const Chat = () => { const [isConfigPanelOpen, setIsConfigPanelOpen] = useState(false); @@ -43,6 +47,7 @@ const Chat = () => { const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState(false); const lastQuestionRef = useRef(""); + const lastAttachementsRef = useRef([]); const chatMessageStreamEnd = useRef(null); const [isLoading, setIsLoading] = useState(false); @@ -53,10 +58,10 @@ const Chat = () => { const [activeAnalysisPanelTab, setActiveAnalysisPanelTab] = useState(undefined); const [selectedAnswer, setSelectedAnswer] = useState(0); - const [answers, setAnswers] = useState<[user: string, response: ChatAppResponse][]>([]); - const [streamedAnswers, setStreamedAnswers] = useState<[user: string, response: ChatAppResponse][]>([]); + const [answers, setAnswers] = useState<[user: string, attachments: string[], response: ChatAppResponse][]>([]); + const [streamedAnswers, setStreamedAnswers] = useState<[user: string, attachments: string[], response: ChatAppResponse][]>([]); - const handleAsyncRequest = async (question: string, answers: [string, ChatAppResponse][], setAnswers: Function, responseBody: ReadableStream) => { + const handleAsyncRequest = async (question: string, attachments: string[], answers: [string, string[],ChatAppResponse][], setAnswers: Function, responseBody: ReadableStream) => { let answer: string = ""; let askResponse: ChatAppResponse = {} as ChatAppResponse; @@ -68,7 +73,7 @@ const Chat = () => { ...askResponse, choices: [{ ...askResponse.choices[0], message: { content: answer, role: askResponse.choices[0].message.role } }] }; - setStreamedAnswers([...answers, [question, latestResponse]]); + setStreamedAnswers([...answers, [question,attachments, latestResponse]]); resolve(null); }, 33); }); @@ -97,8 +102,9 @@ const Chat = () => { const client = useLogin ? useMsal().instance : undefined; - const makeApiRequest = async (question: string) => { - lastQuestionRef.current = question; + const makeApiRequest = async (questionContext: QuestionContextType) => { + lastQuestionRef.current = questionContext.question; + lastAttachementsRef.current = questionContext.attachments || []; error && setError(undefined); setIsLoading(true); @@ -109,13 +115,13 @@ const Chat = () => { try { const messages: ResponseMessage[] = answers.flatMap(a => [ - { content: a[0], role: "user" }, - { content: a[1].choices[0].message.content, role: "assistant" } + { content: a[0], role: "user", attachments: a[1]}, + { content: a[2].choices[0].message.content, role: "assistant" } ]); const stream = streamAvailable && shouldStream; const request: ChatAppRequest = { - messages: [...messages, { content: question, role: "user" }], + messages: [...messages, { content: questionContext.question, role: "user", attachments: questionContext.attachments }], stream: stream, context: { overrides: { @@ -133,7 +139,7 @@ const Chat = () => { }, approach: approach, // ChatAppProtocol: Client must pass on any session state received from the server - session_state: answers.length ? answers[answers.length - 1][1].choices[0].session_state : null + session_state: answers.length ? answers[answers.length - 1][2].choices[0].session_state : null }; const response = await chatApi(request, token?.accessToken); @@ -141,14 +147,14 @@ const Chat = () => { throw Error("No response body"); } if (stream) { - const parsedResponse: ChatAppResponse = await handleAsyncRequest(question, answers, setAnswers, response.body); - setAnswers([...answers, [question, parsedResponse]]); + const parsedResponse: ChatAppResponse = await handleAsyncRequest(questionContext.question,questionContext.attachments || [], answers, setAnswers, response.body); + setAnswers([...answers, [questionContext.question,questionContext.attachments || [], parsedResponse]]); } else { const parsedResponse: ChatAppResponseOrError = await response.json(); if (response.status > 299 || !response.ok) { throw Error(parsedResponse.error || "Unknown error"); } - setAnswers([...answers, [question, parsedResponse as ChatAppResponse]]); + setAnswers([...answers, [questionContext.question,questionContext.attachments || [], parsedResponse as ChatAppResponse]]); } } catch (e) { setError(e); @@ -159,6 +165,7 @@ const Chat = () => { const clearChat = () => { lastQuestionRef.current = ""; + lastAttachementsRef.current = []; error && setError(undefined); setActiveCitation(undefined); setActiveAnalysisPanelTab(undefined); @@ -222,7 +229,7 @@ const Chat = () => { }; const onExampleClicked = (example: string) => { - makeApiRequest(example); + makeApiRequest({question:example}); }; const onShowCitation = (citation: string, index: number) => { @@ -282,17 +289,17 @@ const Chat = () => { {isStreaming && streamedAnswers.map((streamedAnswer, index) => (
- +
onShowCitation(c, index)} onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab, index)} onSupportingContentClicked={() => onToggleTab(AnalysisPanelTabs.SupportingContentTab, index)} - onFollowupQuestionClicked={q => makeApiRequest(q)} + onFollowupQuestionClicked={q => makeApiRequest({question:q})} showFollowupQuestions={useSuggestFollowupQuestions && answers.length - 1 === index} />
@@ -301,17 +308,17 @@ const Chat = () => { {!isStreaming && answers.map((answer, index) => (
- +
onShowCitation(c, index)} onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab, index)} onSupportingContentClicked={() => onToggleTab(AnalysisPanelTabs.SupportingContentTab, index)} - onFollowupQuestionClicked={q => makeApiRequest(q)} + onFollowupQuestionClicked={q => makeApiRequest({question:q})} showFollowupQuestions={useSuggestFollowupQuestions && answers.length - 1 === index} />
@@ -319,7 +326,7 @@ const Chat = () => { ))} {isLoading && ( <> - +
@@ -327,9 +334,9 @@ const Chat = () => { )} {error ? ( <> - +
- makeApiRequest(lastQuestionRef.current)} /> + makeApiRequest({question:lastQuestionRef.current})} />
) : null} @@ -353,7 +360,7 @@ const Chat = () => { activeCitation={activeCitation} onActiveTabChanged={x => onToggleTab(x, selectedAnswer)} citationHeight="810px" - answer={answers[selectedAnswer][1]} + answer={answers[selectedAnswer][2]} activeTab={activeAnalysisPanelTab} /> )} diff --git a/app/frontend/src/pages/layout/Layout.tsx b/app/frontend/src/pages/layout/Layout.tsx index 2d1145e..415d890 100644 --- a/app/frontend/src/pages/layout/Layout.tsx +++ b/app/frontend/src/pages/layout/Layout.tsx @@ -14,7 +14,7 @@ const Layout = () => {
-

GPT + Enterprise data | Java Sample

+

Agents Java Sample