support for image attachments

.github/workflows/app-ci.yaml

@@ -51,11 +51,6 @@ jobs:
           mkdir -p ../backend/src/main/resources/static
           cp -r ./build/* ../backend/src/main/resources/static
       
-      - name: Verify Indexer project
-        run: |
-          echo "Testing indexer project."
-          cd ./app/indexer
-          mvn test
 
       - name: Build Spring Boot App
         run: |

.github/workflows/nightly-jobs.yaml

@@ -91,12 +91,6 @@ jobs:
           java-version: '17'
           cache: 'maven'
 
-      - name: Verify Indexer project
-        run: |
-          echo "Testing indexer project."
-          cd ./app/indexer
-          mvn test
-
       - name: Build Spring Boot App
         run: |
           echo "Building Spring Boot app."

app/backend/pom.xml

@@ -18,7 +18,7 @@
 
 		<spring-cloud-azure.version>4.9.0</spring-cloud-azure.version>
 		<azure-search.version>11.6.0-beta.8</azure-search.version>
-		<semantic-kernel.version>1.0.1</semantic-kernel.version>
+		<semantic-kernel.version>1.0.2</semantic-kernel.version>
 		<mockito-inline.version>4.5.1</mockito-inline.version>
 		<maven.compiler-plugin.version>3.11.0</maven.compiler-plugin.version>
 

app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/AgentContext.java

@@ -1,21 +1,25 @@
 package com.microsoft.openai.samples.assistant.agent;
-public class AgentContext {
 
-    private String result;
+import java.util.HashMap;
+
+public class AgentContext extends HashMap<String,Object>{
 
+    private String result;
 
-    public void AgentContext() {
+    public AgentContext() {
+        super();
     }
 
-    public void AgentContext(String result) {
-          this.result = result;
+    public AgentContext(String result) {
+        super();
+        this.put("result", result);
      }
 
     public String getResult() {
-        return result;
+        return (String)this.get("result");
     }
     public void setResult(String result) {
-        this.result = result;
+        this.put("result", result);
     }
 
 }

app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/HistoryReportingAgent.java

@@ -78,11 +78,9 @@ public HistoryReportingAgent(String azureClientKey, String clientEndpoint, Strin
                 .build();
     }
 
-     public AgentContext run (ChatHistory userChatHistory) {
+     public AgentContext run (ChatHistory userChatHistory, AgentContext agentContext){
          LOGGER.info("======== HistoryAndTransaction Agent: Starting ========");
 
-         AgentContext agentContext = new AgentContext();
-
          var agentChatHistory = new ChatHistory(HISTORY_AGENT_SYSTEM_MESSAGE);
          userChatHistory.forEach( chatMessageContent -> {
              if(chatMessageContent.getAuthorRole() != AuthorRole.SYSTEM)
@@ -155,7 +153,7 @@ public static void main(String[] args) throws NoSuchMethodException {
 
         HistoryReportingAgent agent = new HistoryReportingAgent(AZURE_CLIENT_KEY, CLIENT_ENDPOINT, MODEL_ID);
 
-        agent.run(new ChatHistory());
+        agent.run(new ChatHistory(), new AgentContext());
 
 
 

app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentAgent.java

@@ -11,6 +11,7 @@
 import com.microsoft.semantickernel.orchestration.*;
 import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService;
 import com.microsoft.semantickernel.services.chatcompletion.ChatHistory;
+import org.json.JSONException;
 import org.json.JSONObject;
 
 public class IntentAgent {
@@ -22,26 +23,23 @@ public class IntentAgent {
     private ChatCompletionService chat;
 
     private String INTENT_SYSTEM_MESSAGE = """
-     You are a personal financial advisor who help the user with their recurrent bill payments.
-             The user may want to pay the bill uploading a photo of the bill, or it may start the payment checking payments history for a specific payee.
-             In other cases it may want to just review the payments history.
-             Based on the conversation you need to identify the user intent and ask the user for the missing information.
-             The available intents are:
-             "BillPayment", "RepeatTransaction","TransactionHistory"
-             If none of the intents are identified provide the user with the list of the available intents.
-             
-             If an intent is identified return the output as json format as below
-             {
-               "intent": "BillPayment"
-             }
-             
-             If none of the intents are identified ask the user for more clarity and list of the available intents. Use always a json format as output 
-             {
-               "intent": "None"
-               "clarify_sentence": ""
-             }
-             
-             Don't add any comments in the output or other characters, just the json format.
+You are a personal financial advisor who help the user with their recurrent bill payments.
+The user may want to pay the bill uploading a photo of the bill, or it may start the payment checking payments history for a specific payee.
+In other cases it may want to just review the payments history.
+Based on the conversation you need to identify the user intent.
+The available intents are:
+"BillPayment", "RepeatTransaction","TransactionHistory"
+If none of the intents are identified provide the user with the list of the available intents.
+
+If an intent is identified return the output as json format as below
+{
+"intent": "BillPayment"
+ }
+
+If you don't understand or if an intent is not identified be polite with the user, ask clarifying question also using the list the available intents. 
+
+Don't add any comments in the output or other characters, just the use a json format.
+            
     """;
 
     public IntentAgent(OpenAIAsyncClient client, String modelId){
@@ -73,10 +71,8 @@ public IntentAgent(String azureClientKey, String clientEndpoint, String modelId)
     }
 
     public IntentResponse run(ChatHistory userChatHistory){
-        var agentChatHistory = new ChatHistory();
+        var agentChatHistory = new ChatHistory(INTENT_SYSTEM_MESSAGE);
         agentChatHistory.addAll(userChatHistory);
-        agentChatHistory.addSystemMessage(INTENT_SYSTEM_MESSAGE);
-
 
         var messages = chat.getChatMessageContentsAsync(
                         agentChatHistory,
@@ -92,10 +88,27 @@ public IntentResponse run(ChatHistory userChatHistory){
 
         var message = messages.get(0);
 
-        JSONObject json = new JSONObject(message.getContent());
-        IntentType intentType = IntentType.valueOf(json.get("intent").toString());
+        JSONObject jsonData = new JSONObject();
+
+        /**
+         * Try to see if the model answered with a formatted json. If not it is just trying to move the conversation forward to understand the user intent
+         * but without answering with a formatted output. In this case the intent is None and the clarifying sentence is not used.
+         */
+        try{
+            jsonData = new JSONObject(message.getContent());
+        }catch (JSONException e){
+            return new IntentResponse(IntentType.None,message.getContent());
+        }
+
+        IntentType intentType = IntentType.valueOf(jsonData.get("intent").toString());
+        String clarifySentence = "";
+        try {
+            clarifySentence = jsonData.get("clarify_sentence").toString();
+        } catch(Exception e){
+             // this is the case where the intent has been identified and the clarifying sentence is not present in the json outpu
+             }
 
-        return new IntentResponse(intentType,json);
+        return new IntentResponse(intentType, clarifySentence != null ? clarifySentence.toString() : "");
     }
 
     public static void main(String[] args) throws NoSuchMethodException {

app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/IntentResponse.java

@@ -8,20 +8,20 @@ public class IntentResponse {
 
     private IntentType intentType;
 
-    private JSONObject jsonData;
+    private String message;
 
 
 
-    public IntentResponse(IntentType intentType, JSONObject jsonData) {
+    public IntentResponse(IntentType intentType, String message) {
         this.intentType = intentType;
-        this.jsonData = jsonData;
+        this.message = message;
     }
 
     public IntentType getIntentType() {
         return intentType;
     }
 
-    public JSONObject getJsonData() {
-        return jsonData;
+    public String getMessage() {
+        return this.message;
     }
 }

app/backend/src/main/java/com/microsoft/openai/samples/assistant/agent/PaymentAgent.java

@@ -46,10 +46,6 @@ public class PaymentAgent {
      you are a personal financial advisor who help the user with their recurrent bill payments. The user may want to pay the bill uploading a photo of the bill, or it may start the payment checking transactions history for a specific payee.
      For the bill payment you need to know the: bill id or invoice number, payee name, the total amount and the bill expiration date.
      if you don't have enough information to pay the bill ask the user to provide the missing information.
-     you have the below functions available:
-        - paymentHistory: returns the list of the last payments based on the payee name
-        - payBill: it pays the bill based on the bill id or invoice number, payee name, total amount
-        - invoiceScan: it scans the invoice or bill photo to extract data
      
      Always check if the bill has been paid already based on payment history before asking to execute the bill payment.
      Always ask for the payment method to use: direct debit, credit card, or bank transfer 
@@ -83,18 +79,16 @@ public PaymentAgent(OpenAIAsyncClient client, String modelId, DocumentIntelligen
     }
 
 
-     public AgentContext run (ChatHistory userChatHistory) {
+     public void run (ChatHistory userChatHistory, AgentContext agentContext){
          LOGGER.info("======== Payment Agent: Starting ========");
 
-         AgentContext agentContext = new AgentContext();
-
          var agentChatHistory = new ChatHistory(PAYMENT_AGENT_SYSTEM_MESSAGE);
 
          userChatHistory.forEach( chatMessageContent -> {
             if(chatMessageContent.getAuthorRole() != AuthorRole.SYSTEM)
              agentChatHistory.addMessage(chatMessageContent);
-         });
 
+         });
 
 
          while (true) {
@@ -149,7 +143,6 @@ public AgentContext run (ChatHistory userChatHistory) {
                          }));
              }
             }
-             return agentContext;
          }
 
 

app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatAppRequest.java

@@ -5,6 +5,8 @@
 
 public record ChatAppRequest(
         List<ResponseMessage> messages,
+
+        List<String> attachments,
         ChatAppRequestContext context,
         boolean stream,
         String approach) {}

app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatController.java

@@ -7,8 +7,10 @@
 
 
 import com.microsoft.openai.samples.assistant.proxy.BlobStorageProxy;
+import com.microsoft.semantickernel.services.chatcompletion.AuthorRole;
 import com.microsoft.semantickernel.services.chatcompletion.ChatHistory;
 
+import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -61,42 +63,62 @@ public ResponseEntity<ChatResponse> openAIAsk(@RequestBody ChatAppRequest chatRe
             return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(null);
         }
 
-        ChatHistory chatHistory = convertSKChatHistory(chatRequest.messages());
+        ChatHistory chatHistory = convertSKChatHistory(chatRequest);
 
         LOGGER.info("Processing chat conversation..", chatHistory.getLastMessage().get().getContent());
 
         IntentResponse response = intentAgent.run(chatHistory);
 
         LOGGER.info("Intent Type for chat conversation: {}", response.getIntentType());
         if (response.getIntentType() == IntentType.None) {
-            chatHistory.addAssistantMessage(response.getJsonData().get("clarify_sentence").toString());
+            chatHistory.addAssistantMessage(response.getMessage());
         }
 
+        var agentContext = new AgentContext();
+        agentContext.put("requestContext", chatRequest.context());
+        agentContext.put("attachments", chatRequest.attachments());
+        agentContext.put("approach", chatRequest.approach());
 
         if (response.getIntentType() == IntentType.BillPayment || response.getIntentType() == IntentType.RepeatTransaction) {
-            var agentContext = paymentAgent.run(chatHistory);
+            paymentAgent.run(chatHistory,agentContext);
             chatHistory.addAssistantMessage(agentContext.getResult());
         }
 
         if (response.getIntentType() == IntentType.TransactionHistory) {
-            var agentContext = historyReportingAgent.run(chatHistory);
+            historyReportingAgent.run(chatHistory,agentContext);
             chatHistory.addAssistantMessage(agentContext.getResult());
         }
 
         return ResponseEntity.ok(
-                ChatResponse.buildChatResponse(chatHistory));
+                ChatResponse.buildChatResponse(chatHistory, agentContext));
     }
 
-    private ChatHistory convertSKChatHistory(List<ResponseMessage> protocolChatHistory) {
+    private ChatHistory convertSKChatHistory(ChatAppRequest chatAppRequest) {
        ChatHistory chatHistory = new ChatHistory(false);
-       protocolChatHistory.forEach(
-               historyChat -> {
-                   if("user".equals(historyChat.role()))
-                   chatHistory.addUserMessage(historyChat.content());
+       /*
+       ChatMessageContent lastUserMessage = new ChatMessageContent(AuthorRole.USER,
+                chatAppRequest.messages().remove(chatAppRequest.messages().size()-1).content());
+
+       if(chatAppRequest.attachments() != null && !chatAppRequest.attachments().isEmpty()) {
+           // prepare last user message including attachments. Append list of attachments to the last user message content
+           lastUserMessage = new ChatMessageContent(AuthorRole.USER,
+                   chatAppRequest.messages().remove(chatAppRequest.messages().size()-1).content() + " " +chatAppRequest.attachments().toString());
+       }
 
+        */
+
+        chatAppRequest.messages().forEach(
+               historyChat -> {
+                   if("user".equals(historyChat.role())) {
+                     if(historyChat.attachments() == null || historyChat.attachments().isEmpty())
+                         chatHistory.addUserMessage(historyChat.content());
+                     else
+                         chatHistory.addUserMessage(historyChat.content() + " " + historyChat.attachments().toString());
+                   }
                    if("assistant".equals(historyChat.role()))
                    chatHistory.addAssistantMessage(historyChat.content());
                });
+       //-chatHistory.addUserMessage(lastUserMessage.getContent());
 
        return chatHistory;
 

app/backend/src/main/java/com/microsoft/openai/samples/assistant/controller/ChatResponse.java

@@ -2,6 +2,7 @@
 package com.microsoft.openai.samples.assistant.controller;
 
 
+import com.microsoft.openai.samples.assistant.agent.AgentContext;
 import com.microsoft.openai.samples.assistant.common.ChatGPTMessage;
 import com.microsoft.semantickernel.services.chatcompletion.ChatHistory;
 
@@ -10,25 +11,31 @@
 
 public record ChatResponse(List<ResponseChoice> choices) {
 
-    public static ChatResponse buildChatResponse(ChatHistory chatHistory) {
+    public static ChatResponse buildChatResponse(ChatHistory chatHistory, AgentContext agentContext) {
         List<String> dataPoints = Collections.emptyList();
+        String thoughts = "";
+        List<String> attachments = Collections.emptyList();
 
+        if(agentContext.get("dataPoints") != null) dataPoints.addAll((List<String>) agentContext.get("dataPoints"));
+        if(agentContext.get("thoughts") != null) thoughts = (String)agentContext.get("thoughts");
+        if(agentContext.get("attachments") != null) attachments.addAll((List<String>) agentContext.get("attachments"));
 
 
-        String thoughts = "";
-
 
         return new ChatResponse(
                 List.of(
                         new ResponseChoice(
                                 0,
                                 new ResponseMessage(
                                         chatHistory.getLastMessage().get().getContent(),
-                                        ChatGPTMessage.ChatRole.ASSISTANT.toString()),
+                                        ChatGPTMessage.ChatRole.ASSISTANT.toString(),
+                                        attachments
+                                          ),
                                 new ResponseContext(thoughts, dataPoints),
                                 new ResponseMessage(
                                         chatHistory.getLastMessage().get().getContent(),
-                                        ChatGPTMessage.ChatRole.ASSISTANT.toString()))));
+                                        ChatGPTMessage.ChatRole.ASSISTANT.toString(),
+                                        attachments))));
     }
 
 }