Marlene ignite updates

.devcontainer/Dockerfile

@@ -13,6 +13,9 @@ RUN sudo apt-get update && sudo apt-get install -y \
  && sudo apt-get clean \
  && sudo rm -rf /var/lib/apt/lists/*
 
+# COPY packages folder
+COPY packages packages
+
 # Copy requirements.txt and install the Python packages
 # Install keyring-related and IPython packages in the same layer to reduce the image size and build time
 COPY requirements.txt .

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
-// README at: https://github.com/devcontainers/templates/tree/main/src/python 
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
 {
     "name": "Creative Writing Assistant (ACA)",
     "build": {
@@ -40,4 +40,4 @@
         5173,
 	8000
 	]
-}
+}

.github/workflows/azure-dev.yml

@@ -32,7 +32,6 @@ jobs:
       AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
       AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME }}
       AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
-      AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME }}
       AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }}
       AZURE_CONTAINER_REGISTRY_NAME: ${{ vars.AZURE_CONTAINER_REGISTRY_NAME }}    
       AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} 

.github/workflows/evaluate.yml

@@ -7,6 +7,12 @@ on:
     # Set this to the mainline branch you are using
     branches:
       - main
+      - ignite2024
+  pull_request:
+    # Run when pull requests are opened or updated
+    branches:
+      - main
+      - ignite2024
 
 # Set up permissions for deploying with secretless Azure federated credentials
 # https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication
@@ -21,17 +27,21 @@ jobs:
       AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}
       AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
       AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
+      AZURE_AI_PROJECT_NAME: ${{ vars.AZURE_AI_PROJECT_NAME }}
       AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
+      AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
       AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
       AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
       AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME }}
       AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
-      AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME }}
       AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
       AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }}
       AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }}
       BING_SEARCH_ENDPOINT: ${{ vars.BING_SEARCH_ENDPOINT }}
       BING_SEARCH_KEY: ${{ secrets.BING_SEARCH_KEY }}
+      EVAL_CONFIG_FILE_PATH: ${{ github.workspace }}/evaluate-config.json
+      GENAI_EVALS_INPUT_FILE_PATH: ${{ github.workspace }}/src/api/evaluate/eval_results.jsonl
+      GENAI_EVALS_OUTPUT_FILE_PATH: ${{ github.workspace }}/src/api/evaluate/genai_evals_data.jsonl
 
     steps:
       - name: checkout repo content
@@ -73,13 +83,48 @@ jobs:
         with:
           name: eval_result
           path: ./src/api/evaluate/eval_results.jsonl
+
+      - name: Upload image eval results as build artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval_image_result
+          path: ./src/api/evaluate/image_eval_results.jsonl
+
+      - name: Convert to microsoft/genai-evals Data Format
+        run: |
+          python -m src.api.evaluate.genai_evals_convert ${{ env.GENAI_EVALS_INPUT_FILE_PATH }} ${{ env.GENAI_EVALS_OUTPUT_FILE_PATH }}
+
+      - name: Prepare AI evaluation configuration file
+        run: |
+          cat > ${{ env.EVAL_CONFIG_FILE_PATH }}<<EOF
+          {
+            "data": "${{ env.GENAI_EVALS_OUTPUT_FILE_PATH }}",
+            "evaluators": {
+              "coherence": "CoherenceEvaluator",
+              "fluency": "FluencyEvaluator"
+            },
+            "ai_model_configuration": {
+              "type": "azure_openai",
+              "azure_endpoint": "${{ vars.AZURE_OPENAI_ENDPOINT }}",
+              "azure_deployment": "${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}",
+              "api_version": "${{ vars.AZURE_OPENAI_API_VERSION }}"
+            }
+          }
+          EOF
 
-      - name: GitHub Summary Step
+      - name: Text Evaluation Summary
+        id: run-ai-evaluation
+        uses: microsoft/genai-evals@users/daviwu/ignite2024
+        with:          
+          evaluate-configuration: ${{ env.EVAL_CONFIG_FILE_PATH }}
+          show-summary: true
+          show-raw-output: true
+
+      - name: Image Evaluation Summary
         if: ${{ success() }}
         working-directory: ./src/api
         run: |
           echo "" >> $GITHUB_STEP_SUMMARY
 
-          echo "📊 Promptflow Evaluation Results" >> $GITHUB_STEP_SUMMARY
-          cat evaluate/eval_results.md >> $GITHUB_STEP_SUMMARY
-
+          echo "# 📊 Image Evaluation Results" >> $GITHUB_STEP_SUMMARY
+          cat evaluate/image_eval_results.md >> $GITHUB_STEP_SUMMARY

.vscode/launch.json

@@ -4,7 +4,7 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
-    {
+       {
       "name": "Python: FastAPI",
       "type": "debugpy",
       "request": "launch",
@@ -28,6 +28,33 @@
       "module": "api.evaluate.evaluate",
       "cwd": "${workspaceFolder}/src/api",
       "justMyCode": true
+    },
+    {
+      "name": "Web",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/web/node_modules/vite/bin/vite.js",
+      "args": ["--host"],
+      "cwd": "${workspaceFolder}/src/web",
+      "outFiles": ["${workspaceFolder}/src/web/dist/**/*.js"],
+      "sourceMaps": true,
+      "skipFiles": ["<node_internals>/**"],
+      "serverReadyAction": {
+        "action": "debugWithEdge",
+        "killOnServerStop": true,
+        "pattern": "Local: (http://localhost:\\d+)",
+        "uriFormat": "%s",
+        "webRoot": "${workspaceFolder}/src/web"
+      }
+    } 
+  ],
+  "compounds": [
+    {
+      "name": "Start API(backend) and Web(fronted)",
+      "configurations": [
+        "Python: FastAPI",
+        "Web"
+      ]
     }
   ]
 }

README.md

@@ -45,7 +45,7 @@ description: Using Azure OpenAI agent with Python, integrating Bing Search API a
 
 ![App preview](images/app_preview.png)
 
-![Agent workflow preview](images/agent.png)
+![Agent workflow preview](images/updated_app_image.png)
 
 Contoso Creative Writer is an app that will help you write well researched, product specific articles. Enter the required information and then click "Start Work". To watch the steps in the agent workflow select the debug button in the bottom right corner of the screen. The result will begin writing once the agents complete the tasks to write the article.
 
@@ -68,8 +68,8 @@ This project template provides the following features:
 
 * **Azure account**. If you're new to Azure, [get an Azure account for free](https://azure.microsoft.com/free/cognitive-search/) and you'll get some free Azure credits to get started. See [guide to deploying with the free trial](docs/deploy_lowcost.md).
 * **Azure subscription with access enabled for the Azure OpenAI Service**. You can request access with [this form](https://aka.ms/oaiapply). If your access request to Azure OpenAI Service doesn't match the [acceptance criteria](https://learn.microsoft.com/legal/cognitive-services/openai/limited-access?context=%2Fazure%2Fcognitive-services%2Fopenai%2Fcontext%2Fcontext), you can use [OpenAI public API](https://platform.openai.com/docs/api-reference/introduction) instead.
-    - Ability to deploy `gpt-35-turbo-0613`,`gpt-4-1106-Preview` and `gpt-4o-2024-05-13`.
-    - We recommend using Canada East, as this region has access to all models and services required. 
+    - Ability to deploy `gpt-4o` and `gpt-4o-mini`.
+    - We recommend using `East US 2`, as this region has access to all models and services required. 
 * **Azure subscription with access enabled for [Bing Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api)**
 * **Azure subscription with access enabled for [Azure AI Search](https://azure.microsoft.com/en-gb/products/ai-services/ai-search)**
 
@@ -105,12 +105,14 @@ The easiest way to get started is GitHub Codespaces, since it will setup all the
     azd up
     ```
 
-    You will be prompted to select some details about your deployed resources, including location. As a reminder we recommend Canada East as the region for this project.
+    You will be prompted to select some details about your deployed resources, including location. As a reminder we recommend `East US 2` as the region for this project.
     Once the deployment is complete you should be able to scroll up in your terminal and see the url that the app has been deployed to. It should look similar to this 
     `Ingress Updated. Access your app at https://env-name.codespacesname.eastus2.azurecontainerapps.io/`. Navigate to the link to try out the app straight away! 
 
 5. Once the above steps are completed you can [test the sample](#testing-the-sample). 
 
+NOTE: If the api is failing due to permissions, add the AppID as Contributor in the IAM for the Resource Group.
+
 ### VS Code Dev Containers
 
 A related option is VS Code Dev Containers, which will open the project in your local VS Code using the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers):
@@ -146,7 +148,7 @@ A related option is VS Code Dev Containers, which will open the project in your
 1. Create a new folder and switch to it in the terminal, then run this command to download the project code:
 
     ```shell
-    azd init -t agent-openai-python-prompty
+    azd init -t contoso-creative-writer
     ```
     Note that this command will initialize a git repository, so you do not need to clone this repository.
 
@@ -185,7 +187,7 @@ Once you've opened the project in [Codespaces](#github-codespaces), [Dev Contain
     azd up
     ```
 
-    This project uses `gpt-35-turbo-0613`,`gpt-4-1106-Preview` and `gpt-4o-2024-05-13` which may not be available in all Azure regions. Check for [up-to-date region availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability) and select a region during deployment accordingly. We recommend using Canada East for this project.
+    This project uses `gpt-4o` and `gpt-4o-mini`. which may not be available in all Azure regions. Check for [up-to-date region availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability) and select a region during deployment accordingly. We recommend using `East US 2` for this project.
 
    After running azd up, you may be asked the following question during `Github Setup`:
 
@@ -262,7 +264,7 @@ To test the sample:
 To activate the Prompty tracing server:
 
 ```
-export LOCAL_TRACING=true
+export LOCAL_TRACING_ENABLED=true
 ```
 
 Then start the orchestrator:
@@ -277,7 +279,7 @@ This shows you all the Python functions that were called in order to generate th
 
 ## Evaluating results
 
-Contoso Creative Writer uses evaluators to assess application response quality. The 4 metrics the evaluators in this project assess are Coherence, Fluency, Relevance and Groundedness. A custom `evaluate.py` script has been written to run all evaulations for you.
+Contoso Creative Writer uses evaluators to assess application response quality. The 5 metrics the evaluators in this project assess are Coherence, Fluency, Relevance, Groundedness and Content Safety. A custom `evaluate.py` script has been written to run all evaulations for you.
 
 1. To run the script run the following commands:
 
@@ -286,7 +288,7 @@ cd ./src/api
 python -m evaluate.evaluate
 ```
 
-- Check: You see scores for Coherence, Fluency, Relevance and Groundedness.
+- Check: You see scores for Coherence, Fluency, Relevance, Groundedness and Content Safety.
 - Check: The scores are between 1 and 5
 
 
@@ -303,13 +305,30 @@ To set up CI/CD with GitHub actions on your repository, run the following comman
 ```shell
 azd pipeline config
 ```
+Add permission to Service Principal that is created:
+```
+Azure AI Administrator
+AzureML Data Scientist
+Cognitive Services OpenAI Contributor
+Contributor
+Key Vault Administrator
+Key Vault Contributor
+Key Vault Reader
+Search Index Data Contrbutor
+Storage Blob Data Contributor
+```
 
 ## Guidance
 
 ### Region Availability
 
-This template uses `gpt-35-turbo-0613`,`gpt-4-1106-Preview` and `gpt-4o-2024-05-13` which may not be available in all Azure regions. Check for [up-to-date region availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability) and select a region during deployment accordingly
-  * We recommend using Canada East
+This template uses `gpt-4o` and `gpt-4o-mini`. which may not be available in all Azure regions. Check for [up-to-date region availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability) and select a region during deployment accordingly
+  * We recommend using `East US 2`
+
+We use Risk and Safety Evaluators with region availability [here](https://learn.microsoft.com/en-gb/azure/ai-studio/how-to/develop/evaluate-sdk#region-support)
+We use AI Content Safety Evaluations,including image evaluations with region availability [here](https://learn.microsoft.com/en-us/azure/ai-services/content-safety/overview#region-availability)
+
+For the most comprehensive use we `East US 2`
 
 ### Costs
 

api-test.http

@@ -0,0 +1,9 @@
+
+post http://127.0.0.1:8000/api/article HTTP/1.1
+Content-Type: application/json
+
+{
+    "research": "Can you find the latest camping trends and what folks are doing in the winter?",
+    "products": "Can you use a selection of tents and sleeping bags as context?",
+    "assignment": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+}

azure.yaml

@@ -36,7 +36,6 @@ pipeline:
     - AZURE_RESOURCE_GROUP
     - AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME
     - AZURE_OPENAI_DEPLOYMENT_NAME
-    - AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME
     - AZURE_SEARCH_ENDPOINT
     - BING_SEARCH_ENDPOINT
     - AZURE_OPENAI_NAME
@@ -47,4 +46,4 @@ pipeline:
     - AZURE_SEARCH_NAME
     - WEB_SERVICE_ACA_URI
   secrets:
-    - BING_SEARCH_KEY
+    - BING_SEARCH_KEY

data/creativeevaldataset.jsonl

@@ -0,0 +1,57 @@
+[
+    {
+        "research_context": "Can you find the latest camping trends and what folks are doing in the winter?",
+        "product_context": "Can you use a selection of tents and sleeping bags as context?",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "Can you investigate the rise of glamping and its impact on traditional camping?",
+        "product_context": "Include examples of luxury tents and high-end sleeping bags in your discussion.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "What are the latest trends in family camping and how are families choosing their gear?",
+        "product_context": "Use a variety of family-sized tents and sleeping bags as context.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "Can you analyze the trend of digital detox camping and how it influences gear choices?",
+        "product_context": "Include examples of tents and sleeping bags that promote a tech-free experience.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "Can you delve into the trend of adventure camping and how it influences gear choices for extreme conditions?",
+        "product_context": "Use a selection of rugged tents and specialized sleeping bags as context.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "What are the current trends in pet-friendly camping and how are campers choosing gear that accommodates their furry friends?",
+        "product_context": "Include examples of pet-friendly tents and sleeping bags.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "What are the emerging trends in camping technology and how are they changing the way campers choose their gear?",
+        "product_context": "Include examples of tech-integrated tents and sleeping bags.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "Can you explore the trend of eco-conscious hiking and how it influences gear choices?",
+        "product_context": "Use a selection of sustainable hiking apparel and gear as context.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "What are the current trends in solo hiking and how are individuals choosing their gear for safety and comfort?",
+        "product_context": "Include examples of solo-friendly tents and compact cooking gear.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "Can you analyze the rise of family hiking and how families are selecting gear that accommodates all ages?",
+        "product_context": "Use a variety of family-sized backpacks and kid-friendly hiking gear as examples.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    },
+    {
+        "research_context": "What are the emerging trends in hiking technology and how are they changing the way hikers choose their gear?",
+        "product_context": "Include examples of GPS devices and smart hiking apparel.",
+        "assignment_context": "Write a fun and engaging article that includes the research and product information. The article should be between 800 and 1000 words. Make sure to cite sources in the article as you mention the research not at the end."
+    }
+]