Merge branch 'refs/heads/main' into leaderboard-gemma

.github/workflows/leaderboard-submission.yml

@@ -65,6 +65,14 @@ jobs:
           FANOUTQA_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: python leaderboard-submissions/hydrate.py
 
+      # ensure the PR comment exists so we can edit it in future steps
+      - name: Ensure PR comment
+        if: steps.eval.outputs.changed > 0 || failure()
+        run: |
+          if [[ -z $(gh pr view ${{ github.event.number }} --json comments | jq '.comments[].author.login | select(. == "github-actions")') ]]; then
+            gh pr comment ${{ github.event.number }} -b "Results incoming...";
+          fi
+
       - name: Add PR comment (failure)
         if: failure()
         run: gh pr comment ${{ github.event.number }} --edit-last -b "It looks like this eval run failed. Please check the [workflow logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) to see what went wrong, then push a new commit to your PR to rerun the eval."

leaderboard-submissions/hydrate.py

@@ -1,6 +1,7 @@
 import hashlib
 import json
 import os
+import traceback
 from collections import namedtuple
 from dataclasses import asdict, dataclass
 from pathlib import Path
@@ -81,6 +82,7 @@ def hydrate_all():
         except Exception as e:
             # if invalid, log a check annotation and mark job failure
             print(f"::error file={metadata_fp},title=Could not eval submission::{e}")
+            traceback.print_exc()
             exit_code = 1
 
     print(f"Done. Wrote {len(written_files)} results files.")