chore: some travelplanner eval fixes

zhudotexe · Dec 2, 2024 · 5632993 · 5632993
1 parent b32ec03
commit 5632993
Show file tree

Hide file tree

Showing 9 changed files with 905 additions and 189 deletions.
diff --git a/experiments/travelplanner/cohere-hf/baseline/results_for_tp_eval.jsonl b/experiments/travelplanner/cohere-hf/baseline/results_for_tp_eval.jsonl
diff --git a/experiments/travelplanner/cohere-hf/full/results_for_tp_eval.jsonl b/experiments/travelplanner/cohere-hf/full/results_for_tp_eval.jsonl
diff --git a/experiments/travelplanner/cohere-hf/root-fc/results_for_tp_eval.jsonl b/experiments/travelplanner/cohere-hf/root-fc/results_for_tp_eval.jsonl
diff --git a/experiments/travelplanner/cohere-hf/small-all/results_for_tp_eval.jsonl b/experiments/travelplanner/cohere-hf/small-all/results_for_tp_eval.jsonl
diff --git a/experiments/travelplanner/cohere-hf/small-baseline/results_for_tp_eval.jsonl b/experiments/travelplanner/cohere-hf/small-baseline/results_for_tp_eval.jsonl
diff --git a/...ents/travelplanner/mistral/id_to_idx.json → experiments/travelplanner/id_to_idx.json b/...ents/travelplanner/mistral/id_to_idx.json → experiments/travelplanner/id_to_idx.json
diff --git a/experiments/travelplanner/validation/id_to_idx.json b/experiments/travelplanner/validation/id_to_idx.json
diff --git a/score_travelplanner.py b/score_travelplanner.py
@@ -12,10 +12,8 @@
 
 from redel.utils import read_jsonl
 
-EXPECTED_RESULTS = {
-    "validation": 180,
-    "mistral": 180,
-}
+EXPECTED_RESULTS = 180
+ID_TO_IDX_MAP = Path(__file__).parent / "experiments/travelplanner/id_to_idx.json"
 
 
 async def transform_submission(fp: Path):
@@ -24,7 +22,7 @@ async def transform_submission(fp: Path):
     split = fp.parents[1].name
     transformed = []
     try:
-        with open(fp.parents[1] / "id_to_idx.json") as f:
+        with open(ID_TO_IDX_MAP) as f:
             id_to_idx = json.load(f)
     except FileNotFoundError:
         id_to_idx = {}
@@ -45,7 +43,7 @@ async def transform_submission(fp: Path):
         transformed.append({"idx": idx, "query": query, "plan": plan})
 
     # ensure there are the right number of results, and they're sorted by idx
-    missing_idxs = set(range(EXPECTED_RESULTS[split])).difference({t["idx"] for t in transformed})
+    missing_idxs = set(range(EXPECTED_RESULTS)).difference({t["idx"] for t in transformed})
     if missing_idxs:
         print(f"WARN: Submission is missing indices {missing_idxs}")
         for idx in missing_idxs:

diff --git a/slurm/gen_slurm.py b/slurm/gen_slurm.py
@@ -49,7 +49,7 @@
     # ModelConfig(model_class="openai", large="gpt-4o-2024-05-13", small="gpt-3.5-turbo-0125", size=0, extras=""),
     ModelConfig(
         model_class="mistral",
-        large="mistralai/Mistral-Large-Instruct-2407",
+        large="mistralai/Mistral-Large-Instruct-2411",
         small="mistralai/Mistral-Small-Instruct-2409",
         size=8,
         extras="--engine-timeout 1800",  # 30 min timeout per trial