chore: token padding for claude

bench_engines.py

@@ -79,9 +79,13 @@ def get_engine(model_class: str, model_id: str, context_size: int = None):
         from kani.engines.anthropic import AnthropicEngine
 
         if model_id == "claude-3-5-sonnet-20241022":
-            return AnthropicEngine(model="claude-3-5-sonnet-20241022", temperature=0, max_context_size=context_size)
+            return AnthropicEngine(
+                model="claude-3-5-sonnet-20241022", temperature=0, max_context_size=context_size or 150000
+            )
         if model_id == "claude-3-5-haiku-20241022":
-            return AnthropicEngine(model="claude-3-5-haiku-20241022", temperature=0, max_context_size=context_size)
+            return AnthropicEngine(
+                model="claude-3-5-haiku-20241022", temperature=0, max_context_size=context_size or 150000
+            )
     # todo: qwen
     # todo: cohere
     raise ValueError("unknown engine")

experiments/fanoutqa/claude/full/2120afba8009bad3/events.jsonl

@@ -0,0 +1,4 @@
+{"type":"kani_spawn","timestamp":1731474330.483934,"id":"cd384e92-201b-4449-a6c6-0c82b89cf8f5","depth":0,"parent":null,"children":[],"always_included_messages":[],"chat_history":[],"state":"stopped","name":"root","engine_type":"AnthropicEngine","engine_repr":"AnthropicEngine(client=<anthropic.AsyncAnthropic object at 0x107ceeec0>, model='claude-3-5-sonnet-20241022', max_tokens=512, max_context_size=200000, hyperparams={'temperature': 0}, tokenizer=None)","functions":[{"name":"delegate","desc":"Ask a capable helper for help looking up a piece of information or performing an action.\nDo not simply repeat what the user said as instructions.\nYou should use this to break up complex user queries into multiple simpler steps.\nNOTE: Helpers cannot see previous parts of your conversation.","auto_retry":true,"auto_truncate":null,"after":"assistant","json_schema":{"properties":{"instructions":{"description":"Detailed instructions on what your helper should do to help you.","type":"string"}},"required":["instructions"],"type":"object"}}]}
+{"type":"kani_state_change","timestamp":1731474330.484055,"id":"cd384e92-201b-4449-a6c6-0c82b89cf8f5","state":"running"}
+{"type":"kani_message","timestamp":1731474330.4840791,"id":"cd384e92-201b-4449-a6c6-0c82b89cf8f5","msg":{"role":"user","content":"What were box office values of the Star Wars films in the prequel and sequel trilogies?","name":null,"tool_call_id":null,"tool_calls":null,"is_tool_call_error":null}}
+{"type":"root_message","timestamp":1731474330.484084,"msg":{"role":"user","content":"What were box office values of the Star Wars films in the prequel and sequel trilogies?","name":null,"tool_call_id":null,"tool_calls":null,"is_tool_call_error":null}}

experiments/fanoutqa/claude/full/2120afba8009bad3/state.json

@@ -0,0 +1,50 @@
+{
+  "id": "1731474330-cd33e067-602e-4fcb-82c2-4e39b4c15e49",
+  "title": "fanoutqa: What were box office values of the Star Wars films in the prequel and sequel trilogies? (2120afba8009bad3)",
+  "last_modified": 1731474330.487097,
+  "n_events": 4,
+  "state": [
+    {
+      "id": "cd384e92-201b-4449-a6c6-0c82b89cf8f5",
+      "depth": 0,
+      "parent": null,
+      "children": [],
+      "always_included_messages": [],
+      "chat_history": [
+        {
+          "role": "user",
+          "content": "What were box office values of the Star Wars films in the prequel and sequel trilogies?",
+          "name": null,
+          "tool_call_id": null,
+          "tool_calls": null,
+          "is_tool_call_error": null
+        }
+      ],
+      "state": "stopped",
+      "name": "root",
+      "engine_type": "AnthropicEngine",
+      "engine_repr": "AnthropicEngine(client=<anthropic.AsyncAnthropic object at 0x107ceeec0>, model='claude-3-5-sonnet-20241022', max_tokens=512, max_context_size=200000, hyperparams={'temperature': 0}, tokenizer=None)",
+      "functions": [
+        {
+          "name": "delegate",
+          "desc": "Ask a capable helper for help looking up a piece of information or performing an action.\nDo not simply repeat what the user said as instructions.\nYou should use this to break up complex user queries into multiple simpler steps.\nNOTE: Helpers cannot see previous parts of your conversation.",
+          "auto_retry": true,
+          "auto_truncate": null,
+          "after": "assistant",
+          "json_schema": {
+            "properties": {
+              "instructions": {
+                "description": "Detailed instructions on what your helper should do to help you.",
+                "type": "string"
+              }
+            },
+            "required": [
+              "instructions"
+            ],
+            "type": "object"
+          }
+        }
+      ]
+    }
+  ]
+}

experiments/fanoutqa/claude/full/results.jsonl

@@ -0,0 +1 @@
+{"id": "7dcbbbdc7f1120cd", "answer": "Here are the batting hands for the first five picks in the 1998 MLB draft:\n\n1. Pat Burrell - Right-handed\n2. Mark Mulder - Left-handed\n3. Corey Patterson - Left-handed\n4. Jeff Austin - Right-handed\n5. J.D. Drew - Left-handed", "question": "What is the batting hand of each of the first five picks in the 1998 MLB draft?", "log_dir": "/Users/andrew/Desktop/Code/_penn/redel-experiments/experiments/fanoutqa/claude/full/7dcbbbdc7f1120cd"}