{"built":"2026-10-15","fig4a":{"points":[{"slug":"e12","name":"AgentDecision-4B","group":"ours","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.0045,"acc":84.8,"log10_usd_per_decision":-5.3513,"src":{"usd_per_1k":{"key":"e30/e12/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/e12/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/e12/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":31.0},{"slug":"e28","name":"AgentDecision-0.8B","group":"ours","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.0018,"acc":80.5,"log10_usd_per_decision":-5.7387,"src":{"usd_per_1k":{"key":"e30/e28/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/e28/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/e28/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":27.0},{"slug":"jev","name":"Jev","group":"dm","basis":"e9","hosted":true,"marker":"disc","usd_per_1k":0.016,"acc":86.4,"log10_usd_per_decision":-4.7877,"src":{"usd_per_1k":{"key":"e9/jev/usdk","file":"research/paper/generated/e9_frontier.tex"},"acc":{"key":"e9/jev/acc","file":"research/paper/generated/e9_frontier.tex"},"log10_usd_per_decision":{"key":"e9/jev/lg","file":"research/paper/generated/e9_frontier.tex"}},"lat_p50":206.0},{"slug":"oaidec","name":"OpenAI Decisions API","group":"dm","basis":"e9","hosted":true,"marker":"disc","usd_per_1k":0.027,"acc":79.2,"log10_usd_per_decision":-4.5619,"src":{"usd_per_1k":{"key":"e9/oaidec/usdk","file":"research/paper/generated/e15_baselines.tex"},"acc":{"key":"e9/oaidec/acc","file":"research/paper/generated/e15_baselines.tex"},"log10_usd_per_decision":{"key":"e9/oaidec/lg","file":"research/paper/generated/e15_baselines.tex"}},"lat_p50":158.0},{"slug":"sl0.8b","name":"StartLux-Decision-0.8B","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.0017,"acc":69.3,"log10_usd_per_decision":-5.769,"src":{"usd_per_1k":{"key":"e30/sl0.8b/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/sl0.8b/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/sl0.8b/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":32.0},{"slug":"sl2b","name":"StartLux-Decision-2B","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.0027,"acc":76.0,"log10_usd_per_decision":-5.5625,"src":{"usd_per_1k":{"key":"e30/sl2b/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/sl2b/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/sl2b/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":34.0},{"slug":"sl4b","name":"StartLux-Decision-4B","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.0068,"acc":83.0,"log10_usd_per_decision":-5.1696,"src":{"usd_per_1k":{"key":"e30/sl4b/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/sl4b/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/sl4b/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":44.0},{"slug":"sl9b","name":"StartLux-Decision-9B","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.011,"acc":83.3,"log10_usd_per_decision":-4.9414,"src":{"usd_per_1k":{"key":"e30/sl9b/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/sl9b/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/sl9b/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":66.0},{"slug":"sl27b","name":"StartLux-Decision-27B","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.17,"acc":87.5,"log10_usd_per_decision":-3.7801,"src":{"usd_per_1k":{"key":"e30/sl27b/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/sl27b/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/sl27b/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"Q4_K_M","engine":"llama.cpp","lat_p50":649.0},{"slug":"intern","name":"Intern-Decision-4B","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.012,"acc":73.5,"log10_usd_per_decision":-4.9062,"src":{"usd_per_1k":{"key":"e30/intern/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/intern/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/intern/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":61.0},{"slug":"decider","name":"decider-4b","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.006,"acc":79.4,"log10_usd_per_decision":-5.2201,"src":{"usd_per_1k":{"key":"e30/decider/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/decider/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/decider/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":39.0},{"slug":"openjev","name":"OpenJev","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.011,"acc":79.1,"log10_usd_per_decision":-4.94,"src":{"usd_per_1k":{"key":"e30/openjev/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/openjev/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/openjev/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"its own image (g6e.xlarge)","lat_p50":144.0},{"slug":"openjevhf","name":"OpenJev-27B","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.18,"acc":83.0,"log10_usd_per_decision":-3.7507,"src":{"usd_per_1k":{"key":"e30/openjevhf/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/openjevhf/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/openjevhf/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"Q4_K_M","engine":"llama.cpp","lat_p50":744.0},{"slug":"jevk5","name":"JevK5","group":"dm","basis":"e30","hosted":false,"marker":"disc","usd_per_1k":0.0089,"acc":70.1,"log10_usd_per_decision":-5.0511,"src":{"usd_per_1k":{"key":"e30/jevk5/usdk","file":"research/paper/generated/e30_cost.tex"},"acc":{"key":"e30/jevk5/acc","file":"research/paper/generated/e30_cost.tex"},"log10_usd_per_decision":{"key":"e30/jevk5/lg","file":"research/paper/generated/e30_cost.tex"}},"prec":"FP8","engine":"vLLM","lat_p50":46.0},{"slug":"flash","name":"Gemini 3.5 Flash (multiple choice)","group":"llm","basis":"e9","hosted":true,"marker":"diamond","usd_per_1k":0.81,"acc":88.1,"log10_usd_per_decision":-3.0911,"src":{"usd_per_1k":{"key":"e9/flash/usdk","file":"research/paper/generated/e9_frontier.tex"},"acc":{"key":"e9/flash/acc","file":"research/paper/generated/e9_frontier.tex"},"log10_usd_per_decision":{"key":"e9/flash/lg","file":"research/paper/generated/e9_frontier.tex"}},"lat_p50":2570.0},{"slug":"lite","name":"Gemini 3.5 Flash-Lite (multiple choice)","group":"llm","basis":"e9","hosted":true,"marker":"diamond","usd_per_1k":0.19,"acc":83.3,"log10_usd_per_decision":-3.7302,"src":{"usd_per_1k":{"key":"e9/lite/usdk","file":"research/paper/generated/e9_frontier.tex"},"acc":{"key":"e9/lite/acc","file":"research/paper/generated/e9_frontier.tex"},"log10_usd_per_decision":{"key":"e9/lite/lg","file":"research/paper/generated/e9_frontier.tex"}},"lat_p50":681.0},{"slug":"flashfc","name":"Gemini 3.5 Flash, own tool call","group":"llm","basis":"e9","hosted":true,"marker":"diamond-open","usd_per_1k":0.73,"acc":78.7,"log10_usd_per_decision":-3.1384,"src":{"usd_per_1k":{"key":"e9/flashfc/usdk","file":"research/paper/generated/e9_frontier.tex"},"acc":{"key":"e9/flashfc/acc","file":"research/paper/generated/e9_frontier.tex"},"log10_usd_per_decision":{"key":"e9/flashfc/lg","file":"research/paper/generated/e9_frontier.tex"}},"lat_p50":884.0}],"frontier":["sl0.8b","e28","e12","jev","sl27b","flash"],"frontier_src":[{"key":"e30/front/first","file":"research/paper/generated/e30_cost.tex"},{"key":"e30/front/rest","file":"research/paper/generated/e30_cost.tex"}],"meta":{"gpu":"NVIDIA L4 24 GB","instance":"g6.xlarge","margin_pt":1.0,"src":[{"key":"e30/meta/gpu","file":"research/paper/generated/e30_cost.tex"},{"key":"e30/meta/margin","file":"research/paper/generated/e30_cost.tex"}]},"items_note":"3,991 isolated questions (BFCL v4 tool choice and arguments, When2Call)"},"fig4b":{"rows":[{"slug":"e12","name":"AgentDecision-4B","group":"ours","L1":89.3,"L1_key":"e12/e12/e1/acc/toolchoice","L2":94.9,"L2_key":"e21/e12/l2/all","L3":91.5,"L3_key":"e21/e12/l3/success","L3_n":200.0},{"slug":"e28","name":"AgentDecision-0.8B","group":"ours","L1":87.1,"L1_key":"e28/l1/tool","L2":95.6,"L2_key":"e28/l2/all","L3":82.0,"L3_key":"e28/l3/success"},{"slug":"jev","name":"Jev","group":"dm","L1":89.9,"L1_key":"explore/startlux_live/jev/e1_acc","L2":91.8,"L2_key":"e21/jev/l2/all","L3":82.0,"L3_key":"e21/jev/l3/success","L3_n":200.0},{"slug":"oaidec","name":"OpenAI Decisions API","group":"dm","L1":90.6,"L1_key":"e15/oaidec/e1_acc","L2":86.1,"L2_key":"e21/oaidec/l2/all","L3":66.0,"L3_key":"e21/oaidec/l3/success","L3_n":200.0},{"slug":"openjevhf","name":"OpenJev-27B","group":"dm","L1":90.2,"L1_key":"e15/openjevhf/e1_acc","L2":92.4,"L2_key":"e21/openjevhf/l2/all","L3":69.5,"L3_key":"e21/openjevhf/l3/success","L3_n":200.0},{"slug":"sl27b","name":"StartLux-Decision-27B","group":"dm","L1":90.8,"L1_key":"explore/startlux_live/sl27b/e1_acc","L2":94.7,"L2_key":"e21/sl27b/l2/all","L3":79.9,"L3_key":"e21/sl27b/l3/success","L3_n":164.0},{"slug":"sl9b","name":"StartLux-Decision-9B","group":"dm","L1":92.2,"L1_key":"explore/startlux_live/sl9b/e1_acc","L2":90.9,"L2_key":"e21/sl9b/l2/all","L3":69.5,"L3_key":"e21/sl9b/l3/success","L3_n":200.0},{"slug":"sl4b","name":"StartLux-Decision-4B","group":"dm","L1":91.3,"L1_key":"explore/startlux_live/sl4b/e1_acc","L2":86.5,"L2_key":"e21/sl4b/l2/all","L3":49.0,"L3_key":"e21/sl4b/l3/success","L3_n":200.0},{"slug":"intern","name":"Intern-Decision-4B","group":"dm","L1":88.2,"L1_key":"explore/startlux_live/intern/e1_acc","L2":75.9,"L2_key":"e21/intern/l2/all","L3":34.0,"L3_key":"e21/intern/l3/success","L3_n":200.0}],"levels":[{"level":"L1","task_set":"isolated: BFCL v4 tool choice only","metric":"accuracy (%)"},{"level":"L2","task_set":"DecisionGym-Shift open-loop states (100 tasks)","metric":"step accuracy, all states (%)"},{"level":"L3","task_set":"DecisionGym-Shift closed-loop (200 tasks)","metric":"task success (%)"}],"kendall_l2_l3":0.84,"kendall_l2_l3_n":11.0,"kendall_src":{"key":"e21/kendall/l2_vs_l3","file":"research/paper/generated/e21_shift.tex"},"flash_alone_l3":100.0},"links":{"e30":"https://github.com/JevAgentHarness/JevAgentHarness/blob/main/research/paper/generated/e30_cost.tex","e9":"https://github.com/JevAgentHarness/JevAgentHarness/blob/main/research/paper/generated/e9_frontier.tex","e15":"https://github.com/JevAgentHarness/JevAgentHarness/blob/main/research/paper/generated/e15_baselines.tex","e21":"https://github.com/JevAgentHarness/JevAgentHarness/blob/main/research/paper/generated/e21_shift.tex"}}