{"built":"2026-10-15","benchmark":"Terminal-Bench 2.0, held-out test split","settings":[{"id":"flash_lite","results_file":"bench/results/test_flash_lite.json","config_file":"bench/configs/gemini_argon_lite.yaml","arm":"jah-auto-t75","strong":"gemini-4-argon","strong_effort":"high","fast":"gemini-3.5-flash-lite","fast_effort":"low","decision":{"type":"jev","provider":"openrouter"},"tau":0.75,"results":{"dsh:jah-auto-t75":{"n":74,"success":0.6621621621621622,"cost_per_task":0.6820989625675675,"agent_s_per_task":684.0904856216218,"strong_share":0.5011354825867953,"infra_errors":7,"timeouts":8,"vs":"jah-strong","paired_n":74,"delta_pt":0.0,"delta_ci_pt":[-8.108108108108109,8.108108108108109],"wins":5,"losses":5,"cost_saving":0.27780272711533305,"time_saving":0.1501551549546133},"dsh:jah-strong":{"n":74,"success":0.6621621621621622,"cost_per_task":0.944477344594594,"agent_s_per_task":804.9592694594593,"strong_share":1.0,"infra_errors":6,"timeouts":16},"mswe:jah-auto-t75":{"n":74,"success":0.6351351351351351,"cost_per_task":0.3296905855405405,"agent_s_per_task":471.0289423378377,"strong_share":0.453769013726647,"infra_errors":3,"timeouts":7,"vs":"jah-strong","paired_n":74,"delta_pt":-2.7027027027027026,"delta_ci_pt":[-14.864864864864865,8.108108108108109],"wins":8,"losses":10,"cost_saving":0.6374004092086328,"time_saving":0.428176982970752},"mswe:jah-strong":{"n":74,"success":0.6621621621621622,"cost_per_task":0.9092414716216216,"agent_s_per_task":823.7320434999999,"strong_share":1.0,"infra_errors":3,"timeouts":16},"pi:jah-auto-t75":{"n":74,"success":0.5675675675675675,"cost_per_task":0.4242397687837837,"agent_s_per_task":650.1429811891894,"strong_share":0.5460778457604228,"infra_errors":3,"timeouts":15,"vs":"jah-strong","paired_n":74,"delta_pt":4.054054054054054,"delta_ci_pt":[-6.756756756756757,14.864864864864865],"wins":10,"losses":7,"cost_saving":0.31089019737557944,"time_saving":0.044510951889717765},"pi:jah-strong":{"n":74,"success":0.527027027027027,"cost_per_task":0.6156344999999999,"agent_s_per_task":680.4295480675673,"strong_share":1.0,"infra_errors":4,"timeouts":17},"t2:jah-auto-t75":{"n":74,"success":0.4189189189189189,"cost_per_task":0.13467201459459455,"agent_s_per_task":966.7975510810811,"strong_share":0.44379119864061617,"infra_errors":0,"timeouts":31,"vs":"jah-strong","paired_n":74,"delta_pt":14.864864864864865,"delta_ci_pt":[4.054054054054054,25.675675675675674],"wins":15,"losses":4,"cost_saving":-0.27310441435215194,"time_saving":0.2875468148262573},"t2:jah-strong":{"n":74,"success":0.2702702702702703,"cost_per_task":0.10578237972972976,"agent_s_per_task":1356.9980052027033,"strong_share":1.0,"infra_errors":0,"timeouts":53}}},{"id":"argon_flash","results_file":"bench/results/test_argon_flash.json","config_file":"bench/configs/gemini_argon_flash.yaml","arm":"jah-auto","strong":"gemini-4-argon","strong_effort":"high","fast":"gemini-3.8-flash","fast_effort":"low","decision":{"type":"jev","provider":"openrouter"},"tau":0.5,"results":{"dsh:jah-auto":{"n":74,"success":0.7027027027027027,"cost_per_task":0.8818495847297296,"agent_s_per_task":629.0847851351351,"strong_share":0.5035077949158693,"infra_errors":7,"timeouts":11,"vs":"jah-strong","paired_n":74,"delta_pt":4.054054054054054,"delta_ci_pt":[-5.405405405405405,13.513513513513514],"wins":8,"losses":5,"cost_saving":0.06630943582002657,"time_saving":0.21848867513809256},"dsh:jah-strong":{"n":74,"success":0.6621621621621622,"cost_per_task":0.944477344594594,"agent_s_per_task":804.9592694594593,"strong_share":1.0,"infra_errors":6,"timeouts":16},"mswe:jah-auto":{"n":74,"success":0.7432432432432432,"cost_per_task":0.7881114602702701,"agent_s_per_task":686.0418654729727,"strong_share":0.522843147551415,"infra_errors":3,"timeouts":9,"vs":"jah-strong","paired_n":74,"delta_pt":8.108108108108109,"delta_ci_pt":[0.0,17.56756756756757],"wins":9,"losses":3,"cost_saving":0.13322094859500577,"time_saving":0.1671540874408477},"mswe:jah-strong":{"n":74,"success":0.6621621621621622,"cost_per_task":0.9092414716216216,"agent_s_per_task":823.7320434999999,"strong_share":1.0,"infra_errors":3,"timeouts":16},"pi:jah-auto":{"n":74,"success":0.5945945945945946,"cost_per_task":0.5536130917567568,"agent_s_per_task":563.1531817027027,"strong_share":0.5654949372973315,"infra_errors":4,"timeouts":9,"vs":"jah-strong","paired_n":74,"delta_pt":6.756756756756757,"delta_ci_pt":[-2.7027027027027026,17.56756756756757],"wins":10,"losses":5,"cost_saving":0.10074388008346358,"time_saving":0.17235636914641905},"pi:jah-strong":{"n":74,"success":0.527027027027027,"cost_per_task":0.6156344999999999,"agent_s_per_task":680.4295480675673,"strong_share":1.0,"infra_errors":4,"timeouts":17},"t2:jah-auto":{"n":74,"success":0.33783783783783783,"cost_per_task":0.19234971432432432,"agent_s_per_task":1262.7118713918917,"strong_share":0.5174124492775015,"infra_errors":0,"timeouts":46,"vs":"jah-strong","paired_n":74,"delta_pt":6.756756756756757,"delta_ci_pt":[0.0,13.513513513513514],"wins":6,"losses":1,"cost_saving":-0.8183530642416159,"time_saving":0.06948140929413349},"t2:jah-strong":{"n":74,"success":0.2702702702702703,"cost_per_task":0.10578237972972976,"agent_s_per_task":1356.9980052027033,"strong_share":1.0,"infra_errors":0,"timeouts":53}}},{"id":"codex_openai","results_file":"bench/results/test_codex_openai.json","config_file":"bench/configs/openai_sol_luna.yaml","arm":"jah-auto","strong":"gpt-6.1-sol","strong_effort":"medium","fast":"gpt-6-luna","fast_effort":"low","decision":{"type":"jev","provider":"openrouter"},"tau":0.5,"results":{"codex:jah-auto":{"n":20,"success":0.85,"cost_per_task":0.1580649185,"agent_s_per_task":272.94254635000004,"strong_share":0.5288985140676317,"infra_errors":0,"timeouts":1,"vs":"jah-strong","paired_n":20,"delta_pt":0.0,"delta_ci_pt":[-15.0,15.0],"wins":1,"losses":1,"cost_saving":0.06522068881499166,"time_saving":-0.46866787083322725},"codex:jah-strong":{"n":20,"success":0.85,"cost_per_task":0.16909330000000006,"agent_s_per_task":185.843615,"strong_share":1.0,"infra_errors":0,"timeouts":0}}}],"links":{"repo":"https://github.com/JevAgentHarness/JevAgentHarness/blob/main/","protocol":"https://github.com/JevAgentHarness/JevAgentHarness/blob/main/bench/PROTOCOL.md"}}