{"schema_version":1,"default_baseline":"latest","default_ticket_head":"cpu","dates":{"clm":"2026-09-24","kev":"2026-09-26","jev_original":"2026-09-24","jev_latest":"2026-09-26"},"suite_hashes":{"decisions":"6ef6c158aba336836457f59c1e15e64487eb18c34b14128082bac3b6a2bdade0","bfcl":{"description":"4fed399cd694cf4008f9c9ba57d89862206e4712049594c083465013dbb715a1","schema":"e3f42a8a16daade9cdd11ebdcd16bdc7f6a09fba1f5c8477e24448acd7634188","prose_schema":"79897ecc1a078a94671335b7af1e22aaec6ea93d4d249ff8721aa255fd61b979"},"tickets":"fc3f42b2590f0883391eed3a79d487d5b60302b608d1c2afcaffe83c938e0dbe"},"groups":{"decisions":{"quality":{"id":"narrow-quality","title":"Quality at each decision","rows":[{"id":"tool_routing_accuracy","label":"Choose one of 15 tools / accuracy","kind":"percent","direction":"higher","values":{"clm":0.5,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":40},{"id":"trajectory_selection_accuracy","label":"Keep one of four trajectories / accuracy","kind":"percent","direction":"higher","values":{"clm":0.5833333333333334,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":24},{"id":"result_relevance_f1","label":"Tool-result relevance / positive F1","kind":"f1","direction":"higher","values":{"clm":0.18181818181818182,"jev_original":0.8695652173913043,"kev":0.9523809523809523,"jev_latest":0.851063829787234},"denominator":40},{"id":"result_relevance_recall","label":"Tool-result relevance / recall","kind":"percent","direction":"higher","values":{"clm":0.1,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":40},{"id":"subgoal_satisfaction_f1","label":"Subgoal completion / positive F1","kind":"f1","direction":"higher","values":{"clm":0.5714285714285714,"jev_original":1.0,"kev":0.975609756097561,"jev_latest":1.0},"denominator":40},{"id":"subgoal_satisfaction_recall","label":"Subgoal completion / recall","kind":"percent","direction":"higher","values":{"clm":0.4,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":40},{"id":"file_context_f1","label":"File-context selection / positive F1","kind":"f1","direction":"higher","values":{"clm":0.36363636363636365,"jev_original":0.7346938775510204,"kev":0.85,"jev_latest":0.6666666666666666},"denominator":225},{"id":"file_context_recall","label":"File-context selection / recall","kind":"percent","direction":"higher","values":{"clm":0.2222222222222222,"jev_original":1.0,"kev":0.9444444444444444,"jev_latest":0.9444444444444444},"denominator":225}],"note":"F1 balances precision and recall. Recall measures how much relevant evidence was recovered. Accuracy is used only for the two choice tasks."},"timing":{"id":"narrow-timing","title":"Client response times","rows":[{"id":"tool_routing_latency_ms_p50","label":"Choose one of 15 tools / median","kind":"ms","direction":"lower","values":{"clm":35.63,"jev_original":116.27,"kev":37.33,"jev_latest":125.8}},{"id":"tool_routing_latency_ms_p95","label":"Choose one of 15 tools / p95","kind":"ms","direction":"lower","values":{"clm":51.67,"jev_original":154.62,"kev":89.95,"jev_latest":272.16}},{"id":"trajectory_selection_latency_ms_p50","label":"Keep one of four trajectories / median","kind":"ms","direction":"lower","values":{"clm":90.22,"jev_original":136.29,"kev":32.37,"jev_latest":124.31}},{"id":"trajectory_selection_latency_ms_p95","label":"Keep one of four trajectories / p95","kind":"ms","direction":"lower","values":{"clm":99.11,"jev_original":162.91,"kev":33.53,"jev_latest":171.91}},{"id":"result_relevance_latency_ms_p50","label":"Tool-result relevance / median","kind":"ms","direction":"lower","values":{"clm":36.32,"jev_original":115.64,"kev":31.12,"jev_latest":124.75}},{"id":"result_relevance_latency_ms_p95","label":"Tool-result relevance / p95","kind":"ms","direction":"lower","values":{"clm":51.97,"jev_original":174.37,"kev":83.71,"jev_latest":173.07}},{"id":"subgoal_satisfaction_latency_ms_p50","label":"Subgoal completion / median","kind":"ms","direction":"lower","values":{"clm":37.79,"jev_original":115.98,"kev":31.1,"jev_latest":132.31}},{"id":"subgoal_satisfaction_latency_ms_p95","label":"Subgoal completion / p95","kind":"ms","direction":"lower","values":{"clm":59.39,"jev_original":208.58,"kev":31.91,"jev_latest":173.93}},{"id":"file_context_latency_ms_p50","label":"File-context selection / median","kind":"ms","direction":"lower","values":{"clm":35.56,"jev_original":114.29,"kev":30.97,"jev_latest":128.16}},{"id":"file_context_latency_ms_p95","label":"File-context selection / p95","kind":"ms","direction":"lower","values":{"clm":51.38,"jev_original":148.62,"kev":31.3,"jev_latest":179.56}}],"note":"Latency includes network and serving caches. CLM and Kev were local; Jev was hosted. These are deployment measurements from separate runs."},"diagnostics":{"id":"narrow-diagnostics","title":"Read the errors together","rows":[{"id":"result_relevance_precision","label":"Tool-result relevance / precision","kind":"percent","direction":"higher","values":{"clm":1.0,"jev_original":0.7692307692307693,"kev":0.9090909090909091,"jev_latest":0.7407407407407407}},{"id":"result_relevance_fp","label":"Tool-result relevance / false positives","kind":"count","direction":"lower","values":{"clm":0,"jev_original":6,"kev":2,"jev_latest":7}},{"id":"result_relevance_fn","label":"Tool-result relevance / false negatives","kind":"count","direction":"lower","values":{"clm":18,"jev_original":0,"kev":0,"jev_latest":0}},{"id":"result_relevance_brier","label":"Tool-result relevance / brier error","kind":"score","direction":"lower","values":{"clm":0.253863594072625,"jev_original":0.11012499999999999,"kev":0.05196611425000001,"jev_latest":0.10923000000000001}},{"id":"subgoal_satisfaction_precision","label":"Subgoal completion / precision","kind":"percent","direction":"higher","values":{"clm":1.0,"jev_original":1.0,"kev":0.9523809523809523,"jev_latest":1.0}},{"id":"subgoal_satisfaction_fp","label":"Subgoal completion / false positives","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":1,"jev_latest":0}},{"id":"subgoal_satisfaction_fn","label":"Subgoal completion / false negatives","kind":"count","direction":"lower","values":{"clm":12,"jev_original":0,"kev":0,"jev_latest":0}},{"id":"subgoal_satisfaction_brier","label":"Subgoal completion / brier error","kind":"score","direction":"lower","values":{"clm":0.150140884146575,"jev_original":0.006657500000000001,"kev":0.021987827,"jev_latest":0.006495000000000001}},{"id":"file_context_precision","label":"File-context selection / precision","kind":"percent","direction":"higher","values":{"clm":1.0,"jev_original":0.5806451612903226,"kev":0.7727272727272727,"jev_latest":0.5151515151515151}},{"id":"file_context_fp","label":"File-context selection / false positives","kind":"count","direction":"lower","values":{"clm":0,"jev_original":13,"kev":5,"jev_latest":16}},{"id":"file_context_fn","label":"File-context selection / false negatives","kind":"count","direction":"lower","values":{"clm":14,"jev_original":0,"kev":1,"jev_latest":1}},{"id":"file_context_brier","label":"File-context selection / brier error","kind":"score","direction":"lower","values":{"clm":0.08701198612464,"jev_original":0.06420488888888888,"kev":0.05287302182222222,"jev_latest":0.06408044444444445}}],"note":"A low false-positive count can come from rejecting almost everything. Read precision alongside recall and false negatives. Lower Brier probability error is better."},"consistency":{"id":"narrow-consistency","title":"Consistency and the mixed total","rows":[{"id":"consistency","label":"Same semantic answer after a paired variation","kind":"count","direction":"higher","values":{"clm":52,"jev_original":72,"kev":71,"jev_latest":71},"denominator":72},{"id":"narrow_total","label":"Correct across all narrow requests","kind":"count","direction":"higher","values":{"clm":295,"jev_original":350,"kev":360,"jev_latest":345},"denominator":369}],"note":"The 72 task pairs are related variants, not independent examples. The overall count is dominated by 225 file judgments, of which 207 are negative. Rejecting every file scores 92% on that category."}},"bfcl":{"quality":{"id":"bfcl-quality","title":"Function-selection accuracy","rows":[{"id":"bfcl_description_accuracy","label":"Name + description","kind":"percent","direction":"higher","values":{"clm":0.7685554668794893,"jev_original":0.9760574620909817,"kev":0.9696727853152434,"jev_latest":0.9776536312849162},"denominator":1253},{"id":"bfcl_schema_accuracy","label":"Full JSON schema","kind":"percent","direction":"higher","values":{"clm":0.4748603351955307,"jev_original":0.9888268156424581,"kev":0.9704708699122107,"jev_latest":0.985634477254589},"denominator":1253},{"id":"bfcl_prose_schema_accuracy","label":"Parameters in prose","kind":"percent","direction":"higher","values":{"clm":0.5841979249800479,"jev_original":0.9880287310454908,"kev":0.8451715881883479,"jev_latest":0.9864325618515563},"denominator":1253}],"note":"Each cell uses the same 1,253 requests within its view. Only the selected function name is scored, not arguments, execution, or BFCL's overall leaderboard."},"timing":{"id":"bfcl-timing","title":"Client response times","rows":[{"id":"bfcl_description_latency_ms_p50","label":"Name + description / median","kind":"ms","direction":"lower","values":{"clm":35.43,"jev_original":111.44,"kev":35.34,"jev_latest":130.69}},{"id":"bfcl_description_latency_ms_p95","label":"Name + description / p95","kind":"ms","direction":"lower","values":{"clm":92.24,"jev_original":152.41,"kev":53.75,"jev_latest":183.2}},{"id":"bfcl_schema_latency_ms_p50","label":"Full JSON schema / median","kind":"ms","direction":"lower","values":{"clm":66.34,"jev_original":119.26,"kev":70.65,"jev_latest":137.22}},{"id":"bfcl_schema_latency_ms_p95","label":"Full JSON schema / p95","kind":"ms","direction":"lower","values":{"clm":173.52,"jev_original":162.87,"kev":137.22,"jev_latest":187.51}},{"id":"bfcl_prose_schema_latency_ms_p50","label":"Parameters in prose / median","kind":"ms","direction":"lower","values":{"clm":42.25,"jev_original":117.04,"kev":65.75,"jev_latest":134.55}},{"id":"bfcl_prose_schema_latency_ms_p95","label":"Parameters in prose / p95","kind":"ms","direction":"lower","values":{"clm":139.61,"jev_original":156.06,"kev":122.22,"jev_latest":181.18}}],"note":"Both candidate length and serving stack affect latency. Local-versus-hosted timings are not isolated model-compute comparisons."},"splits":{"description":{"id":"bfcl-split-description","title":"Name + description / the two official splits","rows":[{"id":"bfcl_description_multiple","label":"multiple / accuracy","kind":"percent","direction":"higher","values":{"clm":0.875,"jev_original":0.995,"kev":0.99,"jev_latest":0.995},"denominator":200},{"id":"bfcl_description_multiple_latency","label":"multiple / median","kind":"ms","direction":"lower","values":{"clm":70.9,"jev_original":112.21,"kev":32.41,"jev_latest":127.12}},{"id":"bfcl_description_live_multiple","label":"live_multiple / accuracy","kind":"percent","direction":"higher","values":{"clm":0.748338081671415,"jev_original":0.9724596391263058,"kev":0.9658119658119658,"jev_latest":0.9743589743589743},"denominator":1053},{"id":"bfcl_description_live_multiple_latency","label":"live_multiple / median","kind":"ms","direction":"lower","values":{"clm":31.99,"jev_original":111.16,"kev":35.56,"jev_latest":131.49}}],"note":""},"schema":{"id":"bfcl-split-schema","title":"Full JSON schema / the two official splits","rows":[{"id":"bfcl_schema_multiple","label":"multiple / accuracy","kind":"percent","direction":"higher","values":{"clm":0.525,"jev_original":1.0,"kev":0.995,"jev_latest":1.0},"denominator":200},{"id":"bfcl_schema_multiple_latency","label":"multiple / median","kind":"ms","direction":"lower","values":{"clm":99.94,"jev_original":111.84,"kev":46.01,"jev_latest":132.13}},{"id":"bfcl_schema_live_multiple","label":"live_multiple / accuracy","kind":"percent","direction":"higher","values":{"clm":0.46533713200379867,"jev_original":0.9867046533713201,"kev":0.9658119658119658,"jev_latest":0.9829059829059829},"denominator":1053},{"id":"bfcl_schema_live_multiple_latency","label":"live_multiple / median","kind":"ms","direction":"lower","values":{"clm":55.55,"jev_original":120.84,"kev":74.8,"jev_latest":138.34}}],"note":""},"prose_schema":{"id":"bfcl-split-prose_schema","title":"Parameters in prose / the two official splits","rows":[{"id":"bfcl_prose_schema_multiple","label":"multiple / accuracy","kind":"percent","direction":"higher","values":{"clm":0.76,"jev_original":1.0,"kev":0.98,"jev_latest":1.0},"denominator":200},{"id":"bfcl_prose_schema_multiple_latency","label":"multiple / median","kind":"ms","direction":"lower","values":{"clm":89.48,"jev_original":116.34,"kev":37.96,"jev_latest":135.85}},{"id":"bfcl_prose_schema_live_multiple","label":"live_multiple / accuracy","kind":"percent","direction":"higher","values":{"clm":0.5508072174738842,"jev_original":0.9857549857549858,"kev":0.8195631528964862,"jev_latest":0.9838556505223172},"denominator":1053},{"id":"bfcl_prose_schema_live_multiple_latency","label":"live_multiple / median","kind":"ms","direction":"lower","values":{"clm":36.25,"jev_original":117.37,"kev":66.86,"jev_latest":134.53}}],"note":""}},"sensitivity":{"id":"bfcl-sensitivity","title":"The effect of the candidate adapter","rows":[{"id":"format-delta-schema","label":"Full JSON schema minus name + description","kind":"pp","direction":"none","values":{"jev_latest":0.7980845969672745,"clm":-29.369513168395855,"kev":0.07980845969672856,"jev_original":1.276935355147646},"note":"Representation and included information both change; this is not a pure formatting test."},{"id":"format-delta-prose_schema","label":"Parameters in prose minus name + description","kind":"pp","direction":"none","values":{"jev_latest":0.8778930566640031,"clm":-18.43575418994414,"kev":-12.450119712689546,"jev_original":1.1971268954509173},"note":"Representation and included information both change; this is not a pure formatting test."}],"note":"These are changes within each model relative to its description view. Neither a positive nor a negative change is an overall performance ranking."}},"tickets":{"new_states":{"id":"tickets-new_states","title":"New ticket states","rows":[{"id":"tickets-new_states-accuracy","label":"Correct routing / accuracy","kind":"percent","direction":"higher","values":{"clm":0.675,"clm_cuda":0.65,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":40},{"id":"tickets-new_states-latency_ms_p50","label":"Median client latency","kind":"ms","direction":"lower","values":{"clm":35.39,"clm_cuda":29.86,"jev_original":252.49,"kev":31.83,"jev_latest":122.14}},{"id":"tickets-new_states-latency_ms_p95","label":"p95 client latency","kind":"ms","direction":"lower","values":{"clm":37.18,"clm_cuda":35.34,"jev_original":287.68,"kev":82.46,"jev_latest":168.36}}],"note":"The same 40 tickets and four team choices were sent serially in a fixed order. No client answer cache was used."},"repeated_states":{"id":"tickets-repeated_states","title":"Exact ticket replay","rows":[{"id":"tickets-repeated_states-accuracy","label":"Correct routing / accuracy","kind":"percent","direction":"higher","values":{"clm":0.675,"clm_cuda":0.65,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":40},{"id":"tickets-repeated_states-latency_ms_p50","label":"Median client latency","kind":"ms","direction":"lower","values":{"clm":1.95,"clm_cuda":2.1,"jev_original":263.1,"kev":31.42,"jev_latest":124.02}},{"id":"tickets-repeated_states-latency_ms_p95","label":"p95 client latency","kind":"ms","direction":"lower","values":{"clm":2.43,"clm_cuda":2.55,"jev_original":305.3,"kev":83.16,"jev_latest":220.62}}],"note":"The same 40 tickets and four team choices were sent serially in a fixed order. No client answer cache was used."}},"policy":{"noshield":{"id":"policy-noshield","title":"Model actions without a safety shield","rows":[{"id":"policy-noshield-survived","label":"Courses survived with zero deaths","kind":"count","direction":"higher","values":{"clm":0,"jev_original":1,"kev":0,"jev_latest":2},"denominator":5},{"id":"policy-noshield-deaths","label":"Total deaths","kind":"count","direction":"lower","values":{"clm":33,"jev_original":7,"kev":48,"jev_latest":5}},{"id":"policy-noshield-agreement","label":"Agreement with the physics planner","kind":"percent","direction":"higher","values":{"clm":0.888,"jev_original":0.994,"kev":0.933,"jev_latest":0.992}},{"id":"policy-noshield-latency","label":"Median answer arrival","kind":"ms","direction":"lower","values":{"clm":16.4,"jev_original":132.0,"kev":35.0,"jev_latest":133.7}},{"id":"policy-noshield-errors","label":"Recorded API errors","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":0,"jev_latest":3}}],"note":"Survival means zero deaths over a 60-second course. After a crash, play restarts, so one seed can contain multiple deaths. Median arrival is the median of the five per-seed p50 values."},"shield":{"id":"policy-shield","title":"Model plus the deterministic safety shield","rows":[{"id":"policy-shield-survived","label":"Courses survived with zero deaths","kind":"count","direction":"higher","values":{"clm":5,"jev_original":5,"kev":5,"jev_latest":5},"denominator":5},{"id":"policy-shield-deaths","label":"Total deaths","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":0,"jev_latest":0}},{"id":"policy-shield-agreement","label":"Agreement with the physics planner","kind":"percent","direction":"higher","values":{"clm":0.639,"jev_original":0.988,"kev":0.596,"jev_latest":0.988}},{"id":"policy-shield-interventions","label":"Safety-shield interventions","kind":"count","direction":"lower","values":{"clm":5259,"jev_original":20,"kev":3536,"jev_latest":27},"note":"Lower means less reliance on the shield; it is not a standalone safety score."},{"id":"policy-shield-latency","label":"Median answer arrival","kind":"ms","direction":"lower","values":{"clm":16.4,"jev_original":132.1,"kev":66.1,"jev_latest":132.9}},{"id":"policy-shield-errors","label":"Recorded API errors","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":0,"jev_latest":0}}],"note":"Survival means zero deaths over a 60-second course. After a crash, play restarts, so one seed can contain multiple deaths. Median arrival is the median of the five per-seed p50 values."},"seeds":{"noshield":{"id":"policy-seeds-noshield","title":"Per-course deaths / shield off","rows":[{"id":"policy-noshield-seed-0","label":"Seed 0 / deaths","kind":"count","direction":"lower","values":{"clm":7,"jev_original":1,"kev":10,"jev_latest":2}},{"id":"policy-noshield-seed-1","label":"Seed 1 / deaths","kind":"count","direction":"lower","values":{"clm":8,"jev_original":1,"kev":10,"jev_latest":0}},{"id":"policy-noshield-seed-2","label":"Seed 2 / deaths","kind":"count","direction":"lower","values":{"clm":5,"jev_original":3,"kev":9,"jev_latest":1}},{"id":"policy-noshield-seed-3","label":"Seed 3 / deaths","kind":"count","direction":"lower","values":{"clm":7,"jev_original":0,"kev":9,"jev_latest":0}},{"id":"policy-noshield-seed-4","label":"Seed 4 / deaths","kind":"count","direction":"lower","values":{"clm":6,"jev_original":2,"kev":10,"jev_latest":2}}],"note":""},"shield":{"id":"policy-seeds-shield","title":"Per-course deaths / shield on","rows":[{"id":"policy-shield-seed-0","label":"Seed 0 / deaths","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":0,"jev_latest":0}},{"id":"policy-shield-seed-1","label":"Seed 1 / deaths","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":0,"jev_latest":0}},{"id":"policy-shield-seed-2","label":"Seed 2 / deaths","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":0,"jev_latest":0}},{"id":"policy-shield-seed-3","label":"Seed 3 / deaths","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":0,"jev_latest":0}},{"id":"policy-shield-seed-4","label":"Seed 4 / deaths","kind":"count","direction":"lower","values":{"clm":0,"jev_original":0,"kev":0,"jev_latest":0}}],"note":""}}}},"overview":{"id":"overview","title":"Primary quality and survival results","rows":[{"id":"tool_routing_accuracy","label":"Decision / Choose one of 15 tools / accuracy","kind":"percent","direction":"higher","values":{"clm":0.5,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":40},{"id":"trajectory_selection_accuracy","label":"Decision / Keep one of four trajectories / accuracy","kind":"percent","direction":"higher","values":{"clm":0.5833333333333334,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":24},{"id":"result_relevance_f1","label":"Decision / Tool-result relevance / positive F1","kind":"f1","direction":"higher","values":{"clm":0.18181818181818182,"jev_original":0.8695652173913043,"kev":0.9523809523809523,"jev_latest":0.851063829787234},"denominator":40},{"id":"subgoal_satisfaction_f1","label":"Decision / Subgoal completion / positive F1","kind":"f1","direction":"higher","values":{"clm":0.5714285714285714,"jev_original":1.0,"kev":0.975609756097561,"jev_latest":1.0},"denominator":40},{"id":"file_context_f1","label":"Decision / File-context selection / positive F1","kind":"f1","direction":"higher","values":{"clm":0.36363636363636365,"jev_original":0.7346938775510204,"kev":0.85,"jev_latest":0.6666666666666666},"denominator":225},{"id":"bfcl_description_accuracy","label":"BFCL / Name + description","kind":"percent","direction":"higher","values":{"clm":0.7685554668794893,"jev_original":0.9760574620909817,"kev":0.9696727853152434,"jev_latest":0.9776536312849162},"denominator":1253},{"id":"bfcl_schema_accuracy","label":"BFCL / Full JSON schema","kind":"percent","direction":"higher","values":{"clm":0.4748603351955307,"jev_original":0.9888268156424581,"kev":0.9704708699122107,"jev_latest":0.985634477254589},"denominator":1253},{"id":"bfcl_prose_schema_accuracy","label":"BFCL / Parameters in prose","kind":"percent","direction":"higher","values":{"clm":0.5841979249800479,"jev_original":0.9880287310454908,"kev":0.8451715881883479,"jev_latest":0.9864325618515563},"denominator":1253},{"id":"tickets-new_states-accuracy","label":"New tickets / routing accuracy","kind":"percent","direction":"higher","values":{"clm":0.675,"clm_cuda":0.65,"jev_original":1.0,"kev":1.0,"jev_latest":1.0},"denominator":40},{"id":"policy-noshield-survived","label":"Unshielded / courses survived","kind":"count","direction":"higher","values":{"clm":0,"jev_original":1,"kev":0,"jev_latest":2},"denominator":5},{"id":"policy-shield-survived","label":"Shielded / courses survived","kind":"count","direction":"higher","values":{"clm":5,"jev_original":5,"kev":5,"jev_latest":5},"denominator":5}],"note":""}}