[ { "model_config": "openrouter/anthropic/claude-fable-5[low]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 15342.833333333334, "tokens_per_success": 15342.833333333334, "conditional_tokens_among_solved_loops": 15342.833333333334, "mean_tokens_per_solved_task": 15342.833333333334, "mean_input_tokens_per_repair_loop": 14076.888888888889, "mean_output_tokens_per_repair_loop": 1265.9444444444443, "mean_cache_read_tokens_per_repair_loop": 9607.925925925925, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 4.62962962962963, "p95_turns_per_repair_loop": 7.0, "mean_agent_steps_per_repair_loop": 4.62962962962963, "p95_agent_steps_per_repair_loop": 7.0, "mean_verifier_submissions_per_repair_loop": 1.037037037037037, "mean_verifier_submissions_to_success": 1.037037037037037, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 6.350118, "mean_cost_per_repair_loop": 0.11759477777777778, "p95_cost_per_repair_loop": 0.326736, "cpsc": 0.11759477777777778, "realized_cpsc": 0.11759477777777778, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 6.350118, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.11759477777777778, "mean_spend_per_solved_task": 0.11759477777777778, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.12874403703703705, "mean_cost_delta_vs_gateway_per_repair_loop": -0.011149259259259256, "cost_delta_vs_gateway_ratio": -0.08660019924691224 }, { "model_config": "openrouter/anthropic/claude-opus-4.8[low]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 24816.444444444445, "tokens_per_success": 24816.444444444445, "conditional_tokens_among_solved_loops": 24816.444444444445, "mean_tokens_per_solved_task": 24816.444444444445, "mean_input_tokens_per_repair_loop": 23081.703703703704, "mean_output_tokens_per_repair_loop": 1734.7407407407406, "mean_cache_read_tokens_per_repair_loop": 17497.962962962964, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 5.962962962962963, "p95_turns_per_repair_loop": 10.0, "mean_agent_steps_per_repair_loop": 5.962962962962963, "p95_agent_steps_per_repair_loop": 10.0, "mean_verifier_submissions_per_repair_loop": 1.0925925925925926, "mean_verifier_submissions_to_success": 1.0925925925925926, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 4.321955, "mean_cost_per_repair_loop": 0.0800362037037037, "p95_cost_per_repair_loop": 0.1950455, "cpsc": 0.0800362037037037, "realized_cpsc": 0.0800362037037037, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 4.321955, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.0800362037037037, "mean_spend_per_solved_task": 0.0800362037037037, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.08700097222222222, "mean_cost_delta_vs_gateway_per_repair_loop": -0.006964768518518522, "cost_delta_vs_gateway_ratio": -0.08005391595772933 }, { "model_config": "openrouter/anthropic/claude-opus-4.8[medium]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 38642.851851851854, "tokens_per_success": 38642.851851851854, "conditional_tokens_among_solved_loops": 38642.851851851854, "mean_tokens_per_solved_task": 38642.851851851854, "mean_input_tokens_per_repair_loop": 35743.48148148148, "mean_output_tokens_per_repair_loop": 2899.3703703703704, "mean_cache_read_tokens_per_repair_loop": 28233.90740740741, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 7.5, "p95_turns_per_repair_loop": 14.0, "mean_agent_steps_per_repair_loop": 7.5, "p95_agent_steps_per_repair_loop": 14.0, "mean_verifier_submissions_per_repair_loop": 1.0185185185185186, "mean_verifier_submissions_to_success": 1.0185185185185186, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 6.7040505, "mean_cost_per_repair_loop": 0.12414908333333334, "p95_cost_per_repair_loop": 0.3171925, "cpsc": 0.12414908333333334, "realized_cpsc": 0.12414908333333334, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 6.7040505, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.12414908333333334, "mean_spend_per_solved_task": 0.12414908333333334, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.13351730092592592, "mean_cost_delta_vs_gateway_per_repair_loop": -0.009368217592592588, "cost_delta_vs_gateway_ratio": -0.07016482154466246 }, { "model_config": "openrouter/anthropic/claude-sonnet-5[low]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 67624.85185185185, "tokens_per_success": 67624.85185185185, "conditional_tokens_among_solved_loops": 67624.85185185185, "mean_tokens_per_solved_task": 67624.85185185185, "mean_input_tokens_per_repair_loop": 65288.7037037037, "mean_output_tokens_per_repair_loop": 2336.1481481481483, "mean_cache_read_tokens_per_repair_loop": 58230.555555555555, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 8.518518518518519, "p95_turns_per_repair_loop": 18.0, "mean_agent_steps_per_repair_loop": 8.518518518518519, "p95_agent_steps_per_repair_loop": 18.0, "mean_verifier_submissions_per_repair_loop": 1.6481481481481481, "mean_verifier_submissions_to_success": 1.6481481481481481, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 3.979035, "mean_cost_per_repair_loop": 0.07368583333333334, "p95_cost_per_repair_loop": 0.1609584, "cpsc": 0.07368583333333334, "realized_cpsc": 0.07368583333333334, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 3.979035, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.07368583333333334, "mean_spend_per_solved_task": 0.07368583333333334, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.052644444444444445, "mean_cost_delta_vs_gateway_per_repair_loop": 0.02104138888888889, "cost_delta_vs_gateway_ratio": 0.3996886872097932 }, { "model_config": "openrouter/anthropic/claude-sonnet-5[medium]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 47832.07407407407, "tokens_per_success": 47832.07407407407, "conditional_tokens_among_solved_loops": 47832.07407407407, "mean_tokens_per_solved_task": 47832.07407407407, "mean_input_tokens_per_repair_loop": 45099.40740740741, "mean_output_tokens_per_repair_loop": 2732.6666666666665, "mean_cache_read_tokens_per_repair_loop": 35097.81481481482, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 9.185185185185185, "p95_turns_per_repair_loop": 15.0, "mean_agent_steps_per_repair_loop": 9.185185185185185, "p95_agent_steps_per_repair_loop": 15.0, "mean_verifier_submissions_per_repair_loop": 1.037037037037037, "mean_verifier_submissions_to_success": 1.037037037037037, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 4.4023026, "mean_cost_per_repair_loop": 0.08152412222222222, "p95_cost_per_repair_loop": 0.2018985, "cpsc": 0.08152412222222222, "realized_cpsc": 0.08152412222222222, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 4.4023026, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.08152412222222222, "mean_spend_per_solved_task": 0.08152412222222222, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.059341025925925925, "mean_cost_delta_vs_gateway_per_repair_loop": 0.022183096296296292, "cost_delta_vs_gateway_ratio": 0.3738239430505795 }, { "model_config": "openrouter/google/gemini-3.5-flash[medium]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 307752.6296296296, "tokens_per_success": 307752.6296296296, "conditional_tokens_among_solved_loops": 307752.6296296296, "mean_tokens_per_solved_task": 307752.6296296296, "mean_input_tokens_per_repair_loop": 290188.8333333333, "mean_output_tokens_per_repair_loop": 17563.796296296296, "mean_cache_read_tokens_per_repair_loop": 198208.55555555556, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 24.90740740740741, "p95_turns_per_repair_loop": 49.0, "mean_agent_steps_per_repair_loop": 24.90740740740741, "p95_agent_steps_per_repair_loop": 49.0, "mean_verifier_submissions_per_repair_loop": 1.2037037037037037, "mean_verifier_submissions_to_success": 1.2037037037037037, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 17.5918968, "mean_cost_per_repair_loop": 0.3257758666666667, "p95_cost_per_repair_loop": 1.39996725, "cpsc": 0.3257758666666667, "realized_cpsc": 0.3257758666666667, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 17.5918968, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.3257758666666667, "mean_spend_per_solved_task": 0.3257758666666667, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.30927439444444443, "mean_cost_delta_vs_gateway_per_repair_loop": 0.016501472222222223, "cost_delta_vs_gateway_ratio": 0.053355442670461405 }, { "model_config": "openrouter/moonshotai/kimi-k2.7-code", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 117502.38888888889, "tokens_per_success": 117502.38888888889, "conditional_tokens_among_solved_loops": 117502.38888888889, "mean_tokens_per_solved_task": 117502.38888888889, "mean_input_tokens_per_repair_loop": 111682.42592592593, "mean_output_tokens_per_repair_loop": 5819.962962962963, "mean_cache_read_tokens_per_repair_loop": 91205.33333333333, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 14.148148148148149, "p95_turns_per_repair_loop": 28.0, "mean_agent_steps_per_repair_loop": 14.148148148148149, "p95_agent_steps_per_repair_loop": 28.0, "mean_verifier_submissions_per_repair_loop": 1.2407407407407407, "mean_verifier_submissions_to_success": 1.2407407407407407, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 2.65700082, "mean_cost_per_repair_loop": 0.04920371888888889, "p95_cost_per_repair_loop": 0.22189722, "cpsc": 0.04920371888888889, "realized_cpsc": 0.04920371888888889, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 2.65700082, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.04920371888888889, "mean_spend_per_solved_task": 0.04920371888888889, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.054128717629629626, "mean_cost_delta_vs_gateway_per_repair_loop": -0.004924998740740738, "cost_delta_vs_gateway_ratio": -0.09098679880871283 }, { "model_config": "openrouter/moonshotai/kimi-k3[max]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 353648.72222222225, "tokens_per_success": 353648.72222222225, "conditional_tokens_among_solved_loops": 353648.72222222225, "mean_tokens_per_solved_task": 353648.72222222225, "mean_input_tokens_per_repair_loop": 339439.8703703704, "mean_output_tokens_per_repair_loop": 14208.851851851852, "mean_cache_read_tokens_per_repair_loop": 317098.6666666667, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 9019.203703703704, "mean_turns_per_repair_loop": 15.351851851851851, "p95_turns_per_repair_loop": 62.0, "mean_agent_steps_per_repair_loop": 15.351851851851851, "p95_agent_steps_per_repair_loop": 62.0, "mean_verifier_submissions_per_repair_loop": 1.537037037037037, "mean_verifier_submissions_to_success": 1.537037037037037, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 20.2654434, "mean_cost_per_repair_loop": 0.37528598888888887, "p95_cost_per_repair_loop": 2.1540828, "cpsc": 0.37528598888888887, "realized_cpsc": 0.37528598888888887, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 20.2654434, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.37528598888888887, "mean_spend_per_solved_task": 0.37528598888888887, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.34628535555555556, "mean_cost_delta_vs_gateway_per_repair_loop": 0.02900063333333328, "cost_delta_vs_gateway_ratio": 0.08374779027778038 }, { "model_config": "openrouter/openai/gpt-5.5[low]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 26019.925925925927, "tokens_per_success": 26019.925925925927, "conditional_tokens_among_solved_loops": 26019.925925925927, "mean_tokens_per_solved_task": 26019.925925925927, "mean_input_tokens_per_repair_loop": 23711.314814814814, "mean_output_tokens_per_repair_loop": 2308.6111111111113, "mean_cache_read_tokens_per_repair_loop": 12202.666666666666, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 7.5, "p95_turns_per_repair_loop": 10.0, "mean_agent_steps_per_repair_loop": 7.5, "p95_agent_steps_per_repair_loop": 10.0, "mean_verifier_submissions_per_repair_loop": 1.0, "mean_verifier_submissions_to_success": 1.0, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 7.176757, "mean_cost_per_repair_loop": 0.1329029074074074, "p95_cost_per_repair_loop": 0.223198, "cpsc": 0.1329029074074074, "realized_cpsc": 0.1329029074074074, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 7.176757, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.1329029074074074, "mean_spend_per_solved_task": 0.1329029074074074, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.13215687037037038, "mean_cost_delta_vs_gateway_per_repair_loop": 0.0007460370370370379, "cost_delta_vs_gateway_ratio": 0.005645087046524822 }, { "model_config": "openrouter/openai/gpt-5.5[medium]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 61552.59259259259, "tokens_per_success": 61552.59259259259, "conditional_tokens_among_solved_loops": 61552.59259259259, "mean_tokens_per_solved_task": 61552.59259259259, "mean_input_tokens_per_repair_loop": 57491.648148148146, "mean_output_tokens_per_repair_loop": 4060.9444444444443, "mean_cache_read_tokens_per_repair_loop": 39168.0, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 9.277777777777779, "p95_turns_per_repair_loop": 14.0, "mean_agent_steps_per_repair_loop": 9.277777777777779, "p95_agent_steps_per_repair_loop": 14.0, "mean_verifier_submissions_per_repair_loop": 1.1296296296296295, "mean_verifier_submissions_to_success": 1.1296296296296295, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 12.583651, "mean_cost_per_repair_loop": 0.23303057407407407, "p95_cost_per_repair_loop": 0.370568, "cpsc": 0.23303057407407407, "realized_cpsc": 0.23303057407407407, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 12.583651, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.23303057407407407, "mean_spend_per_solved_task": 0.23303057407407407, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.23303057407407407, "mean_cost_delta_vs_gateway_per_repair_loop": 0.0, "cost_delta_vs_gateway_ratio": 0.0 }, { "model_config": "openrouter/openai/gpt-5.6-luna[low]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 53, "solve_rate": 0.9814814814814815, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 22369.685185185186, "tokens_per_success": 22791.754716981133, "conditional_tokens_among_solved_loops": 20416.924528301886, "mean_tokens_per_solved_task": 20416.924528301886, "mean_input_tokens_per_repair_loop": 20254.314814814814, "mean_output_tokens_per_repair_loop": 2115.3703703703704, "mean_cache_read_tokens_per_repair_loop": 14975.24074074074, "mean_cache_write_tokens_per_repair_loop": 4485.722222222223, "mean_reasoning_tokens_per_repair_loop": 306.44444444444446, "mean_turns_per_repair_loop": 6.2592592592592595, "p95_turns_per_repair_loop": 8.0, "mean_agent_steps_per_repair_loop": 6.2592592592592595, "p95_agent_steps_per_repair_loop": 8.0, "mean_verifier_submissions_per_repair_loop": 1.1111111111111112, "mean_verifier_submissions_to_success": 1.0188679245283019, "stop_reasons": { "agent_exit_repeatedformaterror": 1, "passed": 53 }, "total_model_spend_usd": 1.11187355, "mean_cost_per_repair_loop": 0.020590250925925927, "p95_cost_per_repair_loop": 0.036359050000000004, "cpsc": 0.020978746226415095, "realized_cpsc": 0.020978746226415095, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 1.11187355, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.019982904716981133, "mean_spend_per_solved_task": 0.019982904716981133, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.02008630648148148, "mean_cost_delta_vs_gateway_per_repair_loop": 0.0005039444444444473, "cost_delta_vs_gateway_ratio": 0.025088955249640225 }, { "model_config": "openrouter/openai/gpt-5.6-sol[low]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 28640.87037037037, "tokens_per_success": 28640.87037037037, "conditional_tokens_among_solved_loops": 28640.87037037037, "mean_tokens_per_solved_task": 28640.87037037037, "mean_input_tokens_per_repair_loop": 25403.12962962963, "mean_output_tokens_per_repair_loop": 3237.740740740741, "mean_cache_read_tokens_per_repair_loop": 18137.40740740741, "mean_cache_write_tokens_per_repair_loop": 6434.240740740741, "mean_reasoning_tokens_per_repair_loop": 459.5, "mean_turns_per_repair_loop": 6.796296296296297, "p95_turns_per_repair_loop": 9.0, "mean_agent_steps_per_repair_loop": 6.796296296296297, "p95_agent_steps_per_repair_loop": 9.0, "mean_verifier_submissions_per_repair_loop": 1.0, "mean_verifier_submissions_to_success": 1.0, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 8.13090625, "mean_cost_per_repair_loop": 0.150572337962963, "p95_cost_per_repair_loop": 0.25486475, "cpsc": 0.150572337962963, "realized_cpsc": 0.150572337962963, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 8.13090625, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.150572337962963, "mean_spend_per_solved_task": 0.150572337962963, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.150572337962963, "mean_cost_delta_vs_gateway_per_repair_loop": 0.0, "cost_delta_vs_gateway_ratio": 0.0 }, { "model_config": "openrouter/openai/gpt-5.6-terra[low]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 53, "solve_rate": 0.9814814814814815, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 32534.185185185186, "tokens_per_success": 33148.03773584906, "conditional_tokens_among_solved_loops": 28962.245283018867, "mean_tokens_per_solved_task": 28962.245283018867, "mean_input_tokens_per_repair_loop": 28993.98148148148, "mean_output_tokens_per_repair_loop": 3540.203703703704, "mean_cache_read_tokens_per_repair_loop": 21668.90740740741, "mean_cache_write_tokens_per_repair_loop": 6276.444444444444, "mean_reasoning_tokens_per_repair_loop": 777.4629629629629, "mean_turns_per_repair_loop": 7.111111111111111, "p95_turns_per_repair_loop": 10.0, "mean_agent_steps_per_repair_loop": 7.111111111111111, "p95_agent_steps_per_repair_loop": 10.0, "mean_verifier_submissions_per_repair_loop": 1.1296296296296295, "mean_verifier_submissions_to_success": 1.0377358490566038, "stop_reasons": { "agent_exit_repeatedformaterror": 1, "passed": 53 }, "total_model_spend_usd": 4.36081025, "mean_cost_per_repair_loop": 0.08075574537037038, "p95_cost_per_repair_loop": 0.147829125, "cpsc": 0.08227943867924528, "realized_cpsc": 0.08227943867924528, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 4.36081025, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.07793531839622642, "mean_spend_per_solved_task": 0.07793531839622642, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.07916358101851852, "mean_cost_delta_vs_gateway_per_repair_loop": 0.001592164351851853, "cost_delta_vs_gateway_ratio": 0.020112333618149517 }, { "model_config": "openrouter/thinkingmachines/inkling[high]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 46, "solve_rate": 0.8518518518518519, "cap_hits": 8, "cap_hit_rate": 0.14814814814814814, "mean_tokens_per_repair_loop": 611072.4814814815, "tokens_per_success": 717345.9565217391, "conditional_tokens_among_solved_loops": 154253.63043478262, "mean_tokens_per_solved_task": 154253.63043478262, "mean_input_tokens_per_repair_loop": 602030.1111111111, "mean_output_tokens_per_repair_loop": 9042.37037037037, "mean_cache_read_tokens_per_repair_loop": 524335.4074074074, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 5283.2962962962965, "mean_turns_per_repair_loop": 31.685185185185187, "p95_turns_per_repair_loop": 110.0, "mean_agent_steps_per_repair_loop": 31.685185185185187, "p95_agent_steps_per_repair_loop": 110.0, "mean_verifier_submissions_per_repair_loop": 3.7222222222222223, "mean_verifier_submissions_to_success": 1.326086956521739, "stop_reasons": { "agent_step_cap": 2, "passed": 46, "verifier_submission_cap": 6 }, "total_model_spend_usd": 10.98647944, "mean_cost_per_repair_loop": 0.20345332296296295, "p95_cost_per_repair_loop": 0.95064648, "cpsc": 0.2388365095652174, "realized_cpsc": 0.2388365095652174, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 10.98647944, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.07470325543478261, "mean_spend_per_solved_task": 0.07470325543478261, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.19769627944444446, "mean_cost_delta_vs_gateway_per_repair_loop": 0.005757043518518513, "cost_delta_vs_gateway_ratio": 0.02912064675519767 }, { "model_config": "openrouter/x-ai/grok-4.5[high]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 54, "solve_rate": 1.0, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 40413.40740740741, "tokens_per_success": 40413.40740740741, "conditional_tokens_among_solved_loops": 40413.40740740741, "mean_tokens_per_solved_task": 40413.40740740741, "mean_input_tokens_per_repair_loop": 36682.11111111111, "mean_output_tokens_per_repair_loop": 3731.296296296296, "mean_cache_read_tokens_per_repair_loop": 29660.444444444445, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 961.8888888888889, "mean_turns_per_repair_loop": 7.462962962962963, "p95_turns_per_repair_loop": 17.0, "mean_agent_steps_per_repair_loop": 7.462962962962963, "p95_agent_steps_per_repair_loop": 17.0, "mean_verifier_submissions_per_repair_loop": 1.0555555555555556, "mean_verifier_submissions_to_success": 1.0555555555555556, "stop_reasons": { "passed": 54 }, "total_model_spend_usd": 2.768112, "mean_cost_per_repair_loop": 0.05126133333333333, "p95_cost_per_repair_loop": 0.167036, "cpsc": 0.05126133333333333, "realized_cpsc": 0.05126133333333333, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 2.768112, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.05126133333333333, "mean_spend_per_solved_task": 0.05126133333333333, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.05112577777777778, "mean_cost_delta_vs_gateway_per_repair_loop": 0.00013555555555555543, "cost_delta_vs_gateway_ratio": 0.00265141307277042 }, { "model_config": "openrouter/z-ai/glm-5.2[high]", "total_trials": 54, "repair_loops": 54, "excluded_repair_loops": 0, "successes": 53, "solve_rate": 0.9814814814814815, "cap_hits": 0, "cap_hit_rate": 0.0, "mean_tokens_per_repair_loop": 52482.74074074074, "tokens_per_success": 53472.981132075474, "conditional_tokens_among_solved_loops": 52712.8679245283, "mean_tokens_per_solved_task": 52712.8679245283, "mean_input_tokens_per_repair_loop": 48078.27777777778, "mean_output_tokens_per_repair_loop": 4404.462962962963, "mean_cache_read_tokens_per_repair_loop": 39243.75925925926, "mean_cache_write_tokens_per_repair_loop": 0.0, "mean_reasoning_tokens_per_repair_loop": 0.0, "mean_turns_per_repair_loop": 11.037037037037036, "p95_turns_per_repair_loop": 19.0, "mean_agent_steps_per_repair_loop": 11.037037037037036, "p95_agent_steps_per_repair_loop": 19.0, "mean_verifier_submissions_per_repair_loop": 1.0925925925925926, "mean_verifier_submissions_to_success": 1.1132075471698113, "stop_reasons": { "agent_exit_repeatedformaterror": 1, "passed": 53 }, "total_model_spend_usd": 1.53864186, "mean_cost_per_repair_loop": 0.02849336777777778, "p95_cost_per_repair_loop": 0.08790534, "cpsc": 0.029030978490566038, "realized_cpsc": 0.029030978490566038, "reference_budget_cpsc": null, "escalation_cpsc": null, "total_reference_budget_charged_spend_usd": null, "total_realized_charged_spend_usd": 1.53864186, "total_escalation_charged_spend_usd": null, "reference_budget_complete_rows": 0, "replacement_cost_complete_rows": 0, "conditional_spend_among_solved_loops": 0.027443896981132077, "mean_spend_per_solved_task": 0.027443896981132077, "gateway_reported_repair_loops": 54, "mean_gateway_reported_cost_per_repair_loop": 0.022784761327407407, "mean_cost_delta_vs_gateway_per_repair_loop": 0.005708606450370372, "cost_delta_vs_gateway_ratio": 0.25054493081319157 } ]