{ "schemaVersion": "coding-agent-benchmark-public-v1", "generatedFrom": { "repository": "nwags/cc-deepseek-benchmark", "commit": "79f790fdd9067e33a29e0409c15f8890068ff22d", "authorityPath": "results/phase3/reporting/phase3_current_reviewed_comparison_20260825.json", "authorityBlob": "cab9730ac38cb10fe571004c13d82dcf193c393e", "authoritySchema": "phase3-current-reviewed-comparison-v4", "reviewedAt": "2026-08-25" }, "scopes": { "phase3-core": { "label": "Core reviewed scope", "armCount": 15, "trialCount": 900, "successCount": 515 }, "phase3-extended": { "label": "Extended reviewed scope", "armCount": 16, "trialCount": 960, "successCount": 562 } }, "arms": [ { "armId": "router-kimi-k3", "displayLabel": "Kimi K3", "backendModel": "kimi-k3", "provider": "moonshot", "providerLabel": "moonshot", "trialCount": 60, "successCount": 47, "cleanSuccessCount": 44, "passRate": 0.7833333333333333, "selectedCostUsd": "26.570403", "selectedCostRelation": "estimate", "selectedCostBasis": "provider_rate_reconstructed_selected_run", "selectedCostConfidence": "medium_qualified_pricing_provenance", "selectedCostPerAttemptUsd": "0.44284005", "selectedCostPerCleanSuccessUsd": "0.6038728", "evidenceClass": "complete_retained_usage_plus_retained_rate_constants_qualified_provenance", "providerBillingReconciliationStatus": "provider_log_not_invoice_level_allocation_low_confidence", "selectedTrialCostAllocationStatus": "available_provider_rate_reconstruction", "selectedOutcomeCostAllocationStatus": "unavailable_no_reviewed_outcome_join", "scopeMembership": [ "phase3-extended" ] }, { "armId": "router-gpt-5.5", "displayLabel": "GPT-5.5", "backendModel": "gpt-5.5", "provider": "openai", "providerLabel": "OpenAI", "trialCount": 60, "successCount": 44, "cleanSuccessCount": 42, "passRate": 0.7333333333333333, "selectedCostUsd": "48.604914", "selectedCostRelation": "exact", "selectedCostBasis": "provider_billed", "selectedCostConfidence": "exact_provider_arm_total", "selectedCostPerAttemptUsd": "0.8100819", "selectedCostPerCleanSuccessUsd": "1.15725986", "evidenceClass": "exact_provider_arm_total", "providerBillingReconciliationStatus": "exact_arm_total", "selectedTrialCostAllocationStatus": "unavailable_provider_aggregate", "selectedOutcomeCostAllocationStatus": "unavailable_provider_aggregate", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-deepseek-flash", "displayLabel": "DeepSeek V4 Flash", "backendModel": "deepseek-v4-flash", "provider": "deepseek", "providerLabel": "DeepSeek", "trialCount": 60, "successCount": 40, "cleanSuccessCount": 40, "passRate": 0.6666666666666666, "selectedCostUsd": "1.0798358032", "selectedCostRelation": "estimate", "selectedCostBasis": "provider_rate_reconstructed_selected_run", "selectedCostConfidence": "high", "selectedCostPerAttemptUsd": "0.01799726", "selectedCostPerCleanSuccessUsd": "0.0269959", "evidenceClass": "provider_rate_reconstruction_with_same_day_provider_crosscheck", "providerBillingReconciliationStatus": "same_day_model_aggregate_not_run_isolated", "selectedTrialCostAllocationStatus": "available_provider_rate_reconstruction", "selectedOutcomeCostAllocationStatus": "available_provider_rate_reconstruction", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-gpt-5.4", "displayLabel": "GPT-5.4", "backendModel": "gpt-5.4", "provider": "openai", "providerLabel": "OpenAI", "trialCount": 60, "successCount": 39, "cleanSuccessCount": 38, "passRate": 0.65, "selectedCostUsd": "29.7919335", "selectedCostRelation": "exact", "selectedCostBasis": "provider_billed", "selectedCostConfidence": "exact_provider_arm_total", "selectedCostPerAttemptUsd": "0.49653223", "selectedCostPerCleanSuccessUsd": "0.78399825", "evidenceClass": "exact_provider_arm_total", "providerBillingReconciliationStatus": "exact_arm_total", "selectedTrialCostAllocationStatus": "unavailable_provider_aggregate", "selectedOutcomeCostAllocationStatus": "unavailable_provider_aggregate", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-anthropic-opus", "displayLabel": "Claude Opus 4.7", "backendModel": "claude-opus-4-7", "provider": "anthropic", "providerLabel": "Anthropic", "trialCount": 60, "successCount": 39, "cleanSuccessCount": 39, "passRate": 0.65, "selectedCostUsd": "50.28831125", "selectedCostRelation": "lower_bound", "selectedCostBasis": "provider_rate_reconstructed_retained_usage_lower_bound", "selectedCostConfidence": "high_for_retained_accounting_lower_bound", "selectedCostPerAttemptUsd": "0.83813852", "selectedCostPerCleanSuccessUsd": "1.28944388", "evidenceClass": "verified_retained_artifacts_plus_official_provider_rates", "providerBillingReconciliationStatus": "provider_invoice_unavailable", "selectedTrialCostAllocationStatus": "available_with_exception_path_lower_bounds", "selectedOutcomeCostAllocationStatus": "available_lower_bound", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-anthropic-fable-5", "displayLabel": "Claude Fable 5", "backendModel": "claude-fable-5", "provider": "anthropic", "providerLabel": "Anthropic", "trialCount": 60, "successCount": 39, "cleanSuccessCount": 37, "passRate": 0.65, "selectedCostUsd": "64.80504500", "selectedCostRelation": "exact", "selectedCostBasis": "provider_rate_reconstructed_retained_usage", "selectedCostConfidence": "high", "selectedCostPerAttemptUsd": "1.08008408", "selectedCostPerCleanSuccessUsd": "1.7514877", "evidenceClass": "verified_retained_artifacts_plus_official_provider_rates", "providerBillingReconciliationStatus": "provider_invoice_unavailable", "selectedTrialCostAllocationStatus": "available_provider_rate_reconstruction", "selectedOutcomeCostAllocationStatus": "available_provider_rate_reconstruction", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-glm-5.2", "displayLabel": "GLM 5.2", "backendModel": "glm-5.2", "provider": "zai-glm", "providerLabel": "Z.AI / GLM", "trialCount": 60, "successCount": 38, "cleanSuccessCount": 35, "passRate": 0.6333333333333333, "selectedCostUsd": "8.9016736", "selectedCostRelation": "estimate", "selectedCostBasis": "provider_rate_reconstructed_selected_run", "selectedCostConfidence": "high", "selectedCostPerAttemptUsd": "0.14836123", "selectedCostPerCleanSuccessUsd": "0.25433353", "evidenceClass": "complete_retained_usage_plus_retained_published_rates", "providerBillingReconciliationStatus": "selected_run_provider_invoice_unavailable", "selectedTrialCostAllocationStatus": "available_provider_rate_reconstruction", "selectedOutcomeCostAllocationStatus": "available_provider_rate_reconstruction", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-gemini-3.1-pro", "displayLabel": "Gemini 3.1 Pro Preview", "backendModel": "gemini-3.1-pro-preview", "provider": "google-gemini", "providerLabel": "Google Gemini", "trialCount": 60, "successCount": 38, "cleanSuccessCount": 36, "passRate": 0.6333333333333333, "selectedCostUsd": "19.6968138", "selectedCostRelation": "estimate", "selectedCostBasis": "provider_rate_reconstructed_selected_run_request_tier", "selectedCostConfidence": "high", "selectedCostPerAttemptUsd": "0.32828023", "selectedCostPerCleanSuccessUsd": "0.54713372", "evidenceClass": "verified_retained_trajectories_plus_official_provider_rates", "providerBillingReconciliationStatus": "preselected_family_billing_context_not_selected_run", "selectedTrialCostAllocationStatus": "available_provider_rate_reconstruction", "selectedOutcomeCostAllocationStatus": "available_provider_rate_reconstruction", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-deepseek-pro", "displayLabel": "DeepSeek V4 Pro", "backendModel": "deepseek-v4-pro", "provider": "deepseek", "providerLabel": "DeepSeek", "trialCount": 60, "successCount": 37, "cleanSuccessCount": 37, "passRate": 0.6166666666666667, "selectedCostUsd": "1.899724634", "selectedCostRelation": "estimate", "selectedCostBasis": "provider_rate_reconstructed_selected_run", "selectedCostConfidence": "high", "selectedCostPerAttemptUsd": "0.03166208", "selectedCostPerCleanSuccessUsd": "0.05134391", "evidenceClass": "provider_rate_reconstruction_with_same_day_provider_crosscheck", "providerBillingReconciliationStatus": "same_day_model_aggregate_not_run_isolated", "selectedTrialCostAllocationStatus": "available_provider_rate_reconstruction", "selectedOutcomeCostAllocationStatus": "available_provider_rate_reconstruction", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-glm-5.1", "displayLabel": "GLM 5.1", "backendModel": "glm-5.1", "provider": "zai-glm", "providerLabel": "Z.AI / GLM", "trialCount": 60, "successCount": 36, "cleanSuccessCount": 35, "passRate": 0.6, "selectedCostUsd": "5.3316552", "selectedCostRelation": "estimate", "selectedCostBasis": "provider_rate_reconstructed_retained_usage_partial", "selectedCostConfidence": "medium", "selectedCostPerAttemptUsd": "0.08886092", "selectedCostPerCleanSuccessUsd": "0.15233301", "evidenceClass": "partial_retained_usage_plus_retained_published_rates_cache_accounting_unverified", "providerBillingReconciliationStatus": "preselected_provider_context_not_selected_run", "selectedTrialCostAllocationStatus": "partial_selected_usage_reconstruction_with_unresolved_trials", "selectedOutcomeCostAllocationStatus": "available_partial_estimate", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-grok-build-0.1", "displayLabel": "Grok Build 0.1", "backendModel": "grok-build-0.1", "provider": "xai", "providerLabel": "xAI", "trialCount": 60, "successCount": 36, "cleanSuccessCount": 36, "passRate": 0.6, "selectedCostUsd": "6.418694", "selectedCostRelation": "lower_bound", "selectedCostBasis": "provider_rate_reconstructed_retained_usage_lower_bound", "selectedCostConfidence": "high_for_retained_accounting_lower_bound", "selectedCostPerAttemptUsd": "0.10697823", "selectedCostPerCleanSuccessUsd": "0.17829706", "evidenceClass": "verified_retained_trajectories_plus_official_provider_rates", "providerBillingReconciliationStatus": "preselected_provider_context_not_selected_run", "selectedTrialCostAllocationStatus": "available_with_unresolved_usage_lower_bounds", "selectedOutcomeCostAllocationStatus": "available_lower_bound", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-qwen-3.7-plus", "displayLabel": "Qwen 3.7 Plus", "backendModel": "qwen3.7-plus", "provider": "dashscope-qwen", "providerLabel": "Alibaba / Qwen", "trialCount": 60, "successCount": 33, "cleanSuccessCount": 32, "passRate": 0.55, "selectedCostUsd": "2.50442432", "selectedCostRelation": "lower_bound", "selectedCostBasis": "provider_rate_reconstructed_retained_usage_lower_bound", "selectedCostConfidence": "high_for_retained_accounting_lower_bound", "selectedCostPerAttemptUsd": "0.04174041", "selectedCostPerCleanSuccessUsd": "0.07826326", "evidenceClass": "verified_retained_trajectories_plus_provider_bill_validated_rates", "providerBillingReconciliationStatus": "preselected_provider_context_not_selected_run", "selectedTrialCostAllocationStatus": "available_with_unresolved_usage_lower_bounds", "selectedOutcomeCostAllocationStatus": "available_lower_bound", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-kimi-k2.6", "displayLabel": "Kimi K2.6", "backendModel": "kimi-k2.6", "provider": "moonshot-kimi", "providerLabel": "Moonshot / Kimi", "trialCount": 60, "successCount": 30, "cleanSuccessCount": 30, "passRate": 0.5, "selectedCostUsd": "6.34692415", "selectedCostRelation": "estimate", "selectedCostBasis": "provider_rate_reconstructed_selected_run", "selectedCostConfidence": "high", "selectedCostPerAttemptUsd": "0.10578207", "selectedCostPerCleanSuccessUsd": "0.21156414", "evidenceClass": "complete_retained_usage_plus_provider_dashboard_validated_rates", "providerBillingReconciliationStatus": "preselected_provider_context_not_selected_run", "selectedTrialCostAllocationStatus": "available_provider_rate_reconstruction", "selectedOutcomeCostAllocationStatus": "available_provider_rate_reconstruction", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-anthropic-sonnet", "displayLabel": "Claude Sonnet 4.6", "backendModel": "claude-sonnet-4-6", "provider": "anthropic", "providerLabel": "Anthropic", "trialCount": 60, "successCount": 28, "cleanSuccessCount": 28, "passRate": 0.4666666666666667, "selectedCostUsd": "38.38591710", "selectedCostRelation": "lower_bound", "selectedCostBasis": "provider_rate_reconstructed_retained_usage_lower_bound", "selectedCostConfidence": "high_for_retained_accounting_lower_bound", "selectedCostPerAttemptUsd": "0.63976529", "selectedCostPerCleanSuccessUsd": "1.37092561", "evidenceClass": "verified_retained_artifacts_plus_official_provider_rates", "providerBillingReconciliationStatus": "provider_invoice_unavailable", "selectedTrialCostAllocationStatus": "available_with_exception_path_lower_bounds", "selectedOutcomeCostAllocationStatus": "available_lower_bound", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-anthropic-haiku-sanitized", "displayLabel": "Claude Haiku 4.5", "backendModel": "claude-haiku-4-5-20251001", "provider": "anthropic", "providerLabel": "Anthropic", "trialCount": 60, "successCount": 25, "cleanSuccessCount": 25, "passRate": 0.4166666666666667, "selectedCostUsd": "16.70224485", "selectedCostRelation": "exact", "selectedCostBasis": "provider_rate_reconstructed_retained_usage", "selectedCostConfidence": "high", "selectedCostPerAttemptUsd": "0.27837075", "selectedCostPerCleanSuccessUsd": "0.66808979", "evidenceClass": "verified_retained_artifacts_plus_official_provider_rates", "providerBillingReconciliationStatus": "provider_invoice_unavailable", "selectedTrialCostAllocationStatus": "available_provider_rate_reconstruction", "selectedOutcomeCostAllocationStatus": "available_provider_rate_reconstruction", "scopeMembership": [ "phase3-core", "phase3-extended" ] }, { "armId": "router-gemini-flash", "displayLabel": "Gemini 3.5 Flash", "backendModel": "gemini-3.5-flash", "provider": "google-gemini", "providerLabel": "Google Gemini", "trialCount": 60, "successCount": 13, "cleanSuccessCount": 9, "passRate": 0.21666666666666667, "selectedCostUsd": "16.12091625", "selectedCostRelation": "lower_bound", "selectedCostBasis": "provider_rate_reconstructed_retained_usage_lower_bound", "selectedCostConfidence": "high_for_retained_accounting_lower_bound", "selectedCostPerAttemptUsd": "0.26868194", "selectedCostPerCleanSuccessUsd": "1.79121292", "evidenceClass": "retained_artifacts_plus_official_provider_rates_trajectory_archive_unavailable", "providerBillingReconciliationStatus": "preselected_family_billing_context_not_selected_run", "selectedTrialCostAllocationStatus": "available_with_unresolved_usage_lower_bounds", "selectedOutcomeCostAllocationStatus": "available_lower_bound", "scopeMembership": [ "phase3-core", "phase3-extended" ] } ] }