ai-research-survey

scan.json (23304B)

---

 {
   "paper": {
     "title": "CooperBench: Why Coding Agents Cannot be Your Teammates Yet",
     "authors": ["Arpandeep Khatua", "Hao Zhu", "Peter Tran", "Arya Prabhudesai", "Frederic Sadrieh", "Johann K. Lieberwirth", "Xinkai Yu", "Yicheng Fu", "Michael J. Ryan", "Jiaxin Pei", "Diyi Yang"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.13295"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We open-source this framework' and provides the URL https://cooperbench.com. The benchmark platform is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "CooperBench comprises 652 tasks across 12 repositories and is released as an open benchmark. The paper states 'We release CooperBench as an open benchmark.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 Stage III describes containerized environments with automated setup scripts that clone repos at exact base commits, install dependencies, and verify test suites. Docker-based containers are used for agent execution (§3)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the full evaluation pipeline (§2.2), agent framework setup based on OpenHands v0.54 (§3), and Stage III describes automated setup scripts. The open-sourced framework is intended for reproducibility."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All error bars in Fig. 4 are 95% Wilson confidence intervals computed over task sets (App. C).'"
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The difference between \"with comm\" and \"no comm\" settings is not statistically significant' (§5), indicating significance testing was performed. Communication reducing merge conflicts is described as 'significantly' reducing them."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported throughout: 'around 50% lower than a Solo baseline', 'performance drops from 68.6% with 2 agents to 46.5% with 3 agents and further to 30.0% with 4 agents', specific success rates per model (e.g., GPT-5: 0.48 Solo vs 0.28 Coop)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 652 tasks across 12 repositories, but no justification is given for why this size is sufficient. The 3-to-4 agent scaling experiment uses only 46 tasks from 3 task sets with no power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "95% Wilson confidence intervals are reported for the main results in Fig. 4, which convey the spread/uncertainty of the estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The Solo baseline (one agent doing both features) is the primary comparison. A 'no comm' baseline is also used to isolate the effect of communication (§5)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates GPT-5, Claude Sonnet 4.5, MiniMax-M2, and Qwen3 models — all contemporary frontier models at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The communication ablation (with comm vs. no comm, §5) and the agent scaling experiment (2 to 4 agents, §4) serve as ablation-style studies isolating specific factors."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports success rate (task pass rate), conflict rate (merge compatibility), communication overhead (% of actions), and per-model breakdowns. Solo vs Coop gap is a distinct metric."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual qualitative coding of 50 failed Coop traces was performed (§6.2). The LLM-as-judge annotation for failure symptoms was validated with human evaluation (§6.1, App. G)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 652 tasks are used for evaluation. There is no mention of a held-out test set or dev/test split — the entire benchmark is used for reporting results."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model, by difficulty level (Fig. 4 Right), by communication type (Fig. 5c), by failure symptom category (Tab. 1), and by failure cause (Tab. 2)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides extensive failure analysis with a taxonomy of failure symptoms (Tab. 1), root causes (Tab. 2), and representative examples of each capability gap (§6.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The main finding IS a negative result: agents perform worse when cooperating. Communication does not improve success rates (§5). Prompt optimization yields only marginal improvements (App. D)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims agents achieve '30% lower success rates when working together' — the results show ~50% lower for top models (0.48→0.28 for GPT-5). The 30% is described as 'on average' across all models including weaker ones, which is consistent with the data shown."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'communication reduces merge conflicts' are supported by controlled ablation (with/without communication). The 'curse of coordination' is demonstrated by comparing Solo vs Coop under controlled conditions with the same tasks and models."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'our findings generalize to any domain involving role and resource conflicts under partial observability' (§8) — a very broad generalization from software engineering tasks only. The title 'Why Coding Agents Cannot be Your Teammates Yet' generalizes beyond the 5 models and specific agent framework tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: the spatial vs semantic coordination gap (§5), the trust paradox as an alternative explanation for expectation failures (§6.3), and that prompt optimization might help but yields only marginal improvements (App. D). The agent framework itself is acknowledged as a potential confound (§3)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-5', 'Claude 4.5 Sonnet', 'MiniMax-M2', 'Qwen3-Coder-30B-A3B-Instruct', and 'Qwen3-30B-A3B-Instruct-2507'. No API snapshot dates or version identifiers are given for GPT-5, Claude, or MiniMax."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The system prompts and task prompts given to agents are described in natural language but the actual prompt text is not provided in the paper or appendix. The paper says agents receive feature descriptions in markdown but does not show the system prompt or scaffolding prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or other API hyperparameters for the model calls. The action limit of 100 is stated but sampling parameters are absent."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agent framework is described: OpenHands v0.54 with a custom communication tool using SQL database for message passing (§3). Docker containers for isolation, real-time async communication, and the action space (communication + computer-use tools) are described in §2.1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage dataset construction pipeline is thoroughly documented (§2.3, Fig. 3): repository/PR selection criteria, feature extraction and augmentation process, and environment setup. Inclusion criteria for PRs are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion (§8) mentions future directions but does not substantively discuss limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential confounds like the specific agent framework choice, the 100-action limit, or whether results would differ with different communication mechanisms."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Instead, §8 claims broad generalization: 'our findings generalize to any domain involving role and resource conflicts under partial observability.' No explicit scope boundaries are drawn."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is open-sourced at cooperbench.com, which should include the task specifications, tests, and ground-truth solutions. Agent trajectories are analyzed and the framework is released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 describes the three-stage pipeline in detail: repository selection criteria (1K+ stars, not in SWE-bench), PR selection constraints, feature extraction, and environment validation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are recruited for evaluation. The benchmark evaluates AI agents, not humans. The dataset creators are the paper's co-authors."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from PR selection through feature pool creation to evaluation is documented in §2.3 with Fig. 3. The evaluation pipeline (merge + test) is in §2.2. Statistics on task composition (77.3% conflicting ground-truth solutions) are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists funding: ONR grant N000142412532, NSF grant IIS-2247357, DSO National Laboratories, SAP support, Google Cloud Platform and Modal Platform credits."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Stanford University and SAP Labs US. The paper evaluates third-party models (GPT-5, Claude, MiniMax, Qwen), not SAP products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (ONR, NSF, DSO, SAP) do not have a direct financial stake in whether coding agents succeed or fail at cooperation. The paper evaluates third-party models, not products from its funders."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training cutoff dates are stated for any of the five models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination: 'Each repository exceeds one thousand GitHub stars and does not appear in SWE-Bench or Multi-SWE-Bench, reducing data contamination risk' (§2.3). Features are newly created by co-authors, reducing overlap with training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is newly constructed with expert-written features, tests, and ground-truth code. The paper explicitly avoids repositories in SWE-Bench/Multi-SWE-Bench and creates new features rather than using existing issues/PRs directly, addressing contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are studied. The paper evaluates AI agents on benchmark tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates AI agents."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or per-task inference costs are reported despite using multiple commercial API models (GPT-5, Claude, MiniMax) at scale across 652 tasks with multiple settings."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. Google Cloud Platform and Modal credits are acknowledged but the actual compute expenditure is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "Agents achieve on average 30% lower success rates when cooperating compared to solo performance (the 'curse of coordination').",
       "evidence": "Fig. 4 shows Solo vs Coop success rates: GPT-5 (0.48 vs 0.28), Claude (0.47 vs 0.26), MiniMax (0.36 vs 0.14), Qwen Coder (0.22 vs 0.13), Qwen (0.06 vs 0.05). 95% Wilson CIs reported.",
       "supported": "strong"
     },
     {
       "claim": "Communication does not lead to better cooperation success despite heavy usage.",
       "evidence": "Fig. 5(a) shows with-comm vs no-comm success rates are not statistically significantly different across all models, while Fig. 5(c) shows agents spend up to 20% of actions on communication.",
       "supported": "strong"
     },
     {
       "claim": "Communication reduces merge conflicts but not task success.",
       "evidence": "Fig. 5(b) shows significant reduction in conflict rates with communication for Claude, GPT-5, MiniMax, and Qwen Instruct. This is attributed to spatial vs semantic coordination gap (§5).",
       "supported": "strong"
     },
     {
       "claim": "Coordination failures stem from three capability gaps: expectation (42%), commitment (32%), and communication (26%).",
       "evidence": "Table 2 reports these percentages based on manual review of 50 failed Coop traces (§6.2). Representative examples provided in §6.3.",
       "supported": "moderate"
     },
     {
       "claim": "Increasing the number of agents monotonically decreases success: 68.6% (2 agents) → 46.5% (3) → 30.0% (4).",
       "evidence": "Small-scale experiment using 46 tasks from 3 task sets (§4). No confidence intervals or significance tests reported for this specific experiment.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "CooperBench demonstrates a 'curse of coordination' where frontier coding agents (GPT-5, Claude Sonnet 4.5) achieve only ~25% success when two agents cooperate, roughly half their solo performance on the same workload. Communication between agents reduces merge conflicts but does not improve task success, because agents solve spatial coordination (avoiding overlapping edits) but fail at semantic coordination (ensuring compatible implementations). Qualitative analysis of failures reveals three root causes: expectation failures (42%), commitment failures (32%), and communication breakdowns (26%). Rare emergent coordination behaviors (role division, resource division, negotiation) appear in successful traces, suggesting the capability exists but is unreliable.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section. Potential confounds like the specific agent framework (OpenHands v0.54), communication mechanism (SQL-based message passing), and 100-action limit are not discussed as threats to validity."
     },
     {
       "flag": "Overbroad generalization",
       "detail": "Section 8 claims 'our findings generalize to any domain involving role and resource conflicts under partial observability' based solely on software engineering tasks with a specific agent framework and communication tool."
     },
     {
       "flag": "Small sample for scaling claim",
       "detail": "The 2-to-4 agent scaling experiment uses only 46 tasks from 3 task sets with no confidence intervals or significance tests reported, yet the result is presented as reinforcing the curse of coordination."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the five models, despite these settings significantly affecting agent behavior."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent llm systems fail?",
       "authors": ["Mert Cemri", "Melissa Z. Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Directly relevant: analyzes failure modes in multi-agent LLM systems, finding inter-agent misalignment as a major failure category."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "relevance": "The agent framework used in CooperBench experiments; a major open-source coding agent platform."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Foundational single-agent coding benchmark that CooperBench extends to multi-agent cooperative settings."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2024,
       "relevance": "Multi-agent framework that emulates software organizations with structured interaction, contrasted with CooperBench's free-form coordination."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "Multi-agent coding framework using communicative agents, relevant to evaluating structured vs free-form agent collaboration."
     },
     {
       "title": "Magentic-One: A generalist multi-agent system for solving complex tasks",
       "authors": ["Adam Fourney", "Gagan Bansal", "Hussein Mozannar"],
       "year": 2024,
       "arxiv_id": "2411.04468",
       "relevance": "Multi-agent system using explicit orchestrators, representing the scaffolded approach that CooperBench argues masks coordination problems."
     },
     {
       "title": "CAMEL: Communicative agents for 'mind' exploration of large language model society",
       "authors": ["Guohao Li", "Hasan Abed Al Kader Hammoud"],
       "year": 2023,
       "relevance": "Foundational multi-agent conversation framework relevant to understanding agent communication and coordination."
     },
     {
       "title": "Agentless",
       "authors": ["Chunqiu Steven Xia"],
       "year": 2024,
       "relevance": "Single-agent coding approach achieving strong SWE-bench results, providing context for the solo vs cooperative performance gap."
     },
     {
       "title": "The Collaboration Gap",
       "authors": ["Tim R. Davidson", "Adam Fourney", "Saleema Amershi"],
       "year": 2025,
       "arxiv_id": "2511.02687",
       "relevance": "Directly related work finding that solo-capable models degrade when required to collaborate, corroborating CooperBench findings."
     },
     {
       "title": "Collaborative Gym: A framework for enabling and evaluating human-agent collaboration",
       "authors": ["Yijia Shao", "Vinay Samuel"],
       "year": 2025,
       "arxiv_id": "2412.15701",
       "relevance": "Framework for human-agent collaboration evaluation, complementary to CooperBench's agent-agent focus."
     },
     {
       "title": "Sotopia-pi: Interactive learning of socially intelligent language agents",
       "authors": ["Ruiyi Wang", "Haofei Yu"],
       "year": 2024,
       "relevance": "Training method for social intelligence in agents, proposed by CooperBench authors as a path to improving coordination."
     },
     {
       "title": "Scaling agents via continual pre-training",
       "authors": ["Liangcai Su", "Zhen Zhang"],
       "year": 2025,
       "arxiv_id": "2509.13310",
       "relevance": "Finding that multi-agent configurations degrade performance by 39-70% relative to single-agent baselines, corroborating curse of coordination."
     }
   ]
 }