ai-research-survey

scan.json (28327B)

---

 {
   "paper": {
     "title": "Can Vibe Coding Beat Graduate CS Students? An LLM vs. Human Coding Tournament on Market-driven Strategic Planning",
     "authors": [
       "Panayiotis Danassis",
       "Naman Goel"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.20613"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a code URL: 'Code available at: https://panayiotisd.github.io/apdp_bench/' (page 1, Section 1). The benchmark is described as open-sourced."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark platform and all agent code (including human-coded agents) are released at the same URL. The tournament configurations (network topologies of Switzerland, France, Great Britain, Netherlands) and task generation parameters are described. The benchmark itself is the data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is mentioned. The paper states 'All code is implemented in Java' (Section 4.1) but provides no version details, dependency specifications, or environment configuration."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While the code is released at a URL, there is no description of how to run the tournaments, set up the environment, or replicate the specific experimental configurations used."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports standard deviations for wins/losses per tournament in Table 1, but does not report confidence intervals or error bars for the main comparison claims (e.g., that human agents dominate). The SD values are descriptive variance across tournaments, not inferential uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'a clear superiority of human-coded agents' and that the LLM 'makes the solution significantly worse' (Section 5.3), but no statistical significance tests are reported. Comparisons are made by rank ordering win rates without any formal test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides win rates, total wins/losses, and average wins per tournament with baseline context. For example, Student 1 has 96.58% win rate vs. LLM(O,IR,1) at 85.19%. The Section 5.3 improvement experiment reports a drop from 1st to 10th place (89.667 average wins vs. 108.167). These provide magnitude of differences with clear baselines."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 40 LLM agents and 17 human agents were chosen. The 12 student agents were selected from a single year's course (2020), with no discussion of whether this sample size is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports standard deviation of wins and losses per tournament for all 57 agents across 12 tournaments. For example, 'SD #Wins / Tour' and 'SD #Losses / Tour' columns are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Five baseline agents are included: Naive, ExpCostFixedBid, Honest, ModelOpponent, and RiskSeeking (Section 4.1). These range from very simple to moderately sophisticated strategies."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses state-of-the-art LLMs as of 2025 (GPT-5 Thinking, Gemini 2.5 Pro, Claude Opus 4.1, DeepThink R1) and compares against pre-LLM-era human agents. The baselines are appropriate because the research question is whether LLMs can match human performance on this task, and the human agents serve as the benchmark."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "While not a traditional ablation, the paper systematically varies prompting strategies (A1, A2, IR, CR, GEN) across all four LLMs, effectively ablating the prompt component. Additionally, preliminary experiments (Section 5.1) test simpler variants (Reactive, Deliberative, Centralized) to isolate where LLMs fail."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics in Table 1: average wins per tournament, standard deviation, total wins, total losses, and win rate. The preliminary evaluation also uses profit comparison (Section 5.1, Centralized variant: '19% higher to 194% lower')."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "There is no human evaluation of the quality of the generated code beyond the automated tournament results. The paper does not have humans inspect or rate the LLM-generated agents' code quality, strategic sophistication, or design decisions. The tournament is an automated competitive evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Tasks are drawn randomly at tournament time, but there is no distinction between development and test configurations. The same tournament setup was used for debugging and final evaluation. The paper notes LLMs were 'tested/debugged in self-play and tournament settings' (Section 5.2), meaning the evaluation environment was also used for development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-agent breakdowns for all 57 agents. The paper also breaks down by LLM model and prompting strategy (using the naming scheme explained in Table 1). Results across 4 network topologies are aggregated but per-topology results are not shown separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 (Debugging) discusses specific failure cases: time-out violations, not delivering won tasks, capacity constraint violations, and LLMs failing to fix persistent bugs. Section 5.1 discusses inadmissible heuristics and suboptimal design decisions. Section 5.3 discusses the degradation of the winning solution."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is effectively a negative result for LLMs. Additionally, the iterative refinement and critic strategies often did not improve performance, and providing the best human solution to GPT-5 made it worse (Section 5.3). The paper is transparent about LLM failures."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (i) top 5 spots held by human agents, (ii) 33/40 LLM agents beaten by simple baselines, and (iii) LLM degrades winning solution are all directly supported by Table 1 and Section 5.3 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies causal claims about LLMs' 'limited reasoning and planning capabilities' (Section 6) causing poor performance, but the study design cannot distinguish whether poor results are due to reasoning limitations, prompt quality, or the Java-specific implementation requirements. The paper does not systematically control for these confounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly bounds the conclusions: 'Our conclusions are bounded by a single, albeit rich, domain (logistics PDP). We also do not claim that LLM performance observed in this paper is the optimal performance the best LLM can ever achieve in this setting with any prompt.' The paper acknowledges dedicated coding models and better prompts could yield different results."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses multiple alternative explanations: the results may not generalize beyond the logistics domain, dedicated fine-tuned coding models might perform better, optimal prompting strategies were not explored, and professional engineers might outperform students. The paper also acknowledges that student agents were not debugged while LLM agents were, which could bias results in favor of LLMs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses marketing names only: 'GPT-5 Thinking', 'Gemini 2.5 Pro', 'Claude Opus 4.1', and 'DeepThink R1' (Section 4.2). No API version numbers, snapshot dates, or specific model IDs are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'The supplement contains the complete prompts' (Section 4.2) but only describes the prompts in natural language within the paper itself. The supplement is referenced but not included in the paper text. The description of Author Prompt #1 is a summary, not the actual prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported: no temperature, top-p, max tokens, or other API settings are mentioned for any of the LLM calls. The paper does not state what sampling parameters were used."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses LLMs for one-shot code generation with iterative bug-fixing prompts, not agentic scaffolding with tools, memory, or feedback loops. The approach is direct prompting, not scaffolded agent architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents how student agents were selected: 'top 8 agents from the 2020 tournament' plus '4 additional student agents that performed well against baseline agents' (Section 4.1). The LLM agent generation process is described: 4 LLMs x 5 prompting strategies x 2 runs = 40 agents (Section 4.2). Tournament parameters (4 topologies, 3 per topology, 50 tasks per match) are specified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'IMPLICATIONS, LIMITATIONS, AND FUTURE WORK' and contains substantive discussion of limitations spanning multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: 'Our conclusions are bounded by a single, albeit rich, domain', 'We also do not claim that LLM performance observed in this paper is the optimal performance', acknowledges 'dedicated models, specifically fine-tuned for coding' might differ, and notes that 'professional software engineers could presumably be better than the students in this study'."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states: 'Our work is not concerned with finding the optimal prompting strategy, the best LLM, or to create a leader-board of LLMs.' It also states the scope is bounded to 'a single, albeit rich, domain (logistics PDP)' and explains why model names are not highlighted in the ranking."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark code is released, the raw match results (individual match outcomes for all ~40k matches), LLM conversation logs, generated code, and debugging iterations are not described as being available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how data was collected: student agents from the 2020 EPFL Intelligent Agents course, baseline agents from the AI Lab, LLM agents generated via 5 prompting strategies with 4 LLMs, each run twice. Tournament setup (12 double all-play-all tournaments, 4 topologies, ~40k matches) is fully described."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Student agent recruitment is described: agents from the 2020 postgraduate Intelligent Agents course at EPFL, selected as top 8 from the single-elimination tournament plus 4 additional based on wins against baselines (Section 4.1). The paper acknowledges selection may not capture the true top human agents due to single-elimination format."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from agent generation to tournament results is documented: LLM code generation → debugging → self-play testing → tournament evaluation. The paper specifies how many agents were generated vs. evaluated and notes significant manual effort was required to get bug-free agents."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is mentioned. The Acknowledgments section thanks Professor Faltings and the AI Lab at EPFL, but does not disclose funding. The paper notes the work was partially conducted while P.D. was at Telenor Research, but no explicit funding disclosure is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Panayiotis Danassis (University of Southampton), Naman Goel (University of Oxford and Alan Turing Institute). The paper also notes P.D. was previously at Telenor Research. Neither author is affiliated with the LLM companies being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so it is impossible to assess funder independence. The absence of a funding disclosure does not establish that the work was unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the four LLMs used (GPT-5 Thinking, Gemini 2.5 Pro, Claude Opus 4.1, DeepThink R1). This is relevant because the student code was developed in 2020 and is being released publicly."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 explicitly discusses contamination: 'Many benchmarks are often accompanied with data contamination concerns (i.e., the correct answers were already in the training data). On the other hand, this experiment shows that, in our benchmark, even when we expose a good solution in-context, the LLM is still unable to utilize it.' The paper argues contamination is less of a concern because LLMs fail even with the solution provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination by design: the APDP benchmark is novel and was not previously published as a standard benchmark. Additionally, the in-context experiment (Section 5.3) demonstrates that even with direct access to a winning solution, the LLM cannot improve it, arguing contamination is not a confound."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses pre-existing student code from a 2020 course assignment, not a new human subjects study. No new human participants were recruited for this research."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The student agents were course submissions from 2020, used as benchmark data. No new human participants were involved in the study, so IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No new human participants were recruited. The student agents are from a postgraduate course, but demographic reporting is not applicable since this is not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for the study. Student agent selection criteria are described under data collection, not human subjects methodology."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment was conducted. The tournament uses random task generation and company swapping for fairness, but this is experimental design, not human subjects randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment was conducted requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were involved in the study. Agent crash rates are reported (Section 5.2) but this is system reliability, not participant attrition."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock times are reported for the LLM code generation process. The paper does not mention how much it cost to generate and debug the 40 LLM agents."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report the compute resources used for running ~40k tournament matches or the cost of LLM API calls during agent generation and debugging."
       }
     }
   },
   "claims": [
     {
       "claim": "The top 5 spots in the tournament are consistently won by human (student)-coded agents.",
       "evidence": "Table 1 shows Students 1-5 occupying ranks 1-5 across all 12 double all-play-all tournaments, with win rates from 96.58% to 86.16%.",
       "supported": "strong"
     },
     {
       "claim": "The majority of LLM-coded agents (33 out of 40) are beaten by very simple baseline agents.",
       "evidence": "Table 1 shows ExpCostFixedBid (a simple baseline) at rank 18 with 68.9% win rate. Only 7 LLM agents rank above it; 33 rank below. All are GPT-5 variants except one DeepSeek and one Gemini.",
       "supported": "strong"
     },
     {
       "claim": "Given the best human solution as input and prompted to improve upon it, the best performing LLM makes the solution significantly worse.",
       "evidence": "Section 5.3 describes providing Student 1's code to GPT-5 Pro and asking for improvements. The resulting agent dropped from 1st to 10th place, with average wins falling from 108.167 to 89.667.",
       "supported": "strong"
     },
     {
       "claim": "LLMs can generate syntactically correct code but struggle with reasoning-driven tasks requiring planning, optimization, and strategic interaction.",
       "evidence": "Section 5.1 preliminary results show LLMs produced running code but with semantic bugs (inadmissible heuristics, ignoring constraints). Section 4.2 notes 'significant manual effort' to achieve bug-free code. Tournament results (Table 1) show poor competitive performance.",
       "supported": "moderate"
     },
     {
       "claim": "This is the first comparison of LLMs vs. humans in the domain of real-world optimization code generation.",
       "evidence": "Section 1 states 'This is the first work, to the best of our knowledge, that answers two crucial questions.' Related work (Section 2) surveys existing benchmarks and shows none involve direct LLM vs. human competition on multi-agent optimization problems.",
       "supported": "moderate"
     },
     {
       "claim": "Even with in-context access to a good solution, LLMs cannot utilize it effectively, mitigating data contamination concerns.",
       "evidence": "Section 5.3 describes the experiment where GPT-5 was given the winning student solution and asked to improve it, but the result was significantly worse. The paper argues this shows contamination is not a confound for their benchmark.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "This paper introduces the Auction, Pickup, and Delivery Problem (APDP) benchmark, a multi-agent competitive logistics optimization problem, and evaluates 40 LLM-coded agents against 17 human-coded agents across ~40k tournament matches. Human (graduate student)-coded agents consistently dominated the top 5 positions, while 33 of 40 LLM agents lost to simple baseline strategies. Even when the best-performing LLM (GPT-5) was given the winning human solution and asked to improve it, the result dropped from 1st to 10th place, highlighting a significant gap between LLMs' syntactic code generation ability and their capacity for reasoning-driven optimization in competitive settings.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "Despite claiming 'clear superiority' and that LLMs make solutions 'significantly worse,' no formal statistical tests are conducted. With 12 tournaments and known variance, significance tests would strengthen the claims considerably."
     },
     {
       "flag": "No hyperparameters or model versions reported",
       "detail": "The paper uses only marketing names for LLMs (GPT-5 Thinking, Gemini 2.5 Pro, etc.) without API versions, snapshot dates, or temperature/sampling settings. These can substantially affect code generation quality."
     },
     {
       "flag": "Asymmetric debugging treatment",
       "detail": "LLM agents were thoroughly debugged while student agents were not debugged at all (crashes counted as LLM wins). The paper acknowledges this favors LLMs, but the asymmetry complicates fair comparison. Student agents with crashes (e.g., Student 9 with 184 crashes) might rank much higher if debugged."
     },
     {
       "flag": "Student agent selection may not be representative",
       "detail": "Student agents were selected from a single year (2020) of a single course at EPFL. The top 8 were chosen via single-elimination (which the paper acknowledges may not select the best agents), and 4 more were added by a different criterion. This selection process is not guaranteed to be representative of graduate student capability."
     },
     {
       "flag": "Prompts described but not shown",
       "detail": "The actual prompts are in a supplement, but the paper provides only a natural language description of Prompt A1. Without seeing the exact prompts, it's impossible to assess whether prompt quality explains the results."
     },
     {
       "flag": "No cost reporting",
       "detail": "Generating 40+ LLM agents with iterative debugging across 4 frontier models likely involved significant API costs. This is not reported, limiting reproducibility and practical assessment."
     }
   ],
   "cited_papers": [
     {
       "title": "Measuring the impact of early-2025 AI on experienced open-source developer productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT finding developers were 19% slower with AI tools, directly relevant to AI coding productivity claims."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and pass@k metric, foundational to LLM code generation evaluation."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang"],
       "year": 2024,
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2025,
       "relevance": "Contamination-free code evaluation benchmark, relevant to evaluation methodology."
     },
     {
       "title": "Large Language Models for Code Generation: A Comprehensive Survey of Challenges, Techniques, Evaluation, and Applications",
       "authors": ["Nam Huynh", "Beiyu Lin"],
       "year": 2025,
       "arxiv_id": "2503.01245",
       "relevance": "Survey of LLM code generation covering challenges, techniques, and evaluation methods."
     },
     {
       "title": "Can large language models reason and plan?",
       "authors": ["Subbarao Kambhampati"],
       "year": 2024,
       "relevance": "Directly relevant to the reasoning and planning limitations of LLMs discussed in this paper."
     },
     {
       "title": "Competition-level code generation with alphacode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "relevance": "Demonstrated LLM competitive coding performance, claims this paper challenges."
     },
     {
       "title": "The Effects of Generative AI on High-Skilled Work: Evidence from Three Field Experiments with Software Developers",
       "authors": ["Zheyuan Kevin Cui"],
       "year": 2025,
       "relevance": "Field experiments on AI impact on developer productivity, relevant to AI-assisted coding evaluation."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2025,
       "relevance": "Code generation benchmark with complex instructions, relevant to evaluation methodology."
     },
     {
       "title": "SWE-bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?",
       "authors": ["Xiang Deng", "Jeff Da"],
       "year": 2025,
       "arxiv_id": "2509.16941",
       "relevance": "Evaluates AI agents on long-horizon software engineering tasks, directly relevant to agentic coding capabilities."
     },
     {
       "title": "To the Cutoff... and Beyond? A Longitudinal Perspective on LLM Data Contamination",
       "authors": ["Manley Roberts", "Himanshu Thakur"],
       "year": 2024,
       "relevance": "Addresses data contamination in LLM benchmarking, a key concern this paper discusses."
     },
     {
       "title": "AlphaEvolve: A Gemini-powered coding agent for designing advanced algorithms",
       "authors": ["Alexander Novikov"],
       "year": 2025,
       "relevance": "LLM-powered system for algorithm design, relevant to LLM code generation capabilities beyond autocomplete."
     }
   ]
 }