ai-research-survey

scan.json (26807B)

---

 {
   "paper": {
     "title": "CATArena: Evaluating Evolutionary Capabilities of Code Agents via Iterative Tournaments",
     "authors": [
       "Lingyue Fu",
       "Xin Ding",
       "Linyue Pan",
       "Yaoming Zhu",
       "Shao Zhang",
       "Lin Qiu",
       "Weiwen Liu",
       "Weinan Zhang",
       "Xuezhi Cao",
       "Xunliang Cai",
       "Jiaxin Ding",
       "Yong Yu"
     ],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.26852"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides the official website https://catarena.ai and references the Google ADK framework at https://github.com/google/adk-python. The framework is presented as a released artifact."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release the tournament logs, match data, or the raw scoring matrices. The six task environments reference existing benchmarks (OIBench, SWE-Perf) but the specific tournament data generated by CATArena is not made publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes Docker environments for SWE-Perf and sandboxed evaluation arenas, but does not provide requirements.txt, Dockerfile, conda environment, or detailed dependency specifications for reproducing the framework."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. There is no README with commands to run, no 'Reproducing Results' section. The paper describes the framework at a high level but lacks specific instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2, 11, 12, and 13 report only point estimates (Sbase and Sevo values) with no confidence intervals or error bars. The stability analysis in Appendix E reports standard deviations of rankings across 4 repetitions but does not provide CIs for the main metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., evolutionary capability is 'orthogonal' to static performance, commercial agents do not have decisive advantage in Sevo) but provides no statistical significance tests. SWE-Perf uses Mann-Whitney U internally for speedup calculation, but no significance tests are applied to the paper's own comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported in context: e.g., Gemini 'low-start, high-growth trajectory in Gomoku (0.25 → +0.156)', Claude Chess regression (-0.061), Qwen OIBench advance from G1=0.10 to G4=0.43. The Sevo metric itself quantifies rate of improvement with baselines provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The number of agents (6 minimal + 5 commercial) and number of rounds (N=4 main, N=7 extended) are not justified with any formal reasoning. The four-round window is justified post-hoc by observing diminishing returns, but there is no power analysis or formal justification for the number of agents or repetitions."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E (Table 8) reports standard deviations of agent rankings across 4 independent repetitions of Rounds 1 and 2. The stability analysis explicitly reports ranking variance. DISstd in Table 3 quantifies population score dispersion."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes baselines: minimal agents (isolating LLM capabilities) vs. commercial agents (state-of-the-art systems). The Sbase metric serves as a static baseline against which Sevo is compared. Ablation in Table 4 compares full feedback vs. self-reflection-only."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated agents include contemporary state-of-the-art: Claude-4-Sonnet, GPT-5, Gemini-2.5-Pro, DeepSeek-3.1, Qwen3-Coder-480B, and commercial tools like Claude Code, Codex, Gemini CLI, and Qwen Code. These are current as of 2025-2026."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents an ablation study isolating peer-learning from self-reflection by comparing full feedback (G2) vs. self-reflection-only (G'2). Section 4.4 explicitly labels this as an ablation study."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two core metrics: Sbase (static base performance) and Sevo (evolutionary capability), plus the global performance score Gn. Additional metrics include DISstd, Trendmean, code similarity measures, and HTTP error rates."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper measuring code agent performance through automated tournament outcomes. Human evaluation of agent outputs is not relevant to the claims about evolutionary capability metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The variant tasks (Section 4.5, Table 11) serve as held-out evaluation: rule perturbations (inverted hand rankings, Chess960, modified OIBench objectives) that invalidate memorized strategies and test generalization to unseen settings."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across all six tasks (Gomoku, Hold'em, Bridge, Chess, OIBench, SWE-Perf) in Tables 2 and 11, and per-agent evolutionary trajectories are shown in Figures 2 and 3. Additional breakdowns by programming language (Table 12) and ML track (Table 13)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses three failure modes: Blind Copying, Stubborn Stagnation, and Chaotic Reconstruction. Appendix M discusses the Pommerman failure case where all agents failed to generate functional strategies. HTTP error rates are reported in Appendix F."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results are reported: Chess shows negative Trendmean (-0.74) indicating performance collapse; most agents show negative Sevo in Chess and SWE-Perf; Pommerman is an outright failure for all agents; the ML track shows limited technical depth. The paper explicitly states agents 'struggle to concurrently leverage both peer-learning and self-reflection.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) evolutionary potential is not strictly correlated with initial proficiency — supported by Table 2 showing Gemini low Sbase/high Sevo; (2) agents struggle to leverage both peer-learning and self-reflection — supported by Table 4; (3) high extensibility and resistance to variance — supported by Tables 11 and 12."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study in Table 4 provides controlled manipulation to isolate self-reflection vs. peer-learning contributions. The variant task experiments control for memorization effects. Most causal language ('driven by feedback', 'enables evolution') is grounded in ablation or controlled comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to evaluate 'evolutionary capabilities of code agents' broadly but tests on only 6 tasks (4 competitive games + 2 optimization benchmarks) with 11 agents. The title and abstract frame this as a general evaluation standard, but the paper does not explicitly bound claims to these specific task types. Claims like 'evolutionary capability is a distinct dimension of intelligence' are not bounded to the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its results. For instance, the weak correlation between Sbase and Sevo could be due to task difficulty confounds, prompt sensitivity, or API non-determinism rather than a genuine orthogonal capability dimension. No threats-to-validity section or consideration of confounds is present."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 lists model families (e.g., 'Claude-4-Sonnet', 'GPT-5', 'Gemini-2.5-Pro', 'DeepSeek-3.1') but does not provide specific API snapshot dates or version identifiers (e.g., no 'gpt-5-0613' equivalent). Marketing names without snapshot dates do not count as specified versions per the schema criteria."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix O states 'Full prompt details are in Tables 18 for the main leaderboard, 19 for ML track and 20 for multilingual track.' The paper references providing the full prompts used for agents in the appendix tables."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D states: 'For all LLMs used in our work, we set the temperature to 0.1 and the max tokens to the official API defaults. We set top-p to 1.0, top-k to 100, and the presence penalty to the API default.'"
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the minimal agent framework using Google ADK with 'only core tools (e.g., file I/O and scripting).' Commercial agents are described as 'CLI-based commercial agents' with 'proprietary toolchains.' Table 7 details agent types and frameworks. The case study in Appendix I shows concrete tool usage patterns."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The tournament configurations are detailed in Table 6 (Appendix B) with specific settings for each task: board sizes, number of matches, time limits, repetition counts, scoring rules. The performance matrix construction (Section 3.3) documents how raw match results are transformed into scores."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Impact Statement is a single paragraph that states 'There are many potential societal consequences of our work, none of which we feel must be specifically highlighted here' which does not address methodological limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as: the single-chain evolutionary experiments, prompt sensitivity, API non-determinism effects beyond the limited stability analysis, or whether results generalize beyond these specific tasks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It acknowledges difficulty in SWE-Perf and Chess but frames these as agent limitations rather than scope boundaries of the evaluation framework. No explicit statements about what populations, settings, or claims are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw tournament logs, match-level data, and full scoring matrices are not made available. Only aggregated results (Sbase, Sevo, G scores) are reported in the paper. Independent verification of the underlying data is not possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is well described: tournament configurations in Table 6, scoring rules in Table 5, repetition protocols in Appendix B, and the full evaluation protocol in Section 4.1. The process from code submission to score computation is documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The evaluated agents are selected models and commercial tools, which are standard benchmarks rather than recruited participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: initial development → execution feedback → iterative evolution (Section 3.1), with scoring rules (Table 5), tournament formats (Table 6), and metric calculation (Section 3.3 and Appendix A). The transformation from raw match results to Sbase and Sevo is formalized."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section listing grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Shanghai Jiao Tong University, Tsinghua University, and Meituan. The Meituan affiliation (a major Chinese technology company) is disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. Authors from Meituan (a tech company) are involved, and the paper evaluates multiple commercial LLM products, but there is no statement about whether any evaluated companies funded the research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. Authors from Meituan may have interests related to the Doubao-Seed model evaluated in the paper, but this is not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. This is relevant because competitive game strategies and OIBench problems may exist in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 directly addresses contamination through variant tasks: 'To verify that CATArena evaluates genuine adaptive capabilities rather than the retrieval of memorized solutions, we conducted experiments on unseen variants... that invalidate standard strategies present in the agents' pre-training data.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The variant task experiments (Section 4.5, Table 11) are explicitly designed to address benchmark contamination by introducing rule perturbations that invalidate memorized strategies. The paper states results confirm evaluation is 'independent of data contamination or specific task familiarity.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix N (Tables 14-17) provides detailed cost statistics including total input tokens, output tokens, total time in seconds, tools used, valid lines of code, and average thinking time for each agent across all tasks and rounds."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-agent token costs and execution times are reported (Appendix N), the total computational budget (total API spend, total GPU hours for tournament execution, total cost of the study) is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Evolutionary capability (Sevo) is not strictly correlated with static base performance (Sbase) — agents with high initial pass rates do not necessarily possess strong evolving abilities.",
       "evidence": "Table 2 shows Claude-4 has Sbase=0.90 in Chess but Sevo=-0.061, while Gemini-2.5-Pro has Sbase=0.25 in Gomoku but Sevo=+0.156. The paper states 'We observe a weak correlation between Sbase and Sevo across all tasks' (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "Current agents struggle to concurrently leverage both peer-learning and self-reflection for effective performance gains.",
       "evidence": "Table 4 ablation study shows most models rely predominantly on a single mechanism. Only Claude and Qwen demonstrate effective peer-learning in specific tasks. The paper identifies failure patterns: 'agents typically exhibit a dependency on a single source' (Section 4.4).",
       "supported": "moderate"
     },
     {
       "claim": "Commercial code agent frameworks primarily enhance static generation capability rather than evolutionary ability.",
       "evidence": "Table 2 shows commercial agents frequently outperform Best ADK Agent in Sbase but do not exhibit decisive advantage in Sevo. For example, Claude Code has Sbase=0.78 but Sevo=-0.035 in Gomoku (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "CATArena is robust to task variants and maintains discriminative power for evolutionary capability evaluation.",
       "evidence": "Table 11 shows variant task results where Sevo maintains discriminative power despite altered baselines. Section 4.5 states 'the evolutionary metric retains its effectiveness in distinguishing agent capabilities.'",
       "supported": "moderate"
     },
     {
       "claim": "Competitive tasks foster population convergence while objective tasks drive divergence.",
       "evidence": "Table 3 shows DISstd is negative for most competitive tasks (Gomoku -0.05, Hold'em -0.81, Bridge -0.82) and positive for objective tasks (OIBench +0.42, SWE-Perf +0.20), with Chess as an exception (+0.55).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "CATArena introduces a tournament-based framework for evaluating code agents' evolutionary capabilities through iterative multi-turn development, using dual metrics (Sbase for static proficiency and Sevo for evolutionary potential). The framework reveals that evolutionary capability is largely orthogonal to single-turn generation performance, with agents like Gemini showing low-start/high-growth trajectories. Current agents predominantly rely on either self-reflection or peer-learning but struggle to synergize both mechanisms. The framework demonstrates robustness to variant tasks and extensibility to new domains, though complex environments like Chess and Pommerman remain challenging for all evaluated agents.",
   "red_flags": [
     {
       "flag": "Single-chain main experiments",
       "detail": "The main evolutionary experiments (N=4 rounds) are conducted as a single chain due to computational cost. While a stability analysis is done on Rounds 1 and 2 (4 repetitions), the actual evolutionary trajectories and Sevo metrics are based on single runs, making it impossible to assess the reliability of the evolutionary conclusions."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated limitations or threats-to-validity section. For a paper proposing a new evaluation framework that makes broad claims about 'evolutionary capability as a distinct dimension of intelligence,' the absence of discussed limitations is a significant omission."
     },
     {
       "flag": "Meituan affiliation undisclosed conflict",
       "detail": "Three authors are affiliated with Meituan, which develops the Doubao-Seed model evaluated in the paper. This potential conflict of interest is not acknowledged, and no competing interests statement is provided."
     },
     {
       "flag": "No significance testing for comparative claims",
       "detail": "Claims about weak correlation between Sbase and Sevo, and about commercial agents not having evolutionary advantages, are stated based on visual inspection of point estimates in Table 2 without any correlation coefficients, confidence intervals, or significance tests across the full set of results."
     },
     {
       "flag": "Broad claims from narrow evaluation",
       "detail": "The paper evaluates 6 minimal agents and 5 commercial agents across 6 tasks (primarily games and algorithm optimization) but makes general claims about 'evolutionary capability' as a dimension of intelligence. The task diversity is limited to games and code optimization, and generalizing to broader software engineering evolution is not justified."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Core baseline for evaluating code agents on software engineering benchmarks."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and Pass@k metric that CATArena aims to extend beyond."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan"],
       "year": 2023,
       "relevance": "Key prior work on self-reflection mechanisms for LLM output refinement."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn"],
       "year": 2023,
       "relevance": "Foundational work on iterative self-correction in LLM agents."
     },
     {
       "title": "GameBench: Evaluating Strategic Reasoning Abilities of LLM Agents",
       "authors": ["Anthony Costarelli"],
       "year": 2024,
       "arxiv_id": "2406.06613",
       "relevance": "Prior benchmark for evaluating LLM agents in game-playing scenarios."
     },
     {
       "title": "OIBench: Benchmarking Strong Reasoning Models with Olympiad in Informatics",
       "authors": ["Yaoming Zhu"],
       "year": 2025,
       "arxiv_id": "2506.10481",
       "relevance": "One of the two objective optimization tasks used within CATArena framework."
     },
     {
       "title": "SWE-Perf: Can Language Models Optimize Code Performance on Real-World Repositories?",
       "authors": ["Xin He"],
       "year": 2025,
       "arxiv_id": "2507.12415",
       "relevance": "Second objective optimization task integrated into CATArena for repository-level performance optimization."
     },
     {
       "title": "RedCode: Risky Code Execution and Generation Benchmark for Code Agents",
       "authors": ["Cheng Guo"],
       "year": 2024,
       "arxiv_id": "2411.07781",
       "relevance": "Specialized code security evaluation benchmark for code agents."
     },
     {
       "title": "A Survey on Code Generation with LLM-based Agents",
       "authors": ["Yifan Dong"],
       "year": 2025,
       "arxiv_id": "2508.00083",
       "relevance": "Survey covering LLM-based code agent capabilities and evaluation."
     },
     {
       "title": "SWE-Compass: Towards Unified Evaluation of Agentic Coding Abilities for Large Language Models",
       "authors": ["Jianghao Xu"],
       "year": 2025,
       "arxiv_id": "2511.05459",
       "relevance": "Unified framework for evaluating agentic coding capabilities."
     },
     {
       "title": "Evolutionary Perspectives on the Evaluation of LLM-based AI Agents: A Comprehensive Survey",
       "authors": ["Jiahao Zhu"],
       "year": 2025,
       "arxiv_id": "2506.11102",
       "relevance": "Survey on evolutionary evaluation perspectives for LLM agents."
     },
     {
       "title": "SWE-PolyBench: A Multi-Language Benchmark for Repository Level Evaluation of Coding Agents",
       "authors": ["Md Saidur Rashid"],
       "year": 2025,
       "arxiv_id": "2504.08703",
       "relevance": "Multi-language benchmark for repository-level code agent evaluation."
     }
   ]
 }