ai-research-survey

scan.json (26551B)

---

 {
   "paper": {
     "title": "OmniCode: A Benchmark for Evaluating Software Development Agents",
     "authors": [
       "Atharv Sonwane",
       "Eng-Shen Tu",
       "Wei-Chung Lu",
       "Claas Beger",
       "Carter Larsen",
       "Debjit Dhar",
       "Simon Alford",
       "Rachel Chen",
       "Ronit Pattanayak",
       "Tuan Anh Dang",
       "Guohao Chen",
       "Gloria Geng",
       "Kevin Ellis",
       "Saikat Dutta"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.02262"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code and data are available at https://github.com/seal-research/OmniCode.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is released via the same GitHub repository as stated in the abstract."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions containerized environments for each instance but does not provide a requirements.txt, Dockerfile, or detailed environment specification for reproducing the evaluation experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README-level guidance for replicating the evaluation is included in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, 5, and 6 are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares model performance across tasks and languages but uses no statistical significance tests. Differences are reported as raw percentages."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Pearson correlations are given in Table 4 but no effect size measures for the main performance comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 1794 tasks total but no justification for why this size is sufficient, nor power analysis for the evaluation claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All results are single-run numbers with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two agent frameworks (SWE-Agent and Aider) are compared, and four LLMs are evaluated. Table 3 provides direct comparison between SWE-Agent and Aider."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models (Gemini 2.5 Flash, GPT-5-mini, DeepSeek-V3.1, Qwen3-32B) and frameworks (SWE-Agent, Aider) are contemporary state-of-the-art systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 ablates the impact of including bad patches in test generation evaluation (Figure 6), showing that gold-patch-only evaluation dramatically overestimates capability."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass rate for bug-fixing/test-gen/review-response, Style-Fixing Score (Equation 2), Fix Rate, Error Ratio (Table 5), complexity analysis, and Pearson correlations."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of agent outputs is performed. All evaluation is automated via test suites and style-checking tools. Manual validation was done for benchmark construction but not for evaluating agent performance."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark instances are newly curated or synthetically crafted to avoid data leakage. The paper states tasks are 'synthetically crafted or recently curated to avoid data leakage issues.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task type (4 categories), language (3 languages), and model (4 models) in Tables 2, 3, 5, and 6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides detailed failure mode analysis (Figure 10, 11) categorizing agent failures into incorrect fixes, localization failure, failed tool calls, no patch, invalid patches, and empty patches."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports substantial negative results: agents struggle with test generation (max 25%), C++ tasks, and style fixing introduces many new errors (Error Ratio >2 for C++ and Java in Table 5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about SWE-Agent achieving max 20.9% on Java Test Generation and poor C++/Java performance are supported by Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'Providing Reviews helps solve issues' (Section 4.2) and that patch complexity is a 'latent factor' in performance (Section 4.4). The review-response comparison is confounded by instance selection: review instances were sampled from higher-complexity gold patches, making the comparison non-controlled."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to evaluate 'Software Development Agents' generally, but only two agent frameworks are tested with a limited set of models. The paper does not explicitly bound claims to the tested systems. Section 6 acknowledges scope limitations but the title and abstract are broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for observed results. For example, the C++ performance gap could be due to training data distribution rather than task complexity, but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It reports pass rates on specific task types and does not frame test-suite pass rates as broader claims about 'software engineering capability.' The style-fixing score explicitly defines what is measured (Equation 2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'Gemini 2.5 Flash', 'GPT-5-mini', 'DeepSeek-V3.1', and 'Qwen3-32B' without snapshot dates or API versions. These are marketing names without precise version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for all task types is provided in Appendix C: bug-fixing, test generation, style-fixing, review-fixing, review generation, and bad patch generation prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only the per-instance cost limit ($2.0 for SWE-Agent) and Aider timeout (20 minutes, 3 retries) are mentioned. No temperature, top-p, or other sampling parameters are reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses SWE-Agent and Aider as agent frameworks but says only 'we use the default settings for SWE-Agent.' No description of the scaffolding internals (tool use, retry logic, feedback mechanisms) is provided."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection (selecting PRs that resolve issues and introduce tests), manual filtering criteria, and Appendix F provides additional filtering details. Section 3.2 documents task-specific augmentation pipelines."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations and Future Work' provides substantive discussion of what the benchmark does not cover."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific limitations: the benchmark doesn't cover config files, multi-language projects, profiling, natural language design conversations, sprint planning. It also identifies specific future extensions (security violations, code migration)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what is not tested: 'Real programmers deal with config files, multiple languages, profiling and optimizing, and engage in natural language conversations to make design decisions, plan sprints, and other forms of team strategy.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data and code are released at the GitHub repository, enabling independent verification of task instances and evaluation results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection from GitHub repositories and existing benchmarks (SWE-Bench-Verified, Multi-SWE-Bench), including PR selection criteria and manual verification."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is from public GitHub repositories and existing benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw PRs to benchmark tasks is documented: Section 3.1 covers base instance collection, Sections 3.2.1-3.2.4 cover task-specific augmentation, and Appendix G details bad patch generation with counts (112 C++, 237 Java, 760 Python)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 acknowledges Google (GCP credits) and Meta (LLM Evaluation Research Grant)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Cornell University, UC Santa Barbara, Jadavpur University, NYU, and one independent contributor."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google and Meta both produce models that could be evaluated on this benchmark. Google's Gemini is directly evaluated. The funders have a financial interest in how their models perform on coding benchmarks."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses this: tasks are 'synthetically crafted or recently curated to avoid data leakage issues' (abstract). The benchmark includes synthetic bad patches and newly collected instances to mitigate overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper frames contamination prevention as a core design goal, using synthetic task generation and recent curation to create tasks that models could not have seen during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only a per-instance cost limit of $2.0 is mentioned for SWE-Agent. Actual costs incurred are not reported. Aider evaluations were limited 'to limit costs' but no figures are given."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or total API spend is reported despite running thousands of agent evaluations across multiple models."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs per configuration is not stated. Results appear to be single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search was discussed. Default settings are used for SWE-Agent but no search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Default settings are used for SWE-Agent (Section 4.1), which avoids cherry-picking. The only adjustment is the per-instance cost limit to $2.0, which is stated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 4 models × 4 tasks × 3 languages but no correction for multiple comparisons is applied (no significance tests at all)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate agents on it without acknowledging potential author-evaluation bias in benchmark construction or evaluation setup."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes are compared (Qwen3-32B vs. proprietary frontier models) without discussing compute budget differences. The $2.0 cost limit applies to all but the underlying compute is not equalized."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 explicitly discusses construct validity of test generation evaluation, showing that gold-patch-only evaluation overestimates capability and introducing bad patches for more robust assessment. Section 4.4 analyzes patch complexity as a latent factor."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares SWE-Agent and Aider as separate frameworks (Table 3) and discusses how their different architectures (agentic vs pipeline) affect results (Section 4.3). For the main model comparison in Table 2, all models use the same SWE-Agent scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses temporal leakage by design: tasks are 'synthetically crafted or recently curated to avoid data leakage issues' and bad patches are newly generated, making them unavailable in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (e.g., issue descriptions, test structures) could leak information beyond what would be available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Some instances come from SWE-Bench-Verified and Multi-SWE-Bench. No discussion of whether evaluated models may have been trained on or tuned against these benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap) is applied. The mitigation is entirely through synthetic/recent data construction."
       }
     }
   },
   "claims": [
     {
       "claim": "SWE-Agent achieves a maximum of 20.9% on Java Test Generation with DeepSeek-V3.1",
       "evidence": "Table 2 shows DeepSeek-V3.1 achieves 20.9% on Java test generation, the highest among all models.",
       "supported": "strong"
     },
     {
       "claim": "Performance on bug-fixing is strongly correlated with review-response (Pearson = 0.921) but weakly with style-fixing (0.512)",
       "evidence": "Table 4 reports Pearson correlations across task types and languages.",
       "supported": "moderate"
     },
     {
       "claim": "Including bad patches in test evaluation reveals that gold-patch-only evaluation dramatically overestimates model testing capability",
       "evidence": "Figure 6 shows Qwen C++ drops from 22.7% to 4.55% and DeepSeek C++ from 43.8% to 25% when bad-patch failure is required.",
       "supported": "strong"
     },
     {
       "claim": "SWE-Agent consistently outperforms Aider across most tasks and languages",
       "evidence": "Table 3 shows SWE-Agent outperforms Aider on most cells, especially for C++ where Aider achieves only 1.8% on bug-fixing vs 8.0%.",
       "supported": "moderate"
     },
     {
       "claim": "Style-fixing attempts introduce many new style errors, particularly for Java and C++",
       "evidence": "Table 5 shows Error Ratios of 2.46-5.46 for C++ and Java, meaning fixes create 2-5x as many issues as originally present.",
       "supported": "strong"
     },
     {
       "claim": "Providing reviews helps solve issues that bug-fixing alone cannot",
       "evidence": "Section 4.2 reports review-response resolved 15 unique Java instances vs 4 for bug-fixing, but the comparison is confounded by instance selection (reviews generated for higher-complexity patches).",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "OmniCode is a 1794-task benchmark spanning bug fixing, test generation, code review response, and style fixing across Python, Java, and C++. Current agents struggle most with test generation (max 25%) and non-Python languages. Style-fixing attempts frequently introduce new errors, with error ratios exceeding 5x for Java. Including bad patches in test evaluation reveals that gold-patch-only metrics overestimate testing capability by 2-3x.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results are single-run point estimates across thousands of agent invocations with stochastic LLMs. No error bars, no repeated runs, no seed sensitivity. Results could vary substantially across runs."
     },
     {
       "flag": "Funder conflict of interest",
       "detail": "Google funded via GCP credits while Gemini 2.5 Flash is directly evaluated. Meta funded via research grant. Neither conflict is acknowledged in the paper."
     },
     {
       "flag": "Confounded review-response comparison",
       "detail": "The claim that reviews help is confounded: review instances were sampled from higher-complexity patches, making the bug-fixing vs review-response comparison non-controlled. The paper acknowledges this partially but still presents it as a finding."
     },
     {
       "flag": "Single-run results for stochastic systems",
       "detail": "Agent-based systems are highly stochastic. SWE-Agent + LLM results without repeated runs make it impossible to distinguish signal from noise, especially for small per-category sample sizes."
     },
     {
       "flag": "Unequal compute across model comparisons",
       "detail": "Qwen3-32B (open, 32B parameters) is compared against proprietary frontier models without discussing compute budget differences. The $2.0 cost cap does not equalize effective compute."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Foundational benchmark for evaluating LLM agents on real-world GitHub issue resolution."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Introduces SWE-Agent, one of the primary agent frameworks evaluated in this paper."
     },
     {
       "title": "Agentless: Demystifying LLM-Based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Non-agentic approach to SWE tasks; used to generate bad patches for the OmniCode benchmark."
     },
     {
       "title": "Multi-SWE-Bench: A Multilingual Benchmark for Issue Resolving",
       "authors": ["Daoguang Zan", "Zhirong Huang", "Wei Liu"],
       "year": 2025,
       "arxiv_id": "2504.02605",
       "relevance": "Extended SWE-Bench to multiple languages; OmniCode builds on this for Java and C++ instances."
     },
     {
       "title": "SWE-Smith: Scaling Data for Software Engineering Agents",
       "authors": ["John Yang", "Kilian Leret", "Carlos E. Jimenez"],
       "year": 2025,
       "arxiv_id": "2504.21798",
       "relevance": "Synthetic bug generation for training coding agents; related to OmniCode's synthetic task creation approach."
     },
     {
       "title": "SWT-Bench: Testing and Validating Real-World Bug-Fixes with Code Agents",
       "authors": ["Niels Mündler", "Mark Müller", "Jingxuan He", "Martin Vechev"],
       "year": 2024,
       "relevance": "Prior test generation benchmark that OmniCode improves upon with bad-patch evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, a foundational code generation benchmark."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li"],
       "year": 2024,
       "relevance": "Open-source agent platform for software development tasks."
     },
     {
       "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million from Real-World Freelance Software Engineering?",
       "authors": ["Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke"],
       "year": 2025,
       "arxiv_id": "2502.12115",
       "relevance": "Evaluates LLM agents on real-world freelance software engineering tasks with monetary stakes."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Autonomous program repair agent evaluated on SWE-Bench."
     },
     {
       "title": "SWE-PolyBench: A Multi-Language Benchmark for Repository Level Evaluation of Coding Agents",
       "authors": ["Muhammad Shihab Rashid"],
       "year": 2025,
       "arxiv_id": "2504.08703",
       "relevance": "Multi-language coding benchmark with improved data quality checks."
     },
     {
       "title": "BugPilot: Complex Bug Generation for Efficient Learning of SWE Skills",
       "authors": ["Atharv Sonwane"],
       "year": 2025,
       "arxiv_id": "2510.19898",
       "relevance": "Synthetic bug generation methodology related to OmniCode's bad patch generation."
     }
   ]
 }