ai-research-survey

scan-v4.json (21732B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "FEA-Bench: A Benchmark for Evaluating Repository-Level Code Generation for Feature Implementation",
     "authors": [
       "Wei Li",
       "Xin Zhang",
       "Zhongxin Guo",
       "Shaoguang Mao",
       "Wen Luo"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.06680",
     "doi": "10.48550/arXiv.2503.06680"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'LLMs perform significantly worse in the FEA-Bench' and 'the best-performing LLM, DeepSeek-R1, successfully resolves only about 10%' are directly supported by Table 2 (9.92% resolved ratio).",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal-like claims (e.g., 'the format of code edits is a critical factor limiting performance') are supported by controlled comparisons varying one factor at a time (same model, different formats in Table 4; same model, different context lengths in Table 3).",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title scopes to 'repository-level code generation for feature implementation.' The Limitations section explicitly states 'our benchmark includes only Python repositories' and acknowledges 'certain scenario limitations.'",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper explores different experimental factors (retrieval, format, context length), it does not substantively discuss alternative explanations for the main finding of low performance — e.g., whether benchmark construction choices, test quality, or task formulation artificially inflate difficulty.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures unit test pass rate and frames claims tightly around 'resolved ratio' of task instances. Claims match the granularity of measurement — they do not extrapolate from test-passing to broader software quality claims.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present between Section 7 (Conclusion) and the Ethics Statement, with substantive discussion across multiple paragraphs.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: Python-only scope, scarcity of high-quality new-feature PRs, early-stage repo development not captured, single-round generation bias, and resource constraints limiting DeepSeek experiments.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states Python-only scope, acknowledges 'certain scenario limitations' from the PR-based construction, and notes that 'high-quality and usable pull requests for new feature development are relatively scarce.'",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments state: 'This work was supported by National Science and Technology Major Project (No. 2022ZD0116308) and National Natural Science Foundation of China (62036001).'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Peking University and Microsoft Research Asia. The paper header marks these with symbols (♡ and ♠).",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Chinese government agencies (NSFC, national project) which have no direct financial stake in the benchmark results or the relative performance of any evaluated model.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is present. Authors from Microsoft Research Asia could have interests related to AI code generation products, but no declaration is made.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "'Repository-level incremental code development' is explicitly defined as implementing new features by adding new components (functions/classes) to existing repositories; 'new components' are defined as newly added classes and functions identified by parsing patches.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit bullet-point contributions in the introduction state: (1) introducing the task, (2) constructing the first benchmark with execution-based evaluation, and (3) scaling to 83 repositories with public data release.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 engages substantively with SWE-bench, HumanEval, MBPP, DevEval, EvoCodeBench, and repository-level completion benchmarks; Table 1 provides quantitative comparison with SWE-bench statistics showing how FEA-Bench differs structurally.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that feature implementation 'requires LLMs to simultaneously possess code completion capabilities for new components and code editing abilities for other relevant parts,' distinguishing this from code completion and bug-fix tasks with quantitative statistics in Table 1.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "Figure 5 shows resolved ratios decrease with more added functions, and a lite subset is defined with lower difficulty criteria, but there is no formal difficulty tier characterization (easy/medium/hard) or difficulty score assigned to individual instances.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "The paper notes that best performance is 9.92% (suggesting no ceiling concern), but CodeLlama 13B achieves 0.14%—near floor—without any discussion of whether the benchmark discriminates among weak models or whether floor effects distort comparisons.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is included; the paper mentions a future plan to 'collaborate with professional software engineers to annotate a verified subset' (Appendix A.3) but this has not been done.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "The all-or-nothing unit-test-pass criterion is adopted from SWE-bench without formal justification; partial credit, test suite quality, or edge cases in scoring (e.g., tests that pass trivially) are not discussed.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "No temporal splits, canary strings, or dynamic generation are implemented; popular repositories like scikit-learn, django, and sympy are almost certainly in the training data of evaluated models, and the paper does not address this.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions continuous updates are possible by releasing the collection pipeline, but does not discuss how future training data will inevitably contaminate static benchmark instances or any plan to detect or mitigate this.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Python-only scope and single-round generation are mentioned as limitations, but failure modes of the benchmark mechanism itself—such as insufficient unit test suites, GPT-4o intent classification errors, or the 25% new-component threshold arbitrariness—are not discussed.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Agentless and Agentless-lite are evaluated as agent-framework baselines (Table 5); Oracle and BM25 retrieval baselines are provided; evaluation code adapted from SWE-bench is described with sufficient detail for reproduction, and will be publicly released.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Table 6 lists all 83 repositories with licenses, topics, and categories; Table 7 provides instance counts at each filtering stage; Appendix A.2 describes the full construction pipeline including fast validation, PR filtering, intent classification, and unit test verification.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "Source repository licenses are documented in Table 6, but the benchmark's own license is never stated; access is conditioned on 'will soon be publicly available' language with no release date, making current usability uncertain.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark's purpose is described (evaluating LLM feature implementation capability), and Docker evaluation is recommended in the Ethics Statement; however, what should NOT be concluded from results (e.g., no generalization to non-Python, no multi-round capability claims) is not specified.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "FEA-Bench is the first benchmark dedicated to evaluating repository-level feature implementation (incremental code development)",
       "evidence": "Comparison with SWE-bench (bug fixes) and code completion benchmarks in Section 2; Table 1 quantitatively shows FEA-Bench has 8× more new-component lines than SWE-bench",
       "supported": "moderate"
     },
     {
       "claim": "Current LLMs perform significantly worse on FEA-Bench than on bug-fix benchmarks; best model (DeepSeek-R1) resolves only ~10% of instances",
       "evidence": "Table 2: DeepSeek-R1 achieves 9.92% in Oracle/Detailed setting; SWE-bench resolution rates for comparable models are substantially higher",
       "supported": "strong"
     },
     {
       "claim": "Natural output format significantly outperforms Patch format across all tested models",
       "evidence": "Table 4: GPT-4o resolves 6.14% in Natural vs 1.86% in Patch mode; the success rate of git apply is also substantially higher in Natural mode",
       "supported": "strong"
     },
     {
       "claim": "More complex feature requests (more added functions) lead to lower resolved ratios",
       "evidence": "Figure 5: resolved ratio drops from 18.96% (1 added function) to 8.24% (2) to 5.47% (3+) for DeepSeek-R1",
       "supported": "strong"
     },
     {
       "claim": "Increasing context length (27K→40K tokens) does not improve and slightly hurts model performance despite marginally higher recall",
       "evidence": "Table 3: GPT-4 and GPT-4o performance is equal or lower at 40K vs 27K despite recall improving from 76.04% to 77.14%",
       "supported": "moderate"
     },
     {
       "claim": "Detailed new component hints generally lead to better performance than brief hints",
       "evidence": "Table 2: Detailed outperforms Brief in most settings for the full benchmark, though the pattern reverses in FEA-Bench lite for several models",
       "supported": "moderate"
     },
     {
       "claim": "Agentless improves resolved ratio over BM25 retrieval for most models due to better code edit format adherence",
       "evidence": "Table 5: GPT-4o improves from 4.0% (BM25) to 9.0% (Agentless); improvement correlates with increased %Apply success rate",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "FEA-Bench introduces 1,401 task instances from 83 Python repositories targeting new feature implementation via merged GitHub PRs, filtered by rule-based and GPT-4o intent classification. The best model (DeepSeek-R1) resolves only 9.92% of instances, substantially below SWE-bench performance levels, confirming that feature implementation is harder than bug fixing. Output format (Natural vs Patch) is a critical performance bottleneck—Natural format yields 3–5× higher success rates due to stricter formatting requirements of Patch mode. Task difficulty scales sharply with number of added functions (18.96% → 5.47% as functions increase from 1 to 3+), and increasing context length beyond 27K tokens does not improve performance despite marginally better file recall.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "The benchmark provides no human performance data, making it impossible to calibrate whether 9.92% represents near-zero capability or a reasonable fraction of human performance; a 'verified subset' with human annotations is only planned for future work."
     },
     {
       "flag": "GPT-4o used for construction and evaluation",
       "detail": "GPT-4o is used in the data collection pipeline for intent classification (filtering PRs as 'new feature'), and is also one of the primary models evaluated on the resulting benchmark—creating circularity in the evaluation."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Highly popular repositories (scikit-learn, django, sympy, matplotlib) are almost certainly in the training data of all evaluated models; no temporal splits or contamination detection are implemented, and the paper does not address this threat."
     },
     {
       "flag": "Missing results in main table",
       "detail": "Table 2 has numerous missing entries marked '×' for API-constrained models, and the paper attributes this to 'scarcity of API resources'; this leaves the main comparison incomplete for smaller open-source models against BM25 retrieval."
     },
     {
       "flag": "Arbitrary 25% threshold for new components",
       "detail": "The filtering criterion requiring new components to constitute >25% of edited lines is described as 'set relatively low' but is not empirically validated or sensitivity-tested."
     },
     {
       "flag": "MSRA affiliation conflict undisclosed",
       "detail": "Multiple authors are from Microsoft Research Asia, and the benchmark evaluates GPT-4 and GPT-4o (Microsoft-affiliated OpenAI models); no competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Direct predecessor benchmark for repository-level code generation (bug fixes); FEA-Bench explicitly positions itself as complementary, covering feature implementation rather than bug fixing"
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Foundational standalone code generation benchmark used as baseline comparison for what FEA-Bench advances beyond"
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "relevance": "Best-performing model on FEA-Bench (9.92% resolved); chain-of-thought reasoning models are a key evaluation subject"
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "relevance": "Agent framework evaluated on FEA-Bench lite (Table 5); represents state-of-the-art software engineering agent methodology"
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Referenced as an example of contamination-aware benchmark design that FEA-Bench does not replicate"
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "relevance": "Repository-level code completion benchmark representing the category FEA-Bench distinguishes itself from"
     },
     {
       "title": "EvoCodeBench: An Evolving Code Generation Benchmark Aligned with Real-World Code Repositories",
       "relevance": "Evolving benchmark design relevant to FEA-Bench's stated goal of continuous updates"
     },
     {
       "title": "Towards More Realistic Evaluation of LLM-Based Code Generation (future context leakage study)",
       "relevance": "Cited as motivation for why code completion benchmarks suffer from future context leakage—a problem FEA-Bench avoids by using PR-based task construction"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark is directly useful for researchers evaluating code LLMs on feature implementation, though not immediately actionable for practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that LLMs struggle with complex repository-level tasks is expected; the specific quantification (~10% max) adds some informational value but is not surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns are raised by this benchmark paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild drama in showing that even the best models fail 90%+ of feature implementation tasks, but no controversy or adversarial framing."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Code and data are not yet publicly available — only a promise of future release at a GitHub URL."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Microsoft Research Asia and Peking University; evaluates well-known models (GPT-4, DeepSeek-R1) and positions against the prominent SWE-bench."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "43021849",
         "title": "Competitive Programming with Large Reasoning Models",
         "points": 16,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43021849"
       },
       {
         "hn_id": "9262882",
         "title": "Exploring Non-Homogeneity and Dynamicity of High Scale Cloud [pdf]",
         "points": 9,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=9262882"
       },
       {
         "hn_id": "43025479",
         "title": "Competitive Programming with Large Reasoning Models",
         "points": 6,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43025479"
       },
       {
         "hn_id": "43072941",
         "title": "OpenAI: Competitive Programming with Large Reasoning Models",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43072941"
       },
       {
         "hn_id": "43022224",
         "title": "OpenAI o3 just scored 99.8% on CodeForces using brute-force",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43022224"
       },
       {
         "hn_id": "43030525",
         "title": "Competitive programming with large language models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43030525"
       },
       {
         "hn_id": "30685387",
         "title": "Infinite Wordle",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=30685387"
       },
       {
         "hn_id": "43055820",
         "title": "Competitive Programming with Large Reasoning Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43055820"
       },
       {
         "hn_id": "42705257",
         "title": "What Hawking Radiation Looks Like as You Fall into a Black Hole",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42705257"
       }
     ],
     "top_points": 16,
     "total_points": 41,
     "total_comments": 3
   }
 }