ai-research-survey

scan-v5.json (22604B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "FEA-Bench: A Benchmark for Evaluating Repository-Level Code Generation for Feature Implementation",
     "authors": [
       "Wei Li",
       "Xin Zhang",
       "Zhongxin Guo",
       "Shaoguang Mao",
       "Wen Luo",
       "Guangyue Peng",
       "Yangyu Huang",
       "Houfeng Wang",
       "Scarlett Li"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.06680",
     "doi": "10.48550/arXiv.2503.06680"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are backed by the paper: the benchmark uses PRs from 83 GitHub repos (Section 3.2), includes unit tests for verification (Section 3.3), and LLMs performing poorly is substantiated by Table 2 (best model ~10%).",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims like 'detailed hints lead to better performance' and 'increasing context beyond 27K reduces performance' are backed by direct ablation comparisons in Tables 2 and 3 with controlled prompt settings; the study design supports these comparisons.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds generalization to Python repositories and single-round generation in its Limitations section, and the benchmark is framed as measuring a specific task (incremental feature development), not general coding ability.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses why brief hints outperform detailed hints on the lite subset (lack of structured presentation in the prompt, Figure 6), and why BM25 sometimes matches Oracle (files containing new components are always included as known conditions).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses execution-based unit test pass rates as the metric and explicitly frames this as directly measuring whether the code change works correctly, not as a proxy — test pass/fail directly evaluates the implemented feature.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "There is a dedicated 'Limitations' section in the paper (before the Ethics Statement) discussing language coverage, data scarcity, and single-round evaluation.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: Python-only scope limits cross-language applicability; single-round generation akin to Pass@1 'may introduce a certain level of bias'; API scarcity caused missing results for some model/setting combinations.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states the benchmark covers only Python repositories and only single-round generation, and the paper's framing throughout restricts claims to incremental feature development (not bug fixing or standalone generation).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in Acknowledgments: National Science and Technology Major Project (No. 2022ZD0116308) and National Natural Science Foundation of China (62036001).",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed in the paper header: Peking University and Microsoft Research Asia.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The named funders (Chinese national science foundations) are independent of benchmark outcomes; however, several authors are from Microsoft Research Asia, and while Microsoft does not appear to have a competing product being advantaged, this affiliation is undisclosed as a potential interest.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement, no declaration of patents, equity, or consulting relationships; the paper only discloses funding and affiliations.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'repository-level incremental code development' is defined in the introduction, 'Oracle' and 'BM25' retrieval settings are explained in Section 4.2, 'resolved ratio' is defined, and 'new components' are defined as newly added functions and classes.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly bulleted in the introduction: introducing the task, constructing the first benchmark for it, and providing a scalable automated data collection pipeline with public release.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 thoroughly covers prior code LLMs and benchmarks, explicitly contrasting FEA-Bench against SWE-bench (bug fixing vs. feature implementation), code completion benchmarks (localized vs. repository-wide changes), and standalone benchmarks (HumanEval, MBPP).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that GitHub pull requests classified as 'new feature' by GPT-4o, verified by unit tests before and after patch application, measure incremental feature implementation capability — contrasting with SWE-bench (bug fixes) and completion benchmarks (localized edits).",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 provides statistics on lines edited, files edited, and added functions; Figure 5 shows that resolved ratio decreases as the number of added functions increases (18.96% for 1 function down to 5.47% for 3+), characterizing difficulty through complexity metrics.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "The best model resolves only ~10% of tasks — a near-floor result — but the paper does not explicitly discuss floor or ceiling effects as a design concern, nor does it assess whether the benchmark discriminates appropriately across model capability levels.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is included in the evaluation; the paper only evaluates LLMs, leaving the question of human-level performance on these tasks unanswered.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "The binary 'resolved' metric (all unit tests pass) is used without justifying it against alternatives such as partial credit or function-level pass rate; edge cases like vacuously passing tests are not discussed.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark uses publicly available GitHub pull requests that may be in LLM training corpora; there is no temporal split, canary strings, or other contamination-mitigation mechanism, and the paper does not discuss this risk.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper briefly notes intent to allow 'continuous updates and the creation of new versions of FEA-Bench' but does not discuss how the benchmark will be kept relevant as model capabilities improve or how gaming will be prevented.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper extensively discusses failure modes of LLMs on the benchmark (format adherence, context length, retrieval), but does not discuss failure modes of the benchmark itself — e.g., tests that are too weak, task instances that are ambiguous, or scenarios where the gold patch is not the only valid solution.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Comprehensive baseline results are provided in Table 2 across 12 models and multiple settings, and evaluation code is released at https://github.com/microsoft/FEA-Bench, enabling reproduction of reported numbers.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A provides detailed collection methodology, filtering criteria, and statistics for all 83 repositories (Tables 6 and 7); the pipeline stages are illustrated in Figure 3 with explicit quantification at each filtering step.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": true,
           "justification": "All 83 source repositories have licenses listed in Table 6; the paper states 'The dataset and code for our proposed method will be made publicly available for academic research' at the Microsoft GitHub repository.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The benchmark is explicitly scoped for evaluating LLMs on repository-level incremental feature development; the lite subset is designated for computationally expensive multi-round systems, and the Ethics Statement notes that Docker-based evaluation is recommended to prevent harm from generated code.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Current LLMs perform significantly worse on FEA-Bench than other benchmarks; the best model (DeepSeek-R1) resolves only ~10% of task instances.",
       "evidence": "Table 2 shows DeepSeek-R1 at 9.92% resolved ratio under Oracle+Detailed settings; the paper compares this unfavorably with LLM performance on HumanEval and SWE-bench.",
       "supported": "strong"
     },
     {
       "claim": "FEA-Bench tasks involve substantially more new code generation than SWE-bench, with new components averaging 87.1 lines (8x more) and constituting 67.8% of edits.",
       "evidence": "Table 1 directly compares FEA-Bench vs. SWE-bench statistics: average lines of added components (87.1 vs. 10.9), percentage of new component lines (67.8% vs. 28.9%).",
       "supported": "strong"
     },
     {
       "claim": "Detailed new component hints generally improve model performance over brief hints.",
       "evidence": "Table 2 shows detailed hints outperform brief in most model/setting combinations on the full benchmark, though the lite version shows the opposite trend, which the paper attributes to presentation issues.",
       "supported": "moderate"
     },
     {
       "claim": "Increasing context length from 27K to 40K tokens does not improve and slightly decreases model performance despite marginally better recall.",
       "evidence": "Table 3 shows GPT-4 and GPT-4o performance unchanged or slightly reduced at 40K vs. 27K, despite recall improving from 76.04% to 77.14%.",
       "supported": "strong"
     },
     {
       "claim": "Natural-format code edit generation significantly outperforms direct patch generation due to higher git apply success rates.",
       "evidence": "Table 4 shows GPT-4o Natural format resolves 6.14% vs. 1.86% for Patch, and apply success rates of 66.38% vs. 19.49% respectively.",
       "supported": "strong"
     },
     {
       "claim": "Task difficulty increases with the number of added functions; resolved ratio drops from 18.96% (1 function) to 5.47% (3+ functions).",
       "evidence": "Figure 5 shows the distribution of resolved vs. all instances by number of added functions with specific percentages reported in Section 6.5.",
       "supported": "strong"
     },
     {
       "claim": "The Agentless framework's improvement over BM25 retrieval is primarily attributable to better adherence to code editing format, not better retrieval.",
       "evidence": "Table 5 shows Agentless improvement correlates with higher %Apply success rates; the paper explicitly states this 'strongly correlates with the increased success rate of applying code edits.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "FEA-Bench introduces 1,401 task instances from 83 Python GitHub repositories specifically targeting repository-level feature implementation, a gap between code completion and bug-fixing benchmarks. Current LLMs perform poorly, with the best model (DeepSeek-R1) resolving only ~10% of tasks, demonstrating that incremental development is a substantially harder capability than issue resolution. Code edit output format is a critical limiting factor — natural format outperforms patch format by 3-4x in resolved ratio due to higher git apply success rates. Counterintuitively, providing more context beyond model window limits decreases performance, suggesting retrieval precision matters more than recall.",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "The benchmark uses publicly available GitHub pull requests that were likely in LLM training corpora. No temporal filtering, canary strings, or contamination testing is performed or discussed, making it impossible to assess how much LLMs are recalling versus reasoning."
     },
     {
       "flag": "No human baseline",
       "detail": "There is no human performance measurement on any subset of tasks. Without a human baseline, it is unclear whether 10% resolved ratio represents a meaningful capability gap or whether the tasks are unreasonably difficult even for humans."
     },
     {
       "flag": "Binary scoring not justified",
       "detail": "The all-or-nothing 'all unit tests must pass' metric is not justified against alternatives. Partial credit, function-level pass rates, or test coverage metrics are not considered; a single failing test disqualifies an otherwise correct implementation."
     },
     {
       "flag": "Single-round evaluation bias",
       "detail": "All experiments use single-round generation (Pass@1). The paper acknowledges this 'may introduce a certain level of bias' but does not report Pass@k or multi-round results, which would better characterize model capability on hard tasks."
     },
     {
       "flag": "Near-floor performance with no ceiling check",
       "detail": "Best model performance (~10%) is very close to floor, but the paper does not discuss whether this reflects benchmark design limitations, task impossibility, or genuine model failure. No easier subsets are fully analyzed beyond the lite version."
     },
     {
       "flag": "GPT-4o used in benchmark construction",
       "detail": "GPT-4o is used to classify PR intent as 'new feature' during dataset construction, and then GPT-4o is also evaluated as a benchmark participant. This creates a mild circularity where the model filtered the training distribution and is then tested on it."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "The most directly related benchmark; FEA-Bench explicitly positions itself as complementary, covering feature implementation where SWE-bench covers bug fixing."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Foundational standalone code generation benchmark used as contrast to motivate repository-level evaluation."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Relevant benchmark addressing contamination concerns in code evaluation — a gap FEA-Bench does not address."
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "relevance": "Prior repository-level code completion benchmark directly compared and contrasted with FEA-Bench."
     },
     {
       "title": "EvoCodeBench: An Evolving Code Generation Benchmark Aligned with Real-World Code Repositories",
       "relevance": "Related evolving benchmark for repository-level code generation; demonstrates temporal update strategies FEA-Bench could adopt."
     },
     {
       "title": "Agentless: Demystifying LLM-Based Software Engineering Agents",
       "relevance": "Agent framework evaluated on FEA-Bench in Section 6.3; results show current SOTA agents have substantial room for improvement on feature implementation."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion through Iterative Retrieval and Generation",
       "relevance": "Prior work on repository-level retrieval-augmented code generation; relevant to FEA-Bench's BM25 retrieval baseline."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "relevance": "Related complex code generation benchmark representing the expanding frontier of evaluation beyond HumanEval-style tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "The benchmark directly addresses a real-world software engineering task (adding features to codebases), is publicly released with evaluation code, and results show concrete performance gaps in current tools like Copilot and Devin."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that best-in-class models (DeepSeek-R1, o1) resolve only ~10% of tasks challenges optimism about LLMs for software engineering; the counter-intuitive result that more context decreases performance is also notable."
     },
     "fear_safety": {
       "score": 1,
       "justification": "The Ethics Statement warns that benchmark inference may generate code harmful to computer systems and recommends Docker isolation, a minor safety concern."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "There is mild competitive framing against SWE-bench and implicit positioning of DeepSeek-R1 outperforming OpenAI's o1, but no explicit controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The benchmark is publicly released on GitHub with evaluation scripts, allowing practitioners to run their own models against it, though the computational cost is high."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Microsoft Research Asia and Peking University are recognized institutions; the benchmark evaluates high-profile models (GPT-4, GPT-4o, o1, DeepSeek-R1) which attract attention."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "43021849",
         "title": "Competitive Programming with Large Reasoning Models",
         "points": 16,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43021849"
       },
       {
         "hn_id": "9262882",
         "title": "Exploring Non-Homogeneity and Dynamicity of High Scale Cloud [pdf]",
         "points": 9,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=9262882"
       },
       {
         "hn_id": "43025479",
         "title": "Competitive Programming with Large Reasoning Models",
         "points": 6,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43025479"
       },
       {
         "hn_id": "43072941",
         "title": "OpenAI: Competitive Programming with Large Reasoning Models",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43072941"
       },
       {
         "hn_id": "43022224",
         "title": "OpenAI o3 just scored 99.8% on CodeForces using brute-force",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43022224"
       },
       {
         "hn_id": "43030525",
         "title": "Competitive programming with large language models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43030525"
       },
       {
         "hn_id": "30685387",
         "title": "Infinite Wordle",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=30685387"
       },
       {
         "hn_id": "43055820",
         "title": "Competitive Programming with Large Reasoning Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43055820"
       },
       {
         "hn_id": "42705257",
         "title": "What Hawking Radiation Looks Like as You Fall into a Black Hole",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42705257"
       }
     ],
     "top_points": 16,
     "total_points": 41,
     "total_comments": 3
   }
 }