ai-research-survey

scan-v4.json (19942B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "FeatBench: Towards More Realistic Evaluation of Feature-level Code Generation",
     "authors": [
       "Haorui Chen",
       "Chengze Li",
       "Jia Li"
     ],
     "year": 2026,
     "venue": "ACM Transactions on Software Engineering and Methodology (preprint)",
     "arxiv_id": "2509.22237",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims — 29.94% max resolved rate (Table 4), scope creep leading to regressions (Section 5.3, Figure 11), 157 tasks from 27 repos (Table 3) — are all directly supported by results in the paper.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims: 'This high regression rate is driven by a behavioral pattern termed aggressive implementation' (Section 5.3). This causal link is inferred from manual case analysis of 122 failures by two author-evaluators, not from controlled experiments isolating the cause.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title frames results broadly as 'Feature-level Code Generation' but the benchmark is Python-only with 27 repositories. The abstract does not mention the Python limitation. Only Section 6.2 acknowledges 'FeatBench focuses primarily on Python repositories' and 'our findings may not fully extrapolate to statically typed languages.'",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses three specific threats: quality of LLM-generated requirements (hallucination risk), reliability of test-based evaluation (false positives), and generalizability limitations. The temporal analysis in Section 5.2 considers data leakage as an alternative explanation for performance patterns.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures test pass rates (Resolved%, FV%, RT%) and frames them as measuring agent performance on feature implementation tasks. The claims match the granularity of the measurements without inflating to broader constructs like 'software engineering capability.'",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Threats to Validity' provides substantive discussion of three specific threats: quality of LLM-generated requirements, reliability of test-based evaluation, and generalizability/scope.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses threats specific to this study: LLM hallucination risk in requirement synthesis (mitigated by human verification showing 93.3% quality), false positive risk from sparse test suites (mitigated by dual F2P+P2P validation averaging 1694.6 tests), and Python-only scope limitation.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states 'Currently, FeatBench focuses primarily on Python repositories' and 'our findings may not fully extrapolate to statically typed languages like Java or C++.' Section 4.3 notes the Agentless regression-testing stage was omitted. Future work section states plans to expand beyond Python.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. No grants, sponsors, or funding agencies are mentioned.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Tsinghua University, University of Electronic Science and Technology of China, and Nanjing University. The authors evaluate third-party products (GPT-5, DeepSeek, Trae-agent, Agentless) rather than their own.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding is disclosed, independence of funding cannot be verified. The lack of any funding statement makes this unanswerable.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "'Feature-level code generation', 'Resolved Rate', 'F2P/P2P tests', 'aggressive implementation', and 'scope creep' are all defined precisely with examples in the paper.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Contributions are explicitly numbered: a benchmark with realistic NL-only inputs, an evolving automated pipeline, a rigorous data collection process, comprehensive test suites, and diverse domain coverage.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 systematically compares FeatBench against seven prior benchmarks across five dimensions; Section 2 explains how FeatBench differs from SWE-bench, FEA-Bench, NoCode-bench, and others in terms of specific design choices.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues that removing code hints forces agents to independently bridge user intent to code (measuring true feature implementation capability), and that F2P+P2P tests measure both functional correctness and backward compatibility—articulating why this design measures the claimed capability.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Figures 8 and 10 characterize difficulty empirically by repository complexity (files, LOC) and patch complexity, showing resolved rates ranging from 60-70% for small repos to 10-30% for large ones—difficulty is measured rather than assumed.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not explicitly check for ceiling or floor effects at the task level; while the 29.94% maximum resolved rate rules out an obvious ceiling, the complexity analysis (Figs. 8-10) suggests near-zero success for many task subsets without flagging this as a floor-effect concern.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human performance baseline is provided; the human evaluation in Section 6.1 assesses requirement solvability (information completeness), not how well human developers would perform on the implementation tasks.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "The dual F2P+P2P validation is justified: F2P verifies the new feature works, P2P ensures no regressions—this mirrors production-grade software engineering standards and the potential false-positive risk is acknowledged and partially mitigated by scale (avg. 1,694.6 test cases per task).",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Contamination resistance is explicitly designed in through a June 2024 cutoff for initial tasks (post-training-data) and a planned 6-month automated refresh cycle; Fig. 9 empirically validates that resolved rates are consistent across task creation periods.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.2 explicitly tests temporal robustness by analyzing resolved rates across five creation periods (2308-2509) and finding no performance trend, validating the evolving benchmark strategy; the automated pipeline for 6-month updates is described.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6.2 discusses three benchmark failure modes: LLM hallucination in synthesized requirements, false positives from sparse F2P test suites, and generalizability limits to Python; the paper does not discuss active gaming strategies beyond data leakage.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The benchmark, automated pipeline, and all experimental results are open-sourced at https://github.com/TsinghuaISE/FeatBench; Section 4.3 details exact agent configurations (temperature=0.0, max 150 steps, specific tool sets) for reproducibility.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 defines all task instance fields, Table 3 provides repository and instance statistics, Appendix A.1 lists all 27 repositories with licenses and instance counts, and Section 3.3 documents the three-stage collection pipeline in detail.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A.1 lists the license for all 27 source repositories (MIT, Apache-2.0, BSD-3-Clause, LGPL-3.0); the benchmark is publicly accessible at the provided GitHub URL.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "The intended use (evaluating LLM agents on feature-level code generation without code hints) is clearly stated; the threats section delimits what cannot be concluded (no generalization to non-Python languages), though the paper lacks explicit guidance on misuse cases.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "FeatBench poses a significant challenge: the top-performing configuration (Trae-agent + GPT-5) achieves a resolved rate of only 29.94%.",
       "evidence": "Table 4 reports all agent-model combinations; Trae-agent + GPT-5 is the highest at 29.94%. All other combinations are lower.",
       "supported": "strong"
     },
     {
       "claim": "Autonomous planning agents substantially outperform rigid pipeline-based agents on feature implementation tasks.",
       "evidence": "Table 4: Trae-agent average resolved rate 22.13% vs. Agentless 10.83%; Trae-agent file localization 76.42% vs. Agentless 48.90%.",
       "supported": "strong"
     },
     {
       "claim": "Regressive implementation is the predominant failure reason, accounting for 73.6% of analyzed failure cases, driven by 'aggressive implementation' scope creep.",
       "evidence": "Manual inspection of 122 Trae-agent failure cases by two computer science researchers from the author team (Fig. 11); two case studies provided.",
       "supported": "moderate"
     },
     {
       "claim": "Agent performance is strictly constrained by repository and patch complexity, with resolved rates dropping to 10-30% for repos exceeding 800 files or 300K LOC.",
       "evidence": "Fig. 8 shows consistent inverse correlation between repo file count/LOC and resolved rate across all four LLMs; Fig. 10 shows near-zero success for patches spanning 5+ files or 50+ LOC.",
       "supported": "strong"
     },
     {
       "claim": "Consistent performance across task creation periods confirms the absence of data leakage.",
       "evidence": "Fig. 9 shows stable resolved rates for Trae-agent + Doubao-Seed-1.6 across five creation periods (2308-2509); only one agent-model pair tested for this claim.",
       "supported": "moderate"
     },
     {
       "claim": "93.3% of sampled synthesized requirements are fully solvable and unambiguous.",
       "evidence": "Human evaluation of 30 randomly sampled tasks by two author-team researchers using a 3-point Likert scale; average score 1.93/2.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "FeatBench is a 157-task benchmark for feature-level code generation that removes code hints and uses an automated evolving pipeline to prevent data contamination, sourcing tasks from 27 actively maintained Python repositories. State-of-the-art coding agents achieve a maximum resolved rate of only 29.94% (Trae-agent + GPT-5), highlighting the difficulty of realistic feature implementation. Autonomous planning agents substantially outperform pipeline-based agents but at much higher token cost. The dominant failure mode is 'regressive implementation' (73.6% of failures), where agents exhibit scope creep by implementing beyond the specified requirements and breaking existing tests—a behavior that occasionally produces superior architectural designs but more commonly introduces defects.",
   "red_flags": [
     {
       "flag": "Author-team failure analysis",
       "detail": "The 73.6% regressive implementation finding is based on manual inspection of 122 failure cases by two researchers who are members of the author team, introducing potential categorization bias and lack of independence."
     },
     {
       "flag": "Temporal consistency tested on one configuration",
       "detail": "The no-data-leakage claim (Fig. 9) is validated using only one agent-model pair (Trae-agent + Doubao-Seed-1.6); other combinations are not tested for temporal consistency."
     },
     {
       "flag": "Small human solvability sample",
       "detail": "Only 30 of 157 tasks (19%) were human-evaluated for requirement quality, and the evaluators were from the author team rather than independent assessors."
     },
     {
       "flag": "No human performance baseline",
       "detail": "The benchmark includes no measurement of how human developers would perform on these tasks, making it impossible to calibrate how far agents lag behind human-level performance."
     },
     {
       "flag": "Python-only scope with broad claims",
       "detail": "All 157 tasks are from Python repositories, but the abstract and conclusions make general statements about 'feature-level code generation' without consistently foregrounding this constraint."
     },
     {
       "flag": "LLM-generated requirements not independently validated",
       "detail": "The natural language requirements are reverse-engineered by an LLM from code diffs; while a human check was done by author-team members, there is no independent verification that the requirements are fully equivalent to the original developer intent."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "relevance": "Primary related benchmark; FeatBench explicitly differentiates from SWE-bench's bug-fixing focus and extends to feature implementation"
     },
     {
       "title": "FEA-Bench: A Benchmark for Evaluating Repository-Level Code Generation for Feature Implementation",
       "relevance": "Direct predecessor benchmark; FeatBench addresses its limitation of providing function signatures as code hints"
     },
     {
       "title": "NoCode-bench: A Benchmark for Evaluating Natural Language-Driven Feature Addition",
       "relevance": "Direct predecessor; FeatBench addresses its use of documentation-update identifier hints rather than pure NL requirements"
     },
     {
       "title": "EvoCodeBench: An Evolving Code Generation Benchmark Aligned with Real-World Code Repositories",
       "relevance": "Precedent for evolving benchmark design; FeatBench extends this concept to feature-level tasks"
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Foundational function-level code generation benchmark; establishes the baseline evaluation tradition FeatBench extends"
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "relevance": "One of two agent frameworks evaluated on FeatBench; represents the pipeline-based agent paradigm"
     },
     {
       "title": "Trae Agent: An LLM-based Agent for Software Engineering with Test-time Scaling",
       "relevance": "The other agent framework evaluated; represents the autonomous planning agent paradigm and achieves best performance on FeatBench"
     },
     {
       "title": "SWE-bench Goes Live!",
       "relevance": "Related evolving benchmark work; FeatBench's environment configuration pipeline is inspired by SWE-bench-Live's two-phase approach"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly useful for evaluating coding agents on realistic tasks; the benchmark and pipeline are released, but practitioners need to set up Docker infrastructure to use it."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expectation that agents struggle with realistic tasks; the 'aggressive implementation' finding about scope creep is a moderately novel observation."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or AI risk angle; the paper is about benchmark methodology for coding agents."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild implicit critique of existing benchmarks (FEA-Bench, NoCode-bench) for being unrealistic, but framed diplomatically without direct confrontation."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and benchmark released on GitHub (https://github.com/TsinghuaISE/FeatBench); requires significant setup (Docker, API keys) but is reproducible."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Tsinghua University (well-known in AI research); evaluates GPT-5 and DeepSeek which have moderate brand recognition."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "44157561",
         "title": "Yambda-5B – A Large-Scale Multi-Modal Dataset for Ranking and Retrieval",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44157561"
       },
       {
         "hn_id": "44427694",
         "title": "Can Large Language Models Help Students Prove Software Correctness?",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44427694"
       }
     ],
     "top_points": 3,
     "total_points": 4,
     "total_comments": 0
   }
 }