ai-research-survey

scan.json (32581B)

---

 {
   "paper": {
     "title": "FeatBench: Towards More Realistic Evaluation of Feature-level Code Generation",
     "authors": ["Haorui Chen", "Chengze Li", "Jia Li"],
     "year": 2026,
     "venue": "ACM TOSEM (submitted)",
     "arxiv_id": "2509.22237"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "FeatBench introduces an evolving benchmark for feature-level code generation with 157 tasks from 27 repositories, using only natural language requirements without code hints. The best agent configuration (Trae-agent + GPT-5) achieves only 29.94% resolved rate, with autonomous planning-based agents substantially outperforming pipeline-based approaches. The dominant failure mode (73.6%) is regressive implementation driven by 'scope creep,' where agents over-engineer beyond the user's explicit intent, breaking existing features.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code is available at https://github.com/TsinghuaISE/FeatBench' and 'We release FeatBench, our automated pipeline, and all experimental results to facilitate further community research.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset (157 tasks) is released along with all experimental results via the GitHub repository. The paper explicitly states all data is released."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Each benchmark task includes a Docker container (image_key field in Table 2) providing a reproducible runtime environment. Model settings are specified: temperature=0.0, top-p=1.0. The environment configuration pipeline (Section 3.3, Stage 2) is documented in detail."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a GitHub link and describes the pipeline in detail, but does not contain step-by-step reproduction instructions (no 'Reproducing Results' section or specific commands to run the evaluation)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 4 reports only point estimates (e.g., 29.94% resolved rate) with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims ('Trae-agent consistently outperforms Agentless') based solely on comparing raw percentages without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides sufficient context for effect sizes: 'average FV% of 41.72% vs 21.66%', 'File% of 76.42% vs 48.90%', and performance breakdowns showing resolution rates of 60-70% for small repos degrading to 10-30% for large ones."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final benchmark size of 157 tasks from 27 repositories is a result of their filtering pipeline, not a justified sample size. No power analysis or discussion of whether 157 tasks is sufficient for the conclusions drawn."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run evaluations. No variance, standard deviation, or spread measures are reported across experimental runs. While temperature=0.0 reduces randomness, agentic systems with tool calls and environment interactions can still vary."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two agent frameworks (Trae-agent and Agentless) are compared across four LLMs, providing cross-comparisons between frameworks and models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated models are very recent: GPT-5 (2025-08-07), DeepSeek V3.1, Doubao-Seed-1.6 (250615), Qwen3-Coder-Flash (2025-07-28). Both agent frameworks are recent SOTA systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not ablate benchmark design choices (e.g., effect of removing code hints vs. including them) or agent components. The comparison across agents and models is not an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six metrics are reported: Resolved%, Patch Apply Rate (Applied%), Regression Tests Pass Rate (RT%), Feature Validation Pass Rate (FV%), File-level Localization Success Rate (File%), and Token consumption."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 describes manual inspection of 122 failure cases by two computer science researchers from the author team, categorizing failures into three types. Section 6.1 has human solvability evaluation of 30 sampled tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark test suites (F2P and P2P tests) are explicitly 'withheld from the agent during inference to prevent data leakage' (Section 3.1). The agents are not tuned on FeatBench; they are evaluated as-is."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by repository complexity (Figures 8, 10), patch complexity (Figure 10), creation time period (Figure 9), and per-repository statistics (Appendix A.1)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 (RQ3) is a dedicated failure analysis. Figure 11 categorizes failures (73.6% regressive, 17.8% incomplete, 8.5% misunderstood). Figure 12 provides detailed case studies of both failure and success patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports significant negative findings: agents struggle with realistic tasks (max 29.94%), introduce widespread regressions (73.6% of failures), and performance collapses for large repos and patches. The 'aggressive implementation' pattern is presented as both positive and negative."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims — 29.94% max resolved rate (Table 4), scope creep leading to regressions (Section 5.3, Figure 11), 157 tasks from 27 repos (Table 3) — are all directly supported by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims: 'This high regression rate is driven by a behavioral pattern termed aggressive implementation' (Section 5.3). This causal link is inferred from manual case analysis of 122 failures by two author-evaluators, not from controlled experiments isolating the cause."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title frames results broadly as 'Feature-level Code Generation' but the benchmark is Python-only with 27 repositories. The abstract does not mention the Python limitation. Only Section 6.2 acknowledges 'FeatBench focuses primarily on Python repositories' and 'our findings may not fully extrapolate to statically typed languages.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses three specific threats: quality of LLM-generated requirements (hallucination risk), reliability of test-based evaluation (false positives), and generalizability limitations. The temporal analysis in Section 5.2 considers data leakage as an alternative explanation for performance patterns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures test pass rates (Resolved%, FV%, RT%) and frames them as measuring agent performance on feature implementation tasks. The claims match the granularity of the measurements without inflating to broader constructs like 'software engineering capability.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Three of four models have specific API identifiers with dates: gpt-5-2025-08-07, doubao-seed-1-6-250615, qwen3-coder-flash-2025-07-28. However, DeepSeek V3.1 is identified only as 'deepseek-chat' — a rolling API endpoint without a snapshot date or version pin."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 provides full prompt templates for the benchmark construction pipeline (feature identification, requirement synthesis, relevant file identification, environment configuration). The evaluation uses published open-source agent frameworks (Agentless, Trae-agent) whose prompts are in their cited repositories."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2: 'all LLMs are configured with a temperature of 0.0 and a top-p value of 1.0.' Section 4.3: 'we imposed a maximum limit of 150 steps per task' for Trae-agent and describe supplementary tools provided."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Agentless is described as 'a rigid, two-stage pipeline consisting of feature localization and patch generation' (Section 4.3). Trae-agent is described as 'an autonomous planning agent, utilizing a ReAct loop to dynamically plan execution paths and utilize tools' with 'supplementary tools, including ckg and json_edit_tool.'"
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 documents the full three-stage pipeline with filtering counts: 44 repos → 27 after repo-level filtering, 675 releases analyzed, 297 PRs extracted after PR-level filtering, → 157 final tasks after F2P validation. Each filtering criterion is specified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Threats to Validity' provides substantive discussion of three specific threats: quality of LLM-generated requirements, reliability of test-based evaluation, and generalizability/scope."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses threats specific to this study: LLM hallucination risk in requirement synthesis (mitigated by human verification showing 93.3% quality), false positive risk from sparse test suites (mitigated by dual F2P+P2P validation averaging 1694.6 tests), and Python-only scope limitation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states 'Currently, FeatBench focuses primarily on Python repositories' and 'our findings may not fully extrapolate to statically typed languages like Java or C++.' Section 4.3 notes the Agentless regression-testing stage was omitted. Future work section states plans to expand beyond Python."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We release FeatBench, our automated pipeline, and all experimental results.' The benchmark data (task instances, test suites, Docker images) and experimental outputs are released via GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the three-stage collection pipeline in detail: repository selection criteria (3+ releases, test suite, no tutorials, cutoff June 2024), release filtering, PR-level criteria (modifies Python files, includes tests, modifies functions without adding/deleting), and F2P validation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Repository selection is described (44 candidate repos filtered to 27 based on 4 criteria, Section 3.3). Human evaluators for solvability are described: 'two evaluators from the author team, both computer science researchers with over 2 years of Python development experience' (Section 6.1), paid $7.5/hour."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: 44 repos → 27 (repo-level filtering) → 675 releases → filtered releases → 297 PRs (PR-level filtering) → 157 tasks (F2P validation). Each stage has explicit criteria and the pipeline is illustrated in Figure 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. No grants, sponsors, or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Tsinghua University, University of Electronic Science and Technology of China, and Nanjing University. The authors evaluate third-party products (GPT-5, DeepSeek, Trae-agent, Agentless) rather than their own."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding is disclosed, independence of funding cannot be verified. The lack of any funding statement makes this unanswerable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the four evaluated models (GPT-5, DeepSeek V3.1, Doubao-Seed-1.6, Qwen3-Coder-Flash). They discuss contamination conceptually but not the specific cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 directly analyzes whether performance varies by task creation period (Figure 9), arguing 'consistent performance across task creation times validates the absence of data leakage.' The evolving benchmark design explicitly addresses train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a core focus of the paper. The evolving benchmark design (new versions every 6 months from latest repositories) specifically mitigates contamination. Section 5.2 validates this with temporal analysis showing no performance trend correlated with task age."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no human subjects study. The solvability evaluation (Section 6.1) uses two author-team members as evaluators in an internal quality check, not a human subjects experiment."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. The evaluators in Section 6.1 are from the author team performing an internal quality assessment."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The paper describes evaluator qualifications (CS researchers with 2+ years Python experience) but this is for an internal quality check, not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment is conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment is conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports token consumption per task for each configuration (ranging from 0.05M to 2.90M). Section 3.2 reports benchmark construction cost of '$0.28 per sample using DeepSeek V3.1.' Figure 7 plots token consumption vs. resolved rate."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-task token averages are reported but total computational budget (total API spend, total GPU hours, wall-clock time for all experiments) is not stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. While temperature=0.0 reduces randomness, agentic systems with tool calls and environment interactions can still exhibit non-determinism. No multi-run analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Results are implicitly single-run evaluations with no indication of repeated trials."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The agent frameworks are used with their default settings plus specified temperature=0.0 and max_steps=150. No search budget or justification for these choices is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 8 configurations (2 agents × 4 models) are evaluated and reported in Table 4. No cherry-picking — all results shown, including poor-performing configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 8 configurations and various breakdown dimensions, but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate third-party agents on their own benchmark. They do not discuss potential bias in benchmark construction that could favor certain agent paradigms or the risk that their benchmark design choices systematically advantage or disadvantage particular approaches."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 explicitly plots token consumption vs. resolved rate for all configurations. The paper discusses the cost-performance tradeoff: 'Trae-agent exhibits a near-linear trend where achieving a higher resolved rate requires proportionally greater token consumption' (Section 5.1)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is framed around benchmark construct validity — arguing existing benchmarks fail to measure realistic feature implementation due to code hints and static data. Section 6.1 validates solvability of task requirements (93.3% rated fully solvable). Section 6.2 discusses validity threats."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically compares 4 models across 2 frameworks (Table 4), allowing isolation of scaffold effects from model effects. The framework paradigm difference (autonomous vs. pipeline) is explicitly analyzed as a variable, not a confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 analyzes performance across five creation time periods (2308 to 2509) in Figure 9, showing consistent resolved rates. The evolving benchmark design explicitly addresses temporal leakage by sourcing tasks from recent repository updates."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Eliminating feature leakage is a core contribution. Task inputs are 'strictly devoid of code hints' (Section 3.2). Test cases are 'withheld from the agent during inference' (Section 3.1). The paper contrasts this against existing benchmarks that leak through function signatures."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Tasks are drawn from 27 repositories with highly uneven distribution: conan-io/conan contributes 56/157 tasks (35.7%). The paper does not discuss whether tasks from the same repository are independent or whether repository-level clustering biases results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 applies temporal analysis as a concrete detection method: comparing resolved rates across five task creation periods to check for performance trends that would indicate training data overlap. Figure 9 shows the analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "FeatBench poses a significant challenge to SOTA agents, with the highest resolved rate reaching only 29.94% (Trae-agent + GPT-5).",
       "evidence": "Table 4 shows Trae-agent + GPT-5 achieves 29.94% Resolved%, the highest among all 8 configurations tested. Average across all Trae-agent configs is 22.13%, Agentless average is 10.83%.",
       "supported": "strong"
     },
     {
       "claim": "Autonomous planning-based agents substantially outperform pipeline-based agents on feature implementation tasks.",
       "evidence": "Table 4: Trae-agent averages 22.13% resolved vs. Agentless 10.83%. FV% averages 41.72% vs 21.66%, File% averages 76.42% vs 48.90%. Consistent advantage across all four models (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "Agent performance is strictly constrained by repository and patch complexity, with resolved rates degrading sharply for large repos.",
       "evidence": "Figures 8 and 10: Resolved rates reach 60-70% for repos with <200 files but drop to 10-30% for repos with >800 files. Performance collapses for patches spanning >5 files or >50 LOC (Section 5.2).",
       "supported": "strong"
     },
     {
       "claim": "Consistent performance across task creation periods confirms the absence of data leakage.",
       "evidence": "Figure 9 shows resolved rates across five time periods (2308-2509) remain remarkably consistent for Trae-agent + Doubao-Seed-1.6, with no upward trend that would suggest memorization (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Regressive implementation is the predominant failure reason, accounting for 73.6% of analyzed failure cases.",
       "evidence": "Manual analysis of 122 Trae-agent failure cases by two CS researchers classified 73.6% as regressive implementation, 17.8% as incomplete implementation, and 8.5% as misunderstood requirements (Section 5.3, Figure 11).",
       "supported": "moderate"
     },
     {
       "claim": "93.3% of synthesized task requirements are comprehensive and unambiguous.",
       "evidence": "Human evaluation of 30 sampled tasks (19% of benchmark): 28/30 scored 2 (fully solvable), 2/30 scored 1, none scored 0. Average score 1.93/2.0 (Section 6.1).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Severe repository distribution imbalance",
       "detail": "conan-io/conan contributes 56 of 157 tasks (35.7%), heavily skewing benchmark results toward a single DevOps project. Overall results may disproportionately reflect agent performance on Conan specifically rather than feature implementation broadly."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (Trae-agent vs. Agentless, model rankings) are based on comparing raw percentages without any significance tests. With only 157 tasks and no repeated runs, observed differences could reflect noise."
     },
     {
       "flag": "Author-team evaluators for human validation",
       "detail": "Both the solvability study (Section 6.1) and the failure analysis (Section 5.3) were conducted by researchers from the author team. No independent evaluators were used, creating potential bias in the quality assessment and failure categorization."
     },
     {
       "flag": "Single-run evaluation without variance",
       "detail": "All experimental results are from single runs. While temperature=0.0 reduces randomness, agentic systems with 150-step ReAct loops, tool calls, and environment interactions can exhibit non-deterministic behavior. No stability analysis is provided."
     },
     {
       "flag": "Temporal analysis on single model only",
       "detail": "The data leakage temporal analysis (Figure 9) is shown only for Trae-agent + Doubao-Seed-1.6, not for all 8 configurations. This selective reporting weakens the data leakage absence claim."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Foundational benchmark for repository-level code generation that FeatBench extends to feature implementation with stricter task formulations."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Pipeline-based agent framework evaluated in FeatBench, representing the rigid localization-repair paradigm for code generation."
     },
     {
       "title": "Trae Agent: An LLM-based Agent for Software Engineering with Test-time Scaling",
       "authors": ["Trae Research Team", "Pengfei Gao", "Zhao Tian"],
       "year": 2025,
       "arxiv_id": "2507.23370",
       "relevance": "Autonomous planning agent evaluated in FeatBench, achieving the highest resolved rate and demonstrating the ReAct-based paradigm."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Influential autonomous agent framework using ReAct loop for software engineering tasks."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark — seminal function-level code generation benchmark that FeatBench builds upon by moving to feature-level."
     },
     {
       "title": "Evocodebench: An evolving code generation benchmark aligned with real-world code repositories",
       "authors": ["Jia Li", "Ge Li", "Xuanming Zhang", "Yihong Dong", "Zhi Jin"],
       "year": 2024,
       "relevance": "Evolving code generation benchmark concept that FeatBench extends to feature-level with an automated renewal pipeline."
     },
     {
       "title": "Fea-bench: A benchmark for evaluating repository-level code generation for feature implementation",
       "authors": ["Wei Li", "Xin Zhang", "Zhongxin Guo"],
       "year": 2025,
       "relevance": "Prior feature-level benchmark that FeatBench directly addresses limitations of (reliance on code hints like function signatures)."
     },
     {
       "title": "NoCode-bench: A Benchmark for Evaluating Natural Language-Driven Feature Addition",
       "authors": ["Le Deng", "Zhonghao Jiang", "Jialun Cao", "Michael Pradel", "Zhongxin Liu"],
       "year": 2025,
       "relevance": "Concurrent feature-level benchmark using documentation-update specifications; FeatBench argues its requirement format is more realistic."
     },
     {
       "title": "SWE-bench Goes Live!",
       "authors": ["Linghao Zhang", "Shilin He", "Chaoyun Zhang"],
       "year": 2025,
       "relevance": "Live evolution of SWE-bench that inspired FeatBench's evolving benchmark approach and environment configuration methodology."
     },
     {
       "title": "SWE-PolyBench: A multi-language benchmark for repository level evaluation of coding agents",
       "authors": ["Muhammad Shihab Rashid", "Christian Bock", "Yuan Zhuang"],
       "year": 2025,
       "relevance": "Multi-language benchmark extension that FeatBench does not yet address; FeatBench is currently Python-only."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik R Narasimhan", "Yuan Cao"],
       "year": 2022,
       "relevance": "Agent reasoning framework underlying Trae-agent's architecture; the autonomous planning paradigm evaluated in FeatBench."
     },
     {
       "title": "Classeval: A manually-crafted benchmark for evaluating llms on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "relevance": "Class-level code generation benchmark representing the intermediate complexity step between function-level and feature-level evaluation."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly useful for evaluating coding agents on realistic tasks; the benchmark and pipeline are released, but practitioners need to set up Docker infrastructure to use it."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expectation that agents struggle with realistic tasks; the 'aggressive implementation' finding about scope creep is a moderately novel observation."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or AI risk angle; the paper is about benchmark methodology for coding agents."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild implicit critique of existing benchmarks (FEA-Bench, NoCode-bench) for being unrealistic, but framed diplomatically without direct confrontation."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and benchmark released on GitHub (https://github.com/TsinghuaISE/FeatBench); requires significant setup (Docker, API keys) but is reproducible."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Tsinghua University (well-known in AI research); evaluates GPT-5 and DeepSeek which have moderate brand recognition."
     }
   }
 }