ai-research-survey

scan.json (30483B)

---

 {
   "paper": {
     "title": "FEA-Bench: A Benchmark for Evaluating Repository-Level Code Generation for Feature Implementation",
     "authors": [
       "Wei Li",
       "Xin Zhang",
       "Zhongxin Guo",
       "Shaoguang Mao",
       "Wen Luo",
       "Guangyue Peng",
       "Yangyu Huang",
       "Houfeng Wang",
       "Scarlett Li"
     ],
     "year": 2025,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2503.06680",
     "doi": "10.48550/arXiv.2503.06680"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "FEA-Bench introduces 1,401 benchmark tasks from 83 GitHub repositories for evaluating LLMs on repository-level feature implementation, requiring substantially more code generation than bug-fix-oriented SWE-bench. The best model (DeepSeek-R1) resolves only 9.92% of tasks in the oracle setting, demonstrating that feature implementation is significantly harder than bug fixing. Natural output format substantially outperforms patch format, and increasing context length from 27K to 40K tokens does not improve performance. Task difficulty scales with complexity: resolve rate drops from ~19% for single-function tasks to ~5% for 3+ function tasks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Our code will soon be publicly available at https://github.com/microsoft/FEA-Bench' — this is a future promise, not an actual release. Per scan criteria, 'will be released' counts as NO."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper promises future release: 'We will publicly release our data collection and evaluation codebase, allowing FEA-Bench to be continuously updated.' The dataset is described but not yet available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions vLLM framework on 8-GPU NVIDIA A100, and gives generation settings (Appendix B.2), but no requirements.txt, Dockerfile, or detailed dependency specification is provided. Mentioning hardware and a framework is insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the pipeline and evaluation methodology but does not include a README or script-level instructions for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-5 are point estimates (e.g., '9.92%') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'DeepSeek-V3 and R1 models achieve the best performance, significantly outperforming OpenAI's GPT-4 and o1 series') based solely on comparing raw numbers with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw resolved ratios in tables. No effect sizes (Cohen's d, odds ratios, or framed percentage improvements) are explicitly computed for model comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 1,401 task instances and the lite version 200, but no justification is given for why these sizes are adequate. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section explicitly acknowledges: 'the experimental results are based on a single round generation, akin to Pass@1, which may introduce a certain level of bias.' No variance across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 13 models across multiple settings (Table 2) and also compares direct generation against the Agentless framework (Table 5). Multiple models serve as baselines for each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Evaluated models include DeepSeek-R1, DeepSeek-V3, GPT-4o, o1, and Qwen2.5-Coder — all recent and competitive models at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies context settings (Oracle vs BM25), hint levels (Detailed vs Brief), output formats (Natural vs Patch), and context lengths (27K vs 40K), analyzing the effect of each factor independently."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports resolved ratio (unit test pass rate), %Apply (patch application success rate), and retrieval metrics (precision, recall) across different experiments."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via unit test execution. No human evaluation of generated code quality is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "FEA-Bench is a fixed benchmark — models are pre-trained and not tuned on the benchmark data. The evaluation uses the benchmark as a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows resolved ratios across repository categories (Testing, Internet, AI, etc.). Section 6.5 and Figure 5 provide breakdown by number of added functions. The lite version is also separately reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses quantitative failure patterns (format issues in Section 6.2, complexity in Section 6.5) but does not show qualitative examples of specific failures — no examples of what models generated incorrectly vs the expected output."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative findings: increasing context from 27K to 40K tokens decreases performance (Table 3), patch output format performs significantly worse than natural format (Table 4), and the main finding itself — LLMs resolve at most ~10% of tasks — is a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'LLMs perform significantly worse in the FEA-Bench' and 'the best-performing LLM, DeepSeek-R1, successfully resolves only about 10%' are directly supported by Table 2 (9.92% resolved ratio)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal-like claims (e.g., 'the format of code edits is a critical factor limiting performance') are supported by controlled comparisons varying one factor at a time (same model, different formats in Table 4; same model, different context lengths in Table 3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The title scopes to 'repository-level code generation for feature implementation.' The Limitations section explicitly states 'our benchmark includes only Python repositories' and acknowledges 'certain scenario limitations.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper explores different experimental factors (retrieval, format, context length), it does not substantively discuss alternative explanations for the main finding of low performance — e.g., whether benchmark construction choices, test quality, or task formulation artificially inflate difficulty."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures unit test pass rate and frames claims tightly around 'resolved ratio' of task instances. Claims match the granularity of measurement — they do not extrapolate from test-passing to broader software quality claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 provides exact API versions: 'gpt-4-turbo-2024-04-09', 'gpt-4o-2024-05-13', 'o1-2024-12-17', 'o1-mini-2024-09-12'. Open-source models specify sizes (e.g., 'Qwen2.5-Coder 14B/32B')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 6 provides the complete prompt structure with actual text for all sections (request, definitions, code, instructions). Figure 7 provides the full output instruction text for both Natural and Patch formats."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 reports: 'temperature and top-p settings are fixed at 0.2 and 0.95, respectively.' Max output tokens are stated for each model family (4096, 8192, 100K, 64K)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses the Agentless framework (Section 6.3) but only cites it without describing its internal pipeline (localization, repair steps, iterative refinement). A reader cannot understand the scaffold from this paper alone."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.2-3.3 and Appendix A.2 describe the full pipeline: repository selection from Top PyPI, PR filtering criteria, new component extraction, intent-based filtering with GPT-4o, and unit test verification. Table 7 provides instance counts at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present between Section 7 (Conclusion) and the Ethics Statement, with substantive discussion across multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific threats: Python-only scope, scarcity of high-quality new-feature PRs, early-stage repo development not captured, single-round generation bias, and resource constraints limiting DeepSeek experiments."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states Python-only scope, acknowledges 'certain scenario limitations' from the PR-based construction, and notes that 'high-quality and usable pull requests for new feature development are relatively scarce.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark data is not yet released. The paper promises future release but no data is currently available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.2-3.3 describe data collection in detail: sourcing from Top PyPI repositories, fast validation of 20 PRs per repo, PR crawling, new component extraction via AST parsing, intent-based GPT-4o classification, and unit test verification."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Repository selection criteria are clearly described: Top PyPI packages with licenses and >1,000 PRs, plus the 18 SWE-bench repositories. The filtering process from ~8,000 packages to ~600 candidates to 119 repositories is documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 documents the full pipeline with exact counts at each stage for all 83 repositories: merged PRs → tasks with tests → candidates after filtering → final instances → lite instances. Appendix A.2 describes each step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments state: 'This work was supported by National Science and Technology Major Project (No. 2022ZD0116308) and National Natural Science Foundation of China (62036001).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Peking University and Microsoft Research Asia. The paper header marks these with symbols (♡ and ♠)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Chinese government agencies (NSFC, national project) which have no direct financial stake in the benchmark results or the relative performance of any evaluated model."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is present. Authors from Microsoft Research Asia could have interests related to AI code generation products, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the evaluated models, despite evaluating them on a benchmark constructed from public GitHub PRs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the GitHub PR data used for FEA-Bench appears in the training data of the evaluated models. Given these are public PRs from popular repositories, overlap is likely."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the benchmark data was available online before the models' training cutoffs. Popular repositories like scikit-learn, django, and sympy are highly likely to appear in training corpora."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates LLMs on an automated benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available GitHub data and evaluates LLMs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, per-example costs, or total inference costs are reported. The paper mentions hardware (A100 workstation) and token limits but not actual cost figures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using '8-GPU NVIDIA A100 workstation' but does not report total GPU hours, API spend, or total computational budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are from single-run generation. The Limitations section acknowledges: 'experimental results are based on a single round generation, akin to Pass@1.' No seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'LLMs perform a single generation for each task instance' (Appendix B.2) and acknowledges this as a limitation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.2 and top-p 0.95 are used without justification for these choices. No hyperparameter search or sensitivity analysis is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results across all configurations tested (Oracle/BM25, Detailed/Brief, Natural/Patch) rather than cherry-picking a single best configuration. Table 2 shows all combinations."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors construct FEA-Bench and design the evaluation, but do not discuss potential bias in benchmark construction (e.g., whether their filtering choices systematically disadvantage certain model architectures)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 13B to 671B parameters are compared without normalization for compute. The paper notes 'models with larger parameter sizes demonstrate better results' but does not control for compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 provides substantial construct validity discussion: FEA-Bench's characteristics vs SWE-bench, the nature of feature implementation (new components + complementary changes), and statistics showing the benchmark captures what it claims (67.8% new component lines)."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "In Table 2, all models use the same prompt/retrieval framework. In Table 5, the Agentless framework is compared against direct retrieval for the same models. Model comparisons are not confounded by scaffold differences."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the GitHub PRs in FEA-Bench predate the training data of the evaluated models. Given popular repos like scikit-learn and django, temporal overlap is highly likely."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides new component signatures as hints but does not discuss whether this constitutes feature leakage or how it affects fair comparison. The 'Detailed' setting additionally provides non-Python file changes that could leak implementation details."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data of evaluated models contains code from the same repositories used in FEA-Bench. Models trained on GitHub data likely saw code from django, scikit-learn, sympy, etc."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, n-gram overlap, membership inference, temporal splits) is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "The best-performing LLM (DeepSeek-R1) resolves only about 10% of FEA-Bench tasks in the Oracle+Detailed setting.",
       "evidence": "Table 2 shows DeepSeek-R1 achieves 9.92% resolved ratio under Oracle/Detailed settings on the full FEA-Bench.",
       "supported": "strong"
     },
     {
       "claim": "FEA-Bench tasks require substantially more code generation than SWE-bench, with 8x more lines of new components on average.",
       "evidence": "Table 1 shows FEA-Bench averages 87.1 lines of added components vs 10.9 in SWE-bench, and 128.5 total edited lines vs 37.71.",
       "supported": "strong"
     },
     {
       "claim": "Natural output format significantly outperforms Patch format for code edit generation.",
       "evidence": "Table 4 shows Natural format yields higher resolved ratios across all models (e.g., DeepSeek-R1: 9.92% Natural vs not tested, GPT-4: 4.71% vs 3.07%) and much higher patch application rates (e.g., GPT-4o: 66.38% vs 19.49%).",
       "supported": "strong"
     },
     {
       "claim": "Increasing context length from 27K to 40K tokens does not improve and may decrease model performance.",
       "evidence": "Table 3 shows GPT-4 performance stays at 3.14% and GPT-4o drops from 5.28% to 4.78% when expanding from 27K to 40K BM25 retrieval, despite slight recall improvement.",
       "supported": "moderate"
     },
     {
       "claim": "Task difficulty increases with the number of new components, with resolved ratio dropping from ~19% for 1 function to ~5% for 3+ functions.",
       "evidence": "Section 6.5 and Figure 5 show DeepSeek-R1's resolved ratio drops from 18.96% (1 added function) to 8.24% (2 functions) to 5.47% (≥3 functions).",
       "supported": "strong"
     },
     {
       "claim": "Agentless framework improves performance over direct BM25 retrieval for most models.",
       "evidence": "Table 5 shows improvements for GPT-4o (4.0%→9.5%), o1-mini (1.0%→4.5%), and o1 (5.0%→10.0%) on the lite subset, though the improvement correlates with better patch application rates.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All model comparisons are based on point estimates without any statistical tests. Claims like 'significantly outperforming' are made without p-values or confidence intervals, making it impossible to assess whether observed differences are meaningful or within noise."
     },
     {
       "flag": "Single-run evaluation",
       "detail": "All results are from single-run generation (acknowledged in Limitations). With stochastic models using temperature=0.2, results could vary across runs, but no variance is reported."
     },
     {
       "flag": "No contamination analysis for benchmark using popular public repositories",
       "detail": "FEA-Bench uses PRs from highly popular repositories (scikit-learn, django, sympy) that are almost certainly in the training data of evaluated models. No contamination analysis, temporal split, or decontamination is performed."
     },
     {
       "flag": "Company affiliation evaluating competitor products",
       "detail": "Authors from Microsoft Research Asia evaluate OpenAI and DeepSeek models but notably exclude Microsoft's own models (e.g., Phi, GitHub Copilot). This asymmetry is not discussed."
     },
     {
       "flag": "Missing results in main experiments",
       "detail": "Table 2 has missing entries (× marks for CodeLlama BM25 results, missing rows for some models). The paper attributes this to 'scarcity of API resources' but this means the experimental matrix is incomplete."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Primary comparison benchmark for repository-level code tasks; FEA-Bench extends this to feature implementation rather than bug fixing."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Agent framework evaluated on FEA-Bench; represents state-of-the-art approach for repository-level code tasks."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Agentic approach to repository-level code tasks; demonstrates how agent scaffolding affects LLM performance on SWE-bench."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Autonomous program improvement agent evaluated on SWE-bench; relevant to LLM-based repository-level code modification."
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "authors": ["Jia Li", "Ge Li", "Yunfei Zhao"],
       "year": 2024,
       "arxiv_id": "2405.19856",
       "relevance": "Repository-level code completion benchmark; provides context for evaluating LLMs on real-world code generation."
     },
     {
       "title": "EvoCodeBench: An Evolving Code Generation Benchmark Aligned with Real-World Code Repositories",
       "authors": ["Jia Li", "Ge Li", "Xuanming Zhang", "Yihong Dong", "Zhi Jin"],
       "year": 2024,
       "arxiv_id": "2404.00599",
       "relevance": "Evolving repository-level code benchmark addressing temporal contamination through continuous updates."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Code generation benchmark designed to address contamination through temporal filtering."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo", "Minh Chien Vu", "Jenny Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Large-scale code benchmark evaluating LLMs on complex coding scenarios with diverse function calls."
     },
     {
       "title": "CodePlan: Repository-Level Coding Using LLMs and Planning",
       "authors": ["Ramakrishna Bairi", "Atharv Sonwane", "Aditya Kanade"],
       "year": 2024,
       "relevance": "Combines LLMs with planning for repository-level coding tasks, relevant to multi-file code changes."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Best-performing model on FEA-Bench; demonstrates impact of reasoning-enhanced LLMs on repository-level code tasks."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Codex; foundational work for evaluating code generation capabilities of LLMs."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "authors": ["Fengji Zhang", "Bei Chen", "Yue Zhang"],
       "year": 2023,
       "relevance": "Iterative retrieval approach for repository-level code completion; directly relevant to context retrieval strategies evaluated in FEA-Bench."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark is directly useful for researchers evaluating code LLMs on feature implementation, though not immediately actionable for practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that LLMs struggle with complex repository-level tasks is expected; the specific quantification (~10% max) adds some informational value but is not surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns are raised by this benchmark paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild drama in showing that even the best models fail 90%+ of feature implementation tasks, but no controversy or adversarial framing."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Code and data are not yet publicly available — only a promise of future release at a GitHub URL."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "From Microsoft Research Asia and Peking University; evaluates well-known models (GPT-4, DeepSeek-R1) and positions against the prominent SWE-bench."
     }
   }
 }