ai-research-survey

scan.json (28078B)

---

 {
   "paper": {
     "title": "NL2Repo-Bench: Towards Long-Horizon Repository Generation Evaluation of Coding Agents",
     "authors": ["Jingzhe Ding", "Shengda Long", "Changxin Pu", "Ge Zhang", "Huan Zhou"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.12730"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "NL2Repo-Bench evaluates coding agents on full repository generation from natural language, finding that even the best models (Claude-Sonnet-4.5) achieve below 40% test pass rates. The paper identifies systematic failure modes including premature termination, loss of global coherence, and fragile cross-file dependencies. Context window size correlates with performance, but is not sufficient alone. Task planning tool usage shows the strongest correlation with model performance (0.711).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Project page with GitHub link provided: https://github.com/multimodal-art-projection/NL2RepoBench. The paper states they 'release the NL2Repo benchmark, including the dataset, docker environments, and evaluation toolkit.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset of 104 tasks is released via the GitHub project page, including task specifications and Docker evaluation environments."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.3 describes Docker-based execution environments for each task with dependency version pinning. The evaluation toolkit includes containerized environments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper releases code and environments, the paper itself does not include step-by-step reproduction instructions for replicating the experimental results (e.g., exact commands to run each model evaluation)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 3, 4, and 8 report only point estimates (pass rates). No confidence intervals or error bars are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims performance differences between models (e.g., 'Claude outperforms other models') but provides no statistical significance tests. Comparisons are based solely on raw score differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance differences are reported with baseline context, e.g., 'pass rate increases from 40.2% to 59.4%' (Table 8), and percentage improvements across difficulty levels. The magnitude of differences is clear from the tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 104 tasks but no justification is provided for why 104 tasks is sufficient for the claims being made. No power analysis or sample size rationale is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports standard deviation of interaction turns across tasks (e.g., Claude-Sonnet-4.5: 181.6 ± 64.1). However, variance in pass rates across tasks or runs is not reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 12 model/framework combinations including open-source (DeepSeek, Qwen, GLM) and closed-source (Claude, GPT-5, Gemini) models as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated models are contemporary SOTA models from 2024-2025: Claude-Sonnet-4.5, GPT-5, DeepSeek-V3.2, Gemini-3-pro, Qwen3, Kimi-k2, GLM-4.6."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 presents two ablation studies: impact of interaction round limits (Fig. 9) and impact of revealing test cases (Table 8), isolating specific evaluation conditions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports overall pass rate, Pass@1 count, per-difficulty breakdown, per-category breakdown, interaction turns, tool usage statistics, and turns-per-score efficiency ratio."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The benchmark evaluates code generation via automated pytest suites. Human evaluation of generated code quality is not relevant — the correctness criterion is binary (tests pass or fail)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Test suites are strictly hidden from agents during development (Section 4.1: 'no test cases are revealed during development'). The ablation in Section 4.4.2 explicitly tests the impact of revealing tests."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides per-category breakdowns across 9 task categories. Table 3 provides per-difficulty breakdowns (Easy/Medium/Hard)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4.3.3 (early termination), 4.3.6 (workflow patterns including 'Navigation Trap' and 'Blind Editing'), and 4.3.7 (failure taxonomy including ImportError, dependency issues, test suite alignment) discuss failure modes in detail."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main finding is essentially negative: all models fail to reliably generate repositories (<40% pass rate). GPT-5's underperformance relative to expectations is specifically discussed. The 'thinking model paradox' (Qwen3-Thinking's 49% early termination) is another negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims agents achieve 'below 40% average test pass rates' — supported by Table 3 (best: 40.2%). Claims about failure modes (premature termination, loss of coherence) are supported by Sections 4.3.3 and 4.3.7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'context window capacity emerges as a crucial factor' (Section 4.3.5) and attributes Claude's advantage to its 1M token context, but this is confounded by other model differences (training data, architecture, RLHF). The ablation on test case visibility is a proper controlled experiment, but most causal claims are observational."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to evaluate 'long-horizon repository generation' broadly but tests only Python libraries (104 tasks). The title and abstract frame results as general statements about 'coding agents' without bounding to Python. Section 3.1.1 restricts to Python but claims in Section 5 generalize beyond this."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "When attributing Claude's performance to context window size, the paper does not adequately discuss alternative explanations (model quality, training data, RLHF alignment). The 'thinking model paradox' hypothesis is speculative without controlled evidence."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit about what it measures (pytest pass rates) and what it claims this indicates (repository generation capability). The distinction between functional correctness (measured) and broader software quality (not measured) is maintained throughout."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are specified by marketing names only: 'Claude-Sonnet-4.5', 'GPT-5', 'DeepSeek-V3.2', 'Gemini-3-pro'. No API version strings, snapshot dates, or model IDs are provided. Footnote 1 mentions Gemini-3-pro was used with Cursor-CLI but no version specifics."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions 'a single initialization instruction from the user' but does not provide the actual prompt text used. The exact instruction given to agents is not disclosed."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported — no temperature, top-p, max tokens, or other sampling settings for any of the models evaluated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The OpenHands CodeAct framework tools are described in Appendix B (8 tools listed with descriptions). Cursor-CLI and Claude Code are acknowledged as commercial agents. The paper describes available tools and their functions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the full benchmark construction pipeline: repository selection criteria (Section 3.1.1), document writing process (Section 3.1.2), environment building (Section 3.1.3), and verification/refinement (Section 3.1.4). Appendix C provides detailed annotator tutorials."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion briefly mentions 'limitations' in passing but does not substantively discuss them."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. Issues like the Python-only restriction, potential annotator bias in specification writing, or the representativeness of the 104 selected repositories are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to Python, to the specific frameworks tested, or acknowledge that repository generation from specifications differs from real-world development."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark release includes task documents, Docker environments, and evaluation toolkit. Generated workspaces could potentially be shared, though it's unclear if raw agent outputs/logs are released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.1 describes repository selection criteria (complexity, maturity, completeness, recency). Section 3.1.2 describes the reverse-engineering process for document creation. Appendix C provides annotator guidelines."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The benchmark evaluates AI models, not humans. Data sources are public GitHub repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from repository selection → document writing → environment building → verification is documented in Section 3.1, with filtering criteria at each stage. Figure 2 visualizes the construction pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: ByteDance Seed, M-A-P, 2077AI, Humanlaya Data, Nanjing University, Peking University, Beijing University of Posts and Telecommunications, Beihang University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Lead authors are from ByteDance Seed, which develops AI models. ByteDance has a vested interest in benchmarking outcomes. The paper evaluates ByteDance-affiliated models (Seed-Thinking is referenced) alongside competitors. No discussion of this potential conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. This is relevant since the benchmark uses existing open-source repositories that could appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark tasks are derived from existing GitHub repositories. No discussion of whether these repositories appeared in the training data of the evaluated models. The 'recency' criterion (3 years) is mentioned but not connected to model training cutoffs."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The source repositories are public GitHub projects that likely appeared in LLM training data. No contamination analysis is performed. While the task requires generating from NL specifications (not recalling code), the models may have memorized repository structures and implementations."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption, or wall-clock time are reported for running the benchmark despite evaluating 12 model/framework combinations across 104 tasks each."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report GPU hours, total API spend, or hardware used for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run (Pass@1 is explicitly reported). No seed sensitivity analysis or multiple-run results are provided for pass rates."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state whether results are from single runs or averaged. Pass@1 counts suggest single runs, but this is not clearly stated."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing models via APIs without hyperparameter tuning, so no search budget applies."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection was performed — models were evaluated with their default settings."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is moot."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ByteDance authors evaluate models from multiple companies but do not acknowledge potential bias in benchmark design that could favor certain model capabilities. The Seed-Thinking model (from ByteDance) is referenced in the paper, and the benchmark construction process could embed implicit preferences."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 plots total tool invocations vs. task pass rate, showing efficiency trade-offs. Table 6 reports turns/score ratios. Section 4.3.5 discusses context window size vs. performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4.2 ablates the benchmark by revealing test cases, testing whether the bottleneck is requirement inference vs. implementation. The paper discusses what the benchmark measures (long-horizon generation) vs. what existing benchmarks measure. Section 4.3.5 analyzes whether context size or model capability drives results."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 explicitly evaluates Claude-Sonnet-4.5 across three frameworks (OpenHands, Claude Code, Cursor) showing <1% variation, directly addressing the scaffold confound. The paper concludes the benchmark is 'model-centric' rather than scaffold-dependent."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses existing GitHub repositories. The 'recency' criterion requires repos updated within 3 years, but no analysis connects this to model training cutoffs. Models may have seen source code of benchmark repositories during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates development (NL specification only) from evaluation (hidden test suite). Section 4.4.2 ablates this by revealing tests, showing the design prevents feature leakage from test suites to agents."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 104 source repositories share structural similarities, use similar libraries, or whether training data of evaluated models includes code from these repositories."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. No canary strings, membership inference, or decontamination analysis is performed despite the benchmark being derived from public GitHub repositories."
       }
     }
   },
   "claims": [
     {
       "claim": "Even the strongest coding agents achieve below 40% average test pass rates on NL2Repo-Bench",
       "evidence": "Table 3: Claude-Sonnet-4.5 (Claude Code) achieves 40.2%, all other models below 40%. Best Pass@1 is 5 repositories out of 104.",
       "supported": "strong"
     },
     {
       "claim": "Context window size is a crucial factor in NL2Repo-Bench performance, with 1M-context models substantially outperforming others",
       "evidence": "Section 4.3.5 and Figure 8: 1M context models (Claude) average 38.5% vs 256K models averaging 21.7%. However, Kimi-k2 with long context achieves only 22.7%.",
       "supported": "moderate"
     },
     {
       "claim": "The benchmark is model-centric rather than framework-dependent, as the same model shows <1% variation across frameworks",
       "evidence": "Table 3: Claude-Sonnet-4.5 scores 40.2% (Claude Code), 39.9% (OpenHands), 39.2% (Cursor).",
       "supported": "moderate"
     },
     {
       "claim": "Task planning (task_tracker tool) has the strongest correlation with model performance",
       "evidence": "Table 5: task_tracker correlation = 0.711, with only 1 model not using it. The 'think' tool has higher correlation (0.816) but 4 models don't use it.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-5 is aligned more towards human-in-the-loop collaboration than autonomous execution, with 84.5% non-finish rate",
       "evidence": "Figure 5 and Section 4.3.3: GPT-5 has 84.5% non-finish rate and only 78.4 average turns (lowest among all models). Case analysis shows it halts for user confirmation.",
       "supported": "strong"
     },
     {
       "claim": "Even with all test cases revealed ('cheating' scenario), the best model achieves only 59.4% pass rate",
       "evidence": "Table 8: Claude-Sonnet-4.5 (Claude Code) improves from 40.2% to 59.4% when test cases are provided, with Pass@1 jumping from 3 to 18.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "ByteDance conflict of interest",
       "detail": "Lead authors are from ByteDance Seed. ByteDance develops AI models including Seed-Thinking (referenced in the paper). While the benchmark evaluates mainly competitor models, the benchmark design could embed implicit biases. No conflict of interest statement is provided."
     },
     {
       "flag": "No contamination analysis despite using public repositories",
       "detail": "The 104 benchmark tasks are derived from public GitHub repositories that likely appeared in the training data of evaluated models. The models may have memorized implementations, inflating or deflating scores unpredictably. No decontamination analysis is performed."
     },
     {
       "flag": "Single-run evaluation",
       "detail": "Results appear to be from single runs with no variance or confidence intervals reported for pass rates. LLM outputs are stochastic, and single-run results may not be representative."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations section despite making broad claims about 'autonomous software engineering capability.' Python-only restriction, potential annotation bias, and contamination risks are not acknowledged."
     },
     {
       "flag": "Causal claims from observational comparisons",
       "detail": "Claims about context window size being a 'crucial factor' are drawn from comparing different models that differ in many ways beyond context size. The observed correlation does not establish causation."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Foundational repair-focused benchmark for coding agents that NL2Repo-Bench positions against."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "relevance": "Primary agent framework used for evaluation in NL2Repo-Bench experiments."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark — foundational function-level code generation evaluation that NL2Repo-Bench argues is insufficient."
     },
     {
       "title": "PaperBench: Evaluating AI's ability to replicate AI research",
       "authors": ["Giulio Starace", "Oliver Jaffe"],
       "year": 2025,
       "relevance": "Repository-level benchmark using LLM-based evaluation that NL2Repo-Bench contrasts with its test-based evaluation."
     },
     {
       "title": "Commit0: Library generation from scratch",
       "authors": ["Wenting Zhao", "Nan Jiang"],
       "year": 2024,
       "relevance": "Most directly comparable benchmark — also targets from-scratch library generation but provides scaffolding/signatures as priors."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez"],
       "year": 2024,
       "relevance": "Introduces Agent-Computer Interface abstraction for repository-level coding agent tasks."
     },
     {
       "title": "RepoBench: Benchmarking repository-level code auto-completion systems",
       "authors": ["Tianyang Liu", "Canwen Xu"],
       "year": 2023,
       "arxiv_id": "2306.03091",
       "relevance": "Repository-level code completion benchmark that NL2Repo-Bench positions as addressing only partial completion."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Class-level code generation benchmark bridging function-level and repository-level evaluation."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Key open-source code LLM with long-context support relevant to repository-level generation."
     },
     {
       "title": "DeepSeek-V3.2: Pushing the frontier of open large language models",
       "authors": ["Aixin Liu"],
       "year": 2025,
       "arxiv_id": "2512.02556",
       "relevance": "One of the evaluated open-source models in NL2Repo-Bench experiments."
     },
     {
       "title": "AutoCodeBench: Large language models are automatic code benchmark generators",
       "authors": ["Jason Chou", "Ao Liu"],
       "year": 2025,
       "arxiv_id": "2508.09101",
       "relevance": "Automated benchmark generation for code LLMs, contrasted with NL2Repo-Bench's human-curated approach."
     },
     {
       "title": "SWE-bench Pro: Can AI agents solve long-horizon software engineering tasks?",
       "authors": ["Xiang Deng", "Jeff Da"],
       "year": 2025,
       "arxiv_id": "2509.16941",
       "relevance": "Extended SWE-bench targeting long-horizon tasks, directly comparable in scope to NL2Repo-Bench."
     }
   ]
 }