ai-research-survey

scan-v4.json (21935B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
     "authors": [
       "Jia Li",
       "Ge Li",
       "Yunfei Zhao",
       "Yongmin Li",
       "Huanyu Liu",
       "Hao Zhu",
       "Lecheng Wang",
       "Kaibo Liu",
       "Zheng Fang",
       "Lanshen Wang",
       "Jiazheng Ding",
       "Xuanming Zhang",
       "Yuqi Zhu",
       "Yihong Dong",
       "Zhi Jin",
       "Binhua Li",
       "Fei Huang",
       "Yongbin Li"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2405.19856",
     "doi": "10.48550/arXiv.2405.19856"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims gpt-4-turbo's highest Pass@1 is 53.04%, which matches Table 5 (Local File Infilling). Claims about benchmark alignment are supported by Tables 2-3.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that code context improves LLM performance. The three experimental settings (no context vs completion vs infilling) form a controlled comparison that adequately supports this. Other causal language is modest.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims alignment with 'Real-World Code Repositories' broadly, but the benchmark is Python-only and English-only. While the Limitations section (Section 9) acknowledges this is monolingual, the title and main framing overgeneralize beyond what was tested.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses two reasons LLMs can generate dependencies without context (reasoning from requirements, guessing from naming conventions). It also explains why gpt-family vs open-source models behave differently (instruction tuning). Error analysis considers context length and heterogeneity as alternative failure explanations.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Pass@k explicitly measures functional correctness via test execution, and Recall@k measures dependency recall. The paper is precise about what these metrics capture and introduces Recall@k specifically to address a gap not captured by Pass@k alone. Claims match measurement granularity.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Limitations' provides substantive discussion of three specific limitations: monolingual scope, Recall@k bias from static analysis, and limited context settings.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: monolingual limitation (Python/English only), Recall@k parser bias quantified at 0.16 from a 50-program sample comparison with human annotators, and the limitation of only using local file context.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it is Python-only, English-only, uses only local file contexts (not imported or sibling files), and that Recall@k has slight bias. Future work lists specific extensions planned (multilingual, more projects, more test cases).",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'This research was supported by the National Natural Science Foundation of China (Nos. 62192731, 62152730).'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Peking University (School of Computer Science, Key Laboratory) and Alibaba Group.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Natural Science Foundation of China is a government funding agency with no commercial stake in benchmark outcomes. While Alibaba authors are involved, Alibaba's own models are not being evaluated.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is included in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms defined precisely: 'standalone' vs 'non-standalone' (Figure 1 with examples), dependency types 'intra-class/intra-file/cross-file' (Figure 2, Table 3), 'repository-level code generation' (Section 2.2), Pass@k and Recall@k (equations in Section 2.3).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Explicit contributions stated: (1) four features a benchmark should have, (2) DevEval benchmark dataset (released), (3) repository-level code generation task, (4) evaluation of 8 LLMs with analysis. Contribution type (dataset + task + empirical study) is unambiguous.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 systematically reviews LLMs for code generation (Codex, ChatGPT, CodeLLaMa, DeepSeek) and code benchmarks (HumanEval, MBPP, APPS, ClassEval, CoderEval). Table 1 provides feature-by-feature comparison. Distinguishes from recent work (CrossCodeEval, RepoBench, RepoEval, SWE-bench) with detailed rationale.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "Argument: because DevEval uses real repository code with realistic distributions and dependencies (Table 2 alignment to 500 repos), it measures real-world coding ability better than hand-crafted benchmarks. The claim is: alignment with real repos = valid measurement of real-world skill.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Difficulty variation evident from results: gpt-4 Pass@1 ranges from 17.4% (no context) to 53% (with context), and Figure 5 shows performance varies by program type (standalone ~40-45% vs non-standalone ~53%). Table 2 reports avg requirement length (91.5 tokens) and repo scale variation. Implicitly characterized.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "Table 5 shows performance range for gpt-4: 17.4% (without context) to 60.65% (Pass@10 with context), avoiding extreme floors (<10%) or ceilings (>90%). Variation across models (12.54% to 53.04%) shows discrimination. No explicit ceiling/floor analysis but results demonstrate both.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "Benchmark uses human-written reference code and test cases, but no human developers were asked to generate code to solve the requirements. Human baseline performance is absent, limiting calibration validation.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Pass@k justified by citation to prior work (Chen et al. 2021, Austin et al. 2021). Recall@k introduced to measure dependency generation as separate dimension from functional correctness. Equation 2 explicitly specifies metric. Parser bias (0.16) is acknowledged and quantified in Section 5.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": true,
           "justification": "Section 5 addresses contamination risk. Mitigations: (1) manually-written requirements never seen during training, (2) empirical test showing 0.36 Pass@1 difference between seen/unseen repos (minimal), (3) future guidance to exclude repos from training. Moderate resistance—relies on new requirements rather than fundamental design.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Paper acknowledges that future LLMs may be trained on released repos and recommends exclusion in future work. Future work section states plan to update DevEval. Temporal concerns discussed but not deeply analyzed—no discussion of gaming strategies or benchmark decay rate.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.4 error analysis discusses LLM failure modes: struggling with long contexts (citing Liu et al. 2023a), heterogeneous context fragments (citing Shi et al. 2023), and hallucinating non-existent dependencies (Figure 4). Discussion of why LLMs fail on the benchmark is provided.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Paper states 'DevEval, prompts, and LLMs' predictions have been released' with GitHub link (https://github.com/seketeam/DevEval). Results in Table 5 cover 8 models, implying reproducible experimental setup is available.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Comprehensive documentation provided: 5-stage collection pipeline detailed in Section 3 and Figure 3, data statistics in Tables 2-3, composition breakdown (27% standalone/73% non-standalone), annotation process (13 developers, dual-check), test construction methodology. Preprocessing and quality criteria specified.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "Paper states benchmark 'has been released' but does not explicitly specify: (1) DevEval's license, (2) terms of use, (3) access restrictions. Repository selection mentions 'open-source licenses' but DevEval's own license is unstated.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Intended use stated in Section 2.4: 'DevEval... can serve as an arena to compare approaches ranging from retrieval and long-context models to decision-making agents.' Section 1: 'practitioners can pick up superior LLMs and facilitate the application of code generation techniques in real-world repositories.' Intent is evaluating code generation on repository-level tasks.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Existing code generation benchmarks are poorly aligned with real-world code repositories in code distribution and dependency patterns.",
       "evidence": "Table 2 and Table 3 show HumanEval, MBPP, and others have 100% standalone code with 0 dependencies, while 500 real repos have 27% standalone/73% non-standalone and 3M dependencies across all types.",
       "supported": "strong"
     },
     {
       "claim": "DevEval achieves alignment with real-world repositories across code distribution, dependency distribution, and repository scale.",
       "evidence": "Table 2 shows DevEval has 27% standalone (vs 27% in 500 repos), 73% non-standalone (vs 73%), 3.41 avg dependencies per sample (vs 3.22), and 243 avg files per repo (vs ~238).",
       "supported": "strong"
     },
     {
       "claim": "LLM performance on repository-level code generation is dramatically lower than on function-level benchmarks.",
       "evidence": "gpt-4-turbo achieves 80% Pass@1 on HumanEval but only 53.04% Pass@1 on DevEval without context (17.4%) and with local context (53.04%).",
       "supported": "strong"
     },
     {
       "claim": "Context from local files improves LLM code generation performance by 2-3x.",
       "evidence": "Table 5 shows gpt-4 improves from 17.4% (no context) to 53.04% (infilling context), a 205% relative improvement; gpt-3.5 goes from 13.98% to 44.50%, a 218% improvement.",
       "supported": "strong"
     },
     {
       "claim": "LLMs struggle to understand long and heterogeneous code contexts, leading to hallucinated dependencies and missed relevant code.",
       "evidence": "Section 4.4 error analysis: Figure 4 shows gpt-3.5 inventing non-existent 'create_connection' function despite available 'connect' function in context. Cross-file dependency Recall@1 is only 16.85% without context.",
       "supported": "strong"
     },
     {
       "claim": "Manually-written requirements help LLMs understand repository context and code generation objectives better than original code comments.",
       "evidence": "Section 4.4 successful case analysis attributes improvements to 'synergy of contexts and requirements.' DevEval requirement avg length 91.5 tokens vs CoderEval 41.5 tokens; Section 2.4 notes original comments are 'vague.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "DevEval is a benchmark of 1,874 code generation samples from 117 real-world Python repositories with manually-written requirements and comprehensive dependency annotations, aligned with real-code distributions (27% standalone, 73% non-standalone). Evaluation of 8 popular LLMs reveals repository-level code generation is substantially harder than function-level benchmarks (gpt-4 drops from 80% to 53% Pass@1), with context from local files improving performance by 2-3x, yet LLMs still fail to consistently generate correct dependencies and often hallucinate non-existent functions, attributed to difficulty understanding long and heterogeneous code contexts.",
   "red_flags": [
     {
       "flag": "Missing human baseline",
       "detail": "No human developers were evaluated on DevEval, preventing validation that the task is appropriately calibrated (non-trivial but not impossible for humans)."
     },
     {
       "flag": "No inter-annotator agreement metrics",
       "detail": "13 developers annotated requirements with dual-check process (mentioned in Section 4), but no Cohen's kappa or agreement percentages reported despite 674 person-hours of annotation effort."
     },
     {
       "flag": "Weak contamination analysis",
       "detail": "Risk that 117 repos appear in LLM training data is acknowledged. Mitigation evidence (0.36 Pass@1 difference between seen/unseen repos) is weak and within expected variance; no strong prevention measures like canary strings."
     },
     {
       "flag": "Parser bias in Recall@k metric",
       "detail": "Static analysis parser misses runtime-determined dependencies in Python. While bias is quantified at 0.16, this introduces systematic error that could favor LLMs that generate fewer dependencies."
     },
     {
       "flag": "Limited scope reduces generalizability",
       "detail": "Monolingual (English + Python only), local-file-only context, only 10 PyPI domains, only generation (not debugging/fixing). Findings may not generalize to other languages, multi-file contexts, or real development workflows."
     },
     {
       "flag": "Tautological construct validity",
       "detail": "Argument that 'real repository code = valid measurement of real-world skill' is somewhat circular. The benchmark measures real code but doesn't establish that real code inherently measures the skills practitioners need."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "relevance": "HumanEval benchmark, foundational function-level code generation evaluation."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "relevance": "MBPP benchmark; prior function-level code generation benchmark."
     },
     {
       "title": "Measuring Coding Challenge Competence with APPS",
       "relevance": "APPS benchmark for competition-style code problems; prior benchmark."
     },
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-Level Code Generation",
       "relevance": "Recent class-level code generation benchmark; direct comparison baseline."
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models",
       "relevance": "CoderEval with non-standalone code and dependencies; most similar prior work."
     },
     {
       "title": "CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion",
       "relevance": "Repository-level code completion benchmark; related task, different from generation."
     },
     {
       "title": "RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems",
       "relevance": "Repository-level completion benchmark; similar scope, different task."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "relevance": "Explains why LLMs struggle with middle of long contexts; cited in DevEval error analysis."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Repository-level software engineering benchmark; different task (bug fixing vs generation)."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Benchmark is publicly released and directly usable for evaluating code generation LLMs on realistic tasks."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Shows LLMs perform much worse on real-world code than HumanEval, which is somewhat expected but well-quantified."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns are raised."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly challenges HumanEval and similar benchmarks as insufficient, but framed constructively rather than controversially."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Benchmark, prompts, and model predictions released on GitHub for immediate use and reproduction."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Peking University and Alibaba Group are moderately well-known in the AI research community."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "40281516",
         "title": "Kan: Kolmogorov-Arnold Networks",
         "points": 28,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=40281516"
       },
       {
         "hn_id": "41522319",
         "title": "Show HN: Ask LLMs to predict anything based on news",
         "points": 25,
         "comments": 9,
         "url": "https://news.ycombinator.com/item?id=41522319"
       },
       {
         "hn_id": "44511458",
         "title": "Large Language Models as Autonomous Spacecraft Operators in Kerbal Space Program",
         "points": 6,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44511458"
       },
       {
         "hn_id": "40234345",
         "title": "Kan: Kolmogorov-Arnold Networks",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40234345"
       },
       {
         "hn_id": "40226580",
         "title": "Kan: Kolmogorov-Arnold Networks",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40226580"
       },
       {
         "hn_id": "40272252",
         "title": "Kan: Kolmogorov–Arnold Networks",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40272252"
       },
       {
         "hn_id": "40261190",
         "title": "Kan: Kolmogorov-Arnold Networks",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40261190"
       },
       {
         "hn_id": "39933799",
         "title": "Approaching Human-Level Forecasting with Language Models",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39933799"
       },
       {
         "hn_id": "40607770",
         "title": "Potential Field Based Deep Metric Learning",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40607770"
       },
       {
         "hn_id": "39948335",
         "title": "Towards a Brazilian History Knowledge Graph",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39948335"
       }
     ],
     "top_points": 28,
     "total_points": 78,
     "total_comments": 14
   }
 }