ai-research-survey

scan-v5.json (23467B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
     "authors": [
       "Jia Li",
       "Ge Li",
       "Yunfei Zhao",
       "Yongming Li",
       "Huanyu Liu",
       "Hao Zhu",
       "Lecheng Wang",
       "Kaibo Liu",
       "Zheng Fang",
       "Lanshen Wang",
       "Jiazheng Ding",
       "Xuanming Zhang",
       "Yuqi Zhu",
       "Yihong Dong",
       "Zhi Jin",
       "Binhua Li",
       "Fei Huang",
       "Yongbin Li"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2405.19856",
     "doi": "10.48550/arXiv.2405.19856"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that existing benchmarks are poorly aligned with real-world repos are evidenced by Table 2 comparison of distributions across 10 benchmarks vs. 500 real-world repos. Claims about DevEval's features are supported by construction details in Section 3 and evaluation results in Table 5.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims contexts improve performance (205% improvement for gpt-4) and supports this with controlled comparison across three settings (no context, local file completion, local file infilling). The study design adequately isolates the effect of context availability.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Scope explicitly bounded in Limitations section: Python only, English requirements only, 8 specific LLMs tested, 117 repositories from 10 domains. The paper appropriately qualifies that results apply to this setting, though some statements could be more cautious.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple explanations offered for key findings: performance drops due to context length AND heterogeneity (Section 4.5); successful cases attributed to domain knowledge AND requirement clarity; dependency generation success explained by reasoning from requirements OR guessing from naming conventions.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Paper clearly distinguishes between measured outcomes (Pass@k for functional correctness, Recall@k for dependency accuracy) and claimed 'coding abilities.' These are specific, measurable proxies. Caveat: dependencies are dynamic in Python (Section 5), so some dependencies are missed by parser.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 is a dedicated Limitations section with three specific limitations: monolingual scope, Recall@k parser bias, local-file-only contexts. This is not boilerplate; each limitation is substantive.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats identified: (1) Monolingual—paper acknowledges Python/English-only limits practitioners' ability to generalize to other languages; (2) Recall@k bias quantified at 0.16 due to dynamic typing; (3) Data leakage risk acknowledged with reasoning about why impact is minimal (Section 5). Section 4.5 identifies specific LLM failure modes (context length, hallucinations).",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit boundaries: Python code, English requirements, 117 repositories from 10 specific domains, evaluation of 8 specified LLMs (Table 4). Monolingual limitation stated in Section 9. Boundaries are stated but not always emphasized in main text.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 Acknowledgments: 'This research was supported by the National Natural Science Foundation of China (Nos. 62192731, 62152730).' Funding source is clearly disclosed.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors list affiliations: Peking University (institutions 1, 2) or Alibaba Group (institution 3). Affiliations are clearly stated on title page with no hidden conflicts regarding evaluated models.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF China is government funding independent of benchmark outcomes. Paper evaluates multiple LLMs (gpt-4, DeepSeek, StarCoder, CodeLLaMa) without favoring Alibaba-affiliated products. Funder has no stake in which LLM performs best.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement provided. No disclosure of patents, equity holdings, or consulting relationships. Standard practice to include such a statement even if none exist.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Code generation: Section 2.2 defines as 'write code based on requirements and repository.' Repository-level code generation defined as 'simulates developers' coding process in a working repository.' Standalone/non-standalone clearly defined in Figure 1 with examples. Dependency types (intra-class, intra-file, cross-file) defined with examples.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Abstract and introduction clearly enumerate contributions: (1) four features for benchmarks, (2) DevEval benchmark itself, (3) repository-level code generation task, (4) evaluation of 8 LLMs with analysis. Contributions are explicit and numbered.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 discusses existing benchmarks (HumanEval, MBPP, APPS, etc.) and their shortcomings. Table 1 compares DevEval to 10 prior benchmarks on four dimensions. Section 6 covers related work comprehensively. Paper clearly positions DevEval as addressing gaps in prior benchmarks.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "Paper argues that benchmarks must align with real-world code distributions to validly measure coding ability. Evidence: analysis of 1M+ functions from 500 real-world repos (Section 3) showing distributions of standalone/non-standalone code and dependency types. However, argument is somewhat circular (real-world code has these properties, so we should measure on real-world code).",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "No characterization of task difficulty distribution. No easy/medium/hard tiers identified. Figure 5 shows performance differences by program type but not difficulty level. No difficulty metrics computed or reported.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "Table 5 shows Pass@1 ranges 12.7%-53.04% across models/settings, suggesting no extreme ceiling/floor effects. However, paper does not explicitly analyze or discuss whether ceiling/floor effects exist. The discussion of this potential issue is absent.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human performance reported on benchmark tasks. Paper mentions 13 developers annotated the data but does not report how well humans complete the code generation tasks themselves. This is a significant omission.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "Pass@k justified by reference to prior work and 'assess functional correctness by executing test cases.' Recall@k justified by 'expect LLMs to invoke relevant dependencies.' Justifications are present but relatively brief and could provide deeper reasoning for metric choices over alternatives.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "Paper acknowledges data leakage risk (Section 5) but does not design anti-gaming measures into the benchmark itself. No temporal splits, canary strings, or dynamic generation employed. Mitigation relies on future LLM developers excluding these repositories—a procedural, not technical, safeguard.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Paper does not discuss whether benchmark will become gamed, obsoleted, or outdated. Future work mentions updating with more projects/languages/tests, but no anti-gaming strategy or discussion of benchmark longevity is provided.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Paper discusses LLM failure modes well (Section 4.5): struggles with long contexts, generates hallucinations, poor at cross-file dependencies. Benchmark failure modes: Recall@k parser bias (0.16 quantified), Python dynamic typing evasion, monolingual limitation. However, discussion focuses more on LLM failures than benchmark limitations.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": false,
           "justification": "Paper states 'DevEval, prompts, and LLMs' predictions have been released' with GitHub link, but does not explicitly confirm baseline implementations for reproducing reported numbers are included. Reproducibility details are unclear from paper.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Source description: 1,874 samples from 117 repos in 10 domains from PyPI. Collection methodology: detailed 5-stage pipeline in Section 3 (repository selection, function parsing, test construction, human annotation, benchmark construction). Preprocessing steps clearly described. Characteristics well documented in Section 2.4 and Tables 2-3.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": true,
           "justification": "Access: GitHub link provided (https://github.com/seketeam/DevEval). Licensing: repositories are from PyPI and noted as open-source with licenses, though specific DevEval license not mentioned in paper. Access terms are clear; licensing could be more explicit.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Intended use: 'evaluate the coding abilities of Large Language Models (LLMs)' and 'practitioners can pick up superior LLMs and facilitate application of code generation in real-world repositories.' Limitations section provides guidance on what should not be concluded (e.g., results apply only to Python/English). More detailed use guidance could be provided.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Existing code generation benchmarks are poorly aligned with real-world code repositories in code distributions and dependency distributions.",
       "evidence": "Table 2 compares DevEval to 10 existing benchmarks and 500 real-world repositories. Previous benchmarks are 100% standalone code with no dependencies; 500 real repos are 27% standalone, 73% non-standalone with 3.22 dependencies per sample.",
       "supported": "strong"
     },
     {
       "claim": "DevEval's code and dependency distributions closely match those of 500 real-world repositories.",
       "evidence": "Table 2 shows DevEval: 27% standalone, 73% non-standalone, 3.41 dependencies per sample vs. 500 repos: 27% standalone, 73% non-standalone, 3.22 dependencies. Table 3 shows dependency type distributions closely aligned.",
       "supported": "strong"
     },
     {
       "claim": "gpt-4-turbo achieves only 53.04% Pass@1 on DevEval compared to ~80% on HumanEval, indicating existing benchmarks overestimate LLM coding ability.",
       "evidence": "Table 5 shows gpt-4 Pass@1=53.04% on DevEval (local file infilling). Paper states 'gpt-4-turbo-1106 achieves a Pass@1 score of 80% on HumanEval' in abstract.",
       "supported": "strong"
     },
     {
       "claim": "Adding local file contexts dramatically improves LLM code generation: gpt-4 Pass@1 improves 205% (no context 17.4% → local file infilling 53.04%).",
       "evidence": "Table 5 directly shows these values across three context settings for all 8 models, with consistent improvements across all models.",
       "supported": "strong"
     },
     {
       "claim": "LLMs struggle to understand long and heterogeneous code contexts, causing them to disregard knowledge in contexts and generate hallucinations.",
       "evidence": "Section 4.5 error case analysis (Figure 4): gpt-3.5 invokes non-existent function 'create_connection' despite available 'connect' in contexts. Paper attributes to context length (9× gpt-4 context window) and heterogeneous code from multiple files, citing Liu et al. (2023a) finding that LLMs 'ignore relevant information in the middle of long contexts.'",
       "supported": "moderate"
     },
     {
       "claim": "Cross-file dependencies are significantly harder for LLMs to generate than intra-file or intra-class dependencies.",
       "evidence": "Figure 6 shows gpt-4 Recall@1: intra-class 73%, intra-file 70%, cross-file 60% (local file infilling). Without context: intra-class 24%, intra-file 15%, cross-file 8%. Cross-file consistently lowest.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "DevEval is a manually-annotated benchmark of 1,874 code samples from 117 real-world repositories, designed to evaluate code generation in repository contexts where models have access to local code files. Evaluation of 8 LLMs reveals that coding ability in realistic settings is 45-53% Pass@1 (gpt-4) compared to 80%+ on isolated function benchmarks, with dramatic improvement when local file contexts are provided (205% improvement for gpt-4). The main bottleneck is LLM inability to understand long, heterogeneous code contexts—models generate hallucinations (non-existent dependencies) and struggle especially with cross-file dependencies. Empirical lesson: context and requirement clarity are crucial for realistic code generation evaluation.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "Paper doesn't report human performance on the benchmark tasks, making it impossible to judge if 53% Pass@1 is strong, weak, or reasonable. 13 developers annotated data but their own performance is not reported."
     },
     {
       "flag": "No task difficulty characterization",
       "detail": "Benchmark lacks easy/medium/hard task tiers or difficulty metrics. Unclear if all tasks are similarly challenging or if benchmark has ceiling/floor effects that would reduce discriminative power."
     },
     {
       "flag": "No built-in contamination resistance",
       "detail": "Paper acknowledges data leakage risk (repositories may be in training data) but designs no anti-gaming mechanisms (temporal splits, canary strings, dynamic generation). Mitigation relies on future LLM developers' voluntary exclusion—a procedural safeguard, not technical."
     },
     {
       "flag": "Limited context evaluation scope",
       "detail": "Paper evaluates only local file contexts, not realistic broader contexts (imports, sibling files, external libraries). Authors acknowledge this as Section 9 limitation but it reduces benchmark's real-world alignment claim."
     },
     {
       "flag": "Recall@k metric has known bias",
       "detail": "Python dynamic typing causes some dependencies to be identified only at runtime, eluding the static parser. Bias quantified at 0.16 Recall@1 but still a systematic underestimation of actual dependency recall."
     },
     {
       "flag": "Reproducibility details unclear",
       "detail": "Paper claims code is released on GitHub but doesn't explicitly confirm baseline implementations for reproducing reported Pass@k/Recall@k numbers are provided. Reproducibility from scratch is unclear."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "relevance": "HumanEval benchmark, foundational prior benchmark for code generation that DevEval compares against"
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "relevance": "MBPP benchmark, another key baseline benchmark used to contextualize DevEval's performance numbers"
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models",
       "relevance": "CoderEval benchmark with non-standalone functions; DevEval builds on and extends this approach with more comprehensive annotations"
     },
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-Level Code Generation",
       "relevance": "Class-level code generation benchmark; DevEval positions itself as extending to repository-level contexts"
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "relevance": "Supports DevEval's finding that LLMs struggle with long contexts; cited as evidence for why models fail on heterogeneous code"
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "relevance": "Repository-level code task for issue resolution; closely related to DevEval's repository-level code generation approach"
     },
     {
       "title": "CrossCodeEval: A Diverse and Multilingual Benchmark for Cross-File Code Completion",
       "relevance": "Repository-level code completion benchmark; related work comparing approaches to evaluate repository-aware code tasks"
     },
     {
       "title": "RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems",
       "relevance": "Repository-level code completion; related benchmark for evaluating contextual code generation"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses practitioners' need to evaluate LLMs for real-world code generation. Provides benchmark to select LLMs and understand their real-world coding ability—immediate practical value."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges narrative that LLMs are good at code (80% HumanEval → 53% DevEval), but this finding is somewhat expected given focus on realism. Moderately contrarian to optimistic benchmark narratives."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, alignment, or risk considerations. Purely a methodological benchmark paper with no safety angle."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Minor implicit conflict: existing benchmarks are misleading (HumanEval overstates ability). Not a major controversy or active conflict in the field."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Benchmark released on GitHub, others can evaluate their models. Not instantly demo-able (requires evaluating on 1,874 samples), but reproducible and usable."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Peking University and Alibaba are well-known institutions with strong CS reputation. Not top-tier (OpenAI, DeepMind, FAIR) but reputable."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "40281516",
         "title": "Kan: Kolmogorov-Arnold Networks",
         "points": 28,
         "comments": 4,
         "url": "https://news.ycombinator.com/item?id=40281516"
       },
       {
         "hn_id": "41522319",
         "title": "Show HN: Ask LLMs to predict anything based on news",
         "points": 25,
         "comments": 9,
         "url": "https://news.ycombinator.com/item?id=41522319"
       },
       {
         "hn_id": "44511458",
         "title": "Large Language Models as Autonomous Spacecraft Operators in Kerbal Space Program",
         "points": 6,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44511458"
       },
       {
         "hn_id": "40234345",
         "title": "Kan: Kolmogorov-Arnold Networks",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40234345"
       },
       {
         "hn_id": "40226580",
         "title": "Kan: Kolmogorov-Arnold Networks",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40226580"
       },
       {
         "hn_id": "40272252",
         "title": "Kan: Kolmogorov–Arnold Networks",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40272252"
       },
       {
         "hn_id": "40261190",
         "title": "Kan: Kolmogorov-Arnold Networks",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40261190"
       },
       {
         "hn_id": "39933799",
         "title": "Approaching Human-Level Forecasting with Language Models",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39933799"
       },
       {
         "hn_id": "40607770",
         "title": "Potential Field Based Deep Metric Learning",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40607770"
       },
       {
         "hn_id": "39948335",
         "title": "Towards a Brazilian History Knowledge Graph",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39948335"
       }
     ],
     "top_points": 28,
     "total_points": 78,
     "total_comments": 14
   }
 }