ai-research-survey

scan-v4.json (23042B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "DOMAINEVAL: An Auto-Constructed Benchmark for Multi-Domain Code Generation",
     "authors": [
       "Qiming Zhu",
       "Jialun Cao",
       "Yaojie Lu",
       "Hongyu Lin",
       "Xianpei Han",
       "Le Sun",
       "Shing-Chi Cheung"
     ],
     "year": 2024,
     "venue": "AAAI Conference on Artificial Intelligence",
     "arxiv_id": "2408.13204",
     "doi": "10.48550/arXiv.2408.13204"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about domain performance gaps (82.44% computation vs 33.08% cryptography), the 68.94% gap in Llama-2-13b-chat, and increasing bias with more samples are all directly supported by Table 1.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'fine-tuning can bring about overall improvement, while the domain gaps still exist' based on comparing Llama-2-13b-chat vs CodeLlama-13b-Instruct. This is a causal claim, but confounds (different instruction tuning, different training data beyond code) are not addressed.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title claims 'Multi-Domain Code Generation' broadly, but all experiments are Python-only. The paper does not caveat that results may not generalize to other programming languages.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for domain performance differences. For example, whether differences stem from training data distribution, domain-specific complexity, or test difficulty is not explored.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@k (functional correctness via test execution) and frames results in terms of 'code generation capability.' Pass@k directly measures functional correctness, and the claims match the granularity of measurements.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no limitations, threats to validity, or similar section. The conclusion mentions 'future research directions' but does not discuss limitations of the current work.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no analysis of potential biases in the benchmark construction, domain classification, or evaluation methodology.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state what the results do NOT show. It does not acknowledge that results are limited to Python, to function-level code generation, or to the specific repositories selected.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or sponsorships are mentioned anywhere in the paper.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Chinese Academy of Sciences (Institute of Software) and Hong Kong University of Science and Technology. None of the authors are affiliated with the companies whose models are evaluated.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be assessed.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms ('domain-specific code generation', 'Pass@k', 'macro-average') are either defined in context, exemplified (Figure 1), or referenced as standard prior work (Chen et al. 2021). Sufficient precision for readers.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Abstract explicitly lists three contributions: (1) benchmark dataset with six domains, (2) automated pipeline, (3) findings on LLM limitations. Each is restated in conclusion.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Related Work (2 pages) systematically compares DOMAINEVAL against HumanEval/MBPP (common vs domain tasks), CoderEval/ClassEval (automation level), and Zhuo et al. 2024 (API usage vs implementation). Shows clear positioning.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": false,
           "justification": "Paper assumes functional correctness on computation/cryptography/system tasks measures LLM capability in those domains, but never argues *why* (e.g., 'implementing RSA measures cryptographic understanding because X'). Validity is implicit, not justified.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Figure 5 shows line-count distribution (4-198 lines, avg 55.69). Pass@1 results reveal difficulty gradient (computation 82.44% vs cryptography 33.08%). Constraints (3-100 lines) are stated. Not explicit tiers, but distribution is characterized.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "Table 1 reveals ceiling effects (computation >75% for most models) and moderate floors (cryptography 33.08%). Paper reports results but does not discuss ceiling/floor as a limitation or validity threat.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation, no human performance data, no validation that benchmark difficulty is calibrated appropriately. Cannot compare LLM performance to human baseline.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "Pass@k borrowed from Chen et al. 2021 without domain-specific justification. Handling missing imports via automated completion is justified pragmatically ('tolerable flaw') but not principled. No discussion of why Pass@k is optimal for domain evaluation.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "Abstract claims pipeline 'fortifies DOMAINEVAL against data contamination threat' but provides no mechanism: no temporal splits, canaries, or versioning strategy described. Claim is aspirational, not engineered.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Paper states pipeline enables 'exceptional scalability, capable of incorporating the ever-evolving code corpus.' Plans for continuous updates mentioned. Lacks detail on managing obsolescence or model overfitting to benchmark.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Case studies (Figs 7-8) show LLM failure modes, not benchmark failure modes. No discussion of what the benchmark cannot measure (e.g., code maintainability, security, real-world complexity).",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "Leaderboard available at domaineval.github.io; evaluation procedure detailed in Section 'Experiment Setup' and 'Evaluation Process'; prompt template provided (Figure 6); sufficient for reproduction. Code availability not explicitly stated but procedural clarity is high.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "Collection methodology detailed (Domain Repository Collection, Test-Method Matching, Instruction Generation). Source description (GitHub >100 stars), preprocessing steps (filtering criteria), and dataset statistics (2454 subjects, 5892 tests) all provided.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "Leaderboard URL given but licensing of benchmark data, commercial use rights, and derivative work permissions are not specified. Unclear if code/test data are available under open license.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": true,
           "justification": "Paper states benchmark 'designed to evaluate LLMs' coding capabilities thoroughly' and can be used for 'custom domain benchmarks.' Caveats (e.g., Pass@k doesn't measure maintainability) not discussed.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs are generally good at computation tasks while falling short on cryptography and system coding tasks",
       "evidence": "Table 1: Computation 82.44% Pass@1, Cryptography 33.08%, System 37.50% (macro-averages across 12 models)",
       "supported": "strong"
     },
     {
       "claim": "Performance gap between domains can exceed 68%",
       "evidence": "Table 1: Llama-2-13b-chat shows 80.94% (Computation) - 12.0% (Cryptography) = 68.94% gap",
       "supported": "strong"
     },
     {
       "claim": "Generating more samples increases overall performance",
       "evidence": "Table 1: Pass@1 average 53.42% vs Pass@5 average 59.60% (6.18pp improvement across all models and domains)",
       "supported": "strong"
     },
     {
       "claim": "Generating more samples increases domain bias",
       "evidence": "Section 'Impact of Generated Samples': Standard deviation (bias measure) decreases slightly on average (18.33→17.72) but CodeLlama-13b shows increase (19.90→20.55), indicating bias may increase for certain models",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o-mini exhibits the most stable performance across domains",
       "evidence": "Table 1: GPT-4o-mini has lowest standard deviation in Pass@5 (14.75) compared to 15.45-24.10 for other models",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning improves overall performance but domain gaps persist",
       "evidence": "Section 'LLMs Biases': CodeLlama-13b (fine-tuned from Llama-2-13b) achieves 11.25% improvement overall but domain gaps remain unresolved",
       "supported": "strong"
     },
     {
       "claim": "DOMAINEVAL provides an automated, scalable pipeline for benchmark construction",
       "evidence": "Section 'Benchmark Construction': Describes fully automated three-step pipeline (repository collection, test-method matching, instruction generation) applied to construct 2454 subjects",
       "supported": "strong"
     },
     {
       "claim": "Code from different domains requires different types of knowledge and skills",
       "evidence": "Figure 1 and case studies show computation involves mathematical operations, cryptography requires algorithm knowledge (RSA attacks), system tasks require OS understanding; all show different error patterns in Figure 7-8",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "DOMAINEVAL reveals pronounced domain biases in LLM code generation: computation tasks average 82.44% Pass@1 while cryptography and system domains average 33.08% and 37.50% respectively, with individual models showing gaps exceeding 68%. GPT-4o-mini and Qwen2-72B-Instruct lead in overall performance with 67.13% and 64.25% Pass@5. Increasing sampling from 1 to 5 uniformly improves performance (+6.18pp average) but paradoxically increases domain bias in some models (CodeLlama-13b), suggesting models amplify their weaknesses rather than fixing them with more attempts.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "Benchmark difficulty not validated against human performance. Unknown if 82% Pass@1 on computation is 'easy' (should be >95%) or 'appropriately hard' (should be 60-80%)."
     },
     {
       "flag": "No limitations section",
       "detail": "Paper omits systematic discussion of threats to validity, benchmark design limitations, or scope constraints beyond stating basic boundaries."
     },
     {
       "flag": "Instructions generated by LLM",
       "detail": "Uses Qwen2-72B to generate task descriptions rather than human-written instructions. Quality variation and consistency not discussed; could introduce artifact-specific biases."
     },
     {
       "flag": "GitHub selection bias",
       "detail": "Repositories selected for >100 stars may skew toward popular, well-maintained code. Small/niche domain projects underrepresented."
     },
     {
       "flag": "Contamination resistance claimed without mechanism",
       "detail": "Abstract claims automated pipeline 'fortifies DOMAINEVAL against data contamination threat' but provides no concrete mechanism (temporal splits, canary strings, versioning) beyond vague 'continuous updates'."
     },
     {
       "flag": "Ceiling effects in computation domain unaddressed",
       "detail": "Computation domain scores >75% for nearly all models, suggesting task difficulty may be too low to discriminate performance. Not flagged as a limitation."
     },
     {
       "flag": "Missing imports corrected automatically",
       "detail": "Paper adds missing import statements during evaluation to prevent 'tolerable flaw.' Raises fairness questions: are models being graded on import knowledge or just logic? Correction not standard in prior benchmarks."
     },
     {
       "flag": "Licensing and access unclear",
       "detail": "Paper mentions leaderboard availability but does not specify whether benchmark data (2454 subjects, 5892 tests) is released, under what license, or with what usage restrictions."
     },
     {
       "flag": "Construct validity assumed not argued",
       "detail": "Paper assumes functional correctness on domain-specific code reveals 'domain capability' but doesn't justify why (e.g., why RSA attack code measures cryptographic understanding)."
     },
     {
       "flag": "Alternative explanations not explored",
       "detail": "Why is computation intrinsically easier? Is it because training data emphasizes math? Because cryptography is rarer? Root causes of domain bias asserted but not analyzed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": "Chen et al.",
       "year": 2021,
       "venue": "NeurIPS",
       "relevance": "Foundational HumanEval benchmark; establishes Pass@k metric and function-level code generation evaluation paradigm that DOMAINEVAL extends to multi-domain setting"
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": "Austin et al.",
       "year": 2021,
       "venue": "NeurIPS",
       "relevance": "MBPP benchmark for programming tasks; one of the primary 'common task' baselines that DOMAINEVAL contrasts against by adding domain diversity"
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": "Hendrycks et al.",
       "year": 2021,
       "venue": "NeurIPS",
       "relevance": "APPS algorithm competition dataset; related benchmark emphasizing complexity but not domain specialization that DOMAINEVAL addresses"
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models",
       "authors": "Yu et al.",
       "year": 2024,
       "venue": "ICML",
       "relevance": "Recent pragmatic code benchmark using GitHub; DOMAINEVAL explicitly compares against CoderEval's approach to real-world code sourcing"
     },
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation",
       "authors": "Du et al.",
       "year": 2023,
       "venue": "ICLR",
       "relevance": "Class-level code generation benchmark; demonstrates escalation from function to class granularity, which DOMAINEVAL complements with domain orthogonalization"
     },
     {
       "title": "Concerned with Data Contamination? Assessing Countermeasures in Code Language Model",
       "authors": "Cao et al.",
       "year": 2024,
       "relevance": "Data contamination threat in code LMs; directly cited in DOMAINEVAL as motivation for automated pipeline's claimed contamination resistance"
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": "Guo et al. et al.",
       "year": 2024,
       "relevance": "Code-specific LLM series; DeepSeek-Coder included in DOMAINEVAL evaluation alongside GPT and Llama models"
     },
     {
       "title": "CERT: Continual Pre-Training on Sketches for Library-Oriented Code Generation",
       "authors": "Zan et al.",
       "year": 2022,
       "relevance": "Library-oriented domain-specific code generation; prior work on domain-tailored evaluation that DOMAINEVAL systematically extends"
     },
     {
       "title": "MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation",
       "authors": "Cassano et al.",
       "year": 2022,
       "relevance": "Multi-language code benchmark via translation; DOMAINEVAL contrasts its multi-domain approach against MultiPL-E's multi-language orthogonalization"
     },
     {
       "title": "CodeBenchGen: Creating Scalable Execution-based Code Generation Benchmarks",
       "authors": "Xie et al.",
       "year": 2024,
       "relevance": "Scalable automated benchmark construction using LLM; directly relevant precedent for DOMAINEVAL's automated pipeline approach"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark helps practitioners understand LLM strengths/weaknesses across programming domains, useful for tool selection decisions."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that LLMs struggle with cryptography and system code is somewhat expected given training data distributions, though the magnitude of the gap (68.94%) is notable."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised by the findings."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with prior claims; the paper positions itself as complementary to existing work."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A leaderboard website exists at https://domaineval.github.io/ but no interactive demo or pip-installable tool is provided."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT-4o-mini, DeepSeek-Coder) but authors are from academic institutions, not major AI labs."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "39831754",
         "title": "GPT-4V(ision) Unsuitable for Clinical Care and Education: An Evaluation",
         "points": 75,
         "comments": 52,
         "url": "https://news.ycombinator.com/item?id=39831754"
       },
       {
         "hn_id": "41663273",
         "title": "Unsafe Impedance: Safe Languages and Safe by Design Software",
         "points": 7,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=41663273"
       },
       {
         "hn_id": "40135927",
         "title": "OpenAI: Training LLMs to Prioritize Privileged Instructions",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40135927"
       },
       {
         "hn_id": "41418082",
         "title": "Data Exposure from LLM Apps: An In-Depth Investigation of OpenAI's GPTs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41418082"
       },
       {
         "hn_id": "41408373",
         "title": "Data Exposure from LLM Apps: An In-Depth Investigation of OpenAI's GPTs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41408373"
       },
       {
         "hn_id": "39139543",
         "title": "Exploring Parent's Needs for Children-Centered AI to Support Preschoolers",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=39139543"
       },
       {
         "hn_id": "37345839",
         "title": "Relighting Neural Radiance Fields with Shadow and Highlight Hints",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37345839"
       },
       {
         "hn_id": "41227450",
         "title": "Τ-Bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41227450"
       },
       {
         "hn_id": "40965488",
         "title": "The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40965488"
       },
       {
         "hn_id": "40157957",
         "title": "The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40157957"
       }
     ],
     "top_points": 75,
     "total_points": 96,
     "total_comments": 54
   }
 }