ai-research-survey

scan-v5.json (23173B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "DOMAINEVAL: An Auto-Constructed Benchmark for Multi-Domain Code Generation",
     "authors": [
       "Qiming Zhu",
       "Jialun Cao",
       "Yaojie Lu",
       "Hongyu Lin",
       "Xianpei Han",
       "Le Sun",
       "Shing-Chi Cheung"
     ],
     "year": 2024,
     "venue": "AAAI Conference on Artificial Intelligence",
     "arxiv_id": "2408.13204",
     "doi": "10.48550/arXiv.2408.13204"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are directly supported by experimental results in Table 1. The 68.94% gap (Llama-2-13b), domain distribution, and Pass@1→Pass@5 dynamics are explicitly reported.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The claim 'generating more samples can increase overall performance' conflates correlation with causation. Pass@1 (greedy decoding) vs Pass@5 (sampling with temperature 0.2) are different metrics, so improvement may stem from sampling strategy, not sample count alone. No controlled ablation isolates sample quantity as the causal variable.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Claims generalize from 12 tested models to 'LLMs' broadly (e.g., 'LLMs are generally good at computation'). No discussion of whether findings apply to other model families, instruction-tuning approaches, or code lengths outside the restricted 3-100 line range.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Two case studies explain specific failures (RSA square root overflow, Linux locale-sensitive command parsing) but no systematic discussion of why cryptography/system domains are hard. Paper attributes failures to 'lack of domain knowledge' without ruling out training data distribution, context length, or task complexity confounds.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Pass@k (functional correctness via test suite execution) is a standard, well-justified proxy for code generation capability. The paper does not claim broader outcomes like 'production-readiness' or 'real-world utility.'",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion lists future work directions but not actual limitations of the benchmark, evaluation setup, or generalizability constraints.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats are discussed. The paper describes security mitigations (banned keywords) but not threats like LLM instruction-generation bias (Qwen2-72B used), test suite comprehensiveness, domain classification reliability, or selection bias in GitHub repos.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Scope boundaries are implicit (Python only, 3-100 lines, 6 specific domains, 12 instruction-tuned models, GitHub repos with ≥100 stars) but never formally stated. No explicit statement of what the benchmark does NOT measure or which model/language families are out-of-scope.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The paper neither states a funder nor declares itself unfunded independent work.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed (Chinese Academy of Sciences, Hong Kong University of Science and Technology). No apparent conflicts with evaluated LLM companies.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder disclosed; independence cannot be assessed. Assumes unfunded if not stated.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided. The leaderboard platform (domaineval.github.io) is mentioned with no disclosure of financial stakes.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "Key terms like 'domain-specific code' are demonstrated via examples (Figure 1) and implicitly defined by the six domains, but not formally defined. 'Pass@k' is cited but not defined in the paper. 'Fully automated pipeline' is explained procedurally but not formally defined.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly listed in the abstract: (1) the DOMAINEVAL dataset, (2) the fully automated test-guided construction pipeline, and (3) identification of LLM limitations in domain-specific code generation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The related work section engages substantively with 10+ prior benchmarks (HumanEval, MBPP, APPS, CoderEval, etc.), explicitly contrasting DOMAINEVAL's design (fully automated, multi-domain, real code from GitHub) vs. prior approaches (manual curation, single-domain, synthetic tasks, API-centric).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": false,
           "justification": "The paper selects six domains referencing prior work (Zhuo et al. 2024) but does not argue why these domains collectively measure domain-specific code generation capability. Design goal (align with 'real-world code requirements') is stated, not construct validity justification.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "Figure 5 shows line count distribution by domain (4-198 lines, avg 55.69) and the paper restricts to 3-100 lines as 'appropriate difficulty,' but no explicit difficulty tiers (easy/medium/hard) are defined, measured, or validated via item-response analysis.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "Table 1 shows ceiling effects (computation: 82-91% for top models) and floor effects (Llama-2-13b at 12% on cryptography), but these are observed, not discussed. No analysis of whether ceiling/floor limits benchmark discriminability or benchmark redesign is warranted.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is provided. The paper reports only LLM performance without human reference data, making it impossible to assess whether 82% in computation or 33% in cryptography represents good or poor model capability.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": false,
           "justification": "Pass@k is cited as standard (Chen et al. 2021) but not justified relative to alternatives (e.g., code similarity, syntax, runtime safety metrics). Import auto-completion in evaluation is a scoring decision ('tolerable flaw') that is not debated—missing imports may be a legitimate failure mode.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "The paper claims continuous updates via the automated pipeline resist data contamination (citing Cao et al. 2024b) but provides no explicit contamination-resistance design (e.g., temporal splits, canary strings, dynamic generation). GitHub repos are selected by star count with no date cutoff, risking overlap with pre-2024 training data.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of benchmark longevity, versioning strategy, gaming risk ('will LLMs memorize this in 6 months?'), or maintenance plan. The pipeline's scalability for future updates is mentioned, but not temporal robustness strategy.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Two case studies (Figures 7-8) explain LLM failures, not benchmark failure modes. The paper does not discuss what the benchmark does NOT measure, biases in domain selection, gaps in test coverage, or how the benchmark could be misused.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": false,
           "justification": "The leaderboard URL is provided, but the paper does not explicitly state whether code for the benchmark construction pipeline, evaluation harness, or baseline models is published or reproducible.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "Source repositories are listed by domain (Figure 2) but not version-pinned or linked. Collection methodology (test-method matching) is explained procedurally but lacks data quality metrics (e.g., how many candidates filtered at each step?). No formal data card provided.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The leaderboard (domaineval.github.io) is referenced but no license (MIT, Apache, CC-BY, etc.) is specified. Access terms are not stated—is the benchmark downloadable, or leaderboard-only evaluation?",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "General intended use is clear ('evaluate LLMs' domain-specific coding capabilities'), but specific constraints are not formally stated. No guidance on what conclusions should NOT be drawn (e.g., 'not for evaluating system design' or 'not applicable to non-instruction-tuned models').",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs are generally good at computation tasks (average 82.44% Pass@1) while falling short on cryptography (33.08%) and system coding (37.50%) domains.",
       "evidence": "Table 1, Pass@1 column: computation domain macro-average 82.44%, cryptography 33.08%, system 37.50% across 12 models.",
       "supported": "strong"
     },
     {
       "claim": "The performance gap between domains can be as much as 68.94% (80.94% - 12.0%) in Llama-2-13b-chat.",
       "evidence": "Table 1, Pass@1: Llama-2-13b computation 80.94%, cryptography 12.0%.",
       "supported": "strong"
     },
     {
       "claim": "Generating more samples increases overall LLM performance (Pass@1 53.42% → Pass@5 59.60%).",
       "evidence": "Table 1, average rows: Pass@1 macro-average 53.42%, Pass@5 macro-average 59.60%.",
       "supported": "strong"
     },
     {
       "claim": "Domain bias may increase with more samples; CodeLlama-13b-instruct standard deviation increases from 19.90 to 20.55.",
       "evidence": "Table 1, Std column: CodeLlama-13b Pass@1 Std 19.90, Pass@5 Std 20.55.",
       "supported": "moderate"
     },
     {
       "claim": "The fully automated pipeline provides contamination resistance by enabling continuous benchmark updates.",
       "evidence": "Abstract and introduction claim updates via pipeline maintain 'integrity and novelty' citing Cao et al. 2024b; no empirical demonstration provided.",
       "supported": "weak"
     },
     {
       "claim": "CodeLlama fine-tuning achieves 11.25% average improvement over Llama-2-13b (57.74% - 46.49% Pass@5) but domain gaps persist.",
       "evidence": "Table 1, Pass@5: CodeLlama-13b 57.74%, Llama-2-13b 46.49%; Std values still substantial (20.55 vs 24.10).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "DOMAINEVAL—a 2,454-problem multi-domain code generation benchmark across six Python domains—reveals stark LLM performance disparities: computation achieves 82.44% Pass@1 (ceiling effects at 90%+), while cryptography and system domains score 33.08% and 37.50% respectively, with gaps exceeding 68% in some models. Increasing sample count (Pass@1 to Pass@5) improves aggregate performance by ~6%, but paradoxically increases domain bias variability in some models (CodeLlama std increases from 19.90 to 20.55). Two case studies highlight specific failure modes—LLMs mishandle large-integer square roots in RSA attack tasks and fail to account for locale-dependent Linux command output—suggesting domain-specific knowledge gaps rather than general capability limitations.",
   "red_flags": [
     {
       "flag": "No human baseline",
       "detail": "Without human performance reference, it is unclear whether 82% on computation or 33% on cryptography reflects LLM capability or artifacts of benchmark construction (e.g., instruction phrasing, test case design)."
     },
     {
       "flag": "Ceiling effects in computation domain",
       "detail": "Computation tasks show ceiling effects (82-91% for top models), limiting discriminative power for ranking models in this domain. Benchmark redesign may be needed."
     },
     {
       "flag": "Contamination resistance unvalidated",
       "detail": "Claims about resisting data contamination via continuous updates are speculative. GitHub repos with ≥100 stars lack date stamps, risking overlap with pre-2024 training data. No temporal validation provided."
     },
     {
       "flag": "No formal limitations section",
       "detail": "Absence of explicit scope boundaries, threats to validity, or generalizability caveats. Readers cannot assess applicability to non-instruction-tuned models, other languages, or different code lengths."
     },
     {
       "flag": "LLM-generated instructions introduce bias",
       "detail": "Instructions are generated by Qwen2-72B-Instruct, which may systematically bias task phrasing toward certain model families, favoring the Qwen and related model series."
     },
     {
       "flag": "Import auto-completion in evaluation",
       "detail": "Missing imports are auto-completed during evaluation ('tolerable flaw'), changing the true failure modes. Models that forget imports are not penalized, inflating scores for careless implementations."
     },
     {
       "flag": "Domain selection not justified",
       "detail": "Six domains are chosen by reference to prior work (Zhuo et al. 2024) but no argument is made for why these six construct domains collectively measure domain-specific capability or whether other domains should be included."
     },
     {
       "flag": "Limited model diversity",
       "detail": "12 models tested are predominantly instruction-tuned variants (GPT, Qwen, CodeLlama, DeepSeek). Generalization to non-instruction-tuned, multilingual, or non-English models is unclear."
     },
     {
       "flag": "No disclosure of funding source",
       "detail": "Funding source is not disclosed, raising questions about independence and whether institutional interests (e.g., promoting Chinese Academy of Sciences research) influence benchmark design."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "relevance": "Foundational HumanEval benchmark for code generation evaluation; establishes Pass@k metric used in DOMAINEVAL."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "relevance": "MBPP benchmark; prior code generation benchmark that DOMAINEVAL extends to multi-domain scenarios."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "relevance": "APPS benchmark sourced from competitive programming; prior dataset for code generation evaluation."
     },
     {
       "title": "Concerned with Data Contamination? Assessing Countermeasures in Code Language Model",
       "relevance": "Addresses data contamination threat in LLM benchmarks; cited to justify DOMAINEVAL's continuous-update contamination resistance claim."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "relevance": "Related multi-domain benchmark exploring LLM capability across domains via API calls; differentiates from DOMAINEVAL's focus on direct implementation."
     },
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation",
       "relevance": "Class-level code generation benchmark; shows progression from function-level (HumanEval) to larger code structures."
     },
     {
       "title": "CodeBenchGen: Creating Scalable Execution-based Code Generation Benchmarks",
       "relevance": "Automated benchmark construction pipeline; directly related to DOMAINEVAL's fully-automated construction approach."
     },
     {
       "title": "RepoCoder: Repository-Level Code Completion Through Iterative Retrieval and Generation",
       "relevance": "Repository-level code generation; contrasts with DOMAINEVAL's function-level scope to clarify contribution boundaries."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Benchmark covers real Python packages (numpy, pandas, cryptography), so practitioners can see LLM capability on familiar libraries. Leaderboard enables direct tool comparison. However, no discussion of production integration or how results translate to development workflows."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "LLMs excelling at computation vs. struggling with cryptography aligns with intuition (training data bias toward tutorials). The observation that more samples increase domain bias contradicts scaling intuitions, but is based on one model's std deviation change and lacks systematic evidence."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Cryptography domain is security-adjacent, and failures in RSA and key derivation are concerning. However, the paper frames this as a capability gap, not a safety risk. No discussion of whether weak cryptography implementation should be flagged as dangerous."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The leaderboard (domaineval.github.io) allows live model submission and comparison. Practitioners can run their own models on the benchmark via the site. However, the paper does not describe how to download the benchmark locally or integrate it into development tools."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "No controversial claims or conflict angles. Straightforward benchmark paper with findings presented as technical observations rather than provocative conclusions."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from Chinese Academy of Sciences (ISCAS) and Hong Kong University of Science and Technology—respected but not tier-1 labs (OpenAI, DeepMind, Meta AI). No prominent researchers named that would drive social media amplification."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "39831754",
         "title": "GPT-4V(ision) Unsuitable for Clinical Care and Education: An Evaluation",
         "points": 75,
         "comments": 52,
         "url": "https://news.ycombinator.com/item?id=39831754"
       },
       {
         "hn_id": "41663273",
         "title": "Unsafe Impedance: Safe Languages and Safe by Design Software",
         "points": 7,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=41663273"
       },
       {
         "hn_id": "40135927",
         "title": "OpenAI: Training LLMs to Prioritize Privileged Instructions",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40135927"
       },
       {
         "hn_id": "41418082",
         "title": "Data Exposure from LLM Apps: An In-Depth Investigation of OpenAI's GPTs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41418082"
       },
       {
         "hn_id": "41408373",
         "title": "Data Exposure from LLM Apps: An In-Depth Investigation of OpenAI's GPTs",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41408373"
       },
       {
         "hn_id": "39139543",
         "title": "Exploring Parent's Needs for Children-Centered AI to Support Preschoolers",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=39139543"
       },
       {
         "hn_id": "37345839",
         "title": "Relighting Neural Radiance Fields with Shadow and Highlight Hints",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37345839"
       },
       {
         "hn_id": "41227450",
         "title": "Τ-Bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41227450"
       },
       {
         "hn_id": "40965488",
         "title": "The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40965488"
       },
       {
         "hn_id": "40157957",
         "title": "The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40157957"
       }
     ],
     "top_points": 75,
     "total_points": 96,
     "total_comments": 54
   }
 }