ai-research-survey

scan.json (31793B)

---

 {
   "paper": {
     "title": "Top General Performance = Top Domain Performance? DomainCodeBench: A Multi-domain Code Generation Benchmark",
     "authors": [
       "Dewu Zheng",
       "Yanlin Wang",
       "Ensheng Shi",
       "Xilin Liu",
       "Yuchi Ma",
       "Hongyu Zhang",
       "Zibin Zheng"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2412.18573"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "DomainCodeBench evaluates 10 LLMs across 12 application domains using 2,400 manually verified tasks and finds that top general-domain performance (e.g., GPT-4 on HumanEval) does not predict top domain-specific performance (DeepSeekCoder-33B dominates most domains despite ranking 4th on HumanEval). Manual error analysis identifies domain knowledge gaps and third-party library misusage as primary failure causes. Augmenting prompts with domain-specific context (similar code, dependency info, API docs) improves CodeBLEU scores by up to ~38% on average, with similar-context retrieval yielding the largest gains.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'Our replication package, including the benchmark, source code, and experimental results, is available at https://github.com/DeepSoftwareAnalytics/DomainCodeBench.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at the GitHub URL includes the benchmark dataset of 2,400 tasks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies hardware ('Ubuntu 18.04.6 LTS with 128 Intel(R) Xeon(R) Platinum 8336C @ 2.30GHz CPUs and 8 NVIDIA A800 80GB PCIe GPUs') but provides no software dependency versions, requirements.txt, or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The replication package is referenced but the paper itself contains no instructions for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 4, 5, and 7 report only point estimates (CodeBLEU scores) with no confidence intervals or error bars, despite averaging over 3 runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'DeepSeekCoder-33B ranks 4th on HumanEval yet achieves top performance in most application domains') based solely on comparing raw CodeBLEU numbers with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute CodeBLEU improvements in Table 7 (e.g., +17.14 for CodeLLaMa-7B with all contexts), percentage improvements ('around 38.17%'), and variance across domains (σ² column in Table 4), providing sufficient context for effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 200 tasks per domain, why 10 LLMs were selected, or why 10% of outputs were sampled for error analysis. The numbers appear chosen by convenience."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'the experimental results presented in this paper are obtained by conducting three repeated experiments and averaging the results' but reports no standard deviation, IQR, or any spread measure across runs. The σ² column in Table 4 is variance across domains, not across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 10 LLMs against each other and against their HumanEval rankings, establishing a baseline comparison framework across general and domain-specific performance."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models used include GPT-4-0125-preview, StarCoder2 (2024), DeepSeekCoder (2024), and CodeLLaMa (2023), all within 1-2 years of the paper's submission. However, notable late-2024 models (GPT-4o, Claude 3.5, Llama 3) are missing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Table 7) systematically tests individual context types (API, Dependency, Similar) and their combinations (API+Dependency, API+Similar, API+Similar+Dependency), effectively ablating each component's contribution to performance improvement."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only CodeBLEU as its evaluation metric. The authors explicitly acknowledge not using execution-based metrics and state 'We use CodeBLEU as the primary metric for evaluating code generation ability.'"
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes manual error analysis by five annotators with 5+ years of programming experience who analyzed LLM-generated outputs to classify failure causes, with case studies in Figure 4."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The authors acknowledge in the threats section that they 'choose the prompt template that yields the best results after testing several options,' indicating the test data was used for prompt selection decisions without a separate validation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides per-domain breakdowns across all 12 domains for every model. Table 5 further breaks down into sub-domains for Enterprise Application and Game development."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 provides an extensive failure taxonomy (Table 6) with three detailed case studies in Figure 4 covering insufficient project understanding, limited domain library knowledge, and unfamiliarity with domain algorithms."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that GPT-4 (best on HumanEval) underperforms domain-specifically, that larger models don't always outperform smaller ones, and unexpectedly that similar context outperforms oracle dependency+API context — 'LLMs perform better in the experiment with the only similar context than in the one where all dependencies and domain-specific third-party library knowledge are provided.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's three main claims — performance decoupling (supported by Table 4), domain-specific weaknesses (supported by Section 5.2), and ~38.17% contextual enhancement (supported by Table 7) — are all backed by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claim — that augmenting prompts with domain-specific knowledge improves performance — is tested through controlled experiments in RQ3 where the only variable changed is the additional context provided, constituting adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Findings generalize beyond the tested scope. Finding 1 states 'Current LLMs exhibit substantial performance gaps in application-domain code generation' — generalizing from 10 tested models to 'current LLMs.' The paper tests 10 models on 12 domains but draws broad conclusions about LLMs generally."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats-to-validity section (Section 6) discusses methodological limitations but does not explore alternative explanations for the observed findings. For example, it does not consider whether performance differences across domains could be explained by CodeBLEU's varying reliability across programming languages, or whether training data distribution effects explain domain performance differences."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges that CodeBLEU 'does not fully capture whether the generated code can execute correctly or pass the corresponding test cases' (Section 6), explicitly recognizing the gap between the measured proxy (CodeBLEU) and functional correctness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 is specified as 'GPT-4-0125-preview.' Open-source models are identified with specific parameter sizes (CodeLLaMa-7B/13B/34B, DeepSeekCoder-6.7B/33B, StarCoder-15.5B, StarCoder2-3B/7B/15B), which uniquely identify the model checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the prompt structure ('we combine the docstring and function signature in accordance with the characteristics of the task's corresponding programming language') but does not provide the actual prompt text or template used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'we set the temperature to 0.4 and top-p to 0.95' following prior work."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper performs direct code generation from prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the full pipeline: topic mining from tech websites (2020-2024), LDA-based domain identification, project selection by GitHub star count, manual function verification for domain relevance, cross-validated docstring annotation by 5 annotators, and automated dependency extraction."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a substantive discussion of internal and external threats spanning multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section identifies study-specific concerns: limited domain coverage (12 of many domains), computational constraints preventing testing all LLMs, CodeBLEU not capturing execution correctness, GitHub data potentially in training sets (mitigated by docstring rewriting), model generation randomness (mitigated by 3 runs), and prompt template selection effects."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes limitations but does not explicitly state what the results do NOT show. There is no explicit bounding of claims — e.g., no statement that results apply only to the tested 10 models and 12 domains and should not be extrapolated to other LLMs or domains."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at GitHub includes 'the benchmark, source code, and experimental results,' making the raw benchmark data and model outputs available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the full data collection process: crawling posts from 7 tech websites (2020-2024), LDA topic mining, project selection by star count on GitHub, manual function sampling with domain-relevance verification, and cross-validated docstring annotation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Five annotators are described as having 'more than five years of programming experience' but no details are provided about how they were recruited, their specific backgrounds, or whether their selection could introduce bias in the annotation process."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from domain identification to final benchmark is well-documented in Section 3.2 and Figure 2, with clear stages: domain mining → project selection → function sampling → docstring annotation → dependency extraction, resulting in 2,400 verified tasks (200 per domain)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments are mentioned anywhere in the paper, despite three co-authors being from Huawei Cloud Computing Technologies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Sun Yat-sen University, Huawei Cloud Computing Technologies Co., Ltd, and Chongqing University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. Three co-authors are from Huawei Cloud, which has commercial interests in LLM development, though the paper does not evaluate Huawei products specifically."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the 10 evaluated models, despite acknowledging that 'DomainCodeBench contains data from GitHub before 2023, which may have already been included in the training datasets.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (External Threats) explicitly discusses that benchmark data from GitHub before 2023 'may have already been included in the training datasets of the evaluated models' and describes mitigation via manual docstring rewriting."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges that benchmark code originates from pre-2023 GitHub projects that may be in training data, and mitigates by providing 'each task instance in DomainCodeBench with a manually annotated docstring, avoiding the use of docstrings from online code repositories to prevent LLMs from directly associating training data with the programming tasks.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental evaluation. Annotators are part of benchmark construction, not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental evaluation."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental evaluation."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental evaluation."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental evaluation."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite running GPT-4 API calls on 2,400 tasks across multiple context configurations and repeating each experiment 3 times."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (128 CPUs, 8 NVIDIA A800 GPUs) but no total compute budget (GPU hours, API spend, wall-clock time) is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper conducts 3 repeated experiments and averages results but reports no variance, standard deviation, or any seed sensitivity analysis across these runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 states: 'the experimental results presented in this paper are obtained by conducting three repeated experiments and averaging the results.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges testing 'several' prompt templates and selecting the best-performing one, but does not report how many were tested, what alternatives were tried, or the selection methodology."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The threats section admits 'We ultimately choose the prompt template that yields the best results after testing several options' without describing the selection process, whether a validation set was used, or what alternatives were tested."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes dozens of comparisons across 10 models and 12 domains but uses no statistical tests at all, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of author-evaluation bias. The authors constructed the benchmark, selected the tasks, wrote the docstrings, chose which models to test, and selected the best prompt template — all without acknowledging potential self-comparison biases."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes that 'the larger parameter scale of LLMs does not necessarily correlate with better code generation performance' but does not analyze performance as a function of compute budget or normalize results by model cost/size."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges that CodeBLEU 'does not fully capture whether the generated code can execute correctly or pass the corresponding test cases' and discusses why execution-based metrics were impractical, demonstrating awareness of construct validity limitations."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used; all models are evaluated via direct code generation from prompts."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses that 'DomainCodeBench contains data from GitHub before 2023, which may have already been included in the training datasets of the evaluated models' and mitigates by manually rewriting docstrings."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The RQ3 experiments intentionally provide oracle dependency and API information extracted from the ground truth, but the paper does not discuss whether the base evaluation setup (RQ1) might inadvertently leak information through function signatures or import statements."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether tasks sampled from the same repository share dependencies or correlated solutions, or whether the 200-per-domain sampling introduces structural dependencies."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Manual docstring rewriting is used as a concrete leakage prevention method: 'we manually rewrite the docstrings for every programming problem in DomainCodeBench... to prevent LLMs from directly associating training data with the programming tasks.'"
       }
     }
   },
   "claims": [
     {
       "claim": "Top general-domain models do not consistently excel in specific application domains (performance decoupling).",
       "evidence": "Table 4 shows DeepSeekCoder-33B (ranked 4th on HumanEval at 50.6) achieves top performance in most domains, while GPT-4 (ranked 1st on HumanEval at 83.5) underperforms domain-specifically. CodeLLaMa-34B surpasses DeepSeekCoder-33B on HumanEval but trails in all application domains except Mobile.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs fail at domain-specific code generation primarily due to insufficient project understanding (FOC 0.97), limited domain library knowledge (FOC 0.77), and unfamiliarity with domain algorithms (FOC 0.66).",
       "evidence": "Section 5.2 presents a failure taxonomy (Table 6) based on manual error analysis by 5 annotators, with detailed case studies in Figure 4 for each failure category.",
       "supported": "moderate"
     },
     {
       "claim": "Augmenting prompts with domain-specific knowledge improves performance by around 38.17% on average.",
       "evidence": "Table 7 shows that combining API + Similar + Dependency context yields mean CodeBLEU improvements of 10.70-17.64 across all models (Section 5.3).",
       "supported": "moderate"
     },
     {
       "claim": "Similar context (retrieval-based) yields more substantial performance improvement than oracle dependency context or oracle API context.",
       "evidence": "Table 7 shows mean improvements of 7.24-12.65 for similar context alone vs. 3.87-6.08 for dependency context and 1.85-4.18 for API context across all models.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs generally perform well in blockchain but show weaker performance in web development and enterprise applications.",
       "evidence": "Table 4 shows blockchain scores consistently highest across models (e.g., DeepSeekCoder-6.7B: 49.04 blockchain vs. 34.93 web), with web and enterprise application scores among the lowest.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs with similar general-domain performance can exhibit significant divergences in domain-specific performance.",
       "evidence": "Figure 3 shows DeepSeekCoder-6.7B and StarCoder2-15B (both 45.1 on HumanEval) diverge substantially across domains, with different strengths in different application areas.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single evaluation metric (CodeBLEU only)",
       "detail": "The entire evaluation relies solely on CodeBLEU, a textual/semantic similarity metric, without any execution-based evaluation. CodeBLEU may not capture functional correctness, especially for domain-specific code where API calling conventions matter more than textual similarity. The authors acknowledge this but proceed with no alternative metric."
     },
     {
       "flag": "No variance reported across runs",
       "detail": "Despite conducting 3 repeated experiments, the paper reports only averaged results with no standard deviation, confidence intervals, or any measure of result stability. This makes it impossible to assess whether reported differences between models are meaningful or within noise."
     },
     {
       "flag": "Prompt selection on test data",
       "detail": "The authors admit selecting 'the prompt template that yields the best results after testing several options,' which means the reported results reflect an optimized prompt without a separate validation set. This inflates reported performance for all models."
     },
     {
       "flag": "Missing contemporary models",
       "detail": "For a December 2024 paper, the evaluation misses widely available models including GPT-4o, Claude 3.5, Llama 3.1, and DeepSeek-Coder-V2. Only one closed-source model (GPT-4) is tested, weakening claims about 'mainstream LLMs' generally."
     },
     {
       "flag": "Oracle contexts inflate RQ3 improvement claims",
       "detail": "The 'Dependency Context' and 'API Context' experiments in RQ3 use oracle information extracted from the ground truth, making the reported improvements upper bounds rather than achievable real-world gains. The ~38.17% improvement claim in the abstract does not clearly distinguish oracle vs. realistic settings."
     },
     {
       "flag": "No funding disclosure with industry co-authors",
       "detail": "Three of seven co-authors are from Huawei Cloud Computing Technologies, yet no funding sources, grants, or competing interests are disclosed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the foundational code generation benchmark that DomainCodeBench's analysis directly contrasts against."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP, a widely-used general-domain code generation benchmark for LLMs."
     },
     {
       "title": "EvoCodeBench: An Evolving Code Generation Benchmark with Domain-Specific Evaluations",
       "authors": ["Jia Li"],
       "year": 2024,
       "relevance": "Contemporary evolving code generation benchmark with domain-specific evaluation, directly related to benchmark design in this space."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu"],
       "year": 2024,
       "relevance": "Repository-level code generation benchmark evaluating LLMs on practical coding tasks."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Code generation benchmark with diverse function calls across 139 libraries, closely related domain-aware evaluation."
     },
     {
       "title": "DOMAINEVAL: An Auto-Constructed Benchmark for Multi-Domain Code Generation",
       "authors": ["Qiming Zhu"],
       "year": 2024,
       "arxiv_id": "2408.13204",
       "relevance": "Most closely related work — multi-domain code generation benchmark focused on programming domains rather than application domains."
     },
     {
       "title": "Top Leaderboard Ranking = Top Coding Proficiency, Always? EvoEval: Evolving Coding Benchmarks via LLM",
       "authors": ["Chunqiu Steven Xia"],
       "year": 2024,
       "relevance": "Questions whether leaderboard rankings reflect true coding proficiency, directly parallel to this paper's research question."
     },
     {
       "title": "RepoCoder: Repository-level code completion through iterative retrieval and generation",
       "authors": ["Fengji Zhang"],
       "year": 2023,
       "arxiv_id": "2303.12570",
       "relevance": "Repository-level code generation method using retrieval augmentation, the basis for the similar-context approach tested in RQ3."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "One of the evaluated open-source LLM families for code generation."
     },
     {
       "title": "DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence",
       "authors": ["Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2406.11931",
       "relevance": "Successor to DeepSeekCoder evaluated in this paper; a notably absent contemporary baseline."
     },
     {
       "title": "On the effectiveness of large language models in domain-specific code generation",
       "authors": ["Xiaodong Gu"],
       "year": 2024,
       "relevance": "Directly studies LLM effectiveness in domain-specific code generation, a core theme of DomainCodeBench."
     },
     {
       "title": "CodeBLEU: a Method for Automatic Evaluation of Code Synthesis",
       "authors": ["Shuo Ren"],
       "year": 2020,
       "arxiv_id": "2009.10297",
       "relevance": "The sole evaluation metric used in this paper for measuring code generation quality."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners can use the benchmark to evaluate which LLM is best for their specific application domain, with the benchmark publicly released."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that HumanEval rankings predict real-world domain performance — GPT-4 (#1 general) loses to DeepSeekCoder-33B (#4) in most domains."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns raised."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild implicit criticism of general-domain benchmarks as insufficient, but no direct controversy or confrontational framing."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Benchmark and code are released on GitHub; researchers can evaluate their own models, though it requires significant compute to run."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "University + Huawei Cloud authors; evaluates GPT-4 and well-known open-source models but is not from a high-profile AI lab."
     }
   }
 }