ai-research-survey

scan.json (30053B)

---

 {
   "paper": {
     "title": "DOMAINEVAL: An Auto-Constructed Benchmark for Multi-Domain Code Generation",
     "authors": [
       "Qiming Zhu",
       "Jialun Cao",
       "Yaojie Lu",
       "Hongyu Lin",
       "Xianpei Han",
       "Le Sun",
       "Shing-Chi Cheung"
     ],
     "year": 2024,
     "venue": "AAAI Conference on Artificial Intelligence 2025",
     "arxiv_id": "2408.13204",
     "doi": "10.48550/arXiv.2408.13204"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DOMAINEVAL is a multi-domain code generation benchmark with 2454 subjects across six domains (computation, network, visualization, basic, system, cryptography), constructed via a fully automated pipeline from GitHub repositories. Evaluation of 12 LLMs shows strong computation performance (82.44% avg Pass@1) but poor cryptography (33.08%) and system (37.50%) performance, with gaps up to 68.94% within a single model. Generating more samples (Pass@5) improves overall performance but does not reduce domain bias. GPT-4o-mini achieves the highest and most stable performance across domains.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a leaderboard URL (https://domaineval.github.io/) but no explicit source code repository URL (e.g., GitHub link) for the benchmark construction pipeline or evaluation code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a leaderboard at https://domaineval.github.io/, which is a GitHub Pages site for the benchmark. As a published AAAI benchmark paper with a public leaderboard, the benchmark data is made available for submission and evaluation."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a basic Python environment and necessary packages' and 'torch.bfloat16 when loading LLMs' but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts for replicating experiments are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 1 report only point estimates (e.g., '82.44% Pass@1'). The 'Std' column measures standard deviation across domains (not across experimental runs), and no confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims performance differences between domains and models (e.g., 'LLMs are generally good at computation tasks while falling short on cryptography') but uses no statistical significance tests to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute performance values with context, e.g., 'The performance gap can be as much as 68.94% (80.94% - 12.0%) Pass@1 in Llama-2-13b-chat' and provides full baseline context in Table 1."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 2454 subjects total but with severe domain imbalance (1705 computation vs 100 for system and cryptography). No justification for these sample sizes or discussion of whether 100 subjects per domain provides sufficient statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "For Pass@1, greedy decoding is deterministic (no variance to report). For Pass@5, temperature sampling is used but no variance across runs or seeds is reported. The 'Std' column measures cross-domain variation, not experimental variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 12 LLMs including GPT-4o-mini, GPT-3.5-turbo, DeepSeek-Coder series, Llama-2, CodeLlama series, Qwen2, Phi-3, and CodeQwen1.5, providing extensive cross-model comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include contemporary models as of 2024: GPT-4o-mini, DeepSeek-Coder-V2-Lite-Instruct, Qwen2-72B, and Phi-3, alongside older models for comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark construction pipeline has multiple components (test-method matching, three selection criteria, instruction generation) but no ablation study examines the contribution of each component to benchmark quality."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both Pass@1 (greedy decoding) and Pass@5 (sampling), along with macro-average and standard deviation across domains."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality is performed. Evaluation is entirely automated via test case execution (Pass@k)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated zero-shot with no tuning on the benchmark data. The entire benchmark serves as a held-out test set since no development or validation split is used for model selection."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides complete per-domain breakdowns across all six domains (computation, network, visualization, basic, system, cryptography) for every model and metric."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Case Study section (Figures 7-8) discusses two specific failure cases: GPT-4o-mini's incorrect use of float exponentiation for large numbers in cryptography, and incorrect parsing of the 'free' command output in the system domain."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that LLMs perform poorly on cryptography (33.08%) and system (37.50%) domains, and that 'the domain bias may even increase' with more samples (CodeLlama-13b-Instruct deviation increased from 19.90 to 20.55)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about domain performance gaps (82.44% computation vs 33.08% cryptography), the 68.94% gap in Llama-2-13b-chat, and increasing bias with more samples are all directly supported by Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'fine-tuning can bring about overall improvement, while the domain gaps still exist' based on comparing Llama-2-13b-chat vs CodeLlama-13b-Instruct. This is a causal claim, but confounds (different instruction tuning, different training data beyond code) are not addressed."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title claims 'Multi-Domain Code Generation' broadly, but all experiments are Python-only. The paper does not caveat that results may not generalize to other programming languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for domain performance differences. For example, whether differences stem from training data distribution, domain-specific complexity, or test difficulty is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures Pass@k (functional correctness via test execution) and frames results in terms of 'code generation capability.' Pass@k directly measures functional correctness, and the claims match the granularity of measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models are specified with detail (e.g., 'DeepSeek-Coder-33b-instruct', 'Qwen2-72B-Instruct-GPTQ-Int4'), but closed-source models are listed only as 'GPT-3.5-turbo' and 'GPT-4o-mini' without API version dates or snapshot identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides the full instruction generation prompt template, and Figure 6 provides the full code generation task prompt, both with actual text used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'For Pass@1 metric, we use greedy decoding, i.e. set temperature to 0.0. For Pass@5 metric, we opt for the minimum sample size N = 5 and maintain temperature at 0.2 and top-p at 0.95. We use torch.bfloat16 when loading LLMs.'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is direct prompt-to-completion code generation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-step pipeline is documented in detail: Step I (domain repository collection from 100+ star GitHub repos), Step II (test-method matching via AST parsing and selection via executable/significant/appropriate difficulty criteria), Step III (LLM-based instruction generation)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no limitations, threats to validity, or similar section. The conclusion mentions 'future research directions' but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no analysis of potential biases in the benchmark construction, domain classification, or evaluation methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state what the results do NOT show. It does not acknowledge that results are limited to Python, to function-level code generation, or to the specific repositories selected."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The leaderboard URL is provided but the paper does not provide a direct download link for the raw benchmark data (subjects, test cases, reference solutions) or intermediate pipeline outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is described in detail: GitHub repositories with 100+ stars classified into six domains, AST-based test-method matching, filtering via three criteria (executable, significant, appropriate difficulty with 3-100 line limits), and LLM-based instruction generation."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Repository selection criteria are described: 'GitHub repositories... particularly those with at least 100 stars, as these are considered high-quality code data.' Domain assignment uses 'the repository's topic labels and README files.' Figure 2 lists all 91 repositories used."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from collection to final dataset is documented with three explicit steps (Figure 2), including filtering criteria and the final count of 2454 subjects with 5892 test cases across six domains."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or sponsorships are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Chinese Academy of Sciences (Institute of Software) and Hong Kong University of Science and Technology. None of the authors are affiliated with the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 12 evaluated models, despite evaluating them on benchmark data sourced from public GitHub repositories."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions that the pipeline 'fortifies DOMAINEVAL against the data contamination threat' through continuous updates, but does not analyze whether the current benchmark data from well-known repos (numpy, pandas, scikit-learn, tensorflow) overlaps with any model's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark draws from extremely popular Python repositories (numpy, pandas, scikit-learn, tensorflow, django, matplotlib, etc.) that are almost certainly in every evaluated LLM's training data. No contamination analysis is performed for the current evaluation, only a conceptual claim about future pipeline updates."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark construction and automated model evaluation study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or wall-clock time are reported for evaluating 12 models on 2454 subjects."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (GPU hours, API spend, hardware used) is stated for either benchmark construction or model evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Pass@1 uses greedy decoding (deterministic, no seed sensitivity). For Pass@5 with temperature=0.2, no seed sensitivity analysis is reported across different random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states N=1 for Pass@1 (greedy) and N=5 for Pass@5 (sampling), which fully specifies the number of runs per evaluation setting."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether alternative evaluation settings (different temperatures, prompts, or decoding strategies) were tried. The chosen settings follow prior work but no search budget is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation configuration follows established protocol from prior work (Zhuo et al. 2024): greedy decoding for Pass@1, temperature=0.2 and top-p=0.95 for Pass@5. The choices are referenced and standard."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 12 models and 6 domains but uses no statistical tests at all, so no multiple comparison correction is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Qwen2-72B-Instruct-GPTQ-Int4 is used to generate task instructions AND is one of the 12 evaluated models. This circular construction-evaluation dependency, which could bias Qwen2's performance, is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 6.7B to 72B parameters are compared without discussing compute requirements. No performance-per-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Pass@k on code extracted from GitHub repositories actually measures 'domain-specific code generation capability' as claimed, or whether the 6-domain classification scheme has construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models are evaluated via direct prompt-to-completion code generation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses code from established repositories (numpy, pandas, scikit-learn, etc.) that have been public for years before any evaluated model's training cutoff. This temporal leakage risk is not analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation provides import statements and class context as input (mentioned in 'Evaluation Process' section), which could leak information about expected solutions. This is not discussed as a potential leakage concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple subjects are drawn from the same repositories (e.g., 1705 computation subjects from 15 repos), creating dependencies between test examples. This non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, n-gram overlap, membership inference) is applied. The paper only mentions the pipeline's potential for future updates as a conceptual defense against contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs are generally good at computation tasks (82.44% avg Pass@1) while falling short on cryptography (33.08%) and system (37.50%) domains.",
       "evidence": "Table 1 shows average Pass@1 across 12 LLMs for each domain, with computation consistently highest and cryptography consistently lowest.",
       "supported": "strong"
     },
     {
       "claim": "The performance gap across domains can be as much as 68.94% within a single LLM (Llama-2-13b-chat: 80.94% computation vs 12.0% cryptography).",
       "evidence": "Table 1, Llama-2-13b-chat row shows 80.94% Pass@1 on computation and 12.0% on cryptography, a 68.94 percentage point gap.",
       "supported": "strong"
     },
     {
       "claim": "Generating more samples increases overall performance but domain bias may even increase.",
       "evidence": "Table 1 comparison of Pass@1 (avg 53.42%, std 18.33) vs Pass@5 (avg 59.60%, std 17.72). CodeLlama-13b-Instruct std increased from 19.90 to 20.55.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o-mini exhibits the highest and most stable performance across domains.",
       "evidence": "Table 1 shows GPT-4o-mini achieves 67.13% Pass@5 (highest average) with 14.75 standard deviation across domains (lowest variance).",
       "supported": "strong"
     },
     {
       "claim": "Fine-tuning (CodeLlama from Llama-2) brings overall improvement but domain gaps persist.",
       "evidence": "Table 1 shows CodeLlama-13b achieves 57.74% Pass@5 vs Llama-2-13b-chat at 46.49% (11.25% improvement), but standard deviation remains high (20.55 vs 24.10).",
       "supported": "moderate"
     },
     {
       "claim": "The automated pipeline enables scalable benchmark construction that can mitigate data contamination through continuous code updates.",
       "evidence": "The pipeline is described in detail (Section 'Benchmark Construction') but no empirical evidence demonstrates that updates actually mitigate contamination.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Severely imbalanced domain sizes",
       "detail": "Computation has 1705 subjects (69.5% of all 2454) while system and cryptography each have only 100. This 17:1 imbalance means the benchmark is dominated by one domain. While macro-averaging addresses this for overall scores, statistical power differs dramatically across domains."
     },
     {
       "flag": "No contamination analysis despite high-risk data sources",
       "detail": "The benchmark draws from some of the most popular Python repositories in existence (numpy, pandas, scikit-learn, tensorflow, django, matplotlib). These repositories and their test suites are almost certainly in every evaluated LLM's training data. The paper's only defense — 'the pipeline can be updated' — is aspirational, not actual."
     },
     {
       "flag": "Circular construction-evaluation dependency",
       "detail": "Qwen2-72B-Instruct-GPTQ-Int4 generates all task instructions (Figure 4) and is also one of the 12 evaluated models. This creates a potential self-favoring bias: the model may perform better on instructions it generated because they align with its own understanding and phrasing. This conflict is not disclosed or discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or scope boundaries discussion. For a benchmark paper proposing a novel evaluation methodology, this is a significant omission."
     },
     {
       "flag": "Python-only with broad domain claims",
       "detail": "All 2454 subjects are Python-only, yet the paper makes general claims about 'multi-domain code generation' capabilities without acknowledging this language restriction. Domain-specific challenges may differ significantly in other languages (e.g., system programming in C/Rust)."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval and the Pass@k metric, which DOMAINEVAL directly builds upon for evaluation."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "relevance": "Introduces MBPP, another foundational code generation benchmark that DOMAINEVAL aims to complement."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["Terry Yue Zhuo", "Minh Chien Vu", "Jenny Chim"],
       "year": 2024,
       "relevance": "Closely related multi-domain code benchmark focusing on API usage; DOMAINEVAL positions itself as complementary by testing code implementation rather than API invocation."
     },
     {
       "title": "MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Multi-language code generation benchmark addressing linguistic diversity, a different dimension from DOMAINEVAL's domain diversity."
     },
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "relevance": "Class-level code generation benchmark, representing the code-scale diversity dimension that complements DOMAINEVAL's domain diversity."
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models",
       "authors": ["Hao Yu", "Bo Shen", "Dezhi Ran"],
       "year": 2024,
       "relevance": "Pragmatic code generation benchmark from real-world code, sharing DOMAINEVAL's goal of testing non-standalone functions."
     },
     {
       "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
       "authors": ["Jia Li", "Ge Li", "Yunfei Zhao"],
       "year": 2024,
       "relevance": "Repository-aligned code generation benchmark requiring significant human effort, contrasting with DOMAINEVAL's automated pipeline."
     },
     {
       "title": "CodeBenchGen: Creating Scalable Execution-based Code Generation Benchmarks",
       "authors": ["Yiqing Xie", "Alex Xie", "Divyanshu Sheth"],
       "year": 2024,
       "relevance": "Automated benchmark construction approach using LLMs to curate CodeSearchNet, directly comparable to DOMAINEVAL's automated pipeline."
     },
     {
       "title": "Concerned with Data Contamination? Assessing Countermeasures in Code Language Model",
       "authors": ["Jialun Cao", "Wuqi Zhang", "Shing-Chi Cheung"],
       "year": 2024,
       "arxiv_id": "2403.16898",
       "relevance": "Directly addresses data contamination in code LLMs, the key threat to DOMAINEVAL's validity that the paper cites but does not empirically address."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming–The Rise of Code Intelligence",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "relevance": "One of the evaluated model families in DOMAINEVAL, representative of specialized code LLMs."
     },
     {
       "title": "Execution-Based Evaluation for Open-Domain Code Generation",
       "authors": ["Zhiruo Wang", "Shuyan Zhou", "Daniel Fried"],
       "year": 2023,
       "relevance": "Execution-based code evaluation benchmark (ODEX) for open-domain code generation, sharing DOMAINEVAL's goal of testing beyond algorithmic problems."
     },
     {
       "title": "McEval: Massively Multilingual Code Evaluation",
       "authors": ["Linzheng Chai", "Shukai Liu", "Jian Yang"],
       "year": 2024,
       "relevance": "Multilingual code evaluation benchmark addressing linguistic diversity dimension complementary to DOMAINEVAL's domain diversity."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark helps practitioners understand LLM strengths/weaknesses across programming domains, useful for tool selection decisions."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Finding that LLMs struggle with cryptography and system code is somewhat expected given training data distributions, though the magnitude of the gap (68.94%) is notable."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised by the findings."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with prior claims; the paper positions itself as complementary to existing work."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A leaderboard website exists at https://domaineval.github.io/ but no interactive demo or pip-installable tool is provided."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT-4o-mini, DeepSeek-Coder) but authors are from academic institutions, not major AI labs."
     }
   }
 }