ai-research-survey

scan.json (30046B)

---

 {
   "paper": {
     "title": "DevEval: A Manually-Annotated Code Generation Benchmark Aligned with Real-World Code Repositories",
     "authors": [
       "Jia Li",
       "Ge Li",
       "Yunfei Zhao",
       "Yongmin Li",
       "Huanyu Liu",
       "Hao Zhu",
       "Lecheng Wang",
       "Kaibo Liu",
       "Zheng Fang",
       "Lanshen Wang",
       "Jiazheng Ding",
       "Xuanming Zhang",
       "Yuqi Zhu",
       "Yihong Dong",
       "Zhi Jin",
       "Binhua Li",
       "Fei Huang",
       "Yongbin Li"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2405.19856",
     "doi": "10.48550/arXiv.2405.19856"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "DevEval is a new code generation benchmark of 1,874 samples from 117 real-world Python repositories, annotated by 13 developers, with distributions aligned to 500 real-world repos. Evaluation of 8 LLMs shows dramatic performance drops compared to HumanEval — gpt-4-turbo achieves only 53.04% Pass@1 (vs 80% on HumanEval). Context from local files improves gpt-4's Pass@1 by up to 205%, but LLMs still struggle with long heterogeneous contexts, often hallucinating non-existent functions instead of using available dependencies.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A GitHub repository is provided: https://github.com/seketeam/DevEval. The abstract states 'DevEval, prompts, and LLMs' predictions have been released.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data is released at the same GitHub repository. The paper states the benchmark, prompts, and predictions have been released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions using setuptools and Pytest but does not provide reproducible environment details for the LLM evaluation."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. While the GitHub repo contains data and prompts, the paper lacks a 'Reproducing Results' section or specific instructions for rerunning the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 reports only point estimates (e.g., 53.04% Pass@1) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparisons between LLMs (e.g., 'gpt-4 achieves higher Pass@1') are based solely on comparing point estimates without any tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baseline context, e.g., 'the Pass@1 of gpt-4 is improved by 205% and 173% in two settings' (Section 4.4), and provides both baseline and improved values throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final benchmark contains 1,874 samples but no justification for why this number is sufficient for the claims made. The construction pipeline is described but no power analysis or sample size reasoning is given."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across experimental runs. For k=1 greedy search is used (single deterministic output), and for k>1 only 20 samples are drawn per example with no repeated trials or spread measures reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Eight LLMs are compared (Table 5), and performance is compared against prior benchmarks (e.g., gpt-4's 80% Pass@1 on HumanEval vs 53.04% on DevEval). Three experimental settings serve as internal baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models (gpt-4-turbo-1106, DeepSeek Coder, StarCoder 2, CodeLLaMa) were among the most capable code LLMs available at the time of writing (late 2023 / early 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Three experimental settings ablate the role of context: without context, local file completion (code above), and local file infilling (code above and below). Results in Table 5 and Figures 5-6 also break down by program type and dependency type."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two complementary metrics are used: Pass@k (functional correctness via test execution) and Recall@k (recall of reference dependencies). Both are reported at k=1,3,5,10."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation of LLM outputs is entirely automated via test case execution and static dependency extraction. The 13 developers annotated the benchmark data but did not evaluate LLM-generated code."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "DevEval serves as the test set. The 8 LLMs are evaluated off-the-shelf without any fine-tuning on DevEval data, so there is no dev/test contamination within the experimental protocol."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by program type (standalone vs non-standalone, Figure 5), dependency type (intra-class, intra-file, cross-file, Figure 6), and experimental setting (Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 includes 'Error Case Analyses' with Figure 4 showing a specific failure where gpt-3.5 invokes a non-existent function 'create_connection' instead of the available 'connect'. Two root causes are identified."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that all LLMs perform poorly on DevEval compared to HumanEval, that LLMs struggle with long heterogeneous contexts, and that cross-file dependency generation remains particularly weak."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims gpt-4-turbo's highest Pass@1 is 53.04%, which matches Table 5 (Local File Infilling). Claims about benchmark alignment are supported by Tables 2-3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that code context improves LLM performance. The three experimental settings (no context vs completion vs infilling) form a controlled comparison that adequately supports this. Other causal language is modest."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims alignment with 'Real-World Code Repositories' broadly, but the benchmark is Python-only and English-only. While the Limitations section (Section 9) acknowledges this is monolingual, the title and main framing overgeneralize beyond what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses two reasons LLMs can generate dependencies without context (reasoning from requirements, guessing from naming conventions). It also explains why gpt-family vs open-source models behave differently (instruction tuning). Error analysis considers context length and heterogeneity as alternative failure explanations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Pass@k explicitly measures functional correctness via test execution, and Recall@k measures dependency recall. The paper is precise about what these metrics capture and introduces Recall@k specifically to address a gap not captured by Pass@k alone. Claims match measurement granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 specifies model versions: gpt-4-turbo-1106, gpt-3.5-turbo-1106 (with snapshot dates), and open-source model sizes (StarCoder 2 15B/7B, DeepSeek Coder 33B/6.7B, CodeLLaMa 13B/7B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'DevEval, prompts, and LLMs' predictions have been released' at https://github.com/seketeam/DevEval. The GitHub repository is stated to contain the prompts used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 reports: greedy search for k=1, nucleus sampling with temperature 0.4 and top-p 0.95 for k>1, max generation length 500, n=20 programs sampled per requirement."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. LLMs are prompted directly with requirement, signature, and context to generate code."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full pipeline across five stages with filtering at each step: 500 repos → 590,365 functions → 3,764 with tests → 2,846 after annotation → 1,874 final samples. Criteria at each stage are described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 'Limitations' provides substantive discussion of three specific limitations: monolingual scope, Recall@k bias from static analysis, and limited context settings."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: monolingual limitation (Python/English only), Recall@k parser bias quantified at 0.16 from a 50-program sample comparison with human annotators, and the limitation of only using local file context."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it is Python-only, English-only, uses only local file contexts (not imported or sibling files), and that Recall@k has slight bias. Future work lists specific extensions planned (multilingual, more projects, more test cases)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (samples, repositories, test cases, prompts, and LLM predictions) is released at https://github.com/seketeam/DevEval, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in detail across five stages: repository selection from PyPI (top 10 topics, top 50 repos each, three selection criteria), function parsing, test construction, human annotation (674 person-hours, dual annotation), and final benchmark construction."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The 13 developers who annotated requirements and dependencies are mentioned but their recruitment is not described — no information on how they were selected, their qualifications, programming experience levels, or affiliations beyond 'countries of residence' for payment purposes."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section 3 with counts at each stage: 500 repos (10 topics × 50) → 590,365 candidate functions → 3,764 with tests → 2,846 after annotation → 1,874 final samples (73% non-standalone, 27% standalone)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'This research was supported by the National Natural Science Foundation of China (Nos. 62192731, 62152730).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Peking University (School of Computer Science, Key Laboratory) and Alibaba Group."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Natural Science Foundation of China is a government funding agency with no commercial stake in benchmark outcomes. While Alibaba authors are involved, Alibaba's own models are not being evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the 8 evaluated models. Section 5 uses 'release dates' as a proxy for training data cutoffs but does not report the actual cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Data leakage' discusses train/test overlap: repos are divided into 'unseen' (released after LLM) and 'potentially seen' (released before LLM). Average Pass@1 difference of 0.36 between groups is reported as slight."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 provides three arguments: (1) requirements are newly written and never in training data, (2) no overfitting tendencies between seen/unseen repos (0.36 average difference), (3) repo links released to help future LLM trainers omit them."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted. The 13 developers were annotators for the benchmark, not participants in a study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted. The developers were compensated annotators, not research participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported for any of the 8 LLMs evaluated, despite using commercial APIs (gpt-4, gpt-3.5) and generating 20 programs per requirement across 1,874 samples."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper mentions 674 person-hours for annotation but does not report GPU hours, API spend, or hardware used for the LLM evaluation experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. The nucleus sampling introduces randomness but results are not reported across multiple random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states: 'we generate n≥k programs per requirement' and specifies n=20 for sampling. For k=1, greedy search generates a single deterministic program."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The generation parameters (temperature=0.4, top-p=0.95, max length=500) are stated but no justification or search process for selecting these values is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All three experimental settings (without context, completion, infilling) are reported in Table 5 with full results. No cherry-picking of configurations — all conditions are transparently shown."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so correction for multiple comparisons is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes DevEval and evaluates it against prior benchmarks (Tables 1-2) but does not acknowledge potential bias from authors designing a benchmark that supports their narrative that existing benchmarks are insufficient."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes (7B to gpt-4) are compared without any discussion of compute budget differences. No performance-per-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 and Tables 2-3 extensively analyze alignment with 500 real-world repos across code distributions, dependency types, and repository scale. The paper explicitly discusses what makes a benchmark valid for evaluating real-world coding ability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. LLMs are prompted directly for code generation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 addresses temporal leakage by dividing repos into 'unseen repositories released later than LLMs and potentially seen repositories released earlier than LLMs,' finding an average Pass@1 difference of 0.36."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup (providing local file context) might leak answer information or provide hints not available in realistic development scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data of the LLMs might contain code from the same repositories or structurally similar code, beyond the temporal analysis."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a concrete temporal split method: comparing Pass@1 on repos released before vs after each LLM's release date, finding an average difference of 0.36 (vs 5.15 average variation between LLMs)."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing code generation benchmarks are poorly aligned with real-world code repositories in code distribution, dependencies, and annotations.",
       "evidence": "Tables 1-3 compare DevEval with prior benchmarks across multiple dimensions. Most benchmarks are 100% standalone functions with 0 dependencies, while 500 real-world repos show 73% non-standalone code with 3.22 dependencies per sample.",
       "supported": "strong"
     },
     {
       "claim": "gpt-4-turbo achieves a highest Pass@1 of only 53.04% on DevEval, compared to 80% on HumanEval.",
       "evidence": "Table 5 shows Pass@1 of 53.04% for gpt-4 in the Local File (Infilling) setting. The 80% HumanEval figure is cited from prior work.",
       "supported": "strong"
     },
     {
       "claim": "Code context from local files improves gpt-4's Pass@1 by up to 205%.",
       "evidence": "Table 5: gpt-4 without context achieves 17.40 Pass@1, vs 53.04 in Local File (Infilling), a 205% improvement. Reported in Section 4.4.",
       "supported": "strong"
     },
     {
       "claim": "DevEval's distributions are aligned with 500 real-world repositories.",
       "evidence": "Table 2 shows DevEval has 27% standalone / 73% non-standalone code matching 500 repos exactly. Table 3 shows dependency type distribution (38%/32%/30%) close to 500 repos (42%/29%/30%). Average dependencies per sample: 3.41 vs 3.22.",
       "supported": "strong"
     },
     {
       "claim": "LLMs struggle with understanding long and heterogeneous contexts, leading to hallucinations.",
       "evidence": "Section 4.4 error case analysis shows gpt-3.5 generating non-existent function 'create_connection' despite 'connect' being in context. Two root causes identified: context length and heterogeneity. Supported by one qualitative example (Figure 4).",
       "supported": "moderate"
     },
     {
       "claim": "Data leakage has only a slight impact on DevEval results.",
       "evidence": "Section 5 reports average Pass@1 difference of 0.36 between seen and unseen repos, compared to 5.15 average variation between LLMs. Three arguments provided, but analysis uses release dates as proxy for training cutoffs.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results in Table 5 are point estimates with no confidence intervals, standard deviations, or significance tests. Claims of superiority between models (e.g., gpt-4 vs gpt-3.5) rest entirely on comparing single numbers."
     },
     {
       "flag": "No significance testing for model comparisons",
       "detail": "Multiple claims about which models are better are made without any statistical tests. Small differences (e.g., DeepSeek Coder 33B vs CodeLLaMa 13B at 46.32 vs 41.94) may not be significant."
     },
     {
       "flag": "Annotator recruitment and qualifications undisclosed",
       "detail": "The 13 developers who annotated requirements and dependencies — a critical component of benchmark quality — are not characterized beyond 'developers.' Their experience levels, selection criteria, and inter-annotator agreement are not reported."
     },
     {
       "flag": "Training cutoff dates not established",
       "detail": "The data leakage analysis uses model release dates as a proxy for training data cutoffs, which is imprecise. Actual training cutoff dates for the 8 models are not stated."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the most widely used code generation benchmark that DevEval aims to improve upon."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark and the Pass@k metric used throughout DevEval's evaluation."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu", "Bo Shen", "Dezhi Ran"],
       "year": 2023,
       "arxiv_id": "2302.00288",
       "relevance": "Prior real-world code generation benchmark that DevEval extends with better alignment to repository distributions."
     },
     {
       "title": "ClassEval: A manually-crafted benchmark for evaluating LLMs on class-level code generation",
       "authors": ["Xueying Du", "Mingwei Liu", "Kaixin Wang"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "Class-level code generation benchmark; DevEval addresses its limitations in repository context and dependency coverage."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Repository-level benchmark for issue resolution; contrasted with DevEval's focus on code generation from requirements."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the evaluated code LLMs, demonstrating competitive performance on DevEval."
     },
     {
       "title": "StarCoder 2 and the Stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Open-source code LLM evaluated on DevEval, representing the state of open-source code generation."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM evaluated on DevEval; widely used baseline for code generation."
     },
     {
       "title": "RepoBench: Benchmarking repository-level code auto-completion systems",
       "authors": ["Tianyang Liu", "Canwen Xu", "Julian J. McAuley"],
       "year": 2023,
       "arxiv_id": "2306.03091",
       "relevance": "Repository-level code completion benchmark; lacks natural language requirements that DevEval provides."
     },
     {
       "title": "CrossCodeEval: A diverse and multilingual benchmark for cross-file code completion",
       "authors": ["Yangruibo Ding", "Zijian Wang", "Wasi Uddin Ahmad"],
       "year": 2023,
       "arxiv_id": "2310.11248",
       "relevance": "Cross-file code completion benchmark; complements DevEval's focus on code generation with requirements."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt"],
       "year": 2023,
       "arxiv_id": "2307.03172",
       "relevance": "Study on LLMs ignoring information in long contexts, directly relevant to DevEval's error analysis findings."
     },
     {
       "title": "Repository-level prompt generation for large language models of code",
       "authors": ["Disha Shrivastava", "Hugo Larochelle", "Daniel Tarlow"],
       "year": 2023,
       "relevance": "Proposes methods for extracting relevant context from repositories, directly motivating DevEval's experimental settings."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Competition-style code generation benchmark; DevEval argues such benchmarks are poorly aligned with real-world development."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Benchmark is publicly released and directly usable for evaluating code generation LLMs on realistic tasks."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Shows LLMs perform much worse on real-world code than HumanEval, which is somewhat expected but well-quantified."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns are raised."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly challenges HumanEval and similar benchmarks as insufficient, but framed constructively rather than controversially."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Benchmark, prompts, and model predictions released on GitHub for immediate use and reproduction."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Peking University and Alibaba Group are moderately well-known in the AI research community."
     }
   }
 }