ai-research-survey

scan.json (31227B)

---

 {
   "paper": {
     "title": "Mercury: A Code Efficiency Benchmark for Code Large Language Models",
     "authors": [
       "Mingzhe Du",
       "Anh Tuan Luu",
       "Bin Ji",
       "Qian Liu",
       "See-Kiong Ng"
     ],
     "year": 2024,
     "venue": "Preprint (arXiv)",
     "arxiv_id": "2402.07844",
     "doi": "10.48550/arXiv.2402.07844"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Mercury is the first code efficiency benchmark for Code LLMs, comprising 1,889 Python tasks from LeetCode with a runtime-percentile-based metric called Beyond. Leading Code LLMs achieve up to 65% on functional correctness (Pass) but below 50% on Beyond, revealing a significant efficiency gap. Direct Preference Optimization (DPO) consistently improves both correctness and efficiency, especially for models over 15B parameters, while Supervised Fine-Tuning (SFT) often induces catastrophic forgetting in larger models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link in the abstract footnote: 'Our code and data are available on GitHub: https://github.com/Elfsong/Mercury.' Code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Mercury dataset is hosted on HuggingFace (Section A.12): 'https://huggingface.co/datasets/Elfsong/Mercury'. The dataset includes tasks, solutions, test case generators, and prompts."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 mentions 'two A100-80G GPUs' and names libraries (Accelerate, DeepSpeed, BitsandBytes) but does not provide version numbers, requirements.txt, or Dockerfile. Library names without versions are insufficient for environment recreation."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository with code, a HuggingFace dataset, and Section 4.3 describes experimental configuration in detail (LoRA parameters, optimizer, learning rates, training steps, sampling settings). Combined with the public code and data, this is sufficient for a competent researcher to reproduce."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2 and 3 (the main results) report only point estimates without confidence intervals, error bars, or ± notation. Appendix Figure 11 shows bootstrapped distributions for 3 of 10 models, but the main results tables lack any uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparative claims like 'DPO invariably enhances the overall Pass scores' (Section 4.4) and 'SFT detracts most Beyond scores' are based solely on comparing point estimates without p-values or any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 3 report absolute differences from baseline in parentheses (e.g., '+8.4', '-6.3') and Gap metric showing the difference between Beyond and Pass. Results are contextualized with both baseline and improved values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation set contains 256 tasks and the training set 1,633 tasks. No justification is given for why 256 evaluation tasks suffice, and no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results tables (2, 3, 4) show single point estimates without standard deviation or variance. Appendix Figure 11 shows bootstrapped distributions but only for 3 of 10 models and only for the Beyond metric, not for training experiments (SFT/DPO)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares original (pre-trained) models against SFT and DPO baselines. It also includes HumanEval and MBPP as supplementary functional correctness benchmarks (Section 4.4, Tables 2 and 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 10 evaluated models include contemporary Code LLMs from 2023-2024: DeepSeek-Coder, StarCoder2, CodeQwen1.5, and CodeLlama (Table 7). These represent the state of the art for open-source Code LLMs at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares original, SFT, and DPO training strategies but does not ablate individual components of the Mercury benchmark design (e.g., the runtime distribution modeling, test case generator quality, or metric formulation)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pass (functional correctness), Beyond (efficiency-weighted correctness), and Gap (Beyond minus Pass difference) as metrics. Results are also reported on HumanEval and MBPP."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via test case execution and runtime measurement. No human evaluation of code quality, readability, or maintainability is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Mercury-eval (256 tasks) is a separate held-out set from Mercury-train (1,633 tasks) as described in Section 2 and Table 6. SFT and DPO training use Mercury-train; evaluation uses Mercury-eval."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by difficulty level (Easy, Medium, Hard) in Tables 2, 3, and 4, showing significant performance variation across categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 provides a detailed failure analysis with Table 4 breaking down errors into three categories: Generation Errors, Execution Errors, and Test Case Errors, with counts per model and difficulty level."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that SFT diminishes functional correctness on larger models: deepseek-coder-33b-base (-6.3) and CodeLlama-34b-hf (-4.9) in Table 2. SFT also detracts most Beyond scores (Table 3). These negative findings are explicitly discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims Code LLMs 'achieve 65% on Pass, while less than 50% on Beyond' — supported by Tables 2 and 3 (e.g., deepseek-coder-33b achieves 65% Pass and 48.53% Beyond). The claim that DPO is a robust baseline compared to SFT is supported across Tables 2 and 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims DPO 'enhances' code efficiency and SFT 'may induce catastrophic forgetting.' These are supported by controlled single-variable manipulation: the same base model is trained with either SFT or DPO, with all other variables held constant (Section 4.3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to be 'A Code Efficiency Benchmark for Code Large Language Models' but the benchmark is Python-only, sourced exclusively from LeetCode algorithmic problems. The limitations section (Section 6) acknowledges runtime distribution simplification and contamination but does not bound claims to Python or algorithmic tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes DPO's superiority over SFT to catastrophic forgetting in SFT but does not explore other explanations (e.g., training data quality differences, convergence properties, or task-specific effects). No robustness checks against confounds are performed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines code efficiency explicitly as execution time (Section 1), explains why absolute runtime is insufficient (hardware variability), and justifies the Beyond metric as a runtime-percentile approach. The measurement (runtime percentile) matches the claim (code efficiency) with minimal proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 (Appendix A.6) lists all 10 models with exact HuggingFace model identifiers and links (e.g., 'deepseek-ai/deepseek-coder-1.3b-base'). These are precise, versioned model references. Note: GPT-4 used for test case generation is not versioned beyond 'GPT-4'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 9 (Appendix A.9) provides the full one-shot prompt template. The placeholders (<task_content>, <code_starter>, <code_completion>) map to specific dataset fields (pretty_content, prompt, solution), and the dataset is publicly available on HuggingFace, allowing full prompt reconstruction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 reports LoRA parameters (alpha=16, dropout=0.05, r=8), optimizer (AdamW), learning rates (1e-4 SFT, 5e-5 DPO), training steps (200 SFT, 500 DPO), β=0.1 for DPO, temperature=0.2 for generation, and K=5 for Beyond."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly for code completion without tools, retry logic, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the data filtering pipeline: tasks collected from LeetCode, filtered by number of solutions (≥2), restricted data structures (Python built-ins + TreeNode/ListNode), unique outputs verified across solutions, and Locality-Sensitive Hashing to deduplicate solutions."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations' and discusses the uniform runtime distribution assumption and data contamination concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 identifies specific threats: (1) the uniform runtime distribution assumption may not hold in practice, requiring more solution samples; (2) data contamination from LeetCode tasks appearing in training data compromises benchmark precision. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to Python, LeetCode-style algorithmic tasks, or open-source models. The limitations section discusses what could be improved but not what the current results cannot support."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset including tasks, solutions, test case generators, and prompts is released on HuggingFace (Section A.12). The code is on GitHub. Raw data is available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes data collection: tasks gathered from LeetCode's public problem set, solutions sampled from historical LeetCode submissions, test case generators created via GPT-4 and validated on LeetCode OJ. Filtering criteria and pipeline are documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is LeetCode, a well-known public programming platform. Solutions are from public historical submissions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: collect LeetCode tasks → filter by solution count (≥2) → filter by data structure constraints → verify unique outputs → deduplicate solutions via LSH → split into train (1,633) and eval (256). Table 6 shows final distribution."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. Authors are from NTU, NUS, and Sea AI Lab, but no funding is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Nanyang Technological University, National University of Singapore, and Sea AI Lab. None of the evaluated models are products of these organizations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so funder independence cannot be assessed. One author is from Sea AI Lab (a commercial entity), but since funding is undisclosed, independence cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the 10 evaluated models. This is critical because LeetCode tasks are publicly available and may appear in training corpora."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 acknowledges 'the presence of data contamination during the model training phase compromises the precision of the Mercury benchmark' but provides no analysis of whether specific tasks appeared in training data. No overlap detection is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode tasks have been publicly available for years before all evaluated models' training cutoffs. The paper acknowledges contamination risk in Section 6 and mentions plans to add new tasks dynamically but does not address contamination for the current evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is purely a benchmark evaluation of Code LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on code generation tasks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API spend, tokens consumed, or wall-clock time per example is reported. The paper evaluates 10 models with SFT and DPO variants but provides no cost information."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.3 mentions 'two A100-80G GPUs' and training steps (200 for SFT, 500 for DPO) but does not report total GPU hours, training time, or computational cost."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix Figure 11 shows bootstrapped Beyond distributions for 3 of 10 models (StarCoder2 3B/7B/15B) over 50 runs. However, the main results tables (2, 3) report single-run point estimates, and training experiments (SFT/DPO) do not report seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 caption states 'We sample one solution for each task to calculate pass score.' Table 3 caption states 'we sample 5 solutions for each task to calculate Beyond score.' K=5 is stated in Section 4.3."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Fixed hyperparameters are used (Section 4.3) with no mention of whether any search was conducted or why these specific values were chosen. The paper states 'extensive parameter optimization and prompt engineering were not pursued' but does not document the search budget for the chosen values."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters appear to be chosen without explanation. No justification is given for LoRA r=8, α=16, dropout=0.05, or the specific learning rates and training steps."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 10 models × 3 conditions (original, SFT, DPO) × 3 difficulty levels across multiple metrics without any correction for multiple comparisons. No statistical tests are applied at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the Mercury benchmark and evaluate models on it without acknowledging potential author-evaluation bias. No independent evaluation or discussion of this bias is provided."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 1.3B to 34B parameters with vastly different compute requirements. While model size trends are discussed informally (Section 4.4: 'larger models tend to provide better functional correctness'), performance is not explicitly reported as a function of matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses what Mercury measures (runtime efficiency via percentile ranking) and why absolute runtime is inadequate (hardware variability, Section 3). It explains the Beyond metric's design, demonstrates hardware-agnostic consistency (Appendix Figure 10), and discusses the uniform distribution assumption (Section 6)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are prompted directly for code completion with a uniform prompt template (Figure 9). The same prompting format is applied across all models."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode tasks and solutions have been publicly available for years before the evaluated models' training data collection. The paper does not discuss whether models may have seen these exact tasks and solutions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the prompt format (which includes LeetCode task descriptions) could leak information that models have memorized from training on LeetCode content."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Mercury-eval and Mercury-train tasks share structural similarities with the models' training data or with each other beyond the difficulty-balanced random split."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. Section 6 mentions contamination as a concern and proposes future dynamic task updates, but no canary strings, membership inference, or decontamination analysis is performed on the current evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "Leading Code LLMs achieve up to 65% on Pass (functional correctness) but less than 50% on Beyond (efficiency), indicating a significant gap between correctness and efficiency.",
       "evidence": "Table 2 shows deepseek-coder-33b-base achieves 65.0% Pass; Table 3 shows its Beyond is 48.53%. The Gap column in Table 3 quantifies this difference across all models (Section 4.4).",
       "supported": "strong"
     },
     {
       "claim": "DPO serves as a robust baseline for enhancing both code efficiency and functional correctness compared with SFT.",
       "evidence": "Tables 2 and 3 show DPO improves Pass for 8/10 models and improves or maintains Beyond for 6/10 models. SFT often hurts Pass on larger models (deepseek-coder-33b: -6.3, CodeLlama-34b: -4.9) and detracts most Beyond scores (Section 4.4).",
       "supported": "strong"
     },
     {
       "claim": "Mercury's difficulty stratification effectively differentiates Code LLM capabilities, with scores declining consistently from Easy to Hard.",
       "evidence": "Tables 2 and 3 show consistent performance decline across difficulty levels for all models. For example, deepseek-coder-33b-base: 70.9% Easy, 67.9% Medium, 62.3% Hard on Pass (Section 4.4).",
       "supported": "strong"
     },
     {
       "claim": "DPO substantially narrows the Gap between Beyond and Pass for models larger than 15B parameters.",
       "evidence": "Table 3 Gap column: starcoder2-15b Gap drops from 12.55 to 10.81 with DPO; deepseek-coder-33b from 18.50 to 5.79; CodeLlama-34b from 15.49 to 8.01. However, Gap widens for some smaller models.",
       "supported": "moderate"
     },
     {
       "claim": "The Beyond metric remains consistent across different hardware configurations, making it environment-agnostic.",
       "evidence": "Appendix Figure 10 shows Beyond scores for two models across three CPU tiers (micro, small, standard) with consistent results. However, this is demonstrated for only 2 models.",
       "supported": "moderate"
     },
     {
       "claim": "SFT may induce catastrophic forgetting in the pursuit of code efficiency for larger models.",
       "evidence": "Table 2 shows SFT decreases Pass for deepseek-coder-33b (-6.3) and CodeLlama-34b (-4.9). The paper attributes this to catastrophic forgetting (Section 4.4) but provides no direct evidence of forgetting beyond performance drops.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Severe LeetCode contamination risk",
       "detail": "All 1,889 tasks and their solutions come from LeetCode, which is widely scraped and included in code training datasets. Models like DeepSeek-Coder and StarCoder are trained on large code corpora that likely include LeetCode content. The paper acknowledges this in Section 6 but performs no contamination analysis, making all Pass scores potentially inflated."
     },
     {
       "flag": "No error bars on main results",
       "detail": "Tables 2 and 3 report point estimates without confidence intervals, standard deviations, or error bars. Appendix Figure 11 shows bootstrapped distributions for only 3 of 10 models. Without uncertainty quantification, reported differences between models and methods may not be statistically meaningful."
     },
     {
       "flag": "No statistical significance tests for comparative claims",
       "detail": "All claims about DPO outperforming SFT and model comparisons are based on comparing raw numbers without any statistical test. Given the variance shown in Figure 11 for the few models tested, some reported differences may be within noise."
     },
     {
       "flag": "Python-only scope presented as general benchmark",
       "detail": "Mercury uses only Python tasks from LeetCode algorithmic problems, but the title and framing position it as a general 'Code Efficiency Benchmark for Code Large Language Models.' Efficiency characteristics differ across languages and task types (e.g., systems programming vs. algorithms)."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and Codex, foundational code generation benchmark and model against which Mercury is positioned."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark for program synthesis, one of the supplementary benchmarks used in Mercury evaluation."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Large-scale code generation benchmark from online platforms, directly compared to Mercury in Table 1."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2024,
       "relevance": "EvalPlus benchmark that augments HumanEval/MBPP test coverage; motivates Mercury's approach to test case generation."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark addressing the same contamination concerns that affect Mercury's LeetCode-based evaluation."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source Code LLM evaluated in Mercury experiments."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source Code LLM family (7B, 13B, 34B) evaluated across all Mercury experiments."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Next-generation open-source Code LLM evaluated in Mercury experiments."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma"],
       "year": 2023,
       "arxiv_id": "2305.18290",
       "relevance": "DPO method used as the primary training baseline for improving code efficiency in Mercury experiments."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi"],
       "year": 2022,
       "relevance": "Competition-level code generation system relevant to understanding Code LLM capabilities on algorithmic tasks."
     },
     {
       "title": "SecurityEval dataset: mining vulnerability examples to evaluate machine learning-based code generation techniques",
       "authors": ["Mohammed Latif Siddiq", "Joanna CS Santos"],
       "year": 2022,
       "relevance": "Security-focused code generation benchmark, representing evaluation of code quality dimensions beyond correctness."
     },
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai", "Chengxi Li"],
       "year": 2023,
       "relevance": "Data science code generation benchmark evaluating LLMs on domain-specific tasks beyond algorithmic problems."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Mercury provides a usable benchmark and metric (Beyond) for researchers evaluating Code LLM efficiency, and the DPO finding is actionable for model trainers."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Confirms the expected intuition that Code LLMs prioritize correctness over efficiency; the DPO vs SFT finding is modestly surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper focuses on code efficiency measurement."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with existing claims or institutions."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and dataset publicly available on GitHub and HuggingFace; researchers can run the benchmark themselves."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from NTU, NUS, and Sea AI Lab — recognized institutions in Asia but not top-tier AI lab brand recognition globally."
     }
   }
 }