ai-research-survey

scan.json (31336B)

---

 {
   "paper": {
     "title": "Mercury: A Code Efficiency Benchmark for Code Large Language Models",
     "authors": [
       "Mingzhe Du",
       "Anh Tuan Luu",
       "Bin Ji",
       "Qian Liu",
       "See-Kiong Ng"
     ],
     "year": 2024,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2402.07844",
     "doi": "10.52202/079017-0529"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Mercury is the first code efficiency benchmark for Code LLMs, comprising 1,889 Python tasks from LeetCode with a novel Beyond metric that normalizes runtime via percentile ranking against historical solutions. Leading Code LLMs achieve ~65% Pass (functional correctness) but less than 50% Beyond (efficiency), revealing a substantial efficiency gap. DPO consistently outperforms SFT for improving code efficiency, especially in models above 15B parameters, while SFT often causes catastrophic forgetting in larger models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in the abstract footnote: https://github.com/Elfsong/Mercury. The paper states 'Our code and data are available on GitHub.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset hosted on HuggingFace (Appendix A.12): https://huggingface.co/datasets/Elfsong/Mercury. Croissant metadata also provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions specific libraries (Accelerate, DeepSpeed, BitsandBytes) and hardware (two A100-80G GPUs) in Section 4.3, but does not provide a requirements.txt, Dockerfile, or detailed version listing of dependencies sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released, the paper itself contains no step-by-step reproduction instructions, 'Reproducing Results' section, or specific commands to run. A reader must rely on the GitHub repository's documentation, which is not described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 2 and 3 report point estimates only (e.g., '65.0' Pass, '48.53' Beyond) with no confidence intervals or error bars. Appendix Figure 11 shows bootstrapped distributions for only 3 of 10 models, not integrated into the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used anywhere in the paper. Claims like 'DPO yields a stable enhancement' and comparisons between training methods rely solely on comparing raw numbers without any tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 3 report absolute scores and deltas showing the magnitude of change (e.g., 'deepseek-coder-33b-base: 65.0 → 73.4 (+8.4) with DPO'). The Gap metric in Table 3 explicitly measures the difference between Beyond and Pass, providing context for effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Mercury-eval uses 256 tasks with no justification for this sample size. No power analysis or reasoning for why 256 tasks (88 Easy, 81 Medium, 87 Hard per Table 6) is adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 2, 3) are single-run point estimates. Appendix Figure 11 shows bootstrapped variance for only 3 of 10 models (StarCoder2 3B/7B/15B). The remaining 7 models have no variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each of the 10 models is evaluated in three conditions: original, +SFT, and +DPO (Tables 2, 3). HumanEval and MBPP are included as auxiliary benchmarks for functional correctness comparison (Section 4.2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include DeepSeek-Coder (2023), StarCoder2 (2024), CodeQwen1.5 (2023/2024), and CodeLlama (2023), which were recent at the time of submission. Table 7 lists all models with HuggingFace links."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of the Mercury benchmark or Beyond metric design is performed. The comparison of SFT vs DPO is a baseline comparison, not an ablation of their system's components (e.g., removing the runtime normalization, varying K, or testing alternative percentile calculations)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: Pass (functional correctness), Beyond (efficiency-weighted pass), and Gap (difference between Pass and Beyond). Results are also reported on HumanEval and MBPP."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated via test case execution and runtime measurement. No human evaluation of code quality, readability, or efficiency is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Mercury-eval (256 tasks) is explicitly separated from Mercury-train (1,633 tasks) as described in Section 2 and Table 6. Training uses Mercury-train, evaluation uses Mercury-eval."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by difficulty level (Easy/Medium/Hard) in both Tables 2 and 3. Table 4 provides per-difficulty failure analysis across three error categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 'Failure Analysis' with Table 4 provides a detailed breakdown of Generation Errors, Execution Errors, and Test Case Errors across all models and difficulty levels."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that SFT 'diminishes functional correctness on the largest two Code LLMs' (Section 4.4) and 'detracts most Beyond scores from original models' (Table 3). DPO hurts smaller models' Gap scores. These negative results are prominently discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'leading Code LLMs can achieve 65% on Pass' matches Table 2 (deepseek-coder-33b-base: 65.0%). 'Less than 50% on Beyond' matches Table 3 (same model: 48.53%). 'DPO serves as a robust baseline' is supported by Tables 2-3 showing DPO improvements."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about DPO and SFT effects are made via controlled fine-tuning experiments: same base models, same data, different training methods. This controlled single-variable manipulation is adequate for the causal claims being made (e.g., 'DPO enhances code efficiency', 'SFT may induce catastrophic forgetting')."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title frames this as 'A Code Efficiency Benchmark for Code Large Language Models' broadly, but the benchmark is Python-only, LeetCode-only, and tests only 10 open-source models (no commercial models like GPT-4 or Claude). The limitations section does not bound generalization to Python or algorithmic tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, DPO uses 5 solution pairs vs SFT's single solution per task — the data volume difference is not discussed. The superior DPO results on large models could reflect compute allocation (500 vs 200 steps) rather than method superiority."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly defines code efficiency as runtime (Section 1: 'execution time is the performance bottleneck'), acknowledges that this excludes space complexity, and discusses why absolute runtime is an inadequate proxy (Section 3), motivating the percentile-based Beyond metric. The measurement matches the claim granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 (Appendix A.6) lists all 10 evaluated models with exact HuggingFace model IDs and links (e.g., 'deepseek-ai/deepseek-coder-1.3b-base', 'bigcode/starcoder2-3b'). However, GPT-4 used for test case generation (Section 2) lacks a version/snapshot."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix Figure 9 provides the full one-shot prompt template used for code generation, including all placeholders and what fills them (pretty_content, prompt, code_completion). The template is detailed enough to reconstruct every prompt sent to each model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 reports: LoRA alpha=16, dropout=0.05, r=8; AdamW optimizer; learning rates 1e-4 (SFT) and 5e-5 (DPO); 200 training steps (SFT) and 500 (DPO); β=0.1 for DPO; temperature=0.2; K=5 for Beyond. Hardware: two A100-80G GPUs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models generate code directly from prompts in a single pass."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the full filtering pipeline: LeetCode tasks → filter by number of solutions (≥2) → restrict to Python built-in types + TreeNode/ListNode → filter by unique outputs → difficulty stratification → random selection of 256 eval tasks. Criteria at each stage are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Limitations' is a dedicated section discussing two specific limitations: the uniform runtime distribution assumption and data contamination during model training."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses a specific threat: 'we measure code efficiency under the assumption that the code runtime is uniformly distributed. The simplification streamlines code efficiency evaluation via limited solution samples. However, the distribution of code runtime in real-world scenarios is more intricate.' This is specific to their methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section does not explicitly state what the results do NOT show. It does not bound the scope to Python, to algorithmic problems, or to open-source models. No explicit statements about what populations or settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset with tasks, solutions, test case generators, and prompts is available on HuggingFace (Appendix A.12). Croissant metadata is also provided for structured access."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes data collection from LeetCode in detail: public programming tasks, filtering criteria (solution count, data structures, unique outputs), Locality-Sensitive Hashing for deduplication, GPT-4 for test case generators with LeetCode OJ validation."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from LeetCode, a standard public programming platform. Historical solutions are scraped from public submissions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from LeetCode collection through filtering stages to final split is documented in Section 2. Table 6 shows the final data distribution (1,633 train / 256 eval). Each filter criterion is specified with the resulting effect on the dataset."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present in the paper. One author is from Sea AI Lab (a commercial entity), but no funding disclosure is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Nanyang Technological University, National University of Singapore, and Sea AI Lab. However, the paper does not evaluate Sea products, so no self-evaluation conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, making it impossible to assess funder independence. Sea AI Lab, the employer of one author, has commercial interest in code generation but is not evaluating its own products here."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "None of the 10 evaluated models have their training data cutoff dates stated in the paper. The models were trained on large code corpora that likely include LeetCode data, but no cutoff information is provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 mentions data contamination as a limitation ('the presence of data contamination during the model training phase compromises the precision') but performs no actual analysis of whether LeetCode problems appeared in the models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode problems have been publicly available for years and are widely used in code training corpora. The paper acknowledges contamination risk in Section 6 and proposes future dynamic updates, but does not quantify or address current contamination. No canary strings, membership inference, or decontamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. All evaluation is automated benchmark testing of Code LLMs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency per model, tokens consumed, or cost per task is reported. The paper evaluates 10 models × 3 conditions × 256 tasks without any cost information."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is stated (two A100-80G GPUs, Section 4.3) and training steps are given (200 SFT, 500 DPO), but total GPU hours, wall-clock training time, and total evaluation time are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix Figure 11 shows bootstrapped Beyond distributions across 50 runs for only 3 of 10 models (StarCoder2 3B/7B/15B). The remaining 7 models have no seed sensitivity analysis. Main results tables report single-configuration numbers."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'We sample one solution for each task to calculate pass score' (Table 2 caption) and 'we sample 5 solutions for each task to calculate Beyond score' (Table 3 caption, Section 4.3: K=5)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Specific hyperparameters are chosen (LoRA alpha=16, r=8, learning rates, etc.) but no search budget, search method, or number of configurations tried is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters appear to be fixed choices without explanation of how they were selected. No validation-based selection, no comparison of configurations, no justification for the specific values chosen."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed in the paper, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own benchmark and metric design without acknowledging potential bias. No independent evaluation or discussion of author-evaluation bias is present."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "DPO uses 500 training steps vs SFT's 200 steps, a 2.5x difference in training compute. This is not discussed as a potential confound when comparing DPO vs SFT performance. No performance-vs-compute curves are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether LeetCode algorithmic problems are a valid proxy for real-world code efficiency. LeetCode tasks are predominantly algorithmic (sorting, dynamic programming, etc.) and may not reflect efficiency concerns in production code (I/O, database queries, system design)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models generate code directly from prompts without agentic frameworks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode problems and solutions have been publicly available for years before the evaluated models were trained. The paper does not discuss whether models may have seen these exact problems during training, despite this being a significant concern."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, the prompt format or function signatures might cue models that have memorized LeetCode solutions."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether Mercury-train and Mercury-eval tasks are sufficiently independent. Both come from LeetCode and may share algorithmic patterns that models can exploit."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is used. Section 6 mentions plans to 'update our benchmark via our open-sourced data collection framework to import new tasks dynamically' as future mitigation, but no current detection (canary strings, membership inference, n-gram overlap) is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Leading Code LLMs can achieve ~65% on Pass (functional correctness) but less than 50% on Beyond (efficiency)",
       "evidence": "Table 2 shows deepseek-coder-33b-base achieves 65.0% Pass. Table 3 shows the same model achieves 48.53% Beyond. Section 4.4 discusses these results.",
       "supported": "strong"
     },
     {
       "claim": "DPO is a robust baseline for enhancing code efficiency compared with SFT",
       "evidence": "Table 3 shows DPO improves Beyond scores for most models above 6.7B (e.g., deepseek-coder-33b: 48.53→66.47, +17.94), while SFT often decreases Beyond scores (e.g., deepseek-coder-33b: 48.53→38.32, -10.21). Section 4.4.",
       "supported": "moderate"
     },
     {
       "claim": "SFT may induce catastrophic forgetting in large models when pursuing code efficiency",
       "evidence": "Table 2 shows SFT decreases Pass on deepseek-coder-33b-base (65.0→58.7, -6.3) and CodeLlama-34b-hf (52.4→47.5, -4.9). Section 4.4 states 'SFT may induce catastrophic forgetting in the pursuit of heightened code efficiency.'",
       "supported": "moderate"
     },
     {
       "claim": "The Beyond metric is environment-agnostic, remaining consistent across hardware configurations",
       "evidence": "Appendix Figure 10 shows Beyond scores for two models across three CPU tiers (micro/small/standard) remaining consistent. However, only 2 models and 3 configurations are tested.",
       "supported": "weak"
     },
     {
       "claim": "Larger models have greater capacity to improve efficiency while retaining correctness (narrower Gap)",
       "evidence": "Table 3 shows DPO substantially narrows Gap in models larger than 15B (e.g., deepseek-coder-33b Gap: 18.50→5.79, CodeLlama-34b Gap: 15.49→8.01), while Gap widens in smaller models. Section 4.4.",
       "supported": "moderate"
     },
     {
       "claim": "Mercury's difficulty stratification effectively probes the upper limits of Code LLM capabilities",
       "evidence": "Table 2 shows consistent Pass score decline from Easy to Hard across all models (e.g., deepseek-coder-6.7b: 69.3 Easy → 56.1 Hard). Section 4.4.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Severe LeetCode contamination risk",
       "detail": "All 10 evaluated models were trained on internet-scale code corpora that almost certainly include LeetCode problems and solutions. The paper acknowledges contamination in Section 6 but performs no analysis to quantify it. Performance numbers may reflect memorization rather than code generation capability, directly undermining the benchmark's validity."
     },
     {
       "flag": "No error bars on main results",
       "detail": "Tables 2 and 3 report point estimates for all 30 model configurations (10 models × 3 conditions) without any uncertainty quantification. Appendix Figure 11 shows variance for only 3 of 10 models. The reported differences (e.g., +1.0 vs -2.2) may not be meaningful given unmeasured variance."
     },
     {
       "flag": "Unequal training compute between SFT and DPO",
       "detail": "DPO uses 500 training steps while SFT uses only 200 steps (Section 4.3), a 2.5x difference. DPO also uses 5 solution pairs per task vs SFT's single solution, providing more training signal. These compute and data differences are not acknowledged when concluding that DPO is superior to SFT."
     },
     {
       "flag": "Uniform runtime distribution assumption acknowledged but not validated",
       "detail": "The Beyond metric assumes code runtimes are uniformly distributed between min and max historical solutions (Section 6). Real runtime distributions are often multimodal (reflecting O(n), O(n log n), O(n²) clusters). This could systematically bias the efficiency metric, but no validation of the assumption is performed."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval and Codex, foundational code generation benchmark and model used as a comparison point in Mercury."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for code generation evaluation, used as a secondary benchmark in Mercury experiments."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Introduces EvalPlus with augmented test cases for code generation, directly motivating Mercury's test case coverage approach."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Large-scale coding challenge benchmark from online platforms, a key comparison point for Mercury's benchmark design."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Addresses benchmark contamination in code evaluation, directly relevant to Mercury's acknowledged contamination limitations."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell"],
       "year": 2023,
       "arxiv_id": "2305.18290",
       "relevance": "DPO method used as Mercury's primary training baseline for improving code efficiency in LLMs."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open code LLM family (7B/13B/34B) evaluated as key baselines on Mercury benchmark."
     },
     {
       "title": "Starcoder 2 and the stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "StarCoder2 model family (3B/7B/15B) evaluated on Mercury, representing the latest open code LLMs at time of publication."
     },
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "First-generation StarCoder model, predecessor to StarCoder2 models evaluated in Mercury."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Competition-level code generation system relevant to evaluating code LLM capabilities on challenging programming tasks."
     },
     {
       "title": "SecurityEval dataset: mining vulnerability examples to evaluate machine learning-based code generation techniques",
       "authors": ["Mohammed Latif Siddiq", "Joanna CS Santos"],
       "year": 2022,
       "relevance": "Security-focused code generation benchmark, representing the broadening of code evaluation beyond functional correctness."
     },
     {
       "title": "Learning to improve code efficiency",
       "authors": ["Binghong Chen", "Daniel Tarlow", "Kevin Swersky"],
       "year": 2022,
       "arxiv_id": "2208.05297",
       "relevance": "Prior work on improving code efficiency with ML, directly relevant to Mercury's efficiency optimization goal."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Benchmark and dataset are released and usable by practitioners evaluating code LLM efficiency, though efficiency optimization via DPO requires ML training expertise."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that LLMs generate functionally correct but inefficient code is somewhat expected; the 65% Pass vs <50% Beyond gap quantifies a known intuition rather than overturning it."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised. The paper is about code efficiency, not vulnerabilities or risks."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict. The paper introduces a straightforward new benchmark without challenging existing claims or institutions."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code on GitHub and dataset on HuggingFace allow researchers to run the benchmark, though setting up the sandbox environment requires effort."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from NTU, NUS, and Sea AI Lab are respectable institutions but not top-tier brand names in the LLM space."
     }
   }
 }