ai-research-survey

scan.json (21681B)

---

 {
   "paper": {
     "title": "CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution",
     "authors": ["Alex Gu", "Baptiste Rozière", "Hugh Leather", "Armando Solar-Lezama", "Gabriel Synnaeve", "Sida I. Wang"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2401.03065"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references code and data via links (e.g., 'checking the numsteps variable in our code here' in Appendix A.3), and provides HuggingFace checkpoints in Appendix B. The benchmark is publicly available."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark of 800 Python functions with input-output pairs is released. The paper references public availability and HuggingFace checkpoints for models in Appendix B."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions HuggingFace model checkpoints but not the software environment needed to reproduce experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README with commands are described in the paper. Prompts are provided in appendices, but there is no guide for replicating the full evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap confidence intervals are reported. Fig. 2 shows (2.5, 97.5) percentile whiskers and (25, 75) percentile boxes from 10000 bootstrap samples. Section 3.3 discusses noise measurement."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Paired bootstrap tests are used to compare models. Section 4 states 'if the median bar clears the whisker in Fig. 2, then the difference actually holds with >97.5% probability under paired bootstrap.' Section 3.3 reports significance at alpha=0.05 level."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute performance differences are reported with context. E.g., 'GPT-4 with CoT achieves pass@1 of 75% and 81%' vs 'Code Llama 34B achieves pass@1 of 50% and 46%'. Fig. 1 shows performance differences between model pairs."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 explicitly justifies the 800-sample benchmark size via bootstrap analysis, measuring noise at various dataset sizes (200, 400, 800, 1600) and showing 800 is sufficient to detect meaningful differences."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 reports type 1 noise ~1.5% and type 2 noise ~0.2% at N=800. Bootstrap percentile ranges are shown in figures. The Limitations section notes standard deviations of ~1.5% for individual models."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Twenty models are evaluated including base models (StarCoder, Code Llama, DeepSeek), instruction-tuned/distilled models (WizardCoder, Phind, Phi), and proprietary models (GPT-3.5, GPT-4), providing extensive baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4, Code Llama, DeepSeek Coder, and other models that were state-of-the-art at time of writing (late 2023/early 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates CoT vs direct prediction (Sec 5.1), fine-tuning effects (Sec 5.2), and analyzes the impact of dataset size (Sec 3.3). Confusion matrices in Fig. 7 show where CoT helps vs hurts."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Both pass@1 (T=0.2) and pass@5 (T=0.8) are reported. Two separate tasks (input prediction CRUXEval-I and output prediction CRUXEval-O) are evaluated."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The authors manually checked the 22 problems where GPT-4 scored 0/10 on both tasks to verify they pass the criteria of being simple problems (Sec 6). This constitutes manual inspection of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 800-sample benchmark is the test set. For fine-tuning experiments (Sec 5.2), training and testing accuracy are reported separately (Fig. 8), with decontamination applied."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model family, size, base vs instruct, input vs output prediction. Correlation matrices (Fig. 5) show per-model-pair correlations. Dataset statistics show distributions by code length, step count."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides detailed qualitative analysis of GPT-4 failures with specific code examples (Listings 3-10), categorizing failure types (string concatenation errors, misleading variable names, counting limitations)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: distilled models (WizardCoder, Phind, Phi) don't improve on CRUXEval despite HumanEval gains. CoT does not help Code Llama 13B or GPT-3.5 on input prediction. Fine-tuning plateaus at ~70%."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about GPT-4 CoT achieving 75%/81%, Code Llama 34B achieving 50%/46%, and distilled models not showing the same improvements are all supported by results in Sections 4-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are limited and well-supported. Claims like 'CoT improves performance' are backed by controlled comparisons (same model with/without CoT). The paper is careful to say 'suggests' for correlational findings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section (Sec 7) explicitly bounds generalization: 'our benchmark only measures the input and output prediction accuracy of relatively simple and self-contained Python functions distilled from a single model.' Prompt sensitivity is acknowledged."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses alternative explanations: prompt sensitivity (Mizrahi et al. 2023), information loss from pass@k, potential contamination in fine-tuning, and that interpreters could replace model execution. Sec 6 considers tokenization as an alternative explanation for GPT-4 failures."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Model sizes are specified (e.g., 'Code Llama 34B', 'StarCoder 15.5B') and HuggingFace checkpoints listed in Appendix B, but GPT-3.5 and GPT-4 are referenced without API version or snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "All prompts are provided in Appendix D.2 (evaluation prompts) and D.3 (CoT prompts). The benchmark generation prompt is shown in Listing 11 and Appendix A.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings are reported: T=0.2 for pass@1, T=0.8 for pass@5. Number of samples: N=100 for non-GPT models, N=10 for GPT models. Generation temperature T=1 for benchmark construction."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated via direct prompting and CoT prompting without any agent framework."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the filtering pipeline in detail: compile-time filters (argument usage, code length 75-300 chars, syntax), runtime filters (no floats, time limits), and quality filters (no imports, deterministic, no side effects)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations and Future Work' provides extensive discussion across multiple subsections covering benchmark scope, prompt sensitivity, pass@k limitations, fine-tuning caveats, and more."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: prompt sensitivity may favor certain models (Sec 7), standard deviations of ~1.5% mean close model pairs cannot be distinguished (e.g., Phind 47.9% vs CodeLlama 46.5%), weak decontamination in fine-tuning may inflate scores."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states what was NOT tested: longer/harder code, other programming languages, open-domain code, different prompting techniques, different temperatures. States the benchmark only measures 'relatively simple and self-contained Python functions.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (800 functions with input-output pairs) is publicly released. The paper provides links to code and data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data collection process: Code Llama 34B generates candidate functions using 69 standard library functions as seeds, with 25 few-shot prompt combinations, producing 102,000 functions and 489,306 input-output pairs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The benchmark is synthetically generated from a language model."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: generation (Sec 3.1, 102K functions, 489K pairs) → filtering (Sec 3.2, compile-time, runtime, quality filters) → statistical sizing (Sec 3.3, ~1700 → 800 samples via bootstrap analysis)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 Acknowledgements discloses NSF Graduate Research Fellowship Grant No. 2141064 and NSF Grant CCF:2217064 with Intel Corporation."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: MIT CSAIL and Meta AI. The lead author's work was primarily done during a Meta AI internship."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSF and Intel, neither of which has a direct stake in the benchmark results. Meta AI is an affiliation but the paper evaluates multiple companies' models, not just Meta's."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Authors from Meta AI evaluate Code Llama (a Meta model) but this potential conflict is not explicitly acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. The paper does not discuss when models' training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is newly generated by Code Llama 34B, making overlap with existing training data unlikely. Section 5.2 explicitly addresses decontamination for fine-tuning experiments. The novel generation approach inherently mitigates this risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is newly created (not pre-existing public data), so contamination risk is inherently low. The generate-and-filter approach using Code Llama 34B produces novel functions. Section 5.2 discusses decontamination for fine-tuning."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API spend, or tokens consumed are reported despite using GPT-3.5 and GPT-4 APIs with N=10 samples across 800 problems."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or hardware specifications are reported for running the 20 model evaluations or the fine-tuning experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 with CoT achieves pass@1 of 74.8% on input prediction and 81.9% on output prediction, the best among all evaluated models.",
       "evidence": "Section 5.1 and Table 3 in the Appendix report these numbers. Fig. 2 shows GPT-4 CoT as the top-performing configuration.",
       "supported": "strong"
     },
     {
       "claim": "Models distilled on GPT-4 data (WizardCoder, Phind, Phi) show large HumanEval improvements over base models but do not show similar improvements on CRUXEval.",
       "evidence": "Section 5 and Fig. 3 show WizardCoder 34B improves HumanEval from 53.7% to 73.2% over Code Llama 34B but shows no significant CRUXEval improvement. Statistical significance confirmed via paired bootstrap.",
       "supported": "strong"
     },
     {
       "claim": "800 benchmark examples are sufficient to reliably distinguish model performance differences.",
       "evidence": "Section 3.3 provides bootstrap analysis at various dataset sizes (Fig. 1), showing that at N=800, type 1 noise is ~1.5% per model and paired differences are statistically significant at alpha=0.05 for many model pairs.",
       "supported": "strong"
     },
     {
       "claim": "CoT improves output prediction more than input prediction, and does not help all models equally.",
       "evidence": "Section 5.1 shows CoT does not improve Code Llama 13B or GPT-3.5 on input prediction. Fig. 4b and confusion matrices in Fig. 7 show CoT can hurt individual samples for weaker models.",
       "supported": "strong"
     },
     {
       "claim": "Simple fine-tuning on execution data improves CRUXEval performance but plateaus at under 70%, far from solving the benchmark.",
       "evidence": "Section 5.2 and Fig. 8 show training vs test accuracy curves for Code Llama 34B fine-tuned on 140K samples, with test accuracy plateauing.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CRUXEval is a benchmark of 800 Python functions testing code reasoning via input and output prediction tasks. Evaluating 20 code LMs reveals that GPT-4 with CoT achieves 75%/82% pass@1 but no model comes close to solving the benchmark. Models distilled on GPT-4 data (WizardCoder, Phind, Phi) show inflated HumanEval scores without corresponding CRUXEval improvements, suggesting HumanEval-focused training does not transfer to code understanding. The benchmark construction uses a rigorous generate-and-filter pipeline with bootstrap-justified sample sizes.",
   "red_flags": [
     {
       "flag": "Meta authors evaluating Meta model",
       "detail": "Four of six authors are from Meta AI, and Code Llama (a Meta model) is both used to generate the benchmark and is one of the evaluated models. This potential conflict is not explicitly acknowledged."
     },
     {
       "flag": "No GPT model version specified",
       "detail": "GPT-3.5 and GPT-4 are used without specifying API versions or snapshot dates. Model behavior changes across versions, making these results non-reproducible for the proprietary models."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the primary code generation benchmark that CRUXEval complements and compares against."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark and early work on measuring execution ability of code LMs."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM software engineering capabilities on real-world tasks."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used both to generate the benchmark and as a key evaluated model."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "HumanEval+ augments HumanEval with better test cases, related to benchmark quality concerns."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang"],
       "year": 2022,
       "relevance": "Foundational CoT prompting method evaluated as an improvement technique on CRUXEval."
     },
     {
       "title": "The false promise of imitating proprietary LLMs",
       "authors": ["Arnav Gudibande", "Eric Wallace"],
       "year": 2023,
       "arxiv_id": "2305.15717",
       "relevance": "Questions whether distilling from proprietary models leads to genuine capability improvements, directly relevant to CRUXEval's findings on distilled models."
     },
     {
       "title": "WizardCoder: Empowering code large language models with Evol-Instruct",
       "authors": ["Ziyang Luo"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Key distilled model showing HumanEval gains without CRUXEval improvements, central to the paper's findings."
     },
     {
       "title": "Reasoning or reciting? Exploring the capabilities and limitations of language models through counterfactual tasks",
       "authors": ["Zhaofeng Wu"],
       "year": 2023,
       "arxiv_id": "2307.02477",
       "relevance": "Examines LLM reasoning failures through counterfactual tasks, related to CRUXEval's investigation of code reasoning failures."
     },
     {
       "title": "Show your work: Scratchpads for intermediate computation with language models",
       "authors": ["Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2112.00114",
       "relevance": "Introduces scratchpad approach for training LMs on execution traces, directly related to CRUXEval's code execution evaluation."
     }
   ]
 }