ai-research-survey

scan.json (20148B)

---

 {
   "paper": {
     "title": "CRUXEVAL-X: A Benchmark for Multilingual Code Reasoning, Understanding and Execution",
     "authors": ["Ruiyang Xu", "Jialun Cao", "Yaojie Lu", "Ming Wen", "Hongyu Lin", "Xianpei Han", "Ben He", "Shing-Chi Cheung", "Le Sun"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2408.13001"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the benchmark is available and references a GitHub repository. The abstract and construction pipeline describe publicly available data."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset (CRUXEVAL-X with 19 languages, 500 aligned entries, 19K test cases) is described as publicly available. Built on the public CruxEval dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions using various LLM APIs and models but does not specify library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The construction pipeline is described at a high level but no README or runnable scripts are referenced."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2 are point estimates (Pass@1 percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., Deepseekcoder-V2 is better than GPT-4o-mini) based solely on comparing numbers without statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute Pass@1 scores with baselines for context (e.g., 'phi-1, trained solely on Python, scored 11.8% Pass@1 on Python input prediction and 23.6% on Perl'), providing magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '500 entries are sufficient to distinguish the effectiveness of the LLMs' but provides no power analysis or formal justification for this claim."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 with greedy decoding, so single-run results are reported. No variance across multiple runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates 24 LLMs across multiple categories (general, multilingual code, instruction-tuned, single-language), providing extensive comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4o, Deepseekcoder-V2, Qwen2, Llama-3, and other contemporary models from 2024."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows ablation of the construction pipeline (w/o Iter vs w/ Iter). Section 4.1 analyzes key factors affecting code reasoning. The data bias analysis in Appendix A also serves as an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two tasks are evaluated: input reasoning and output reasoning, both measured with Pass@1. Additionally, syntactic correctness rate and semantic correctness rate are analyzed."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of LLM outputs is performed. Evaluation is entirely automated via test case pass/fail."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is used as a test set for evaluating LLMs. The paper also checks for data contamination against Stack v2, finding only 0.8% overlap."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-language breakdowns for all 19 programming languages across all 24 models. Figure 3 breaks down by code complexity factors."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 provides case studies of failures (e.g., phi-1.5 failing on Racket due to distinct syntax). Appendix C discusses per-language translation difficulties."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that single-LLM translation has low success rates, that some languages (Racket) have consistently worst results, and that the initial overlap was only 333 questions across all PLs."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about 19 PLs, 600+ subjects per language, 19K tests, correlation between language pairs, and cross-language generalization (34.4% Pass@1) are all supported by Tables 1-2 and Figures 3-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'improvement in NL reasoning positively impacts code reasoning' (Section 4.2.3) based on comparing phi-1 to phi-1.5, but this is observational — the models differ in many ways beyond NL training data."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested models and languages. The title says 'Multilingual' which is accurate for 19 languages. The paper acknowledges that translated code cannot reflect language-specific features (Limitations section)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed correlations or cross-language generalization. For instance, the phi-1.5 cross-language finding could be due to shared tokenization or natural language overlap rather than true code reasoning transfer."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names: 'GPT-4o', 'GPT-3.5-Turbo', 'GPT-4o-mini' without snapshot dates or API versions. Open-source models are specified by name and parameter count but not exact checkpoint versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper references prompts in Appendix D and shows prompt templates in Figure 2. The evaluation task format is clearly specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is set to 0 with greedy decoding for evaluation (Section 3.1). Construction pipeline also specifies temperature 0 for repair (Section 2.3.2)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly for code reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full construction pipeline is documented in Section 2 with three steps: function signature translation, test suite translation, and iterative generation & repair. Table 1 shows counts at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is a dedicated 'Limitations' section discussing three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section discusses specific threats: model-generated data may introduce bias, translation cannot guarantee perfect accuracy (500 out of 800 aligned), and translated code cannot reflect language-specific features."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states that translated codes from Python cannot reflect language-specific features, that this is a trade-off for alignment, and that 500 entries were deemed sufficient though derived from 800."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (code subjects and test cases in 19 languages) is released publicly."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the full construction pipeline: starting from CruxEval (800 Python subjects), translating through three steps, with specific LLMs (GPT-3.5-Turbo, DeepseekCoder-33B, GPT-4o) used at each stage."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is the existing CruxEval benchmark, a standard public dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 documents the pipeline from 800 subjects through Step I, Step II, w/o Iter, and w/ Iter stages with exact counts per language at each stage. Section 2.3.3 documents the overlap-based refinement (333 → 462 → 500)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is visible in the paper text."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Chinese Academy of Sciences, HKUST, Huazhong University. None of the authors appear to be affiliated with the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 24 evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 (Quality Analysis - Data Leakage) compares the benchmark against Stack v2 (67.5TB of GitHub data), finding only 0.8% overlap, indicating minimal leakage risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination: the benchmark is constructed via automated translation rather than sourcing from contest websites, and the 0.8% overlap with Stack v2 is reported. The paper also notes contest website solutions suffer from contamination (Section 1)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs or API costs are reported for evaluating 24 models across 19 languages, despite this being a substantial evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated for either benchmark construction or evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "CRUXEVAL-X contains 19 programming languages with at least 600 subjects each and 19K content-consistent test cases total.",
       "evidence": "Table 1 shows the construction pipeline results with per-language counts, all reaching 600+ after iterative generation and repair.",
       "supported": "strong"
     },
     {
       "claim": "Even GPT-4o can only achieve Pass@1 around 70% on CRUXEVAL-X, indicating the benchmark is challenging.",
       "evidence": "Table 2 shows GPT-4o achieving 64.6-75.4% across languages for input reasoning and 70.8-77.6% for output reasoning.",
       "supported": "strong"
     },
     {
       "claim": "A model trained solely on Python (phi-1) can achieve at most 34.4% Pass@1 in other languages, demonstrating cross-language generalization.",
       "evidence": "Table 2 shows phi-1 and phi-1.5 results across all 19 languages. phi-1.5 achieves up to 34.4% in shell scripting for input reasoning despite being trained only on Python.",
       "supported": "moderate"
     },
     {
       "claim": "JavaScript and TypeScript show the strongest positive correlation among all PL pairs (0.87 and 0.91 on both tasks).",
       "evidence": "Figure 5 shows the correlation heatmap calculated via cosine similarity of LLM performance vectors.",
       "supported": "strong"
     },
     {
       "claim": "The benchmark has minimal data leakage risk, with only 0.8% overlap with Stack v2.",
       "evidence": "Section 2.4 reports comparison against Stack v2 (67.5TB of GitHub data).",
       "supported": "moderate"
     },
     {
       "claim": "Improvement in natural language reasoning positively impacts code reasoning, as shown by phi-1 to phi-1.5 comparison.",
       "evidence": "Section 4.2.3 compares phi-1 (Python-only) to phi-1.5 (Python + NL) showing 10.7% vs 21.7% average input reasoning.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CRUXEVAL-X is a multilingual code reasoning benchmark spanning 19 programming languages, constructed via an automated test-guided translation pipeline from the Python-only CruxEval dataset. Evaluation of 24 LLMs reveals that code reasoning capabilities are highly correlated across programming languages, with JavaScript-TypeScript showing the strongest correlation and Racket the weakest. Models trained solely on Python (phi-1, phi-1.5) demonstrate unexpected cross-language generalization, achieving 16-26% success rates in unseen languages. The automated pipeline successfully translated 500 of 800 original subjects to all 19 languages with 0.8% overlap against Stack v2 training data.",
   "red_flags": [
     {
       "flag": "No statistical rigor on comparisons",
       "detail": "All comparative claims between models are based on point estimates without confidence intervals, error bars, or significance tests, despite using greedy decoding (single run)."
     },
     {
       "flag": "Weak causal claim about NL-to-code transfer",
       "detail": "The claim that NL training improves code reasoning (phi-1 vs phi-1.5) is based on comparing two models that differ in multiple dimensions, not just NL data."
     },
     {
       "flag": "No cost reporting despite large-scale evaluation",
       "detail": "Evaluating 24 models across 19 languages and constructing benchmarks using GPT-3.5, DeepseekCoder, and GPT-4o involves substantial costs that are not reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, the foundational code generation benchmark that CRUXEVAL-X addresses biases in."
     },
     {
       "title": "CruxEval: A benchmark for code reasoning, understanding and execution",
       "authors": ["Alex Gu", "Baptiste Rozière"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "The Python-only code reasoning benchmark that CRUXEVAL-X extends to 19 languages."
     },
     {
       "title": "MultiPL-E: A scalable and polyglot approach to benchmarking neural code generation",
       "authors": ["Federico Cassano"],
       "year": 2023,
       "relevance": "Provided multilingual translation rules adopted by CRUXEVAL-X for test suite translation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E Jimenez"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major LLM code benchmark for real-world software engineering tasks."
     },
     {
       "title": "McEval: Massively multilingual code evaluation",
       "authors": ["Linzheng Chai"],
       "year": 2024,
       "arxiv_id": "2406.07436",
       "relevance": "Competing multilingual code benchmark using human annotation at $12K cost."
     },
     {
       "title": "Concerned with data contamination? Assessing countermeasures in code language model",
       "authors": ["Jialun Cao"],
       "year": 2024,
       "arxiv_id": "2403.16898",
       "relevance": "Addresses benchmark contamination risks in code LLM evaluation."
     },
     {
       "title": "Rectifier: Code translation with corrector via LLMs",
       "authors": ["Xin Yin"],
       "year": 2024,
       "arxiv_id": "2407.07472",
       "relevance": "LLM-based code translation approach showing low success rates that CRUXEVAL-X's pipeline improves upon."
     },
     {
       "title": "Deepseek-coder: When the large language model meets programming",
       "authors": ["Daya Guo"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the key code LLMs evaluated in the benchmark."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu"],
       "year": 2024,
       "relevance": "Addresses rigor in evaluating LLM-generated code, directly relevant to benchmark methodology."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM family evaluated across multiple variants in the benchmark."
     }
   ]
 }