ai-research-survey

scan.json (28772B)

---

 {
   "paper": {
     "title": "THROWBENCH: Benchmarking LLMs by Predicting Runtime Exceptions",
     "authors": [
       "Julian Aron Prenner",
       "Romain Robbes"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2503.04241",
     "doi": "10.48550/arXiv.2503.04241"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "THROWBENCH is a multilingual benchmark of 2,466 programs (Python, Java, C#, Ruby) where LLMs must predict runtime exception types. Evaluating six open-weight code LLMs (7B–34B), performance is modest (F1 19–38%), with Qwen2.5 Coder (32B) performing best. Performance varies substantially by language and exception type: ZeroDivision is relatively easy across languages, while overflow and null-pointer exceptions remain very challenging. Model size does not strongly correlate with performance (ρ=0.39).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark and all model outputs are released at https://github.com/giganticode/throwbench/ as stated in the abstract and Section V."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark dataset of 2,466 programs is released at the same GitHub repository. The underlying data also comes from the publicly available RunBugRun and CodeNet datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Ollama and lists model names, sizes, and quantization levels (Table I), but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The approach is described at a high level (Section III-B) but lacks specific commands or a README-style guide for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results (Tables I and II) are reported as point estimates of precision, recall, and F1. No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares F1 scores across 6 models and claims performance differences (e.g., Qwen2.5 Coder is best) without any statistical significance tests. Only a Pearson correlation (ρ=0.39) for model size vs. performance is reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw F1 scores are reported. No formal effect sizes (Cohen's d, odds ratios) are given. The Pearson ρ=0.39 for model size vs. performance is the only measure of association."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The final benchmark size (2,466 programs) is stated but not justified. The filtering steps are described (Section III-A) but no rationale is given for why this number is sufficient to draw reliable conclusions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single inference runs. No standard deviations, variance across seeds, or any spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six different code LLMs are evaluated (Table I), providing comparison across models. However, no non-LLM baselines (random classifier, majority class, static analysis) are included."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All six models are recent (2024): CodeGemma, DeepSeek Coder 2, Codestral, Qwen2.5 Coder, DeepSeek Coder, and CodeLlama. These represent the contemporary open-weight code LLM landscape."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper, not a system with components to ablate. There is no proposed method with separable components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Precision, Recall, and F1 are reported (Table I), providing three complementary evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant for this benchmark — ground truth is determined objectively by program execution, not human judgment."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The models are pre-trained and used as-is without any fine-tuning on the benchmark data. The entire benchmark serves as a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides a detailed breakdown of F1 scores per programming language (4 languages) and per exception type (37 types across languages) for all 6 models."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV discusses exception types where all models struggle: overflow errors, null dereference (NullPointerException, NullReferenceException), and Ruby's RangeError. Specific patterns of failure are identified."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The overall framing is that performance is modest (19–38% F1). The paper reports that model size doesn't correlate strongly with performance (ρ=0.39), and that the largest model (CodeLlama 34B) performed worst."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'modest performance ranging from 19 to 38% (F1 score)' which matches Table I exactly. The claim that THROWBENCH consists of 'over 2,400 short user-written programs' matches the 2,466 count. The contamination-free claim is supported by the execution-based ground truth design."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It reports descriptive benchmark results without claiming that any factor causes performance differences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the six tested open-weight models (7B–34B) and four programming languages. The paper does not overclaim — it explicitly notes the benchmark is 'intended to be used in addition to other benchmarks' and frames results as specific to the evaluated models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for observed performance differences. For instance, it does not consider whether quantization levels, model training data composition, or prompt sensitivity could explain the results (except a brief CodeGemma quantization check)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures exception type prediction accuracy (F1) and claims to assess this exact capability. No proxy gap exists — the measurement matches the claimed outcome."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table I lists specific model names, sizes, and quantization levels (e.g., 'Qwen2.5 Coder Instr. 32B Q4_K_M', 'CodeGemma Instr. 7B FP16'). These identify specific model variants retrievable via Ollama."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 provides the full prompt text used in evaluation, including the exception type list, instructions, and placeholder format for code and input. Since the benchmark data is released, every prompt can be reconstructed."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No generation hyperparameters (temperature, top-p, max tokens) are reported. The paper states evaluation was done via Ollama's Python API but does not specify inference settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are prompted directly with a single-turn prompt (Figure 1)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A documents the benchmark creation pipeline: selecting buggy programs from RunBugRun in 4 languages, executing them, filtering for exception-throwing programs, subsampling for balance, discarding programs >53 LOC (95th percentile), and adding 10% correct programs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section V ('Conclusions') briefly mentions future directions (omitting answer choices, using chain-of-thought) but does not discuss limitations of the current work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as the effect of quantization on results, the representativeness of competition programming problems, or prompt sensitivity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not discuss limitations of testing only open-weight models, only competition-style programs, or only exception prediction as a measure of code understanding."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data and all model outputs are released at https://github.com/giganticode/throwbench/, enabling independent verification of reported results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the data source (RunBugRun, based on CodeNet), the selection criteria (buggy programs that throw exceptions), and the filtering process (language selection, balancing, length cutoff)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public datasets (RunBugRun/CodeNet programming contest submissions)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline steps are listed (Section III-A) but intermediate counts are not provided. The paper gives the starting source (RunBugRun) and final counts (2,466 total) but does not state how many programs existed at each filtering stage or how many were removed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments or grant information appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Free University of Bozen-Bolzano and Univ. Bordeaux/CNRS. Neither author is affiliated with the companies that produced the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence of the funder from outcomes cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interest statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff dates for the six evaluated models are not stated anywhere in the paper."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues that while the source code may appear in training data (from CodeNet/RunBugRun), the ground truth labels (exception types) were determined by execution and are not in training data: 'the code in training corpora hardly contains any information on their runtime behavior' (Section II)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Contamination is a central design consideration. The paper explicitly argues THROWBENCH is 'currently free from any contamination issues' because ground-truth answers were obtained through program execution, not from information that would appear in training data (Sections II and V)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or time-per-example is reported despite evaluating 2,466 examples across 6 models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No hardware specifications, GPU hours, or total compute budget is stated. The paper mentions using Ollama but does not describe the evaluation infrastructure."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. No mention of multiple random seeds or sensitivity analysis across runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It is unclear whether results are from a single run or averaged over multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Generation parameters (temperature, top-p) are not even stated, let alone any search over them."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Different quantization levels are used for different models (FP16 for CodeGemma, Q4_0 for others) without full justification for these choices. A brief CodeGemma quantization comparison is provided but not systematic across models."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors introduce the benchmark and evaluate models on it without discussing potential biases in their benchmark construction or evaluation methodology."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Different models use different quantization levels (FP16 vs Q4_0), affecting compute requirements, but performance is not reported as a function of compute budget. The CodeGemma quantization test is the only partial check."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims the benchmark tests 'code understanding' and 'runtime behavior' prediction but does not validate whether exception type prediction actually measures deep code comprehension versus surface-level pattern matching."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All models are evaluated with the same prompt structure (Figure 1) and the same infrastructure (Ollama). The evaluation is consistent across models, isolating model differences."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues that even if the source code (from CodeNet, published 2021) appears in training data, the ground-truth exception types were determined by execution and would not be in training data. This structurally addresses temporal leakage of labels."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompt provides the list of possible exception types as answer options, which constrains the output space. The paper notes this makes the task 'slightly easier' but does not analyze whether this constitutes feature leakage or how much it inflates scores."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether benchmark programs share structural similarities (e.g., same contest problems, similar code patterns from CodeNet) that could violate independence assumptions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes a conceptual argument about contamination-free design but does not apply any concrete leakage detection method (no canary strings, membership inference, or n-gram overlap analysis)."
       }
     }
   },
   "claims": [
     {
       "claim": "THROWBENCH is challenging for current state-of-the-art LLMs, with the best model achieving only 38% F1.",
       "evidence": "Table I shows Qwen2.5 Coder achieves the highest F1 of 38.2% among six models. Performance ranges from 19.0% (CodeLlama) to 38.2%.",
       "supported": "strong"
     },
     {
       "claim": "Model size and performance are not strongly correlated (Pearson's ρ=0.39).",
       "evidence": "Section IV: The 7B CodeGemma outperformed the 34B CodeLlama (21.1% vs 19.0% F1). The correlation across 6 models is ρ=0.39.",
       "supported": "moderate"
     },
     {
       "claim": "Performance varies substantially across programming languages, with Ruby lowest (22%) and Python highest (29%).",
       "evidence": "Table II micro-averages across models: Ruby 13.7–35.0%, Python 18.9–49.3% (from per-model averages). Averaged over all models: Ruby 22%, Python 29%.",
       "supported": "moderate"
     },
     {
       "claim": "ZeroDivision exceptions are recognized with relatively high success, while overflow and null dereference errors pose major challenges.",
       "evidence": "Table II shows DivideByZero/Arithmetic exceptions have green bands across models and languages. NullPointer (Java), NullReference (C#), StackOverflow, and Overflow exceptions show very low F1 scores across all models.",
       "supported": "strong"
     },
     {
       "claim": "THROWBENCH is free from contamination because ground-truth answers were determined through program execution.",
       "evidence": "Section II argues that 'the code in training corpora hardly contains any information on their runtime behavior.' Section V: 'ground-truth answers were obtained through program execution.'",
       "supported": "moderate"
     },
     {
       "claim": "Quantization (FP16 vs 4-bit) does not significantly affect performance for CodeGemma.",
       "evidence": "Section IV: 'We also evaluated CodeGemma with 4-bit quantization and found no significant performance difference (in fact, the 4-bit version performed 0.2% better).'",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "Performance differences between models are reported as raw F1 score comparisons without any significance tests. With 2,466 examples, small differences could be statistically meaningful or due to chance — this is impossible to assess without tests."
     },
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results appear to be single-run point estimates with no error bars, confidence intervals, or variance across runs. LLM outputs can be stochastic, and the stability of these results is unknown."
     },
     {
       "flag": "Only open-weight models tested",
       "detail": "The evaluation covers only 7B–34B open-weight models. No proprietary models (GPT-4, Claude, Gemini) are tested, limiting the generalizability of claims about LLM capabilities on this task."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations or threats-to-validity discussion. Issues such as the representativeness of competition code, prompt sensitivity, quantization confounds, and benchmark construct validity are not addressed."
     },
     {
       "flag": "Inconsistent quantization across models",
       "detail": "CodeGemma uses FP16 while all other models use 4-bit quantization. Although a brief check for CodeGemma shows minimal difference, this was not verified for the other five models, creating a potential confound."
     },
     {
       "flag": "Missing trivial baseline",
       "detail": "No random-guess or majority-class baseline is included. Without this, it is difficult to assess whether the 19–38% F1 scores reflect genuine code understanding or could be partially achieved by simple heuristics."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and Codex, foundational code LLM benchmark and model."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["J. Austin", "A. Odena", "M. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP, a widely-used code synthesis benchmark."
     },
     {
       "title": "MultiPL-E: A Scalable and Polyglot Approach to Benchmarking Neural Code Generation",
       "authors": ["F. Cassano", "J. Gouwar", "D. Nguyen"],
       "year": 2023,
       "doi": "10.1109/TSE.2023.3267446",
       "relevance": "Extended code benchmarks to 18+ programming languages, relevant to multilingual LLM evaluation."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["N. Jain", "K. Han", "A. Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-aware code benchmark using live programming competition problems."
     },
     {
       "title": "CRUXEval: A Benchmark for Code Reasoning, Understanding and Execution",
       "authors": ["A. Gu", "B. Rozière", "H. Leather", "A. Solar-Lezama", "G. Synnaeve", "S. I. Wang"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Closest related benchmark — tests runtime behavior prediction (input/output) for code understanding."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "relevance": "EvalPlus — enhanced evaluation rigor for code generation benchmarks with additional test cases."
     },
     {
       "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions",
       "authors": ["T. Y. Zhuo", "M. C. Vu", "J. Chim"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "Large-scale code generation benchmark evaluating diverse function calls."
     },
     {
       "title": "RunBugRun – An Executable Dataset for Automated Program Repair",
       "authors": ["J. A. Prenner", "R. Robbes"],
       "year": 2023,
       "arxiv_id": "2304.01102",
       "relevance": "Source dataset for THROWBENCH; executable bug dataset used for automated program repair research."
     },
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "LLM-based automated program repair with cost analysis."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Rozière", "J. Gehring", "F. Gloeckle"],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Open-weight code LLM evaluated in THROWBENCH; foundational model for code tasks."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming – The Rise of Code Intelligence",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Open-weight code LLM evaluated in THROWBENCH."
     },
     {
       "title": "Qwen2.5-Coder Technical Report",
       "authors": ["B. Hui", "J. Yang", "Z. Cui"],
       "year": 2024,
       "arxiv_id": "2409.12186",
       "relevance": "Best-performing model on THROWBENCH; recent open-weight code LLM."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": ["D. Hendrycks", "S. Basart", "S. Kadavath"],
       "year": 2021,
       "relevance": "Large code benchmark from programming competitions, similar sourcing to THROWBENCH."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "A benchmark that could be used to evaluate code LLMs, but not a directly usable tool or technique for practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "LLMs struggling with runtime exception prediction is mildly surprising given claims of code understanding, but not deeply contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or conflicts with other work."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Benchmark and code released on GitHub; anyone with Ollama and a GPU can run the evaluation."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors from Free University of Bozen-Bolzano and University of Bordeaux; no major AI lab branding."
     }
   }
 }