ai-research-survey

scan.json (24783B)

---

 {
   "paper": {
     "title": "Collu-Bench: A Benchmark for Predicting Language Model Hallucinations in Code",
     "authors": ["Nan Jiang", "Qi Li", "Lin Tan", "Tianyi Zhang"],
     "year": 2024,
     "venue": "Preprint Under Review",
     "arxiv_id": "2410.09997"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL is provided in the paper. The paper provides a HuggingFace dataset link (https://huggingface.co/datasets/lt-asset/collu-bench) but no code repository for the benchmark construction pipeline or prediction experiments."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace at https://huggingface.co/datasets/lt-asset/collu-bench, stated explicitly in Section 1: 'Availability: https://huggingface.co/datasets/lt-asset/collu-bench.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment specification is provided. The paper mentions using scikit-learn and tree-sitter but does not provide version details or a reproducible environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided to reproduce the benchmark construction or prediction experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2-5 report only point accuracy estimates with no confidence intervals, error bars, or uncertainty measures despite using 5-fold cross-validation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares multiple ML and NN models (e.g., 'RF produces higher accuracy than SVC, AB, GB, and MLP') but no statistical significance tests are reported to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as raw accuracy percentages (e.g., 33.09%, 33.15%) without effect sizes, baseline context for improvement magnitude, or standardized measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 13,234 instances but no justification is given for why this sample size is sufficient. For the human evaluation of error rate, only 100 samples were checked with no power analysis or justification for that number."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Although 5-fold cross-validation is used, no standard deviation, variance, or spread measures across folds are reported. Only single accuracy numbers are shown in Tables 2-5."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: five ML methods (SVC, AB, RF, GB, MLP) and four NN architectures (CNN, GRU, LSTM, Transformer) are compared against each other, plus a simple string matching baseline (57% vs 86% agreement) for the data collection pipeline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The prediction baselines are all traditional ML/NN methods (Random Forest, SVC, LSTM, CNN, GRU, Transformer). No contemporary LLM-based or specialized hallucination detection methods are used as baselines, even though the paper's related work discusses methods like RARR and Chain-of-Verification."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper conducts implicit ablation through feature analysis: it tests different data split settings (All-in-one, One-per-dataset, One-per-LLM) and examines different feature types (log probabilities, token types). The three data split configurations serve as ablations over data organization."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy is reported as an evaluation metric. No other metrics such as precision, recall, F1, or AUC are used despite this being a prediction task where class imbalance is acknowledged (3:1 downsampling ratio mentioned in Appendix A.5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes a human evaluation where two developers reviewed 100 randomly selected samples to assess the accuracy of the automated hallucination token localization pipeline, finding 86% agreement."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 5-fold cross-validation with 80% training and 20% test data per fold, as described in Section 5."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (MBPP, HumanEval, HumanEval-Java, Defects4J, SWE-bench) in Tables 2 and 4, and per LLM in Tables 3 and 5. Token type analysis is also broken down by category in Table 1."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses the 14 error cases in hallucination labeling and identifies the cause (missing equivalent canonical solutions). Section 6 (Limitation) also discusses failure modes of the automated pipeline."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that overall accuracy is only 22.03-33.15%, explicitly framing this as a challenging task. GPT-4o-mini's unique patterns that cause cross-model prediction to fail (~0% accuracy) are also reported as negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about 13,234 instances, 11 LLMs, five datasets, 22.03-33.15% accuracy, and the benchmark's features (log probabilities, token types, execution feedback) are all supported by the experimental results in Sections 4 and 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are modest and well-supported. Claims like 'LLMs are less confident when hallucinating' are supported by probability distribution analysis (Figure 3). The paper avoids strong causal language, mostly using observational framing."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is careful about generalization. Section 6 (Limitation) explicitly acknowledges limits in the range of selected LLMs and datasets. The title and claims are specifically about the benchmark, not about hallucination in general."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the observation that 'LLMs are less confident when hallucinating' could have alternative explanations (e.g., lower confidence may correlate with harder problems). No such alternatives are considered."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Model families and sizes are specified (e.g., DeepSeekCoder-1.3B/6.7B/33B, StarCoder2-3B/7B/15B, CodeLlama-7B/13B/34B, Llama3-8B, GPT-4o-mini) with release dates in Table 7, but specific model version identifiers or snapshot dates for GPT-4o-mini are not provided. For open-source models, the exact checkpoint versions are not specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full few-shot prompt templates are provided in Appendix A.2 (Figures 4 and 5), including the system prompt, five-shot examples with actual code, and the test sample format for both code generation and automated program repair tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.5 reports hyperparameters for ML models (default scikit-learn), NN architectures (number of layers, hidden dimensions, batch size, epochs, optimizer), and generation parameters (temperature 0.8 for sampling, greedy decoding for generation) in Sections 3.1 and 3.3."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper uses direct LLM prompting for code generation/repair and standard ML/NN training for prediction."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the full data preprocessing pipeline in detail: diverse canonical solution collection via sampling (Section 3.1), program normalization using tree-sitter (Section 3.1), hallucination token localization (Section 3.3), and additional signal collection (Section 3.4). Filtering criteria and the downsampling ratio (3:1) are also documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is a dedicated 'Limitation' section that discusses two specific limitations: errors in automated hallucination token indices and the limited range of LLMs and datasets."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 identifies specific threats: (1) the automated pipeline's non-perfect accuracy (14% error rate on 100-sample check), (2) limited to 11 LLMs and 5 datasets which may not represent all patterns. These are specific to this study rather than generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what was not tested: 'There exist lots of different LLMs and code generation or program repair datasets, we select the set of state-of-the-art, widely-used LLMs.' The paper also clarifies its focus on token-level first hallucination prediction rather than all hallucinations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full benchmark dataset is available on HuggingFace (https://huggingface.co/datasets/lt-asset/collu-bench), which includes the raw LLM-generated code, log probabilities, token types, and hallucination indices."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes the data collection in detail: datasets selected (Section 3.2), LLMs used (Section 3.2), generation procedure with greedy decoding and few-shot prompting (Section 3.3), and hallucination token localization through normalization and comparison (Sections 3.1, 3.3)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited as subjects. The two developers who reviewed 100 samples in Section 4.2 were for data validation, not as study participants. The data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from dataset selection to final benchmark is documented: Section 3.1 (code equivalence handling and normalization), Section 3.2 (dataset and LLM selection), Section 3.3 (generation and hallucination localization), Section 3.4 (additional signal collection). Table 8 shows instance counts per LLM and dataset."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section states: 'This research was supported in part by NSF 1901242 and 2006688 and a CFI fund.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from Purdue University, USA, as stated in the author information. They are not affiliated with any of the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are NSF (National Science Foundation) and CFI, which are government/academic funding agencies with no financial stake in the evaluation of specific LLMs."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists release dates for the LLMs in Table 7 but does not state their training data cutoff dates. For GPT-4o-mini, even the training data and process are explicitly noted as 'unknown.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses data leakage for Defects4J by including HumanEval-Java, which was 'transformed from HumanEval to overcome the data leakage threat of Defects4J' (Section 3.2). However, this is acknowledged as an issue rather than comprehensively addressed for all benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published before most models' training cutoffs and could be in training data. While HumanEval-Java is used to mitigate Defects4J leakage, the paper does not discuss contamination risk for HumanEval and MBPP themselves, despite these being well-known contamination risks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted. The two developers who reviewed 100 samples were performing data validation, not participating as research subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite using GPT-4o-mini API and running 11 LLMs for generation. The cost of sampling 100 programs per problem per LLM is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget, GPU hours, or hardware specifications are mentioned despite the significant computational cost of running 11 LLMs across 5 datasets with extensive sampling (100 samples per problem per LLM)."
       }
     }
   },
   "claims": [
     {
       "claim": "Collu-Bench includes 13,234 code hallucination instances collected from five datasets and 11 diverse LLMs.",
       "evidence": "Table 8 in Appendix A.4 provides detailed counts per LLM and dataset, totaling 13,234 instances.",
       "supported": "strong"
     },
     {
       "claim": "LLMs are less confident when hallucinating, as hallucinated tokens have lower probability than correct tokens.",
       "evidence": "Figure 3 shows probability distributions of correct vs. hallucinated tokens across all LLMs, datasets, and token types (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "LLMs are more likely to hallucinate when generating certain types of tokens such as Keyword, Identifier, and Type Identifier.",
       "evidence": "Table 1 (Section 4.1) shows hallucination rates by token type, with Keyword having the highest rate (8.24-15.35%) across most LLMs.",
       "supported": "strong"
     },
     {
       "claim": "Random forest produces the highest overall accuracy of 33.09% for per-token prediction; LSTM produces the highest overall accuracy of 33.15% for per-sample prediction.",
       "evidence": "Tables 2 and 4 (Sections 5.1 and 5.2) show these results under the All-in-one setting.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4o-mini has the most unique hallucination pattern compared to other LLMs.",
       "evidence": "Tables 3 and 5 show that predictors trained on other LLMs perform near 0% on GPT-4o-mini's data, and vice versa. This is consistent across both per-token (RF) and per-sample (LSTM) predictions.",
       "supported": "strong"
     },
     {
       "claim": "The proposed pipeline of sampling diverse canonical solutions and program normalization reduces the error rate from 43% to 14%.",
       "evidence": "Section 4.2 reports that simple string/token matching achieves only 57/100 agreement with human developers, while the proposed pipeline achieves 86/100.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Collu-Bench is a benchmark of 13,234 code hallucination instances from 11 LLMs across 5 datasets, designed for predicting and localizing the first hallucinated token in LLM-generated code. Key findings include that LLMs show lower confidence (log probability) when hallucinating and are more error-prone on Keyword tokens. Preliminary prediction experiments using ML and NN approaches achieve only 22-33% accuracy, indicating the task is challenging. GPT-4o-mini exhibits distinctly different hallucination patterns from open-source models, making cross-model prediction ineffective.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty reported despite cross-validation",
       "detail": "All results use 5-fold cross-validation but only report single accuracy numbers with no standard deviation across folds. This makes it impossible to assess the reliability of comparisons between models."
     },
     {
       "flag": "Single metric evaluation",
       "detail": "Only accuracy is used to evaluate hallucination prediction despite acknowledged class imbalance (3:1 downsampling ratio). Precision, recall, and F1 would provide a more complete picture of model performance."
     },
     {
       "flag": "Incomplete contamination analysis",
       "detail": "HumanEval (2021) and MBPP (2021) are well-known contamination risks for models released in 2023-2024, but the paper does not discuss whether models may have memorized these benchmarks' solutions, which could affect what constitutes a 'hallucination.'"
     },
     {
       "flag": "No compute cost reported for extensive sampling",
       "detail": "The pipeline samples 100 programs per problem per LLM to collect canonical solutions, which is computationally expensive, but no cost or compute budget is reported."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan"],
       "year": 2024,
       "relevance": "Major benchmark for project-level automated program repair used as one of the five datasets in Collu-Bench."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Prominent agentic approach to software engineering referenced as a key application of LLMs in code."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2404.05427",
       "relevance": "Autonomous program repair agent relevant to understanding LLM capabilities in code modification."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduced the HumanEval benchmark, a foundational dataset used in Collu-Bench and widely used for code generation evaluation."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduced EvalPlus for rigorous LLM code evaluation, used as the test harness in Collu-Bench."
     },
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00125",
       "relevance": "Evaluates LLMs for automated program repair at ICSE, directly relevant to understanding LLM coding capabilities and limitations."
     },
     {
       "title": "A Deep Dive into Large Language Models for Automated Bug Localization and Repair",
       "authors": ["Soneya Binta Hossain", "Nan Jiang", "Qiang Zhou"],
       "year": 2024,
       "relevance": "Studies LLMs for bug localization and repair; pointer network approach from this work is adopted for per-sample prediction in Collu-Bench."
     },
     {
       "title": "Exploring and Evaluating Hallucinations in LLM-Powered Code Generation",
       "authors": ["Fang Liu"],
       "year": 2024,
       "arxiv_id": "2404.00971",
       "relevance": "HalluCode benchmark for code hallucination taxonomy, a key related work for understanding LLM hallucinations in code."
     },
     {
       "title": "CodeHalu: Code Hallucinations in LLMs Driven by Execution-Based Verification",
       "authors": ["Yuchen Tian", "Weixiang Yan", "Qian Yang"],
       "year": 2024,
       "arxiv_id": "2405.00253",
       "relevance": "Another code hallucination benchmark using execution-based verification, directly comparable to Collu-Bench."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "One of the main LLM families evaluated in Collu-Bench, relevant to understanding code-specialized LLM capabilities."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Another major code LLM family evaluated in Collu-Bench."
     },
     {
       "title": "Can LLM Replace Stack Overflow? A Study on Robustness and Reliability of Large Language Model Code Generation",
       "authors": ["Li Zhong", "Zilong Wang"],
       "year": 2024,
       "doi": "10.1609/aaai.v38i19.30185",
       "relevance": "Studies API misuse in LLM code generation, relevant to understanding code hallucination patterns."
     }
   ]
 }