ai-research-survey

scan.json (25697B)

---

 {
   "paper": {
     "title": "Assessing Correctness in LLM-Based Code Generation via Uncertainty Estimation",
     "authors": ["Arindam Sharma", "Cristina David"],
     "year": 2025,
     "venue": "arXiv preprint (under review)",
     "arxiv_id": "2502.11620"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. There is no mention of code availability."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses LiveCodeBench, a publicly available benchmark (cited as [20], with arXiv link https://arxiv.org/abs/2403.07974). The dataset is publicly accessible and was not modified."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Ubuntu 20.04.5 LTS (Focal Fossa) with one NVIDIA A100 GPU (80GB)' and the Crosshair tool [7], but does not provide a requirements.txt, Dockerfile, or detailed dependency/library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The experimental setup section describes the methodology but does not give executable instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 2, 3, and 4 report point estimates (Pearson correlation coefficients, accuracy percentages) without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports p-values alongside Pearson correlation coefficients in Table 2 and Table 4, and highlights statistically significant results. This constitutes significance testing for their comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pearson correlation coefficients are reported (e.g., -0.56, -0.51) alongside p-values, providing the magnitude of the effect. Abstention metrics in Table 3 report accuracy percentages, false positive rates (e.g., 0.01%), and false negative rates (e.g., 11.9%-20.3%), giving context for the magnitude of improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 831 problems from LiveCodeBench split by difficulty (Easy: 239, Medium: 332, Hard: 260) but does not justify why this sample size is adequate for the correlation analyses, nor is any power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. The 2-fold cross-validation for abstention policies is mentioned but results are reported as single numbers without variance across folds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares symbolic clustering against multiple baselines: NLG embeddings (DeBERTa), code embeddings (GPT text-embedding-ada-002), CodeBLEU, and raw LLM probability. SE-CODEEMBED and LLM-PROBABILITY serve as explicit baselines in Table 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are drawn from recent state-of-the-art NLG uncertainty estimation methods: Kuhn et al. (ICLR 2023), Abbasi et al. (NeurIPS 2024), and OpenAI's text-embedding-ada-002 embeddings. These are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 functions as a systematic ablation: it tests the same uncertainty estimation frameworks (SE, MI, CC) with different clustering methods (NLG, CODEEMBED, CODEBLEU, SYMB) and distribution assumptions (norm vs. uniform), isolating the contribution of symbolic clustering and token probabilities."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Pearson correlation (for uncertainty-correctness relationship) and abstention policy metrics (accuracy, false positive rate, false negative rate) as distinct evaluation dimensions."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates code correctness via automated test suites from the LiveCodeBench benchmark. Human evaluation is not relevant to the claims, which concern statistical correlation between uncertainty estimates and automated correctness scores."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For abstention policies, the paper uses a 50/50 train/validation split with 2-fold cross-validation, alternating between training and validation sets. The correlation analysis in Table 2 uses the full dataset without tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by difficulty level (Easy, Medium, Hard) for all three models in Tables 2, 3, and 4. Table 1 provides model-specific statistics per difficulty level."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases: NLG, CODEEMBED, and CODEBLEU clustering methods all fail to achieve significant correlation. The motivating example in Section 3 illustrates a specific failure where embedding-based and CodeBLEU-based clustering incorrectly separates equivalent programs. The Limitations section discusses false positives from bounded symbolic execution."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative results are prominently reported: SE-NLG, SE-CODEEMBED, SE-CODEBLEU, MI-NLG, MI-CODEEMBED, MI-CODEBLEU, CC-CODEEMBED, and LLM-PROBABILITY all fail to achieve statistically significant correlation with correctness (Table 2). The paper centers on explaining and overcoming these negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims: (1) existing NLG uncertainty techniques fail for code — supported by Table 2 (non-significant p-values for NLG/CODEEMBED/CODEBLEU variants); (2) symbolic clustering restores predictive power — supported by significant correlations for SYMB variants; (3) uniform distribution performs comparably — supported by comparing SYMBNORM vs. SYMBUNIFORM; (4) false positive rate below 0.02% — supported by Table 3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claim is that semantic clustering is the key factor enabling correlation between uncertainty and correctness. This is justified through controlled ablations: the same uncertainty framework (SE, MI) is tested with different clustering methods while holding other variables constant (Table 2). The ablation design supports the causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to address 'LLM-Based Code Generation' broadly, but results are only on three models and one benchmark (LiveCodeBench, which is Python-focused competitive programming problems). The paper does not bound its claims to this specific setting. Section 7 (Conclusion) refers to 'code generation' generally without qualification."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why symbolic clustering works (e.g., could the bounded equivalence checking be introducing systematic biases? Could the timeout-based assumption of equivalence inflate cluster merging?). The Limitations section discusses bounded symbolic execution but does not consider alternative explanations for the observed correlations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'gpt-3.5-turbo-instruct', 'DeepSeek-R1', and 'claude-3-7-sonnet' but does not provide snapshot dates or API version identifiers for any of these. 'gpt-3.5-turbo-instruct' has no version suffix. 'DeepSeek-R1' and 'claude-3-7-sonnet' are marketing names without version pinning."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We use the natural language description in the query provided to the LLM' (Section 5) but does not provide the actual prompt text, system instructions, or any wrapping around the problem description. The iterative prompting approach (Appendix B) is described conceptually but the actual prompts are not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports: M=5 sampled responses, 2 iterations for iterative prompting, Crosshair per-condition timeout of 5s, per-path timeout of 5s, overall equivalence timeout of 10s, correctness test case timeout of 5s, and 90% correctness score threshold. However, LLM sampling parameters (temperature, top-p) are not explicitly stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. The LLMs are queried directly for code generation; the symbolic execution pipeline is a post-processing analysis step, not an agentic scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents the data pipeline: LiveCodeBench problems are used as prompts, 5 responses are sampled per problem, the top-ranked response is tested against test cases, correctness scores are computed as percentage of passing tests. For abstention, downsampling of incorrect responses and 2-fold cross-validation are described (Appendix C.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' subsection appears at the end of Section 5.1, discussing bounded symbolic execution and the limited set of NLG techniques evaluated."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) bounded symbolic execution may cause false positives in equivalence checking because behavioral differences may only manifest beyond the unrolling limit; (2) only two NLG-based techniques (Kuhn et al. and Abbasi et al.) are evaluated, mitigated by their prominence in the literature."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound its claims to Python, competitive programming, or the specific models tested. The limitations discuss technical constraints of symbolic execution but do not state, for instance, that results may not generalize to other programming languages, real-world codebases, or non-competitive-programming tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (generated code samples, uncertainty scores, correctness scores per problem) is released. Only aggregated statistics in tables are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes data collection: 831 problems from LiveCodeBench, categorized by difficulty, with natural language descriptions used as prompts and I/O test cases for evaluation. Table 1 provides detailed statistics per model and difficulty level."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is a standard public benchmark (LiveCodeBench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: (1) sample 5 responses per problem from each LLM, (2) compute token-level log-probabilities or apply uniform distribution, (3) perform symbolic execution for clustering, (4) compute uncertainty scores, (5) compute correctness scores via test cases. Timeouts and thresholds are specified. The downsampling procedure for GPT-3.5-turbo-instruct is documented in Appendix C.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or sponsors are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are affiliated with the University of Bristol, clearly stated on the first page. No evaluated product is from Bristol, so there is no product-affiliation conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. The absence of any funding disclosure makes this NO rather than NA — funding may exist but is undisclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the three models (gpt-3.5-turbo-instruct, DeepSeek-R1, claude-3-7-sonnet). This is relevant because they evaluate model performance on LiveCodeBench."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination by choosing LiveCodeBench specifically because it is described as a 'contamination-free benchmark' (Section 5). LiveCodeBench is designed with temporally-gated problems to avoid train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "LiveCodeBench is explicitly chosen as a 'contamination-free benchmark for code-related tasks' (Section 5). The benchmark is designed to mitigate contamination by sourcing problems that postdate model training cutoffs."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The approach requires multiple LLM calls (5 samples per problem, plus iterative prompting for MI variants) and symbolic execution per pair of snippets. No API costs, token counts, or wall-clock times per problem are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using an NVIDIA A100 GPU (80GB) but does not state total GPU hours, API costs, or overall computational budget. Only per-pair symbolic execution timeouts (10s) are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Existing NLG-based uncertainty estimation techniques (semantic entropy and mutual information) fail to exhibit statistically significant correlation with correctness when applied to code generation, even with code-specific embeddings or CodeBLEU.",
       "evidence": "Table 2 shows non-significant p-values (all >0.05) for SE-NLG, SE-CODEEMBED, SE-CODEBLEU, MI-NLG, MI-CODEEMBED, MI-CODEBLEU, and CC-CODEEMBED across all difficulty levels and both models.",
       "supported": "strong"
     },
     {
       "claim": "Symbolic clustering via symbolic execution enables NLG-based uncertainty techniques to achieve statistically significant correlation with correctness in code generation.",
       "evidence": "Table 2 shows all SYMB variants (SE-SYMBNORM, SE-SYMBUNIFORM, MI-SYMBNORM, MI-SYMBUNIFORM, CC-SYMB) achieve p-values well below 0.05 with Pearson coefficients ranging from -0.19 to -0.56.",
       "supported": "strong"
     },
     {
       "claim": "Token-level log-probabilities are not essential; uncertainty estimates assuming uniform distributions perform comparably to those using actual log-probabilities.",
       "evidence": "Table 2 shows SE-SYMBUNIFORM and MI-SYMBUNIFORM achieve comparable or near-comparable correlation coefficients to SE-SYMBNORM and MI-SYMBNORM. For example, on Easy problems with gpt-3.5-turbo-instruct, SE-SYMBNORM achieves -0.56 vs. SE-SYMBUNIFORM at -0.51.",
       "supported": "strong"
     },
     {
       "claim": "Abstention policies based on symbolic clustering achieve false positive rates below 0.02%.",
       "evidence": "Table 3 shows SE-SYMBNORM (0.01%), SE-SYMBUNIFORM (0.01%), and CC-SYMB (0.02% and 0.01%) false positive rates for gpt-3.5-turbo-instruct and DeepSeek-R1.",
       "supported": "strong"
     },
     {
       "claim": "The symbolic cluster count (CC-SYMB), despite its simplicity, correlates with correctness and performs on par with more complex information-theoretic approaches.",
       "evidence": "Table 2 shows CC-SYMB achieving correlation coefficients comparable to SE-SYMBNORM (e.g., -0.51 vs. -0.56 on Easy, -0.44 vs. -0.52 on Medium for gpt-3.5-turbo-instruct). Table 3 shows comparable abstention metrics.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper demonstrates that existing NLG uncertainty estimation techniques fail to correlate with correctness in code generation because learned embeddings and heuristic metrics cannot capture functional equivalence of programs. Symbolic clustering via symbolic execution restores this correlation, and a simple symbolic cluster count metric performs comparably to complex information-theoretic approaches. Abstention policies based on symbolic clustering achieve near-zero false positive rates (below 0.02%) with false negative rates of 11.9-20.3%, tested across three LLMs and 831 LiveCodeBench problems.",
   "red_flags": [
     {
       "flag": "No code or raw data released",
       "detail": "Despite proposing a novel technique (symbolic clustering for uncertainty estimation), no implementation code, generated samples, or raw experimental data are released. This prevents independent verification of results."
     },
     {
       "flag": "Missing LLM sampling parameters",
       "detail": "Temperature, top-p, and other sampling parameters for the three LLMs are not reported. These significantly affect the diversity of generated code and thus the clustering results. The paper mentions 'sampling M code snippets' but does not specify the sampling configuration."
     },
     {
       "flag": "No variance or reproducibility measures",
       "detail": "All results are reported as single numbers without standard deviations or confidence intervals. The 2-fold cross-validation for abstention is mentioned but fold-level results are not shown. There is no indication of whether results are stable across random seeds or sampling runs."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The paper claims to address 'LLM-Based Code Generation' in the title but only evaluates on Python competitive programming problems from LiveCodeBench. No discussion of whether symbolic clustering generalizes to other programming languages, real-world codebases, or non-algorithmic tasks."
     }
   ],
   "cited_papers": [
     {
       "title": "Semantic uncertainty: Linguistic invariances for uncertainty estimation in natural language generation",
       "authors": ["Lorenz Kuhn", "Yarin Gal", "Sebastian Farquhar"],
       "year": 2023,
       "relevance": "Foundational work on semantic entropy for NLG uncertainty estimation, which this paper adapts to code generation."
     },
     {
       "title": "To believe or not to believe your LLM: Iterative prompting for estimating epistemic uncertainty",
       "authors": ["Yasin Abbasi-Yadkori", "Ilja Kuzborskij", "András György", "Csaba Szepesvari"],
       "year": 2024,
       "relevance": "State-of-the-art mutual information approach for epistemic uncertainty estimation in LLMs, adapted in this paper for code."
     },
     {
       "title": "Detecting hallucinations in large language models using semantic entropy",
       "authors": ["Sebastian Farquhar", "Jannik Kossen", "Lorenz Kuhn", "Yarin Gal"],
       "year": 2024,
       "relevance": "Extension of semantic entropy to hallucination detection in LLMs, closely related to correctness estimation."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code generation benchmark used for evaluation in this paper."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "State-of-the-art open-source reasoning model evaluated in this paper for code generation."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Large-scale evaluation of LLM-generated code correctness using external oracles, relevant to the broader code quality assessment space."
     },
     {
       "title": "Look before you leap: An exploratory study of uncertainty analysis for large language models",
       "authors": ["Yuheng Huang", "Jiayang Song", "Zhijie Wang"],
       "year": 2025,
       "relevance": "Explores lightweight uncertainty proxies (e.g., CodeBLEU) for code generation and finds them ineffective, directly motivating this paper's approach."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "doi": "10.1145/3650212.3680384",
       "relevance": "Autonomous program repair system relevant to the broader agentic AI coding landscape."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Zhiyu Fan", "Xiang Gao", "Martin Mirchev", "Abhik Roychoudhury", "Shin Hwei Tan"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00128",
       "relevance": "LLM-based automated program repair, relevant to code quality and correctness assessment."
     },
     {
       "title": "Exploring and evaluating hallucinations in LLM-powered code generation",
       "authors": ["Fang Liu", "Yang Liu", "Lin Shi"],
       "year": 2024,
       "relevance": "Establishes taxonomy of hallucinations in LLM-generated code, directly related to correctness estimation."
     },
     {
       "title": "De-Hallucinator: Iterative grounding for LLM-based code completion",
       "authors": ["Aryaz Eghbali", "Michael Pradel"],
       "year": 2024,
       "relevance": "Technique for improving LLM code reliability through iterative grounding, related to oracle-free quality estimation."
     }
   ]
 }