ai-research-survey

scan.json (24660B)

---

 {
   "paper": {
     "title": "Reducing Hallucinations in LLM-Generated Code via Semantic Triangulation",
     "authors": ["Yihan Dai", "Sijie Liang", "Haotian Xu", "Peichu Xie", "Sergey Mechtaev"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.12288",
     "doi": "10.48550/arXiv.2511.12288"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code, data, and mechanized proofs are available at http://github.com/msv-lab/just-tri-it/' (Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states all code and data are available at the GitHub link. They also use public benchmarks (LiveCodeBench and CodeElo)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of environment specifications, requirements.txt, Dockerfile, or dependency versions in the paper text."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no instructions on how to run experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Figures 9, 10, 11, and 12 report point estimates (percentages) without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims just-tri-it outperforms baselines but no statistical significance tests are reported. Comparisons are based on raw percentage differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in context, e.g., 'increases reliability of generated code by 21%' (abstract), '29% on average' (RQ1), percentage improvements with baseline values throughout Section 6."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The sample size of 30 programs is justified via entropy stabilization analysis (Appendix C, Figure 14), showing semantic entropy plateaus around N=30."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be single-run."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines compared: Plurality, Majority0.5, CodeT, Postcondition, Syntactic, OffByOne (Section 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include CodeT (2023), postcondition autoformalization, and majority voting approaches. These are contemporary and relevant methods for sample consensus."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 6.4) provides an ablation study showing contributions of individual triangulation schemes (FWD-INV, FWD-SINV, ENUM-SINV) and the impact of removing bijective mapping constraints."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: Reliable Accuracy, Overall Accuracy, Abstention Rate, Abstention F1-score, and conditional probability of correctness under agreement (Figure 8)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via test suites and correctness judges."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "LiveCodeBench v6 uses problems published Feb-Apr 2025 to ensure contamination-free evaluation. The angelic value threshold was tuned on MBPP (a separate benchmark), not on the test benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per-method (Figures 9, 10, 11), per triangulation scheme (Figure 12), and separately for exact (LCB) vs inexact (CEI) problems."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Threats to Validity) discusses where the approach fails: boolean predicates with intractable domains, optimization problems, and errors in LLM-generated transformations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Syntactic and OffByOne transformations 'showed no significant difference compared to plurality' (Section 6.2 footnote). The ablation shows 23% drop when bijective mapping is removed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 21% reliable accuracy improvement and selection at 0.14 probability are supported by results in Section 6 (Figures 9, 11, and the motivating example)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'triangulation increases confidence' are justified through theoretical proofs (Propositions 4.2, 4.3) and controlled ablation experiments (RQ4) that isolate components."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly bounds generalization: 'limited generalization beyond programming-contest problems', mentions that programs interacting with users/environment may need different methods, and notes results may vary across LLM architectures."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses alternative explanations: transformation errors from LLM prompting, the possibility that reasoning models may shift program distribution structures, and that not all problems are naturally invertible."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper carefully defines its metrics (reliable accuracy, overall accuracy, abstention F1) in Figure 8 and measures exactly what it claims — correctness of code selection and abstention decisions, not broader 'code quality' claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o' and 'DeepSeek-V3' without specifying exact API versions or snapshot dates. Marketing names without version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows a prompt template fragment for SINV transformation. The paper states prompts are used for problem transformation and code generation, and the full prompts would be in the released code repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature set to 1.0, sample size N=30, angelic threshold T=1/3 (tuned on MBPP pilot study). Reported in Section 6 and Appendix C."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The just-tri-it pipeline is described in detail: problem transformation via LLM prompting, sampling, RANSAC consensus, cascading triangulation, and the STREAM meta-method (Section 5)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "CodeElo-Inexact subset selection is documented: 'we systematically selected 31 inexact problems from CodeElo, choosing those where the acceptability of multiple solutions is explicitly stated' (Section 6). LCB v6 segment selection is also described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides a dedicated discussion of limitations across multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: limited to programming-contest problems, boolean predicates have intractable inversions, optimization problems are hard to invert, LLM-generated transformations may introduce errors (Section 7)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states it focuses on 'generating individual functions... rather than whole projects interacting with users and environment' (Assumption 2), and bounds its scope to pure deterministic functions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All code, data, and mechanized proofs are available at http://github.com/msv-lab/just-tri-it/' suggesting raw data is available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection described: LCB v6 (175 problems, Feb-Apr 2025), CEI (31 inexact problems from CodeElo with manually written correctness judges verified against CodeForces submissions). 100 trials for probability estimation (Figure 2)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data comes from standard benchmarks (LiveCodeBench, CodeElo)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: sample 30 programs per problem, transform problems via LLM, sample witnesses, check hyperproperties, apply RANSAC consensus, with the angelic/demonic value handling detailed in Section 5.3 and Figure 7."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. The acknowledgements (Section 10) thank individuals for feedback but do not mention funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Peking University, Beijing Forestry University, and Independent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for GPT-4o or DeepSeek-V3."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses this by using LiveCodeBench v6 (problems from Feb-Apr 2025) 'to ensure contamination-free evaluation' (Section 6)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Using the most recent LCB segment (published Feb-Apr 2025) specifically mitigates contamination. The paper also notes that 'traditional code generation benchmarks HumanEval and MBPP are reaching their saturation' (Section 6)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The method samples 30 programs plus additional witnesses per problem via GPT-4o/DeepSeek-V3 API calls, but no cost, latency, or token consumption is reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware details are stated despite significant API usage (30 samples + witnesses for 175+31 problems across 2 models)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as single-run point estimates. No seed sensitivity analysis or multi-seed results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "100 trials for probability estimation (Figure 2) and 30 samples per problem are explicitly stated (Section 6, Appendix C)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The angelic threshold T=1/3 was tuned on MBPP but no search budget (how many values tried) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Configuration selection is justified: sample size via entropy analysis (Figure 14), threshold via pilot study on MBPP (a separate benchmark, not the test set), temperature at default 1.0 with justification for challenging benchmarks."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons across methods, benchmarks, and models without any correction for multiple testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines themselves (CodeT, Postcondition, etc.) without acknowledging potential bias in their implementations of competitor methods."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "just-tri-it requires significantly more compute than baselines (multiple problem transformations, additional sampling for witnesses) but this cost difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses construct validity by noting LCB excludes inexact problems and introducing CEI to fill this gap. It also discusses the limitation that programming-contest problems may not generalize (Section 7)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates its own tool (just-tri-it) as a post-generation selection method, not comparing models across different scaffolds. The scaffold IS the thing being tested."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "LCB v6 uses problems from Feb-Apr 2025 to ensure they postdate model training. The paper explicitly states this is 'to ensure contamination-free evaluation' (Section 6)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information through context or whether prompts contain implicit hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether LCB or CEI problems share structural similarities with training data problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "Temporal splitting is used (recent LCB segment) but no concrete leakage detection method (canary strings, membership inference, etc.) is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Semantic triangulation yields plausibility witnesses that increase the probability of correctness by 29% over the strongest baseline on average.",
       "evidence": "Figure 10 shows conditional probability of correctness under agreement: ENUM-SINV 91.4%, FWD-INV 88.7%, FWD-SINV 88.3% vs. Postcondition 63.6% and Another solution 59.2% on 175 LCB problems with GPT-4o.",
       "supported": "strong"
     },
     {
       "claim": "just-tri-it increases reliable accuracy by 21% compared to Majority0.5 (probability threshold ≥0.5).",
       "evidence": "Figure 9: GPT-4o LCB reliable accuracy 60.0% (just-tri-it) vs 52.9% (Majority0.5); DeepSeek-V3 76.0% vs 69.8%. Average improvement ~11% on LCB, higher on CEI.",
       "supported": "moderate"
     },
     {
       "claim": "just-tri-it can select correct solutions with sampling probabilities as low as 0.14.",
       "evidence": "The motivating example (Figure 3, Section 2.3) demonstrates selection of a correct solution with probability 0.07 in the presence of a dominant error with probability 0.23 for CodeElo problem 1999D.",
       "supported": "moderate"
     },
     {
       "claim": "On inexact problems, just-tri-it achieves 76% higher reliable accuracy than Majority0.5 on average.",
       "evidence": "Figure 11: GPT-4o CEI 44.4% vs 40.0%; DeepSeek-V3 30.0% vs 12.5%. The 76% average improvement is mainly driven by the DeepSeek-V3 gap.",
       "supported": "moderate"
     },
     {
       "claim": "Semantic triangulation provably outperforms plurality voting under stochastic parrot and correlated errors assumptions.",
       "evidence": "Propositions 4.2 and 4.3 with proofs, mechanized in Lean. The proof uses the rearrangement inequality to show bijective error mapping reduces matching bug probability.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "The paper introduces semantic triangulation, a framework that transforms coding problems via partial inversion and answer enumeration to detect LLM code hallucinations. On LiveCodeBench and CodeElo benchmarks with GPT-4o and DeepSeek-V3, the approach achieves ~90% conditional correctness probability under agreement vs ~60% for baselines. It enables reliable code selection even at low sampling probabilities (0.14) and on inexact problems with multiple valid solutions, where prior methods fail. Theoretical proofs (mechanized in Lean) establish that triangulation strictly improves upon plurality voting under correlated error assumptions.",
   "red_flags": [
     {
       "flag": "No cost/compute reporting",
       "detail": "The method requires sampling 30 programs plus additional witnesses (enumerators, inverses) per problem, likely costing many times more than baselines. No cost comparison is provided, making practical applicability hard to assess."
     },
     {
       "flag": "No error bars or significance tests",
       "detail": "All results are point estimates without confidence intervals or statistical significance tests. Given the relatively small benchmark sizes (175 LCB, 31 CEI), differences may not be statistically significant."
     },
     {
       "flag": "Very small inexact benchmark",
       "detail": "The CodeElo-Inexact benchmark has only 31 problems with manually written judges. Claims about inexact problem handling rest on this small sample, making percentage comparisons unreliable."
     },
     {
       "flag": "Self-implemented baselines",
       "detail": "All baselines are re-implemented by the authors. The paper does not use official CodeT implementations or acknowledge potential bias in baseline implementation."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Pass@k metrics for LLM code generation evaluation."
     },
     {
       "title": "CodeT: Code Generation with Generated Tests",
       "authors": ["Bei Chen"],
       "year": 2023,
       "relevance": "Key baseline using dual execution agreement between sampled programs and tests for consensus."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain"],
       "year": 2025,
       "relevance": "Primary benchmark used for evaluation, designed for contamination-free LLM code assessment."
     },
     {
       "title": "CodeElo: Benchmarking Competition-level Code Generation of LLMs with Human-comparable Elo Ratings",
       "authors": ["Shanghaoran Quan"],
       "year": 2025,
       "arxiv_id": "2501.01257",
       "relevance": "Source of the inexact problems benchmark (CEI) used for multi-solution evaluation."
     },
     {
       "title": "Correlated Errors in Large Language Models",
       "authors": ["Elliot Myunghoon Kim"],
       "year": 2025,
       "relevance": "Provides empirical evidence for correlated LLM errors and abstention evaluation metrics used in this paper."
     },
     {
       "title": "Exploring and evaluating hallucinations in llm-powered code generation",
       "authors": ["Fang Liu"],
       "year": 2024,
       "relevance": "Studies hallucination patterns in LLM code generation, directly relevant to the problem addressed."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce"],
       "year": 2022,
       "relevance": "Documents security vulnerabilities in LLM-generated code, motivating the need for hallucination detection."
     },
     {
       "title": "Can large language models transform natural language intent into formal method postconditions?",
       "authors": ["Madeline Endres"],
       "year": 2024,
       "relevance": "Baseline approach using autoformalized specifications for code verification."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduced MBPP benchmark used for hyperparameter tuning in this paper."
     },
     {
       "title": "Calibration and Correctness of Language Models for Code",
       "authors": ["Claudio Spiess"],
       "year": 2025,
       "relevance": "Studies calibration and confidence measures for LLM code generation, related to uncertainty quantification."
     },
     {
       "title": "Incoherence as Oracle-less Measure of Error in LLM-Based Code Generation",
       "authors": ["Thomas Valentin"],
       "year": 2025,
       "relevance": "Proposes uncertainty measure for code correctness that assumes unique correct solutions, a limitation addressed by triangulation."
     }
   ]
 }