ai-research-survey

scan.json (27738B)

---

 {
   "paper": {
     "title": "Pragmatic Reasoning improves LLM Code Generation",
     "authors": [
       "Zhuchen Cao",
       "Sven Apel",
       "Adish Singla",
       "Vera Demberg"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2502.15835",
     "doi": "10.48550/arXiv.2502.15835"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeRSA applies the Rational Speech Act pragmatic reasoning framework to code candidate reranking, achieving 59.53% accuracy on MBPP with Llama-3-8B-Instruct vs. Coder (~55%) and CoderReviewer (~57%) baselines. Semantic clustering of paraphrased instructions is critical — removing it causes a substantial accuracy drop. The approach is robust to the calibration parameter α across a wide band. Results are consistent across two models (Llama-3-8B, Qwen-2.5-7B) and two benchmarks (HumanEval, MBPP), with a sanity check on Llama-3-70B.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to released source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: HumanEval (Chen et al., 2021) and MBPP (Austin et al., 2021). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA Tesla A100 (PCIe 4.0, 80GB HBM2e, 300W)' in Section 8 but provides no requirements.txt, dependency specifications, or library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described algorithmically but not in a reproducible recipe format."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Figures 3, 5, 6, 7) report single accuracy values across α sweeps with no confidence intervals or error bars. Appendix A.4 (Figure 8) shows standard deviation bands for the sensitivity analysis, but the primary reported results lack uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims CodeRSA 'outperforms' and 'surpasses' baselines based solely on comparing accuracy percentages. No statistical significance tests (p-values, t-tests, bootstrap tests) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented as accuracy curves across α values (Figures 3, 5, 6, 7) and as raw accuracy numbers (e.g., 59.53% in Section 5.1, Table 2). No explicit effect sizes (e.g., percentage point improvements with baseline context) are calculated or reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses standard benchmark sizes (HumanEval: 164, MBPP: 974) without justifying whether these are sufficient for the comparisons being made. No power analysis is performed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main experimental results (Figures 3, 5, 6, 7) appear to be single-run numbers without variance or standard deviation. Only Appendix A.4's sensitivity analysis (Figure 8) reports standard deviation across 10 repetitions with random candidate subsampling."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Coder Reranking (Chen et al., 2021) and CoderReviewer Reranking (Zhang et al., 2023a), described as the state-of-the-art content-driven reranking method."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CoderReviewer (Zhang et al., 2023a) is described as the current state-of-the-art for content-driven code reranking. The paper also references but excludes execution-driven methods (CodeT, AgentCoder) for methodological reasons."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "All four main result figures (3, 5, 6, 7) include 'CodeRSA w/o Clustering' as an ablation, showing the contribution of the clustering component. The α calibration parameter sweep also functions as a form of ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses only accuracy (proportion of instances where the selected candidate passes all test cases) as the evaluation metric. No additional metrics (e.g., code quality, runtime, rank position) are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via test-case pass/fail. The qualitative analysis in Section 5.2 is performed by the authors to illustrate RSA behavior, not a systematic human evaluation of outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The calibration parameter α is swept across the full benchmark test sets (Figures 3, 5, 6, 7) with best values selected on the same data used for evaluation. No separate validation set is used for hyperparameter selection."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (Llama-3-8B-Instruct, Qwen-2.5-7B-Instruct), dataset (HumanEval, MBPP), and number of candidates (Figure 8). Appendix A.3 includes a 70B model breakdown."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The qualitative analysis (Section 5.2) shows only a successful case where CodeRSA corrects Coder's mistake. No examples of where CodeRSA fails or makes worse selections are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation shows CodeRSA without clustering performs substantially worse, sometimes below baselines. The paper also notes 'some fluctuations in performance' on HumanEval (Appendix A.2) and acknowledges that Schuster et al. (2024) obtained negative results with RSA on spreadsheets."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CodeRSA 'consistently outperforms common baselines, surpasses the state-of-the-art approach in most cases.' The results show consistent improvement over Coder and improvement over CoderReviewer in most (not all) settings. The hedging ('most cases') matches the evidence."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('improves', 'enhancing') supported by controlled comparisons: all methods operate on the same candidate set per problem, with only the reranking method varying. The ablation study (removing clustering) isolates component contributions through controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Pragmatic Reasoning improves LLM Code Generation' and abstract claim about 'enhancing code generation quality in LLMs' are broader than the evidence: two mid-sized open-source models on two Python benchmarks. The Limitations section acknowledges narrow scope but the title and abstract overreach."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Discussion (Section 6) addresses uniform cost assumption and applicability conditions, but does not consider alternative explanations for the observed improvements. For example, could the clustering step alone (without RSA) explain improvements? Could the results be specific to these benchmark difficulties? No confound analysis is provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy (test-case pass rate) and discusses this as selecting the correct candidate from sampled solutions. Claims stay at the level of 'reranking quality' and 'candidate selection,' which match the measurement granularity. No inflated framing beyond what test-case passing demonstrates."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'Llama-3-8B-Instruct' and 'Qwen-2.5-7B-Instruct' (Section 4.1), plus 'Llama-3-70B-Instruct' for the sanity check (Appendix A.3). These are specific open-source releases with defined parameter counts."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows the prompts for calculating Coder score and generating additional instructions. Appendix A.6 provides the full prompt templates including a one-shot example for instruction generation and the Reviewer score prompt format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: temperature=1.0 for candidate sampling, temperature=0.7 for instruction generation, n=10 candidates, m=1 additional instruction per candidate, α calibration parameter swept and analyzed (Section 4.2)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. CodeRSA is a reranking method applied post-generation, not an agent with tools, memory, or iterative loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: sample n=10 candidates per problem at temperature 1.0, generate m=1 instruction per candidate, perform pairwise semantic equivalence clustering, compute literal listener scores, aggregate by cluster, and apply pragmatic listener reranking (Sections 3-4)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' provides substantive discussion of computational complexity, candidate pool size constraints, and scope of models/benchmarks."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: computational cost (6 hours on A100 for 500 instances), restriction to 10 candidates which 'inevitably narrows the variety of solutions,' deliberate choice of mid-sized models, and plan to expand to DS-1000, Mistral, and newer Qwen releases."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'we deliberately focus on mid-sized models and well-established benchmarks as a proof-of-concept design choice' and 'We do not include specialized coder models in this study.' Specific exclusions (model families, benchmark types) are enumerated."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (generated candidates, instructions, scores, clustering outputs) is made available. Only aggregated accuracy numbers are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation procedure is described: n=10 candidates sampled at temperature 1.0, m=1 instruction generated per candidate at temperature 0.7, 3-shot prompt for semantic equivalence clustering (Sections 3-4)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is standard public benchmarks (HumanEval, MBPP)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from candidate generation to instruction generation to pairwise clustering to literal listener scoring to pragmatic speaker/listener reranking is documented step-by-step in Sections 3 and 4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Max Planck Institute for Informatics, Saarland University, and Max Planck Institute for Software Systems. No products being evaluated are produced by these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Authors are at public academic institutions (Max Planck Institutes, Saarland University). These funders have no financial stake in the outcome of CodeRSA's performance evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for Llama-3-8B-Instruct or Qwen-2.5-7B-Instruct. Both models were released in 2024, well after the benchmarks were published."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval or MBPP problems appeared in the training data of either model. Both benchmarks were published in 2021 and are widely known."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published years before Llama-3 (2024) and Qwen-2.5 (2024) were trained. The contamination risk is high and completely unaddressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study is a purely computational evaluation of reranking methods."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'on a single NVIDIA Tesla A100 (PCIe 4.0, 80GB HBM2e, 300W), performing complete CodeRSA inference on 500 instances takes nearly 6 hours.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The GPU hardware (NVIDIA Tesla A100, 80GB) and approximate runtime (6 hours for 500 instances) are stated in Section 8."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Figures 3, 5, 6, 7) are presented as single runs without seed variation. Appendix A.4 varies random candidate subsampling across 10 seeds, but this tests candidate selection sensitivity, not generation seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The main experiments do not state how many runs were performed. Only Appendix A.4's sensitivity analysis explicitly states 'ten times with different random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The α parameter is swept across a range shown in figures, but no explicit search budget (number of configurations, compute spent) is reported. The sweep granularity and total configurations tried are not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Best α=1.0 is selected based on test set performance (Figure 3, MBPP with Llama-3-8B). No separate validation set is used for hyperparameter selection — α is effectively tuned on the test data."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all methods (Coder, CoderReviewer, CodeRSA) themselves. No discussion of author-evaluation bias or whether their baseline implementations match published results."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "CodeRSA requires generating additional instructions, clustering, and computing scores across all instruction-candidate pairs — substantially more compute than Coder or CoderReviewer. Performance is not compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 discusses why HumanEval and MBPP are suitable difficulty levels for evaluating reranking, but does not question whether test-case pass rate actually measures code generation quality or whether these benchmarks have known validity limitations."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. CodeRSA is a post-generation reranking method, not an agent scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. HumanEval and MBPP were published in 2021. Llama-3 and Qwen-2.5 were trained in 2024 and could have seen solutions in their training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether the evaluation setup leaks information (e.g., function signatures in HumanEval providing strong hints)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether training data contains problems structurally similar to HumanEval or MBPP problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or temporal splits."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeRSA consistently outperforms Coder Reranking and CoderReviewer Reranking baselines across models and benchmarks.",
       "evidence": "Figures 3, 5, 6, 7 show CodeRSA above baselines on MBPP and HumanEval with both Llama-3-8B-Instruct and Qwen-2.5-7B-Instruct. E.g., 59.53% vs baselines on MBPP/Llama-3-8B at α=1.0 (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "Semantic clustering is crucial for CodeRSA's performance — removing it causes a substantial accuracy drop.",
       "evidence": "All four main result figures (3, 5, 6, 7) include 'CodeRSA w/o Clustering' ablation showing performance drops below baselines in some cases.",
       "supported": "moderate"
     },
     {
       "claim": "CodeRSA is robust to the calibration parameter α within a stable band of [0.90, 1.15].",
       "evidence": "Figure 3 shows stable performance within the shaded band. Appendix A.2 figures show similar stability for other model/dataset combinations.",
       "supported": "moderate"
     },
     {
       "claim": "CodeRSA scales to larger models (Llama-3-70B-Instruct) without abnormal behavior.",
       "evidence": "Table 2 (Appendix A.3): CodeRSA achieves 51.2% accuracy vs CoderReviewer's 48.8% and Coder's 39.3% on 84 MBPP instances unsolved at pass@1 but solved at pass@10.",
       "supported": "weak"
     },
     {
       "claim": "CodeRSA corrects the length bias inherent in Coder Reranking by normalizing over alternative instruction clusters.",
       "evidence": "Qualitative example in Section 5.2 (Figure 4) shows Coder preferring a shorter, incomplete code_09 while CodeRSA correctly selects code_01 via pragmatic reasoning over clusters.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('outperforms', 'surpasses') are based on comparing raw accuracy percentages without any statistical tests. With single-run results on modestly sized benchmarks, observed differences could be due to chance."
     },
     {
       "flag": "Hyperparameter tuned on test set",
       "detail": "The calibration parameter α is swept and selected based on test set performance (Figures 3, 5, 6, 7). No validation set is used. The 'best α=1.0' is effectively selected post-hoc on the evaluation data."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HumanEval (2021) and MBPP (2021) are well-known benchmarks published years before Llama-3 (2024) and Qwen-2.5 (2024). The models likely encountered these problems during training, potentially inflating absolute performance numbers and complicating interpretation of reranking improvements."
     },
     {
       "flag": "No error bars on main results",
       "detail": "Main experimental results are presented as single-run accuracy values. Only the sensitivity analysis in Appendix A.4 reports standard deviation across runs. The stability of the main findings cannot be assessed."
     },
     {
       "flag": "Compute cost asymmetry not controlled",
       "detail": "CodeRSA requires substantially more compute than baselines (generating instructions, pairwise clustering, cross-evaluating all instruction-candidate pairs). Performance is not compared at matched compute budgets, making it unclear whether the improvement justifies the ~6x+ cost increase."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Coder Reranking baseline used in this paper's evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark used as one of two primary evaluation datasets."
     },
     {
       "title": "Coder reviewer reranking for code generation",
       "authors": ["Tianyi Zhang", "Tao Yu", "Tatsunori B Hashimoto"],
       "year": 2023,
       "relevance": "State-of-the-art content-driven code reranking baseline that CodeRSA aims to surpass."
     },
     {
       "title": "CodeT: Code generation with generated tests",
       "authors": ["Bei Chen", "Fengji Zhang", "Anh Nguyen"],
       "year": 2022,
       "arxiv_id": "2207.10397",
       "relevance": "Execution-driven code reranking method representing an alternative approach to candidate selection."
     },
     {
       "title": "AgentCoder: Multi-agent-based code generation with iterative testing and optimisation",
       "authors": ["Dong Huang", "Jie M. Zhang", "Michael Luck"],
       "year": 2024,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent code generation with iterative testing, an execution-driven reranking alternative."
     },
     {
       "title": "Large language monkeys: Scaling inference compute with repeated sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Explores scaling inference compute via repeated sampling for code generation — directly relevant to the sample-then-rerank paradigm."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Rigorous evaluation methodology for LLM code generation, directly relevant to benchmark validity."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Aaron Grattafiori"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Technical report for Llama-3, one of the primary models evaluated in this work."
     },
     {
       "title": "A performance study of LLM-generated code on LeetCode",
       "authors": ["Tristan Coignion", "Clément Quinton", "Romain Rouvoy"],
       "year": 2024,
       "relevance": "Empirical study of LLM code generation performance on competitive programming problems."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "More challenging code generation benchmark referenced as too difficult for meaningful reranking comparison."
     },
     {
       "title": "Evaluating the code quality of AI-assisted code generation tools: An empirical study on GitHub Copilot, Amazon CodeWhisperer, and ChatGPT",
       "authors": ["Burak Yetiştiren", "Işık Özsoy", "Miray Ayerdem", "Eray Tüzün"],
       "year": 2023,
       "arxiv_id": "2304.10778",
       "relevance": "Evaluates code quality of commercial AI code generation tools, relevant to AI programming assessment."
     }
   ]
 }