ai-research-survey

scan.json (28077B)

---

 {
   "paper": {
     "title": "Beyond Synthetic Benchmarks: Evaluating LLM Performance on Real-World Class-Level Code Generation",
     "authors": ["Musfiqur Rahman", "SayedHassan Khatoonabadi", "Emad Shihab"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.26130",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub replication package at https://github.com/mrsumitbd/RealClassEval-Replication (reference [3]). Section 11 states 'We make our data and scripts available to facilitate replication and future research.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The RealClassEval dataset is released as part of the replication package (reference [3]). The paper explicitly states data and scripts are available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or specific library versions. The paper mentions using Python's ast library and PYNGUIN but does not provide environment setup details sufficient to recreate the experimental environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While a replication package is referenced, the paper itself does not describe how to run the experiments, what commands to execute, or provide a README-level walkthrough."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap 95% confidence intervals are reported throughout for Cliff's Delta, mean differences, and proportion differences. Tables 3 and 4 include 95% CIs for all comparisons (e.g., 'Mean Diff (95% CI)' column)."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses appropriate statistical tests throughout: Kruskal-Wallis, Mann-Whitney U, Friedman, Wilcoxon signed-rank, chi-square, and sign tests. FDR correction via Benjamini-Hochberg is consistently applied for multiple comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cliff's Delta with established thresholds (negligible, small, medium, large) is reported for per-class analyses, and Cramér's V for chi-square analyses. Both are reported with bootstrap 95% CIs. Mean percentage point differences are also provided throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 9.1.1 explicitly discusses sample size limitations and justification. Power analysis is reported for all RQs (e.g., 'Power analysis across all seven models indicated high statistical power (0.974–0.983)' in Section 4.2.1). The paper also acknowledges budget constraints limiting to 200 classes per version."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports distributions via violin plots (Figure 3), bootstrap confidence intervals across analyses, and discusses skewness across comparisons (e.g., 'High skewness (|skewness| > 1 in 11 of 28 comparisons)' in Section 5.1.3). Results are reported with spread measures throughout."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares real-world performance against the synthetic ClassEval benchmark as a baseline. Seven LLMs are compared against each other, and ablation conditions (with/without docstrings, with/without RAG) serve as controlled baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The seven evaluated models include GPT-5, GPT-4.1, GPT-OSS (August 2025), Llama 4 Maverick (April 2025), DeepSeek-V3 (August 2025), and other recent models. ClassEval (2023) is the most recent comparable class-level benchmark."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 is an ablation study of docstring completeness (full, partial, no docstrings). RQ3 ablates the presence/absence of RAG across docstring conditions. Both systematically vary individual factors while controlling others."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary metric is pass rate (proportion of passed tests). While the paper also analyzes error type distributions, it does not use multiple quantitative metrics for code generation quality (e.g., no CodeBLEU, edit distance, or other complementary metrics)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "There is no human evaluation of the generated code quality. All evaluation is automated through test suite execution. The paper makes claims about code quality and practical capability that could benefit from human review of generated outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The Post-Cutoff dataset (200 classes from repositories created after March 31, 2025) serves as a held-out set unseen by any evaluated model. This temporal split guarantees no training data contamination."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (ClassEval, Pre-Cutoff, Post-Cutoff), by model (7 models), by docstring condition (full, partial, no), and by error type (13 error categories). Tables 3, 4, 6, 9, and 12 provide detailed per-condition breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 7) is entirely dedicated to error analysis. The paper provides detailed failure mode analysis including error type distributions, qualitative analysis of test suite differences, and the RAG error substitution mechanism."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: docstrings provide only negligible improvements (RQ2), RAG provides no benefit with full docstrings (RQ3), GPT-5 shows negative effects with RAG, and RAG introduces new dependency errors (ImportError, KeyError increases)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 84-89% synthetic vs 25-34% real-world correctness are supported by Tables 3-4 and Figure 2. The 1-3% docstring improvement and 4-7% RAG gains are supported by Tables 5-6 and 8-9. Error analysis claims match Table 10."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (RQ2: docstring impact, RQ3: RAG impact) with within-subjects designs where each class is tested under all conditions, controlling for snippet-specific characteristics. The language 'RAG improves' and 'docstrings improve' are supported by controlled single-variable manipulations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 10.2 explicitly bounds generalization: 'All evaluated code is Python-based, and findings may not generalize to statically typed languages.' Section 9.1.3 states findings 'may not generalize to other granularities.' The title specifies 'Class-Level Code Generation' rather than making broad claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative explanations: the performance gap could be due to code complexity (investigated via mixed-effects model in Section 4.3), test suite quality differences (qualitatively analyzed), and model-specific architectural differences explaining GPT-5's anomalous RAG behavior (Section 6.3.3)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 lists model names and release dates but not exact API versions or snapshot dates. 'GPT-4.1', 'GPT-5', 'GPT-OSS', 'DeepSeek-V3' are listed without specific API version strings (e.g., no 'gpt-4.1-2025-04-14' or equivalent). Codestral's knowledge cutoff is listed as 'Unknown'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.2 provides the exact prompt template used: 'You are an expert Python programmer who can correctly implement complete Python classes based on the provided class skeletons. Implement the following class. Do not explain the code. The given class skeleton is as follows: [CLASS SKELETON].' The class skeleton content is fully specified by the dataset."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report temperature, top-p, max tokens, or other generation hyperparameters used for the LLM API calls. Section 9.1.5 acknowledges 'consistent generation settings' but does not specify what those settings were."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper uses direct prompt-to-completion generation for each class skeleton. RAG retrieval is a simple embedding-based similarity search, not an agentic workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the full data pipeline: data collection from CodeSearchNet and GitHub API (3.1.1), filtering criteria including license, project size/activity, and code quality (3.1.2), static analysis and class extraction (3.1.3), and post-processing with random sampling (3.1.4). Figure 1 provides a workflow diagram."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 is titled 'Limitations and Future Work' with Section 9.1 dedicated to limitations. Section 10 is titled 'Threats to Validity' with detailed subsections on internal, external, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses specific threats: sample size of 200 classes per version with detailed budget analysis (~$1000 API cost, 500+ CPU hours), Python-only evaluation limiting generalization to other languages, Pynguin test suite limitations, standardized prompts vs model-specific optimization, and open-source vs proprietary codebase differences."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 9.1.3 explicitly states: 'Findings may not generalize to other granularities: single-function generation may show higher success rates... while generating entire projects likely shows even larger challenges.' Section 10.2 states 'findings may not generalize to statically typed languages' and 'potentially limiting generalizability to proprietary enterprise codebases.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at https://github.com/mrsumitbd/RealClassEval-Replication includes the dataset and scripts. The paper states 'We make our data and scripts available to facilitate replication and future research' (Section 11)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 provides detailed data collection procedures: Pre-cutoff data from CodeSearchNet, post-cutoff from GitHub API for repos created after March 31, 2025, with specific criteria (10+ commits, Python primary, not forks). Figure 1 illustrates the workflow."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Data sources are standard benchmarks and public GitHub repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section 3.1: collection from two sources, filtering for engineered projects (license, stars 10+, commits 50+, code quality), static analysis with Understand tool, class extraction via ast library, post-processing removal of problematic classes, and random sampling of 200 per version."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. The authors are affiliated with Concordia University but no grants or funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed with their affiliation at Concordia University, Canada. No evaluated product is affiliated with their institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The paper evaluates commercial products (GPT-4.1, GPT-5, GPT-OSS from OpenAI; Codestral from Mistral; etc.) without disclosing whether any API credits or funding came from these companies."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 lists knowledge cutoff dates for each model: Qwen 2.5 Coder (June 2024), GPT-4.1 (June 2024), GPT-5 (October 2024), GPT-OSS (June 2024), Codestral (Unknown), DeepSeek-V3 (July 2024), Llama 4 Maverick (August 2024). The post-cutoff data boundary (March 31, 2025) is explicitly stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The entire study design addresses contamination through seen/unseen partitions. Section 3.1.1 explains: pre-cutoff data from CodeSearchNet is 'likely encountered during training' while post-cutoff data from repos created after March 31, 2025 guarantees they 'were not in training data.' The finding that seen vs unseen performance is equivalent is a key result."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses this through the seen/unseen design. Section 3.1.1 states pre-cutoff data uses CodeSearchNet which 'has been used as a pre-training corpus for numerous prominent models.' Post-cutoff data guarantees no contamination via temporal separation. The result that contamination does not explain performance gaps is a central finding."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. This is a benchmark evaluation study using automated test suites."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not report per-class or per-model inference costs, API token usage, or wall-clock time for generation. Section 6.1.1 mentions budget constraints ('it was beyond our budget to use k > 2') but does not quantify per-inference costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 9.1.1 states: 'the total generation cost, including trial and error, was approximately $1000 for API calls alone, plus over 500 CPU hours for test execution.' This provides the total computational budget."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs achieve 84-89% correctness on synthetic benchmarks (ClassEval) but only 25-34% on real-world class-level code generation tasks.",
       "evidence": "Figure 2 and Tables 3-4 show ClassEval pass rates of 83.60-88.27% vs Pre-Cutoff 24.59-33.60% and Post-Cutoff 27.40-31.00%, with medium to large effect sizes (Cliff's Delta 0.46-0.64). All comparisons statistically significant after FDR correction.",
       "supported": "strong"
     },
     {
       "claim": "There is no meaningful performance difference between seen (pre-cutoff) and unseen (post-cutoff) real-world code, indicating memorization does not explain real-world failures.",
       "evidence": "Table 3 shows 6 of 7 models have no significant Pre-Cutoff vs Post-Cutoff differences after FDR correction. All effect sizes are negligible. GPT-OSS shows statistical significance (p=0.019) but with negligible effect size (Cliff's Delta=-0.125). Table 4 pooled analysis confirms: all FDR-adjusted p > 0.07.",
       "supported": "strong"
     },
     {
       "claim": "Complete docstrings provide only marginal improvements (1-3%) in code generation correctness, with only 2 of 7 models showing statistically significant benefits.",
       "evidence": "Table 5 shows only 2/28 comparisons are significant after FDR correction, both for full vs partial (Codestral 2.72%, DeepSeek-V3 1.51%). All Cliff's Delta values are negligible (0.003-0.077). Pooled chi-square shows no significant differences (all pFDR > 0.05).",
       "supported": "strong"
     },
     {
       "claim": "RAG provides significant benefits (4-7%) specifically when documentation is incomplete (partial docstrings), but minimal value with complete or absent documentation.",
       "evidence": "Table 8 shows 5/7 models significant with partial docstrings, 0/7 with full, 0/7 with no. Table 9 details: Codestral 5.27% (p=0.030), DeepSeek-V3 6.94% (p=0.023), GPT-4.1 4.63% (p=0.023), Llama 4 Maverick 4.25% (p=0.024), Qwen 2.5 5.18% (p=0.024). Power analysis confirms adequate detection (0.634-0.838).",
       "supported": "strong"
     },
     {
       "claim": "AttributeError, TypeError, and AssertionError account for 84% of all failures in LLM-generated class-level code.",
       "evidence": "Table 10 shows AttributeError 43.84%, TypeError 21.65%, AssertionError 18.51%, totaling 84%. Chi-square goodness-of-fit confirms non-uniform distribution (chi-squared = 37847.3, p < 10^-300).",
       "supported": "strong"
     },
     {
       "claim": "RAG operates through an error substitution mechanism, reducing logic and object access errors while occasionally introducing dependency errors.",
       "evidence": "Table 13 and Figure 5 show: RAG reduces AttributeError (-60), Other (-53), AssertionError (-43), but increases ImportError (+5, +250%) and KeyError (+5, +45%). Chi-square tests significant for 3 models with partial docstrings (all pFDR < 0.05).",
       "supported": "strong"
     },
     {
       "claim": "Synthetic benchmarks fundamentally mischaracterize real-world failure modes, with ClassEval dominated by AssertionError (71.80%) versus real-world code dominated by AttributeError (45-49%) and TypeError (22-24%).",
       "evidence": "Table 12 shows ClassEval: 71.80% AssertionError, 2.14% AttributeError vs Pre-Cutoff: 5.34% AssertionError, 49.41% AttributeError. Table 11 shows large effect sizes (Cramér's V 0.59-0.85) for all synthetic vs real-world comparisons.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "LLMs achieve 84-89% correctness on synthetic class-level benchmarks but only 25-34% on real-world classes from open-source projects, with no meaningful difference between seen and unseen codebases. This performance gap stems from fundamentally different error patterns: synthetic benchmarks test logical correctness (AssertionError-dominated) while real-world code requires object attribute mastery and type consistency (AttributeError and TypeError-dominated). Docstrings provide only marginal improvements (1-3%, mostly non-significant), while RAG yields 4-7% gains specifically when documentation is incomplete, operating through an error substitution mechanism that trades logic failures for dependency errors.",
   "red_flags": [
     {
       "flag": "Missing hyperparameters",
       "detail": "Temperature, top-p, max tokens, and other generation settings are not reported despite the paper acknowledging 'consistent generation settings.' These parameters significantly affect LLM output quality and reproducibility."
     },
     {
       "flag": "Missing model version specifics",
       "detail": "Models are identified by marketing names (GPT-4.1, GPT-5, etc.) without API version strings or snapshot dates. Codestral's knowledge cutoff is listed as 'Unknown,' making contamination assessment incomplete for that model."
     },
     {
       "flag": "Single-metric primary evaluation",
       "detail": "Pass rate is the sole quantitative metric for code generation quality. No complementary metrics (CodeBLEU, semantic similarity, etc.) are used, which could provide different perspectives on code quality beyond test-passing."
     },
     {
       "flag": "No funding disclosure",
       "detail": "The paper evaluates commercial API products costing approximately $1000 but does not disclose whether API credits were provided by any of the companies whose products were evaluated."
     },
     {
       "flag": "Automated test suite limitations acknowledged but not mitigated",
       "detail": "Section 9.1.4 acknowledges Pynguin-generated test suites 'primarily assess that code executes without exceptions, but may not validate whether methods compute correct results.' No manual validation of test quality was performed for any subset."
     }
   ],
   "cited_papers": [
     {
       "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-Level Code Generation",
       "authors": ["X. Du", "M. Liu", "K. Wang", "H. Wang", "J. Liu", "Y. Chen", "J. Feng", "C. Sha", "X. Peng", "Y. Lou"],
       "year": 2023,
       "arxiv_id": "2308.01861",
       "relevance": "The primary baseline benchmark for class-level code generation that this paper extends with real-world evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval benchmark and Pass@k metric widely used in code generation evaluation."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "relevance": "Introduced EvalPlus showing HumanEval overestimates LLM capabilities due to weak test suites, directly motivating this paper's real-world benchmark."
     },
     {
       "title": "CodeRAG: Supportive Code Retrieval on Bigraph for Real-World Code Generation",
       "authors": ["J. Li", "X. Shi", "K. Zhang"],
       "year": 2025,
       "arxiv_id": "2504.10046",
       "relevance": "Repository-level RAG framework for code generation, directly related to this paper's RAG evaluation."
     },
     {
       "title": "Retrieval-Augmented Code Generation: A Survey with Focus on Repository-Level Approaches",
       "authors": ["Y. Tao", "Y. Qin", "Y. Liu"],
       "year": 2025,
       "arxiv_id": "2510.04905",
       "relevance": "Survey of RAG for code generation providing context for this paper's RAG evaluation methodology."
     },
     {
       "title": "Bugs in Large Language Models Generated Code: An Empirical Study",
       "authors": ["F. Tambon", "A. Moradi-Dakhel", "A. Nikanjam", "F. Khomh", "M. C. Desmarais", "G. Antoniol"],
       "year": 2025,
       "relevance": "Prior work on error patterns in LLM-generated function-level code that this paper extends to class-level generation."
     },
     {
       "title": "Towards Understanding the Characteristics of Code Generation Errors Made by Large Language Models",
       "authors": ["Z. Wang", "Z. Zhou", "D. Song"],
       "year": 2025,
       "relevance": "Analysis of 557 incorrect code snippets from six LLMs on HumanEval, providing error taxonomy context for this paper's class-level error analysis."
     },
     {
       "title": "HumanEval Pro and MBPP Pro: Evaluating Large Language Models on Self-Invoking Code Generation",
       "authors": ["Z. Yu", "Y. Zhao", "A. Cohan", "X.-P. Zhang"],
       "year": 2024,
       "arxiv_id": "2412.21199",
       "relevance": "Extended HumanEval with self-invoking tasks showing performance degradation, related to this paper's finding that complex code generation is harder."
     },
     {
       "title": "CodeRAG-Bench: Can Retrieval Augment Code Generation?",
       "authors": ["Z. Z. Wang", "A. Asai", "X. V. Yu"],
       "year": 2024,
       "arxiv_id": "2406.14497",
       "relevance": "Comprehensive RAG benchmark for code generation evaluating retrievers and LLMs, directly relevant to this paper's RAG analysis."
     },
     {
       "title": "A Survey on Large Language Models for Code Generation",
       "authors": ["J. Jiang", "F. Wang", "J. Shen", "S. Kim", "S. Kim"],
       "year": 2024,
       "arxiv_id": "2406.00515",
       "relevance": "Survey covering LLM code generation capabilities and benchmarks, providing context for this study's contributions."
     },
     {
       "title": "CoderEval: A Benchmark of Pragmatic Code Generation with Generative Pre-trained Models",
       "authors": ["H. Yu", "B. Shen", "D. Ran"],
       "year": 2024,
       "relevance": "Benchmark for non-standalone function-level code generation from real-world projects, addressing similar real-world evaluation concerns."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["I. Bouzenia", "P. Devanbu", "M. Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Agentic approach to LLM-based program repair, relevant to understanding LLM code capabilities in software engineering."
     }
   ]
 }