ai-research-survey

scan.json (28278B)

---

 {
   "paper": {
     "title": "PythonSaga: Redefining the Benchmark to Evaluate Code Generating LLMs",
     "authors": [
       "Ankit Yadav",
       "Himanshu Beniwal",
       "Mayank Singh"
     ],
     "year": 2024,
     "venue": "Conference on Empirical Methods in Natural Language Processing (EMNLP 2024 Findings)",
     "arxiv_id": "2401.03855",
     "doi": "10.18653/v1/2024.findings-emnlp.996"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval",
     "qualitative"
   ],
   "key_findings": "HumanEval and MBPP are heavily biased toward basic programming concepts (72-77% of problems in 5 concepts) and easy difficulty (85-90% Easy), with 14 of 38 programming concepts entirely absent. PythonSaga, a new 185-problem benchmark with balanced concept and difficulty representation, reveals dramatically lower Code-LLM performance: open-source models achieve <4.5% pass@1 and GPT-4 achieves only 12.4% pass@1, far below scores on existing benchmarks. Six programming concepts were unsolved by all tested models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'The code and data set are openly available to the NLP community at https://anonymous.4open.science/r/PythonSaga.' A URL is provided, though it is an anonymous review link."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released at the same URL mentioned in the abstract. PythonSaga's 185 problems are described as openly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a single Tesla V100' as hardware but provides no software environment details — no requirements.txt, Dockerfile, library versions, or Python version."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. There is no description of how to run the evaluation pipeline or replicate the benchmarking experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports only point estimates for pass@1 and pass@10. No confidence intervals or error bars are provided for any results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'closed-source models performed considerably better than open-source models' and discusses performance differences across concept categories, but no statistical significance tests are used anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only raw pass@k scores are reported. No formal effect sizes (Cohen's d, odds ratios, etc.) are computed for any comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 5 problems per concept (185 total) is not justified with any statistical reasoning or power analysis. The 5-annotator sample for human evaluation is also unjustified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite generating n=20 samples per problem, no variance, standard deviation, or spread measures are reported across the samples or across problems."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares 14 open-source and 2 closed-source models, and contrasts PythonSaga results against HumanEval and MBPP performance (Figure 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models tested include Llama 3, StarCoder2, DeepSeek Coder, GPT-4 — all contemporary at time of writing (2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "PythonSaga is a benchmark, not a multi-component system. There is nothing to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Both pass@1 and pass@10 are reported for all models in Table 2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The human annotation study evaluates the benchmark problems themselves (concept labels and difficulty), not model outputs. No human evaluation of generated code quality is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "PythonSaga is a newly created benchmark. Models are evaluated on it without any tuning on PythonSaga data, making it a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 provides a detailed heatmap of per-concept performance for each model. Section 5.3 also reports breakdowns by basic/intermediate/advanced categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 identifies and categorizes 9 error types including invalid syntax, incomplete code, hallucination, and non-compliance with problem statements."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that no model could solve any problem in 6 specific concepts (Basic Data Structures, Recursion, Hashing, Context Managers, Concurrency and Parallelism, Max Flow)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about concept bias (supported by Section 4.3, Figure 2), prevalence of easy tasks (supported by Section 4.3 difficulty analysis), and poor Code-LLM performance (supported by Table 2) are all backed by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper asserts that 'current benchmarks overstate the generalization abilities of existing code-LLMs' and that easy/biased benchmarks 'inflate model performance estimations.' However, the performance difference between PythonSaga and HumanEval/MBPP could be due to confounds other than concept diversity (e.g., problem rephrasing style, source platform differences, test case quality). No controlled analysis isolates the effect of concept diversity."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'we focus on Python Programming language' and the Limitations section acknowledges that 'the generalizability of these findings to other languages requires further investigation.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for poor model performance on PythonSaga, such as the rephrasing style making problems harder to parse, platform-specific problem formulations, low test case coverage, or other confounds beyond concept diversity."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k and discusses results in terms of pass@k without inflating it to broader claims about 'code generation ability' in general. Claims match the measurement granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed by general name and size (e.g., 'Code Llama 7B', 'GPT-3.5', 'GPT-4') without specific version identifiers or snapshot dates. No API version strings are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark problems serve as the prompts in HumanEval-style evaluation. Representative examples are provided in Appendices A.2-A.4, and the full dataset is released."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states n=20 samples per problem but does not report temperature, top-p, max tokens, or any other generation hyperparameters for any model."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models generate code directly from prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes the curation methodology: problems selected from GeeksForGeeks and LeetCode, 5 per concept, manually rephrased without AI assistance. Section 4.2 describes the annotation protocol including guidelines and constraints."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section discusses three specific limitations of the study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: (1) random sampling of 164/974 MBPP problems may introduce selection bias, (2) annotators are all postgrad CS students limiting annotator diversity, (3) findings limited to Python."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states that findings may not generalize to the full MBPP benchmark, other annotator populations, or other programming languages."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark dataset is released, but raw model outputs (the 20 generated code samples per problem per model) and raw annotation data from the 5 annotators are not made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes curation from GeeksForGeeks and LeetCode with manual rephrasing. Section 4.2 describes the annotation procedure including annotator qualifications, guidelines, and constraints."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Annotators are described as 'postgraduate students in Computer Science with at least three years of experience' who 'willingly volunteered,' but how they were recruited (likely convenience sampling from the same lab) is not described, and selection bias is not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from problem selection to final benchmark is described at a high level, but key details are missing: how many candidate problems were initially considered, how many were filtered at each stage, and what criteria were used to select the final 5 per concept."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The Acknowledgements section thanks annotators and the LINGO research group but does not mention any funding sources, grants, or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with IIT Gandhinagar are clearly disclosed in the paper header. No commercial affiliation with any evaluated model is present."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder is identified. This appears to be unfunded academic work from a university research group."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 16 evaluated models. This is critical since PythonSaga problems are sourced from public coding platforms."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether PythonSaga problems (or their originals from GeeksForGeeks/LeetCode) appeared in any model's training data. The manual rephrasing is framed as making problems harder, not as decontamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP were published in 2021; many tested models were trained after that. PythonSaga draws from public platforms that are common training data sources. Contamination is not discussed for any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the human annotation study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The Ethics Statement mentions informed consent but does not mention IRB or ethics board approval."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Annotators are described as 'postgraduate students in Computer Science with at least three years of experience in Python programming and competitive programming' (Section 4.2)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 specifies annotators must be postgraduate CS students with at least 3 years of Python and competitive programming experience."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an annotation study, not an experimental study with treatment/control conditions. Randomization of conditions is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is an annotation study where blinding is not applicable — annotators are classifying problems by concept and difficulty, not evaluating experimental conditions."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states annotators 'willingly volunteered throughout the entire duration' but does not explicitly report whether any annotators started but did not complete the task, or how many total were initially recruited."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example latency figures are reported despite evaluating 16 models on 185 problems with 20 samples each."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'a single Tesla V100' as a hardware constraint but does not quantify total GPU hours, training time, or API spend."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Results appear to be from a single experimental run with n=20 samples per problem."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'we consistently generated n = 20 samples from both open and closed source models' (Section 5.3)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper does not report what generation settings were used or whether any tuning was performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how generation configurations were chosen for each model. Temperature, top-p, and other settings are not reported or justified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed in the paper, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created PythonSaga and evaluate all models on it. They do not acknowledge or address the bias of evaluating using their own benchmark."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 2.7B to 13B parameters (plus unknown-size closed-source), but performance is not analyzed as a function of compute budget. Closed-source models are compared with open-source without accounting for compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central contribution is a construct validity analysis: it argues that HumanEval and MBPP fail to measure general code generation ability due to biased concept coverage and difficulty distribution, and designs PythonSaga to address these validity gaps."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models generate code directly from prompts."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "PythonSaga problems are sourced from GeeksForGeeks and LeetCode, which predate most tested models' training data. This temporal overlap is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The problem format (function signature + docstring) is standard but feature leakage is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Problems from GeeksForGeeks and LeetCode may have structural overlap with model training data from these same platforms. This non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection method is applied. The manual rephrasing is presented as making problems harder, not as a decontamination technique, and its effectiveness at preventing memorization is not tested."
       }
     }
   },
   "claims": [
     {
       "claim": "HumanEval and MBPP are biased toward a limited set of programming concepts, with 72.1% and 77.3% of problems in just 5 concepts (Mathematics, Control Flow, Basic Data Structures, Variables & Data Types, In-Built Functions)",
       "evidence": "Section 4.3 and Figure 2 show concept distributions based on 5-annotator majority voting across 164 HumanEval and 164 MBPP problems.",
       "supported": "moderate"
     },
     {
       "claim": "14 out of 38 programming concepts (37.8%) are completely absent from both HumanEval and MBPP",
       "evidence": "Figure 2 shows zero problems in concepts including OOPs, Linked Lists, Tree, Graph, Backtracking, and 9 others.",
       "supported": "moderate"
     },
     {
       "claim": "More than 80% of problems in both benchmarks are perceived as Easy (84.8% HumanEval, 89.6% MBPP)",
       "evidence": "Section 4.3 reports difficulty distributions from 5-annotator majority voting. Inter-annotator agreement: 39% complete agreement on HumanEval, 40.2% on MBPP.",
       "supported": "moderate"
     },
     {
       "claim": "Existing Code-LLMs perform poorly on PythonSaga: open-source models achieve <4.5% pass@1, closed-source <13% pass@1",
       "evidence": "Table 2 shows pass@1 scores: best open-source (Code Llama Python 13B) at 4.05%, GPT-4 at 12.43%, with n=20 samples per problem.",
       "supported": "moderate"
     },
     {
       "claim": "No model could solve any problem in six specific concepts: Basic Data Structures, Recursion, Hashing, Context Managers, Concurrency and Parallelism, and Max Flow",
       "evidence": "Figure 3 heatmap shows zero solved problems for these concepts across all 16 models.",
       "supported": "moderate"
     },
     {
       "claim": "Current benchmarks overestimate LLM generalization abilities for code generation",
       "evidence": "Comparison between high pass@1 on HumanEval/MBPP (Figure 6) and low pass@1 on PythonSaga (Table 2), combined with concept diversity analysis showing benchmark bias.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small annotator pool",
       "detail": "Only 5 annotators from the same institution (IIT Gandhinagar, same research group) classify all problems. With such a small, homogeneous pool, the concept and difficulty classifications may not generalize. No inter-annotator reliability metric beyond agreement percentage is reported."
     },
     {
       "flag": "No statistical tests for any comparisons",
       "detail": "All comparisons (model vs model, concept vs concept, benchmark vs benchmark) are made by comparing raw numbers without any statistical significance testing."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "PythonSaga problems are sourced from GeeksForGeeks and LeetCode — among the most common code training data sources. Manual rephrasing may not prevent memorization of solution patterns. No contamination analysis is performed."
     },
     {
       "flag": "Very few test cases per problem",
       "detail": "PythonSaga averages 3.7 test cases per problem (max 4). This is notably low and may not adequately verify functional correctness, potentially over- or underestimating model performance."
     },
     {
       "flag": "Missing generation hyperparameters",
       "detail": "Temperature, top-p, and other generation settings are not reported for any model. These settings significantly affect pass@k scores and make results non-reproducible."
     },
     {
       "flag": "Anonymous review URL as dataset link",
       "detail": "The dataset URL (anonymous.4open.science) is an anonymous review platform link, which may not be a permanent or reliable distribution method for the published version."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex model, the primary baseline benchmark analyzed in this paper."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark, the second primary baseline benchmark analyzed in this paper."
     },
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Major open-source Code-LLM evaluated in the study; established pass@k evaluation conventions."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Leading open-source Code-LLM family evaluated across multiple variants (base, instruct, Python)."
     },
     {
       "title": "Deepseek-coder: When the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Open-source Code-LLM evaluated in the study; representative of recent code-focused models."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "Code generation benchmark with difficulty levels from competitive programming platforms, directly relevant to benchmark design."
     },
     {
       "title": "DS-1000: A natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai", "Chengxi Li"],
       "year": 2023,
       "relevance": "Python-specific code generation benchmark focused on data science, relevant to benchmark diversity discussions."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Multi-language extension of HumanEval, directly relevant to code generation benchmark design."
     },
     {
       "title": "Dynabench: Rethinking benchmarking in NLP",
       "authors": ["Douwe Kiela", "Max Bartolo"],
       "year": 2021,
       "arxiv_id": "2104.14337",
       "relevance": "Addresses benchmark saturation and dynamic evaluation in NLP, a key motivation for PythonSaga."
     },
     {
       "title": "OpenCodeInterpreter: Integrating code generation with execution and refinement",
       "authors": ["Tianyu Zheng", "Ge Zhang"],
       "year": 2024,
       "arxiv_id": "2402.14658",
       "relevance": "Code generation model with execution feedback, evaluated in the study."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Recent open-source Code-LLM evaluated in the study; represents latest generation of code models."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The benchmark could be used by researchers evaluating Code-LLMs, but is primarily an academic contribution rather than a practitioner tool."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that models achieving 60-80% on HumanEval score below 13% on a more balanced benchmark challenges the narrative that Code-LLMs are approaching human-level capability."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised by the paper."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly criticizes the HumanEval/MBPP ecosystem as misleadingly easy, but does so in measured academic language."
     },
     "demo_ability": {
       "score": 1,
       "justification": "The dataset is released and could be downloaded and used, but there is no live demo or easy-to-use evaluation tool."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From IIT Gandhinagar (not a top-tier AI lab), but evaluates recognizable models like GPT-4 and Code Llama."
     }
   }
 }