ai-research-survey

scan.json (25831B)

---

 {
   "paper": {
     "title": "AI-Generated Code Is Not Reproducible (Yet): An Empirical Study of Dependency Gaps in LLM-Based Coding Agents",
     "authors": ["Bhanu Prakash Vangala", "Ali Adibifar", "Ashish Gehani", "Tanu Malik"],
     "year": 2025,
     "venue": "AAAI 2026",
     "arxiv_id": "2512.22387"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The authors describe their experimental infrastructure (AWS EC2 instances, SciUnit) and iterative resolution protocol but do not release the code implementing these experiments."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 300 generated projects, 100 prompts, and dependency analysis results are not released. No download link or data archive is provided. The prompt template is shown in Figure 1 but is only a template with placeholders, not the actual 100 prompts used."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'AWS EC2 instances (t2.large with 4 vCPUs and 16GB RAM running Ubuntu 22.04 LTS)' and SciUnit v0.4.post135, but does not provide a requirements.txt, Dockerfile, or detailed dependency list for reproducing the experimental infrastructure itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Algorithm 1 describes the iterative resolution protocol conceptually, but there are no scripts, commands, or README explaining how to replicate the full study."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (68.3% overall, 89.2% Python, etc.) are reported as point estimates without confidence intervals or error bars. The runtime multipliers in Equation 17 include ± values (e.g., '12.3 ± 4.2') but these appear to be standard deviations across projects, not confidence intervals on the main reproducibility claims."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims differences between agents (Claude 73% vs Codex 60%) and between languages (Python 89.2% vs Java 44.0%) without any statistical significance tests. No p-values, chi-squared tests, or other tests are reported for these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context: e.g., '68.3% of projects execute out-of-the-box', '13.5× average expansion from declared to actual runtime dependencies', 'Python 89.2% vs Java 44.0%'. Tables 1-6 provide raw counts and percentages enabling readers to assess magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 100 prompts (40 Python, 35 JavaScript, 25 Java) is not justified by power analysis or formal reasoning. The allocation is described as 'informed by the high prevalence of these languages' but no justification is given for why 100 prompts (or 300 total projects) is sufficient to support the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The runtime multipliers include ± notation (Equation 17: '12.3 ± 4.2' for Python), but the main reproducibility results are single-run numbers with no variance or spread measures. Each prompt-agent pair was tested once; no repeated runs are described."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three LLM agents (Claude Code, OpenAI Codex, Gemini) are compared against each other as baselines. The comparison is structured across agents and languages in Tables 1-3 and the success rate matrix (Equation 15)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The three agents evaluated (Claude Code with Opus 4.1, OpenAI Codex 0.52.0, Gemini 2.5 Pro) are all 2025 state-of-the-art coding agents, representing the most contemporary options available."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not test which components of its methodology matter (e.g., whether prompt wording affects results, whether environment reset procedures matter, or whether results change with different prompt formulations)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: executable reliability (success rate), completeness gap, runtime multiplier, error type distribution, and per-language/per-agent breakdowns. Tables 1, 2, 4, 5, 6 cover different evaluation dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the generated code quality is included. The evaluation is entirely automated (does the code execute or not). Human evaluation could have assessed code readability, correctness beyond execution, or documentation quality, which are relevant to the reproducibility claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning study with train/test splits. The paper generates and tests projects; there is no held-out evaluation scenario applicable here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-category breakdowns are provided: by language (Table 2), by agent (Table 1), by agent-language combination (Table 3), by error type (Table 5), and by missing dependency distribution (Table 4)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure cases are discussed extensively. Table 5 categorizes all 95 failures into Code Bugs (52.6%), Not Processed (16.8%), Other (15.8%), Dependency (10.5%), and Environment (4.2%). Specific examples are given (e.g., 'malformed Maven XML configurations', 'compressed multi-file projects into single files')."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is essentially a negative result: AI-generated code is not reproducible. Additionally, the paper reports that the majority of failures are NOT dependency issues as initially hypothesized, but code generation errors (52.6%), which counters their own initial framing."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: '68.3% of projects execute out-of-the-box' (Table 1/6), 'Python 89.2%, Java 44.0%' (Table 2), '13.5× average expansion' (Table 6). All numerical claims in the abstract match the results section."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about why languages differ ('Python's success stems from its simple flat dependency structure', 'Java's challenges arise from complex XML configuration') and about agent specializations ('Gemini likely trained heavily on data science notebooks'). These are speculative explanations not supported by the study design, which only observes outcomes without testing causal mechanisms."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper title and abstract make broad claims about 'AI-Generated Code' and 'LLM-Based Coding Agents' generally, but tests only 3 agents on 100 prompts covering specific task types. The conclusion states 'AI coding agents generate code that looks complete but isn't reproducible' as a general claim. The task distribution (web scraping, data analysis, ML pipelines, etc.) is not justified as representative of all development work."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed results. For example, the paper does not consider whether prompt design could explain the failure rates, whether the clean-environment assumption is realistic, or whether different prompting strategies could improve results. No threats-to-validity or alternative-explanations section exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Model versions are specified: 'Claude Code Agent(Opus.4.1)', 'OpenAI Codex Agent(0.52.0)', and 'Gemini Code Agent(2.5.Pro)' in the Methodology section. These include version identifiers beyond just marketing names."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 1 shows only a prompt TEMPLATE with placeholders like '[Task Description]', '[Environment]', and '[Language]'. The actual 100 task descriptions used are not provided. The reader cannot reconstruct the exact prompts sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported for the LLM API calls. Temperature, top-p, max tokens, and other sampling parameters are not mentioned. These settings significantly affect output quality and reproducibility."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party coding agents (Claude Code, Codex, Gemini) as black boxes. The authors cannot be expected to describe internal scaffolding they have no access to."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: 100 prompts designed across 3 languages (40/35/25 split), each sent to 3 agents producing 300 projects, tested in standardized AWS EC2 environments with 91 baseline packages. Algorithm 1 documents the iterative resolution process. The environment reset procedure is formalized in Equations 9-10."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The Discussion section discusses implications but does not acknowledge limitations of the study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as: prompt design bias, single-run evaluation (no repeated trials), limited prompt diversity, or whether the clean-environment assumption matches real developer workflows."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show (e.g., that results apply only to these 3 agents, these specific task types, single-run generation without iteration, etc.). The title and conclusion generalize broadly without bounding the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (the 300 generated projects, dependency analysis outputs, error logs, SciUnit traces) is not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in detail: 100 prompts distributed across 3 languages, sent to 3 LLM agents, tested on standardized AWS EC2 instances. The prompt template is shown in Figure 1, the environment baseline is specified (91 packages, Ubuntu 22.04 LTS), and the iterative resolution protocol is formalized in Algorithm 1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study evaluates LLM-generated code, not human subjects. Data sources are standard LLM APIs, not human-derived."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is well documented: prompt design → code generation by 3 agents → environment standardization (Equations 9-10) → execution attempt → iterative resolution (Algorithm 1) → SciUnit runtime capture (Equations 11-13) → error classification (Table 5). Each stage is described with specific tools and counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: University of Missouri (Vangala, Adibifar, Malik) and SRI International (Gehani). Neither institution is a vendor of the evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence of the funder cannot be assessed. The absence of funding disclosure is itself a concern."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Note that co-author Tanu Malik is associated with SciUnit, which is prominently used and promoted in the paper as a key tool, but this potential conflict is not acknowledged."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This study does not evaluate pre-trained model capability on a benchmark. It tests whether LLM-generated code is reproducible (i.e., whether it runs in a clean environment). The concern is code generation quality, not whether models memorized test answers."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The study generates new projects from new prompts; there is no pre-existing benchmark that could overlap with training data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The study uses original prompts, not a published benchmark. There is no contamination concern as the tasks are novel."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It evaluates LLM-generated code execution, not human behavior."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or per-project generation costs are reported for running the 300 code generation requests across three commercial LLM agents."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The AWS EC2 instance type is mentioned (t2.large) but total compute budget, number of instance-hours, or total API spend is not stated."
       }
     }
   },
   "claims": [
     {
       "claim": "Only 68.3% of AI-generated projects execute successfully out-of-the-box without any manual intervention.",
       "evidence": "Table 1 and Table 6 report 205 of 300 projects succeeding. The methodology section defines success as execution in a clean environment using only LLM-specified dependencies with zero manual intervention.",
       "supported": "strong"
     },
     {
       "claim": "Python achieves 89.2% reproducibility while Java achieves only 44.0%.",
       "evidence": "Table 2 reports Python 107/120 success (89.2%) and Java 33/75 success (44.0%). Results are broken down by language in Equation 14.",
       "supported": "strong"
     },
     {
       "claim": "There is a 13.5x average expansion from declared to actual runtime dependencies.",
       "evidence": "Table 6 and the Conclusion report this multiplier. The SciUnit Provenance Analysis section gives a concrete example: 3 claimed dependencies → 52 runtime packages for an ML project (17x). Equation 17 gives per-language multipliers.",
       "supported": "moderate"
     },
     {
       "claim": "52.6% of failures stem from code generation errors, not dependency issues (only 10.5%).",
       "evidence": "Table 5 classifies all 95 failed projects: Code Bugs 50 (52.6%), Not Processed 16 (16.8%), Other 15 (15.8%), Dependency 10 (10.5%), Environment 4 (4.2%).",
       "supported": "strong"
     },
     {
       "claim": "Each LLM agent has hidden language specializations not advertised by vendors.",
       "evidence": "Table 3 and Equation 15 show Gemini achieves 100% Python but only 28% Java; Claude achieves 80% Java vs. Codex at 24%. The Discussion section speculates about training data explanations.",
       "supported": "moderate"
     },
     {
       "claim": "Each failed project requires approximately 15 minutes of manual debugging.",
       "evidence": "Stated in the Introduction ('we found takes 15 minutes on average') and Discussion ('Our manual processing averaged 15 minutes per failed project'). No formal timing methodology or variance is reported for this estimate.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Only 68.3% of 300 AI-generated projects (by Claude Code, Codex, and Gemini) execute out-of-the-box in clean environments, with Python at 89.2% and Java at 44.0%. The majority of failures (52.6%) stem from code generation errors rather than missing dependency declarations (10.5%). A three-layer dependency framework reveals a 13.5x average expansion from declared to runtime dependencies. Agents show distinct, unadvertised language specializations: Gemini achieves perfect Python reproducibility but struggles with Java, while Claude excels at Java where others fail.",
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any limitations, threats-to-validity, or scope-bounding section. This is a significant omission for a study making broad claims about 'AI-Generated Code' based on 100 prompts across 3 agents."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Differences between agents (Claude 73% vs Codex 60%) and languages (Python 89.2% vs Java 44.0%) are stated without any significance tests. With sample sizes of 25-40 per cell, some observed differences may not be statistically significant."
     },
     {
       "flag": "Single-run evaluation with no variance reporting",
       "detail": "Each prompt-agent pair appears to have been tested only once. LLM outputs are stochastic, so results may vary across runs. No repeated trials, temperature settings, or seed information is provided."
     },
     {
       "flag": "No artifacts released",
       "detail": "Despite studying reproducibility, the paper itself is not reproducible: no code, data, prompts, or generated projects are released. This is ironic given the paper's central thesis."
     },
     {
       "flag": "Unbounded generalization",
       "detail": "The title and conclusion generalize to 'AI-Generated Code' and 'LLM-Based Coding Agents' broadly, but the study tests only 3 specific agents on 100 prompts of specific task types. No scope boundaries are acknowledged."
     },
     {
       "flag": "Potential undisclosed conflict of interest with SciUnit",
       "detail": "Co-author Tanu Malik appears to be associated with the SciUnit tool (cited as 'Ton That, Fils, Yuan, and Malik 2017'), which is prominently featured as a key tool in the methodology. This connection is not disclosed as a potential conflict."
     },
     {
       "flag": "Speculative causal explanations",
       "detail": "The Discussion contains speculative causal claims about why agents differ ('Gemini likely trained heavily on data science notebooks', 'Claude appears optimized for enterprise patterns') without evidence. These are plausible hypotheses stated as explanations."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation evaluation, which this paper critiques for ignoring reproducibility."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for code generation, another benchmark critiqued in this paper for missing reproducibility evaluation."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Major LLM code generation benchmark that this paper cites as recognizing environment configuration as a bottleneck."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["N. Jain", "K. Han", "A. Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Code generation benchmark addressing contamination but still assuming reproducible environments exist."
     },
     {
       "title": "A reproducibility and generalizability study of large language models for query generation",
       "authors": ["M. Staudinger", "W. Kusa", "F. Piroi", "A. Lipani", "A. Hanbury"],
       "year": 2024,
       "relevance": "Studies LLM output consistency and reproducibility, directly related to the reproducibility crisis in LLM-generated artifacts."
     },
     {
       "title": "Assessing consistency and reproducibility in the outputs of large language models: Evidence across diverse finance and accounting tasks",
       "authors": ["J. J. Wang", "V. X. Wang"],
       "year": 2025,
       "arxiv_id": "2503.16974",
       "relevance": "Studies LLM output inconsistency across domains, relevant to understanding reproducibility challenges of AI systems."
     },
     {
       "title": "Hidden Technical Debt in Machine Learning Systems",
       "authors": ["D. Sculley", "G. Holt", "D. Golovin"],
       "year": 2015,
       "relevance": "Seminal paper on technical debt in ML systems, referenced in context of hidden costs of AI-generated code."
     },
     {
       "title": "On Reproducible AI: Towards Reproducible Research, Open Science, and Digital Scholarship in AI Publications",
       "authors": ["O. E. Gundersen", "Y. Gil", "D. W. Aha"],
       "year": 2018,
       "relevance": "Foundational work on AI reproducibility that this paper extends to AI-generated code."
     },
     {
       "title": "Improving Reproducibility in Machine Learning Research (A Report from the NeurIPS 2019 Reproducibility Program)",
       "authors": ["J. Pineau", "P. Vincent-Lamarre", "K. Sinha"],
       "year": 2021,
       "relevance": "Major effort to improve ML reproducibility, directly relevant to the reproducibility methodology assessment in this survey."
     },
     {
       "title": "An empirical comparison of dependency network evolution in seven software packaging ecosystems",
       "authors": ["A. Decan", "T. Mens", "P. Grosjean"],
       "year": 2019,
       "relevance": "Empirical study of software dependency ecosystems, providing context for the dependency complexity findings in this paper."
     }
   ]
 }