ai-research-survey

scan.json (24867B)

---

 {
   "paper": {
     "title": "More Code, Less Reuse: Investigating Code Quality and Reviewer Sentiment towards AI-generated Pull Requests",
     "authors": [
       "Haoming Huang",
       "Pongchai Jaisri",
       "Shota Shimizu",
       "Lingfeng Chen",
       "Sota Nakashima",
       "Gema Rodríguez-Pérez"
     ],
     "year": 2026,
     "venue": "MSR 2026 (23rd International Conference on Mining Software Repositories)",
     "arxiv_id": "2601.21276",
     "doi": "10.1145/3793302.3793622"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or Zenodo archive is provided anywhere in the paper. The paper uses tools like Radon, CodeSage-Large, and PyRef but does not release its own analysis code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses the publicly available AIDev dataset [13] and the publicly available Emotion English DistilRoBERTa-base model [8]. The underlying data source is referenced with citations and is accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions tools (Radon, CodeSage-Large, PyRef) but does not specify versions or dependencies needed to replicate the analysis."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology section describes the approach conceptually but does not include commands, scripts, or a README for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports point estimates (e.g., AMR of 0.2867 vs. 0.1532) and p-values but no confidence intervals or error bars. Table 1 shows averages without uncertainty bounds."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Mann-Whitney test for redundancy comparison (p < 0.001, Section 3.1) and reports significance levels in Table 1 with * (p < 0.05) and *** (p < 0.001) annotations."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports the AMR for AI agents as 0.2867 vs. 0.1532 for humans, explicitly noting 'a nearly 1.87x increase' (Section 3.1). Table 1 also provides absolute differences. This provides enough context to assess magnitude, though no formal effect size metric (Cohen's d) is used."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for sample sizes is provided. Dataset A has 3,858 PRs and Dataset B has 617 PRs from a single repository, but no rationale is given for why these sizes are sufficient. The choice of crewAI as the sole repository for redundancy analysis is acknowledged as a limitation but not justified with a power calculation."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, interquartile ranges, or variance measures are reported for the main results. Table 1 shows only averages. The AMR values are reported as single numbers without spread measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares AI-generated PRs against human-generated PRs as the baseline across all metrics (code quality, redundancy, and sentiment). Human-PRs serve as the comparison group throughout."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is between AI-generated and human-generated PRs from the same dataset (AIDev, 2025), making the baselines contemporary and relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The MRS metric pipeline includes several components (CodeSage-Large embeddings, PyRef filtering, cosine similarity) but none are ablated to measure their individual contribution. It would be informative to know how results change without the PyRef refactoring filter, for example."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: LOC, multi-line strings, blank lines, cyclomatic complexity, the proposed Max Redundancy Score/AMR, and sentiment scores across seven emotion categories."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs is conducted. The sentiment analysis is automated using DistilRoBERTa. The paper analyzes existing human review comments as data, but humans do not evaluate the paper's own method outputs (e.g., whether detected redundancies are true positives)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is an observational mining study analyzing an existing dataset, not a machine learning evaluation requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by PR type (Human vs. Agent) across multiple categories: addition vs. removal in Table 1, complexity levels in Table 2, MRS distribution in Figure 2, and per-emotion sentiment scores in Figure 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases of the MRS approach are discussed. The paper does not show examples where the redundancy detection produced false positives or false negatives, nor does it discuss cases where the sentiment classifier may have been wrong."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that traditional code metrics (LOC, CC) show minimal differences between AI and human PRs (Table 1, Table 2), which is effectively a null result for those metrics. The interesting finding emerged only from the novel redundancy metric."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that LLM Agents 'disregard code reuse opportunities, resulting in higher levels of redundancy' (supported by AMR 0.2867 vs. 0.1532, p < 0.001) and that 'reviewers tend to express more neutral or positive emotions towards AI-generated contributions' (supported by Figure 3 sentiment analysis). These claims are backed by the results sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'the surface-level plausibility of AI code masks redundancy, leading to the silent accumulation of technical debt' (abstract) and 'reviewers may lower their vigilance' (Section 4.1). These are causal mechanisms proposed from observational data without controlling for confounders. The study design is correlational (comparing two groups on existing data) and cannot establish that plausibility causes lower vigilance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract make broad claims about 'AI-generated Pull Requests' and 'AI agents' generally, but the redundancy analysis (RQ1) is conducted on a single Python repository (crewAI). The paper acknowledges this in threats to validity but the title and abstract do not bound claims to Python or to crewAI specifically."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes that RLHF sycophancy explains the disconnect (Section 4.1) but does not consider alternative explanations for the observed patterns. For example, AI-generated code might receive more neutral sentiment simply because it is more uniform in style, or because reviewers apply different standards to bot-authored PRs. No alternative explanations for the redundancy finding are discussed either (e.g., whether the crewAI repository structure encourages certain patterns)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies CodeSage-Large [24] for code embeddings and Emotion English DistilRoBERTa-base [8] for sentiment analysis, with citations to the specific model releases. However, it does not specify which LLM agents generated the Agentic-PRs (this is inherited from the AIDev dataset)."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. It uses CodeSage-Large for embeddings and DistilRoBERTa for sentiment classification, neither of which requires prompt engineering in this context."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters are reported. The cosine similarity threshold for redundancy is not stated (MRS uses max similarity with no threshold). The DistilRoBERTa model's maximum token length of 512 is mentioned but no other parameters. No temperature or embedding generation settings are specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. It runs standard analysis tools (Radon, CodeSage-Large, PyRef, DistilRoBERTa) in a pipeline, not an agent framework."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents the data strategy: filtering to Python repositories with 500+ stars, separation into Human-PRs and Agentic-PRs, and the two-level dataset strategy (Dataset A: 3,858 PRs; Dataset B: 617 PRs from crewAI). Section 2.2 describes extracting pre/post file pairs from PRs. Section 2.3 describes filtering comments longer than 512 tokens and excluding bot comments."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated subsection discussing internal and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: PyRef may miss refactorings leading to false redundancy labels, crewAI may not represent all coding styles, Python-only focus may not generalize to other languages, and the sentiment model may misinterpret technical code review language (Section 6)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the threats section mentions limitations, the paper does not explicitly state what the results do NOT show. The broad title and abstract claims are not bounded to the tested setting. There is no explicit statement like 'our results should not be interpreted as...' or 'we did not test...' beyond the brief threats section."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The study uses the publicly available AIDev dataset [13], which provides the underlying PR data. An independent researcher could access the same data to verify the analysis."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes how the dataset was obtained (AIDev dataset), the filtering criteria (Python repositories, 500+ stars), and the two-level data strategy with specific counts (3,858 PRs for Dataset A, 617 PRs from crewAI for Dataset B)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study mines public GitHub data from the AIDev dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: AIDev dataset -> filter to Python repos with 500+ stars -> separate Human-PRs and Agentic-PRs -> extract file pairs (Figure 1) -> compute metrics. For redundancy: extract functions -> generate embeddings -> filter refactorings with PyRef -> compute MRS. For sentiment: extract review comments -> filter bots -> filter by token length -> classify with DistilRoBERTa."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. The Acknowledgments section mentions 'discussions held during the AI-Driven SE Summit 2025' but does not mention any grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Institute of Science Tokyo, Nara Institute of Science and Technology, Ritsumeikan University, Kyushu University, and University of British Columbia. These are academic institutions with no obvious conflict with the products being studied."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of a funding disclosure means this criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a mining study analyzing PR data. It does not evaluate a pre-trained model's capability on a benchmark. The models used (CodeSage-Large, DistilRoBERTa) are analysis tools, not subjects being benchmarked."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable for the same reason: this is a mining study, not a benchmark evaluation of model capabilities."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable: the study does not evaluate a pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a mining study of public GitHub data."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study mines publicly available repository and review data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants recruited for the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study with treatment assignment."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or computation time is reported for the CodeSage-Large embedding generation, PyRef refactoring detection, or DistilRoBERTa sentiment classification. The paper mentions 'computational costs' as a reason for restricting RQ1's redundancy analysis to one repository (Section 6), implying significant cost, but does not quantify it."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget, GPU hours, or hardware specifications are stated. The paper acknowledges computational constraints drove the Dataset B scoping decision but never quantifies the actual resources used."
       }
     }
   },
   "claims": [
     {
       "claim": "AI agents produce significantly more redundant code than human developers, with Average Max Redundancy (AMR) of 0.2867 for AI vs. 0.1532 for humans (1.87x increase), statistically significant at p < 0.001.",
       "evidence": "Section 3.1, Figure 2: Mann-Whitney test on MRS distributions from 617 PRs in the crewAI repository (Dataset B).",
       "supported": "moderate"
     },
     {
       "claim": "Traditional code metrics (LOC, cyclomatic complexity) show minimal differences between AI-generated and human-generated PRs.",
       "evidence": "Section 3.1, Tables 1 and 2: Most metrics show small or non-significant differences. 85% of code changes have zero complexity score for both groups.",
       "supported": "strong"
     },
     {
       "claim": "Reviewers tend to express more neutral or positive emotions towards AI-generated contributions than human ones.",
       "evidence": "Section 3.2, Figure 3: Sentiment analysis using DistilRoBERTa on review comments from Dataset A (3,858 PRs). Agentic-PRs had higher proportion of neutral, joy, and sadness; Human-PRs had higher disgust, anger, fear, and surprise.",
       "supported": "moderate"
     },
     {
       "claim": "The surface-level plausibility of AI code masks redundancy, leading to silent accumulation of technical debt.",
       "evidence": "Section 4.1-4.2: This is an interpretive claim connecting the RQ1 (higher redundancy) and RQ2 (positive/neutral sentiment) findings. No direct causal evidence is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "observational"
   ],
   "key_findings": "AI-generated pull requests contain significantly more semantic code redundancy than human-written PRs (AMR 0.2867 vs. 0.1532, p < 0.001) despite similar traditional code metrics. However, human reviewers express more neutral or positive sentiment toward AI-generated code compared to human-written code. The authors argue this disconnect reveals a blind spot where AI-generated code's surface plausibility masks underlying redundancy issues, potentially leading to silent technical debt accumulation.",
   "red_flags": [
     {
       "flag": "Single-repository redundancy analysis",
       "detail": "The key redundancy finding (RQ1, MRS/AMR) is based entirely on a single repository (crewAI). The paper acknowledges this but makes broad claims about 'AI agents' in the title and abstract without bounding the generalization."
     },
     {
       "flag": "No validation of the MRS metric",
       "detail": "The Max Redundancy Score is a novel metric proposed in this paper, but no validation is provided. There is no ground truth evaluation, no precision/recall analysis, and only 10 manually verified samples for PyRef filtering. It is unknown how well cosine similarity of CodeSage-Large embeddings captures true semantic redundancy."
     },
     {
       "flag": "Causal interpretation from observational data",
       "detail": "The paper interprets the disconnect between quality and sentiment causally (RLHF sycophancy -> plausible code -> reduced reviewer vigilance -> technical debt) from purely observational, cross-sectional data. Many confounders are possible but not discussed."
     },
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "Main results (AMR values, average metrics in Table 1) are reported as point estimates without confidence intervals, standard deviations, or error bars, making it impossible to assess result stability."
     },
     {
       "flag": "Sentiment classifier not validated for code review domain",
       "detail": "The Emotion English DistilRoBERTa-base model was trained on general English text. The paper acknowledges it 'may misinterpret technical discussions' but does not validate it on code review comments. Domain mismatch could systematically bias sentiment results."
     }
   ],
   "cited_papers": [
     {
       "title": "The Rise of AI Teammates in Software Engineering (SE) 3.0: How Autonomous Coding Agents Are Reshaping Software Engineering",
       "authors": ["Hao Li", "Haoxiang Zhang", "Ahmed E. Hassan"],
       "year": 2025,
       "arxiv_id": "2507.15003",
       "doi": "10.48550/arXiv.2507.15003",
       "relevance": "Introduces the AIDev dataset of autonomous AI agent pull requests used as the primary data source in this study."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Joel Becker", "Nate Rush", "Elizabeth Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "doi": "10.48550/arXiv.2507.09089",
       "relevance": "Studies adverse effects of AI on developer productivity, directly relevant to the survey's scope of LLM impact on software development."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R. Narasimhan"],
       "year": 2023,
       "relevance": "Key benchmark for evaluating LLM coding agents; this paper critiques its sole focus on pass rates rather than code quality."
     },
     {
       "title": "Do Users Write More Insecure Code with AI Assistants?",
       "authors": ["Neil Perry", "Megha Srivastava", "Deepak Kumar", "Dan Boneh"],
       "year": 2023,
       "doi": "10.1145/3576915.3623157",
       "relevance": "Studies adverse effects of AI coding assistants on code security, directly relevant to evaluating AI-assisted code quality."
     },
     {
       "title": "SycEval: Evaluating LLM Sycophancy",
       "authors": ["Aaron Fanous", "Jacob Goldberg", "Ank A. Agarwal", "Joanna Lin", "Anson Zhou", "Roxana Daneshjou", "Sanmi Koyejo"],
       "year": 2025,
       "arxiv_id": "2502.08177",
       "doi": "10.48550/arXiv.2502.08177",
       "relevance": "Evaluates LLM sycophancy behavior, relevant to the paper's claim that RLHF-trained models produce plausible but low-quality code."
     },
     {
       "title": "Towards Understanding Sycophancy in Language Models",
       "authors": ["Mrinank Sharma", "Meg Tong", "Tomasz Korbak", "David Duvenaud", "Amanda Askell"],
       "year": 2023,
       "relevance": "Foundational work on LLM sycophancy, cited as explanation for why AI-generated code appears superficially correct."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "Core RLHF paper cited as the mechanism behind AI code's tendency to optimize for human preference over correctness."
     },
     {
       "title": "Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback",
       "authors": ["Stephen Casper", "Xander Davies", "Claudia Shi"],
       "year": 2023,
       "relevance": "Discusses fundamental limitations of RLHF that may cause AI agents to prioritize perceived helpfulness over code quality."
     },
     {
       "title": "Code Representation Learning At Scale",
       "authors": ["Dejiao Zhang", "Wasi Ahmad", "Ming Tan", "Hantian Ding"],
       "year": 2024,
       "arxiv_id": "2402.01935",
       "doi": "10.48550/arXiv.2402.01935",
       "relevance": "Describes CodeSage-Large, the code embedding model used for semantic redundancy detection in this paper."
     }
   ]
 }