ai-research-survey

scan.json (27060B)

---

 {
   "paper": {
     "title": "DeputyDev - AI Powered Developer Assistant: Breaking the Code Review Logjam through Contextual AI to Boost Developer Productivity",
     "authors": [
       "Vishal Khare",
       "Vijay Saini",
       "Deepak Sharma",
       "Anand Raj",
       "Ankit Rana",
       "Anshul Yadav"
     ],
     "year": 2025,
     "venue": "arXiv (preprint, under review)",
     "arxiv_id": "2508.09676",
     "doi": "10.48550/arXiv.2508.09676"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["rct", "case-study"],
   "key_findings": "DeputyDev, an AI code review assistant using multi-agent LLM architecture with AST-based context retrieval, was evaluated in a 30-day double-controlled A/B experiment at TATA 1mg with ~721 PRs across 3 groups. The test group showed reductions of up to 28.8% in average review time, 42.2% per LOC, and 47.5% in median review time compared to controls. DeputyDev was most effective on smaller PRs (0-50 LOC: ~42% reduction) but showed mixed or negative results on medium and extra-large PRs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or experiment scripts are released. DeputyDev is a commercial SaaS product (deputydev.ai). No repository URL is provided for the tool or the experimental analysis code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The experiment data (PR review times, telemetry data) is proprietary to TATA 1mg and is not released. No dataset download or supplementary data is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided. The paper does not mention any reproducibility artifacts."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The experiment relies on internal TATA 1mg infrastructure and proprietary telemetry, making reproduction by external researchers impossible without detailed instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Only point estimates are reported (averages, medians, percentage changes in Table 2 and Table 3). No confidence intervals, error bars, or uncertainty measures are provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'statistically significant reduction' but no statistical significance test is reported anywhere in the paper — no p-values, t-tests, Mann-Whitney U, or any test statistic. The claim of statistical significance is entirely unsupported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports percentage changes with baseline context: e.g., average review time went from 239.57/278.14 hrs (controls) to 197.97 hrs (test), yielding -17.36%/-28.82%. Per-LOC and median reductions are similarly contextualized. Table 3 breaks this down by PR size category."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the sample size is given. The experiment yielded ~240 PRs per group (721 total after filtering) but no power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, interquartile ranges, or variance measures are reported for any metric. Only averages and medians are provided in Table 2."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two control sets (ControlSet1 and ControlSet2) serve as baselines, with 33% allocation each alongside the test set. Results are compared against both controls in Tables 2 and 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The two control groups are contemporaneous — they are from the same 30-day period (July 27 - August 27, 2024) as the test group, ensuring temporal validity."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "DeputyDev has multiple components (6 agents, reflection, blending engine, AST-based context retrieval) but no ablation study tests which components contribute to the observed improvements. The paper does not isolate the effect of any individual component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: average review time per PR, average review time per LOC, and median review time (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of DeputyDev's review quality is performed. The experiment measures only review time metrics. There is no assessment of whether DeputyDev's comments were useful, accurate, or accepted by developers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Post-hoc outlier exclusion thresholds (top 25%, bottom 10% by LOC) and repository balance criteria were applied to the same data on which results are reported. No pre-specified analysis plan or separation between exploratory and confirmatory analysis is documented."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides a breakdown by PR size category (S: 0-50 LOC, M: 51-100, L: 101-200, XL: 201-500), showing DeputyDev's differential effectiveness across categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes DeputyDev has 'mixed performance' in M and XL categories and mentions 'areas where DeputyDev's performance is suboptimal' (Section 2), but provides no qualitative examples of failed or unhelpful AI reviews, no error analysis of review quality, and no discussion of where the AI's feedback was wrong or misleading."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows DeputyDev increased review time per LOC vs ControlSet1 in the M category (+34.01%) and XL category (+100.30%). The paper acknowledges 'mixed performance' in these categories."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'statistically significant reduction' but no statistical significance test is reported in the paper. The results show percentage differences (Table 2) but 'statistically significant' is asserted without any supporting statistical test or p-value."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a double-controlled A/B experiment with random 33% allocation of PRs to three groups (Section 9). While the design has limitations (no blinding, PR-level rather than engineer-level randomization), the basic causal design — random assignment with concurrent controls — is adequate for the causal claims about review time reduction."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to 'Break the Code Review Logjam' and 'Boost Developer Productivity' without qualification. Results come from a single company (TATA 1mg) using one LLM (GPT-4o) in one ecosystem (Bitbucket), but claims are framed broadly. The abstract mentions 'external companies' using it but provides no data from those deployments."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider: Hawthorne effect (engineers aware of the experiment), novelty effect, the possibility that immediate AI feedback simply reminds authors to self-fix rather than providing quality reviews, or that reduced review time might reflect reviewers rubber-stamping after seeing AI feedback."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures review time reduction but frames this as 'boosting developer productivity' (title) and 'improving development workflow timelines and code quality' (abstract). Faster review time is a proxy for productivity, and no measurement of code quality is performed despite quality being claimed as an improvement. The gap between the proxy (time) and the claimed outcomes (productivity, quality) is never acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'GPT-4o' (Section 8) and 'Claude 3.5 Sonnet' (Section 6.2.3) without snapshot dates or API version identifiers. Model behavior varies across versions; neither model is pinned to a specific version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described only in natural language (e.g., 'This agent is responsible to identify and recommend corrective code for any security issues,' Section 6.2.3). No actual prompt text is provided in the paper or appendix. The XML output schema in Appendix A is a response format, not the input prompt."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, etc.) are reported for any LLM call. The paper uses both GPT-4o and Claude 3.5 Sonnet without stating any API parameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in substantial detail: 6 specialized agents (security, code communication, performance, maintainability, errors, business validation), reflection design pattern for iterative improvement, AST-based semantic chunking for context retrieval (Section 6.1), lexical+semantic search fusion (Section 6.1), blending engine with confidence scoring and comment overlap summarization (Section 6.4), and mathematical formalization (Section 6.5)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 documents: outlier exclusion (top 25% and bottom 10% by LOC), repository balance filtering (≥10 PRs per set or equal count), experiment duration (30 days, July 27 - August 27, 2024), and 33% allocation per set."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section, threats-to-validity section, or any dedicated discussion of the study's shortcomings. The conclusion mentions 'areas where DeputyDev's performance is suboptimal' only in passing in the introduction (Section 2)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. Obvious threats like Hawthorne effect, single-company bias, lack of blinding, and PR-level vs engineer-level randomization are not mentioned."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not acknowledge that results are limited to one company, one LLM, one version control system, or that code quality was not measured despite being claimed as an improvement."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (PR review times, telemetry) is not available. Only aggregated statistics are presented in Tables 2-3. No supplementary data files or download links are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 describes data collection: Bitbucket telemetry at TATA 1mg, 30-day experiment period (July 27 - August 27, 2024), three-group allocation at 33% each, over 200 engineers involved. Review time metrics (pick-up time, review time, closure cycle) are defined in the abstract."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'over 200 engineers' participated but does not describe how they were selected — whether all TATA 1mg engineers were included, whether participation was voluntary, or whether specific teams were targeted. No recruitment or selection process is described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While filtering criteria are stated (outlier exclusion by LOC percentiles, repository balance), the paper does not document how many PRs were collected initially, how many were removed at each filtering stage, or the full pipeline from raw telemetry to the final 721 PRs analyzed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or acknowledgments are present. The work was conducted at TATA 1mg, a commercial entity, but no funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: all authors are from 'TATA 1mg Healthcare Solutions Private Limited' with institutional email addresses."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "TATA 1mg employees are evaluating DeputyDev, which TATA 1mg sells as a SaaS product to external companies. The employer has a direct financial interest in DeputyDev being shown to be effective."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. The authors work at the company that commercializes DeputyDev as SaaS, but this conflict is not explicitly declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It is an A/B experiment measuring real-world PR review times, not model performance on test data."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study is an A/B experiment measuring review times in production, not a benchmark evaluation of model knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation is performed. The study measures the effect of an AI tool on human review workflows."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration. The experiment involves 200+ engineers but no pre-registration link (OSF, AsPredicted, etc.) is provided."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval for the experiment involving 200+ engineer participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'over 200 engineers' but provides no demographic information: no experience levels, team distribution, programming languages used, or any participant characterization."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "Inclusion/exclusion criteria are described for repositories (balance requirement) and PRs (outlier exclusion by LOC) but not for engineer participants. No criteria for which engineers were included or excluded from the study."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '33% allocation to each' set but does not describe the randomization mechanism — how PRs were assigned to groups (hash, random number generator, manual), whether randomization was stratified, or what unit was randomized (PR, repository, engineer)."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Engineers likely knew whether their PR received DeputyDev review (it posts comments on the PR), creating potential for behavioral changes in the test group."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is provided. The paper does not report how many PRs or engineers were in the initial pool vs. the final analysis, beyond the final counts in Table 2 (244, 238, 239)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API costs, or latency figures are reported. Section 5 mentions cost as a reason not to send entire codebases but does not quantify DeputyDev's actual per-review cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated — no API spend, no token counts, no hardware specifications for the experiment or the production system."
       }
     }
   },
   "claims": [
     {
       "claim": "DeputyDev reduces average PR review time by up to 28.82% compared to controls.",
       "evidence": "Table 2 shows TestSet average review time of 197.97 hrs vs ControlSet1 (239.57 hrs, -17.36%) and ControlSet2 (278.14 hrs, -28.82%), based on 721 PRs across three groups.",
       "supported": "moderate"
     },
     {
       "claim": "DeputyDev reduces average review time per LOC by up to 42.19%.",
       "evidence": "Table 2 shows TestSet average of 7.50 hrs/LOC vs ControlSet1 (12.97 hrs/LOC, -42.19%) and ControlSet2 (12.29 hrs/LOC, -38.98%).",
       "supported": "moderate"
     },
     {
       "claim": "DeputyDev reduces median review time by up to 47.52%.",
       "evidence": "Table 2 shows TestSet median of 0.41 hrs vs ControlSet1 (0.76 hrs, -46.36%) and ControlSet2 (0.78 hrs, -47.52%).",
       "supported": "moderate"
     },
     {
       "claim": "DeputyDev is most effective on smaller PRs.",
       "evidence": "Table 3 shows the largest review time reduction for small PRs (0-50 LOC: -43.87%/-41.40%) while medium and extra-large categories show mixed or negative results (+34.01%/+100.30% vs CS1).",
       "supported": "moderate"
     },
     {
       "claim": "The results are statistically significant.",
       "evidence": "Claimed in the abstract ('statistically significant reduction') but no statistical test, p-value, or test statistic is reported anywhere in the paper.",
       "supported": "unsupported"
     },
     {
       "claim": "There is weak correlation between lines of code and code review time.",
       "evidence": "Figure 5 shows correlation coefficients of 0.095, 0.004, and 0.052 for test, control 1, and control 2 sets respectively.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own commercial product",
       "detail": "All authors are TATA 1mg employees evaluating DeputyDev, which TATA 1mg sells as SaaS to external companies. This is a textbook conflict of interest: the evaluators have a direct financial interest in positive results. No independent evaluation or external replication is provided."
     },
     {
       "flag": "Statistical significance claimed without any significance test",
       "detail": "The abstract asserts 'statistically significant reduction' but the paper contains no p-values, no test statistics, no significance tests of any kind. The claim is stated as fact with zero statistical support."
     },
     {
       "flag": "Post-hoc outlier exclusion may bias results",
       "detail": "PRs in the top 25% and bottom 10% by LOC were excluded before analysis (Section 9, point 4). These thresholds appear arbitrary and are not justified. Post-hoc exclusion of 35% of data could be tuned to favor the test group. No sensitivity analysis shows results hold under different thresholds."
     },
     {
       "flag": "No blinding in the A/B experiment",
       "detail": "Engineers in the test group received visible AI comments on their PRs. They knew they were getting AI assistance, which could change behavior (Hawthorne effect). Reviewers might also spend less time on PRs already reviewed by AI, regardless of AI quality."
     },
     {
       "flag": "Negative results partially obscured by aggregation",
       "detail": "Table 3 shows DeputyDev INCREASED review time per LOC in medium PRs (+34.01% vs CS1) and extra-large PRs (+100.30% vs CS1). These negative results are acknowledged as 'mixed performance' but the paper's framing and conclusion emphasize the positive aggregate results."
     },
     {
       "flag": "Experiment model differs from production model",
       "detail": "The A/B experiment used GPT-4o (Section 9, point 6: 'comprehensive workflow and while using OpenAI's GPT-4o as LLM'), but the production system uses Claude 3.5 Sonnet for code reviews (Section 6.2.3, Section 8). Results from one model cannot be assumed to transfer to another."
     },
     {
       "flag": "No measurement of review quality",
       "detail": "The paper claims to improve 'code quality' (Section 3, hypothesis 3) but measures only review time. Faster reviews could indicate rubber-stamping or lower-quality human reviews after seeing AI feedback. No quality metric (defect escape rate, comment acceptance rate, post-merge bug rate) is measured."
     }
   ],
   "cited_papers": [
     {
       "title": "Code review automation: Strengths and weaknesses of the state of the art",
       "authors": ["Rosalia Tufano", "Ozren Dabić", "Antonio Mastropaolo", "Matteo Ciniselli", "Gabriele Bavota"],
       "year": 2024,
       "relevance": "Directly surveys AI-based code review automation approaches, identifying strengths and limitations of current methods."
     },
     {
       "title": "Using pre-trained models to boost code review automation",
       "authors": ["Rosalia Tufano", "Simone Masiero", "Antonio Mastropaolo", "Luca Pascarella", "Denys Poshyvanyk", "Gabriele Bavota"],
       "year": 2022,
       "relevance": "Evaluates pre-trained language models for automating code review, directly relevant to LLM-based code analysis."
     },
     {
       "title": "CommentFinder: A simpler, faster, more accurate code review comments recommendation",
       "authors": ["Yang Hong", "Chakkrit Tantithamthavorn", "Patanamon Thongtanunam", "Aldeida Aleti"],
       "year": 2022,
       "relevance": "Proposes an automated system for recommending code review comments, a closely related task to DeputyDev's core functionality."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu", "Nuo Chen"],
       "year": 2024,
       "relevance": "Multi-agent framework for software development that inspired DeputyDev's agentic architecture; directly relevant to agentic AI workflows."
     },
     {
       "title": "Self-Refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Foundational work on LLM self-reflection and iterative improvement, the pattern used in DeputyDev's reflection design."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Edward Berman"],
       "year": 2023,
       "relevance": "Proposes verbal self-reflection for LLM agents, directly relevant to the reflection design pattern used in agentic AI systems."
     },
     {
       "title": "Let me speak freely? A study on the impact of format restrictions on performance of large language models",
       "authors": ["Zhi Rui Tam", "Cheng-Kuang Wu", "Yi-Lin Tsai", "Chieh-Yen Lin", "Hung yi Lee", "Yun-Nung Chen"],
       "year": 2024,
       "relevance": "Studies how enforcing structured output formats affects LLM reasoning quality, directly relevant to LLM deployment practices."
     },
     {
       "title": "AI-assisted assessment of coding practices in modern code review",
       "authors": ["Manushree Vijayvergiya", "Małgorzata Salawa", "Ivan Budiselić"],
       "year": 2024,
       "relevance": "Evaluates AI-assisted code review practices with human studies, directly comparable to DeputyDev's goals and approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "DeputyDev is a deployed tool available as SaaS, already in production use at TATA 1mg and external companies — immediately usable by practitioners."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected finding that AI code review tools can reduce review turnaround time; no surprising or counterintuitive results."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns are raised by the paper."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with existing claims or practices."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Available as SaaS at deputydev.ai but not open source; users can sign up but cannot inspect or self-host the system."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "TATA 1mg is a known Indian healthcare company but not a major AI research lab or globally recognized tech brand."
     }
   }
 }