ai-research-survey

scan.json (24436B)

---

 {
   "paper": {
     "title": "From Horizontal Layering to Vertical Integration: A Comparative Study of the AI-Driven Software Development Paradigm",
     "authors": ["Chi Zhang", "Zehan Li", "Ziqian Zhong", "Haibing Ma", "Dan Xiao", "Chen Lin", "Ming Dong"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.22667",
     "doi": "10.48550/arXiv.2601.22667"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["case-study", "qualitative"],
   "key_findings": "Through a multiple-case comparative study of a traditional enterprise (brownfield) and an AI-native startup (greenfield), the paper claims 8x to 33x reductions in resource consumption when transitioning from horizontal functional silos to vertically integrated 'Super-Cells' where AI-augmented engineers own end-to-end delivery. The efficiency baselines are counterfactual estimates, not observed controls. The paper proposes Human-AI Collaboration Efficacy as a new optimization metric and identifies an 'AI Distortion Effect' that suppresses returns to labor scale.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code, analysis scripts, or repository links are provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No interview transcripts, coding logs, git commit data, or other qualitative data are released. The paper references project artifacts (git logs, meeting minutes) but does not make them available."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative case study with no computational experiments requiring environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The interview guides, coding schemes, and detailed case selection procedures are not included."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative case study with no statistical analysis. The quantitative claims (8x, 33x) are simple ratios from counterfactual estimates, not statistical results."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical comparisons are made; this is qualitative research."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No statistical effect sizes are relevant to this qualitative case study design."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper studies two cases but provides no justification for why two cases are sufficient, no discussion of theoretical saturation, and no reference to case study sample size guidelines (e.g., Eisenhardt recommends 4-10 cases)."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs or repeated measurements; qualitative case study design."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The 'traditional baseline' for each case is a counterfactual estimate (what would have happened under traditional methods), not an observed baseline. No actual control group or prior measured project was used for comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are hypothetical counterfactuals based on function point analysis and 'historical data from similar past projects' (Section 5.1.2), but no specific past projects are identified or their data presented."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative case study examining organizational transformation, not a system with decomposable components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The primary metric is person-months reduction. While the paper discusses cognitive load, role transformation, and organizational complexity conceptually, these are not measured with distinct metrics — only the person-month ratio is quantified."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "There is no system output to evaluate. The paper is about organizational structure, not a technical system."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark or test set is used; this is qualitative organizational research."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides a cross-case comparison broken down by five dimensions: Structure, Process, Role, Efficiency, and Key Issues, comparing traditional vs. AI-driven paradigm across both cases."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 discusses risks including ethical concerns, workforce displacement, and dynamic instability. Section 7.4 discusses limitations including elite bias, Hawthorne effect, and long-term technical debt. The 'Black Box Effect' risk is discussed in Section 5.4."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that AI can sometimes slow experienced developers (citing Becker et al. 2025), discusses cognitive overload risks, and acknowledges that the efficiency gains may not generalize to median-level developers (Section 7.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims '8× to 33× reductions in resource consumption' but these are based on comparing actual project effort against counterfactual estimates, not observed controls. The abstract presents these as validated findings without qualifying the counterfactual nature of the comparison."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims throughout — 'transitioning from Horizontal Layering to Vertical Integration yields 8× to 33× reductions' — but the study design (two cases, no control group, counterfactual baselines) is inadequate for causal inference. Confounds include the Hawthorne effect (acknowledged in limitations), team selection bias, and project novelty."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper generalizes from two cases to sweeping claims about 'the new paradigm' for software engineering organizations broadly. While limitations mention elite bias and scaling concerns, the title and abstract frame results as a general 'paradigm shift' rather than bounding them to the specific contexts studied."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 discusses several alternative explanations: the Hawthorne Effect (pilot team awareness), elite bias (high-performing engineers may not be representative), Organization B's dual role introducing observer bias, and model dependency affecting generalizability."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures person-months and frames this as 'efficiency' and 'resource consumption reduction' without discussing what these proxies miss — code quality, long-term maintainability, defect rates, customer satisfaction. Section 7.4 briefly mentions maintainability concerns but does not frame person-month reduction as a proxy for broader organizational effectiveness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not run any AI models as part of its research methodology. It studies organizations that use AI tools but does not specify or evaluate particular model versions."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting as part of its research methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No AI models are run as part of the research; this is qualitative organizational research."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in the research methodology itself."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes three data sources (participant observation, semi-structured interviews, project documentation) in Section 4.3 but does not describe how qualitative data was coded, analyzed, or reduced. No coding scheme, thematic analysis procedure, or inter-coder reliability is reported."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 is a dedicated 'Limitations' subsection covering elite bias, methodological confounds, long-term technical debt, and model dependency."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 discusses specific threats: the Hawthorne effect from the Seed Team's pilot status, Organization B's dual role as partner and subject introducing observer bias, elite bias from high-performing engineers, and the 3-month observation window being too short to assess burnout."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.4 states specific scope boundaries: results may not apply to median-level developers, cognitive sustainability over periods longer than 3 months is untested, scaling from a 4-person unit to enterprise-scale is not captured, and efficiency gains are contingent on current LLM capabilities."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No interview transcripts, observation notes, git logs, or project documentation are made available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes three data collection methods: immersive participant observation, semi-structured interviews with key stakeholders, and analysis of project documentation (git logs, resource reports, meeting minutes)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states a 'Seed Team was selected from their existing workforce' (Section 4.1.1) but does not describe the selection criteria, how many candidates were considered, or whether the selection could introduce bias toward favorable outcomes."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how qualitative data was transformed from raw interviews and observations into the findings presented. No coding process, theme derivation, or analytical steps are described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed. The Declarations section explicitly states that Zhang, Li, Zhong, Xiao, and Lin are full-time employees of Organization B, and Ma serves as part-time Chief Scientist."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Organization B serves as both the research partner/subject and the employer of 5 of 7 authors. The company has a direct financial interest in demonstrating the superiority of its AI-driven organizational model. The competing interests statement acknowledges this dual role but the funder is clearly not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The Declarations section includes a 'Competing Interests' statement that explicitly acknowledges the employment relationship and dual role, though it claims adherence to 'academic standards of objectivity.' Dong M. is stated to have 'no employment ties to Organization B.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for this qualitative study."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite conducting interviews with employees."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Interviewees are described only by role (Seed Team members, Company G management, AI engineers from Organization B). No demographic details, experience levels, or number of participants are provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No criteria for selecting interview participants are described. The 'Seed Team' selection process is not documented."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative case study, not an experimental study requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not feasible in qualitative case study research."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information is provided about how many people were interviewed, whether any declined, or whether any data was excluded."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative organizational study, not a system with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments are conducted."
       }
     }
   },
   "claims": [
     {
       "claim": "Transitioning from Horizontal Layering to Vertical Integration yields 8x to 33x reductions in resource consumption.",
       "evidence": "Case A (Project W): 12 person-months actual vs. ~100 person-months estimated traditional (8.3x). Case B (AI-CRM): 1.5 person-months actual vs. ~50 person-months estimated traditional (33x). Sections 5.1-5.2. Traditional estimates based on function point analysis and historical data.",
       "supported": "weak"
     },
     {
       "claim": "AI adoption enables engineers to transition from execution-focused coding to architectural supervision roles.",
       "evidence": "Qualitative interview quotes from Seed Team engineer (Section 5.4): 'Previously, I spent approximately nine hours writing code and thirty minutes on planning and design. Now the ratio has inverted.' Supported by participant observation.",
       "supported": "moderate"
     },
     {
       "claim": "The AI Distortion Effect diminishes returns to labor scale while amplifying technological leverage in TFP.",
       "evidence": "Theoretical analysis in Section 6.3 based on Solow TFP framework. Figure 4 shows changing TFP component weights, but values are described as 'developed through expert consensus during qualitative interviews' — not empirically measured.",
       "supported": "weak"
     },
     {
       "claim": "Traditional cognitive bandwidth is progressively underutilized as engineers gain experience (Table 1).",
       "evidence": "Table 1 presents a 'theoretical conceptual model developed through expert consensus.' The note explicitly states values are 'illustrative estimations intended to visualize the trend' rather than 'precise psychometric measurements.'",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Severe conflict of interest",
       "detail": "5 of 7 authors are employees of Organization B, which is simultaneously the 'Research Partner' providing the paradigm and one of the two case subjects. The paper is essentially Organization B evaluating Organization B's own methodology and finding it produces 8-33x efficiency gains. This is analogous to a drug company running its own clinical trial with no independent oversight."
     },
     {
       "flag": "Counterfactual baselines presented as empirical evidence",
       "detail": "The 8x and 33x efficiency claims compare actual project effort against hypothetical estimates of what a traditional team would have needed. No actual traditional team delivered the same project. The 'traditional baseline' is a thought experiment using function point analysis, not an observed comparison. The paper presents these ratios as empirical 'findings' without adequately qualifying the counterfactual nature."
     },
     {
       "flag": "Fabricated quantitative data in Table 1",
       "detail": "Table 1 presents specific numerical values for 'Cognitive Capacity' and 'Utilized Load' across career stages (e.g., Year 10+ expert has '50 units capacity, 2.5 utilized'). The paper's own note admits these are 'illustrative estimations' from 'expert consensus,' yet they are presented in a formal table with precise numbers, lending false precision to a conceptual argument."
     },
     {
       "flag": "Extreme claims from minimal evidence",
       "detail": "33x efficiency gain claimed from a single project done by a single engineer at the company that authored the paper. No independent verification, no code quality assessment, no long-term maintenance evaluation, no customer satisfaction data. The magnitude of the claim far exceeds what two uncontrolled case studies can support."
     },
     {
       "flag": "Hawthorne effect acknowledged but not controlled",
       "detail": "Section 7.4 acknowledges the Seed Team knew they were a pilot unit, which likely boosted motivation and performance. Despite acknowledging this confound, the paper still presents the efficiency numbers as evidence of paradigm superiority rather than as potentially inflated by participant awareness."
     },
     {
       "flag": "Qualitative analysis methodology absent",
       "detail": "Despite claiming to use triangulation and semi-structured interviews (Section 4.3), the paper provides no coding scheme, no inter-coder reliability, no thematic analysis procedure, no interview guide, and no count of participants. The qualitative methodology is described at a high level but not actually documented or executed with rigor."
     },
     {
       "flag": "Self-citation of unpublished work",
       "detail": "The paper cites 'Zhang et al. (2025). The rise of the super employee' from Zenodo, which appears to be a prior version of the same work by the same authors from the same company, used to support the 'Super Employee' concept as if it were independently established."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Models for Software Engineering: A Systematic Literature Review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu", "Zhen Yang", "Kailong Wang", "Li Li", "Haoyu Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs in software engineering covering code generation, testing, and maintenance."
     },
     {
       "title": "AI Agentic Programming: A Survey of Techniques, Challenges, and Opportunities",
       "authors": ["Hao Wang", "Jiacheng Gong", "Haofeng Zhang", "Jiaqi Xu", "Zhichao Wang"],
       "year": 2025,
       "arxiv_id": "2508.11126",
       "relevance": "Survey of agentic AI programming techniques relevant to the AI-driven development paradigm."
     },
     {
       "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
       "authors": ["Jonas Becker", "Natalie Rush", "Eleanor Barnes", "David Rein"],
       "year": 2025,
       "arxiv_id": "2507.09089",
       "relevance": "RCT finding AI tools can slow experienced developers in complex open-source tasks — counter-evidence to uniform productivity gains."
     },
     {
       "title": "GPTs are GPTs: Labor Market Impact Potential of LLMs",
       "authors": ["Tyna Eloundou", "Sam Manning", "Pamela Mishkin", "Daniel Rock"],
       "year": 2024,
       "relevance": "Foundational analysis of LLMs as general-purpose technologies with labor market implications for software engineering."
     },
     {
       "title": "Advancing GenAI Assisted Programming – A Comparative Study on Prompt Efficiency and Code Quality between GPT-4 and GLM-4",
       "authors": ["Andrew Yang", "Zhouli Li", "Jian Li"],
       "year": 2024,
       "arxiv_id": "2402.12782",
       "relevance": "Empirical comparison of LLM code generation efficiency claiming 30-100x improvements over manual coding."
     },
     {
       "title": "A Review on Vibe Coding: Fundamentals, State-of-the-Art, Challenges and Future Directions",
       "authors": ["Partha Pratim Ray"],
       "year": 2025,
       "relevance": "Discusses the 'vibe coding' phenomenon where developers rely on AI output without rigorous verification."
     },
     {
       "title": "Understanding Code Quality: A Qualitative Evaluation of LLM-Generated vs. Human-Written Code",
       "authors": ["Adnan Naqvi", "Anish Jain", "Arjun Goyal", "Ankit Verma"],
       "year": 2025,
       "relevance": "Qualitative evaluation of AI-generated code quality finding subtle high-risk vulnerabilities distinct from human errors."
     },
     {
       "title": "Generative AI at Work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey Raymond"],
       "year": 2025,
       "relevance": "Empirical study of AI productivity effects showing disproportionate benefits for lower-skilled workers."
     },
     {
       "title": "The Simple Macroeconomics of AI",
       "authors": ["Daron Acemoglu"],
       "year": 2025,
       "relevance": "Economic analysis of AI's impact on total factor productivity and labor markets."
     },
     {
       "title": "The Impact of Generative AI on Critical Thinking",
       "authors": ["Harsh Puppala Lee", "Advait Sarkar", "Lev Tankelevitch", "Ian Drosos", "Sean Rintel", "Richard Banks", "Nadia Wilson"],
       "year": 2025,
       "relevance": "Survey finding GenAI reduces information gathering effort but increases cognitive demand for verification — relevant to cognitive load shifts."
     }
   ]
 }