ai-research-survey

scan-v4.json (22057B)

---

 {
   "scan_version": 4,
   "paper_type": "position",
   "paper": {
     "title": "Five Fatal Assumptions: Why T-Shirt Sizing Systematically Fails for AI Projects",
     "authors": [
       "Raja Soundaramourty",
       "Ozkan Kilic",
       "Ramu Chenchaiah"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.17734",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to identify five fatal assumptions and ground them in empirical literature. The paper delivers this in Section 4 with citations to specific studies for each assumption.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims throughout, e.g., 'these failures aren't about poor execution' (Section 1.1), AI development 'breaks these rules' (abstract). These causal claims about why estimation fails are argued analytically from cited literature but not tested with original data or controlled study.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.4 explicitly defines scope: LLM applications, agentic workflows, RAG systems, and model adaptation. Section 5.2 acknowledges 'simpler ML projects may violate fewer assumptions.'",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for estimation failures in AI projects (e.g., team inexperience, poor requirements, organizational factors). It attributes failures entirely to the five assumptions without considering confounds.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames cited metrics like '39% performance degradation' and 'N(N-1) complexity' as direct evidence of estimation failure, without discussing whether these proxy measurements actually predict estimation accuracy.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 'Limitations' explicitly lists three limitations.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 lists specific limitations: (1) evidence is qualitative/analytical not from a new controlled study, (2) analysis focuses on LLM/multi-agent and may not apply to simpler ML, (3) Checkpoint Sizing is not empirically validated.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.4 explicitly defines what counts as an 'AI project' and Section 5.2 acknowledges the analysis focuses on LLM and multi-agent systems specifically.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section. All authors are from Cisco Systems, a major technology company.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations at Cisco Systems are clearly stated in the header.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement is provided. Cisco, as a technology company developing AI products, has potential interest in outcomes related to AI project estimation methodology. The absence of disclosure is itself a concern.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Authors work at Cisco which develops and sells AI products and services.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "Only 'AI projects' is precisely defined (1.4). Key terms like 'T-shirt sizing', 'LLMs', 'multi-agent systems', 'RAG', 'evaluation', and 'estimation failure' are used but not formally defined; relies on reader familiarity.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.2 explicitly states four contributions: (1) identify five fatal assumptions, (2) ground in empirical literature, (3) provide quantitative evidence (N(N−1) complexity, 39% degradation), (4) propose Checkpoint Sizing framework.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": false,
         "justification": "Section 2 lists related work on agile estimation (2.1), AI project management (2.2), and LLM-integrated systems (2.3), but engagement is shallow. Cites papers without deeply discussing how this work builds on, differs from, or extends prior insights.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "position": {
       "argument_quality": {
         "argument_internally_consistent": {
           "applies": true,
           "answer": true,
           "justification": "Logical structure: T-shirt sizing rests on 5 assumptions → these assumptions fail for AI → therefore T-shirt sizing fails for AI → Checkpoint Sizing is alternative. No internal contradictions or logical fallacies detected.",
           "source": "haiku"
         },
         "counterarguments_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Section 5.1 dismisses buffer argument ('Buffers Aren't Enough') but doesn't engage with strongest opposing views. A skeptic might argue 'this is just poor team execution' or 'AI estimation is just messier, not fundamentally different'—these are not addressed.",
           "source": "haiku"
         },
         "analogies_appropriate": {
           "applies": true,
           "answer": true,
           "justification": "Analogies between traditional software (linear scaling, repeatability, fungibility) and AI (exponential curves, dataset uniqueness, sequential bottlenecks) are apt. Figures 1–13 illustrate contrasts clearly without false equivalence.",
           "source": "haiku"
         },
         "prescriptions_proportional": {
           "applies": true,
           "answer": false,
           "justification": "Paper proposes Checkpoint Sizing as solution, which is proportional to the problem identified. However, the prescription itself is not validated—Section 5.2 admits 'Checkpoint Sizing... [is] not empirically validated.' Proportionality undermined by lack of evidence for the cure.",
           "source": "haiku"
         },
         "evidence_for_claims_cited": {
           "applies": true,
           "answer": true,
           "justification": "Factual claims reference sources. Examples: scaling laws [11], [12]; multi-turn degradation [4]; multi-agent complexity [1], [3]; ML technical debt [10]; data work challenges [13], [14]. Citations include arXiv IDs and DOIs.",
           "source": "haiku"
         },
         "alternatives_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Paper discusses story points and planning poker (2.1) but does not engage deeply with why these methods might succeed or fail for AI. Section 5.2 explicitly notes 'Checkpoint Sizing... [is] not empirically validated [relative to] other emerging AI estimation approaches,' indicating no comparison provided.",
           "source": "haiku"
         },
         "historical_context_accurate": {
           "applies": true,
           "answer": true,
           "justification": "Historical references are accurate: Brooks's Law cited correctly (4.3.1); agile estimation history (Cohn 2005, Grenning 2002, McConnell 2006) matches publication records; ML technical debt (Sculley et al. 2015) appropriately cited.",
           "source": "haiku"
         }
       },
       "clarity_and_scope": {
         "key_terms_defined_precisely": {
           "applies": true,
           "answer": false,
           "justification": "'AI projects' is precisely defined (1.4: initiatives where primary risk is model behavior). Other key terms ('T-shirt sizing', 'estimation', 'multi-agent', 'agents') are explained contextually but not formally defined. Assumes reader familiarity.",
           "source": "haiku"
         },
         "engages_with_existing_literature": {
           "applies": true,
           "answer": false,
           "justification": "Section 2 reviews agile estimation (2.1), AI project management (2.2), and LLM-integrated systems (2.3), but engagement is superficial. Papers are listed with brief summaries, not substantively discussed or integrated into the argument. Reads as background survey, not integrated analysis.",
           "source": "haiku"
         },
         "intended_audience_clear": {
           "applies": true,
           "answer": true,
           "justification": "Abstract explicitly states: 'This paper is intended for engineering managers, technical leads, and product owners responsible for planning and delivering AI initiatives.' Specific, actionable audience definition.",
           "source": "haiku"
         },
         "assumptions_stated": {
           "applies": true,
           "answer": false,
           "justification": "Paper derives five assumptions underlying T-shirt sizing (Section 3.1) but does not state its own assumptions. Implicit assumptions: (1) estimation happens upfront; (2) team size/composition is relatively fixed; (3) scope changes are not inherent; (4) traditional software practices are common baseline. These should be explicit.",
           "source": "haiku"
         },
         "scope_of_applicability_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Section 1.4 states scope but does not discuss edge cases or where argument breaks down. Boundaries stated (LLM apps, agentic workflows, RAG, fine-tuning) but not discussed: simple prompt-based systems, fine-tuning on fixed datasets, or internal tools with deterministic outputs.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Improving LLM accuracy from 85% to 95% requires ~10x increase in effort (exponential, not linear scaling)",
       "evidence": "Section 4.1.2 cites scaling laws research [11], [12] showing model performance follows power-law relationships with compute and data.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs exhibit 39% average performance degradation in multi-turn conversations vs. single-turn",
       "evidence": "Section 4.2.2 and Table 3 reference [4] (Laban et al., 'LLMs Get Lost In Multi-Turn Conversation'); stated as 'Large-scale simulations find an average 39% performance drop.'",
       "supported": "strong"
     },
     {
       "claim": "Multi-agent interaction complexity grows as N(N−1), not linearly with agent count",
       "evidence": "Section 4.1.2 and Table 1 show combinatorial growth: 2 agents = 2 interactions, 3 agents = 6, 10 agents = 90. Supported by [1], [3].",
       "supported": "strong"
     },
     {
       "claim": "AI development contains mandatory sequential phases (data→train→eval) that cannot be parallelized",
       "evidence": "Section 4.3.2 cites [13] (Paleyes et al., ML deployment survey) and [14] (Sambasivan et al., data cascades) documenting sequential bottlenecks.",
       "supported": "strong"
     },
     {
       "claim": "Agent systems can enter infinite correction loops (Supervisor rejects Worker, Worker retries indefinitely)",
       "evidence": "Section 4.5.2 mentions 'Infinite correction loops' as an emergent failure mode; supported by [1] (multi-agent failure taxonomy). Illustrated in Figure 12 but no quantitative evidence provided.",
       "supported": "moderate"
     },
     {
       "claim": "Data work is the longest and most unpredictable phase in ML projects and cannot be easily parallelized",
       "evidence": "Section 4.3.2 cites [14] (Sambasivan et al., 'Everyone wants to do the model work, not the data work') documenting data cascades and bottlenecks.",
       "supported": "moderate"
     },
     {
       "claim": "Token starvation: one verbose agent can exhaust shared context window, starving others of reasoning capacity",
       "evidence": "Section 4.4.2 and Figure 10 describe scenario but no quantitative evidence provided. Illustrated conceptually but not measured in practice.",
       "supported": "weak"
     },
     {
       "claim": "Guardrail tuning requires weeks of iteration (tightening reduces utility, loosening increases risk)",
       "evidence": "Section 4.5.2 mentions 'weeks of RLHF or prompt engineering' but cites no source and provides no data to support duration claim.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "position",
     "theoretical",
     "case-study"
   ],
   "key_findings": "T-shirt sizing systematically fails for AI projects because five foundational assumptions that hold in traditional software break down: (1) effort scales linearly (not exponentially in AI), (2) past experience predicts future effort (every dataset is unique), (3) effort and duration are interchangeable (sequential bottlenecks exist), (4) tasks decompose independently (data/model/prompt layers are tightly coupled), and (5) completion criteria are deterministic (AI outcomes are probabilistic). The paper proposes Checkpoint Sizing—an iterative framework with explicit decision gates (data readiness, evaluation harness, safety/reliability, cost/latency, operationalization) where scope and feasibility are reassessed based on evidence rather than assumptions.",
   "red_flags": [
     {
       "flag": "No empirical validation",
       "detail": "Authors acknowledge (5.2) that 'assumption violations are characterized analytically rather than through a new controlled study.' All claims are theoretical or synthesized from other papers, not validated through original empirical work."
     },
     {
       "flag": "Proposed solution unvalidated",
       "detail": "Checkpoint Sizing is presented as alternative but Section 5.2 admits it is 'not empirically validated [relative to] other emerging AI estimation approaches.' The synthetic case study (5.4) does not validate the framework's effectiveness."
     },
     {
       "flag": "Heavy reliance on unreviewed preprints",
       "detail": "Primary citations [1]–[5] are all 2024–2025 arXiv preprints (not peer-reviewed). Only supporting references [11], [12], [13], [14] are published. This weakens evidence grounding."
     },
     {
       "flag": "Potential strawman of T-shirt sizing",
       "detail": "Paper assumes organizations use T-shirt sizing for AI projects as-is without adaptation. No evidence that teams actually rely on this method for AI without compensatory adjustments or other estimation techniques."
     },
     {
       "flag": "Conflicts of interest not disclosed",
       "detail": "Authors from Cisco Systems but no funding disclosure or statement on whether Cisco has business interests in estimation tools, project management software, or AI platforms. Cisco's motivation is unclear."
     },
     {
       "flag": "Synthetic case study only",
       "detail": "The 'Support Copilot with RAG + Tool Use' example (5.4) is explicitly synthetic ('Illustrative Case Study'). Outcomes are illustrative, not based on real project data. No retrospective validation against actual AI projects."
     },
     {
       "flag": "Limited engagement with counterarguments",
       "detail": "Paper dismisses buffer-based estimation (5.1) but does not deeply engage with skeptics who might argue this is poor execution, not a property of AI itself, or that teams already use different methods."
     },
     {
       "flag": "Generalization to all 'AI projects' may be overstated",
       "detail": "Analysis focuses on LLM and multi-agent systems. Simpler ML projects (fixed-data fine-tuning, deployment-only tasks) may not face all five assumption violations, but paper presents framework as universal to 'AI projects.'"
     }
   ],
   "cited_papers": [
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": "Cemri et al.",
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Validates assumptions 4 (decomposability via inter-agent coupling) and 5 (deterministic completion via verification/termination failures). Core reference for multi-agent failure modes."
     },
     {
       "title": "An LLM-based multi-agent framework for agile effort estimation",
       "authors": "Bui, Dam, Hoda",
       "year": 2025,
       "arxiv_id": "2509.14483",
       "relevance": "Validates assumptions 2 (repeatability via subjective inconsistency) and 5 (deterministic completion via estimation instability). Directly addresses AI effort estimation challenges."
     },
     {
       "title": "Towards a Science of Scaling Agent Systems",
       "authors": "Kim et al.",
       "year": 2025,
       "arxiv_id": "2512.08296",
       "relevance": "Validates assumptions 1 (non-linear scaling), 3 (effort-duration tradeoffs), and 4 (coordination overhead/error amplification). Core reference for agent scaling complexity."
     },
     {
       "title": "LLMs Get Lost In Multi-Turn Conversation",
       "authors": "Laban, Hayashi, Zhou, Neville",
       "year": 2025,
       "arxiv_id": "2505.06120",
       "relevance": "Validates assumption 2 (multi-turn unreliability, 39% degradation) and 5 (non-recovering error trajectories). Key evidence for conversation state drift."
     },
     {
       "title": "Effort and Size Estimation in Software Projects with Large Language Model-based Intelligent Interfaces",
       "authors": "Coelho et al.",
       "year": 2024,
       "arxiv_id": "2402.07158",
       "relevance": "Validates assumptions 2 (hidden uncertainty) and 5 (evolving spec/AI interface behavior). Directly addresses LLM-integrated project estimation."
     },
     {
       "title": "Hidden Technical Debt in Machine Learning Systems",
       "authors": "Sculley et al.",
       "year": 2015,
       "venue": "NIPS",
       "relevance": "Foundational reference on ML-specific technical challenges (data dependencies, feedback loops, configuration debt). Supports overall argument that AI differs from traditional software."
     },
     {
       "title": "Everyone wants to do the model work, not the data work: Data Cascades in High-Stakes AI",
       "authors": "Sambasivan et al.",
       "year": 2021,
       "venue": "CHI",
       "relevance": "Validates assumption 3 (sequential data bottlenecks cannot be parallelized). Empirical study of data work challenges in high-stakes ML projects."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": "Kaplan et al.",
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational work on power-law relationships between compute/data and performance. Validates assumption 1 (exponential effort for marginal gains)."
     },
     {
       "title": "Challenges in Deploying Machine Learning: A Survey of Case Studies",
       "authors": "Paleyes, Urma, Lawrence",
       "year": 2022,
       "venue": "ACM Computing Surveys",
       "relevance": "Survey of real-world ML deployment challenges including sequential bottlenecks. Supports assumption 3 (irreducible sequential dependencies)."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Checkpoint Sizing framework is conceptually actionable for engineering managers, but no tooling, templates, or implementation guide provided. Practitioners would need to operationalize the framework themselves."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The core insight (AI estimation differs from software) is not surprising to practitioners who have shipped AI projects. The 5-assumption framework adds structure but doesn't fundamentally challenge existing practice."
     },
     "fear_safety": {
       "score": 0,
       "justification": "Paper focuses on estimation methodology, not AI safety, risk, alignment, or societal impact. Safety is mentioned only as a checkpoint (Gate C) but not as a risk concern."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Title promises 'fatal assumptions,' but the framing is methodological rather than dramatic. Failure modes (infinite loops, guardrail oscillation) have some conflict angle but are not positioned sensationally."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Position paper with no code, tooling, or interactive demo. Checkpoint Sizing could theoretically be applied to a real team, but no template or software to 'try now.' Synthetic case study is illustrative, not runnable."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from Cisco Systems, a recognizable company but not a top-tier AI research lab (not OpenAI, Anthropic, DeepMind, Google, Meta). Limited brand pull in academic or AI circles."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "47279778",
         "title": "Nested Training for Mutual Adaptation in Human-AI Teaming",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47279778",
         "created_at": "2026-03-06T19:21:19Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 0
   }
 }