ai-research-survey

scan-v5.json (21232B)

---

 {
   "scan_version": 5,
   "paper_type": "position",
   "paper": {
     "title": "Five Fatal Assumptions: Why T-Shirt Sizing Systematically Fails for AI Projects",
     "authors": [
       "Raja Soundaramourty",
       "O. Kilic",
       "R. Chenchaiah"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.17734",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All five fatal assumptions claimed in the abstract are systematically analyzed with evidence in Sections 4.1–4.5. Checkpoint Sizing is presented in Section 5.3 with framework and pseudocode.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Causal mechanisms (e.g., 'T-shirt assumptions fail → estimation error') rely on cited literature rather than original causal studies by the authors. Paper synthesizes rather than independently validates causal claims.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.4 explicitly scopes 'AI projects' to LLM applications, agentic workflows, RAG, and model adaptation. Limitations (5.2) acknowledge analysis focuses on LLM/multi-agent systems; simpler ML may differ.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Paper attributes estimation failures to violated T-shirt sizing assumptions but does not explore alternative explanations: team inexperience, tool misuse, organizational factors, or whether problem is inherent to AI rather than the methodology.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": false,
         "answer": false,
         "justification": "Paper discusses direct concepts (effort, duration, completion criteria) without relying on problematic proxies. No confusion between measured variables and claimed outcomes.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 titled 'Limitations' explicitly states three limitations: qualitative evidence (no controlled study), generalization scope (LLM/multi-agent focus), and alternative methods not empirically validated.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific: 'assumption violations are characterized analytically rather than through a new controlled study,' 'analysis focuses on LLM and multi-agent systems,' and 'Checkpoint Sizing effectiveness not empirically validated.'",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.4 explicitly bounds scope to AI projects where model/data-dependent uncertainty drives delivery risk. Limitations note simpler ML may violate fewer assumptions.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement provided. Paper does not disclose whether work was independently funded, sponsored, or supported by employer.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors list 'Cisco Systems, Inc.' as affiliation. However, potential conflicts of interest with product positioning or corporate interests are not discussed.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No explicit funder identified beyond employer. Independence from Cisco's interests in AI/estimation tooling cannot be assessed.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. No disclosure of patents, equity stakes, consulting relationships, or financial interests in estimation tools/frameworks.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: T-shirt sizing explained in introduction, 'AI projects' formally scoped in Section 1.4 (LLM applications, agentic workflows, RAG, model adaptation), Checkpoint Sizing framework presented in Section 5.3.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.2 explicitly lists four contributions: (1) identification of five fatal assumptions, (2) empirical grounding in literature, (3) quantitative evidence, (4) alternative framework (Checkpoint Sizing).",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": false,
         "justification": "Section 2 surveys prior work on agile estimation and AI project management but engages superficially—mostly summarizing what prior work found rather than critically positioning new contribution relative to existing knowledge.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "position": {
       "argument_quality": {
         "argument_internally_consistent": {
           "applies": true,
           "answer": true,
           "justification": "Argument is logically consistent: T-shirt sizing assumes X, AI development violates X, therefore estimation fails. Each assumption is traced to failure mode. However, Checkpoint Sizing also relies on untested assumptions (that decision gates provide actionable evidence).",
           "source": "haiku"
         },
         "counterarguments_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No serious engagement with counterarguments: Could teams improve AI estimation accuracy through experience? Could larger buffers solve the problem? Are unknown unknowns fixable via any methodology? Are there downsides to Checkpoint Sizing?",
           "source": "haiku"
         },
         "analogies_appropriate": {
           "applies": true,
           "answer": true,
           "justification": "Analogies are valid: linear vs. non-linear effort curves (Figure 1), circular dependency trap (Figure 3), guardrail oscillation as 'whack-a-mole' (Figure 13). Parallels are apt, not false equivalences.",
           "source": "haiku"
         },
         "prescriptions_proportional": {
           "applies": true,
           "answer": false,
           "justification": "Paper prescribes abandoning T-shirt sizing and adopting Checkpoint Sizing, but provides no evidence Checkpoint Sizing works. Recommending a replacement methodology without proof it succeeds is disproportionate to the evidence presented.",
           "source": "haiku"
         },
         "evidence_for_claims_cited": {
           "applies": true,
           "answer": true,
           "justification": "Factual claims are cited: exponential scaling cites [11][12], 39% multi-turn degradation cites [4], N(N-1) agent complexity cites [1]. Appendix A documents reference validation.",
           "source": "haiku"
         },
         "alternatives_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Paper proposes Checkpoint Sizing as the alternative framework without discussing other emerging AI estimation methodologies or competing proposals.",
           "source": "haiku"
         },
         "historical_context_accurate": {
           "applies": true,
           "answer": true,
           "justification": "Historical references are accurate: T-shirt sizing correctly explained, Story Points and Planning Poker accurately characterized, Brooks's Law and Cone of Uncertainty correctly cited.",
           "source": "haiku"
         }
       },
       "clarity_and_scope": {
         "key_terms_defined_precisely": {
           "applies": true,
           "answer": false,
           "justification": "Key terms lack precision: 'deterministic vs. probabilistic completion' discussed informally, 'effort' undefined, 'checkpoint readiness' criteria vague. Algorithm 1 requires 'Evidence' but does not specify what constitutes sufficient evidence.",
           "source": "haiku"
         },
         "engages_with_existing_literature": {
           "applies": true,
           "answer": false,
           "justification": "Section 2 reviews literature descriptively rather than critically. Paper summarizes prior work (Amershi, Sculley, etc.) but does not substantively discuss how this contribution differs from, extends, or contradicts existing research.",
           "source": "haiku"
         },
         "intended_audience_clear": {
           "applies": true,
           "answer": true,
           "justification": "Abstract explicitly states: 'This paper is intended for engineering managers, technical leads, and product owners responsible for planning and delivering AI initiatives.'",
           "source": "haiku"
         },
         "assumptions_stated": {
           "applies": true,
           "answer": false,
           "justification": "Implicit assumptions are not made explicit: (1) five assumptions framework correctly diagnoses root cause, (2) Checkpoint Sizing will outperform T-shirt sizing, (3) evidence from recent arXiv papers generalizes beyond LLM/multi-agent systems.",
           "source": "haiku"
         },
         "scope_of_applicability_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Scope explicitly bounded to LLM/multi-agent systems but does not discuss applicability to other AI domains (vision, robotics), team structures, organizational maturity, or regulatory/safety-critical contexts.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "T-shirt sizing assumes linear effort scaling, but AI systems exhibit exponential effort curves where incremental performance gains require disproportionate resources",
       "evidence": "Section 4.1 presents Figure 1 (linear vs. non-linear comparison) and cites [11][12] on power-law relationships between compute/data and model performance.",
       "supported": "strong"
     },
     {
       "claim": "Multi-agent interaction complexity grows combinatorially as N(N-1), not linearly with agent count",
       "evidence": "Section 4.1 presents Table 1 and Figure 2 documenting combinatorial growth. Cites [1] on multi-agent failure modes.",
       "supported": "strong"
     },
     {
       "claim": "LLMs exhibit 39% average performance degradation in multi-turn conversations compared to single-turn interactions",
       "evidence": "Section 4.2.2 cites [4] as primary source. Figure 5 illustrates context degradation in multi-turn settings.",
       "supported": "moderate"
     },
     {
       "claim": "AI development contains irreducible sequential dependencies (data→train→evaluate) that cannot be compressed with added headcount",
       "evidence": "Section 4.3.2 lists mandatory sequential phases. Cites [13][14] on ML deployment bottlenecks and data work as longest phase.",
       "supported": "strong"
     },
     {
       "claim": "Data engineering, model architecture, and prompt engineering form a tightly coupled system where changes cascade unexpectedly",
       "evidence": "Section 4.4.2 discusses tight coupling across stack (Figure 9). Cites [1][3] on system interdependencies and error amplification.",
       "supported": "moderate"
     },
     {
       "claim": "AI project completion criteria are probabilistic, not deterministic; safety/legal constraints can turn 'done' projects into major rework",
       "evidence": "Section 4.5 discusses moving goalpost problem, termination failures, guardrail oscillation. Cites [1][2].",
       "supported": "moderate"
     },
     {
       "claim": "Each new AI dataset introduces unique 'unknown unknowns' (data corruption, bias, distribution shifts) that emerge only during training/evaluation",
       "evidence": "Section 4.2.2 argues dataset uniqueness prevents reliable analogies. Cites [2][5] on hidden uncertainty in AI projects.",
       "supported": "moderate"
     },
     {
       "claim": "Checkpoint Sizing with explicit decision gates and evidence-based reassessment provides a better estimation framework for AI projects than T-shirt sizing",
       "evidence": "Section 5.3 proposes framework (Algorithm 1, gate checklist). Section 5.4 presents synthetic case study showing estimate evolution across gates.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "theoretical",
     "qualitative"
   ],
   "key_findings": "T-shirt sizing systematically fails for AI projects because five foundational assumptions do not hold: (1) linear effort scaling (AI exhibits exponential curves), (2) repeatability from prior experience (every dataset is unique), (3) effort-duration fungibility (sequential dependencies create timeline floors), (4) task decomposability (tight coupling across data/model/prompt layers), and (5) deterministic completion criteria (probabilistic, evolving constraints). The paper proposes Checkpoint Sizing as an alternative: an iterative methodology using explicit decision gates (data readiness, evaluation harness, safety/reliability, cost/latency budgets, operationalization) to reassess scope and timeline based on empirical evidence rather than analogy. Checkpoint Sizing treats initial estimates as testable hypotheses rather than commitments.",
   "red_flags": [
     {
       "flag": "No empirical validation of core claims",
       "detail": "Paper is analytical synthesis of literature. No original experiments, real case studies, or quantitative comparison between T-shirt sizing and Checkpoint Sizing on actual AI projects."
     },
     {
       "flag": "Checkpoint Sizing effectiveness unproven",
       "detail": "Proposed alternative lacks evidence of effectiveness. Synthetic case study (5.4) illustrates framework mechanics but does not prove it reduces estimation error, schedule slip, or improves outcomes."
     },
     {
       "flag": "Counterarguments unaddressed",
       "detail": "Paper does not explore alternative explanations for AI estimation failure (team inexperience, tool misuse, organizational dysfunction). Does not discuss whether Checkpoint Sizing itself introduces overhead or new failure modes."
     },
     {
       "flag": "Heavy reliance on recent arXiv papers",
       "detail": "Evidence grounded in 5 recent (2024–2025) arXiv papers [1–5]. Limited engagement with older/canonical ML systems literature. Only Sculley et al. (2015) provides established baseline; generalizability of recent papers uncertain."
     },
     {
       "flag": "Potential conflict of interest not disclosed",
       "detail": "All three authors from Cisco Systems, Inc. No statement on funding independence or conflict of interest. Unclear whether work is independent research or influenced by Cisco's interests in AI estimation/tooling products."
     },
     {
       "flag": "Checkpoint readiness criteria vague",
       "detail": "Algorithm 1 and Gate Checklist specify artifacts (data inventory, eval pipeline, etc.) but not acceptance criteria. Who decides when 'data readiness' is sufficient? How to handle disagreement? No guidance on checkpoint failure/iteration."
     },
     {
       "flag": "Overclaimed 'fatality' of assumptions",
       "detail": "Paper frames assumptions as 'fatal' without proving they're always problematic or impossible to address within T-shirt sizing (e.g., adding larger buffers, breaking projects into smaller chunks)."
     },
     {
       "flag": "Limited scope and generalization",
       "detail": "Analysis focuses on LLM/multi-agent systems. Applicability unclear for other AI domains (vision, robotics), smaller/mature teams, or non-commercial research contexts."
     }
   ],
   "cited_papers": [
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": "M. Cemri et al.",
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Primary evidence for Assumption 4 (task decomposability via inter-agent coupling) and 5 (deterministic completion via verification/termination failures)."
     },
     {
       "title": "An LLM-based multi-agent framework for agile effort estimation",
       "authors": "T.-L. Bui, H. K. Dam, R. Hoda",
       "year": 2025,
       "arxiv_id": "2509.14483",
       "relevance": "Validates Assumption 2 (repeatability via subjective inconsistency) and 5 (deterministic completion via estimation instability in LLM-based systems)."
     },
     {
       "title": "Towards a Science of Scaling Agent Systems",
       "authors": "Y. Kim et al.",
       "year": 2025,
       "arxiv_id": "2512.08296",
       "relevance": "Primary evidence for Assumption 1 (non-linear scaling), 3 (effort-duration tradeoffs), and 4 (coordination overhead/error amplification in multi-agent systems)."
     },
     {
       "title": "LLMs Get Lost In Multi-Turn Conversation",
       "authors": "P. Laban, H. Hayashi, Y. Zhou, J. Neville",
       "year": 2025,
       "arxiv_id": "2505.06120",
       "relevance": "Primary evidence for 39% multi-turn performance degradation (Assumption 2: repeatability) and non-recovering error trajectories (Assumption 5: deterministic completion)."
     },
     {
       "title": "Effort and Size Estimation in Software Projects with Large Language Model-based Intelligent Interfaces",
       "authors": "C. N. Coelho Jr et al.",
       "year": 2024,
       "arxiv_id": "2402.07158",
       "relevance": "Validates Assumption 2 (repeatability via hidden uncertainty in AI projects) and 5 (deterministic completion via evolving specifications and AI interface behavior)."
     },
     {
       "title": "Software Engineering for Machine Learning: A Case Study",
       "authors": "S. Amershi et al.",
       "year": 2019,
       "venue": "IEEE/ACM ICSE-SEIP",
       "relevance": "Foundational work identifying nine ML workflow characteristics (data dependencies, experimental iteration, model decay) that differ from traditional software, establishing that AI development violates software estimation assumptions."
     },
     {
       "title": "Hidden Technical Debt in Machine Learning Systems",
       "authors": "D. Sculley et al.",
       "year": 2015,
       "venue": "NeurIPS",
       "relevance": "Established literature on ML technical debt showing conventional software engineering intuitions systematically mislead practitioners, supporting Assumptions 2, 4, and 5."
     },
     {
       "title": "Challenges in Deploying Machine Learning: A Survey of Case Studies",
       "authors": "A. Paleyes, R.-G. Urma, N. Lawrence",
       "year": 2022,
       "venue": "ACM Computing Surveys",
       "relevance": "Documents sequential bottlenecks in real-world ML deployments (data→train→evaluate), supporting Assumption 3 (irreducible sequential dependencies)."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners could attempt Checkpoint Sizing, but lack of empirical validation makes adoption risky. Framework is intuitive but unproven relative to alternatives."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges widespread T-shirt sizing use, but difficulties with AI estimation are well-known to experienced practitioners. Not surprising to practitioners who have lived the problem."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Mentions safety validation and hallucination rates as hidden costs, but frames as project management challenge rather than AI safety/existential risk concern."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Moderate controversy over whether T-shirt sizing is broken for AI. Practical methodology debate rather than high-drama industry conflict or heated disagreement."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Checkpoint Sizing is a conceptual framework with no accompanying software, tooling, or live demo. Would require manual implementation and interpretation to evaluate."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Cisco Systems is well-known company, but authors are not prominent AI researchers. Work not associated with top-tier AI research lab or famous research group."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "47279778",
         "title": "Nested Training for Mutual Adaptation in Human-AI Teaming",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47279778",
         "created_at": "2026-03-06T19:21:19Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 0
   }
 }