ai-research-survey

scan.json (28551B)

---

 {
   "paper": {
     "title": "Codified Context: Infrastructure for AI Agents in a Complex Codebase",
     "authors": ["Aristidis Vasilopoulos"],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2602.20478"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The companion repository is published at https://github.com/arisvas4/codified-context-infrastructure, including representative agent specifications, the MCP retrieval server, example documents, factory agents, and analysis scripts (Section 6, final paragraph)."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Raw milestone data is available in the companion repository' (Section 4.2) and 'extraction methodology, scripts, and dataset schema are described in the companion repository' (Section 4.3), but the actual conversation history dataset (1,457 JSONL files) and the full project codebase (108,000 lines of C#) are not released. Only analysis scripts and representative examples are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, Dockerfile, or dependency versions are mentioned in the paper. The MCP retrieval server is described as '~1,600 lines Python' but no Python version or dependencies are specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The companion repository includes factory agents for bootstrapping the architecture on new projects, but no step-by-step reproduction instructions for replicating the quantitative metrics or case study results are described. The underlying C# project is not available for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports point estimates only (e.g., '2,801 human prompts', '1,197 agent invocations', '~9.9 human prompts per session') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper explicitly states 'No causal relationships are claimed' (Section 4.1) and makes no comparative claims requiring significance tests. It is a systems paper and experience report."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make comparative claims between conditions. It reports descriptive metrics about a single system and explicitly disclaims causal or comparative claims (Section 4.1)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why 283 sessions, 4 case studies, or 1,457 conversation files represent an adequate sample. The paper acknowledges some data was lost ('some early files were lost during a cache cleanup') but does not discuss implications for coverage."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any of the interaction metrics. Only totals and averages are given (e.g., '~9.9 human prompts per session', '~6 autonomous agent turns per human prompt')."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. There is no comparison against development without the codified context infrastructure, against single-file manifests, or against alternative knowledge organization approaches. The paper acknowledges this: 'It is not possible to quantify with statistical rigor how much the architecture improved development speed or code quality' (Section 5.3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so contemporaneity is moot. The paper mentions Google Conductor as a concurrent approach but does not compare against it."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is presented. The paper does not test what happens when individual tiers (constitution, agents, knowledge base) are removed. The paper acknowledges 'Controlled benchmarking—measuring task completion rates and error rates with and without each architecture tier—is the most immediate priority' (Section 5.3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple descriptive metrics are reported: infrastructure growth (lines, files by tier), interaction metrics (human prompts, agent invocations, agent turns, sessions), retrieval metrics (1,478 MCP calls across 218 sessions), and per-agent invocation counts (Section 4.2, 4.3, Table 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs is conducted. The case studies are the author's own observations of their own system. No independent human evaluators assessed the quality of agent outputs with vs. without the infrastructure."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate model performance on a benchmark or test set. It is a systems paper with descriptive metrics and case studies."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Breakdowns are provided by tier (T1/T2/T3 in Table 3), by agent capability class (8 higher-capability vs. 11 standard-capability agents in Section 3.2), by session type (structured ~13% vs. ad-hoc ~87% in Section 4.3), and per-agent invocation counts (code reviewer 154, network-protocol-designer 85)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 discusses specification staleness as the primary failure mode, with two specific examples: a combat specification referencing legacy stat fields and an undescribed second case. Case Study 4 describes a bug that required 'five context window exhaustions and 84 code edits' before resolution."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that specification staleness caused agents to generate incorrect code on at least two occasions (Section 5.2), that some early conversation files were lost during a cache cleanup reducing data coverage (Section 4.3), and that keyword matching retrieval has precision limitations (Section 5.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are modest: it presents a three-component architecture, reports quantitative metrics across 283 sessions, describes four case studies, and states the framework is open-source. All of these are supported in the paper body (Sections 3-4, companion repository link in Section 6)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly disclaims causal claims: 'No causal relationships are claimed; confounding factors including developer experience growth cannot be isolated' (Section 4.1). The case studies use language like 'illustrate' rather than 'prove'. This appropriate hedging means the study design is adequate for the claims actually made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds generalization: 'single developer on a single project; its effectiveness in team settings, other project types, or larger scales has not been evaluated' (Section 5.3). It also notes that the project domain 'demands more extensive documentation than many application types, which may limit generalizability to simpler projects.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 discusses that developer experience growth cannot be isolated as a confound, that the observational methodology prevents quantifying the architecture's contribution, and that the tool-specific implementation limits transferability claims. The paper explicitly acknowledges 'confounding factors including developer experience growth cannot be isolated' (Section 4.1)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Claude Code' was used as the sole code-generation tool but does not specify which Claude model version, API version, or snapshot date. No version identifier beyond 'Claude Code' is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides representative prompt/specification content: the constitution structure (Section 3.1), agent specification format with an abbreviated example in Appendix B (328-line coordinate-wizard agent), knowledge base document format with a specification example (Section 3.3), and orchestration trigger tables (Table 1). Full specifications are available in the companion repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported for the Claude Code sessions used during development."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in substantial detail: the three-tier architecture (Section 3), orchestration protocols with trigger tables (Section 3.1.1, Table 1), MCP retrieval service with five search tools (Section 3.3.1), workflow diagrams (Figures 1 and 3), and structured vs. ad-hoc session modes (Section 4.3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The interaction data extraction methodology is described only at a high level: 'Interaction data was extracted from 1,457 JSONL conversation history files' (Section 4.3). The paper acknowledges 'some early files were lost during a cache cleanup, and agent chain data is available only for a 31-day window' but does not document filtering criteria, classification methodology, or transformation steps in the paper itself. It references the companion repository for details."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 'Threats to Validity and Future Work' provides a substantive discussion of limitations spanning multiple paragraphs, covering single-developer evaluation, observational methodology, and tool-specific implementation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: 'single developer on a single project' (not generic), 'The project domain (real-time distributed simulation) demands more extensive documentation than many application types' (not generic), 'some early files were lost during a cache cleanup' (specific data loss), and confounding with developer experience growth (Section 5.3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries are stated: 'its effectiveness in team settings, other project types, or larger scales has not been evaluated' (Section 5.3), 'Transferability has not been evaluated', and 'No causal relationships are claimed' (Section 4.1). The evaluation methodology section also explicitly states 'This is a systems paper and experience report, where the primary contribution is the architecture rather than statistical evidence of effectiveness' (Section 4.1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw conversation history data (1,457 JSONL files) is not released. The paper references analysis scripts and dataset schema in the companion repository but not the underlying data itself."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes data collection: interaction data from 1,457 JSONL conversation history files, with the caveat that 'some early files were lost during a cache cleanup, and agent chain data is available only for a 31-day window.' Section 4.2 describes infrastructure growth metrics reconstructed from Git history."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. This is a single-developer experience report using data from the author's own development sessions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states extraction methodology and scripts are in the companion repository (Section 4.3) but does not document in the paper itself how raw JSONL files were processed into the reported metrics. There is no description of filtering steps, classification criteria for structured vs. ad-hoc sessions, or how the 757 'classifiable' agent invocations were selected from the total 1,197."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This work was conducted independently with no external funding.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author is listed as 'Independent Researcher, USA' with no corporate affiliation. The paper evaluates Claude Code, and the author has no disclosed connection to Anthropic."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "The work is explicitly stated as independently conducted with no external funding (Acknowledgments). There is no funder to evaluate for independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The companion repository is open-source, but there is no explicit declaration about absence of financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. It is a systems paper and experience report describing an architecture for organizing project knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model on any benchmark. No train/test overlap concern applies."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate model performance on any benchmark. No contamination concern applies."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a single-developer experience report using the author's own development sessions."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Single-developer experience report."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The author does describe their own background ('primary background is in chemistry rather than software engineering') but this is self-description, not participant demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and not an experimental study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption totals, or per-session costs are reported despite using Claude Code extensively across 283 sessions with 16,522 agent turns. Only maintenance overhead is described in time terms ('approximately 1-2 hours per week', Section 5.2)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, API spend, or hardware specifications are stated. The paper reports development time (70 days part-time) but not computational costs."
       }
     }
   },
   "claims": [
     {
       "claim": "The three-tier codified context infrastructure supported a single developer in constructing a 108,000-line distributed system in under 70 days of part-time development using AI agents as the sole code-generation tool.",
       "evidence": "Table 3 reports project scale: 108,256 C# LOC, 405 source files, 148 commits over 70 days. Section 4.1 states Claude Code was the sole code-generation tool. Context infrastructure totaled ~26,200 lines across 54 files (Section 4.2, Table 3).",
       "supported": "moderate"
     },
     {
       "claim": "The codified context infrastructure reduced the need for in-prompt explanation, with over 80% of human prompts being 100 words or fewer.",
       "evidence": "Section 4.3 reports 'Over 80% of human prompts were 100 words or fewer, consistent with the hypothesis that pre-loaded context reduces the need for in-prompt explanation.' However, no baseline comparison without context infrastructure is provided.",
       "supported": "weak"
     },
     {
       "claim": "Meta-infrastructure prompts (building the knowledge architecture itself) accounted for only 4.3% of substantive prompts.",
       "evidence": "Section 4.3 states: 'Meta-infrastructure prompts—building the knowledge architecture itself—accounted for 4.3% of substantive prompts, representing the direct overhead of the approach.'",
       "supported": "moderate"
     },
     {
       "claim": "Specialist agents were the dominant invocation type, with 57% of classifiable agent invocations being project-specific specialists.",
       "evidence": "Section 4.3 reports: 'Of 757 classifiable agent invocations, 432 (57%) were project-specific specialists defined in the context infrastructure and 325 were built-in tool agents.'",
       "supported": "moderate"
     },
     {
       "claim": "Coordination across 74 independent sessions over four weeks produced no save-related bugs when agents had access to the save-system specification.",
       "evidence": "Case Study 1 (Section 4.4.1) describes the save-system.md specification appearing in 74 sessions and 12 agent conversations. Five subsequent features touching persistence were implemented correctly. However, this is observational with no controlled comparison.",
       "supported": "weak"
     },
     {
       "claim": "The context infrastructure's maintenance overhead averaged approximately 1-2 hours per week.",
       "evidence": "Section 5.2 states: 'Total maintenance overhead averaged approximately 1-2 hours per week,' comprising ~5 minutes per session when specifications were affected plus biweekly ~30-45 minute review passes.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["case-study", "observational"],
   "key_findings": "The paper presents a three-tier codified context infrastructure (constitution, specialized agents, knowledge base) for maintaining AI coding agent consistency across sessions in a large codebase. Developed iteratively during construction of a 108,000-line C# distributed system across 283 sessions, the architecture treats documentation as machine-readable infrastructure rather than human-oriented artifacts. Quantitative metrics show 57% of agent invocations used project-specific specialists, 80%+ of human prompts were under 100 words, and meta-infrastructure overhead was 4.3% of prompts. Four observational case studies illustrate distinct mechanisms (coordination, experience capture, gap detection, domain-expert debugging) but no controlled comparisons are provided.",
   "red_flags": [
     {
       "flag": "Single-developer self-evaluation",
       "detail": "The sole author developed, used, and evaluated the system. There is no independent assessment of effectiveness, no external users, and no separation between the developer and evaluator roles. The author acknowledges this limitation."
     },
     {
       "flag": "No controlled comparison",
       "detail": "No baseline comparison is provided (e.g., development with single-file manifests, without context infrastructure, or using alternative approaches). The paper cannot quantify whether the architecture actually improved outcomes versus the alternatives. The paper acknowledges this explicitly."
     },
     {
       "flag": "Survivorship bias in case studies",
       "detail": "Case studies were 'selected for qualitative diversity' from successful uses of the system. Instances where the context infrastructure failed to prevent errors or was counterproductive may be underrepresented, beyond the two staleness incidents mentioned in Section 5.2."
     },
     {
       "flag": "Missing cost data",
       "detail": "Despite 283 sessions with 16,522 agent turns, no API costs or token consumption data is reported. This makes it impossible to assess the practical economics of the approach."
     },
     {
       "flag": "Incomplete interaction data",
       "detail": "Some early conversation files were lost during a cache cleanup, and agent chain data is available only for a 31-day window within the 70-day measurement period. The impact of this data loss on reported metrics is not assessed."
     }
   ],
   "cited_papers": [
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Empirical study of failure modes in multi-agent LLM systems, directly relevant to understanding agentic coding system reliability."
     },
     {
       "title": "Agent READMEs: An Empirical Study of Context Files for Agentic Coding",
       "authors": ["W. Chatlatanagulchai", "H. Li", "Y. Kashiwa"],
       "year": 2025,
       "arxiv_id": "2511.12884",
       "relevance": "Empirical characterization of agent context files across Claude Code, Codex, and GitHub Copilot—directly studies the manifest pattern this paper extends."
     },
     {
       "title": "On the Use of Agentic Coding Manifests: An Empirical Study of Claude Code",
       "authors": ["W. Chatlatanagulchai", "K. Thonglek", "B. Reid"],
       "year": 2025,
       "arxiv_id": "2509.14744",
       "relevance": "Empirical study of CLAUDE.md manifests in Claude Code projects, providing baseline data on how developers configure AI coding agents."
     },
     {
       "title": "Promptware Engineering: Software Engineering for Prompt-Enabled Systems",
       "authors": ["Z. Chen", "C. Wang", "W. Sun"],
       "year": 2025,
       "arxiv_id": "2503.02400",
       "relevance": "Proposes treating prompts as engineered software artifacts, a principle extended by the codified context infrastructure approach."
     },
     {
       "title": "Context Engineering for Multi-Agent LLM Code Assistants Using Elicit, NotebookLM, ChatGPT, and Claude Code",
       "authors": ["M. Haseeb"],
       "year": 2025,
       "arxiv_id": "2508.08322",
       "relevance": "Studies integrated multi-tool workflows for context engineering in multi-file code generation, showing higher success rates than single-agent systems."
     },
     {
       "title": "Agentic Software Engineering: Foundational Pillars and a Research Roadmap",
       "authors": ["A. E. Hassan", "H. Li", "D. Lin"],
       "year": 2025,
       "arxiv_id": "2509.06216",
       "relevance": "Provides foundational framework and roadmap for agentic software engineering research including trust and scaffolding mechanisms."
     },
     {
       "title": "Professional Software Developers Don't Vibe, They Control: AI Agent Use for Coding in 2025",
       "authors": ["R. Huang", "A. Reyna", "S. Lerner"],
       "year": 2025,
       "arxiv_id": "2512.14012",
       "relevance": "Empirical study of how professional developers actually use AI coding agents, characterizing knowledge transfer as an open interaction design problem."
     },
     {
       "title": "Beyond the Prompt: An Empirical Study of Cursor Rules",
       "authors": ["S. Jiang", "D. Nam"],
       "year": 2025,
       "arxiv_id": "2512.18925",
       "relevance": "Empirical classification of instruction types developers include in Cursor configuration files, providing baseline data on manifest patterns."
     },
     {
       "title": "On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents",
       "authors": ["J. L. Lulla", "S. Mohsenimofidi", "M. Galster"],
       "year": 2026,
       "arxiv_id": "2601.20404",
       "relevance": "Quantitative evidence that AGENTS.md files reduce runtime by 29% and token consumption by 17%, providing effectiveness data for the manifest pattern."
     },
     {
       "title": "Context Engineering for AI Agents in Open-Source Software",
       "authors": ["S. Mohsenimofidi", "M. Galster", "C. Treude"],
       "year": 2025,
       "arxiv_id": "2510.21413",
       "relevance": "Studies adoption of context engineering in open-source repositories, finding only ~5% have adopted any context file format."
     },
     {
       "title": "Decoding the Configuration of AI Coding Agents: Insights from Claude Code Projects",
       "authors": ["H. V. F. Santos", "V. Costa", "J. E. Montandon"],
       "year": 2025,
       "arxiv_id": "2511.09268",
       "relevance": "Characterizes CLAUDE.md configurations in Claude Code projects, finding 72.6% specify application architecture."
     },
     {
       "title": "Agentic Software Engineers: Programming with Trust",
       "authors": ["A. Roychoudhury", "C. Pasareanu", "M. Pradel"],
       "year": 2025,
       "arxiv_id": "2502.13767",
       "relevance": "Calls for formal scaffolding, grounding, and trust mechanisms for agentic AI software engineering."
     },
     {
       "title": "Agentic Context Engineering: Evolving Contexts for Self-Improving Language Models",
       "authors": ["Q. Zhang", "C. Hu", "S. Upasani"],
       "year": 2026,
       "arxiv_id": "2510.04618",
       "relevance": "Formalizes Agentic Context Engineering (ACE) as evolving playbooks with a generate-reflect-curate cycle, identifying brevity bias in context optimization."
     },
     {
       "title": "Agentic Much? Adoption of Coding Agents on GitHub",
       "authors": ["R. Robbes", "T. Matricon", "T. Degueule"],
       "year": 2026,
       "arxiv_id": "2601.18341",
       "relevance": "Documents adoption of fully agentic systems for coding on GitHub, providing context for the emerging agentic development ecosystem."
     }
   ]
 }