ai-research-survey

calibration.json (19166B)

---

 {
   "calibration": {
     "paper_slug": "agentspawn-adaptive-multiagent-2026",
     "calibration_date": "2026-02-28",
     "sonnet_scan_date": "2026-02-28",
     "model": "opus",
     "agreement_rate": 0.94,
     "total_questions": 50,
     "agreements": 47,
     "disagreements": 3,
     "disagreement_details": [
       {
         "category": "evaluation_design",
         "question": "multiple_metrics",
         "sonnet": {"applies": true, "answer": true},
         "opus": {"applies": true, "answer": false},
         "direction": "sonnet_generous",
         "reasoning": "Sonnet credits the paper for proposing multiple evaluation metrics in Section 4.1 (task completion rate, memory overhead, spawn count, coherence violations, cost per success). However, the conclusion explicitly lists empirical validation as future work, meaning no experiments were conducted and no metrics were actually measured. Proposing metrics in a design document is not the same as using multiple metrics in an evaluation. The schema asks 'Are multiple evaluation metrics used?' — they were not used because no evaluation occurred."
       },
       {
         "category": "evaluation_design",
         "question": "per_category_breakdown",
         "sonnet": {"applies": true, "answer": true},
         "opus": {"applies": true, "answer": false},
         "direction": "sonnet_generous",
         "reasoning": "Sonnet credits Tables 4 (breakdown by difficulty: Easy/Medium/Hard) and 5 (breakdown by task type: Simple refactor/Multi-file fix/Complex feature) as per-category breakdowns. While these tables visually contain breakdowns, the conclusion confirms no experiments were conducted — these numbers are fabricated. Crediting presentation of invented data as satisfying a methodological criterion is incorrect. The criterion evaluates whether actual results are broken down, not whether hypothetical numbers are formatted in a table."
       },
       {
         "category": "cost_and_practicality",
         "question": "inference_cost_reported",
         "sonnet": {"applies": true, "answer": true},
         "opus": {"applies": true, "answer": false},
         "direction": "sonnet_generous",
         "reasoning": "Sonnet credits Table 6 (cost-benefit analysis: $2.10 vs. $2.30 cost per success, 23 vs. 12 API calls, 26 vs. 18 minutes average time) and Section 5.3's O(n·ℓ) cost scaling analysis. However, Table 6's numbers are not from actual experiments — the conclusion lists 'empirical validation' as future work. These are projected or hypothetical costs, not measured inference costs. The schema asks 'Is inference cost or latency reported?' — reporting invented projections of what costs might be does not satisfy this criterion."
       }
     ],
     "disagreement_summary": {
       "sonnet_generous": 3,
       "opus_generous": 0,
       "applies_boundary": 0,
       "interpretive": 0
     },
     "notes": "All three disagreements follow the same pattern: Sonnet credits the paper for presenting tables and numbers that appear to satisfy the criterion on their surface, even though the conclusion explicitly states that no experiments were conducted (empirical validation is listed as future work item #2). This is a case of Sonnet evaluating the form of the paper (tables exist, numbers are present, categories are shown) rather than the substance (the numbers are fabricated or hypothetical). This paper is unusual in that it presents extensive fake experimental results as if they were real, which creates a trap for checklist evaluation: a literal reading of 'are multiple metrics used' would credit the paper because multiple metrics appear in the tables, but a substantive reading recognizes that metrics from non-existent experiments don't count as 'used.' Opus consistently applies the substantive reading."
   },
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code, repository, or archive is provided. The conclusion (Section 6) explicitly states 'implementing the AgentSpawn prototype using Claude 3.5 Sonnet or GPT-4' as future work, confirming no implementation exists."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No datasets are released. The custom refactoring task set (100 tasks) mentioned in Section 4.1 is not available. SWE-bench and Defects4J are referenced as benchmarks but no data artifacts are provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, dependency list) is provided. The paper has no implementation, so there is no environment to specify."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions exist. The conclusion lists 'empirical validation on SWE-bench and Defects4J with proper baseline configurations' as future work item (2), confirming experiments were never conducted."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars appear anywhere. All tables (2-6) report single point estimates with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparative claims (e.g., '+97% over GPT-4 Single' in Table 2) lack any statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports percentage improvements over baselines (e.g., +97%, +85%, +74%) but the GPT-4 Single baseline absolute values are not stated — only 'baseline' appears. Without knowing the absolute rates, effect sizes are uninterpretable. Furthermore, the results are from non-existent experiments."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 300 SWE-bench tasks, 200 Defects4J tasks, and 100 custom tasks (Section 4.1) with no justification for these sizes and no power analysis. The experiments were never conducted."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures appear for any result. All numbers are single-point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 lists baselines (GPT-4 Single, AutoGen, CrewAI, AFLOW) but the conclusion states 'empirical validation on SWE-bench and Defects4J with proper baseline configurations' is future work, meaning baseline comparisons were never actually conducted."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines listed (AutoGen, CrewAI, AFLOW) are contemporary systems, but since no experiments were conducted (per the conclusion's future work list), no actual baseline comparison was performed."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 presents an ablation study removing memory slicing, skill inheritance, adaptive spawning, and coherence manager. However, the conclusion confirms experiments were not conducted, so these ablation numbers are hypothetical."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 proposes five metrics (task completion rate, memory overhead, spawn count, coherence violations, cost per success) but no experiments were conducted. Proposing metrics in a design is not the same as using them in an evaluation. No metrics were actually measured."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates automated code generation via benchmark test suites. Human evaluation is not applicable and not claimed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No train/test split methodology is described. The paper references existing benchmarks as test sets but no experiments were conducted and no splitting procedure is documented."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Tables 4 and 5 present breakdowns by task difficulty (Easy/Medium/Hard) and task type (Simple refactor/Multi-file fix/Complex feature). However, since the conclusion confirms no experiments were run, these numbers are fabricated. Presenting invented data in categorized tables does not constitute providing per-category breakdowns of actual results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No actual failure cases are shown or analyzed. Section 5.4 lists design limitations (hyperparameter sensitivity, semantic merge complexity) but presents no empirical failure examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every result presented shows AgentSpawn outperforming all baselines on all benchmarks. No negative results, failed approaches, or configurations that didn't work are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims '34% higher completion rates than static baselines on benchmarks like SWE-bench' and '42% memory overhead reduction.' The conclusion contradicts these by listing empirical validation as future work, meaning the abstract's claims are unsupported by actual experiments."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims: 'memory slicing reduces context overflow failures by 42%' (Section 4.2), the ablation in Table 3 implies causal component contributions. No experiments were conducted, so no causal claims can be justified."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims results 'on benchmarks like SWE-bench' without clarifying experiments were never performed. Section 5.6 generalizes to 'document authoring, data analysis, system administration' without any evidence. The title ('Long-Horizon Code Generation') is broad given no empirical validation exists."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the presented results are discussed. Section 5.4 mentions design limitations but does not consider alternative explanations for why the system might appear to perform well (e.g., benchmark selection, baseline configuration)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "No model version is specified. The conclusion mentions 'Claude 3.5 Sonnet or GPT-4' as future implementation candidates. No model was actually used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompts are provided. Section 3.3 describes skills as 'parameterized prompts s = (p, θs)' with abstract examples like 'Write unit tests for {function}' — these are templates, not actual prompts. No experiments were conducted."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides hyperparameter specifications: spawn threshold δ=0.7, memory threshold θ=0.5, metric weights w1-w5, max spawn depth=3, concurrent spawn limit=4, timeout=600s, relevance weights α=0.3, β=0.3, γ=0.2, δ=0.2. These are proposed design values, but they are explicitly stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively describes agentic scaffolding: five components (Section 3.1), three formal algorithms (Algorithms 1-3), data structures for spawn/resume packages (Appendix A), memory tiers (Section 3.2), complexity metrics (Section 3.4), and coherence protocols (Section 3.6)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing is described. Section 4.1 states benchmark sizes (300 SWE-bench, 200 Defects4J, 100 custom) without describing how tasks were selected, filtered, or prepared."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 'Limitations' provides a dedicated subsection discussing four specific limitations: hyperparameter sensitivity, semantic merge complexity (73% success rate varies), domain generalization, and spawn depth scaling."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 discusses design limitations but not threats to validity of the experimental results. Critically, it does not acknowledge the most significant validity threat: that the presented results are from experiments that were never conducted."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not clearly state what the results do NOT show. The abstract and body present fabricated results as real without bounding what was actually demonstrated (architectural design only) versus what was not (any empirical validation)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The experimental results in Tables 2-6 are not from actual experiments — the conclusion lists empirical validation as future work. No underlying data exists to verify."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "No data collection procedure is described. The paper references existing benchmarks and a 'custom refactoring task set' without describing how any data was gathered or prepared."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper evaluates automated code generation benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. There is no description of how raw data flows to final results, because no experiments were conducted."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. There is no acknowledgments section, no grant information, and no mention of any institutional or corporate support."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author lists 'AutoHand Evolve' as affiliation with email igor@autohand.ai on the title page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The author is affiliated with AutoHand Evolve, which presumably has commercial interest in the proposed architecture's success."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interests declaration appears anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for any model. The conclusion mentions 'Claude 3.5 Sonnet or GPT-4' as future implementation options, but no model was actually used in experiments."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. SWE-bench tasks are publicly available and likely in LLM training data, but this is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench (published 2023) and Defects4J are public benchmarks likely seen during training of contemporary LLMs. The paper does not address contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates automated code generation benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or randomized experimental design."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 6 presents cost figures ($2.10 vs. $2.30 per success, API call counts, timing) and Section 5.3 discusses O(n·ℓ) cost scaling. However, these are hypothetical projections from experiments that were never conducted (per the conclusion's future work list), not measured inference costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. Since no experiments were conducted, there is no actual compute expenditure to report."
       }
     }
   }
 }