ai-research-survey

calibration.json (18343B)

---

 {
   "calibration_metadata": {
     "paper_slug": "agentmesh-cooperative-multiagent-2025",
     "calibration_date": "2026-02-28",
     "sonnet_scan_date": "2026-02-28",
     "calibrator": "opus",
     "agreement_rate": 0.88,
     "total_questions": 50,
     "agreements": 44,
     "disagreements": 6
   },
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "baselines_contemporary",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false because no formal baselines are included. Opus sets applies=true because the paper does make informal comparative claims (multi-agent vs single-agent GPT-4) and could have included contemporary baselines like ChatDev or MetaGPT. Per scan-agent instructions: 'Do NOT set applies=false when a paper is weak in an area.' The absence of baselines is a weakness, not structural inapplicability."
     },
     {
       "category": "evaluation_design",
       "question": "multiple_metrics",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false because no quantitative metrics are reported. Opus sets applies=true because the paper presents a system that generates code and could have measured success rates, code quality metrics, token usage, etc. The paper chose not to report metrics, which is a weakness, not structural inapplicability. A case-study paper demonstrating a code generation system can and should report metrics."
     },
     {
       "category": "evaluation_design",
       "question": "human_evaluation",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false reasoning it is a 'system demonstration' where human evaluation is not in scope. Opus sets applies=true because the paper claims the system generates working software, and human evaluation of code quality/correctness would be directly relevant to validating those claims. The schema says NA only 'if human evaluation is clearly irrelevant to the claims' — for a code generation system, it is clearly relevant."
     },
     {
       "category": "evaluation_design",
       "question": "per_category_breakdown",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false because no quantitative evaluation across categories is performed. Opus sets applies=true because the paper tests on multiple tasks (to-do app, REST API, 2048 game) and could have provided per-task breakdowns of success/failure. The absence of breakdown is a methodological weakness, not structural inapplicability."
     },
     {
       "category": "contamination",
       "question": "training_cutoff_stated",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=true because GPT-4 is used and its training cutoff matters for the case study tasks. Opus sets applies=false because the schema's NA rule says 'NA if the paper does not evaluate a pre-trained model's capability on any benchmark.' The paper uses GPT-4 within a framework demonstration on self-constructed example tasks, not on a recognized benchmark. This is a genuine boundary case — the to-do app case study implicitly tests GPT-4's coding capability but is not a formal benchmark evaluation."
     },
     {
       "category": "contamination",
       "question": "train_test_overlap_discussed",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Same reasoning as training_cutoff_stated. Sonnet treated the case study as implicitly evaluating GPT-4's capability, making contamination relevant. Opus applied the schema's NA rule strictly: self-constructed example tasks are not a benchmark, so contamination questions do not apply. Both positions are defensible."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided. The paper describes the implementation and provides pseudo-code snippets and prompt excerpts, but no link to working code."
       },
       "data_released": {
         "applies": false,
         "answer": false,
         "justification": "This is a system architecture paper with a case study demonstration. There is no collected dataset to release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Python-based framework' and 'OpenAI API (GPT-4 model)' but provides no requirements.txt, Dockerfile, or version specifications for any libraries."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions. Only architectural descriptions and simplified pseudo-code (Listing 1)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "The paper explicitly states 'We did not perform a rigorous quantitative evaluation here.' No quantitative metrics are reported, making confidence intervals structurally inapplicable."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative comparisons or numeric results are reported. Significance tests are structurally inapplicable."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative results reported at all. Effect sizes are structurally inapplicable."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Single case study plus two briefly mentioned examples. No statistical sample requiring justification."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative results reported. Variance is structurally inapplicable."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No systematic baseline comparison. The paper anecdotally mentions that 'asking GPT-4 to do the entire to-do app in one go often resulted in missing persistence,' but this is informal and unmeasured."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (multi-agent vs single-agent) and cites contemporary multi-agent frameworks (ChatDev, MetaGPT) but does not include them as formal baselines. Could and should have compared against these systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has four components (Planner, Coder, Debugger, Reviewer). No ablation study removing any component is performed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative metrics are reported at all. The paper could have measured success rates, code quality, token usage, or time, but reports only qualitative narrative from a case study."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's outputs. The paper claims the system generates working software, making human evaluation of code quality directly relevant. Only informal developer observation is described."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No train/test split. The paper uses self-constructed example tasks as demonstrations, not a formal evaluation on held-out data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on multiple tasks (to-do app, REST API, 2048 game) but provides no per-task breakdown of results. Only the to-do app is described in detail; the other two are mentioned in a single sentence."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion and Limitations section discusses specific failure modes: incomplete plans causing missing features, error propagation, Debugger getting stuck in loops, hallucinations, and context window limitations."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly describes failures: 'we did encounter cases where the plan was incomplete, causing the final software to lack a feature,' and discusses hallucinations, debugging loops, and inability to guarantee correctness."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are appropriately hedged: 'We propose AgentMesh,' 'A case study illustrates,' 'We discuss... limitations.' These claims are supported by the architecture description, case study walkthrough, and limitations section."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims without controlled experiments: 'the multi-agent approach ensured that planning was done before coding (preventing the model from jumping straight into code),' 'the Debugger agent was invaluable in fixing syntax errors.' No ablation, no controlled comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'AgentMesh can handle multi-faceted tasks more robustly than a single prompt' and envisions systems that 'one day operate as a full-fledged software development team,' but only tested a single Python CLI to-do app in detail plus brief mentions of two other Python tasks. The title says 'Software Development Automation' generally."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations considered. The benefit of multi-agent over single-agent could be due to more total tokens, different prompt engineering, API non-determinism, or task simplicity rather than the multi-agent architecture itself. None of these confounds are discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Specifies 'GPT-4' and mentions 'up to 8K or 32K tokens depending on version,' but does not provide an exact model version (e.g., 'gpt-4-0613') or snapshot date."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided for PlannerAgent and DebuggerAgent including system and user messages in the 'Example Prompt Design' section. The case study shows what the placeholders were filled with."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other sampling parameters reported despite using the OpenAI API."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed description of sequential orchestration pipeline, shared project state dictionary, Debugger retry logic, artifact-centric communication, and pseudo-code orchestration loop (Listing 1)."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "System demonstration paper with no dataset collection or preprocessing pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated 'Discussion and Limitations' section covering six specific limitations: LLM output quality/error propagation, hallucinations, context window/scalability, lack of learning, evaluation guarantees, and domain constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific to this system: 'the plan was incomplete, causing the final software to lack a feature,' 'context window (for GPT-4, can be up to 8K or 32K tokens),' 'we cannot guarantee the correctness, completeness, or security of the generated software.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly delineate what the results do NOT show. It does not state boundaries like 'we only tested Python CLI tasks' or 'we cannot claim results for other languages/domains.' Domain constraints are mentioned as a limitation but not as explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Case study results (generated code, agent outputs) are shown only as excerpts. No raw logs, complete generated programs, or full agent interaction traces are made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The example tasks (to-do app, REST API, 2048 game) are chosen informally with no description of how or why these specific tasks were selected."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants and no benchmark dataset. Self-generated example tasks. Recruitment is not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No systematic documentation of how many tasks were tried, success/failure rates, or how the presented examples were selected from any larger set of attempts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section. No mention of grants, institutional support, or funding of any kind."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The single author's affiliation (Toronto Metropolitan University) is clearly listed on the title page."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding disclosed. Appears to be unfunded academic work by a single researcher."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate GPT-4's capability on a recognized benchmark. It uses GPT-4 within a framework demonstration on self-constructed example tasks. Per schema NA rule: 'NA if the paper does not evaluate a pre-trained model's capability on any benchmark.'"
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning as training_cutoff_stated. Self-constructed example tasks are not a benchmark, so train/test overlap is not applicable per the schema's NA rule."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No standard benchmark used. Self-constructed example tasks (to-do app, REST API, 2048 game)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses cost conceptually ('scales roughly with the number of subtasks times the cost of each agent's LLM calls') but provides no actual figures — no token counts, API costs, or wall-clock times."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, API spend, hardware specifications, or total computational cost reported."
       }
     }
   }
 }