ai-research-survey

calibration.json (17347B)

---

 {
   "paper_slug": "agentsnet-coordination-collaborative-2025",
   "calibration_date": "2026-02-28",
   "sonnet_scan_date": "2026-02-28",
   "agreement_rate": 0.94,
   "total_questions": 50,
   "agreements": 47,
   "disagreements": 3,
   "disagreement_details": [
     {
       "category": "statistical_methodology",
       "question": "significance_tests",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "The paper makes explicit comparative claims ('best performing models are Claude 3.7 Sonnet, Gemini 2.5 Pro, and Gemini 2.5 Flash', 'Gemini 2.5 Flash is roughly on par with Claude 3.7'). The schema states 'NA if the paper makes no comparative claims' — since comparative claims are present, significance tests apply. Sonnet treats benchmark evaluations as inherently non-comparative, but the paper does rank and compare models without statistical testing."
     },
     {
       "category": "statistical_methodology",
       "question": "effect_sizes_reported",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Same reasoning as significance_tests: the paper makes comparative claims between models. Effect sizes (relative improvements, magnitude of differences) are applicable when claims of difference are made. While raw benchmark scores are reported, the paper never explicitly computes or frames relative differences as effect sizes. Applies because comparisons are made; answer is false because no formal effect sizes are provided."
     },
     {
       "category": "limitations_and_scope",
       "question": "threats_to_validity_specific",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": true, "answer": true},
       "direction": "opus_generous",
       "explanation": "Appendix G discusses specific threats beyond Section 6: (1) synchronous communication limits ecological validity, (2) binary metric may obscure partial progress in tasks with substantial near-correct solutions, (3) homogeneous agents do not capture heterogeneous real-world deployments, (4) no adversarial or faulty agents considered. These are specific to this study's design, not generic boilerplate. Sonnet focused on Section 6's narrower discussion of implementation choices and noted the absence of threats like graph size/topology bias, but Appendix G contains substantive, study-specific validity discussions."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided at https://github.com/floriangroetschla/AgentsNet (Section 5.1). Open-source code explicitly stated."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace at https://huggingface.co/datasets/disco-eth/AgentsNet (Section 5.1)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions LangChain and NetworkX as frameworks but provides no requirements.txt, Dockerfile, or detailed dependency version specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. Code is on GitHub but no 'Reproducing Results' section or specific run commands are documented in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports standard error of the mean in parentheses (e.g., '0.14 (0.04)'). Figure 1 explicitly states 'Error bars indicate standard error of the mean.' Appendix C details the computation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims ('best performing models are Claude 3.7 Sonnet, Gemini 2.5 Pro, and Gemini 2.5 Flash'; 'Gemini 2.5 Flash is roughly on par with Claude 3.7') but uses no significance tests (p-values, t-tests, bootstrap, etc.) to support these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw benchmark scores but never explicitly computes or frames relative differences as effect sizes. Comparisons like 'roughly on par' and 'cheaper by a factor of 20' are made informally without formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "27 network topologies with at least one repeat per graph (Section 5.1). No power analysis or justification for why this number of graphs and repeats is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard error of the mean reported in Table 2 for all task-model combinations. The statistical methodology follows Miller [32] as described in Appendix C."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Ten frontier LLM models are compared against each other on the benchmark (Table 2), including Claude, GPT, Gemini, Llama 4, and o4-mini variants."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All evaluated models are 2024-2025 frontier models (Claude 3.5 Haiku, Claude 3.7 Sonnet, GPT-4.1 mini, Gemini 2.0/2.5 variants, Llama 4, o4-mini)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study examines which components of the benchmark design matter (e.g., number of message-passing rounds, chain-of-thought prompting, graph topology effects). The scaling experiment (Section 5.3) increases graph size but does not ablate system components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Binary (fully correct) evaluation is the main metric (Table 2), and soft evaluation scores providing continuous quality measures are reported in Appendix B / Table 4."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to this benchmark paper evaluating LLM agents on structured mathematical/graph problems with objective ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Graphs are procedurally generated and no model tuning is performed on them. The evaluation is zero-shot with no development/test split confound."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 provides per-task and per-graph-size breakdown. Table 2 provides per-task scores. Both breakdowns complement the aggregate AGENTSNET score."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 presents three key findings about failure modes: strategy coordination failures, agents accepting erroneous information, and conflict resolution behaviors. Appendix E provides detailed transcript examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance drops to near zero for 100-agent networks (Figure 5). Many models perform poorly on VERTEXCOVER and COLORING. These negative results are reported candidly."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: 'strong performance for small networks but fall off as network scales' (Table 2, Figures 4-5); '100 agents' (Figure 5); 'existing benchmarks cover at most 2-5 agents' (Section 2, Related Work)."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a benchmark evaluation that reports performance scores. No causal claims are made about the authors' own system — statements describe observed performance patterns (e.g., 'performance drops') rather than asserting causal mechanisms."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are appropriately scoped to specific models tested on the AGENTSNET benchmark. Results are presented as performance on specific tasks and graph sizes, not generalized to all multi-agent coordination."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 describes failure patterns qualitatively but does not systematically discuss alternative explanations for performance differences between models (e.g., context length limitations vs. reasoning capability vs. instruction-following quality)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 in Appendix A lists exact model versions: claude-3-5-haiku-20241022, claude-3-7-sonnet-20250219, gpt-4.1-mini, gemini-2.5-flash-preview-04-17, etc. These are the actual API model identifiers used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full system prompt template provided in Appendix A with all template variables. Task-specific prompt texts ([task1] and [task2]) for all five tasks provided in Appendix B. Template variables are deterministic from the graph instance, and the dataset is released, enabling full prompt reconstruction."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM API hyperparameters (temperature, top-p, max tokens) are reported anywhere in the paper or appendices."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Algorithm 1 describe the message-passing protocol in detail: synchronous round structure, JSON message formatting, retry logic for invalid JSON, chain-of-thought prompting option."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes graph generation: 27 topologies from 3 graph models (small-world, scale-free, Delaunay) x 3 sizes (4, 8, 16) x 3 instances. Appendix D details the graph generation models."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 is titled 'Limitations' and Appendix G provides an extended 'Limitations' discussion covering synchronous communication, binary metrics, agent homogeneity, scalability, and adversarial settings."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G discusses specific threats: (1) synchronous communication model limits ecological validity for real-world transfer, (2) binary metric may obscure partial progress, (3) homogeneous agents miss heterogeneous deployment challenges, (4) no adversarial/faulty agents tested. These are specific to this study's design choices."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Appendix G identifies specific design limitations, the paper does not explicitly state what claims the results do NOT support or what the benchmark does NOT measure about multi-agent coordination capability more broadly."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace at https://huggingface.co/datasets/disco-eth/AgentsNet, allowing independent verification of benchmark instances."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Graph generation procedurally described in Section 5.1 and Appendix D: three graph models (Watts-Strogatz, Barabasi-Albert, Delaunay), three sizes, three instances each."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Study uses procedurally generated graphs and LLM API calls."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: graph generation (Section 5.1, Appendix D), agent instantiation and message-passing (Section 4, Algorithm 1), response extraction and scoring (Section 3, Appendix C)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments or funding disclosure section in the paper. Two authors are affiliated with Google Research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed on the title page: ETH Zurich, RWTH Aachen University, and Google Research."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two of five authors are from Google Research, which develops Gemini models — four of ten evaluated models. This represents a non-independent relationship. No conflict acknowledgment is made."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement in the paper. The Google Research affiliation and potential interest in Gemini model performance is not explicitly declared as a conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLM models on the benchmark but does not state training data cutoff dates for any evaluated model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark tasks or graph structures could appear in training data. The procedural generation makes exact instance contamination unlikely, but the underlying problem types (graph coloring, leader election, etc.) are classical and well-studied."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is novel (2025), but classical distributed computing problems (graph coloring, vertex cover, etc.) are extensively documented in textbooks and papers likely in training data. The paper does not address whether familiarity with these problem types affects performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this benchmark evaluation paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 plots model performance versus API cost per repeat in USD (as of May 15, 2025), ranging from ~$1 to ~$200 per repeat. Pareto-optimal models identified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-repeat API costs shown in Figure 1, but total computational budget (total number of API calls, total cost across all experiments) is not explicitly stated."
       }
     }
   }
 }