ai-research-survey

calibration.json (17791B)

---

 {
   "paper_slug": "agentask-multiagent-systems-2025",
   "calibration_date": "2026-02-28",
   "model": "opus",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "category": "conflicts_of_interest",
       "question": "funder_independent_of_outcome",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false reasoning that no funding is disclosed and hence the question is NA (unfunded). Opus set applies=true because authors are affiliated with major research institutions (USTC, Shanghai AI Lab, NUS) where it is implausible the work was entirely unfunded. The schema says 'NA if unfunded' but absent explicit statement that the work was unfunded, the more conservative reading is that funding was simply not disclosed — making funder independence unassessable (applies=true, answer=false) rather than inapplicable."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The word 'Code' appears at the top of the abstract without any URL, repository link, or archive. No working code release is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 824 annotated execution logs used for the taxonomy and the SFT training dataset are not released. The evaluation benchmarks (GSM8K, MATH, MMLU, HumanEval, MBPP) are public but the paper's own collected data is not available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda file, or environment specification is provided. Model names (Qwen-3-4B, Llama-3.2-3B, GPT-4o-mini-0718) are listed but no dependency or software environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 provides training pseudocode but no step-by-step reproduction instructions, README, or runnable scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports standard deviations for +GPT-5 and +AgentAsk conditions (e.g., '94.52 ± 1.27'). Origin baselines are single point estimates without uncertainty, but the main experimental conditions include error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AgentAsk improves accuracy by +3.29 to +3.45 points but reports no statistical significance tests — no p-values, t-tests, Mann-Whitney U, or bootstrap tests. All comparative claims are based solely on numeric differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Improvements are reported as percentage-point differences (e.g., +3.29 to +3.45) but no standardized effect sizes (Cohen's d, odds ratios) are provided. While baseline context is given, this does not meet the bar for formal effect size reporting."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the sample size of 824 execution logs used for taxonomy annotation. No power analysis for benchmark evaluations. The number of training examples N is referenced symbolically but never stated."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in Table 1 for +GPT-5 and +AgentAsk conditions across all five benchmarks and four frameworks. Origin baselines are single-run numbers, but the main experimental conditions report spread."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines included: single-model prompting (IO, CoT, Self-Refine) and four multi-agent frameworks (GPTSwarm, AFlow, MaAS, MasRouter). Also compares +GPT-5 as an upper-bound clarifier."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All four MAS baselines (GPTSwarm, AFlow, MaAS, MasRouter) are from 2024-2025 and represent state-of-the-art multi-agent systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and Table 3 ablate reward components (rpar, reff, rfmt) with full 'R only' comparison. Table 9 compares E-GRPO vs PPO and GRPO. Table 8 provides extended ablation across three frameworks."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Accuracy (GSM8K, MATH, MMLU), Pass@1 (HumanEval, MBPP), plus latency and extra cost as efficiency metrics. Four distinct metrics used."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates on automated benchmarks with objective ground-truth answers (math solutions, code correctness, multiple choice). Human evaluation of system outputs is not relevant to the claims made."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly describe whether standard test splits were used for the benchmarks or whether any tuning decisions were made using the evaluation data. The relationship between training data (execution logs) and benchmark evaluation splits is not clarified."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down per benchmark (5 benchmarks) in Tables 1, 4-7; per framework (4 frameworks); and per error type (Data Gap, Signal Corruption, Referential Drift, Capability Gap) with Resolved@Edge rates in Section 5.5 and Figure 4."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 discusses where AgentAsk is less effective: Referential Drift (58.3% resolved) and Capability Gap (49.5% resolved). The Limitations section acknowledges inability to handle hallucinations. Case studies in Appendix D illustrate specific error patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows AgentAsk decreasing GSM8K accuracy by -0.20 on MaAS. Tables 5-6 show slight decreases in some SFT configurations (e.g., -0.11 MBPP on Llama-3.2-3B SFT, -0.08 MMLU on MaAS Qwen-3-4B SFT). These are reported transparently."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'improving accuracy by up to 4.69%' confirmed in Table 1 (MaAS +GPT-5 average +4.69). 'Latency and extra costs below 10%' supported by Tables 2 and 5-7 showing normalized extra costs of 3.3-9.7 for AgentAsk."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies in Section 5.4 provide controlled single-variable manipulation removing individual reward components. The experimental design holds orchestration fixed and varies only the clarifier, isolating its causal effect. Comparison with +GPT-4o-mini and +GPT-5 further supports the causal design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Multi-Agent Systems Need to Ask' and claims like 'improving the reliability of multi-agent systems' are broad. Evaluation is limited to 5 benchmarks (math reasoning, QA, code), one executor model (GPT-4o-mini-0718), and four specific MAS frameworks. Claims are not bounded to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the observed gains. Could improvements stem from simply adding more LLM calls regardless of clarification content? From increased token budget? The +GPT-4o-mini baseline partially controls for model-call overhead but is not framed as testing an alternative explanation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions provided: 'GPT-4o-mini-0718' (with date stamp), 'Qwen-3-4B', 'Llama-3.2-3B'. GPT-5 referenced with OpenAI URL accessed 2025-08-07. These are sufficiently specific identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix F, Figure 7 provides the complete edge-level clarifier prompt including all four error types (DG, SC, RD, CG), decision criteria for when/what/whom to ask, and the NONE condition. This is the full prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 lists hyperparameters as required inputs (lr eta, lambda_ask, lambda_R, epsilon, beta, weights wt, baseline b) but their actual values are never reported. Temperature=0.3 is mentioned only for robustness experiments in Appendix C.1. No hyperparameter table is provided."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4.1-4.3 describe the edge-local state (Eq. 9), action space (Eq. 10), factorized policy (Eq. 11), two-stage training (SFT then E-GRPO RL), and reward shaping. Algorithm 1 provides full pseudocode. Appendix A details modeling and SFT."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The training corpus is described as built 'from a set of logged multi-agent executions' with labels from 'a teacher model' (Appendix A.2), but the paper does not document how execution logs were collected, filtered, or preprocessed. The number of training examples N is referenced but never stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the Conclusion, discussing that AgentAsk cannot eliminate errors from internal model flaws like hallucinations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section is generic: AgentAsk 'cannot fully eliminate all errors' due to 'well-known limitations in large models, such as hallucinations and inconsistencies.' No specific threats to experimental validity are discussed (e.g., benchmark contamination, selection of favorable frameworks, confounds in cost measurements)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statement of what the results do NOT show. No discussion of excluded domains, task types, or model families. No explicit disclaimers about what claims are NOT being made. The paper presents results on math/QA/code without bounding generalization."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The 824 annotated execution logs forming the taxonomy basis and the SFT training data are not released. The error type distribution claims (Data Gap 29.1%, Referential Drift 27.3%, Signal Corruption 36.8%, Capability Gap 6.8%) cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '824 execution logs' were audited but does not describe which frameworks generated them, which tasks they covered, over what time period, or what inclusion/exclusion criteria were applied."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the research sense. Annotators are described as 'multiple professional annotators with expertise in MAS' performing taxonomy classification — this is annotation work, not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from execution logs to taxonomy annotations to SFT training data is not fully documented. Filtering criteria, number of training examples, intermediate processing steps, and how many examples were removed at each stage are not described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section and no funding sources disclosed. Authors are affiliated with USTC, Shanghai AI Lab, Xi'an Jiaotong University, and NUS — institutions that typically fund research — but no grants or sponsors are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed on the title page: USTC, Shanghai AI Laboratory, Xi'an Jiaotong University, NUS. No author is affiliated with OpenAI or DeepSeek whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed despite authors being affiliated with major research institutions (USTC, Shanghai AI Lab, NUS) where unfunded research is implausible. Since funding is not disclosed, funder independence cannot be assessed. The schema says 'NA if unfunded' but the absence of a funding disclosure from well-resourced institutions is not the same as confirmed unfunded status."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears anywhere in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o-mini-0718 and GPT-5 are used to evaluate on benchmarks but no training data cutoff dates are stated for these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Public benchmarks (HumanEval 2021, MMLU 2021, GSM8K 2021, MATH 2021, MBPP 2021) used with models trained well after these were published. No discussion of potential train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All five benchmarks were published in 2021, well before the training data cutoff of any model used. The paper does not discuss contamination risk despite this being a significant concern for interpreting benchmark results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Annotators classify agent execution logs but are not research participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the research sense. Professional annotators performing taxonomy classification does not constitute a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Annotators are described as professionals with MAS expertise but are not research subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants whose inclusion/exclusion criteria need specification."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment with conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants who could drop out of a study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2, 5-7 report 'Extra Cost' as a normalized metric relative to origin=0. AgentAsk achieves extra costs of 3.3-9.7 while +GPT-5 incurs 24-44 units. These are relative rather than absolute costs but do quantify the overhead."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, total API spend, or training time reported. The SFT and E-GRPO training processes are described algorithmically but computational costs are not quantified."
       }
     }
   }
 }