ai-research-survey

calibration.json (19245B)

---

 {
   "paper_slug": "agent-error-taxonomy-2025",
   "calibration_date": "2026-02-28",
   "calibration_model": "opus",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "question": "statistical_methodology.effect_sizes_reported",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": true,
       "opus_answer": true,
       "direction": "opus_generous",
       "explanation": "Sonnet says the paper 'does not report formal effect sizes such as Cohen's d or odds ratios.' Opus notes that the schema explicitly lists 'percentage improvement with baseline context' as sufficient for YES, giving the example 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper consistently reports percentage improvements with baseline numbers (e.g., '24.3% vs. 0.3%', '45.0% vs. 28.0%', 'from 21 to 55'). These provide the magnitude context the schema requires. On reflection, this is a borderline case where Opus may be slightly generous — while the paper provides raw comparison numbers, it does not report standardized effect sizes (Cohen's d, odds ratios). The schema's example shows raw percentage with context is sufficient, but the paper's comparisons are not truly 'effect sizes' in the formal statistical sense. Opus leans toward YES based on the schema's explicit example."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'The code and data will be available at https://github.com/ulab-uiuc/AgentDebug', which is a promise of future release. Per schema criteria, a promise of future release counts as NO."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper promises data availability at the same GitHub URL, but this is a future release promise ('will be available'). AgentErrorBench is not currently downloadable."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment specification is provided. The paper specifies model names and temperature=0 for GPT-4.1 but nothing sufficient to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "Algorithm 1 provides pseudocode but no step-by-step reproduction instructions (commands, scripts, README). A competent researcher could not replicate the experiments from the paper alone."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 and Figures 5-7 report only point estimates (percentage accuracy, success rates). No confidence intervals, error bars, or uncertainty quantification is provided anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. The paper claims AgentDebug 'consistently surpasses baselines' (Section 4.1) based solely on comparing point estimates without p-values, t-tests, or any other statistical tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout: '24.3% vs. 0.3% all-correct' (Table 1), '45.0% vs. 28.0% step accuracy' (Table 1), 'success from 21 to 55' (Section 4.2). Per the schema, 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper consistently provides this form of context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 200 annotated trajectories (100 ALFWorld, 50 WebShop, 50 GAIA) but no power analysis or justification for why this sample size is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single-run point estimates. No standard deviation, variance across repeated experimental runs, or any spread measure is provided. Figure 7a shows cumulative success across attempts but these are cumulative counts, not variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 compares against Direct Prompting, Brute Force, and Binary Search for error detection. Section 4.2 compares against Self-Refine, Vanilla Debugger, Tree-of-Thought, and Best-of-N for downstream task success."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are contemporary: Self-Refine, Tree-of-Thought, and Best-of-N are recent methods. The paper explicitly controls for compute budget to ensure fair comparison (Section 4.2: 'max number of attempts of all baselines is matched to AgentDebug by total token usage')."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 presents ablation studies on three factors: (1) max number of re-rollout attempts (Figure 7a), (2) different base models for the AgentDebug detector (Figure 7b), and (3) different rollout strategies (Figure 7c)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports three evaluation metrics for error detection: Step accuracy (S), Step+Module accuracy (S+M), and All Correct (Step+Module+Error Type). Task success rate is reported separately in Section 4.2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human experts annotated the benchmark (10 graduate students), but this was for dataset construction, not for evaluating AgentDebug's outputs. All evaluation of the system is automated — comparing predictions to ground-truth labels."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "AgentErrorBench serves as the test set for error detection evaluation (Section 4.1). Downstream evaluation uses standard external benchmarks (ALFWorld, GAIA, WebShop). The paper does not describe using the test data for any tuning decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down results per benchmark (ALFWorld, WebShop, GAIA). Figures 11-13 in Appendix A.4 provide per-module and per-error-type failure distributions for each benchmark."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 analyzes error propagation patterns in detail with Figure 8 illustrating cascading failures. Appendix A.3 provides a concrete qualitative failure case comparison (ALFWorld saltshaker example, Figure 10)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7b shows that alternative base models (Llama-3.3-70B, GPT-4o-mini, Qwen3-Next-80B) perform substantially worse as detectors. Table 1 shows Brute Force performing worse than Direct Prompting in some metrics. Figure 7c shows Act Only achieving only 0.10 success rate."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '24% higher all-correct accuracy' — confirmed by Table 1 (24.3% vs. 0.3%). Claims '17% higher step accuracy' — confirmed (45.0% vs. 28.0%). Claims 'up to 26% relative improvements' — shown in Figures 5-6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'correcting a single root-cause mistake can often flip an otherwise failing trajectory' and 'focusing on root-cause errors...is key to efficient debugging.' The main comparison (AgentDebug vs. baselines) involves multiple simultaneous differences (taxonomy-guided analysis, critical error detection, targeted feedback), making causal attribution to any single mechanism difficult. While the ablations vary individual factors within AgentDebug, they don't isolate the core mechanism."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Where LLM Agents Fail' implies broad scope, and the conclusion frames AgentDebug as 'a foundation for agents that can continuously learn and evolve,' which significantly overgeneralizes from 3 specific benchmarks (ALFWorld, WebShop, GAIA) with a single detector model (GPT-4.1)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section (Appendix A.1) discusses scale and annotation cost constraints but does not consider alternative explanations for the observed results — for example, whether gains are driven by GPT-4.1's specific analytical strengths rather than the framework design, or whether the benchmark construction favored the proposed taxonomy."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4.1', 'GPT-4o-mini', 'Qwen3-8B', and 'Qwen3-Next-80B' without API version numbers, snapshot dates, or specific model identifiers. Per the schema, marketing names without a snapshot date or API version do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.5 provides verbatim prompt content: Detector Prompt (Figure 14), AgentDebug Prompt (Figure 15), Baseline Prompts (Figure 16), and Environment Rollout Prompts for ALFWorld (Figure 17), WebShop (Figure 18), and GAIA (Figure 19). The templates include placeholders that are filled deterministically from the trajectory data and taxonomy."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 states 'temperature set to 0 for deterministic outputs' for GPT-4.1. But no other hyperparameters are reported: temperature/top-p/max tokens for the backbone agents (GPT-4o-mini, Qwen3-8B, Qwen3-Next-80B) in Section 4.2 are not specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.2 and Algorithm 1 describe AgentDebug's three-stage scaffolding in detail: Stage 1 (fine-grained analysis with taxonomy), Stage 2 (critical error detection), Stage 3 (iterative debugging with re-rollouts). The modular rollout design with memory, reflection, planning, and action modules is described in Section 3.1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.2 states 200 trajectories were 'curated' from 500+ failed trajectories but does not document the selection criteria. The annotation procedure is described (decision-step level, three rounds of pilot annotation, double-annotation), but the filtering from 500+ to 200 is an undocumented step."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 is titled 'LIMITATION' and provides substantive discussion of limitations including scale/domain constraints and annotation cost."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 identifies specific limitations: (1) the benchmark 'remains limited in scale and domain diversity,' naming multimodal environments and safety-critical applications as untested domains, and (2) annotation costs preventing a trained debugging model. These are specific to this study rather than generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to the specific models tested, acknowledge that results may not apply to multi-agent settings, or state specific things it did NOT test. The limitations are framed as future directions rather than explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "AgentErrorBench (200 annotated trajectories) and the 500+ failed trajectories used for taxonomy development are promised for future release at GitHub but not currently available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes collecting 200 trajectories: 100 from ALFWorld, 50 from WebShop, 50 from GAIA. Ten graduate student annotators labeled each trajectory at the decision-step level. Annotation included training, double-annotation, and disagreement adjudication. Inter-annotator agreement reported (Cohen's kappa = 0.55)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes annotators as 'ten expert annotators—graduate students with prior experience in NLP and LLMs agent research.' The qualification criteria (graduate-level, relevant domain experience) are stated."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper collected 500+ failed trajectories but used only 200 in the benchmark. The selection criteria for which 200 were chosen from the 500+ are not documented. This represents an unexplained filtering step in the data pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The acknowledgment section thanks the OpenManus team for 'discussion and providing some resources' but does not mention grants, corporate sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Illinois Urbana-Champaign, Stanford University, AMD, OpenManus, University of Toronto, Likelihood Lab. The paper does not evaluate any of these institutions' products directly."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. Without an identified funder, independence cannot be assessed. The schema specifies NA if unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper. Per the schema, 'absence of disclosure is not the same as absence of conflict.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates GPT-4.1, GPT-4o-mini, Qwen3-8B, and Qwen3-Next-80B on agent benchmarks but does not state training data cutoff dates for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether ALFWorld, GAIA, or WebShop benchmarks appeared in the training data of the evaluated models. These public benchmarks from 2020-2023 are plausibly in training data for 2025 models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld (2020), WebShop (2022), and GAIA (2023) are public benchmarks that likely appeared in training data for GPT-4.1 and Qwen3 series models. The paper does not discuss this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not involve human participants in an experimental sense. Human annotators constructed the benchmark but are not study subjects. Pre-registration is not applicable."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The ethical statement explicitly states 'Our study does not involve human subjects.' Graduate student annotators are performing a service, not being studied. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental sense. Annotators are characterized as 'graduate students with prior experience in NLP and LLMs agent research,' which is adequate for their role. Human studies demographic requirements do not apply."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental sense. Annotation is a data construction process, not a human study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental conditions involving human subjects requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the experimental sense."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "AgentDebug involves per-step LLM calls for error analysis plus critical error detection per trajectory, with up to 5 re-rollouts. The paper does not report API costs, tokens consumed, wall-clock time, or cost per example."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions controlling for total token usage between methods (Section 4.2: 'the max number of attempts of all baselines is matched to AgentDebug by total token usage') but does not state the absolute compute budget in terms of total tokens, GPU hours, or API spend."
       }
     }
   }
 }