ai-research-survey

calibration.json (21348B)

---

 {
   "paper_slug": "agent-contracts-formal-2026",
   "calibration_model": "opus",
   "scan_model": "sonnet",
   "timestamp": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 47,
   "disagreement_count": 3,
   "agreement_rate": 0.94,
   "disagreements": [
     {
       "category": "claims_and_evidence",
       "question": "generalization_bounded",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "The title 'Agent Contracts: A Formal Framework for Resource-Bounded Autonomous AI Systems' claims general applicability to 'Autonomous AI Systems,' but empirical validation is limited to Gemini 2.5 Flash/Flash-Lite with Google ADK/LiteLLM on two benchmarks (LiveCodeBench, OpenR1). The schema guidance states: 'broad titles like LLM-based Software Engineering when results are on a single benchmark in a single language is NO.' While Section 7 discusses enforcement limitations, the title and conclusion ('formal governance becomes essential... Agent Contracts provide one such foundation') imply general applicability beyond the tested setting. Sonnet credited the Section 7 limitations as adequate bounding, but the title and conclusion's generalizations are not bounded to the tested models and benchmarks."
     },
     {
       "category": "claims_and_evidence",
       "question": "alternative_explanations_discussed",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet credited Section 7's discussion of enforcement limitations and the Section 8.3 outlier analysis as discussion of alternative explanations. However, Section 7 addresses what contracts cannot enforce (single-call budget overruns), which is a limitation, not an alternative explanation for the observed results. The core alternative explanation — that simple budget-aware prompting or iteration limits alone (without the full formal contract framework) could produce equivalent token reduction and variance reduction — is not discussed. The outlier analysis in Section 8.3 is a sensitivity analysis, not a consideration of confounding factors. Per the schema, 'a threats-to-validity section counts only if it discusses specific alternative explanations for the observed results, not just generic methodological limitations.'"
     },
     {
       "category": "cost_and_practicality",
       "question": "inference_cost_reported",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": true, "answer": true},
       "direction": "opus_generous",
       "explanation": "The schema says 'Look for: API costs, tokens consumed, wall-clock time, cost per example.' The paper reports tokens consumed per task (3,461 vs 34,606 in Code Review), wall-clock time per task (6.9s, 12.5s, 16.9s in Strategy Modes), and iteration/LLM call counts. These are explicitly listed in the schema's 'Look for' guidance as sufficient indicators. Sonnet required dollar-cost conversion, but the schema does not require this — 'tokens consumed' and 'wall-clock time' are listed as standalone acceptable evidence."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided in footnote 3: 'Implementation available at https://github.com/flyersworder/agent-contracts.' The authors state they provide 'experiment code and data for reproducibility.' This is a working URL to a public repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: LiveCodeBench (arXiv:2403.07974) and OpenR1 logic puzzles. The authors also state they provide experiment data via the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions Google ADK and LiteLLM as frameworks and Gemini 2.5 Flash/Flash-Lite as models, but provides no requirements.txt, Dockerfile, conda environment file, or specific library versions. Mentioning framework names without version numbers is insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper links to a GitHub repository but provides no step-by-step reproduction instructions in the paper itself. Experiment designs are described conceptually but not with specific commands, scripts, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states 'bootstrap confidence intervals (10,000 resamples) with BCa correction' for statistical analysis. P-values and significance tests are reported throughout the results tables."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Multiple significance tests reported: paired t-test for token usage (p=0.0007), iterations (p<0.0001), LLM calls (p<0.0001), crisis communication token reduction (p=0.005), quality equivalence (p=0.32), success rate difference (p=0.13 NS), and strategy modes (p~0.05)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes reported with baseline context: '90% token reduction' (from 34,606 to 3,461), '525x lower variance' (5.29B vs 10.1M), '7.1 percentage points' (60.0% to 52.9%), '23% token reduction,' '70%->86% success rate,' and '26.7x lower variance' (sigma 1.75 vs 9.07). Absolute and relative differences provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are stated (n=70, n=50, n=50, n=24) but never justified. No power analysis is provided. No acknowledgment of whether sample sizes are sufficient for the claims, particularly the n=24 Crisis Communication experiment."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Variance is a central result metric: '525x lower variance (5.29B vs 10.1M)' for Code Review and '26.7x lower variance (sigma: 1.75 vs 9.07)' for Research Pipeline. Standard deviation is explicitly reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each experiment includes baseline comparisons: CONTRACTED vs UNCONTRACTED in Code Review, Research Pipeline, and Crisis Communication; three contract modes (URGENT, ECONOMICAL, BALANCED) in Strategy Modes."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The UNCONTRACTED baseline (no contract enforcement) is the natural and appropriate baseline for evaluating a first-of-kind governance framework. Current models (Gemini 2.5 Flash) and frameworks (Google ADK) are used. No prior comparable framework exists to serve as a competitive baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study isolates individual components of the Agent Contract framework. The four experiments test different scenarios but do not systematically remove single components (e.g., resource constraints without success criteria, or budget-aware prompting without formal lifecycle management)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics reported across experiments: token usage, variance, iterations, LLM calls, success rate, timeout rate, reasoning tokens, average time, and quality scores. Code Review alone reports 5 distinct metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The Research Pipeline experiment uses 'multi-judge LLM evaluation' (Section 8.3), not human evaluation. No human evaluation of system outputs is performed in any of the four experiments."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "LiveCodeBench problems 'released post-February 2025, after model cutoff' and OpenR1 problems 'released February 2025, after model cutoff' serve as held-out test sets that the model could not have seen during training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Code Review provides breakdown by difficulty: 'medium-difficulty problems show 92% token savings versus 76% for easy' (Section 8.2). Research Pipeline uses five categories (technology, science, business, health, society). Strategy Modes breaks down by contract mode."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple failure cases discussed: one UNCONTRACTED agent 'failed entirely—stuck in an evaluation loop without submitting output' (Section 8.2); a runaway agent 'exceeded its 40K token budget (56K consumed)' detected and halted (Section 8.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that contracted execution has a 7.1 percentage point lower success rate (52.9% vs 60.0%, p=0.13 NS) — an honest cost of governance. Section 7 discusses fundamental limitations of what contracts cannot guarantee."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: '90% token reduction' (Section 8.2, 34,606 to 3,461), '525x lower variance' (Section 8.2, 5.29B vs 10.1M), 'zero conservation violations' (Section 8.3, 0/50 trials), 'measurable quality-resource tradeoffs' (Section 8.4, 70%->86% across modes)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The Code Review experiment uses a within-subjects design (same 70 problems run in both CONTRACTED and UNCONTRACTED conditions), enabling controlled causal inference about the effect of contracts on resource consumption. The manipulation is the contract treatment (including budget-aware prompting), and the within-subjects design controls for problem difficulty."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'A Formal Framework for Resource-Bounded Autonomous AI Systems' claims general applicability. Empirical results are limited to Gemini 2.5 Flash/Flash-Lite on LiveCodeBench and OpenR1 with Google ADK/LiteLLM. The conclusion states 'formal governance becomes essential... Agent Contracts provide one such foundation' without bounding this to the tested models. Per schema guidance, broad titles when results are on a single model family and two benchmarks is NO."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 discusses enforcement limitations (single-call budget overruns cannot be prevented) and Section 8.3 provides a sensitivity analysis for the outlier. However, the core alternative explanation — that simple budget-aware prompting or iteration limits alone could produce equivalent results without the formal contract framework — is not discussed. The paper does not consider whether the observed token reduction and variance reduction are attributable to the formal framework versus the simpler mechanisms it bundles together."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Gemini 2.5 Flash and Flash-Lite (knowledge cutoff: January 2025)' but provides no API version identifier or snapshot date. Per schema guidance, marketing names like 'Gemini 2.5' without a snapshot date or API version do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes budget-aware prompting and shows 'Budget: {used}/{total}' as an example template, but does not provide the full prompt text used in any experiment. Prompt templates with placeholders without actual fill values do not satisfy the criterion."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM inference hyperparameters (temperature, top-p, max tokens, sampling settings) are reported for any experiment. The paper reports resource constraint parameters (token budgets, iteration limits) but not the model's generation settings."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: Google ADK with DelegatingAdkAgent for multi-agent delegation, Coder-Reviewer pipeline mechanics, tool descriptions (test_code, web_search), budget-aware prompting mechanism, runtime monitoring, conservation law enforcement, and contract lifecycle management (Sections 5, 6, and 8)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper states which benchmarks were used and their general characteristics (difficulty levels, post-cutoff dates) but does not document how specific problems were selected or filtered. The selection of 31 easy and 39 medium LiveCodeBench problems, 50 medium OpenR1 problems, 50 research topics, and 24 crisis scenarios lacks documented filtering criteria."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Fundamental Limitations and Practical Enforcement' is a dedicated limitations section covering single-call enforcement constraints, enforcement capabilities, and future infrastructure requirements."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific threats: token consumption is unknowable during a call (Section 7.1), contracts provide best-effort not hard guarantees, the outlier sensitivity analysis in Section 8.3 examines whether one catastrophic failure drives variance results, and the Strategy Modes experiment acknowledges the reasoning_effort parameter is the direct mechanism."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 states 'contracts cannot prevent a single expensive call from exceeding budget.' Section 7.3 states 'contracts remain most valuable for multi-call and multi-agent scenarios.' These are specific statements about what the framework does NOT provide."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we provide experiment code and data for reproducibility' via the GitHub repository (footnote 3). The benchmarks (LiveCodeBench, OpenR1) are publicly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark sources are identified: LiveCodeBench (arXiv:2403.07974, post-February 2025 problems), OpenR1 (released February 2025), research topics across five categories, and 24 crisis communication scenarios. Sources, temporal windows, and difficulty levels are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use LLM agents on benchmark problems and synthetically generated research topics."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The selection criteria for specific problems within LiveCodeBench (31 easy, 39 medium from how many available?) and OpenR1 (50 medium-difficulty from how many?) are not documented. How research topics were generated and how crisis scenarios were defined is not explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed anywhere in the paper. While both authors are independent researchers with personal email addresses suggesting unfunded work, the experiments required real API costs (Gemini API for ~244 runs) and there is no acknowledgment of how these were funded. No acknowledgments section exists."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are listed as 'Independent Researcher' with personal emails (yeqi519@gmail.com, jtan@live.de). No institutional affiliation with Google (whose Gemini model is used) or other evaluated systems."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. The authors appear to be self-funded independent researchers. NA per schema guidance for unfunded work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears in the paper. The authors do not declare whether they hold patents, equity, or other financial interests related to the Agent Contracts framework. Absence of disclosure counts as NO per the schema."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly states 'Gemini 2.5 Flash and Flash-Lite (knowledge cutoff: January 2025).' This is used to justify benchmark selection."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly addresses contamination by using benchmarks released after the model's training cutoff: LiveCodeBench 'released post-February 2025, after model cutoff' and OpenR1 'released February 2025, after model cutoff.' Temporal separation is the explicit mitigation strategy."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Benchmarks were specifically chosen to be released after the January 2025 model training cutoff (LiveCodeBench post-February 2025, OpenR1 February 2025). This is an explicit and appropriate contamination mitigation strategy."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use LLM agents on benchmark problems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use LLM agents on benchmark problems."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use LLM agents on benchmark problems."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use LLM agents on benchmark problems."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use LLM agents on benchmark problems."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use LLM agents on benchmark problems."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All experiments use LLM agents on benchmark problems."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports tokens consumed per task (34,606 vs 3,461 in Code Review, Section 8.2), wall-clock time per task (6.9s, 12.5s, 16.9s in Strategy Modes, Section 8.4), reasoning tokens (0, 718, 1519), and LLM call counts. The schema lists 'tokens consumed' and 'wall-clock time' as sufficient indicators of inference cost/latency."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper reports per-task metrics but not total API spend, total tokens across all ~244 experimental runs, or hardware specifications."
       }
     }
   }
 }