ai-research-survey

calibration.json (18351B)

---

 {
   "paper_slug": "agentic-ai-assessment-framework-2025",
   "calibration_date": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 47,
   "disagreement_count": 3,
   "agreement_rate": 0.94,
   "disagreements": [
     {
       "category": "statistical_methodology",
       "question": "significance_tests",
       "sonnet": { "applies": false, "answer": false },
       "opus": { "applies": true, "answer": false },
       "direction": "applies_boundary",
       "note": "Sonnet says no comparative claims requiring significance tests. Opus judges RQ1 ('How effective is the proposed framework compared to baseline evaluations?') as a comparative claim — the paper compares two evaluation approaches on the same data and makes effectiveness claims without statistical tests. The comparison is about measurement scope rather than quantitative differences, but the framing uses comparative language."
     },
     {
       "category": "evaluation_design",
       "question": "ablation_study",
       "sonnet": { "applies": true, "answer": true },
       "opus": { "applies": true, "answer": false },
       "direction": "sonnet_generous",
       "note": "Sonnet credits Table 7 ('Pillar ablation: Average failures per pillar') as an ablation study. Opus disagrees: the table shows a per-category breakdown of failures by pillar, not a proper ablation where components are removed to measure their individual contribution. No pillar is removed to show the framework degrades without it. The paper labels it 'ablation' but the methodology is a breakdown, not an ablation."
     },
     {
       "category": "evaluation_design",
       "question": "human_evaluation",
       "sonnet": { "applies": false, "answer": false },
       "opus": { "applies": true, "answer": false },
       "direction": "applies_boundary",
       "note": "Sonnet sets applies=false, reasoning that the paper uses only automated LLM evaluation. Opus sets applies=true because human evaluation of the framework's outputs (e.g., expert review of whether identified failures are genuine) would be relevant and valuable — the schema says applies=false only when 'human evaluation is clearly irrelevant to the claims.' For a framework claiming to detect behavioral failures, human validation of those detected failures is relevant."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub URL (https://github.com/sa4s-serc/asf) provided in Section 1 and reiterated in Section 4.2: 'Our experiments and results are available on GitHub.'"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Synthetic CloudOps data (instances, logs, policies) was generated using Sonnet 4.5 but no dataset download link or standalone release is mentioned. The GitHub repo contains experiment trajectories, not the underlying synthetic dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Paper mentions GPT-4o with temperature 0.7, ChromaDB, text-embedding-3-small, and Mem0 with 'default settings', but no requirements.txt, Dockerfile, or library version listing is provided. Insufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "A GitHub link is provided but the paper itself contains no step-by-step reproduction instructions, commands, or 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 3-7 report only point estimates (e.g., '33.7% precision', '37.9% recall'). Figure 3 shows visual distributions across three runs but no CIs or numerical error bars are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "RQ1 frames a comparison between baseline and framework evaluation approaches, but no statistical significance tests are used to support the claim that the framework is more effective. The comparison relies on showing the framework reveals additional metrics, without formal hypothesis testing."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper compares measurement scope (baseline metrics vs. framework metrics) rather than two competing methods on the same metrics. There is no meaningful effect size to compute between approaches that measure different things."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only three scenarios (S1, S2, S3), each run three times. No justification for why three scenarios are sufficient and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each scenario was run three times and 'averaged to ensure reliability,' but tables report only averages with no standard deviation or range. Figure 3 shows visual distributions but no numerical variance statistics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares a 'Baseline (B)' evaluation (task completion and tool usage ratio only) against the proposed 'Framework (F)' metrics across all three scenarios."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is the current dominant practice of task-completion and tool-usage-ratio metrics. This is the appropriate comparison for the paper's goal of showing what standard metrics miss."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Table 7 is labeled 'Pillar ablation' but shows a per-category breakdown of failure counts across pillars, not a proper ablation study. No pillar is removed from the framework to measure its contribution — the table simply counts how many failures each category detects. A true ablation would remove a pillar and show the framework's effectiveness degrades."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The framework uses many metrics: task completion, tool usage ratio, tool sequence correctness, expected calls, policy adherence, dependency inquiry, memory P/R/F1, and judge scores across completion/safety/memory/reasoning/overall dimensions."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated (LLM-as-Judge and Agent-as-Judge). No human experts validate whether the framework's detected failures are genuine. For a framework claiming to identify behavioral failures, human validation of those findings would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The three CloudOps scenarios are used directly for framework validation with no separate held-out test set or train/test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down per pillar (Table 7), per scenario (Tables 3-6), and memory retrieval by type (Table 4: single-hop, multi-hop, temporal)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 and Table 7 describe concrete failures: 'Skipped policy validation before instance termination', 'Missed diagnostic or verification steps before applying remediation', 'Did not recall previous role mappings'. Section 5 discusses failure patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Substantial negative results reported: S1 had 0% task completion and 33% policy adherence, S2 had 13.1% memory recall, S3 had 7.67 average tool failures. The paper transparently presents these to demonstrate baseline evaluation gaps."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the framework 'reveal[s] behavioral deviations overlooked by conventional metrics.' Tables 3-7 support this: S2 shows 100% task completion masking 13.1% memory recall and 33% tool sequence correctness."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims (e.g., 'tool sequencing and memory management need particular attention in multi-agent scenarios') derived from observational differences across three scenarios of different complexity. No controlled single-variable manipulation isolates the causes of observed failures."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'An Assessment Framework for Evaluating Agentic AI Systems' implies broad applicability, but validation is only on CloudOps scenarios with a single LLM (GPT-4o). Section 5.1 acknowledges the CloudOps limitation but the title and framing claim generality beyond what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 identifies some threats (uniform temperature, single model, ground truth contracts, judge biases) but these are methodological limitations, not specific alternative explanations for observed results. No consideration of whether failures stem from synthetic data design, prompt engineering choices, or temperature settings rather than genuine system weaknesses."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o' without a version or snapshot date, 'Sonnet 4.5' for data generation, and 'text-embedding-3-small' for embeddings. Per schema criteria, marketing names without API snapshot dates do not count as specified versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided anywhere in the paper or appendix. The LLM-as-Judge uses 'structured prompts' (Section 3.3) but the prompt text is not shown. Agent system prompts are not disclosed."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature (0.7) is reported for GPT-4o and Mem0 uses 'default settings,' but top-p, max tokens, and other API settings are not specified. The default settings of Mem0 are not enumerated. Partial reporting is insufficient."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed description of MOYA framework architecture, memory using Mem0 with ChromaDB, tool invocation protocols, test case generation, static/dynamic/judge evaluation layers, Agent Card mechanism, and Agent-as-Judge auditor. Figures 1 and 2 provide architectural diagrams."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Synthetic data was 'generated using Sonnet 4.5' based on production issues at MontyCloud, but the generation process, prompts used, volume, filtering criteria, and verification procedures are not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 'Threats to Validity' addresses external, internal, and construct validity with substantive multi-paragraph discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 identifies study-specific threats: evaluation limited to CloudOps domain (external), uniform temperature 0.7 and single model GPT-4o (internal), metrics may not capture all reliability aspects and memory assessment excludes storage efficiency (construct)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 notes CloudOps limitation but does not explicitly enumerate what the results do NOT show or what claims the authors are NOT making. There is no equivalent of METR's 'What the evidence does not show' — the paper states its limitations but not its explicit non-claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "GitHub repo contains 'experiments and results' (execution trajectories) but the underlying synthetic CloudOps data (instances, logs, policies) is not described as publicly available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Synthetic data generated 'using Sonnet 4.5 due to its ranking on LMArena' based on 'production issues we encountered,' but prompts, criteria, volume, and verification procedures for the synthetic data are not described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses synthetic CloudOps scenarios, not data collected from human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from synthetic data generation through evaluation metrics is not fully documented. The paper describes framework components but does not trace the full data pipeline with intermediate steps, data volumes, and filtering criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present in the paper. The industry collaboration with MontyCloud is described but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed on the title page: four authors at IIIT-Hyderabad SERC, two at MontyCloud Inc. The industry affiliation is prominently disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two MontyCloud Inc. employees are co-authors and the framework is validated exclusively on MontyCloud's production system (MOYA). MontyCloud has a financial interest in the framework appearing effective."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present. The MontyCloud affiliation creates a potential undisclosed conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4o is used as the agent backbone but no training data cutoff date is stated. The paper does not address whether the model's training data could overlap with the evaluation scenarios."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4o's training data overlaps with the CloudOps scenarios, AWS documentation patterns, or tool descriptions used in evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluation uses custom-designed CloudOps workflows, not published public benchmarks. Standard benchmark contamination does not apply to custom scenarios."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The evaluation is entirely automated using LLM agents and synthetic CloudOps scenarios."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; inclusion/exclusion criteria are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or controlled experiment with human assignment; not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or human evaluators; blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 reports inference costs: average $0.0621 per scenario run, LLM-as-Judge $0.0593 total across all scenarios, Agent-as-Judge $0.9572 total. Figure 4 shows cost breakdowns."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 reports average execution time (183.5s per run), token consumption (19,644 input / 1,301 output per run), and per-scenario costs ($0.0405-$0.0818)."
       }
     }
   }
 }