ai-research-survey

calibration.json (15102B)

---

 {
   "paper_slug": "agentic-ai-security-survey-2025",
   "calibration_model": "opus",
   "scan_model": "sonnet",
   "timestamp": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "category": "conflicts_of_interest",
       "question": "funder_independent_of_outcome",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "note": "Sonnet treats no funding disclosure as equivalent to 'unfunded' (applies=false/NA). Opus treats it as applies=true, answer=false because the schema says 'NA if unfunded' but the paper does not confirm it is unfunded—university researchers typically have some form of funding. The absence of a funding disclosure does not confirm absence of funding. Strict reading: if we cannot confirm the work is unfunded, we cannot claim NA."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or Zenodo archive is mentioned anywhere in the paper. The survey releases no analysis scripts or supporting code."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset of reviewed papers is released. The bibliography is cited inline but no structured, machine-readable corpus is provided for download."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tooling specification is provided. No mention of reference management software, analysis tools, or any computational environment used to conduct the review."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are included. The paper does not describe how a reader could replicate the literature search, paper selection, or taxonomy construction process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a narrative survey with no quantitative meta-analysis conducted by the authors. Statistical uncertainty measures are structurally inapplicable."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The survey authors make no comparative statistical claims of their own. NA for a narrative survey."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No effect sizes are computed by the survey authors. Statistics cited (e.g., '94.4% of agents vulnerable') are from cited works, not the authors' own analysis."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted by the survey authors, so sample size justification is structurally inapplicable."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted by the survey authors. NA for a narrative survey."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not systematically compare its taxonomy or coverage against prior surveys. Section 2 briefly mentions existing surveys [47-52] and claims this work is 'distinct' but provides no structured comparison of coverage, methodology, or scope."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baseline comparison is conducted, so assessing whether baselines are contemporary is structurally inapplicable. There are no baselines to evaluate."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate is proposed. This is a survey paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation is performed by the survey itself that would require multiple metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs are generated by this survey that would warrant human evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No empirical evaluation with train/test splits is conducted."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed per-category breakdowns: Figure 2 and Section 3 organize threats into five major categories with subcategories; Table 1 provides defense coverage across seven dimensions; Table 2 organizes benchmarks by capability vs. security-specific with detailed attribute columns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The survey discusses failure modes of defense approaches throughout. Section 4.1.1 notes fine-tuning 'can degrade the general-purpose capabilities of LLMs without providing significant defensive capabilities against adaptive attacks.' Section 4.1.4 notes training-free defenses 'remain fragile against adaptive attacks.' Section 6 discusses open challenges representing current failure areas."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The survey consistently reports negative results from cited work: defensive fine-tuning degrading general capabilities (Section 4.1.1, citing [179]), prompt augmentation being bypassed by adaptive attacks (Section 4.1.3), sandboxing reducing collaborative efficiency (Section 4.3), and agents' poor CAPTCHA performance (Section 3.4.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the paper 'outlines a taxonomy of threats specific to agentic AI, reviews recent benchmarks and evaluation methodologies, and discusses defense strategies.' All three are delivered in Sections 3, 5, and 4 respectively. No overclaiming is present."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The survey makes no causal claims of its own. It summarizes causal findings from cited works with appropriate attribution (e.g., 'Lupinacci et al. demonstrated...', 'Fang et al. demonstrate...')."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The survey appropriately attributes claims to specific systems, models, or studies (e.g., 'GPT-4 achieving 87% success rate'). Section 6.5 explicitly states 'this survey primarily focuses on software-based agents' and notes physically embodied agents as underexplored."
       },
       "alternative_explanations_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a pure survey/taxonomy paper that presents no original empirical results. The schema states NA for 'papers that present no empirical results (e.g., pure surveys or taxonomies).'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The survey does not run any LLM experiments itself. When discussing cited work, it uses model names as reported in those papers, which is appropriate for a survey."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The survey does not use prompting in its own methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted by the survey authors. Hyperparameters are structurally inapplicable."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used by the survey itself."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No systematic search methodology is described. The paper does not explain how papers were identified for inclusion, what databases were searched, what search terms were used, or what inclusion/exclusion criteria were applied. There is no PRISMA diagram or paper selection pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 6 ('Open Challenges') discusses gaps in the field of agentic AI security, but does not discuss limitations of the survey itself (e.g., potential coverage gaps, selection bias, recency bias)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to the validity of the survey's own methodology are discussed. The paper does not address whether it missed relevant papers, whether its taxonomy is complete, or whether its selection of cited papers is representative."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 explicitly states 'this survey primarily focuses on software-based agents' and notes that 'physically embodied agents introduce additional and largely unexplored security risks.' The introduction also distinguishes the security focus from capability surveys."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No structured dataset of reviewed papers is released. The bibliography provides references but not a machine-readable corpus with coded attributes, making independent verification of coverage claims impossible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "No description of how papers were collected. The survey does not specify search databases, date ranges, search queries, or any systematic selection process. Papers appear selected based on the authors' judgment without a documented protocol."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey. This is a literature review, not a human subjects study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "There is no documented pipeline from paper collection to final taxonomy. The survey presents a taxonomy and cites supporting papers, but the process by which papers were screened, included, or excluded is entirely opaque."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section is present. No funding sources are disclosed anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed on the title page: 'Bellini College of AI, Cybersecurity, and Computing, University of South Florida.' Contact email addresses are also provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so funder independence cannot be assessed. The schema says 'NA if unfunded,' but the paper does not confirm it is unfunded—university researchers typically receive some form of funding support. The absence of a funding disclosure is not the same as confirmation of being unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial disclosure statement in the paper. Absence of declaration is not the same as absence of conflict. The corresponding author (Chhabra) cites two of his own works [145, 146]."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model's capabilities on benchmarks. Contamination questions are structurally inapplicable."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is conducted by the survey itself. NA for a survey paper."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is conducted. NA for a survey paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a literature survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not propose or evaluate a system with inference costs."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper requiring no computational budget for its own methodology."
       }
     }
   }
 }