ai-research-survey

calibration.json (15934B)

---

 {
   "paper_slug": "adversarial-threat-vectors-2025",
   "calibration_date": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 49,
   "disagreement_count": 1,
   "agreement_rate": 0.98,
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "multiple_metrics",
       "sonnet": { "applies": true, "answer": true },
       "opus": { "applies": false, "answer": false },
       "direction": "applies_boundary",
       "explanation": "Sonnet treated the OWASP-derived risk scoring factors (Likelihood Factor, Impact Factor, etc.) as 'multiple evaluation metrics.' Opus disagrees: the schema asks whether multiple evaluation metrics are used to evaluate a system's performance. This paper does not evaluate any system — it applies a qualitative risk-scoring model to a hypothetical RAG architecture. The OWASP factors are dimensions of a risk assessment framework, not evaluation metrics for measuring system output quality. The criterion is structurally inapplicable to a framework/position paper that runs no experiments."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or archive is provided anywhere in the paper. The paper presents a qualitative threat modeling framework with no accompanying software artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No datasets were collected or released. The paper conducts qualitative analysis referencing existing frameworks (MITRE ATLAS, OWASP) rather than any original data collection."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a theoretical/analytical framework paper with no computational experiments, so environment specification is not applicable."
       },
       "reproduction_instructions": {
         "applies": false,
         "answer": false,
         "justification": "No experiments were conducted that would require reproduction instructions. The paper presents a threat modeling methodology applied to a hypothetical system."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents qualitative risk scores (likelihood x impact) derived from OWASP-style factor analysis, not statistical experiments. No confidence intervals are applicable."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "No statistical comparisons between conditions are made. The risk reduction figures are deterministic scoring calculations based on expert judgment, not empirical measurements."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses a qualitative risk-scoring model (inherent vs. residual risk) rather than empirical experiments that would yield effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No sample of participants, examples, or cases is drawn. The analysis is applied to a generic hypothetical RAG system architecture."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs were conducted. The risk scores are deterministic expert assessments, not repeated measurements."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "The paper proposes a threat modeling framework and control prioritization scheme rather than evaluating a system empirically; there are no baselines to compare against."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No baseline comparison is conducted; this criterion does not apply to a framework/position paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No implemented system is evaluated. The paper applies qualitative risk analysis to a generic RAG architecture, so component ablation is not applicable."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "The schema asks whether multiple evaluation metrics are used to evaluate a system's performance. This paper does not evaluate any system — it applies a qualitative risk scoring model (OWASP factors) to a hypothetical RAG architecture. The OWASP factors (Likelihood, Impact, Ease of Exploit, etc.) are dimensions of a risk assessment framework, not evaluation metrics for measuring system output quality. This criterion is structurally inapplicable to a framework paper with no experiments."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "There are no AI system outputs being evaluated; the paper is a framework paper with qualitative risk analysis, making human evaluation of system outputs irrelevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No machine learning evaluation is conducted. The paper does not train or test any model on held-out data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides breakdowns by threat model (Sensitive Information Disclosure in Section 3.3.1 vs. RAG System Poisoning in Section 3.3.2) and by attack stage, with separate risk scoring figures for each (Figures 7-10)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses residual risks that remain after applying controls, including 'sophisticated insider threats or advanced supply chain compromises' that 'will continue to challenge even well-defended RAG architectures.'"
       },
       "negative_results_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not run experiments, so there are no negative empirical results to report. The discussion of residual risk is conceptual, not experimental."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to identify prominent attack vectors and propose a prioritized control list. The paper delivers on both: Section 3.3 identifies prompt injection, data poisoning, and adversarial query manipulation; Table 3 provides the prioritized control list. Claims are descriptive and matched by content."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims about control effectiveness, e.g., 'Adversarial Training... mitigate risks by increasing the Skill Level required' and reports numerical risk reductions (Overall Risk Severity from 19.5 to 10.41). These causal claims rely entirely on expert judgment applied to an OWASP scoring formula with no empirical validation — no red-team tests, measured attack success rates, or before/after comparisons support the specific numerical reductions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames conclusions broadly ('organizations can better protect the integrity and reliability of their systems') without restricting claims to specific RAG architectures, deployment contexts, or adversary capability models. The generic RAG system is hypothetical, and results are presented as generally applicable."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why the proposed controls would or would not work, alternative threat prioritizations, or reasons the risk reduction estimates might be inaccurate. Section 4 acknowledges residual risks but does not consider alternative threat models or limitations of the scoring methodology."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No specific LLM is used or evaluated in experiments. The paper analyzes a generic, hypothetical RAG architecture."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompts are used in experiments. The paper does not conduct any LLM prompting as part of its methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models are trained or queried. This is a framework paper with no computational experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper proposes a threat modeling methodology for RAG systems but does not itself implement or evaluate an agentic scaffold."
       },
       "data_preprocessing_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data collection or preprocessing is performed. The risk scoring is based on expert judgment applied to a hypothetical system, not on processed datasets."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 4 (Discussion and Future Work) briefly notes residual risks in a single paragraph, but this does not constitute substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to the validity of the threat modeling analysis or risk reduction calculations are discussed. The paper does not address whether the OWASP scoring factors are calibrated for LLM systems, whether the Pyramid of Pain framework transfers to AI, or whether the residual risk numbers are meaningful."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the analysis does NOT cover. It analyzes a 'generic RAG system' for enterprise knowledge management but does not delineate which deployment contexts, adversary capabilities, or RAG variants are excluded from its conclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "No empirical data was collected. The risk scores are constructed from expert judgment applied to OWASP scoring factors; there is no raw dataset to verify."
       },
       "data_collection_described": {
         "applies": false,
         "answer": false,
         "justification": "No primary data was collected. The analysis applies existing frameworks (MITRE ATLAS, OWASP Top 10 for LLMs) to a generic hypothetical RAG architecture."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No participants or samples were recruited. The paper is a framework analysis with no human subjects or empirical data collection."
       },
       "data_pipeline_documented": {
         "applies": false,
         "answer": false,
         "justification": "No data pipeline exists. The methodology is a five-stage qualitative threat modeling process applied to a hypothetical system, not a data processing pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "The acknowledgments section thanks Dr. Mike Tan for conversations but discloses no funding source or grant. The authors are from Fire Mountain Labs (a private company) but no sponsoring entity is stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: both are from Fire Mountain Labs, San Diego, CA. Reference [9] reveals the AI Security Pyramid of Pain framework is their own prior work (Ward, Harguess et al., 2024)."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding source is disclosed. The schema says NA if unfunded. The paper appears to be self-funded or unfunded by Fire Mountain Labs."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or financial disclosure. The authors promote their own prior framework (AI Security Pyramid of Pain) as the central organizing structure without disclosing any commercial interests in its adoption."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model's benchmark performance. It proposes a threat modeling framework for generic RAG systems."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation of a pre-trained model is conducted. Contamination is not applicable to this framework paper."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark is used to evaluate a pre-trained model's knowledge. The paper is a threat modeling framework, not a capability evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The paper presents a qualitative threat modeling framework."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or evaluators are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a framework/position paper proposing a threat modeling methodology, not an empirical system. No inference costs are incurred by the paper's own method."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "No computational experiments were run. The methodology is expert-judgment-based qualitative analysis requiring no compute budget."
       }
     }
   }
 }