ai-research-survey

calibration.json (16297B)

---

 {
   "paper_slug": "advevomarl-shaping-internalized-2025",
   "calibration_date": "2026-02-28",
   "calibration_model": "opus",
   "total_questions": 50,
   "agreement_count": 50,
   "disagreement_count": 0,
   "agreement_rate": 1.0,
   "disagreements": [],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or any mention of code release anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses publicly available benchmarks: MATH-500, JailbreakBench, Wild Jailbreak, Strong Reject, AIME, LiveCodeBench, and GPQA-diamond. The custom Dadv training set (~4,000 samples) is not released, but the evaluation datasets are all standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda file, or detailed environment specification. The paper only mentions using QWen2.5 instruction-tuned models (3B and 7B) without library versions or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided. The method is described conceptually but no actionable instructions for replication exist."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 and Figures 2-5 are point estimates. No confidence intervals, error bars, or ± notation are reported anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims such as 'consistently achieves the lowest ASR' and '12% reduction' but no statistical significance tests (p-values, t-tests, bootstrap tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect size measures (Cohen's d, odds ratios) are reported. While percentage differences are mentioned in text (e.g., '12% reduction in ASR'), these are not systematically computed with baseline context across all comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of 4,000 training samples, 1,000 harmful behaviors, 300 adversarial prompts, or benchmark sizes. No power analysis or reasoning for chosen N values."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run numbers. No standard deviation, variance across seeds, interquartile range, or multiple-run spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Vanilla QWen2.5 (3B and 7B), Challenger (self-verification), Inspector (external guard agent), and closed-source GPT-3.5 and GPT-4o-mini as reference models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are contemporary: Challenger from Huang et al. 2025, Inspector approach, QWen2.5 models, and GPT-4o-mini — all from 2024-2025, appropriate for this 2025 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 ablates dynamic vs. static attacker and individual vs. joint (MAS) defender training. Section 5.4 ablates the public baseline mechanism against a no-baseline variant with training dynamics shown in Figure 5."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: Attack Success Rate (ASR), Contagion Rate (CR), and task performance (accuracy for math/reasoning, Pass@1 for coding), measuring both safety and utility."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates automated safety metrics (ASR, CR) and task accuracy via automated benchmarks. Human evaluation of system outputs is not clearly relevant to the claims about attack success rates and task accuracy."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses MATH-500 for defenders and custom adversarial pool for attackers. Evaluation uses separate benchmarks: AIME, GPQA-diamond, LiveCodeBench for task evaluation; JailbreakBench, Wild Jailbreak, Strong Reject, and NetSafe official dataset for safety evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides detailed breakdown by attack scenario (NetSafe, AutoInject, UserHijack) and by system topology (chain, tree, complete), giving 21 distinct evaluation conditions per model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges that CR remains elevated (up to ~35%) in complete graph topology even for AdvEvo-MARL, the 3B variant shows a maximum 3% accuracy drop, and Challenger-7B sometimes performs worse than vanilla (38.33% ASR vs. 22.29% for vanilla in UserHijack complete)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 reports that the no-baseline variant exhibits 'non-stationary behavior and even degraded performance in later stages' with a 13.3% drop in defender response length. The 3B model's accuracy drop of up to 3% is also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'ASR below 20%' — Table 1 shows maximum ASR of 17.68% for AdvEvo-MARL-3B in complete topology. 'baselines reach up to 38.33%' — Table 1 confirms Challenger-7B at 38.33% in UserHijack complete. '+3.67% on reasoning tasks' — approximately consistent with Figure 2, though this exact number is not reported in a table."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made via ablation studies with single-variable manipulation: dynamic vs. static attacker (Section 5.3), individual vs. joint training (Section 5.3), and with vs. without public baseline (Section 5.4). These controlled comparisons adequately support the causal attributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion describes AdvEvo-MARL as 'a promising and unified framework for building safe and capable multi-agent systems' without bounding to the tested conditions: QWen2.5 3B/7B only, three specific attack types, three topologies, and one model family. The title and abstract also make broad claims not bounded to the experimental scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example, the 7B model's improved task performance after safety training could be due to general RL training benefits rather than safety-specific mechanisms. The paper does not consider confounds or alternative interpretations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'QWen2.5 instruction-tuned models (3B and 7B)' without specifying exact checkpoint names (e.g., 'Qwen2.5-7B-Instruct' with a version hash). GPT-3.5 and GPT-4o-mini comparison models also lack snapshot dates or API version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. System prompts for defenders, adversarial prompts for attackers, and evaluation prompts for the LLM-as-judge are all described only in natural language without the actual text used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Reward weights are partially reported (alpha_s=1, beta_t=0.5 in first half, reversed in second; format rewards 0.5/-0.1; safety/task rewards 1/-1). However, critical RL hyperparameters are missing: learning rate, batch size, number of training steps/epochs, clipping parameter epsilon, KL coefficient beta, and sampling temperature."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent scaffolding is described in detail in Section 4 and Figure 1: three agent topologies (chain, tree, complete), attacker warm-up via SFT, adversarial co-evolutionary RL loop with REINFORCE++, public baseline for advantage estimation, and separate reward mechanisms for attackers vs. defenders."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 documents the preprocessing pipeline: sample 1,000 harmful behaviors from public datasets → apply jailbreak strategies to get Dinit → use reasoning model to synthesize multi-step traces → filter with LLM-as-judge removing contradictory, off-topic, or vague trajectories → resulting in ~4,000 samples in Dadv."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Ethics Statement (Section 7) discusses social implications but not methodological limitations. The conclusion contains no limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. Issues such as small model sizes (3B/7B only), limited attack types, single-run results, potential overfitting to specific attack strategies, and single model family evaluation are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The conclusion broadly claims 'promising and unified framework for building safe and capable multi-agent systems' without bounding to the tested conditions (specific models, attack types, topologies)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data, model outputs, evaluation logs, or detailed results in downloadable format are provided. Only aggregated metrics in tables and figures are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes training data collection: sampling 1,000 harmful behaviors from public datasets, applying jailbreak strategies, synthesizing reasoning traces, and filtering. Section 5.1 describes evaluation datasets sourced from JailbreakBench, Wild Jailbreak, Strong Reject, and standard task benchmarks."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All data comes from automated benchmarks and synthetic adversarial prompt construction."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline from seed prompts to final training data is documented with counts: 1,000 harmful behaviors → jailbreak strategy application → reasoning trace synthesis → LLM-as-judge filtering → ~4,000 samples. Each step and transition is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section, no funding disclosure, no mention of grants or corporate sponsors anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the first page: Northwestern University, University of Illinois at Chicago, University of Rochester, and Carnegie Mellon University."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so funder independence cannot be assessed. Without funding information, this criterion is not evaluable."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosure, or declaration of financial interests appears anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "QWen2.5 instruction-tuned models are used but their training data cutoff dates are not stated. GPT-3.5 and GPT-4o-mini comparison models also lack training cutoff information."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "AIME, GPQA-diamond, and LiveCodeBench are used for evaluation but no discussion of whether QWen2.5's pre-training data may have included these benchmarks or their solutions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LiveCodeBench is cited as 'contamination free' (Jain et al., 2024), but GPQA-diamond and AIME are public benchmarks whose solutions could appear in QWen2.5's training data, and this contamination risk is not addressed for these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The Ethics Statement (Section 7) discusses social implications of the research, not IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes a method requiring multi-agent RL training of LLMs but reports no inference cost, API cost, tokens consumed, or wall-clock time for training or evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, total API spend, or training time are stated anywhere in the paper."
       }
     }
   }
 }