ai-research-survey

calibration.json (19842B)

---

 {
   "paper_slug": "aart-aiassisted-redteaming-2023",
   "total_questions": 50,
   "agreement_count": 47,
   "disagreement_count": 3,
   "agreement_rate": 0.94,
   "disagreements": [
     {
       "category": "contamination",
       "question": "training_cutoff_stated",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": false,
       "opus_answer": false,
       "opus_justification": "This paper is a red-teaming data generation study. PaLM 2 is used as a generation tool to produce adversarial prompts, not evaluated on a benchmark. The schema description explicitly lists 'red-teaming study that tests defenses rather than model knowledge' as a case where contamination questions are NA. While PaLM 2 may have seen comparison datasets during training, this is an alternative-explanations concern, not a benchmark contamination concern in the schema's intended sense.",
       "sonnet_justification": "The paper uses PaLM 2 (instruction-tuned) as the generation model. No training data cutoff date is stated for PaLM 2. This matters for assessing whether existing red-teaming datasets used for comparison may have been in PaLM 2's training data.",
       "direction": "applies_boundary"
     },
     {
       "category": "contamination",
       "question": "train_test_overlap_discussed",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": false,
       "opus_answer": false,
       "opus_justification": "Same reasoning as training_cutoff_stated: this is a red-teaming data generation study, not a benchmark evaluation of model knowledge. PaLM 2 is the generation instrument, not the subject being evaluated. The comparison is about keyword coverage of generated datasets, not about PaLM 2's performance on a test set. The schema and scan-agent instructions explicitly exclude red-teaming studies from contamination applicability.",
       "sonnet_justification": "The paper compares AART-generated data against existing public datasets (RealToxicityPrompts 2020, ParlAI 2019, BAD 2021, Anthropic 2022) without discussing whether PaLM 2 may have been trained on these datasets, which would confound the quality and novelty comparisons.",
       "direction": "applies_boundary"
     },
     {
       "category": "contamination",
       "question": "benchmark_contamination_addressed",
       "sonnet_applies": true,
       "sonnet_answer": false,
       "opus_applies": false,
       "opus_answer": false,
       "opus_justification": "Same reasoning: the paper does not evaluate PaLM 2 on a benchmark. It uses PaLM 2 to generate adversarial prompts and compares the generated dataset's keyword coverage against other datasets. This is structurally different from a benchmark evaluation where contamination matters. The concern that PaLM 2 trained on comparison datasets is valid but is better captured under alternative_explanations_discussed (where both Sonnet and Opus agree the paper fails) rather than contamination.",
       "sonnet_justification": "The comparison datasets (especially RealToxicityPrompts published 2020, ParlAI 2019) predate PaLM 2's likely training cutoff. The paper does not address whether PaLM 2 was trained on these datasets, which could influence the model's generation patterns.",
       "direction": "applies_boundary"
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository is provided. The paper mentions a GitHub link for a demonstration dataset but no code implementing the AART pipeline is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'we intend to make available a demonstration dataset' (future tense) at a GitHub URL. This is a promise of future release, not an actual release. As of publication, the dataset is not available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'PaLM API' (Vertex LLM PaLM 2 API) but provides no library versions, requirements file, Docker configuration, or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Appendix A shows the prompts used and Appendix B shows dataset filters, but there is no README or instructions that would allow end-to-end reproduction without significant guesswork."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports keyword coverage rates as single point estimates (e.g., 0.384 for Policy Concepts in AART). Query length is reported with mean and standard deviation, but the main coverage metrics have no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares AART keyword coverage rates against five other datasets (Table 2) and claims 'promising results,' but no statistical significance tests are applied to any of these comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Coverage ratios are compared numerically (e.g., AART's 0.384 vs. Perez adaptation's 0.210) but no standardized effect sizes (Cohen's d, odds ratios, etc.) are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The qualitative analysis uses n=120 prompts (Section 4) with no justification for why this sample size was chosen or whether it is sufficient to support the percentages reported."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Keyword coverage rates in Table 2 are single-run numbers with no variance. The qualitative analysis percentages are from a single pass with no inter-rater reliability or spread measures. Query length std dev is reported but this is not the main result."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares AART against four human red-teaming datasets (RealToxicityPrompts, ParlAI Dialogue Safety, BAD, Anthropic) and one automated method (Perez et al. 2022 adapted to the same scenario)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Perez et al. (2022) and the Anthropic red-teaming dataset (2022), which are the most relevant contemporary comparators for automated red-teaming of LLMs."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper presents the full AART pipeline without isolating individual components' contributions (e.g., effect of structured CoT, geographic dimensions, or the multi-step pipeline vs. a simpler approach)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses quantitative keyword-match metrics across three dimensions (Table 2: Policy Concepts, Task Formats, Geographic Regions) and qualitative human review of n=120 prompts assessing quality, concept reference, and format reference."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes a qualitative analysis on n=120 sampled prompts assessing quality (92.5%), policy concept reference (99.2%), geographic region reference (96.7%), and task format reference (85.0%). This constitutes human review of the system's outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "AART is a data generation pipeline, not a trained model. The evaluation compares generated datasets using keyword coverage metrics; a held-out test set is structurally inapplicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 breaks down keyword coverage across three separate dimensions: Policy Concepts, Task Formats, and Geographic Regions. The qualitative analysis also reports separate percentages for each dimension."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses over-representation of top-5 regions (53.4%), imbalanced task format coverage, and missing formats. Appendix C.1 provides a dedicated list of worst-case examples from the qualitative analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Task Format is described as 'an area of relative weakness' (85.0%), geographic region imbalances are reported, several task formats had zero representation, and the paper acknowledges keyword-based evaluation underestimates actual coverage."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AART 'shows promising results in terms of concept coverage and data quality.' Table 2 shows higher keyword coverage than baselines, and qualitative analysis finds 92.5% quality. The hedged language ('promising results,' 'compared to some state-of-the-art tools') is appropriate for the limited evidence."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims that recipe-guided structured generation causes higher diversity than ad-hoc methods. These are supported only by descriptive comparisons across datasets of different sizes, origins, and purposes, without controlling for any confounds."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 states 'AART enabled us to launch several products with improved safety measures,' generalizing from a single English-language demonstration scenario to broad product safety claims. No results are shown for other applications or languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss that higher coverage of AART-defined keywords in AART-generated data is expected by construction, since AART's prompts are parameterized by those very concepts. The comparison datasets were designed for different contexts with different keyword vocabularies."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'instruction tuned LLM (PALM API)' and cites the PaLM 2 technical report, but does not specify an exact model version, size, or snapshot date. PaLM 2 encompasses multiple model sizes (Gecko, Otter, Bison, Unicorn)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the full prompt text for all three pipeline steps: Step 1 prompts for generating policy concepts, task formats, and geographic regions; the complete Step 3 structured generation prompt with all placeholders; and Appendix A.2 provides the actual keyword lists used as fill values."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.7 is stated only for the Perez et al. adaptations (Appendix B). No hyperparameters (temperature, top-p, max tokens) are reported for AART's own generation steps."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The AART pipeline (Steps 1-4) is described in detail in Section 3.1 with Figure 2 providing an overview diagram. The structured generation process, how dimensions are combined, and the CoT-style instruction creation are documented clearly enough for re-implementation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B documents how comparison datasets were filtered (e.g., RealToxicityPrompts: 'challenging=True or toxicity >= 0.8'; ParlAI: 'first exchanges, adversarial, labeled not ok'). JSON parsing errors are reported (144 lines discarded, 95.8% valid)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations' and provides substantive discussion of long-tail coverage gaps, LLM generation biases, ambiguity in defining 'adversarial,' low keyword-based coverage rates, and computational expense."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 identifies specific threats: keyword-based evaluation underestimates concept presence; LLM-generated data may exhibit systematic biases; the method may miss emerging attack patterns (prompt injection, jailbreaking); the qualitative analysis is limited to n=120."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section does not explicitly state what the results do NOT generalize to. The paper does not bound claims to the specific English-language, text-generation, dangerous-activities demonstration scenario. The conclusion broadly claims real-world product launches."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The generated adversarial dataset is promised as a future release but not available at publication time. Independent verification of the reported quality metrics (92.5%, etc.) is not possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation procedure is described in detail across Section 3 and Appendix A: specific prompts, parameter choices, number of runs per policy concept (twice), sampling strategy for task formats and geographic regions, and JSON parsing step."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses automated LLM generation and existing public datasets with no external human participant recruitment. The qualitative analysis was conducted by the paper's authors."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from Step 1 (concept generation) through Step 2 (scoping/filtering) to Step 3 (structured generation) to Step 4 (review) is documented. JSON parsing step with 144 discarded lines is reported. Comparison dataset filtering is documented in Appendix B."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The acknowledgments section thanks individual colleagues at Google but includes no formal funding statement or grant acknowledgment."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as 'Google Research' on the title page. The affiliation is clearly disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The authors are Google Research employees using Google's PaLM 2 API. Google has a financial interest in demonstrating effective AI safety tools and promoting responsible use of its LLMs. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a red-teaming data generation study. PaLM 2 is used as a generation tool to produce adversarial prompts, not evaluated on a benchmark for its knowledge or capabilities. The schema and scan-agent instructions explicitly list red-teaming studies as cases where contamination questions are NA."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same reasoning: the paper does not evaluate a pre-trained model on a benchmark. PaLM 2 is the generation instrument for producing adversarial prompts, not the subject being evaluated on test data."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate PaLM 2 on any benchmark. It uses PaLM 2 to generate data and then compares that data's keyword coverage against other datasets. Contamination in the traditional sense is structurally inapplicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No external human participants. The qualitative analysis (n=120) was conducted by the paper's authors as an internal quality assessment."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No external human participants were involved. The qualitative analysis was conducted by the authors themselves."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No external human participants. The qualitative analysis was conducted by the authors."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No external human participants were recruited."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study with randomization. The qualitative analysis uses a convenience sample of 20 prompts per dimension category."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring blinding. The authors knew which method generated each prompt."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 notes 'the computational expense of using LLMs is high' but provides no quantitative figures: no API call counts, token counts, dollar costs, or latency measurements for generating the 3,269-example demonstration dataset."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget is stated. Total API calls, generation time, or cloud compute costs are not reported anywhere in the paper."
       }
     }
   }
 }