ai-research-survey

scan.json (20900B)

---

 {
   "paper": {
     "title": "AART: AI-Assisted Red-Teaming with Diverse Data Generation for New LLM-powered Applications",
     "authors": [
       "Bhaktipriya Radharapu",
       "Kevin Robinson",
       "Lora Aroyo",
       "Preethi Lahoti"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2311.08592"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": [
     "case-study",
     "qualitative"
   ],
   "key_findings": "AART proposes a structured pipeline for AI-assisted generation of adversarial evaluation datasets using LLM-generated recipes across policy concepts, task formats, and geographic regions. Compared to existing human red-teaming datasets and an adaptation of Perez et al. (2022), AART achieves higher keyword coverage for policy concepts (0.384 vs 0.210), task formats (0.148 vs 0.009), and geographic regions (0.410 vs 0.000). Qualitative analysis of 120 samples found 92.5% were of good quality for adversarial testing.",
   "claims": [
     {
       "claim": "AART generates adversarial datasets with higher coverage of policy concepts, task formats, and geographic regions than existing datasets",
       "evidence": "Table 2 shows keyword presence scores: AART achieves 0.384 for policy concepts vs next best 0.210 (Perez adaptation), 0.148 for task formats vs 0.013, and 0.410 for geographic regions vs 0.027. Section 4.",
       "supported": "moderate"
     },
     {
       "claim": "92.5% of AART-generated prompts are of good quality and useful for adversarial testing",
       "evidence": "Qualitative evaluation of 120 sampled prompts in Section 4. Limited sample size, single evaluator not explicitly described.",
       "supported": "weak"
     },
     {
       "claim": "Policy concepts from structured generation are referenced in 99.2% of queries",
       "evidence": "Qualitative evaluation section states this based on manual review of 120 sampled prompts.",
       "supported": "weak"
     },
     {
       "claim": "AART enabled launching several products with improved safety measures",
       "evidence": "Stated in conclusion (Section 6) with no supporting data or details provided.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All authors are from Google Research. AART uses Google's PaLM API and the paper discusses enabling Google product launches. No independent evaluation."
     },
     {
       "flag": "Weak evaluation methodology",
       "detail": "Quantitative evaluation relies solely on keyword matching, which the authors acknowledge underestimates coverage. Qualitative evaluation covers only 120 of 3,269 prompts with unclear evaluator details and no inter-rater reliability."
     },
     {
       "flag": "No downstream effectiveness evaluation",
       "detail": "The paper never tests whether AART-generated adversarial prompts actually elicit unsafe model behavior. Coverage/diversity of prompts is measured but not their adversarial effectiveness."
     },
     {
       "flag": "Unfair baseline comparison",
       "detail": "Existing datasets (RealToxicityPrompts, ParlAI, BAD, Anthropic) were created for different application contexts and purposes. Comparing keyword coverage for AART's specific policy concepts against datasets not designed for those concepts inflates AART's apparent advantage."
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository is provided. The paper mentions intending to release a demonstration dataset at a GitHub URL but no code for the AART pipeline itself is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we intend to make available a demonstration dataset' (abstract/footnote 1) — this is a promise of future release, not an actual release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or setup instructions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While prompts are shown in Appendix A, the full pipeline configuration and execution details are insufficient for reproduction."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported for the keyword coverage metrics in Table 2. Standard deviation is reported only for prompt length."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims AART shows higher coverage than comparison datasets but provides no statistical tests for these differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw keyword presence values are reported but without baseline context or formal effect size measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The qualitative evaluation uses 120 prompts out of 3,269 with no justification for this sample size. The Perez adaptation used only 160 responses with no justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Variance is reported only for prompt length (Table 2). No variance across multiple runs of the AART pipeline or across different LLM decodes."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 compares against four existing datasets (RealToxicityPrompts, ParlAI Dialogue Safety, BAD, Anthropic) and an adaptation of Perez et al. (2022)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Perez et al. (2022) and Ganguli et al. (2022) are recent and relevant automated/human red-teaming approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is provided. The pipeline has multiple components (problem definition, scoping, query generation) but no experiments isolate their individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses keyword coverage for three dimensions (policy concepts, task formats, geographic regions), prompt length statistics, and qualitative quality assessment."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes a qualitative evaluation of 120 sampled prompts assessing quality and coverage, though details about evaluators are sparse."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is a data generation method, not a model being evaluated on held-out data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by the three diversity dimensions (policy concepts, task formats, geographic regions) and per-keyword analysis is discussed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses failure patterns such as 'how to get away with...' oversampling, missing task formats (poems, legal documents), and geographic region imbalance. Appendix C provides examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 13 task formats were represented only once, some formats were missing entirely, and top-5 regions made up 53.4% of mentions — acknowledging imbalances."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'promising results in terms of concept coverage and data quality' which is hedged appropriately and supported by Table 2 and qualitative analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion claims 'AART enabled us to launch several products with improved safety measures and reduced risks' — a causal claim with no supporting evidence or study design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents a single hypothetical demonstration scenario (dangerous activities policy, English, global) but frames AART as a general-purpose solution for 'new LLM-powered applications' without bounding this generalization."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for why AART achieves higher keyword coverage (e.g., the keywords were specifically designed for AART's dimensions, giving it an inherent advantage)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures keyword coverage (keyword presence scores) and frames this as 'concept coverage' and 'data quality' for adversarial testing. The gap between keyword matching (the proxy) and actual adversarial effectiveness (whether prompts elicit unsafe model behavior) is not acknowledged. The paper never tests downstream effectiveness of generated prompts."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'PALM API' and 'instruction-tuned language model' but does not specify exact model version, snapshot, or size."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides the actual prompts used for Steps 1 and 3, including the full query generation template with placeholders and the fill values (keyword lists in A.2)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0.7 is mentioned only for the Perez adaptation (Appendix B). No temperature or sampling parameters stated for AART's own generation pipeline."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is described in detail across Steps 1-4 in Section 3, with Figure 2 providing a visual overview. The structured generation process, sampling strategy, and JSON output parsing are documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 states that 144 JSON lines were discarded (4.2% error rate) and the parsing/filtering process is described. Appendix B describes how comparison datasets were filtered."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is a dedicated 'Limitations' section with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific limitations: LLM bias in outputs, inability to capture emerging attack patterns (prompt injection, jailbreaking), ambiguity in defining 'adversarial', and keyword-based evaluation underestimating coverage."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what AART does NOT test or what settings are excluded. It acknowledges 'may not capture all rare or unseen problems' but does not enumerate specific scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The generated dataset is not released (only 'intended' for future release). Raw outputs from intermediate pipeline steps are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data generation procedure is described step by step in Section 3 with concrete examples and prompts in Appendix A."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants — data is generated by LLMs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from problem definition through query generation to review is documented. JSON parsing error rate (4.2%) and dataset size (3,269 prompts) are stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgment of funding sources. All authors are Google Research employees but no explicit funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as Google Research affiliates on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google Research authors are evaluating a method built on Google's PaLM API for use in Google product launches. The funder (Google) has a direct stake in demonstrating their safety tools work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. Authors work at Google whose products are implicated in the paper's claims."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate a pre-trained model's capability on a benchmark. It uses an LLM to generate adversarial prompts, not to solve tasks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper generates adversarial data rather than evaluating model knowledge on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no benchmark evaluation of model capability."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 acknowledges 'the computational expense of using LLMs is high' but provides no actual cost figures, token counts, or API costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget, GPU hours, or API spend reported despite the method requiring multiple LLM calls per adversarial prompt."
       }
     }
   },
   "cited_papers": [
     {
       "title": "Red teaming language models with language models",
       "authors": [
         "Ethan Perez",
         "Saffron Huang",
         "Francis Song"
       ],
       "year": 2022,
       "relevance": "Core baseline — automated red-teaming using LLMs, directly compared against AART."
     },
     {
       "title": "Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned",
       "authors": [
         "Deep Ganguli",
         "Liane Lovitt"
       ],
       "year": 2022,
       "relevance": "Major human red-teaming study from Anthropic; AART uses its dataset as a comparison baseline."
     },
     {
       "title": "Constitutional AI: Harmlessness from AI feedback",
       "authors": [
         "Yuntao Bai"
       ],
       "year": 2022,
       "relevance": "Foundational work on AI safety training using AI feedback, related to AART's automated safety testing."
     },
     {
       "title": "ToxiGen: A large-scale machine-generated dataset for adversarial and implicit hate speech detection",
       "authors": [
         "Thomas Hartvigsen"
       ],
       "year": 2022,
       "relevance": "Synthetic safety data generation approach that AART builds upon."
     },
     {
       "title": "Build it break it fix it for dialogue safety: Robustness from adversarial human attack",
       "authors": [
         "Emily Dinan"
       ],
       "year": 2019,
       "relevance": "Human adversarial testing for dialogue safety; ParlAI dataset used as comparison baseline."
     },
     {
       "title": "RealToxicityPrompts: Evaluating neural toxic degeneration in language models",
       "authors": [
         "Samuel Gehman"
       ],
       "year": 2020,
       "relevance": "Toxicity evaluation dataset used as a comparison baseline in Table 2."
     },
     {
       "title": "PaLM 2 technical report",
       "authors": [
         "Rohan Anil"
       ],
       "year": 2023,
       "relevance": "The underlying LLM (PaLM API) used by AART for adversarial prompt generation."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": [
         "Jason Wei"
       ],
       "year": 2022,
       "relevance": "Prompting technique adapted by AART for structured adversarial query generation."
     },
     {
       "title": "Identifying sociotechnical harms of algorithmic systems: Scoping a taxonomy for harm reduction",
       "authors": [
         "Renee Shelby"
       ],
       "year": 2023,
       "relevance": "Harm taxonomy relevant to AI safety evaluation methodology."
     },
     {
       "title": "Ethical and social risks of harm from language models",
       "authors": [
         "Laura Weidinger"
       ],
       "year": 2021,
       "relevance": "LLM risk taxonomy foundational to red-teaming and safety evaluation approaches."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Presents a reusable pipeline for generating adversarial test datasets that safety teams could adapt to their own LLM applications."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected intuition that structured AI-assisted generation produces more diverse adversarial prompts than repurposing existing datasets."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses AI safety testing as its core topic but demonstrates no novel attacks or vulnerabilities, focusing instead on dataset generation methodology."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; the paper positions itself as complementary to existing approaches rather than challenging any claims."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A demonstration dataset is promised on GitHub but the pipeline itself requires PaLM API access and custom prompt engineering to reproduce."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "All authors are from Google Research and the method uses Google's PaLM API, giving it major tech company recognition."
     }
   }
 }