ai-research-survey

scan.json (23969B)

---

 {
   "paper": {
     "title": "Constitutional AI: Harmlessness from AI Feedback",
     "authors": ["Yuntao Bai", "Saurav Kadavath", "Sandipan Kundu", "Amanda Askell", "Jackson Kernion", "Andy Jones", "Anna Chen", "Anna Goldie", "Azalia Mirhoseini", "Cameron McKinnon", "Carol Chen", "Catherine Olsson", "Christopher Olah", "Danny Hernandez", "Dawn Drain", "Deep Ganguli", "Dustin Li", "Eli Tran-Johnson", "Ethan Perez", "Jamie Kerr", "Jared Mueller", "Jeffrey Ladish", "Joshua Landau", "Kamal Ndousse", "Kamile Lukosuite", "Liane Lovitt", "Michael Sellitto", "Nelson Elhage", "Nicholas Schiefer", "Noemi Mercado", "Nova DasSarma", "Robert Lasenby", "Robin Larson", "Sam Ringer", "Scott Johnston", "Shauna Kravec", "Sheer El Showk", "Stanislav Fort", "Tamera Lanham", "Timothy Telleen-Lawton", "Tom Conerly", "Tom Henighan", "Tristan Hume", "Samuel R. Bowman", "Zac Hatfield-Dodds", "Ben Mann", "Dario Amodei", "Nicholas Joseph", "Sam McCandlish", "Tom Brown", "Jared Kaplan"],
     "year": 2022,
     "venue": "arXiv",
     "arxiv_id": "2212.08073"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository released: https://github.com/anthropics/ConstitutionalHarmlessnessPaper (Section 1.3, footnote 6), containing few-shot prompts, constitutional principles, and model responses."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The red teaming data references prior work (Ganguli et al., 2022) available at https://github.com/anthropics/hh-rlhf, but the actual training data (182,831 constitutional comparisons, SL-CAI finetuning data) generated for this paper is not released. The repository contains prompts and samples but not the full training datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency versions are mentioned. The paper describes model sizes but not the software environment needed to reproduce."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The method is described at a high level but there are no scripts or README with commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Error bars are shown in Figure 3 for Elo scores across model sizes. Figure 2 notes 'Error bars are visible in Figure 3 but are suppressed here for clarity.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims like 'RL-CAI models are significantly more harmless' (Section 4.3) are based on Elo score comparisons without formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are contextualizable via Elo score differences shown in Figures 2, 3, and 8, with baselines for comparison. Absolute harmfulness scores (0-4 scale) in Figure 10 provide magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are stated (10,274 helpfulness and 8,135 harmlessness comparisons for AB testing, Section 3.3; 64 held-out red team prompts for absolute scores, Section 4.5) but no justification or power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations or variance across runs are reported. Error bars appear in Figure 3 but it is not stated whether these represent std dev, confidence intervals, or something else. No mention of multiple experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: helpful RLHF, HH RLHF (trained with human harmlessness feedback), SL-CAI, and pretrained LMs are compared (Figures 2, 3, 8)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines are from the authors' own concurrent work (Bai et al., 2022) and represent the state of the art for RLHF-trained assistants at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Several ablations: critiqued vs. direct revisions (Figure 7), varying number of constitutional principles (Figure 6), varying number of revisions (Figure 5), soft vs. hard vs. clamped preference labels (Section 4.3), with and without chain-of-thought (Figures 2, 3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics: helpfulness Elo, harmlessness Elo (Figures 2, 3), absolute harmfulness scores on 0-4 scale (Figure 10), HHH evaluation accuracy (Figure 4), PM scores (Figure 5), calibration (Figure 9)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Crowdworker evaluations via model comparison tests produce the Elo scores (Section 3.3). Absolute harmfulness ratings from crowdworkers are also used (Section 4.5)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 states comparison test conversations are 'similar in distribution to, but distinct from, those appearing in the PM and RL training data.' Section 4.5 uses '64 hand-picked held-out red team prompts.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down by model size (Figure 3), by number of revisions (Figure 5), by number of principles (Figure 6), by RL training step (Figure 8). Appendix B breaks down harm classification into 9 categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses Goodharting/over-training failure modes with concrete examples of overly harsh or boilerplate responses ('you are valid, valued, and cared for'). Section 3.5 notes critiques are 'sometimes reasonable, but often made inaccurate or overstated criticisms.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reports that helpfulness scores decrease with revisions (Figure 5 center), that SL-CAI is less helpful than RL models (Section 3.3), that CoT is slightly less helpful (Section 4.3), and that over-training causes Goodharting (Section 4.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about training harmless but non-evasive assistants without human harmlessness labels are supported by Elo comparisons (Figures 2, 3) and qualitative examples (Appendix D). Claim that RL-CAI is 'preferred by crowdworkers' is supported by Section 3.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are supported by controlled ablations: critique vs. no critique (Figure 7), varying revisions (Figure 5), varying principles (Figure 6), CoT vs. no CoT (Figures 2, 3). These are single-variable manipulations within the same experimental framework."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims about 'Constitutional AI' as a general method but tests only on their own proprietary models at specific sizes (up to 52B). No testing on other model families, languages, or domains. The title and framing suggest generality beyond what is tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses how instruction changes to crowdworkers may explain differences from prior work. Section 4.5 caveats that 'absolute scores may not be well-calibrated, as different workers may have their own personal biases.' Section 3.4 notes PM scores 'become less calibrated at higher values.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are described only by parameter count (e.g., '52B') and reference to 'pretrained in the way we described in prior work [Bai et al., 2022].' No specific model version names, snapshot dates, or release identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full constitutional principles for both SL-CAI and RL-CAI are provided in Appendix C. Few-shot examples are referenced as being in Appendix E and the GitHub repository. The actual prompt formatting is shown in Sections 3.1 and 4.1."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2: temperature T=1 for sampling, learning rate 0.5 relative to pretraining LR, batch size 1024. Section 4.3: probability clamping at 40-60%. Section 4.2: 'same hyperparameters as our prior work [Bai et al., 2022].'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a training pipeline (SL finetuning + RL), not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents data sources and counts: 42,496 human-written red team prompts + 140,335 model-generated = 182,831 total; 135,296 helpfulness prompts; 4 revisions per red team prompt; 2 responses per helpfulness prompt. Section 4.2 provides PM comparison data counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Discussion) and Section 6.2 (Broader Impacts) discuss limitations including dual-use concerns, risk of deploying insufficiently tested models, and remaining reliance on helpfulness labels."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: PM score calibration issues at high values (Section 3.4), crowdworker instruction changes affecting Elo comparisons (Section 4.4), Goodharting from over-training (Section 4.3), constitutional principles 'selected in a fairly ad hoc manner' (footnote 7)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to specific model families, languages, or deployment contexts. Section 6.1 frames remaining work as 'future directions' rather than explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw crowdworker comparison data and training data are not released. Only the constitutional principles and sample outputs are available in the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes data collection: red team prompts from Ganguli et al. (2022) plus model-generated prompts. Section 3.3 describes crowdworker evaluation collection (10,274 helpfulness + 8,135 harmlessness comparisons). Section 4.4 describes instruction differences for crowdworkers."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Crowdworker platforms identified: Surge AI for current comparison tests, Upwork and MTurk for prior PM data collection (Section 4.4). Red teaming crowdworkers described in Ganguli et al. (2022)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: red team prompts → model responses → critiques → revisions → SL finetuning (Section 3.1-3.2); response pairs → AI feedback labels → PM training → RL (Section 4.1-4.2). Data counts at each stage are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgment of funding sources. All authors are from Anthropic but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors listed as affiliated with Anthropic, clearly stated at the top of the paper."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Anthropic employees are evaluating a method (Constitutional AI) developed at Anthropic for training Anthropic's own models. The funder/employer has a direct commercial interest in the success of the method."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement. No disclosure of equity, patents, or other financial interests related to Constitutional AI or Anthropic's products."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for the pretrained models used. The paper references models 'pretrained in the way we described in prior work' without specifying when training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the HHH evaluation questions (from Askell et al., 2021 and newly written ones) could overlap with pretraining data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The HHH evaluations from Askell et al. (2021) were published before model training, and the BIG Bench evaluations (Srivastava et al., 2022) were also public. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration mentioned. Crowdworker evaluations are used for model comparison but the study was not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval for the crowdworker studies."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No demographics reported for crowdworkers performing evaluations on any of the three platforms (Surge AI, MTurk, Upwork)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion/exclusion criteria for crowdworkers described beyond platform membership."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with human participants assigned to conditions. Crowdworkers evaluate model outputs but are not randomized into treatment/control groups."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No mention of whether crowdworkers knew which model produced which response during comparison tests."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition or dropout information reported for crowdworkers."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures reported despite the method requiring multiple rounds of model sampling (4 critique-revision pairs per prompt, multiple RL training runs)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total compute, or training time reported for any of the experiments (SL finetuning, RL training, preference model training)."
       }
     }
   },
   "claims": [
     {
       "claim": "RL-CAI models trained with AI feedback are preferred by crowdworkers over models trained with human feedback labels for harmlessness.",
       "evidence": "Figures 2 and 3 show Elo scores from crowdworker comparisons where RL-CAI achieves higher harmlessness scores than HH RLHF. Section 3.3 reports 10,274 helpfulness and 8,135 harmlessness comparisons.",
       "supported": "strong"
     },
     {
       "claim": "Constitutional AI achieves a Pareto improvement: higher harmlessness at a given level of helpfulness compared to standard RLHF.",
       "evidence": "Figure 2 shows the helpfulness-harmlessness Elo frontier, with RL-CAI points above and to the right of HH RLHF points.",
       "supported": "moderate"
     },
     {
       "claim": "Chain-of-thought reasoning improves AI identification of harmful behavior, approaching human feedback-trained preference model performance.",
       "evidence": "Figure 4 shows CoT and ensembled CoT accuracy on 438 HHH binary comparisons approaching PM accuracy at 52B parameters.",
       "supported": "strong"
     },
     {
       "claim": "Model-generated critiques and revisions progressively reduce harmfulness.",
       "evidence": "Figure 5 shows monotonically increasing harmlessness PM scores across 0-4 revisions. Figure 7 compares critiqued vs direct revisions.",
       "supported": "strong"
     },
     {
       "claim": "RL-CAI is virtually never evasive, unlike HH RLHF which frequently gives canned refusal responses.",
       "evidence": "Qualitative examples in Appendix D show HH RLHF responding with 'I'm sorry. I won't respond' while RL-CAI gives substantive responses. Section 4.4 discusses this qualitatively.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Constitutional AI (CAI) trains harmless but non-evasive AI assistants using only a set of natural language principles (a 'constitution') instead of human feedback labels for harmlessness. The two-stage process (supervised critique-revision followed by RL from AI feedback) achieves higher harmlessness than RLHF models trained with human harmlessness labels while maintaining comparable helpfulness, as measured by crowdworker Elo scores. Chain-of-thought reasoning improves AI feedback quality and approaches human preference model accuracy. The method significantly reduces evasiveness compared to standard HH RLHF models.",
   "red_flags": [
     {
       "flag": "Company evaluating own method",
       "detail": "All authors are Anthropic employees evaluating a method (Constitutional AI) that directly benefits Anthropic's product development. No external evaluation or independent replication."
     },
     {
       "flag": "No compute costs reported",
       "detail": "The method requires training multiple large models (up to 52B parameters), generating hundreds of thousands of critique-revision pairs, and running RL training, but no compute costs or GPU hours are reported."
     },
     {
       "flag": "Crowdworker instruction change confounds comparison",
       "detail": "Section 4.4 acknowledges that crowdworker instructions were changed from prior work to penalize evasiveness. This makes direct comparison with prior RLHF results unreliable, as the evaluation criteria shifted."
     },
     {
       "flag": "No statistical tests on main claims",
       "detail": "Claims of 'significantly more harmless' are based on Elo score differences without any formal statistical testing."
     }
   ],
   "cited_papers": [
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "relevance": "Foundation work on RLHF for AI assistants; establishes helpfulness-harmlessness tradeoff that CAI addresses."
     },
     {
       "title": "Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned",
       "authors": ["Deep Ganguli", "Liane Lovitt", "Jackson Kernion"],
       "year": 2022,
       "relevance": "Red teaming methodology and datasets used for CAI training and evaluation."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "InstructGPT paper; RLHF method that CAI extends by replacing human harmlessness labels with AI feedback."
     },
     {
       "title": "Improving alignment of dialogue agents via targeted human judgements",
       "authors": ["Amelia Glaese", "Nat McAleese", "Maja Trebacz"],
       "year": 2022,
       "relevance": "Sparrow model using rule-based decomposition of harmlessness, similar in spirit to CAI's constitutional principles."
     },
     {
       "title": "Deep reinforcement learning from human preferences",
       "authors": ["Paul Christiano", "Jan Leike", "Tom B. Brown"],
       "year": 2017,
       "relevance": "Foundational RLHF method that CAI builds upon and partially replaces."
     },
     {
       "title": "Language models (mostly) know what they know",
       "authors": ["Saurav Kadavath", "Tom Conerly", "Amanda Askell"],
       "year": 2022,
       "relevance": "Calibration of language model outputs used to justify soft preference labels in RLAIF."
     },
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Chain-of-thought technique used in CAI to improve AI feedback quality and transparency."
     },
     {
       "title": "Scaling laws for reward model overoptimization",
       "authors": ["Leo Gao", "John Schulman", "Jacob Hilton"],
       "year": 2022,
       "relevance": "Reward model Goodharting analysis relevant to CAI's observed over-training failure modes."
     },
     {
       "title": "Measuring progress on scalable oversight for large language models",
       "authors": ["Samuel R. Bowman", "Jeeyoon Hyun", "Ethan Perez"],
       "year": 2022,
       "relevance": "Empirical work on scalable oversight directly motivating CAI's approach to AI supervision."
     },
     {
       "title": "A general language assistant as a laboratory for alignment",
       "authors": ["Amanda Askell", "Yuntao Bai", "Anna Chen"],
       "year": 2021,
       "relevance": "Defines helpful, harmless, honest (HHH) framework and evaluation data used in CAI."
     },
     {
       "title": "Beyond the imitation game: Quantifying and extrapolating the capabilities of language models",
       "authors": ["Aarohi Srivastava"],
       "year": 2022,
       "relevance": "BIG Bench includes HHH evaluations used to benchmark CAI feedback quality."
     }
   ]
 }