ai-research-survey

scan.json (35089B)

---

 {
   "paper": {
     "title": "GPT-4 Technical Report",
     "authors": ["OpenAI"],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2303.08774"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper open-sources OpenAI Evals (https://github.com/openai/evals), their evaluation framework. However, model weights, training code, and full reproduction pipelines are not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluations use standard public benchmarks (MMLU, HumanEval, HellaSwag, ARC, WinoGrande, DROP, GSM-8K, TruthfulQA) that are all publicly available. The paper does not create new datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 2 explicitly states: 'this report contains no further details about the architecture (including model size), hardware, training compute, dataset construction, training method, or similar.' No environment specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The model is proprietary, architecture is withheld, and the specific evaluation scripts beyond the Evals framework are not released. The paper cannot be reproduced by external researchers."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report point estimates only (e.g., '298 / 400', '86.4%') with no confidence intervals or error bars. The only uncertainty quantification is the ECE in Figure 8's calibration plots."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'GPT-4 outperforms both previous large language models and most state-of-the-art systems' based solely on comparing numbers. No statistical significance tests (p-values, t-tests, bootstrap) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baselines: '19 percentage points higher than our latest GPT-3.5' (Section 5), '82% compared to GPT-3.5' (Section 6), and Tables 1-2 provide both GPT-4 and GPT-3.5 scores enabling magnitude comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of exam questions, benchmark examples, or evaluation prompts used. The 5,214 prompts for the preference evaluation and 10 Codeforces contests are stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Most results are single-run point estimates. The Codeforces section mentions averaging over 100 simulations but reports no standard deviation. Appendix A.3 explicitly states free-response questions were 'only a single time.' No variance or standard deviation is reported across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "GPT-3.5 is the primary baseline throughout. Table 2 also compares to LM SOTA (PaLM, LLaMA, U-PaLM) and overall SOTA with benchmark-specific tuning."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include PaLM (2022), LLaMA (2023), Chinchilla (2022), U-PaLM (2022), and Flan-PaLM (2022), which were all contemporary or near-contemporary at time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B compares GPT-4 base model vs GPT-4 post-RLHF on all exam benchmarks (73.7% vs 74.0% average). Tables 9-10 show results with contaminated questions removed. Table 1 compares vision vs no-vision variants."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on dozens of different metrics across exams (bar exam, SAT, GRE, AP exams) and benchmarks (MMLU accuracy, HumanEval pass rate, HellaSwag, ARC, WinoGrande, DROP F1, GSM-8K, TruthfulQA, RealToxicityPrompts, human preference rate)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human labelers judged GPT-4 vs GPT-3.5 responses on 5,214 prompts (70.2% preference for GPT-4, Section 4). Free-response exam answers were graded by 'qualified third-party contractors.' Over 50 expert red teamers evaluated model outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4: 'The evaluation setup was designed based on performance on a validation set of exams, and we report final results on held-out test exams.' Appendix A.2 describes the holdout/non-holdout exam pair methodology."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down performance by individual exam. Table 2 by benchmark. Figure 5 by language. Figure 6 by factuality category (learning, technology, writing, etc.). Tables 9-10 by contamination per exam."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses hallucinations, reasoning errors, and overconfidence. Table 4 shows an incorrect TruthfulQA answer. Codeforces rating is below 5th percentile. The System Card extensively catalogs failure modes including harmful content generation, bias, and jailbreaks (Figures 1, 10)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Codeforces performance is poor (below 5th percentile). AP English scores are mediocre (2/5). RLHF degrades calibration (Figure 8, ECE from 0.007 to 0.074). Appendix B shows RLHF hurts performance on several individual exams. GPT-4 underperformed predictions on the easiest HumanEval bucket (Section 3.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'human-level performance on various professional and academic benchmarks' (supported by Table 1), 'passing a simulated bar exam with a score around the top 10%' (298/400, ~90th percentile in Table 1), and 'accurately predict some aspects of GPT-4's performance' (supported by Figures 1-2)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'The post-training alignment process results in improved performance on measures of factuality' (abstract) and 'We've decreased the model's tendency to respond to requests for disallowed content by 82%' (Section 6), attributing these to the combined RLHF + RBRM + data filtering pipeline without isolating individual intervention effects. The base vs RLHF ablation (Appendix B) only covers exam performance, not the emphasized safety/factuality claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded: 'human-level performance on various professional and academic benchmarks' (not general intelligence), 'on translated variants of MMLU' for multilingual claims. The abstract explicitly states 'While less capable than humans in many real-world scenarios.' Section 5 bounds capabilities clearly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendices C-D extensively analyze contamination as an alternative explanation for benchmark performance, with per-exam contamination rates and decontaminated scores. Appendix E acknowledges GSM-8K was partially in training data. Figure 3 discusses inverse scaling as an alternative trend."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'While less capable than humans in many real-world scenarios' while claiming 'human-level performance on various professional and academic benchmarks,' explicitly distinguishing benchmark performance from real-world capability. Section 5 reinforces this by discussing hallucinations and reasoning errors despite strong exam scores."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-4' and 'GPT-3.5' without version strings. Appendix A.7 gives internal snapshot dates (March 1, 2023; February 23, 2023; December 16, 2022) that are not externally reproducible identifiers. Architecture and model size are explicitly withheld (Section 2)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.8 provides detailed few-shot prompts used for multiple-choice evaluation (AP Art History example with full chain-of-thought) and free-response evaluation, including actual prompt text, sampling parameters (temperature, max_tokens, stop tokens), and format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings are specified: 0.3 for MCQ explanations, 0.0 for letter extraction, 0.6 for free-response (Appendix A.2-A.3). Max tokens and stop tokens are shown in the example prompts. The Codeforces evaluation reports 10 attempts per problem."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The main evaluations use direct prompting without agentic scaffolding. The ARC autonomous replication tests (System Card Section 2.9) use a scaffold described as a 'read-execute-print loop' but this is a secondary evaluation, not the main methodology."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A describes exam sourcing (publicly available materials, purchased study guides), contamination checking methodology, prompt formatting (separate for MCQ and free-response), image handling (text tags for non-vision models, embedded images for multimodal), and scoring methodology with specific references."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is titled 'Limitations' and provides substantive discussion of hallucinations, knowledge cutoff, inability to learn from experience, reasoning errors, overconfidence, bias, and calibration degradation. The System Card (Section H, 60+ pages) provides extensive additional limitation analysis."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: hallucination ('can suffer from hallucinations'), temporal knowledge cutoff ('September 2021'), post-training calibration degradation (Figure 8, ECE increases from 0.007 to 0.074), reasoning errors on specific problem types, and security vulnerabilities in generated code. The System Card discusses jailbreak vulnerabilities."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 explicitly states: 'this report contains no further details about the architecture... hardware, training compute, dataset construction, training method.' Section 5 states GPT-4 'does not learn from its experience' and 'has a limited context window.' The System Card notes that custom fine-tuning and image capabilities are 'explicitly out of scope.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw evaluation data is released. Individual model responses to exam questions, benchmark examples, or safety evaluations are not available. Training data is proprietary. Only aggregate results are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A details exam sourcing ('most recent publicly-available official past exams, or practice exams in published third-party 2022-2023 study material'), contamination checking procedures, scoring methodologies per exam type, and model snapshot dates. Section 4 describes the preference evaluation data collection."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Red teamer recruitment is described: 'over 50 experts from domains such as long-term AI alignment risks, cybersecurity, biorisk, and international security' (Section 6). Preference evaluation labelers were instructed per footnote 7. Exam graders described as 'qualified third-party contractors with relevant work experience' (Appendix A.3). The System Card notes selection bias in red teamer demographics."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation pipeline is documented: exam sourcing → contamination checking (50-character substring matching, Appendix C) → prompt formatting (MCQ vs free-response) → generation (with specified temperatures) → scoring (using published methodologies and rubrics). The safety evaluation pipeline (RBRM classifiers, Figure 6-9) is also described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges Microsoft's partnership: 'We thank Microsoft for their partnership, especially Microsoft Azure for supporting model training with infrastructure design and management.' This constitutes disclosure of a major funding/resource relationship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper is clearly attributed to 'OpenAI' with extensive authorship credit listings. The company developing and commercially deploying GPT-4 is the same entity conducting the evaluation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI has direct financial interest in GPT-4 performing well — the model is their commercial product. Microsoft, acknowledged as a major partner, is a major investor in OpenAI. Neither entity is independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included. The authors are employees of OpenAI, which commercially deploys GPT-4. No disclosure of equity, patents, or other financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5: 'GPT-4 generally lacks knowledge of events that have occurred after the vast majority of its pre-training data cuts off in September 2021.' Footnote 10 adds: 'The pre-training and post-training data contain a small amount of more recent data.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes a detailed contamination checking methodology using 50-character substring matching. Tables 9-11 report per-exam and per-benchmark contamination rates. BIG-bench was 'inadvertently mixed into the training set' and excluded (footnote 5). Results with contaminated questions removed are reported alongside full results."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Tables 9-10 provide per-exam contamination rates ranging from 0% to 100%. Table 11 covers academic benchmarks. The paper reports both contaminated and non-contaminated scores, and concludes 'contamination overall has very little effect on the reported results.' Appendix E explicitly addresses GSM-8K partial inclusion in training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not conduct a human subjects study. Human labelers and red teamers are evaluators of model outputs, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted. Labelers and red teamers serve as evaluators, not participants in a research study on humans."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The System Card notes red teamers have 'ties to English-speaking, Western countries' and have 'specific educational and professional backgrounds,' but these are evaluators, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects experiment. Preference evaluation does randomize response order (footnote 7) but this is an evaluation methodology detail, not a human subjects study design."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The preference evaluation uses blinding ('labelers were not told which response was generated by which model,' footnote 7) but this is part of model evaluation, not a study of human subjects."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study is conducted."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, token consumption, or latency numbers are reported for any evaluation. The paper does not quantify the cost of running the exam or benchmark evaluations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 2 explicitly withholds this: 'this report contains no further details about the architecture (including model size), hardware, training compute.' Figure 1's x-axis is normalized compute (GPT-4 = 1) with no absolute values."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Exam evaluations appear to be single-run. Codeforces simulations are averaged over 100 iterations but no variance across seeds is shown. Main benchmark results (MMLU, HumanEval, etc.) do not report seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 states free-response questions were run 'only a single time.' Appendix A.6 states Codeforces used '10 attempts per problem' with '100 times' simulation per contest. Appendix A.2 states holdout exams were run 'once for a final score.' While not always explicit for every benchmark, run counts are stated for the major evaluations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 4 states 'The evaluation setup was designed based on performance on a validation set of exams' but does not report how many configurations were tried, what search method was used, or what compute was spent on tuning the evaluation methodology."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2: 'We iterated on our methodology using the nonholdout exam, and then ran each holdout exam once for a final score.' This holdout methodology for configuration selection is clearly described and avoids overfitting to the test set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes dozens of comparisons across exams and benchmarks (Tables 1-2, Figures 4-9) with no correction for multiple comparisons. No Bonferroni, Holm, or other family-wise error rate corrections are applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI evaluates its own commercial product without acknowledging self-evaluation bias. While standard benchmarks and third-party exam graders mitigate this somewhat, the paper does not discuss the inherent bias of a company evaluating its own model, nor does it include independent third-party evaluation of the main results."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1 and 2 are core contributions showing performance as a function of normalized compute budget. The scaling law analysis (Section 3) explicitly plots loss and capability vs compute, demonstrating predictable scaling relationships."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses professional exams and academic benchmarks without questioning whether they measure the claimed capabilities. No discussion of whether bar exam scores reflect legal reasoning ability, whether MMLU scores measure understanding vs pattern matching, or whether these proxies have construct validity for 'human-level performance.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The main evaluations use direct prompting without agentic scaffolding. Model comparisons use consistent prompting methodology across GPT-4 and GPT-3.5. No scaffold confound is present in the primary results."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the September 2021 training cutoff and systematically checks whether exam/benchmark questions appeared in training data. Appendices C-D provide detailed contamination analysis, and results are reported both with and without contaminated questions."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation format itself leaks information. No analysis of whether few-shot prompting examples, answer choice formatting, or exam structure provides hints not available in real deployment. The multiple-choice format ('present all answers ABCD') could differ from real-world usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The contamination analysis (Appendix C) uses substring matching to check for overlap between evaluation and training data. The paper identifies and excludes BIG-bench due to inadvertent training set contamination. Per-exam contamination rates are reported with decontaminated results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes a concrete method: 'We measure cross-contamination between our evaluation dataset and the pre-training data using substring match... we randomly select three substrings of 50 characters.' This yields per-example contamination labels. Limitations of the method (false positives and negatives) are acknowledged."
       }
     }
   },
   "claims": [
     {
       "claim": "GPT-4 exhibits human-level performance on various professional and academic benchmarks, including passing a simulated bar exam with a score around the top 10% of test takers.",
       "evidence": "Table 1 shows 298/400 (~90th percentile) on the Uniform Bar Exam. GPT-4 achieves top scores on many AP exams, SAT, GRE, and professional exams. Figure 4 visualizes performance across exams.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4's final loss can be accurately predicted from models trained with 1,000x to 10,000x less compute using a scaling law.",
       "evidence": "Figure 1 shows the predicted vs observed loss on an internal codebase, with the power law fit from smaller models accurately predicting GPT-4's final loss. Section 3.1 describes the methodology.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 outperforms existing language models on all benchmarks tested, and beats SOTA with benchmark-specific training on all datasets except DROP.",
       "evidence": "Table 2 shows GPT-4 achieving 86.4% on MMLU (vs 70.7% LM SOTA), 95.3% on HellaSwag (vs 84.2%), 96.3% on ARC (vs 85.2%), 67.0% on HumanEval (vs 26.2%), and 92.0% on GSM-8K (vs 58.8%). DROP is the exception (80.9 vs 88.4 SOTA).",
       "supported": "moderate"
     },
     {
       "claim": "RLHF post-training does not substantially alter base model capability on exam benchmarks.",
       "evidence": "Appendix B, Table 8: averaged across all exams, the base model achieves 73.7% while the RLHF model achieves 74.0%. Individual exam comparisons show variation in both directions.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 significantly reduces hallucinations relative to previous GPT-3.5 models, scoring 19 percentage points higher on internal factuality evaluations.",
       "evidence": "Figure 6 shows GPT-4 outperforming ChatGPT versions across nine factuality categories. The evaluation is internal and adversarially designed.",
       "supported": "moderate"
     },
     {
       "claim": "Safety mitigations have decreased the model's tendency to respond to requests for disallowed content by 82% compared to GPT-3.5.",
       "evidence": "Figure 9 shows incorrect behavior rates on disallowed and sensitive prompts for text-davinci-003, gpt-3.5-turbo, and gpt-4. Section 6 reports the 82% figure and 29% improvement on sensitive requests.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 surpasses the English-language performance of existing language models on MMLU in 24 of 26 languages tested.",
       "evidence": "Figure 5 shows GPT-4's accuracy across 26 languages on MMLU compared to Chinchilla (67.0%), PaLM (69.3%), and GPT-3.5 (70.1%) English performance. GPT-4 exceeds these in 24 languages.",
       "supported": "moderate"
     },
     {
       "claim": "Post-training (RLHF) hurts model calibration significantly.",
       "evidence": "Figure 8 shows the pre-trained model has ECE of 0.007 (near-perfect calibration) while the post-trained model has ECE of 0.074 on an MMLU subset.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GPT-4 achieves human-level or better performance on most professional and academic exams tested, including top 10% on the bar exam. Scaling laws allow accurate prediction of GPT-4's loss and capabilities from 1000x-10000x smaller models. RLHF post-training minimally affects base model exam capability (73.7% vs 74.0%) but significantly improves safety metrics (82% reduction in disallowed content responses) while degrading calibration (ECE from 0.007 to 0.074). The paper withholds architecture, model size, training data, and compute details, citing competitive and safety concerns.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "OpenAI evaluates GPT-4, its commercial product, without independent third-party verification of the main results. The company has direct financial interest in GPT-4 appearing capable. While standard benchmarks provide some objectivity, internal evaluations (factuality, safety) have no external validation."
     },
     {
       "flag": "Architecture and training details completely withheld",
       "detail": "Section 2 explicitly states: 'this report contains no further details about the architecture (including model size), hardware, training compute, dataset construction, training method, or similar.' This makes independent verification or reproduction impossible."
     },
     {
       "flag": "No error bars or variance on main results",
       "detail": "Tables 1 and 2 report only point estimates across dozens of exams and benchmarks. No confidence intervals, error bars, or variance across runs. Free-response questions were run only once (Appendix A.3). This makes it impossible to assess result reliability."
     },
     {
       "flag": "Contamination methodology limitations acknowledged but not addressed",
       "detail": "Appendix C acknowledges the 50-character substring method has false negatives (paraphrased content would be missed) and false positives. RLHF post-training data was 'not checked explicitly' for contamination. GSM-8K was acknowledged to be partially in training data (Appendix E)."
     },
     {
       "flag": "Safety improvement claims conflate multiple interventions",
       "detail": "The 82% reduction in disallowed content is attributed to the combined post-training pipeline (RLHF + RBRMs + data filtering + SFT), but no ablation isolates individual intervention effects. The paper does not report which safety improvements come from which intervention."
     },
     {
       "flag": "Selective presentation acknowledged in System Card",
       "detail": "The System Card states: 'the examples included throughout this system card are not zero-shot and are cherry picked from our evaluation efforts to illustrate specific types of safety concerns or harms.' This applies to both positive and negative examples."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing few-shot prompting capabilities of large language models; foundational to GPT-4's lineage."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval benchmark for code generation used in GPT-4's capability evaluation and scaling prediction."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "InstructGPT paper establishing the RLHF methodology used for GPT-4's post-training alignment."
     },
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": ["Aakanksha Chowdhery", "Sharan Narang", "Jacob Devlin"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "Major competing LLM used as baseline in GPT-4's benchmark evaluations."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws paper relevant to GPT-4's predictable scaling approach and used as baseline."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Chain-of-thought prompting methodology used in GPT-4's GSM-8K evaluation."
     },
     {
       "title": "TruthfulQA: Measuring how models mimic human falsehoods",
       "authors": ["Stephanie Lin", "Jacob Hilton", "Owain Evans"],
       "year": 2022,
       "relevance": "Benchmark used to evaluate GPT-4's factuality improvements over GPT-3.5 (Figure 7)."
     },
     {
       "title": "Training a helpful and harmless assistant with reinforcement learning from human feedback",
       "authors": ["Yuntao Bai", "Andy Jones", "Kamal Ndousse"],
       "year": 2022,
       "arxiv_id": "2204.05862",
       "relevance": "Anthropic's RLHF work on AI safety alignment, directly compared to GPT-4's approach on TruthfulQA."
     },
     {
       "title": "Red teaming language models with language models",
       "authors": ["Ethan Perez", "Saffron Huang", "H. Francis Song"],
       "year": 2022,
       "arxiv_id": "2202.03286",
       "relevance": "Red teaming methodology for LLM safety evaluation, related to GPT-4's expert adversarial testing approach."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws work that GPT-4's predictable scaling capability prediction builds upon."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2021,
       "relevance": "MMLU benchmark used extensively in GPT-4's capability evaluation including multilingual testing (Figure 5)."
     },
     {
       "title": "LLaMA: Open and efficient foundation language models",
       "authors": ["Hugo Touvron", "Thibaut Lavril", "Gautier Izacard"],
       "year": 2023,
       "arxiv_id": "2302.13971",
       "relevance": "Open-source LLM baseline used in GPT-4's benchmark comparisons (Table 2, HellaSwag)."
     },
     {
       "title": "Ethical and social risks of harm from Language Models",
       "authors": ["Laura Weidinger", "John Mellor", "Maribeth Rauh"],
       "year": 2021,
       "relevance": "Framework for understanding societal risks of LLMs that informed GPT-4's System Card analysis."
     },
     {
       "title": "Deep reinforcement learning from human preferences",
       "authors": ["Paul F Christiano", "Jan Leike", "Tom Brown"],
       "year": 2017,
       "relevance": "Original RLHF algorithm paper forming the basis of GPT-4's post-training methodology."
     }
   ]
 }