ai-research-survey

calibration.json (22461B)

---

 {
   "paper_slug": "african-woman-rhythmic-2024",
   "calibrator": "opus",
   "calibration_date": "2026-02-28",
   "total_questions": 50,
   "agreement_count": 46,
   "disagreement_count": 4,
   "agreement_rate": 0.92,
   "disagreements": [
     {
       "category": "evaluation_design",
       "question": "baselines_contemporary",
       "sonnet": {"applies": false, "answer": false},
       "opus": {"applies": true, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=false reasoning that since no baselines were included, the question of whether baselines are contemporary is inapplicable. However, the schema says applies=false is for structurally inapplicable criteria. This paper could and should have compared against other LLMs or prior bias detection methods. The fact that no baselines were included makes this applies=true, answer=false (the paper fails the criterion), not applies=false. The question is relevant to this paper type — it just wasn't done."
     },
     {
       "category": "evaluation_design",
       "question": "human_evaluation",
       "sonnet": {"applies": true, "answer": true},
       "opus": {"applies": true, "answer": false},
       "direction": "sonnet_generous",
       "explanation": "Sonnet credited the authors' thematic analysis of stories and word clouds as 'human evaluation'. However, the schema description specifies human evaluation means 'human ratings, manual inspection, user studies, expert review of the system's OUTPUTS. The humans must be evaluating what the system produced.' The authors did analyze outputs qualitatively, but this was the researchers themselves doing an informal thematic analysis with no coding scheme, no inter-rater reliability, and no systematic evaluation protocol. The schema also says 'If evaluation of the system is entirely automated (e.g., pass/fail on test suites), NO.' The paper's approach is neither automated metrics nor a proper human evaluation study — it is informal qualitative observation by the authors. Without a systematic coding scheme or evaluation protocol, this does not constitute human evaluation as described by the schema."
     },
     {
       "category": "contamination",
       "question": "training_cutoff_stated",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=true, arguing the training cutoff is relevant to understanding what biases GPT-4 absorbed. However, the schema says this question is about whether 'the model's training data cutoff date [is] stated' in the context of assessing 'whether test examples could have been in the training set.' The NA rule says: 'NA if the paper does not evaluate a pre-trained model's capability on any benchmark.' This paper does not evaluate GPT-4 on any benchmark — it uses novel author-designed prompts to probe for bias. The concern here is not benchmark contamination but rather what societal biases exist in training data, which is a different question. The contamination category is about 'could the model have seen the test data during training' — which is not relevant to this paper's design."
     },
     {
       "category": "conflicts_of_interest",
       "question": "funding_disclosed",
       "sonnet": {"applies": true, "answer": false},
       "opus": {"applies": false, "answer": false},
       "direction": "applies_boundary",
       "explanation": "Sonnet set applies=true and answer=false. However, the schema says 'NA only if it's clearly unfunded work (e.g., a solo independent researcher).' This paper lists authors from UCL with correspondence emails in the format 'serene.lim.21@ucl.ac.uk' — the '.21' strongly suggests a student email (enrolled 2021). The paper has no acknowledgments section, no grant numbers, and appears to be a student research project. While not exactly a 'solo independent researcher,' a student project at a university with no external funding disclosed is reasonably 'clearly unfunded work' and applies=false is defensible. That said, this is a borderline case — UCL students do sometimes have funded projects. I set applies=false because the indicators point to unfunded student work."
     }
   ],
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code, GitHub repository, or code archive is provided anywhere in the paper. The paper mentions data is stored on a secure database linked to the OpenAI API, but no code for reproducing the experiments is released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. The Ethics section mentions 'experimental data has been recorded on a secure database' but no public access is provided. The raw LLM outputs are not available for download."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. The paper only mentions 'GPT-4 via the interface of ChatGPT' with no library versions, API details, Python version, or dependency files."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The methodology describes the experimental approach in narrative form but does not provide scripts, commands, or a structured protocol that would allow a researcher to replicate the study without guessing."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported anywhere. Results are presented as raw counts (e.g., '9 of 10 trials', '8 of 8 positive words') and approximate percentages (e.g., 'around 80%') with no uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper makes comparative claims (e.g., IAT correlates with traditional methods, gender effects differ from race effects) without any formal statistical testing — no p-values, t-tests, chi-squared tests, or similar."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standardized effect sizes (Cohen's d, odds ratios, etc.) are reported. Results are given as raw counts and percentages without baseline context or standardized measures of effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 10 trials per experiment is not justified anywhere. No power analysis is discussed, and there is no acknowledgment that 10 repetitions may be insufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure is reported across the 10 experimental trials. Results are presented as aggregate counts (e.g., '9 of 10') without any measure of variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. The paper only tests GPT-4 and does not compare against other LLMs, prior bias measurement tools, random baselines, or any reference point for contextualizing the findings."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper could and should have included contemporary baselines (other LLMs, prior bias detection methods). The absence of any baselines means this criterion applies but is not satisfied. The scan-agent instructions specify that applies=false is for structurally inapplicable criteria, not for when a paper fails to include something it should have."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper tests five different experimental tasks but does not systematically isolate variables (e.g., prompt phrasing, name selection, word list composition) to determine which factors drive the observed biases."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple assessment approaches: LLM IAT (word association counts), Decision Bias (profile-based task allocation), Sycophancy (behavioral observation), Word Generation (word cloud analysis), and Story Generation (thematic analysis). These constitute multiple complementary evaluation methods."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The authors informally analyzed GPT-4 outputs through thematic observation, but this does not constitute a proper human evaluation study. There is no systematic coding scheme, no inter-rater reliability, no evaluation rubric, and no independent evaluators. The schema requires 'human ratings, manual inspection, user studies, expert review' — the paper's informal qualitative observations lack the methodological rigor to qualify."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this is not a machine learning training study with train/dev/test splits. The paper uses the ChatGPT interface with manually designed prompts; there is no data split relevant here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported separately for each of the five experimental tasks (LLM IAT, Decision Bias, Sycophancy, Word Generation, Story Generation) and further broken down by attribute type (gender-career, race-valence, Muslim-others IAT)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the approach did not work as expected: GPT-4 rejected the visual IAT '90% of the time' when sensitive characteristics were mentioned, and the race-only Decision Bias results 'did not show much difference.' These qualify as discussion of failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that race-only prompts in the Decision Bias task 'did not show much difference' and that the model assigned gender to gender-neutral names rather than showing racial bias. The visual IAT failure is also a negative result. These count as reporting things that did not work as hypothesized."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the LLM IAT Bias 'correlates with traditional methods and more effectively predicts downstream behaviors' but no quantitative correlation analysis is presented in the paper. The paper shows only qualitative consistency (both methods reveal bias) without statistical correlation. The claim of 'a more comprehensive framework' is not empirically validated against prior frameworks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims including 'LLMs trained on biased data perpetuate biases,' 'RLHF causes sycophancy,' and 'biases become entrenched within AI systems.' The study design is purely observational — prompt-and-observe with no controlled manipulation of training data, RLHF parameters, or other causal factors. The causal claims are not supported by the study design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only GPT-4 via the ChatGPT interface but makes broad claims about 'LLMs' generally. The title, abstract, and discussion all frame findings as applicable to 'Large Language Models' and 'AI systems' without bounding claims to GPT-4 specifically. The paper does not state that results may not generalize to other models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No substantive alternative explanations are discussed. The paper does not consider confounds such as prompt phrasing effects, temperature/sampling settings, whether results reflect RLHF guardrails vs. base model behavior, or whether the specific name choices (Rebecca/John) drive results. The brief note that race was subordinated to gender is an observation, not an analysis of alternative explanations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states only 'GPT-4 via the interface of ChatGPT' without specifying a version number, API snapshot date, or model identifier (e.g., 'gpt-4-0613'). Per the schema, 'GPT-4' without a version does not meet the criterion."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Actual prompt text is provided for the key experiments: the LLM IAT prompt with the word list and name assignment instruction, the Decision Bias profile generation prompt, the Word Generation prompt ('Please brainstorm 10 words for the name Rebecca'), and the Story Generation prompts are quoted directly in the methodology section."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens, etc.) are reported. The paper uses the ChatGPT web interface, and the default settings used are not documented. These settings significantly affect output variability."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper accesses GPT-4 through the ChatGPT interface with direct single-turn prompts (and occasional follow-ups); there is no agentic loop, tool use, or multi-step scaffolding system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data preprocessing documentation is provided. The paper goes from collecting LLM outputs to reporting results without describing how qualitative themes were identified, whether multiple coders were used, how word clouds were generated from outputs, or what coding scheme was applied to the thematic analysis."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper has a brief Ethics section that only addresses privacy and data handling, not methodological limitations. The Conclusion mentions 'challenges' but does not substantively discuss limitations of the study itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address the small sample size (n=10), lack of inter-rater reliability for qualitative coding, potential prompt phrasing effects, model version instability, or generalizability to other LLMs."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No scope boundaries are explicitly stated. The paper does not specify that results apply only to GPT-4, only to the tested prompts, only to the specific names chosen, or only to the English language. Claims generalize broadly to 'LLMs' and 'AI systems' without stating what the paper does NOT claim."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (actual LLM response transcripts) is not released. The Ethics section mentions data is stored on a secure database but no public access is provided for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: experiments were conducted via GPT-4 through the ChatGPT interface, each task was repeated 10 times with specific prompts provided verbatim, and the order of stimuli was randomized. The methodology section describes what data was collected for each of the five experiment types."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — there are no human participants in this study. The research collects and analyzes LLM outputs rather than recruiting human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw LLM outputs to reported findings is not documented. There is no description of how word clouds were constructed from raw outputs, how thematic categories were derived, whether multiple coders were involved, or what criteria were used to identify themes in the story analysis."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": false,
         "answer": false,
         "justification": "The paper appears to be unfunded student work at UCL (author email format 'serene.lim.21@ucl.ac.uk' suggests student enrollment). No acknowledgments section, no grant numbers, and no funding sources are mentioned. Per the schema, NA applies for 'clearly unfunded work.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated as University College London, London, United Kingdom in the paper header. Neither author appears to be affiliated with OpenAI, whose product is being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no funding is disclosed, and the paper appears to be unfunded student work. The schema says NA if unfunded."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears anywhere in the paper. There is no disclosure of patents, equity holdings, or other financial interests related to the findings. The schema states: 'If there is no competing interests statement at all, NO.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — this paper does not evaluate GPT-4's capability on any benchmark. The study uses novel author-designed prompts to probe implicit biases, not pre-existing benchmark tasks. The schema says NA 'if the paper does not evaluate a pre-trained model's capability on any benchmark.'"
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the paper does not evaluate a pre-trained model on any benchmark. The prompts are original author-designed stimuli adapted from psychological literature, not a fixed ML benchmark corpus."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — the study does not use a pre-existing benchmark dataset. The IAT-inspired prompts and decision bias scenarios are novel author-constructed stimuli, not taken from a published ML benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved. The study evaluates LLM outputs, not human subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved. The Ethics section confirms: 'This form of prompt engineering will not have any implications on any other participants, nor will any privacy guidelines be breached.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants or experimental groups. The paper mentions 'randomizing the order of stimuli' but this is in LLM prompting, not human participant randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable — no human participants are involved, so attrition is not relevant."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, API cost, or token consumption is reported. The paper uses the ChatGPT interface for GPT-4 queries but provides no estimate of costs incurred or tokens consumed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No compute budget is stated. The total number of API interactions, wall-clock time, or cost estimate is not provided."
       }
     }
   }
 }