ai-research-survey

scan.json (22096B)

---

 {
   "paper": {
     "title": "AI Assistance in Legal Analysis: An Empirical Study",
     "authors": ["Jonathan H. Choi", "Daniel Schwarcz"],
     "year": 2023,
     "venue": "Journal of Legal Education, Volume 73, Number 2"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or archive is mentioned in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or data release is mentioned. Exam data and scores are not publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies the GPT-4 model version (gpt-4-0314) and temperature=0 in the Appendix, but no environment file, requirements.txt, or detailed software setup is provided for reproducing the analysis pipeline."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The study design is described narratively but there are no runnable scripts or explicit replication guides."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "95% confidence intervals are reported for all main results in Table 1 and in all figures, generated via 50,000 bootstrap iterations (footnote 63)."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Bootstrap-based significance tests are used throughout. The paper states 'All distributions, confidence intervals, and significance tests in this article were conducted by bootstrapping' (footnote 63)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentile-point changes with baselines: e.g., '+28.9 percentile-point improvement' for MC questions (Table 1), '+45 percentile points' for lowest performers, '-20 percentile points' for top performers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or sample size justification is provided. The sample sizes (16 for Insurance Law, 32 for Intro to American Law) are convenience samples based on volunteer recruitment, with no discussion of statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported for both human-only and AI-assisted conditions in Table 1, with confidence intervals on the standard deviations themselves."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human-only performance (without AI) serves as the baseline, with comparisons to AI-assisted and AI-only conditions. Prior year (2022) human exam performance is used as the reference distribution."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is the contemporaneous human performance on the same exams (2022 cohort), which is the most appropriate comparison for this study design."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Four prompting strategies (basic, chain-of-thought, few-shot, grounded) are tested for AI-only conditions, serving as an ablation across prompting methods (Table 2)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: percentile performance, letter grades, speed/time to completion, and qualitative analysis of exam quality."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "All exams were blindly graded by human evaluators (teaching assistants for Intro to American Law, the instructor for Insurance Law). Section III.B provides detailed qualitative evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a human subjects experiment, not a benchmark evaluation with train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by exam type (multiple-choice vs. essay), by course (Intro to American Law vs. Insurance Law), and by baseline student performance level (Figures 4, 12-17)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B discusses specific weaknesses of AI-assisted exams: organizational problems, missed hidden legal issues, conclusory analysis, failure to cite case law covered in class."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key negative results are prominently reported: AI assistance had no effect on essay performance, top-performing students saw ~20 percentile-point declines with AI access."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The introduction's claims (29 percentile-point MC improvement, no essay improvement, 45-point gain for worst students, 20-point decline for best) are all supported by Table 1 and Figure 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The study makes causal claims ('AI assistance improved performance') but uses a within-subjects pre-post design without randomization to conditions. The authors acknowledge mean reversion as a confound (footnote 66) and note this is not an RCT (footnote 12 references their separate RCT study)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section III.C explicitly bounds generalization: 'we should be cautious about extrapolating the results of our study to other settings, particularly nonlegal settings' and discusses four specific limitations on external validity."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section III.C discusses multiple alternative explanations: inadequate training, motivation differences, mean reversion (footnote 66), difficulty of integrating AI with human writing (Section IV), and that 'AI-only' methods involved some human input."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix Section A.4 specifies 'the 8K GPT-4 model through OpenAI's API, using the March 14, 2023, version (gpt-4-0314).'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix Figures 9-11 for chain-of-thought, few-shot, and grounded prompts, plus system prompts for essays and multiple-choice."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix Section A.4: 'we set temperature to 0, as recommended by OpenAI.' Model context window (8K) is also specified."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The study uses direct single-turn prompting of GPT-4."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is well-documented: exam selection process (Appendix A.2), regrading procedure with correction factors (Section II.A), Spearman correlation between original and regraded scores (0.77 and 0.94)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section III.C 'Study Limitations' is a dedicated subsection discussing four specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section III.C discusses specific threats: inadequate one-hour training, motivation differences between study and real exam, non-representative sample (inverse-U shaped distribution, Appendix A.3), mean reversion confound, and limited generalizability to non-legal settings."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section III.C states specific scope boundaries: results may not generalize to non-legal settings, law school exams may not reflect real lawyering skills, the 'AI-only' label overstates autonomy since prompts required human selection of materials."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (exam scores, individual responses, grading rubrics) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II.A and Appendix A describe the data collection procedure in detail: recruitment via email, training process, exam administration, blind grading procedure, and regrading for cross-year consistency."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 describes recruitment: emails sent to all enrolled students, flat fee paid for participation, participation rates (31% for Insurance Law, 23% for Intro to American Law), and attrition rates."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: recruitment → training → exam administration → blind grading mixed with 2022 exams → regrading of 2022 exams for calibration → correction factor applied → percentile calculation. Specific numbers provided at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Choi at USC Gould School of Law, Schwarcz at University of Minnesota Law School. Schwarcz taught the courses used in the study."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure is present, so independence of funder cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4 (gpt-4-0314) on law school exam questions but does not state the model's training data cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses this: 'These exams were not publicly available' (Section II.A), indicating the 2022 exams used were not in GPT-4's training data. The exams were administered in prior years and not published online."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "By using unpublished prior-year exams that were 'not publicly available and had not been previously provided to students' (Section II.A), the authors mitigated contamination risk, though they do not explicitly discuss this as a contamination concern."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned. No link to OSF, AsPredicted, or any pre-registration platform."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 39 states: 'This experiment received approval from the University of Minnesota's institutional review board. IRB #STUDY00019012.'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "No participant demographics are reported beyond their enrollment in specific classes (undergraduates vs. law students). No age, gender, experience level, or other demographic information is provided."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Inclusion criteria are stated: enrolled students in the two specific courses. Appendix A.3 describes recruitment and that participants agreed before receiving final grades. Footnote 41 notes one undergraduate in Insurance Law."
       },
       "randomization_described": {
         "applies": true,
         "answer": false,
         "justification": "This study does not randomize participants to conditions. All participants receive AI assistance on the second exam (within-subjects design without randomization). The authors explicitly note in footnote 12 that their separate follow-up study was the 'first randomized controlled trial' on this topic."
       },
       "blinding_described": {
         "applies": true,
         "answer": true,
         "justification": "Blind grading is described: 'Exams produced by AI-assisted humans and by AIs alone were then mixed with exams written by real students in 2022 without AI assistance, and blindly graded' (Section II.A)."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 42 reports attrition: Insurance Law had 18 sign up and 16 complete; Intro to American Law had 39 sign up and 32 complete."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs or inference costs are reported despite using the GPT-4 API for both AI-only and AI-assisted conditions."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget or API spend is reported."
       }
     }
   },
   "claims": [
     {
       "claim": "AI assistance improved multiple-choice performance by 29 percentile points on average.",
       "evidence": "Table 1 shows mean change of +28.9 percentile points (CI: +20.3, +38.4) for Introduction to American Law multiple-choice questions.",
       "supported": "strong"
     },
     {
       "claim": "AI assistance had no significant effect on essay question performance.",
       "evidence": "Table 1 shows +3.5 percentile points (CI: -6.7, +15.3) for Intro essay and -1.3 (CI: -9.7, +7.7) for Insurance Law, both including zero in the confidence interval.",
       "supported": "strong"
     },
     {
       "claim": "The worst-performing students benefited enormously from AI (~45 percentile-point gains) while the best-performing students saw ~20 percentile-point declines.",
       "evidence": "Figure 4 shows the relationship between baseline performance and AI benefit, with confidence intervals. However, the authors acknowledge this may be partially driven by mean reversion (footnote 66).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 with grounded prompting outperformed both humans and AI-assisted humans.",
       "evidence": "Table 2 shows grounded GPT-4 at 100th percentile for MC, 93rd for Intro essay, 65th for Insurance Law, compared to AI-assisted means of 88, 53, and 46 respectively.",
       "supported": "moderate"
     },
     {
       "claim": "AI assistance significantly reduced exam completion time.",
       "evidence": "Section III.A.3: average time dropped from 74.5 to 62.9 minutes for Intro to American Law, described as 'a large and statistically significant drop' with bootstrap CIs shown in Figure 8.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["observational", "case-study"],
   "key_findings": "AI assistance dramatically improved law student performance on multiple-choice questions (+29 percentile points) but had no significant effect on essay exam performance. The benefit varied by student skill level: low-performing students gained ~45 percentile points while top students declined ~20 points, though mean reversion may partially explain this. GPT-4 alone with grounded prompting outperformed both unassisted humans and AI-assisted humans, suggesting optimal prompting may matter more than human-AI collaboration for certain legal analysis tasks.",
   "red_flags": [
     {
       "flag": "No randomization to conditions",
       "detail": "All participants received AI in the study condition and no AI in the real exam, making it a within-subjects pre-post design without randomization. The authors acknowledge this and note their separate RCT study, but causal claims in this paper rest on a weaker design."
     },
     {
       "flag": "Mean reversion confound",
       "detail": "The finding that low performers benefit most and high performers are harmed could be substantially driven by regression to the mean, as the authors themselves acknowledge (footnote 66). Without parallel forms reliability data, the magnitude of this confound is unknown."
     },
     {
       "flag": "Small sample sizes",
       "detail": "Only 16 students in Insurance Law and 32 in Introduction to American Law participated. No power analysis justifies these sample sizes for the subgroup analyses presented."
     },
     {
       "flag": "Non-representative volunteer sample",
       "detail": "Appendix A.3 acknowledges the sample underrepresented both low- and high-performing students (inverse-U shaped distribution), limiting generalizability of average treatment effects."
     },
     {
       "flag": "Different exam conditions",
       "detail": "Study participants took a different exam (prior year) under different conditions (after training, with different time pressure, lower stakes) than their real exams, introducing multiple confounds beyond AI access."
     }
   ],
   "cited_papers": [
     {
       "title": "ChatGPT Goes to Law School",
       "authors": ["Jonathan H. Choi", "Kristin E. Hickman", "Amy B. Monahan", "Daniel Schwarcz"],
       "year": 2022,
       "relevance": "Earlier study by same authors evaluating GPT-3.5 standalone performance on law school exams, foundational work in AI legal analysis evaluation."
     },
     {
       "title": "GPT-4 Passes the Bar Exam",
       "authors": ["Daniel Martin Katz", "James Michael Bommarito", "Shang Gao", "Pablo Arredondo"],
       "year": 2024,
       "relevance": "Prominent evaluation of GPT-4 on the Uniform Bar Examination, key benchmark for AI legal capability claims."
     },
     {
       "title": "Experimental Evidence on the Productivity Effects of Generative Artificial Intelligence",
       "authors": ["Shakked Noy", "Whitney Zhang"],
       "year": 2023,
       "relevance": "RCT on AI productivity effects in professional writing tasks, finding greatest gains for lowest-skilled workers."
     },
     {
       "title": "Generative AI at Work",
       "authors": ["Erik Brynjolfsson", "Danielle Li", "Lindsey R. Raymond"],
       "year": 2023,
       "relevance": "Large-scale study of AI chatbot impact on customer service productivity, finding greatest gains for least-skilled workers."
     },
     {
       "title": "Lawyering in the Age of Artificial Intelligence",
       "authors": ["Jonathan H. Choi", "Amy B. Monahan", "Daniel Schwarcz"],
       "year": 2024,
       "relevance": "Follow-up RCT by same authors testing AI assistance on realistic legal tasks, the first RCT in this domain."
     },
     {
       "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng"],
       "year": 2023,
       "relevance": "Empirical evaluation of AI coding assistance productivity effects, directly relevant to AI-augmented programming research."
     },
     {
       "title": "Combining Human Expertise with Artificial Intelligence: Experimental Evidence from Radiology",
       "authors": ["Nikhil Agarwal", "Alex Moehring", "Pranav Rajpurkar", "Tobias Salz"],
       "year": 2023,
       "relevance": "Study showing human-AI collaboration can fail even when AI alone outperforms humans, relevant to understanding AI assistance limitations."
     },
     {
       "title": "Large Legal Fictions: Profiling Legal Hallucinations in Large Language Models",
       "authors": ["Matthew Dahl"],
       "year": 2024,
       "relevance": "Study on LLM hallucination rates in legal analysis, relevant to AI reliability and safety in professional contexts."
     },
     {
       "title": "Re-Evaluating GPT-4's Bar Exam Performance",
       "authors": ["Eric Martínez"],
       "year": 2024,
       "relevance": "Methodological critique of GPT-4 bar exam claims, showing inflated performance estimates due to comparison population bias."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei"],
       "year": 2022,
       "relevance": "Foundational prompting technique paper used as basis for the chain-of-thought prompting method in this study."
     }
   ]
 }