ai-research-survey

scan.json (26226B)

---

 {
   "paper": {
     "title": "Epistemic Alignment: A Mediating Framework for User-LLM Knowledge Delivery",
     "authors": ["Nicholas Clark", "Hua Shen", "Bill Howe", "Tanushree Mitra"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2504.01205",
     "doi": "10.48550/arXiv.2504.01205"
   },
   "scan_version": 3,
   "active_modules": [],
   "methodology_tags": ["qualitative", "theoretical"],
   "key_findings": "The paper proposes ten epistemic challenges across three dimensions (epistemic responsibility, epistemic personalization, testimonial reliability) derived from philosophical epistemology. Thematic analysis of 128 Reddit custom instructions shows 92.1% address at least one challenge and 80.3% address multiple, despite no standardized vocabulary. Content analysis of OpenAI and Anthropic policies reveals both providers acknowledge epistemic challenges in documentation but lack structured interface mechanisms for users to specify citation standards, uncertainty expression, or perspective balance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository or archive is provided. The paper uses GPT-4o-mini and GPT-4o for analysis but does not release any analysis scripts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The 128 custom instructions dataset extracted from Reddit is not released. Appendix B provides only examples (Table 2), not the full dataset."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification is provided. The paper mentions using GPT-4o-mini and GPT-4o but provides no software versions, API details, or reproducibility setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The Reddit API query parameters are listed in Table 1 and prompts are in Appendix A, but there are no step-by-step instructions for reproducing the full analysis pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports point estimates (92.1%, 80.3%, κ = 0.8875) without any confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Prevalence rates and inter-rater reliability are reported as descriptive statistics only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The Cohen's Kappa (κ = 0.8875) serves as an agreement measure, but no effect sizes are provided for the main findings about challenge prevalence."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample of 128 custom instructions is reported without justification for why this size is sufficient. No power analysis or saturation analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or spread measures are reported for any of the quantitative results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baseline frameworks or alternative taxonomies are systematically compared against. The related work section discusses relevant literature but does not benchmark the proposed framework against prior frameworks for coverage or completeness."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included, so contemporariness cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The contribution is a conceptual framework with ten challenges. There is no multi-component system to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The only quantitative metric reported is Cohen's Kappa for inter-rater reliability (κ = 0.8875). No additional metrics assess framework validity, coverage, or usefulness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Two human experts independently validated the GPT-4o challenge labels, achieving inter-rater reliability of κ = 0.8875 (Section 5). This constitutes human evaluation of the annotation quality."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No train/test paradigm is used. The paper performs qualitative thematic analysis, not predictive modeling."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "While the paper discusses each of the ten challenges qualitatively and provides example instructions in Table 2, no per-challenge quantitative prevalence rates are reported. The reader cannot determine which challenges are most or least frequently addressed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases of the framework are discussed. The paper does not address custom instructions that did not fit any challenge or cases where the framework's categories were ambiguous or overlapping."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "No negative results are reported. Every aspect of the analysis supports the framework. There is no discussion of challenges that were rarely observed or areas where the framework proved inadequate."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the framework identifies ten challenges, that thematic analysis finds users develop workarounds for each, and that providers fail to establish adequate mechanisms. Sections 4, 5, and 6 support these claims with the framework definition, thematic analysis results, and provider content analysis respectively."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes descriptive and evaluative claims about the state of epistemic alignment. It does not make causal claims about what causes epistemic misalignment or what interventions would improve it."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The framework is presented as general ('for AI developers' and 'for users') but the empirical validation is limited to Reddit custom instructions from 4 subreddits and two model providers (OpenAI, Anthropic). The paper does not bound its generalizations to these specific populations and platforms."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example, custom instructions addressing the ten challenges could reflect social copying rather than independent user needs. The paper does not consider whether its framework imposes categories that may not reflect actual user mental models."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Reddit custom instructions as a proxy for user epistemic preferences broadly but does not acknowledge the gap between Reddit power users who share prompting strategies and typical LLM users. The framework is presented as addressing general user needs but validated only on a self-selected online community."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'GPT-4o-mini' and 'GPT-4o' without specifying snapshot dates or API versions. These models change behavior across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix A: Prompt 1 for custom instruction extraction and Prompt 2 for identifying epistemic challenges, including complete instructions and output format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'zero-shot prompting with GPT-4o-mini' but reports no temperature, top-p, or other generation parameters for either GPT-4o-mini or GPT-4o."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The analysis pipeline consists of straightforward API calls for classification."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and Table 1 document the data pipeline: Reddit API query with specific search terms and subreddits, filtering to top-level comments >100 characters, GPT-4o-mini classification to identify custom instructions (yielding 128), followed by GPT-4o analysis and human validation."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 7 is titled 'Discussion & Conclusion' and discusses contributions and proposed interface designs but does not substantively address limitations of the work."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address the representativeness of Reddit data, the circular use of LLMs to study LLM interaction patterns, or the subjectivity in framework construction."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not identify what settings, user populations, or platform types the framework does NOT apply to."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw Reddit custom instructions dataset is not released. Only selected examples appear in Table 2."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides detailed Reddit API query parameters including search query, keyword filters, subreddits (ChatGPT, ChatGPTPro, ClaudeAI, OpenAI), time frame (past 2 years), and comment/instruction length filters."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The data sourcing is described: posts from four specific subreddits matching search terms, filtered to top-level comments >100 characters, then GPT-4o-mini classified to identify actual custom instructions. The sourcing method is clear even if selection bias is not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline stages are described (Reddit query → comment extraction → GPT-4o-mini filter → GPT-4o analysis → human validation) but intermediate counts are missing. The paper does not report how many total posts were retrieved, how many comments were extracted before the 100-character filter, or how many passed that filter before GPT-4o-mini classification."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. There is no acknowledgments section or funding statement in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are disclosed as affiliated with the University of Washington. They are evaluating OpenAI and Anthropic products as external academics, not as employees of either company."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funder from outcome cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. GPT-4o-mini and GPT-4o are used as classification tools, not as subjects of evaluation."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The models are used as annotation tools, not tested for capability."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper analyzes publicly posted Reddit content and has two expert annotators for validation, but does not conduct a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the traditional sense. The paper mines public Reddit data."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Reddit users whose posts were analyzed are anonymous and not treated as study subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data selection criteria are covered under data_integrity."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. No experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is primarily a theoretical/qualitative paper proposing a framework. The GPT-4o usage is for annotation, not a proposed method. Cost reporting is not relevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Primarily theoretical/qualitative work. Compute budget is not relevant."
       }
     }
   },
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The framework provides conceptual vocabulary but no tools or techniques a practitioner could directly implement."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that users develop elaborate workarounds is somewhat known in the prompt engineering community; the formalization into epistemic dimensions is new but not surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns are raised beyond general knowledge delivery quality."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Evaluates OpenAI and Anthropic as failing to provide adequate epistemic customization mechanisms, mild critical angle."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool is provided."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Directly evaluates ChatGPT and Claude products from OpenAI and Anthropic, well-known brands, though the authors are from UW (not a famous AI lab)."
     }
   },
   "claims": [
     {
       "claim": "92.1% of analyzed custom instructions address at least one epistemic challenge, and 80.3% address multiple challenges.",
       "evidence": "Section 5 reports these percentages from thematic analysis of 128 custom instructions extracted from Reddit, with GPT-4o classification validated by two human experts (κ = 0.8875).",
       "supported": "moderate"
     },
     {
       "claim": "All ten epistemic challenges in the framework are independently addressed by users through custom instructions and prompting strategies.",
       "evidence": "Section 5 and Table 2 (Appendix B) provide examples of custom instructions corresponding to each of the ten challenges, showing 'independent emergence of solutions to all ten challenges across diverse user instructions.'",
       "supported": "moderate"
     },
     {
       "claim": "Both OpenAI and Anthropic acknowledge epistemic challenges in documentation but fail to provide structured interface mechanisms for specifying epistemic preferences.",
       "evidence": "Sections 6.1 and 6.2 present content analysis of provider documentation. For OpenAI: 'while it mentions reliable sources, it lacks detailed mechanisms for citation verification.' For Anthropic: 'the interface still lacks dimension-specific controls for specifying citation standards.'",
       "supported": "moderate"
     },
     {
       "claim": "Users develop three prominent folk theories of model behavior: 'Suppressing Default Behavior,' 'Expert Persona,' and 'Parameter Configuration.'",
       "evidence": "Section 5 describes these three folk theories with specific examples from the custom instructions dataset (e.g., 'Avoid any language constructs that could be interpreted as expressing remorse' for suppression).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Non-representative sample",
       "detail": "The 128 custom instructions come from 4 Reddit subreddits (r/ChatGPT, r/ChatGPTPro, r/OpenAI, r/Anthropic). Reddit power users who share prompting strategies are not representative of typical LLM users. The paper generalizes to 'users' broadly without acknowledging this selection bias."
     },
     {
       "flag": "Circular LLM-assisted coding",
       "detail": "GPT-4o-mini was used to extract custom instructions from comments, and GPT-4o was used to classify which epistemic challenges each instruction addresses. Using LLMs to classify data about LLM interaction patterns introduces a circular dependency — the model may impose its own understanding of epistemic concepts rather than capturing user intent."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations section despite significant methodological choices that could affect validity: small sample, non-representative population, LLM-assisted coding, evaluation of only two providers, and reliance on publicly documented policies rather than actual system behavior."
     },
     {
       "flag": "Missing intermediate pipeline counts",
       "detail": "The data collection pipeline does not report how many Reddit posts were retrieved, how many comments were extracted, or how many passed the 100-character filter. Only the final count (128 custom instructions) is reported, making it impossible to assess the selectivity of each filtering stage."
     },
     {
       "flag": "Framework validated by its own creators",
       "detail": "The ten-challenge framework was created by the authors and then validated by the same authors finding instances in Reddit data. There is no independent evaluation of whether the framework categories are comprehensive, non-overlapping, or meaningful to actual users."
     }
   ],
   "cited_papers": [
     {
       "title": "Survey of Hallucination in Natural Language Generation",
       "authors": ["Ziwei Ji", "Nayeon Lee", "Rita Frieske", "Tiezheng Yu", "Dan Su", "Yan Xu", "Etsuko Ishii", "Yejin Bang", "Delong Chen", "Ho Shu Chan", "Wenliang Dai", "Andrea Madotto", "Pascale Fung"],
       "year": 2022,
       "arxiv_id": "2202.03629",
       "doi": "10.1145/3571730",
       "relevance": "Foundational survey on LLM hallucination, directly relevant to AI safety and reliability research."
     },
     {
       "title": "Towards Understanding Sycophancy in Language Models",
       "authors": ["Mrinank Sharma", "Meg Tong", "Tomasz Korbak", "David Duvenaud", "Amanda Askell", "Samuel R. Bowman"],
       "year": 2023,
       "arxiv_id": "2310.13548",
       "relevance": "Studies LLM sycophancy behavior (deference to user misinformation), a key AI alignment concern."
     },
     {
       "title": "The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions",
       "authors": ["Eric Wallace", "Kai Xiao", "Reimar Leike", "Lilian Weng", "Johannes Heidecke", "Alex Beutel"],
       "year": 2024,
       "arxiv_id": "2404.13208",
       "relevance": "Addresses LLM instruction following and prioritization, relevant to prompt injection and safety."
     },
     {
       "title": "Towards Bidirectional Human-AI Alignment: A Systematic Review for Clarifications, Framework, and Future Directions",
       "authors": ["Hua Shen", "Tiffany Knearem", "Reshmi Ghosh"],
       "year": 2024,
       "arxiv_id": "2406.09264",
       "relevance": "Systematic review of human-AI alignment approaches, directly relevant to LLM alignment methodology."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Brian Ichter", "Fei Xia", "Ed Chi", "Quoc Le", "Denny Zhou"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Foundational prompting technique paper, central to LLM capability and prompting research."
     },
     {
       "title": "Can Large Language Models Faithfully Express Their Intrinsic Uncertainty in Words?",
       "authors": ["Gal Yona", "Roee Aharoni", "Mor Geva"],
       "year": 2024,
       "arxiv_id": "2405.16908",
       "relevance": "Studies LLM uncertainty expression capability, relevant to AI reliability and calibration."
     },
     {
       "title": "Fine-grained Hallucination Detection and Editing for Language Models",
       "authors": ["Abhika Mishra", "Akari Asai", "Vidhisha Balachandran", "Yizhong Wang", "Graham Neubig", "Yulia Tsvetkov", "Hannaneh Hajishirzi"],
       "year": 2024,
       "arxiv_id": "2401.06855",
       "relevance": "Addresses hallucination detection and mitigation in LLMs, key AI safety research."
     },
     {
       "title": "LLMs Know More Than They Show: On the Intrinsic Representation of LLM Hallucinations",
       "authors": ["Hadas Orgad", "Michael Toker", "Zorik Gekhman", "Roi Reichart", "Idan Szpektor", "Hadas Kotek", "Yonatan Belinkov"],
       "year": 2024,
       "arxiv_id": "2410.02707",
       "relevance": "Studies internal representations of hallucinations in LLMs, relevant to understanding model reliability."
     },
     {
       "title": "The Art of Defending: A Systematic Evaluation and Analysis of LLM Defense Strategies on Safety and Over-Defensiveness",
       "authors": ["Neeraj Varshney", "Pavel Dolin", "Agastya Seth", "Chitta Baral"],
       "year": 2023,
       "arxiv_id": "2401.00287",
       "relevance": "Evaluates LLM defense strategies including over-abstention, directly relevant to AI safety and alignment."
     },
     {
       "title": "Knowledge Conflicts for LLMs: A Survey",
       "authors": ["Rongwu Xu", "Zehan Qi", "Zhijiang Guo", "Cunxiang Wang", "Hongru Wang", "Yue Zhang", "Wei Xu"],
       "year": 2024,
       "arxiv_id": "2403.08319",
       "relevance": "Surveys knowledge conflicts in LLMs, relevant to understanding LLM reliability and knowledge delivery."
     },
     {
       "title": "Enabling Large Language Models to Generate Text with Citations",
       "authors": ["Tianyu Gao", "Howard Yen", "Jiatong Yu", "Danqi Chen"],
       "year": 2023,
       "arxiv_id": "2305.14627",
       "relevance": "Addresses citation generation in LLMs, relevant to verification and reliability of AI outputs."
     },
     {
       "title": "The Art of Saying No: Contextual Noncompliance in Language Models",
       "authors": ["Faeze Brahman", "Sachin Kumar", "Vidhisha Balachandran"],
       "year": 2024,
       "arxiv_id": "2407.12043",
       "relevance": "Studies LLM refusal behavior and contextual noncompliance, relevant to AI safety alignment."
     },
     {
       "title": "A Roadmap to Pluralistic Alignment",
       "authors": ["Taylor Sorensen", "Jared Moore", "Jillian Fisher", "Mitchell Gordon"],
       "year": 2024,
       "arxiv_id": "2402.05070",
       "relevance": "Proposes framework for pluralistic AI alignment accommodating diverse perspectives, directly relevant to alignment methodology."
     },
     {
       "title": "SAFETY-TUNED LLAMAS: Lessons from Improving the Safety of Large Language Models that Follow Instructions",
       "authors": ["Federico Bianchi", "Mirac Suzgun", "Giuseppe Attanasio", "Paul Röttger", "Dan Jurafsky", "Tatsunori Hashimoto", "James Zou"],
       "year": 2024,
       "relevance": "Studies safety tuning of instruction-following LLMs, relevant to alignment and safety research."
     }
   ]
 }