ai-research-survey

scan.json (31171B)

---

 {
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "paper": {
     "title": "Multi-Level Explanations for Generative Language Models",
     "authors": [
       "Lucas Monteiro Paes",
       "Dennis Wei",
       "Hyo Jin Do",
       "Hendrik Strobelt",
       "Ronny Luss",
       "Amit Dhurandhar",
       "Manish Nagireddy",
       "Karthikeyan Natesan Ramamurthy",
       "Prasanna Sattigeri",
       "Werner Geyer",
       "Soumya Ghosh"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2403.14459",
     "doi": "10.48550/arXiv.2403.14459"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'We open-source code for MExGen as part of the ICX360 toolkit: https://github.com/IBM/ICX360.' A working repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation datasets are publicly available: XSUM (MIT license), CNN/DailyMail (Apache-2.0), and SQuAD (CC BY-SA 4.0). The paper uses these standard benchmarks without modification."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C.6 describes the hardware environment (V100/A100 GPUs, 32 GB CPU memory) but does not provide software dependency specifications such as requirements.txt, library versions, or Dockerfile."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The code is open-sourced but the paper itself does not include instructions for reproducing the experimental results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Perturbation curve figures include shading showing 'one standard error above and below' (Section 4.2, Figures 4, 5). Standard errors are consistently reported across all perturbation curve visualizations."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The user study employs binomial tests (p < .05), paired t-tests, Bradley-Terry models, repeated ANOVA, and Bonferroni-adjusted pairwise comparisons (Tables 6-7, Appendix E)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "AUPC values are reported with full baselines for context (e.g., MExGen C-LIME 13.6 vs P-SHAP 9.4 on XSUM/DistilBART in Table 1). User study proportions are reported (e.g., '57% to 35%' for BERT vs Log Prob fidelity perception)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification given for sample sizes: 1000 test examples for DistilBART, 500 for Flan-UL2/Llama-3, 88 participants in user study. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard errors are reported via shading in all perturbation curve figures: 'Shading shows standard error in the mean' (Figures 4, 5, 8-12)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against PartitionSHAP, CaptumLIME, and LLM self-explanations (DeepSeek-V3 and Granite-3.3). Section 4.1 describes the baseline setup in detail."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "PartitionSHAP is from the SHAP library (ongoing development), CaptumLIME from Captum (2023), and DeepSeek-V3 is a very recent LLM used for self-explanation comparison. Baselines represent current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically compares three attribution algorithms (LOO, C-LIME, L-SHAP) and six scalarizers (Log Prob, BERT, Sim, SUMM, BART, Log NLI) as different instantiations of the framework. Section 4.2 evaluates scalarizer choices and Section 4.3 compares explainer algorithms."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple evaluation scalarizers are used (Log Prob, BERTScore, SUMM), along with AUPC, Spearman rank correlation, cosine similarity between scalarizer explanations, and human evaluation (fidelity perception, preference)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes a user study with 88 ML practitioners evaluating explanation quality. Participants assessed fidelity, preference, and granularity of attribution methods."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Evaluation uses standard test/validation splits: 'first 1000 test set examples' for XSUM/CNN-DM (Section 4.1), '1000 validation set examples at random from SQuAD.' No tuning is performed on the evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (XSUM, CNN/DM, SQuAD), by model (DistilBART, Flan-UL2, Llama-3, Flan-T5-Large, DeepSeek-V3, Granite-3.3), and by scalarizer. Table 1 provides full per-dataset, per-model AUPC values."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure case analysis is provided showing where MExGen produces misleading or incorrect explanations. The paper acknowledges the CNN/DM/Flan-UL2 exception (Appendix D.2) but does not analyze what makes specific examples fail."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The (CNN/DM, Flan-UL2) pair is reported as an exception where P-SHAP achieves higher AUPC (Table 1, Figure 11b). Appendix F reports that word infilling with BERT showed no 'quantifiable benefit.' The gap between MExGen and self-explanation is shown to narrow with DeepSeek-V3."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims MExGen provides 'more faithful explanations' than alternatives including LLM self-explanations, which is supported by AUPC comparisons in Tables 1 and 2 and perturbation curves in Figures 5 and 12. The code release claim is supported by the GitHub link."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (e.g., 'C-LIME is more effective' than LIME) are justified through controlled comparisons: 'This is a direct comparison between LIME (represented by CaptumLIME) and our modification C-LIME, using the same number of model queries and input segmentation' (Section 4.3). Single-variable ablations are adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly states: 'the results reported in Section 4 could still change in other experimental settings.' Results are tied to specific models and datasets. The paper tests across multiple datasets and models rather than claiming universal superiority."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses why P-SHAP might outperform at higher token fractions: 'P-SHAP perturbs larger subsets of the input than MExGen... These larger subsets may enable P-SHAP to find larger changes in output' (Appendix D.2). The limitations section discusses risks of post-hoc explanations being steered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between the proxy (perturbation curves measuring scalarizer change) and the outcome (faithful explanations). The limitations section acknowledges: 'Our user study analyzes the perception of participants of how well a method explains the predictions of a model, and not necessarily the fidelity of the explanation itself.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are provided: 'distilbart-xsum-12-6' with HuggingFace link, 'Llama-3-8B-Instruct,' 'Flan-T5-Large' with HuggingFace link, 'Flan-UL2' (20B parameters), 'DeepSeek-V3,' and 'Granite-3.3-8B-Instruct' (Section 4.1, Appendix C.2)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided: system prompts for summarization and QA are quoted in Appendix C.2, and the self-explanation prompt template is shown in Figure 6 with all fill values specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C details all hyperparameters: greedy decoding, max_new_tokens settings, C-LIME parameters (n/d ratios, K values), L-SHAP parameters (M, K), multi-level refinement thresholds (k, ϕ), and scalarizer model choices."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. MExGen is a post-hoc explanation framework that queries an LLM with perturbed inputs and analyzes outputs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents preprocessing: linguistic segmentation via spaCy v3.6, phrase segmentation algorithm (Appendix B.2), units marked as not of interest (punctuation, prompt templates), and dataset selection details (Appendix C.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section discusses four specific limitations: post-hoc explanations are local only, results may change in other settings, user study measures perception not fidelity, and risk of obfuscation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: 'results reported in Section 4 could still change in other experimental settings,' 'our user study analyzes the perception... not necessarily the fidelity,' and 'post hoc explanations come with the risk of being steered to obfuscate undesirable behavior' with specific mitigations proposed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations explicitly state what results do NOT show: explanations 'do not fully characterize how models generate output and only provide local explanations.' The paper also bounds scope to 'context-grounded tasks like summarization and question answering.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw experimental results (individual perturbation curve data, per-example attribution scores, user study responses) are not released. Only aggregate results are reported in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Datasets are well-documented public benchmarks (XSUM, CNN/DM, SQuAD) with licenses stated (Appendix C.1). Selection criteria are explicit: 'first 1000 test set examples,' '1000 validation set examples at random from SQuAD because selecting the first 1000 examples yielded insufficient diversity.'"
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "User study recruitment is described: 'We recruited participants from a large technology company who self-identify as machine learning practitioners using language models and collected data from 88 of them after filtering' (Section 5, Appendix E.1)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents the full pipeline: dataset selection and licensing (C.1), LM inference settings (C.2), MExGen configuration (C.3), baseline configuration (C.4), and perturbation curve computation (C.5)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgments section states: 'The work of Lucas Monteiro Paes was supported by the Apple Scholars in AI/ML Fellowship.' Other authors are IBM Research employees with affiliations listed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Harvard University, IBM Research, and Merck Research Labs. The paper is open about the IBM connection to the ICX360 toolkit where MExGen is released."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Apple fellowship and IBM Research general funding do not have a direct financial stake in MExGen outperforming specific baselines. The paper evaluates a general explanation framework rather than a commercial IBM product."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is found in the paper. IBM authors could hold patents related to the ICX360 toolkit, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the models used (Flan-UL2, Llama-3, DeepSeek-V3, Granite-3.3). DistilBART was trained on the evaluation datasets by design, but this is not discussed as a contamination concern."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. DistilBART is explicitly trained on XSUM/CNN-DM (evaluation datasets), and the API models (Llama-3, DeepSeek-V3) could have seen these public benchmarks during training. Neither issue is addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "XSUM (2018), CNN/DM (2015/2017), and SQuAD (2016) are all publicly available benchmarks predating the training of all models used. No discussion of contamination risk from these benchmarks appearing in model training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the user study (no OSF, AsPredicted, or similar link)."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The paper states: 'Although a formal IRB process does not exist in our institution, we went through an equivalent informal process, including reviewing the study with our peers.' No formal IRB or ethics board approval is reported."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the survey collected demographic information (job title, work location, English proficiency, LLM usage frequency — Appendix E.5), the paper does not report the actual demographic breakdown of the 88 participants beyond stating they are ML practitioners from 'a large technology company.'"
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Eligibility criteria are stated: 'Those who self-identified as machine learning practitioners using language models were eligible for the study. We filtered out 8 participants who did not pass eligibility checks or did not provide valid responses' (Appendix E.1)."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Randomization is described: 'The presentation order of the attribution algorithms was randomized to mitigate order effects.' Input texts were 'randomly drawn from the ten examples' (Section 5)."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly describe blinding. While the survey appears to show explanations without method labels (Figures 14-15), blinding is not explicitly discussed as a design consideration."
       },
       "attrition_reported": {
         "applies": true,
         "answer": true,
         "justification": "'We recruited 96 participants... We filtered out 8 participants who did not pass eligibility checks or did not provide valid responses, resulting in data from 88 participants' (Appendix E.1)."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Computational cost is discussed in terms of number of LLM inferences (the dominant cost factor) and total computation time: 'The total computation time is estimated to be on the order of 1000 hours' (Appendix C.6). The paper also discusses how cost scales linearly with input units."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.6: 'Experiments were run on a computing cluster providing nodes with 32 GB of CPU memory, V100 GPUs with 32 GB of GPU memory, and occasionally A100 GPUs with 40 or 80 GB of GPU memory. One CPU and one GPU were used at a time. The total computation time is estimated to be on the order of 1000 hours.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of running experiments across multiple random seeds. The automated evaluation aggregates across examples but does not report seed sensitivity. For API models, they 'fixed the LLM's random seed' (Appendix C.2) for determinism but do not test sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Standard errors are computed across examples (1000 or 500 depending on the model), not across multiple independent runs of the same experiment."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters (K, M, n/d ratios, ϕ thresholds) are reported but no search budget is stated. It is unclear how these values were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Rather than selecting a single best configuration, the paper reports results for all tested instantiations (LOO, C-LIME, L-SHAP) and all scalarizers. Parameter choices are motivated (e.g., 'a ratio n/d of 5 or 10 can yield good results,' Section 3.2)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Bonferroni correction is applied in the user study: 'Significant p-values after Bonferroni adjustment are noted with ** (p<0.05/3)' (Tables 6-7, Appendix E.3)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own MExGen framework against baselines (P-SHAP, CaptumLIME) without acknowledging potential author-evaluation bias. They do use existing baseline implementations rather than reimplementing baselines, which partially mitigates this."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While baselines are given matched or greater compute budgets ('P-SHAP and CaptumLIME were allocated the same or slightly more inferences,' Section 4.3), performance is not shown as a function of compute budget with systematic variation."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether perturbation curves (the primary evaluation metric) truly measure explanation faithfulness, or whether the selected benchmarks (XSUM, CNN/DM, SQuAD) are appropriate for evaluating explanation methods. The limitations note that the user study measures 'perception' not fidelity, but this gap is not deeply analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. MExGen is a post-hoc explanation framework, not a scaffolded agent system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The evaluation benchmarks (XSUM 2018, CNN/DM 2015/2017, SQuAD 2016) substantially predate the training of the models used (Llama-3, DeepSeek-V3), but temporal leakage implications for explanation quality are not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of whether the evaluation setup (e.g., using model-generated outputs as target outputs) introduces any information leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. No analysis of independence between training and evaluation data, particularly relevant since DistilBART was trained directly on the XSUM and CNN/DM evaluation datasets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "MExGen provides more faithful explanations than baseline methods (PartitionSHAP, CaptumLIME) as measured by perturbation curves and AUPC.",
       "evidence": "Table 1 shows MExGen instantiations achieve higher AUPC than P-SHAP across 7 of 8 dataset-model pairs. Figure 5 shows perturbation curves where MExGen rises more quickly, indicating better identification of important input units. Section 4.3.",
       "supported": "strong"
     },
     {
       "claim": "MExGen outperforms LLM self-explanations in faithfulness, including self-explanations from powerful LLMs like DeepSeek-V3.",
       "evidence": "Table 2 shows MExGen AUPC exceeds self-explanation AUPC in all cases. 'In all cases, MExGen is more faithful to the LLM's behavior (as measured by higher AUPC) than the LLM's self-explanations.' The gap narrows with the larger DeepSeek-V3 model on CNN/DM (14.1 vs 13.5). Section 4.4, Figure 12.",
       "supported": "strong"
     },
     {
       "claim": "The BERT scalarizer (text-only) can approximate Log Prob scalarizer (logit-based) performance, and users perceive BERT as more faithful than Log Prob.",
       "evidence": "Figure 3 shows BERT has the highest Spearman correlation with Log Prob among text-only scalarizers (0.58). User study: 'Significantly more participants perceived BERT to be higher in fidelity than Log Prob (57% to 35%)' with p < .05 (Section 5, Tables 4-5).",
       "supported": "strong"
     },
     {
       "claim": "C-LIME is perceived as more faithful and preferred by users over L-SHAP.",
       "evidence": "Bradley-Terry model shows significant difference between C-LIME and L-SHAP for both perceived fidelity (p = 0.011) and preference (p = 0.007), with Bonferroni adjustment. This is notable because the two performed similarly in automated evaluation. Section 5, Tables 6-7.",
       "supported": "strong"
     },
     {
       "claim": "Text-only scalarizers can provide effective explanations even without access to model logits, implying no loss from text-only API access in some cases.",
       "evidence": "Figure 5 shows MExGen C-LIME with the BERT scalarizer (text-only) can outperform P-SHAP (which uses logits) in perturbation curve fidelity. User study shows BERT preferred over Log Prob. Section 6: 'in some cases, there may be no loss in having text-only access compared to full logit access.'",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MExGen extends perturbation-based attribution methods (LIME, SHAP) to generative LLMs for context-grounded tasks, using scalarizers to handle text outputs and multi-level linguistic segmentation for long inputs. Across summarization (XSUM, CNN/DM) and QA (SQuAD) tasks with six models, MExGen instantiations (LOO, C-LIME, L-SHAP) consistently produce more faithful explanations than PartitionSHAP, CaptumLIME, and LLM self-explanations, as measured by perturbation curve AUPC. A user study with 88 ML practitioners found that the text-only BERT scalarizer was perceived as more faithful than the logit-based Log Prob scalarizer, suggesting text-only API access may suffice for explanation.",
   "red_flags": [
     {
       "flag": "Self-evaluation without bias acknowledgment",
       "detail": "IBM Research authors evaluate their own MExGen framework (released as part of IBM's ICX360 toolkit) against baselines without acknowledging potential author-evaluation bias. While they use existing implementations for baselines rather than reimplementing them, the selection of evaluation metrics and experimental design still favors the proposed method."
     },
     {
       "flag": "DistilBART trained on evaluation data",
       "detail": "One of the primary models (DistilBART) was explicitly trained on the XSUM and CNN/DM evaluation datasets. While this is by design for a summarization model, it is not discussed how this affects explanation quality evaluation — the model's familiarity with training data could systematically affect attribution patterns compared to unseen data."
     },
     {
       "flag": "No seed sensitivity analysis",
       "detail": "Automated experiments do not report results across multiple random seeds. C-LIME and L-SHAP involve randomized perturbation sampling, so results could vary across seeds, but this variability is not quantified."
     }
   ],
   "cited_papers": [
     {
       "title": "Are self-explanations from large language models faithful?",
       "authors": ["Andreas Madsen", "Sarath Chandar", "Siva Reddy"],
       "year": 2024,
       "relevance": "Directly evaluates faithfulness of LLM self-explanations for in-context classification, finding limitations that MExGen addresses for generative tasks."
     },
     {
       "title": "Can large language models explain themselves? A study of LLM-generated self-explanations",
       "authors": ["Shiyuan Huang", "Siddarth Mamidanna", "Shreedhar Jangam", "Yilun Zhou", "Leilani H. Gilpin"],
       "year": 2023,
       "arxiv_id": "2310.11207",
       "relevance": "Studies LLM self-explanation capabilities, showing they may be less faithful than external attribution methods like MExGen."
     },
     {
       "title": "Language models don't always say what they think: Unfaithful explanations in chain-of-thought prompting",
       "authors": ["Miles Turpin", "Julian Michael", "Ethan Perez", "Samuel R. Bowman"],
       "year": 2023,
       "relevance": "Demonstrates unfaithfulness of chain-of-thought explanations, motivating external attribution methods for LLM explainability."
     },
     {
       "title": "Measuring faithfulness in chain-of-thought reasoning",
       "authors": ["Tamera Lanham", "Anna Chen", "Ansh Radhakrishnan"],
       "year": 2023,
       "arxiv_id": "2307.13702",
       "relevance": "Measures faithfulness of chain-of-thought reasoning, relevant to understanding when LLM self-explanations can be trusted."
     },
     {
       "title": "ContextCite: Attributing model generation to context",
       "authors": ["Benjamin Cohen-Wang", "Harshay Shah", "Kristian Georgiev", "Aleksander Madry"],
       "year": 2024,
       "arxiv_id": "2409.00729",
       "relevance": "Concurrent work extending LIME to generative models for context attribution, but operates at single predefined granularity unlike MExGen's multi-level approach."
     },
     {
       "title": "The Llama 3 herd of models",
       "authors": ["Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Describes Llama-3-8B-Instruct used as one of the primary evaluation models in the paper."
     },
     {
       "title": "A survey on hallucination in large language models: Principles, taxonomy, challenges, and open questions",
       "authors": ["Lei Huang", "Weijiang Yu", "Weitao Ma"],
       "year": 2023,
       "relevance": "Surveys LLM hallucination — MExGen addresses the related problem of verifying whether LLM outputs are grounded in input context."
     },
     {
       "title": "Chain of thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational work on chain-of-thought prompting, whose faithfulness limitations motivate external attribution methods."
     },
     {
       "title": "TextGenSHAP: Scalable post-hoc explanations in text generation with long documents",
       "authors": ["James Enouen", "Hootan Nakhost", "Sayna Ebrahimi", "Sercan O. Arik", "Yan Liu", "Tomas Pfister"],
       "year": 2023,
       "arxiv_id": "2312.01279",
       "relevance": "Proposes efficient SHAP estimation for text generation using speculative decoding, a baseline approach that requires logit access unlike MExGen's text-only capability."
     },
     {
       "title": "Semantic uncertainty: Linguistic invariances for uncertainty estimation in natural language generation",
       "authors": ["Lorenz Kuhn", "Yarin Gal", "Sebastian Farquhar"],
       "year": 2023,
       "relevance": "Develops uncertainty estimation for LLM generation using semantic equivalence, related to MExGen's use of semantic similarity for scalarization."
     }
   ]
 }