ai-research-survey

scan.json (29436B)

---

 {
   "paper": {
     "title": "Learning \"Partner-Aware\" Collaborators in Multi-Party Collaboration",
     "authors": ["Abhijnan Nath", "Nikhil Krishnaswamy"],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2510.22462",
     "doi": "10.48550/arXiv.2510.22462"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our code is available at https://github.com/csu-signal/ICR' in Section 1 (footnote 2)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: DeliData (Karadzhov et al., 2023) and the Weights Task (Khebour et al., 2024a). Bootstrap dialogues are drawn from these public sources."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section D.1 mentions specific libraries (PEFT, TRL, bitsandbytes, SFTTrainer) and hardware (NVIDIA A100), but no requirements.txt, Dockerfile, or comprehensive environment specification with library versions is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Code to run the experiment will be provided in supplementary material' but no step-by-step reproduction instructions are included in the paper itself. The GitHub repo is referenced but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports ± standard error for all metrics across all baselines (e.g., 'ICR 14.06±0.13'). Table 2 similarly reports ± values."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Despite claiming ICR 'consistently outperforms all baselines,' no statistical significance tests (t-tests, bootstrap tests, etc.) are reported. Comparisons are based solely on point estimates with standard errors."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., 'ICR agents achieve an high accuracy of 14.06, which represents a dramatic 47% improvement over the next best performer (DPO, at 9.56)' in Section 6."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 100 dialogues per task for evaluation (50 for supplementary experiments) but provides no justification for why these sample sizes are sufficient. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports standard error across 100 collaboration trials. Figure 1b shows ablation results across 3 random seeds. Section 7 (NeurIPS checklist) states 'we report the standard error over 100 collaboration trials.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes five baselines: BC-COLLABORATOR, DPO, IPO, PPO, and PSO-INTENT. Section A adds additional baselines including GPT-4o-based comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DPO (Rafailov et al., 2024b), IPO (Azar et al., 2024), and PPO (Schulman et al., 2017), all standard and contemporary for LLM alignment. PSO-INTENT (Ward et al., 2023) is also recent and relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1b presents an ablation on λ_Intent values (0.01, 0.2, 1.0) across 3 random seeds over 8k training steps. Section A includes ablations: ICR-Masked, ICR-Small, ICR-Phrasing, PPO-CF."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports ACC (accuracy) and CG (common ground gain) for DeliData, and ACC for Weights Task. Figure 1a further breaks down results by proposition type (equality, inequality, order)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section D.4 describes human validation where two annotators evaluated 200 interventions (100 pairs) across both tasks, measuring agreement with LLM-Judge rankings (Cohen's κ = 0.92 on DeliData, κ = 0.58 on Weights Task)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section C states 'training/evaluation splits for both datasets are consistent with prior work [Nath et al., 2024].' Evaluation uses 100 dialogues initialized from bootstrap dialogues, separate from training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1a provides per-relation-type breakdowns (equality, inequality, order propositions) for the Weights Task. Table 1 breaks down by full-press vs. no-press conditions and by task."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section E discusses adoption effects including a 'Misleading Intervention → Poor Outcome' example and an 'Ignored Intervention' example showing cases where the approach produces suboptimal results."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that BC-COLLABORATOR achieves negative CG values (-0.13, -0.15), showing that behavior cloning reduces solution diversity. Footnote 8 mentions that 'rewarding agents with a consensus signal is counterproductive.' PSO-Skeptical shows degraded performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims ICR 'is more capable of promoting successful CG convergence and exploring more diverse solutions,' which is supported by Table 1 showing ICR outperforming all baselines on both ACC and CG metrics across both tasks."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about ICR's counterfactual regularization causing improved performance. The ablation study (Fig 1b, varying λ_Intent) and controlled comparisons (PPO-CF vs ICR in Table 2) provide evidence for the causal mechanism through controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Multi-Party Collaboration' generally but results are limited to two specific tasks (Wason Card Selection and Weights Task) with one base model (Llama-3-8B-Instruct) and one fixed intervention agent (GPT-4o). The limitations section acknowledges this ('we could only test our method on two collaborative domains') but the title and abstract framing is broader."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section A discusses alternative explanations: GPT-4o's potential prior exposure to DeliData tasks, the effect of shared underlying distributions when GPT-4o serves as both agents, LLM limitations with negation (for PSO-Skeptical results), and whether prompt augmentation alone (PPO-CF) can explain results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes proxy training rewards (task-specific accuracy) from gold evaluation rewards (composite of correctness and common ground). Section D.2 describes this in detail: 'we compute a composite reward of task-specific accuracy and common-ground convergence since this accurately measures the quality of the collaborator, and therefore can be treated as the \"gold reward.\"'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Meta-Llama-3-8B-Instruct' and 'GPT-4o' but provides no API version or snapshot date for GPT-4o. 'GPT-4o' is a marketing name that changes over time."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The appendix (Figs. 2-10) provides complete prompt text for all experimental conditions: expert collaborator prompts, intervention agent prompts, full-press and no-press collaborator prompts, counterfactual prompts, and final submission prompts. Table 4 lists all counterfactual prefix variants."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 provides detailed hyperparameters: LoRA α=16, dropout=0.05, rank R=8, 4-bit quantization, AdamW optimizer, cosine scheduler, weight decay 0.05, 100 warmup steps, β=0.1 for DPO/IPO, learning rates, batch sizes, max token lengths. GPT-4o sampling: T=0, top-p=0.9."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The MAMDP interaction framework is described in detail in Section 3: turn-taking between collaborator and intervention agents over T=15 turns, with the collaborator responding to interventions. Algorithm 1 details the full training pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section C describes data collection: bootstrap dialogues from original task datasets, expert trajectories via GPT-4o roleplay, parsing of per-participant utterances, and preference data generation (West-of-N sampling for full-press, synthetic swaps for no-press). Section D.2 details the reward computation pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations and Future Work' section appears after Section 7 (Conclusion), with substantial discussion of constraints including compute budgets, fixed intervention agent, limited domains, and lack of human collaboration data."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section identifies specific threats: only 8B-scale models tested due to compute budgets, fixed GPT-4o as intervention agent (real-world interventions vary), only two collaborative domains tested, and lack of LLM-scale human-collaboration data as a bottleneck."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what was NOT tested: larger models, human interventions, more domains (e.g., Diplomacy), ad hoc collaborators, centralized coordination, multimodal interaction. The limitations section asks 'how would ICR perform in more challenging domains like Diplomacy?'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The expert trajectory data generated via GPT-4o is not released. Only the code repository is provided. The underlying DeliData and Weights Task datasets are public, but the generated MAMDP interaction trajectories used for training are not made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 and Section C describe data collection in detail: GPT-4o generates both intervener and collaborator responses over 15 turns, using specific prompts (Figs. 2-6), with personality traits sampled from a pool (Table 3), and bootstrap dialogues from original task datasets."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section D.4 describes human annotator recruitment: 'Two human annotators—both fluent English-speaking college undergraduates.' For data generation, the expert model (GPT-4o) and its configuration are described. No human participants were recruited for the main study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 documents the full data pipeline: Phase 1 collects expert trajectories, then per-participant utterances are parsed from continuations (Section C), preference data is generated via West-of-N sampling or synthetic swaps, and Phase 2 trains using PPO with the documented loss function."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists funding from DARPA (HR00112490377), NSF (DRL 2019805, DRL 2454151, IIS 2303019), ARO (W911NF-25-1-0096), and ARPA-H (1AY2AX000062)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are from the SIGNAL Lab, Department of Computer Science, Colorado State University. No commercial product is being evaluated—the paper proposes a new method. Affiliations are clearly stated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from government agencies (DARPA, NSF, ARO, ARPA-H) which have no financial stake in whether ICR outperforms baselines. The paper includes 'Views expressed herein do not reflect the policy or position of the National Science Foundation, the Department of Defense, or the U.S. Government.'"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. While this is an academic paper from a university, the absence of an explicit declaration is noted."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-4o for data generation and as the fixed intervention agent but does not state GPT-4o's training data cutoff date. It also uses Llama-3-8B-Instruct without stating its training cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section A acknowledges GPT-4o's 'extensive pretraining on reasoning tasks, potentially including exposure to DeliData or DeliData-like problems' and discusses this as a confounder for the GPT-4o baseline results."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper acknowledges GPT-4o may have seen DeliData, it does not address whether the Llama-3-8B-Instruct model may have been exposed to these tasks during pretraining, nor does it apply any contamination detection methods."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study. The human validation in D.4 is a small annotation task, not a human subjects experiment."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study. The NeurIPS checklist confirms 'No human experiments are conducted in our work.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the main study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or API cost is reported despite extensive use of GPT-4o for data generation and as the fixed intervention agent during evaluation across 100+ dialogues per condition."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section D.1 states: 'Training standard baselines for 2,000 steps typically required around 12 GPU hours, while PPO models—trained over 6,000 mini-batches with an effective batch size of 8—took approximately 24 hours to converge.' Hardware is specified as NVIDIA A100 GPUs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1b shows ablation results 'across 3 random seeds.' The shaded regions in the figure show variance across seeds. Table 1 reports standard error across 100 dialogue trials."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states '100 DeliData and 100 Weights Task dialogues' for evaluation. Section 6 confirms 'over 100 collaboration runs.' Figure 1b states '3 random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section D.1 mentions 'Based on early validation experiments on the DeliData task, we found β = 0.1 to yield consistently strong performance' but does not report how many configurations were tried or the search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "For λ_Intent, Figure 1b shows results across three values (0.01, 0.2, 1.0) with the selection of 0.2 justified by the learning curves. For β, D.1 states it was selected via 'early validation experiments on the DeliData task.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares ICR against 5 baselines across multiple metrics and conditions without applying any multiple comparison correction (Bonferroni, etc.)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (BC, DPO, IPO, PPO, PSO-INTENT) themselves and compare against their own ICR method, but do not acknowledge the bias of evaluating their own system against their own implementations of baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "ICR adds an additional forward pass per sample compared to PPO (Section 4, Computational Cost) but no performance-vs-compute curve is provided. The paper does not compare methods at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses why DeliData (Wason Card Selection) and Weights Task are appropriate for testing collaborative reasoning, and Section 5 explains the choice of evaluation metrics (ACC, CG) and their relationship to the claimed capabilities."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All baselines use the same base model (Llama-3-8B-Instruct), same intervention agent (GPT-4o), same evaluation setup, and same prompt structure. The only variable is the training algorithm. The scaffold is controlled across comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section A acknowledges 'GPT-4o's extensive pretraining on reasoning tasks, potentially including exposure to DeliData or DeliData-like problems' as a confounder, constituting awareness of temporal leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, the system prompts for evaluation include task-specific information, but no analysis of whether this constitutes feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The bootstrap dialogues for evaluation are drawn from the same datasets used for training data collection. No discussion of whether train and test dialogues share structural similarities or are independent."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied (no canary strings, membership inference, n-gram overlap analysis, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "ICR-trained collaborators achieve 47% improvement over the next best performer (DPO) on the Weights Task in full-press conditions (14.06 vs 9.56 ACC).",
       "evidence": "Table 1, Section 6. Results averaged over 100 dialogues with 15 turns each, reported with standard error.",
       "supported": "strong"
     },
     {
       "claim": "Standard RL/preference alignment methods (PPO, DPO) are Bellman-optimal for the underlying MDP but suboptimal in the MAMDP setting.",
       "evidence": "Theorem 3.2 and proof in Theorem B.3, building on Langlois and Everitt (2021). Empirical validation in Table 1 showing all standard baselines underperform ICR.",
       "supported": "strong"
     },
     {
       "claim": "ICR's performance improvements hold across both language-rich (full-press) and language-free (no-press) conditions.",
       "evidence": "Table 1 shows ICR outperforms all baselines in both full-press and no-press settings for both tasks. No-press: 10.87 ACC Weights (vs 7.81 PPO), 0.85 ACC DeliData (vs 0.79 DPO).",
       "supported": "strong"
     },
     {
       "claim": "Llama-3-8B-Instruct trained with ICR performs comparably with GPT-4o acting as both agents.",
       "evidence": "Table 2, Section A. ICR (14.06 Weights, 0.88 DeliData) vs GPT-4o paired (15.23 Weights, 0.91 DeliData). Close but GPT-4o still higher.",
       "supported": "moderate"
     },
     {
       "claim": "λ_Intent = 0.2 provides optimal learning, with too low (0.01) or too high (1.0) values degrading performance.",
       "evidence": "Figure 1b ablation over 8k training steps with 3 random seeds on DeliData no-press variant.",
       "supported": "moderate"
     },
     {
       "claim": "Common ground convergence emerges as an emergent property of counterfactual training without explicitly training for it.",
       "evidence": "Section 6: ICR agents trained with only proxy task rewards (no CG signal) achieve highest CG scores in Table 1 (3.35 full-press, 3.18 no-press on DeliData).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "ICR (Interruptible Collaborative Roleplayer) introduces counterfactual KL-divergence regularization to train LLM-based collaborators that selectively incorporate helpful interventions while resisting misleading ones. On the DeliData Wason Card Selection task and Weights Task, ICR consistently outperforms BC, DPO, IPO, PPO, and PSO-INTENT baselines in both task accuracy and common ground convergence. Notably, common ground convergence emerges without explicit CG-based training rewards, supporting the hypothesis that counterfactual robustness induces partner-aware collaboration. A Llama-3-8B model trained with ICR performs comparably to GPT-4o paired with itself.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "Despite strong claims of superiority ('consistently outperforms all baselines'), no statistical significance tests are reported. Comparisons rely on point estimates with standard errors. Given the variability across runs, some comparisons (e.g., DPO vs IPO vs PPO in DeliData) may not be statistically significant."
     },
     {
       "flag": "Fixed intervention agent introduces confound",
       "detail": "All evaluation uses a single fixed GPT-4o instance as the intervention agent. Results may not generalize to other intervention sources. The paper acknowledges this in limitations but the main claims are not bounded to this setting."
     },
     {
       "flag": "GPT-4o version unspecified",
       "detail": "GPT-4o is used extensively for data generation, as the fixed intervention agent, and as the LLM-Judge for reward modeling, but no API version or snapshot date is provided. GPT-4o behavior changes across versions."
     }
   ],
   "cited_papers": [
     {
       "title": "How RL agents behave when their actions are modified",
       "authors": ["Eric D Langlois", "Tom Everitt"],
       "year": 2021,
       "relevance": "Core theoretical foundation for the MAMDP framework used in ICR; proves Bellman-optimal policies are suboptimal in MAMDPs."
     },
     {
       "title": "Honesty is the best policy: defining and mitigating AI deception",
       "authors": ["Francis Ward", "Francesca Toni", "Francesco Belardinelli", "Tom Everitt"],
       "year": 2023,
       "relevance": "Provides the 'intentionality' concept and Path-Specific Objectives that ICR extends for counterfactual regularization in collaborative settings."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Christopher D Manning", "Stefano Ermon", "Chelsea Finn"],
       "year": 2024,
       "relevance": "Key baseline method; ICR demonstrates limitations of DPO in collaborative MAMDP settings."
     },
     {
       "title": "Safely interruptible agents",
       "authors": ["Laurent Orseau", "M Armstrong"],
       "year": 2016,
       "relevance": "Foundational work on safe interruptibility in RL that ICR builds upon for collaborative agent design."
     },
     {
       "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training",
       "authors": ["Evan Hubinger", "Carson Denison", "Jesse Mu"],
       "year": 2024,
       "relevance": "Cited in safety discussion about risks of partner-aware LLMs potentially being used for covert collusion or manipulation."
     },
     {
       "title": "Alignment faking in large language models",
       "authors": ["Ryan Greenblatt", "Carson Denison", "Benjamin Wright"],
       "year": 2024,
       "relevance": "Cited regarding risks that partner-aware agents could fake alignment; relevant to AI safety evaluation."
     },
     {
       "title": "Social influence as intrinsic motivation for multi-agent deep reinforcement learning",
       "authors": ["Natasha Jaques", "Angeliki Lazaridou", "Edward Hughes"],
       "year": 2019,
       "relevance": "Prior work on multi-agent RL with social influence that ICR extends by addressing intervention quality discrimination."
     },
     {
       "title": "Delidata: A dataset for deliberation in multi-party problem solving",
       "authors": ["Georgi Karadzhov", "Tom Stafford", "Andreas Vlachos"],
       "year": 2023,
       "relevance": "Primary evaluation dataset; provides the Wason Card Selection task dialogues used for both training and evaluation."
     },
     {
       "title": "Language instructed reinforcement learning for human-ai coordination",
       "authors": ["Hengyuan Hu", "Dorsa Sadigh"],
       "year": 2023,
       "relevance": "InstructRL method for human-AI coordination that ICR builds upon for incorporating human priors."
     },
     {
       "title": "From r to Q*: Your Language Model is Secretly a Q-Function",
       "authors": ["Rafael Rafailov", "Joey Hejna", "Ryan Park", "Chelsea Finn"],
       "year": 2024,
       "relevance": "Provides token-MDP framework showing DPO performs credit assignment; theoretical foundation for ICR's analysis of preference-aligned policy limitations."
     },
     {
       "title": "Path-specific objectives for safer agent incentives",
       "authors": ["Sebastian Farquhar", "Ryan Carey", "Tom Everitt"],
       "year": 2022,
       "relevance": "Introduces Path-Specific Objectives used to assign 'intention' to parametric agents; ICR extends this for collaborative settings."
     }
   ]
 }