ai-research-survey

scan.json (29740B)

---

 {
   "paper": {
     "title": "RESCUE: Ranking LLM Responses with Partial Ordering to Improve Response Generation",
     "authors": [
       "Yikun Wang",
       "Rui Zheng",
       "Haoming Li",
       "Qi Zhang",
       "Tao Gui",
       "Fei Liu"
     ],
     "year": 2023,
     "venue": "Annual Meeting of the Association for Computational Linguistics (Student Research Workshop)",
     "arxiv_id": "2311.09136",
     "doi": "10.18653/v1/2024.acl-srw.32"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RESCUE trains LLMs to prioritize high-quality responses using a partial ordering ranking metric combined with supervised fine-tuning. On e-SNLI, partial ordering strategies (label prioritization and human-label hybrid) outperform both full ordering and SFT alone, especially with limited training data (0.4%–3.6% of full set). On Multi-doc QA, label-prioritized ranking helps the model better identify relevant information across document positions. Human evaluation shows a 47% win rate for partial ordering over SFT on explanation quality.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in footnote 1: 'Our code and models are available at: https://github.com/ekonwang/RRescue.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: e-SNLI (Camburu et al., 2018) and NaturalQuestions-Open (Kwiatkowski et al., 2019). Both are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 4xA100 GPUs, BF16 mixed precision, and fully sharded data parallelism, but does not provide a requirements.txt, Dockerfile, or detailed software dependency list with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released and training hyperparameters are described in §6, the paper itself does not contain step-by-step reproduction instructions. A reader would need to consult the GitHub repository."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates (e.g., accuracy percentages) with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims partial ordering 'outperforms' SFT and full ordering methods based solely on comparing accuracy numbers. No statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as accuracy percentages with baseline context (e.g., Table 1 shows SFT at 77.45% vs PO Human-Label Hybrid at 82.86% at 0.4% data). Human evaluation reports win/tie/lose percentages (47%/18%/35%)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why subsets of {2k, 5k, 10k, 20k} were chosen, why 100 samples were used for human evaluation, or why 1k training examples were used for Multi-doc QA."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single experimental runs. No standard deviations, variance across seeds, or spread measures are reported in any table or figure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: SFT, FO Similarity, FO GPT-3.5-Turbo, and the base Llama-2-7b model (Table 1). For Multi-doc QA, comparison is against base Llama-2-7b (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Llama-2-7b was a contemporary model at submission time (2023). The paper also compares with Hsieh et al. (2023) step-by-step distilling results and mentions other contemporary models (Falcon, Mistral, Vicuna, MPT)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablations are conducted: PO vs FO ordering strategies (Table 1), effect of α balancing coefficient (Figure 4 left), number of candidate responses (Figure 4 right), response flipping (Table 1 right columns), and length scaling factor λ (§7)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses automatic accuracy on e-SNLI, answer accuracy on Multi-doc QA, and human evaluation with win/tie/lose ratings on explanation quality (Figure 3)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 describes human evaluation: 'An annotator evaluated responses for 100 randomly selected samples from the e-SNLI test set, using win, tie and lose to rate each response pair.' Results in Figure 3."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on the standard e-SNLI test set of 9,824 examples (§6.1) and a Multi-doc QA test set of 665 examples (§6.3), separate from training subsets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down results by training data proportion (0.4%–3.6%). Table 2 breaks down by position of gold document (1st, 3rd/5th, 5th/10th). Figure 6 shows per-label confusion matrix."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses central tendency bias in LLM predictions (§7, Figure 6), margin violations in training (§7), and model copying behavior in Multi-doc QA (§6.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Response flipping does not consistently improve results beyond 2k training examples (Table 1). Performance saturates beyond 3-4 candidate responses (Figure 4 right). Full ordering can hurt at higher data proportions (Table 1, FO Similarity drops from 86.69 to 86.38)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims partial ordering is 'more robust, less sensitive to noise' and results in 'improved response generation.' Tables 1–2 and Figure 3 support improved accuracy and response quality. The abstract appropriately hedges with 'offers a promising avenue.'"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'training an LLM to rank responses can improve response generation' are supported by controlled ablation studies comparing PO, FO, and SFT variants (Table 1, Figures 3–4), holding other factors constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame the work broadly ('Ranking LLM Responses,' 'enhancing the response generation and task accuracy of LLMs') but experiments use only Llama-2-7b on two tasks (e-SNLI and Multi-doc QA). The authors acknowledge leaving 'extension to other models such as Llama-3 for future work' in a footnote but the framing is broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions SFT may rely on 'data artifacts for predictions (Gururangan et al., 2018)' but does not substantively discuss alternative explanations for why partial ordering improves over full ordering or SFT, such as data augmentation effects, regularization from multiple responses, or selection bias in the human evaluation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on e-SNLI and answer accuracy on Multi-doc QA, and claims improvement in 'task accuracy' and 'response generation.' These measurements align with the claims without proxy gaps. Human evaluation directly assesses explanation quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'Llama-2-7b,' 'GPT-3.5-Turbo-0613,' and 'GPT-4-0613' (§5.1, §6). These include API snapshot dates or size specifications."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Only the response flipping prompt is provided ('Rewrite the sentence to convey the opposite meaning: {Explanation}'). The main prompts used to elicit responses from Llama-2-7b, GPT-3.5-Turbo for response generation, and GPT-3.5-Turbo for full ordering are not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports: AdamW with lr=2e-5, cosine scheduler with 0.03 warmup, BF16 mixed precision, batch size B=64 (g=16, b=1, D=4), single epoch training, temperature 0.8 for Llama-2-7b sampling, λ=0.85 length scaling."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RESCUE is a fine-tuning method combining SFT with a ranking loss."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes candidate response collection: 3 responses from Llama-2-7b at temperature 0.8, 1 from GPT-3.5-Turbo, 1 human explanation per e-SNLI prompt. For Multi-doc QA, 5 candidates per question (1 from gold passage, 4 from distractors). Response flipping process is described in §5.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the Conclusion, discussing domain-specific criteria needs, GPU resource constraints, and candidate response diversity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The Limitations section discusses generic future directions ('organizing candidate responses can benefit from domain-specific criteria,' 'with additional GPU resources, we can improve the variety') rather than specific threats to the validity of the current results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of which tasks, model sizes, or settings the approach may not generalize to, beyond a footnote about leaving Llama-3 extension to future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying datasets (e-SNLI, NaturalQuestions-Open) are publicly available. Code and models are released at https://github.com/ekonwang/RRescue, enabling regeneration of candidate responses."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 details candidate response collection: sources (Llama-2-7b, GPT-3.5-Turbo, human annotations), sampling parameters (temperature 0.8), number of responses per instance (5), and the response flipping procedure."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluator in §6.2 is described only as 'an annotator' with no information about who they are, their qualifications, how they were selected, or potential biases."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from public datasets through candidate response generation, ordering (PO/FO strategies), to combined SFT+ranking training is described across §3–5. Training subsets are specified ({2k, 5k, 10k, 20k})."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements state: 'HL and FL are supported in part by National Science Foundation grant IIS-2303678.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Fudan University and Emory University. They are not evaluating their own commercial product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is an independent government funding agency with no financial stake in the experimental outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for Llama-2-7b, GPT-3.5-Turbo-0613, or GPT-4-0613, despite fine-tuning and evaluating Llama-2-7b on e-SNLI."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether e-SNLI test examples (publicly available since 2018) could appear in Llama-2's pre-training data. The base Llama-2-7b already achieves 33.31% on the NLI task, suggesting some exposure."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "e-SNLI was published in 2018, well before Llama-2's training. NaturalQuestions was published in 2019. No discussion of contamination risk for either benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no human subjects study. The single annotator in §6.2 evaluates model outputs but is not a research participant."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring IRB approval. The evaluation annotator is not a research participant."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The single evaluator annotator is not described, but this is an output evaluation task, not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study with participant recruitment."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No experimental study with human participants requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study with participant attrition."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API costs are reported. The method uses GPT-3.5-Turbo and GPT-4 for candidate generation and ordering but the associated costs are not mentioned."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 4xA100 GPUs and single-epoch training but does not report total GPU hours, training wall-clock time, or API costs for GPT-3.5-Turbo/GPT-4 usage."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or results across multiple seeds. All results appear to be from single experimental runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating whether they represent single runs or averages."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While Figure 4 shows results for different α values (0, 0.01, 0.02, 0.05, 0.1, 0.2, 1), no search budget is reported (total configurations tried, compute spent on search)."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "For human evaluation, §6.2 states 'the highest performing model across all data proportions was chosen' — this is selection based on test set performance, not a held-out validation set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many pairwise comparisons across ordering strategies and data proportions (Table 1, Figure 3) without any statistical testing, let alone multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (SFT, FO Similarity, FO GPT-3.5-Turbo) and their own PO methods without acknowledging the bias of evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "FO methods require GPT-3.5-Turbo for ranking (additional API cost) while PO methods use heuristic orderings, but performance is not compared at matched compute budgets."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether e-SNLI accuracy is a valid measure of explanation quality or whether NaturalQuestions answer accuracy captures the claimed 'response generation' improvement."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. RESCUE is a fine-tuning method, not an agentic system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "e-SNLI (2018) and NaturalQuestions (2019) both predate Llama-2's training. No discussion of whether the model may have seen test examples or solutions during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For Multi-doc QA, training uses labels to determine correct/incorrect responses, but no analysis of whether this creates feature leakage at test time."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between train and test splits beyond using the standard e-SNLI train/test split."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods (canary strings, membership inference, n-gram overlap analysis) are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Partial ordering strategies outperform both full ordering and SFT for response ranking, especially with limited training data.",
       "evidence": "Table 1 shows PO Label Prioritization (86.34% avg) and PO Human-Label Hybrid (86.35% avg) exceed FO Similarity (85.15%), FO GPT-3.5-Turbo (85.14%), and SFT (84.57%) on e-SNLI. The gap is largest at 0.4% training data (82.86% vs 77.45% for SFT).",
       "supported": "moderate"
     },
     {
       "claim": "RESCUE with label prioritization achieves a 47% win rate over SFT in human evaluation of explanation quality.",
       "evidence": "Figure 3 (§6.2) shows 47% win, 18% tie, 35% lose based on a single annotator evaluating 100 randomly selected samples from the e-SNLI test set.",
       "supported": "weak"
     },
     {
       "claim": "Ranking responses allows the LLM to more effectively identify relevant information in long contexts, improving the U-shaped performance curve.",
       "evidence": "Table 2 shows PO Label Prioritization improves average accuracy from 40.96% to 46.92% (5 docs) and 38.84% to 41.42% (10 docs), with large gains in middle positions (34.19%→42.44% at 3rd position).",
       "supported": "moderate"
     },
     {
       "claim": "Optimal performance is achieved with α between 0.01 and 0.1 for balancing SFT and ranking metrics.",
       "evidence": "Figure 4 (left) shows accuracy across α values, with peak performance in the 0.01–0.1 range on e-SNLI. Single experimental configuration, no error bars.",
       "supported": "moderate"
     },
     {
       "claim": "Performance improvement can be achieved with 3-4 candidate responses, with saturation beyond that.",
       "evidence": "Figure 4 (right) shows performance by number of candidate responses, with PO performance plateauing at 3-4 candidates.",
       "supported": "moderate"
     },
     {
       "claim": "Response flipping increases answer variety but does not consistently improve performance.",
       "evidence": "Table 1 shows flipping helps only at 0.4% data (↑5.15 for PO Label Prioritization) but can hurt at 0.9% (↓0.26 for FO Similarity, ↓1.30 for PO Human Prioritization). §6.1 notes it 'might cause a shift in the distribution of ranked responses.'",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Single annotator for human evaluation",
       "detail": "Human evaluation (§6.2) relies on a single unnamed annotator evaluating 100 samples with no inter-annotator agreement reported. This makes the human evaluation results (Figure 3) unreliable — a second annotator might produce substantially different ratings."
     },
     {
       "flag": "No error bars or variance on any result",
       "detail": "All results in Tables 1–2 and Figure 4 are point estimates from apparently single experimental runs. Without variance information, it is impossible to assess whether the differences between methods are meaningful or within noise."
     },
     {
       "flag": "Test-set model selection for human evaluation",
       "detail": "Section 6.2 states 'the highest performing model across all data proportions was chosen for human evaluation,' meaning model selection was done on test set performance rather than a validation set, biasing the human evaluation comparison."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "e-SNLI (2018) and NaturalQuestions (2019) are both publicly available well before Llama-2's training period. The base Llama-2-7b achieves 33.31% on 3-class NLI, which is above random (33.3%), yet no contamination analysis is performed."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Claims of one method outperforming another (e.g., PO vs FO, PO vs SFT) are based solely on comparing point estimates. Without significance tests, it is unclear whether observed differences are statistically meaningful, especially given small margins (e.g., 86.34% vs 85.15%)."
     }
   ],
   "cited_papers": [
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov", "Archit Sharma", "Eric Mitchell", "Stefano Ermon", "Christopher D. Manning", "Chelsea Finn"],
       "year": 2023,
       "relevance": "Core method for aligning LLMs with human preferences without a reward model, directly compared against in this work's positioning."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "InstructGPT/RLHF foundational work on aligning LLMs using human preference rankings."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "Foundational prompting technique for LLM reasoning that RESCUE builds upon for explanation generation."
     },
     {
       "title": "Tree of thoughts: Deliberate problem solving with large language models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "arxiv_id": "2305.10601",
       "relevance": "Alternative reasoning approach for LLMs using tree-structured exploration, discussed as related work."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Beck Labash"],
       "year": 2023,
       "relevance": "Self-reflection approach for LLM agents that represents an alternative to ranking-based improvement."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Base model used in RESCUE experiments; discussion of labeler disagreement in preference data informs partial ordering motivation."
     },
     {
       "title": "Fine-tuning language models from human preferences",
       "authors": ["Daniel M. Ziegler", "Nisan Stiennon", "Jeffrey Wu"],
       "year": 2020,
       "relevance": "Foundational work on RLHF for language model fine-tuning, establishing the reward model training approach that RESCUE aims to simplify."
     },
     {
       "title": "BRIO: Bringing order to abstractive summarization",
       "authors": ["Yixin Liu", "Pengfei Liu", "Dragomir Radev", "Graham Neubig"],
       "year": 2022,
       "relevance": "Prior work on ranking-based training for text generation that directly inspires RESCUE's approach."
     },
     {
       "title": "RRHF: Rank responses to align language models with human feedback without tears",
       "authors": ["Zheng Yuan", "Hongyi Yuan", "Chuanqi Tan"],
       "year": 2023,
       "relevance": "Concurrent work on ranking-based LLM alignment without reinforcement learning, closely related to RESCUE's ranking metric."
     },
     {
       "title": "Distilling step-by-step! Outperforming larger language models with less training data and smaller model sizes",
       "authors": ["Cheng-Yu Hsieh", "Chun-Liang Li", "Chih-Kuan Yeh"],
       "year": 2023,
       "relevance": "Comparison point for state-of-the-art NLI performance (89.51% with 540B model), showing RESCUE achieves competitive results with 7B model."
     },
     {
       "title": "Self-RAG: Learning to retrieve, generate, and critique through self-reflection",
       "authors": ["Akari Asai", "Zeqiu Wu", "Yizhong Wang"],
       "year": 2024,
       "relevance": "Related approach for improving LLM generation through self-critique in retrieval-augmented settings."
     },
     {
       "title": "Lost in the middle: How language models use long contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt"],
       "year": 2023,
       "relevance": "Identifies the U-shaped performance curve in long-context LLMs that RESCUE's Multi-doc QA experiments directly address."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The partial ordering approach offers a practical method for fine-tuning LLMs with limited annotated data, applicable to domain-specific tasks."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Partial ordering outperforming full ordering is a moderately interesting finding but not deeply contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper focuses on improving response quality."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; a straightforward methodological contribution."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and models are released on GitHub (https://github.com/ekonwang/RRescue), enabling reproduction."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Fudan University and Emory University; uses well-known models (Llama-2, GPT-3.5/4) but is not from a major AI lab."
     }
   }
 }