ai-research-survey

scan.json (28927B)

---

 {
   "paper": {
     "title": "Hint-Augmented Re-ranking: Efficient Product Search using LLM-Based Query Decomposition",
     "authors": [
       "Yilun Zhu",
       "Nikhita Vedula",
       "Shervin Malmasi"
     ],
     "year": 2025,
     "venue": "IJCNLP-AACL",
     "arxiv_id": "2511.13994",
     "doi": "10.48550/arXiv.2511.13994"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM-generated structured \"hints\" (brands, features, alternative queries) decomposing superlative e-commerce queries improve retrieval by 10.9 MAP points and ranking by 5.9 MRR points. Fine-tuned small models (0.5B–3B parameters) augmented with hints outperform 72B listwise LLM rankers and DeepSeek-R1 at 14x lower compute cost. Human evaluation with 10 annotators confirms hint-augmented system wins 65% of non-tie comparisons. The approach enables concurrent hint generation and retrieval, adding only 3.5–6.8% latency overhead.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and data is available at: https://github.com/yilunzhu/superhints/' in a footnote on page 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is built on the publicly available Amazon Shopping Queries Dataset (KDD 2022 Cup), and the authors state their code and data are available at their GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.1 mentions hardware (NVIDIA L40s, A100 GPUs, AWS instance types) and training settings (learning rate, batch size, FP16), but no requirements.txt, Dockerfile, or library version specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper itself. The GitHub link is provided but the paper does not contain a 'Reproducing Results' section or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 report only point estimates. No confidence intervals, error bars, or ± notation are provided for any metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., '+6.2 points', '10.9 points in MAP') but no statistical significance tests (p-values, t-tests, bootstrap tests) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute point differences with baseline context: e.g., '3B model improves to 64.39 P@1 (+6.2 points)' and 'QE-BM25 achieves 25.93 P@1, significantly outperforming BM25 (+10.5 points over doc2query).' Table 2 provides full baseline numbers for computing relative improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 21,407 queries and 1,070,350 pairs but no justification is given for these sizes. No power analysis or rationale for why these numbers are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. Results appear to be single-run numbers. Table 3 reports p5 and p95 percentile latencies but no variance for quality metrics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: vanilla BM25, doc2query, dense retriever (gte-Qwen2-7B-instruct), listwise LLM rankers (Qwen2.5-72B, DeepSeek-R1), and pointwise models without hints."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DeepSeek-R1 (2025), Qwen2.5 family (2024), gte-Qwen2-7B-instruct (2023), and doc2query (2023). These are recent and competitive models."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates hint augmentation (P vs P+H) across two model sizes (0.5B, 3B), and compares two query generation methods (features vs queries, Table 10). The with/without hints comparison isolates the contribution of LLM-generated interpretations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six evaluation metrics are used: P@1, P@3, P@5, P@10, MAP, and MRR (Tables 2 and 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 describes human evaluation with 10 expert annotators judging 153 queries in a blind setting, with Cohen's kappa of 0.74 measuring inter-annotator agreement."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows explicit train/dev/test splits (13,914/2,140/5,353 queries). Results in Tables 2 and 4 are reported on the test set. Early stopping uses validation performance."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 (Appendix D.4) provides performance breakdowns across 16 product categories. Table 11 (Appendix D.5) stratifies by number of relevant items per query."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 and Appendix D.1 identify three error categories (brand recognition, feature interpretation, relevancy assessment) with qualitative examples showing where the baseline fails and hints help."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 10 shows that feature-based augmentation performs marginally better at P@1 but worse on other metrics than query reformulations. The listwise 72B model underperforms fine-tuned 0.5B models, demonstrating that larger models don't always win."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '10.9 points in MAP' (35.74 vs 24.79 in Table 2) and '5.9 points in MRR' (74.74 vs 68.82 in Table 2) are directly supported by the results. The claim that small models surpass large models is confirmed (3B P+H 64.39 P@1 vs 72B 35.31 P@1)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's primary causal claim is that hint augmentation improves ranking. This is supported by controlled ablation (P vs P+H) where the only variable changed is the presence of hints, using the same model architecture, training setup, and test data."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly bounds scope: 'our method addresses only English (particularly US) queries' and acknowledges limited model diversity (only Qwen family for pointwise, two open-source LLMs for listwise). Future work for non-superlative and multilingual settings is noted."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why hints improve performance. For example, improvements could be due to query expansion effects (more tokens = better matching) rather than genuine superlative interpretation, but this confound is not addressed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures P@k, MAP, and MRR directly on their ranking task and claims improvements in ranking quality. The human evaluation validates that automated metrics align with human judgment. The measurements match the granularity of the claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are used throughout: 'Claude 3.5 Sonnet v2', 'Claude 3.7 Sonnet', 'qwen2.5-72B-Instruct', 'qwen2.5-0.5B-instruct', 'qwen2.5-3B-instruct', 'gte-Qwen2-7B-instruct', 'DeepSeek-R1'. These are specific versioned model names identifiable on model hubs."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt texts are provided in the appendix: Tables 12 (query generation), 13 (relevance annotation), 14 (hint generation), and 15 (listwise ranking). These include the complete instructions sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Training hyperparameters for fine-tuned models are reported (learning rate 1e-5, batch sizes, early stopping, FP16). However, temperature, top-p, and other sampling settings for LLM inference (Claude hint generation, DeepSeek-R1 and Qwen-72B listwise ranking) are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a standard retrieve-then-rerank pipeline with offline hint generation, not an agent-based system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 3.1-3.2 document the full pipeline: extracting two-level category info, filtering subcategories with <300 products, generating 50 queries per subcategory via Claude, BM25 filtering, dense retrieval of top 50 products per query, filtering queries with 0 or ≥15 relevant products. Numbers are provided at each stage (26,993 → 21,407)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, spanning two substantial paragraphs discussing model diversity, language constraints, and future extensions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses specific issues: restricted to only two open-source LLMs for listwise ranking, pointwise experiments limited to Qwen family, English-only queries, and that superlative interpretations likely differ across languages with varying grammatical structures."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'our method addresses only English (particularly US) queries, constraining the broader applicability' and notes the need to evaluate 'more architecturally diverse models such as Phi-4' to validate the approach's generalizability."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The base dataset is the publicly available Amazon Shopping Queries Dataset (KDD 2022 Cup). The authors' constructed dataset and code are stated to be available at their GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1-3.2 describes in detail: extracting categories from the Amazon Shopping Queries Dataset, filtering subcategories, generating queries via Claude 3.7 Sonnet, BM25 filtering for zero-result queries, dense retrieval for relevance labels, and the three-label annotation scheme."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human evaluation uses '10 expert annotators' and the dataset quality check uses 'an annotator,' but no description is given of how these annotators were recruited, their qualifications, or whether the recruitment process could introduce bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented with counts at each stage: Amazon Shopping Queries Dataset → subcategory filtering (≥300 products) → 50 queries per subcategory via Claude → 26,993 candidates → BM25 filtering → dense retrieval of top 50 → filtering (0 or ≥15 relevant) → 21,407 final queries with 1,070,350 pairs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. All authors are from Amazon.com, Inc. but there is no acknowledgments section listing grants or corporate sponsorship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated as 'Amazon.com, Inc.' with Amazon email addresses. The connection to the evaluated product domain (Amazon product search) is visible."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "All authors are Amazon employees developing a method for product search. Amazon has a direct commercial interest in improved search performance. The funder (employer) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper. Amazon employees working on product search improvement have an inherent financial interest in the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the pre-trained models used (Claude 3.5 Sonnet v2, DeepSeek-R1, Qwen2.5 family, gte-Qwen2-7B-instruct). The Amazon Shopping Queries Dataset (2022) could have been in their training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Amazon Shopping Queries Dataset (from KDD 2022 Cup, publicly available) appeared in the pre-training data of the models used for listwise ranking (Qwen2.5-72B, DeepSeek-R1) or dense retrieval."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Amazon Shopping Queries Dataset was published in 2022 (KDD Cup), well before the training of DeepSeek-R1 (2025) and Qwen2.5 (2024). No discussion of whether these models may have been trained on this benchmark data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper's human evaluation uses annotators as measurement tools to evaluate system outputs, not as study participants. This is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The human evaluation involves expert annotators judging product rankings, not a human subjects study requiring ethics approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The annotators are evaluating system outputs as part of the measurement methodology, not participating as human subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study design sense. Annotators serve as evaluation instruments."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects experiment. The blind setup for annotators is part of the evaluation methodology."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. However, the paper does note the annotation was 'conducted in a blind setting, with annotators unaware of which system was which.'"
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. Annotator participation is part of the measurement methodology."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports end-to-end latency per query (seconds) on a single L40 GPU with avg, p5, and p95. Section 4.4 reports PFlops per query (0.122–0.224 for proposed models vs 1.200–1.720 for baselines). AWS instance costs are also noted ($30.13/hr, $27.45/hr)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 reports hardware (L40s, A100 GPUs), AWS instance costs, and Appendix D.2 states fine-tuning requires '3–4 hours on a single NVIDIA L40s GPU.' PFlops per query are reported in Figure 2."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. It is unclear whether results come from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Learning rate (1e-5) and batch sizes are given but no search budget is described — no mention of how many configurations were tried or what search method was used."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports using early stopping on validation performance but does not describe how the learning rate, batch size, or other hyperparameters were selected. Only the best configuration is presented with no exploration of alternatives."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, making correction for multiple comparisons inapplicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own baselines (BM25, QE-BM25, doc2query) and fine-tune their own models. No acknowledgment of the bias inherent in evaluating one's own system against one's own implementation of baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 and Appendix D.2 explicitly plot MRR/MAP against computational cost (PFlops per query, log scale), showing the efficiency-quality tradeoff across all model configurations."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The authors created their own benchmark with LLM-generated queries and LLM-generated labels. While they validate query naturalness (94.6%) and use human evaluation as a secondary check, they do not discuss whether their benchmark measures real-world superlative search behavior as opposed to LLM-generated approximations of it."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The system is a standard retrieve-and-rerank pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Amazon Shopping Queries Dataset was released in 2022. Models used (Qwen2.5, DeepSeek-R1, trained in 2024-2025) may have seen this data during pre-training. No discussion of this temporal leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the hint generation process (Claude interpreting queries) could introduce feature leakage, e.g., by encoding information about specific products that would not be available in real-time deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Products in train and test sets come from the same Amazon catalog. No discussion of whether product overlap or near-duplicate queries between splits could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is described. No decontamination pipeline, canary strings, membership inference tests, or overlap analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "QE-BM25 with superlative-aware decomposition improves retrieval by 10.9 points in MAP over vanilla BM25.",
       "evidence": "Table 2: QE-BM25 achieves 35.74 MAP vs BM25's 24.79 MAP, a 10.95-point improvement. Also outperforms doc2query (27.84 MAP) by 7.9 points.",
       "supported": "strong"
     },
     {
       "claim": "Hint-augmented 3B model (64.39 P@1) surpasses 72B listwise LLM (35.31 P@1) and DeepSeek-R1 (44.35 P@1) while being 14x more compute-efficient.",
       "evidence": "Table 2 shows ranking results. Figure 2 and Appendix D.2 show 0.122-0.224 PFlops for proposed models vs 1.200-1.720 PFlops for listwise baselines. Note: text inconsistently reports DeepSeek-R1 as '47.06 P@1' while Table 2 shows 44.35.",
       "supported": "strong"
     },
     {
       "claim": "Hint augmentation improves the 3B model by 6.2 points P@1 and 5.9 points MRR, and the 0.5B model by 4.2 points P@1.",
       "evidence": "Table 2: 3B P→P+H is 58.15→64.39 P@1 (+6.24) and 68.82→74.74 MRR (+5.92). 0.5B P→P+H is 52.47→56.71 P@1 (+4.24).",
       "supported": "strong"
     },
     {
       "claim": "Hint augmentation introduces minimal latency overhead: 3.5% for 0.5B and 6.8% for 3B models.",
       "evidence": "Table 3 reports end-to-end latency: 0.5B P 4.215s vs P+H 4.364s (+149ms, 3.5%) and 3B P 6.912s vs P+H 7.385s (+473ms, 6.8%).",
       "supported": "strong"
     },
     {
       "claim": "Human evaluation confirms hint-augmented system wins 65.38% of non-tie comparisons with substantial inter-annotator agreement (Cohen's kappa 0.74).",
       "evidence": "Section 4.3: 10 expert annotators, 153 queries, blind evaluation. 51 wins for hints, 27 for baseline, 75 ties. Win rate excluding ties: 65.38%.",
       "supported": "moderate"
     },
     {
       "claim": "Results are robust to judge choice — Amazon Nova Pro as alternative judge confirms consistent relative performance patterns.",
       "evidence": "Table 4 in Appendix A.2 shows same ranking of methods with Nova Pro judge. 3B+H achieves 72.50 P@1 (vs 64.39 with Claude). Relative hint improvement remains: +4.9 for 3B, +10.4 for 0.5B.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Amazon conflict of interest",
       "detail": "All three authors are Amazon employees developing a method for Amazon product search using the Amazon Shopping Queries Dataset. Amazon has a direct commercial interest in demonstrating improved search performance. No conflict of interest statement is present."
     },
     {
       "flag": "Internal number discrepancy",
       "detail": "The text in Section 4.3 states 'deepseek-r1 achieves superior results (47.06 P@1)' but Table 2 reports 44.35 P@1. This 2.7-point discrepancy suggests possible reporting errors or confusion between different evaluation runs."
     },
     {
       "flag": "No statistical testing on any result",
       "detail": "All performance claims are based on comparing point estimates from apparently single experimental runs. No significance tests, confidence intervals, or variance measures are reported for quality metrics. The observed improvements could be within noise."
     },
     {
       "flag": "Circular evaluation concern",
       "detail": "Claude 3.5 Sonnet v2 generates both the hints and the relevance labels used for primary evaluation. Although the authors validate with Nova Pro as an alternative judge (Appendix A.2), the primary results table uses the same model family for both data generation and evaluation."
     },
     {
       "flag": "LLM-generated benchmark validity",
       "detail": "Both the queries and the relevance labels are generated by LLMs (Claude). The query quality check (94.6% valid) was done by a single annotator on 1,000 samples. The benchmark may capture LLM preferences rather than actual user search behavior."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Directly addresses LLM cost reduction strategies, relevant to efficiency and routing in LLM-based systems."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Major reasoning-focused LLM used as baseline; relevant to LLM capability and reasoning enhancement."
     },
     {
       "title": "Large Language Models are Effective Text Rankers with Pairwise Ranking Prompting",
       "authors": ["Zhen Qin", "Rolf Jagerman", "Kai Hui"],
       "year": 2024,
       "relevance": "Demonstrates LLM zero-shot ranking capability, foundational work on LLM-based information retrieval."
     },
     {
       "title": "Is ChatGPT Good at Search? Investigating Large Language Models as Re-Ranking Agents",
       "authors": ["Weiwei Sun", "Lingyong Yan", "Xinyu Ma"],
       "year": 2023,
       "relevance": "Evaluates LLMs as ranking agents, relevant to LLM capability assessment in applied tasks."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational prompting technique used in the hint generation pipeline; central to LLM reasoning methodology."
     },
     {
       "title": "Query Expansion by Prompting Large Language Models",
       "authors": ["Rolf Jagerman", "Honglei Zhuang", "Zhen Qin"],
       "year": 2023,
       "arxiv_id": "2305.03653",
       "relevance": "Directly relevant baseline approach for LLM-based query expansion in information retrieval."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt"],
       "year": 2024,
       "relevance": "Documents LLM context window limitations that motivate the chunking approach used in listwise ranking."
     },
     {
       "title": "Qwen2.5 Technical Report",
       "authors": ["An Yang", "Baosong Yang", "Beichen Zhang"],
       "year": 2024,
       "arxiv_id": "2412.15115",
       "relevance": "Technical report for the model family used in both pointwise fine-tuning and listwise ranking experiments."
     },
     {
       "title": "Phi-4 Technical Report",
       "authors": ["Marah Abdin", "Jyoti Aneja", "Harkirat Behl"],
       "year": 2024,
       "arxiv_id": "2412.08905",
       "relevance": "Small language model mentioned as future work for validating approach across diverse architectures."
     },
     {
       "title": "ReasoningRank: Teaching Student Models to Rank through Reasoning-Based Knowledge Distillation",
       "authors": ["Yuelyu Ji", "Zhuochun Li", "Rui Meng"],
       "year": 2025,
       "arxiv_id": "2410.05168",
       "relevance": "Closely related work on distilling reasoning capabilities from large to small models for ranking tasks."
     }
   ]
 }