ai-research-survey

scan.json (27489B)

---

 {
   "paper": {
     "title": "RankLLM: A Python Package for Reranking with LLMs",
     "authors": [
       "Sahel Sharifymoghaddam",
       "Ronak Pradeep",
       "Andre Slavescu",
       "Ryan Nguyen",
       "Andrew Xu",
       "Zijian Chen",
       "Yilin Zhang",
       "Yidi Chen",
       "Jasper Xian",
       "Jimmy Lin"
     ],
     "year": 2025,
     "venue": "SIGIR 2025",
     "arxiv_id": "2505.19284",
     "doi": "10.1145/3726302.3730331"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RankLLM is an open-source Python package supporting pointwise, pairwise, and listwise LLM-based reranking in a modular framework. Reproduction experiments on TREC DL19-DL23 show specialized fine-tuned models (RankZephyr, DuoT5) outperform out-of-the-box LLMs, while GPT-4o-mini generates 28-75% malformed responses depending on prompt template yet still achieves competitive nDCG@10 through graceful error handling. The framework emphasizes reproducibility through end-to-end scripts and two-click reproduction pages.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Repository publicly available at rankllm.ai and installable via PyPI ('pip install rank-llm[all]'). GitHub URL github.com/castorini/rank_llm provided. Source installation instructions included."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All experiments use publicly available TREC DL19-DL23 datasets from MS MARCO V1/V2. The retriever component supports these datasets with prebuilt indexes and caching."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific package version (0.25.0) given for pip install with dependency specification via PyPI. GPU hardware specified ('single NVIDIA RTX A6000 GPU'). CUDA-enabled PyTorch required as prerequisite."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 5: 'All reported results are generated by running the end-to-end demos/experimental_results.py script.' Two-click reproduction (2CR) pages provided with reproduction matrices. run_rank_llm.py wrapper script encapsulates retrieval and reranking."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 reports point estimates of nDCG@10 only. No confidence intervals or error bars. The paper explicitly states 'we report single-run results' and acknowledges non-deterministic behavior."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests used. Table 1 compares models by raw nDCG@10 scores without any tests of statistical difference."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 shows raw nDCG@10 scores with BM25 baseline in the same table, but no explicit effect sizes, percentage improvements, or relative gains are reported."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for using DL19-DL23 datasets specifically, or discussion of whether five test collections provide sufficient statistical power for the comparisons shown."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 explicitly states: 'To save costs, we report single-run results.' No variance, standard deviation, or spread measures across runs despite acknowledging that 'out-of-the-box prompt-decoders rank non-deterministically.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "BM25 first-stage baseline shown in Table 1. Multiple reranking approaches compared: pointwise (MonoT5), pairwise (DuoT5), and various listwise methods."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent models: Gemini Flash 2.0, GPT-4o-mini, LLaMA 3.1 8B, Qwen 2.5 7B (all 2024-2025 models), alongside established reranking models."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study of framework components. The sliding window parameters (window size 20, stride 10) and prompt templates are configurable but not ablated. Rows 4d-4f compare prompt templates but this is a comparison, not a systematic ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only nDCG@10 is reported for ranking quality (Table 1). The paper mentions supporting mAP@100 and recall@20 (Section 4.4, Figure 8) but does not report these in the experiments."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to this paper's claims. The framework reproduces benchmark results using TREC relevance judgments as the established ground truth for IR evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results reported on standard TREC DL19-DL23 test sets with established qrels (relevance judgments). These are well-defined test collections separate from any training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 breaks down results per dataset (DL19-DL23) and per model/method. Table 2 breaks down malformed response types (OK, Wrong Format, Repetition, Missing) per model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows malformed response distributions. Paper discusses that GPT-4o-mini generates '28% to 75% missing candidate ids' depending on prompt template. Section 4.3 describes how malformed responses are processed gracefully."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: RankVicuna underperforms most methods, RankGPTAPEER has 75.7% missing documents, Qwen 2.5 has 26.4% repetition errors, and out-of-the-box LLMs frequently generate malformed responses."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about modularity, configurability, diverse model support, reproducibility features, and integration with Pyserini are all demonstrated through the paper's architecture description, code examples, and experimental results."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is primarily a systems/framework paper that describes and demonstrates a toolkit. It does not make causal claims about what causes performance differences — it presents reproduction results and notes that 'comparing model effectiveness is not the main focus.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are appropriately scoped to the reranking task on specific TREC DL19-DL23 datasets. The paper does not overclaim beyond the tested setting and explicitly notes it is a framework paper, not a model comparison paper."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for observed performance differences between models. No consideration of confounding factors such as model size, training data differences, or prompt sensitivity beyond noting non-determinism."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures nDCG@10 and frames it as 'reranking effectiveness,' which is the standard meaning in IR. No proxy gap — the measurement matches the claim granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models have specific HuggingFace paths (e.g., 'castorini/monot5-3b-msmarco-10k', 'castorini/first_mistral'). Gemini uses specific version 'gemini-2.0-flash-001'. However, GPT-4o-mini (used for rows 4d-4f, 3 key experiments) has no snapshot date or API version specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompt templates (RankGPT, RankGPTAPEER, LRL) are named and implemented in the open-source repository at rankllm.ai. While full prompt text is not reproduced in the paper, the code with exact prompts is publicly available."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Reranking-specific parameters reported: window size 20, stride 10, top-100 candidates, context size 4096 for some models. However, LLM generation hyperparameters (temperature, top-p, max tokens) are not stated for any model despite using LLM APIs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RankLLM is a reranking framework that sends prompts to LLMs and processes responses — it does not involve autonomous agents, tool use, or multi-step reasoning loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: BM25 retrieves top 100 candidates, sliding window (size 20, stride 10) processes them, long passages truncated to fit context size, malformed responses processed by removing duplicates and appending missing IDs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section in the paper. Section 6 (Discussion) compares to related tools but does not discuss limitations of RankLLM or the experimental evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity discussed anywhere in the paper. The only acknowledgment of a limitation is the brief note that results are single-run due to cost constraints."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries stated. While the paper notes 'comparing model effectiveness is not the main focus,' it does not state what the results do NOT show or what populations/settings are excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All underlying data is publicly available: TREC DL19-DL23 datasets, MS MARCO corpus, and qrels. The reproduction scripts can regenerate all results from scratch."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are standard IR benchmarks (TREC DL19-DL23 from MS MARCO V1/V2). Collection method is BM25 retrieval of top 100 candidates per query from prebuilt indexes, well described in Section 4.2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public IR benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented: query → BM25 retrieval (top 100) → Request object creation → sliding window reranking → Result objects → TREC eval format → nDCG@10 evaluation. Each step is explained with code examples."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses NSERC, Microsoft Accelerating Foundation Models Research program, and IITP grant from Korean Government (MSIT)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All ten authors are affiliated with University of Waterloo. Author affiliations are clearly listed. The castorini lab created several evaluated models (RankVicuna, RankZephyr, MonoT5, DuoT5, LiT5) but this relationship is implicit from the shared affiliation and citation structure."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (NSERC, Microsoft research program, Korean government IITP grant) are academic funding bodies with no direct stake in which reranking model performs best. The paper is a framework demonstration, not a product evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement in the paper. The authors' lab created several of the evaluated models (RankVicuna, RankZephyr, LiT5, MonoT5, DuoT5) which could constitute a form of interest."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for any model used (GPT-4o-mini, Gemini Flash 2.0, LLaMA 3.1, Qwen 2.5). These models may have been trained on data containing TREC/MS MARCO benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. MS MARCO and TREC DL datasets are widely published and could appear in training data of the evaluated models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. TREC DL19-DL23 and MS MARCO have been publicly available since before the training cutoffs of models like GPT-4o-mini and Gemini Flash 2.0. No discussion of contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, latency measurements, or per-query timing reported. The paper mentions using GPT-4o-mini and Gemini Flash APIs but does not report API costs, wall-clock time, or tokens consumed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware mentioned ('single NVIDIA RTX A6000 GPU') but no total compute budget: no GPU hours, wall-clock time, or API costs quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 explicitly states: 'To save costs, we report single-run results.' No seed sensitivity analysis despite acknowledging that models 'rank non-deterministically.'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'we report single-run results' and 'For more accurate results, we recommend running each experiment multiple times.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search conducted or budget reported. Default settings used throughout without exploration of alternatives."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses default/standard configurations for each model and explicitly states the defaults (window size 20, stride 10, top 100 candidates). Since this is a reproduction study, using original papers' defaults is the appropriate choice."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors' lab (castorini, University of Waterloo) created RankVicuna, RankZephyr, LiT5, MonoT5, DuoT5, and FirstMistral — 6 of 12 evaluated systems. This potential self-evaluation bias is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of performance at matched compute budgets. Models range from small local models (MonoT5-3B) to large proprietary APIs (GPT-4o-mini, Gemini Flash) with vastly different compute requirements, but this is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether nDCG@10 on TREC DL19-DL23 actually measures what matters for practical reranking quality. The benchmarks are used without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Different models use different prompt templates, context sizes, and inference frameworks. Rows 4d-4f compare GPT-4o-mini with different prompts (showing prompt effect), but the scaffold confound across models is not explicitly discussed as a variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. DL19-DL23 benchmarks predate the training of GPT-4o-mini, Gemini Flash 2.0, LLaMA 3.1, and Qwen 2.5 — these models may have seen the benchmark data during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. No discussion of whether the reranking setup leaks information not available in real-world usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. No discussion of independence between training data of evaluated models and the MS MARCO/TREC test collections."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods used. No canary strings, membership inference tests, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "RankLLM supports diverse reranking models (pointwise, pairwise, listwise) across proprietary and open-source LLMs in a modular framework.",
       "evidence": "Architecture described in Sections 3-4 with code examples (Figures 3-9). Table 1 shows results across 12 different model configurations spanning all three reranking paradigms.",
       "supported": "strong"
     },
     {
       "claim": "Graceful handling of malformed LLM responses allows competitive ranking results despite high error rates in out-of-the-box prompt-decoders.",
       "evidence": "Table 2 shows GPT-4o-mini has 28-75% malformed responses depending on prompt template, yet Table 1 shows competitive nDCG@10 scores (e.g., RankGPT at 0.7338 on DL19). Section 4.3 describes the graceful processing pipeline.",
       "supported": "moderate"
     },
     {
       "claim": "The framework enables reproducible results from prior work (RankGPT, RankVicuna, RankZephyr, etc.).",
       "evidence": "Table 1 presents reproduction results. End-to-end script demos/experimental_results.py generates all results. 2CR reproduction pages provided. However, results are single-run with no variance reported, and 'newer model versions' are used where applicable, so exact reproduction is not verified.",
       "supported": "moderate"
     },
     {
       "claim": "Specialized fine-tuned rerankers (RankZephyr, DuoT5) outperform out-of-the-box LLMs on TREC DL benchmarks.",
       "evidence": "Table 1 shows RankZephyr (0.7412 on DL19) outperforms Qwen 2.5 (0.6784) and LLaMA 3.1 (0.6688). However, single-run results without significance testing make this comparison unreliable.",
       "supported": "weak"
     },
     {
       "claim": "RankLLM has achieved widespread community adoption, as evidenced by GitHub stars, PyPI downloads, and integration with frameworks like LlamaIndex, LangChain, and Rerankers.",
       "evidence": "Stated in Section 1 and Section 7 but no specific numbers (download counts, star counts) are provided to quantify adoption.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Self-evaluation bias",
       "detail": "The authors' lab (castorini, University of Waterloo) created 6 of 12 evaluated systems (RankVicuna, RankZephyr, LiT5, MonoT5, DuoT5, FirstMistral) and the framework itself. This self-evaluation bias is not acknowledged. RankZephyr (their model) achieves the best overall results in Table 1."
     },
     {
       "flag": "Single-run results despite known non-determinism",
       "detail": "The paper explicitly acknowledges that 'out-of-the-box prompt-decoders rank non-deterministically' but reports only single-run results 'to save costs,' making all comparative claims unreliable."
     },
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All 60 numbers in Table 1 are point estimates with no confidence intervals, standard deviations, or significance tests despite the acknowledged non-determinism of the evaluated systems."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity section, despite clear limitations including single-run results, potential contamination, self-evaluation bias, and single-metric evaluation."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "All TREC DL benchmarks (2019-2023) and MS MARCO predate the training of the proprietary models (GPT-4o-mini, Gemini Flash). These benchmarks could be in training data, potentially inflating the proprietary models' scores. Not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "Is ChatGPT Good at Search? Investigating Large Language Models as Re-Ranking Agents",
       "authors": ["Weiwei Sun", "Lingyong Yan", "Xinyu Ma"],
       "year": 2023,
       "relevance": "Foundational work on LLM-based listwise reranking (RankGPT), directly reproduced in this paper."
     },
     {
       "title": "RankVicuna: Zero-shot Listwise Document Reranking with Open-Source Large Language Models",
       "authors": ["Ronak Pradeep", "Sahel Sharifymoghaddam", "Jimmy Lin"],
       "year": 2023,
       "arxiv_id": "2309.15088",
       "relevance": "Demonstrates distilling proprietary LLM reranking capability into open-source models, a core use case of RankLLM."
     },
     {
       "title": "RankZephyr: Effective and Robust Zero-Shot Listwise Reranking is a Breeze!",
       "authors": ["Ronak Pradeep", "Sahel Sharifymoghaddam", "Jimmy Lin"],
       "year": 2023,
       "arxiv_id": "2312.02724",
       "relevance": "State-of-the-art open-source listwise reranker achieving better results than its teacher model, key model in RankLLM ecosystem."
     },
     {
       "title": "FIRST: Faster Improved Listwise Reranking with Single Token Decoding",
       "authors": ["Revanth Gangi Reddy", "JaeHyeok Doo", "Yifei Xu"],
       "year": 2024,
       "relevance": "Novel single-token decoding approach for efficient listwise reranking using learning-to-rank losses, integrated into RankLLM."
     },
     {
       "title": "Efficient Memory Management for Large Language Model Serving with Paged-Attention",
       "authors": ["Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang"],
       "year": 2023,
       "relevance": "vLLM inference framework used by RankLLM for efficient open-source model serving."
     },
     {
       "title": "Drowning in Documents: Consequences of Scaling Reranker Inference",
       "authors": ["Mathew Jacob", "Erik Lindgren", "Matei Zaharia"],
       "year": 2024,
       "arxiv_id": "2411.11767",
       "relevance": "Analyzes scalability challenges of LLM-based reranking, directly relevant to the efficiency concerns RankLLM addresses."
     },
     {
       "title": "Zero-Shot Listwise Document Reranking with a Large Language Model",
       "authors": ["Xueguang Ma", "Xinyu Zhang", "Ronak Pradeep", "Jimmy Lin"],
       "year": 2023,
       "arxiv_id": "2305.02156",
       "relevance": "LRL zero-shot listwise reranking method, one of the prompt templates implemented and evaluated in RankLLM."
     },
     {
       "title": "Rankify: A Comprehensive Python Toolkit for Retrieval, Re-Ranking, and Retrieval-Augmented Generation",
       "authors": ["Abdelrahman Abdallah", "Jamshid Mozafari", "Bhawna Piryani"],
       "year": 2025,
       "arxiv_id": "2502.02464",
       "relevance": "Concurrent reranking toolkit covering retrieval, re-ranking, and RAG — direct competitor to RankLLM."
     },
     {
       "title": "The LLaMA 3 Herd of Models",
       "authors": ["Abhimanyu Dubey", "Abhinav Jauhri"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "LLaMA 3.1 8B used as an out-of-the-box reranking model in RankLLM's benchmark evaluation."
     },
     {
       "title": "Building a Culture of Reproducibility in Academic Research",
       "authors": ["Jimmy Lin"],
       "year": 2022,
       "arxiv_id": "2212.13534",
       "relevance": "Foundational work on two-click reproducibility (2CR) methodology adopted by RankLLM."
     }
   ]
 }