ai-research-survey

scan.json (30267B)

---

 {
   "paper": {
     "title": "RAL2M: Retrieval Augmented Learning-To-Match Against Hallucination in Compliance-Guaranteed Service Systems",
     "authors": [
       "Mengze Hong",
       "Di Jiang",
       "Jiangtao Wen",
       "Zhiyang Su",
       "Yawen Li",
       "Yanjie Sun",
       "Guan Wang",
       "Chen Jason Zhang"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.02917",
     "doi": "10.48550/arXiv.2601.02917"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RAL2M repositions LLMs as query-response matching judges rather than generators, eliminating generation hallucination by design. The proposed query-adaptive latent ensemble achieves 70.7% accuracy and 13.9% hallucination rate on a multi-domain QA benchmark, substantially outperforming majority voting (60.2%/49.2%), weighted voting, and a neural ensemble baseline. Analysis reveals heterogeneous inter-model dependencies (Mistral largely uncorrelated with other models) and that 98.5% of samples have at least one correct judge, suggesting the core challenge is recovering minority-correct judgments rather than simply adding more models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 footnote states 'Data and code are publicly available at GitHub Repo' with what appears to be a hyperlinked reference to a repository."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The English benchmark is claimed publicly available (Section 5.1 footnote). The five underlying QA datasets (HotpotQA, MS MARCO, CovidQA, ExpertQA, HAGRID) are all public. The Chinese dataset 'will be publicly released upon acceptance' (Appendix D footnote), which does not count, but the main English dataset does."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'single NVIDIA L20 GPU with 48GB of memory' and specific model names, but no requirements.txt, Dockerfile, or dependency specification is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. Hyperparameters are listed (Appendix A.3) and the prompt is given (Appendix A.4), but no commands, scripts, or README-style instructions for replicating the experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Despite running 5 seeds per ensemble model, Tables 3, 6, 7, and 10 report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'significantly outperforms strong baselines' (abstract) but no statistical significance tests (p-values, t-tests, etc.) are reported anywhere. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context: 70.7% vs 64.2% (neural model), 60.2% (majority vote). Section 6.4.1 states '12.6% accuracy gain and a 26.4% hallucination reduction' from 10% to 90% data scaling. Sufficient context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 82,606 queries but no justification is given for this sample size. No power analysis or discussion of whether the sample is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 states 'Each ensemble model was run with five different seeds and reported the average performance for robustness,' but no standard deviation, variance, or spread measure is reported in any results table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares against multiple baselines: BGE retrieval (with keyphrase and intent augmentation), single LLM inference (average and GPT-5), LLM debate, majority voting, weighted voting, and a neural ensemble model."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent methods: FrugalGPT (2024), AutoMix (2024), LLM-TOPLA (2024), LLM-Blender (2023), GPT-5 for ablation. Table 1 surveys methods from 2023-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 presents ablation studies: training data size scaling (Table 6, from 10% to 90%), and ensemble size scaling (Table 7, from 2 to 5 models). The neural model baseline also serves as an ablation isolating the latent dependency modeling contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are reported: Accuracy, Hallucination Rate (FPR), Precision, and F1 Score (Table 3, Appendix A.1)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human annotators were used for data quality (Appendix B) but not for evaluating system outputs. All evaluation of matching decisions is automated against ground-truth labels."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states 'the dataset is split 70% for training and 30% for testing.' Results are reported on the test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows per-dataset accuracy and hallucination rate across all five QA datasets (CovidQA, ExpertQA, HAGRID, HotpotQA, MS MARCO). Table 4 breaks down single LLM performance per model."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses LLM debate's excessive rejection (1% hallucination but extremely low F1). Section 6.2 notes retrieval augmentation can 'slightly distract models' (Qwen drops from 63.9% to 61.1%). Appendix E's toy example illustrates majority voting failure. Section 6.3 analyzes the 40% of samples where majority vote fails."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Intent augmentation 'biases the system toward accepting most candidates, resulting in extremely high hallucination rates' (98.8%, Table 3). LLM debate yields 'extremely low F1 scores due to excessive rejection.' Retrieval augmentation hurts Qwen performance (Table 4). GPT-5 'does not outperform smaller models' (Section 6.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims: 'eliminates generation hallucination' (supported by design — no open-ended generation), 'significantly outperforms strong baselines' (supported by Table 3, 70.7% vs 64.2% next-best). The 'wisdom of the crowd' claim is supported by Figure 4 analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions (latent variables, query-adaptive weighting, dependency modeling) are supported by controlled ablation studies: training data scaling (Table 6), ensemble size with/without the proposed method (Table 7), and comparison against the neural model that lacks dependency modeling."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper is titled 'Compliance-Guaranteed Service Systems' and motivates with healthcare and finance (Section 1), but evaluates on general QA benchmarks (HotpotQA, MS MARCO, CovidQA, ExpertQA, HAGRID). The Chinese financial dataset is closer to the compliance domain but is proprietary and not the main evaluation. The compliance-guaranteed framing extends substantially beyond what general QA matching benchmarks demonstrate."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. Possible confounds include the latent model having more parameters than baselines, training data distribution effects, or model selection effects. No robustness checks against alternative hypotheses are provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures matching accuracy and hallucination rate (defined as FPR in Appendix A.1) and reports them as such. The empirical measurements are precisely described and match the granularity of the empirical claims. The 'compliance-guaranteed' framing is about the system architecture (retrieval-based, no generation), not about inflating what the metrics measure."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 and footnote 2 specify: 'Qwen2.5-7B, Mistral-7B-v0.2, Llama-3.1-8b, Gemma-2-9b, GLM-4-9b, and GPT-5.' The encoder is 'bge-large-en-v1.5.' These include version numbers and parameter counts."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4 provides the full judgment prompt text used for all LLM inferences, including the exact instruction text, role description, and output format specification."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.3 reports: hidden dimension 512, dropout 0.3, learning rate 1e-3, 256 MC samples for training, 1024 for evaluation, 10 VI iterations for training, 60 for evaluation. Section 5.4 states temperature 0 for all LLMs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a retrieval pipeline with direct LLM inference for binary judgments, followed by ensemble aggregation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes query generation via 'intention-enhanced similar-question generation' under strict and relaxed constraints, LLM classification by two judges, filtering by five human annotators, yielding 82,606 queries. Table 2 shows per-dataset composition. Appendix B describes annotator training."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 ('Limitations') provides substantive discussion of two specific limitations: reliance on knowledge base completeness and computational cost of the LLM ensemble."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses specific threats: (1) 'performance is directly constrained by the completeness and accuracy of this knowledge base' — the system cannot handle novel topics not in D1; (2) 'requires multiple LLM inferences per user query, which can be more computationally expensive.' These are specific to this approach, not generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations discuss method constraints (cannot synthesize new information, needs KB updates for evolving topics) but do not explicitly state what the experimental results do NOT show — e.g., no statement about generalization limits to untested domains, real-world deployment conditions, or adversarial queries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 footnote states 'Data and code are publicly available at GitHub Repo.' The underlying five QA datasets (HotpotQA, MS MARCO, CovidQA, ExpertQA, HAGRID) are all publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes: 5 public QA collections extended to 10,020 QA pairs and 47,975 document chunks; query generation using intention-enhanced similar-question generation; LLM classification by two judges followed by five human annotators; 82,606 total queries with balanced positive/negative classes."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects. The data source is standard public benchmarks (HotpotQA, MS MARCO, etc.). Human annotators were recruited for data quality (Appendix B describes their qualifications) but are not study participants."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 documents the pipeline: 5 public QA collections → 10,020 QA pairs + 47,975 document chunks → query generation (aligned + misaligned) → LLM classification + human filtering → 82,606 queries → 70/30 train/test split. Distinct-N metrics (D5: 89.2%, D7: 73.6%) verify diversity."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is disclosed anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Hong Kong Polytechnic University, NYU Shanghai, HKUST, Beijing University of Posts and Telecommunications. All are academic institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper uses open-source LLMs and does not evaluate any sponsor's product, but without funding disclosure the criterion is not satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosure, or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five LLMs used (Qwen2.5-7B, Mistral-7B-v0.2, Llama-3.1-8b, Gemma-2-9b, GLM-4-9b). These models likely have training data that includes the public QA datasets used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs may have seen HotpotQA (2018), MS MARCO (2016), CovidQA (2020), ExpertQA (2024), or HAGRID (2023) answers during pre-training. This is a significant risk since several datasets predate the models' training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark is constructed from public QA datasets dating back to 2016-2023, all likely within the training data of models released in 2024-2025. No contamination analysis or mitigation is discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects. Human annotators were used for data quality but are not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants as study subjects."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.1 mentions single LLM inference processes 'approximately 23 queries per second,' but no latency or cost figures are provided for the full proposed ensemble method, which requires 5 LLM inferences plus the latent model per query."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 states 'a single NVIDIA L20 GPU with 48GB of memory' but provides no total training time, GPU hours, or computational budget for the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 states 'Each ensemble model was run with five different seeds and reported the average performance for robustness.' Results are averaged across 5 seeds, though per-seed variation is not shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 explicitly states 'five different seeds' for each ensemble model."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.3 states 'Hyperparameter tuning was performed to identify optimal configurations' and lists final values, but does not report the number of configurations tried, the search method, or computational cost of the search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.3 lists the final hyperparameter values but does not describe how they were selected — no mention of validation set performance, selection criterion, or whether all configurations were evaluated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares against 8+ baselines across multiple metrics and datasets without any statistical tests or correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own baselines (neural ensemble model, voting methods) and compare against their proposed method without acknowledging the inherent bias of evaluating one's own system against self-implemented baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The proposed method requires 5 LLM inferences plus ensemble model inference per query, while single LLM baselines require only one inference. Table 7 varies ensemble size but does not report corresponding compute costs. The compute difference is acknowledged only qualitatively in limitations."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses synthetically generated queries to evaluate matching accuracy, but no discussion of whether synthetic query matching performance transfers to real-world compliance system performance. No analysis of construct validity — whether the benchmark measures what the paper claims to evaluate."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All LLMs use the same zero-shot prompt and inference pipeline. Comparisons are between ensemble strategies, not between different model scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses QA pairs from datasets created 2016-2024. The LLMs were released in 2024-2025 and likely trained on data including these QA pairs. No discussion of temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The judgment prompt provides the full QA pair and supporting document to each LLM. No discussion of whether this setup leaks answer information or whether models' prior knowledge of these QA pairs biases their judgments."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Queries are synthetically generated from the same QA pairs used in both training and testing the matching system. No discussion of structural similarity or independence between train and test examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, decontamination, or temporal splits are used."
       }
     }
   },
   "claims": [
     {
       "claim": "RAL2M eliminates generation hallucination by repositioning LLMs as query-response matching judges rather than generators.",
       "evidence": "This is a design claim supported by the system architecture (Section 1, Figure 1): the system returns only verified responses from a knowledge base or a safe fallback, with no open-ended generation.",
       "supported": "strong"
     },
     {
       "claim": "The proposed latent ensemble achieves 70.7% accuracy and 13.9% hallucination rate, outperforming all baselines including the neural ensemble (64.2%/32.5%) and majority voting (60.2%/49.2%).",
       "evidence": "Table 3 (Section 6.1) reports these figures. However, no statistical significance tests or error bars are provided for the 5-seed runs.",
       "supported": "moderate"
     },
     {
       "claim": "98.5% of samples have at least one correct LLM judge, showing the core challenge is recovering minority-correct judgments.",
       "evidence": "Figure 4 (Section 6.3) shows the distribution of correct judges across samples, with 98.5% having ≥1, 90.6% having ≥2, and 60.2% having ≥3 correct judges.",
       "supported": "strong"
     },
     {
       "claim": "Modeling inter-model dependencies is critical for ensemble performance — naive majority voting can amplify errors from correlated models.",
       "evidence": "Table 5 shows heterogeneous dependencies (Mistral uncorrelated with others, Gemma-Qwen strongly correlated). Table 7 shows majority voting degrades severely with 3 models (48.4% accuracy, 87.2% hallucination) while the proposed method improves steadily.",
       "supported": "moderate"
     },
     {
       "claim": "The method generalizes across languages, achieving 95% accuracy on a Chinese financial services dataset.",
       "evidence": "Table 10 (Appendix D) shows 95% accuracy, 94% recall, and 95% F1 on the Chinese dataset, outperforming all baselines. However, this is a single proprietary dataset with potentially different characteristics (Section D notes 'lower overall complexity').",
       "supported": "moderate"
     },
     {
       "claim": "GPT-5 does not outperform smaller open-source models at single-LLM judgment.",
       "evidence": "Table 3 shows GPT-5 at 55.7% accuracy (61.3% hallucination), comparable to the 55.5% LLM average. Section 6.4 notes 'judgment quality is only weakly correlated with model size.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars despite multiple seeds",
       "detail": "The paper runs 5 seeds per ensemble model (Section 5.4) but reports only point estimates in all tables. With no variance information, readers cannot assess result stability or whether differences between methods are meaningful."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims including 'significantly outperforms' (abstract) are based solely on comparing raw numbers. No p-values, confidence intervals, or statistical tests are provided for any comparison."
     },
     {
       "flag": "Synthetic evaluation may not reflect real usage",
       "detail": "All 82,606 queries are synthetically generated via LLM-based 'intention-enhanced similar-question generation' (Section 5.1). Real user queries in compliance systems may have different characteristics (ambiguity, domain jargon, adversarial intent) not captured by synthetic generation."
     },
     {
       "flag": "Contamination risk from public QA datasets",
       "detail": "The benchmark uses QA pairs from HotpotQA (2018), MS MARCO (2016), CovidQA (2020), ExpertQA (2024), and HAGRID (2023). The LLMs used were released in 2024-2025 and likely saw these QA pairs during pre-training, potentially biasing their matching judgments. No contamination analysis is performed."
     },
     {
       "flag": "Compliance framing exceeds experimental scope",
       "detail": "The paper frames itself as solving compliance-guaranteed service systems (title, introduction cites healthcare and finance regulations) but evaluates on general QA benchmarks. The gap between matching accuracy on HotpotQA/MS MARCO and real-world regulatory compliance is not addressed."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "LLM cost optimization through cascading, directly compared as a baseline for LLM ensemble/routing strategies."
     },
     {
       "title": "AutoMix: Automatically mixing language models",
       "authors": ["Pranjal Aggarwal", "Aman Madaan", "Ankit Anand"],
       "year": 2024,
       "relevance": "Automated LLM mixing/routing method compared as a cascade baseline for multi-model strategies."
     },
     {
       "title": "LLM-Blender: Ensembling large language models with pairwise ranking and generative fusion",
       "authors": ["Dongfu Jiang", "Xiang Ren", "Bill Yuchen Lin"],
       "year": 2023,
       "relevance": "Foundational LLM ensemble method using pairwise ranking and fusion, directly compared in Table 1."
     },
     {
       "title": "Are more LLM calls all you need? towards the scaling properties of compound AI systems",
       "authors": ["Lingjiao Chen", "Jared Quincy Davis", "Boris Hanin"],
       "year": 2024,
       "relevance": "Studies scaling properties of compound AI systems with multiple LLM calls, directly relevant to ensemble and routing research."
     },
     {
       "title": "Harnessing multiple large language models: A survey on LLM ensemble",
       "authors": ["Zhijun Chen", "Jingzheng Li", "Pengpeng Chen"],
       "year": 2025,
       "arxiv_id": "2502.18036",
       "relevance": "Comprehensive survey on LLM ensemble methods covering strategies, architectures, and applications."
     },
     {
       "title": "ChatEval: Towards better LLM-based evaluators through multi-agent debate",
       "authors": ["Chi-Min Chan", "Weize Chen", "Yusheng Su"],
       "year": 2024,
       "relevance": "Multi-agent debate for LLM-based evaluation, used as the LLM debate baseline in this paper."
     },
     {
       "title": "Don't hallucinate, abstain: Identifying LLM knowledge gaps via multi-LLM collaboration",
       "authors": ["Shangbin Feng", "Weijia Shi", "Yike Wang"],
       "year": 2024,
       "relevance": "Multi-LLM approach to hallucination mitigation through abstention, directly relevant to the paper's hallucination reduction goals."
     },
     {
       "title": "PromptWizard: Optimizing prompts via task-aware, feedback-driven self-evolution",
       "authors": ["Eshaan Agarwal", "Raghav Magazine", "Joykirat Singh"],
       "year": 2025,
       "relevance": "Prompt optimization framework used in this paper's pipeline for judgment prompt tuning."
     },
     {
       "title": "JudgeBench: A benchmark for evaluating LLM-based judges",
       "authors": ["Sijun Tan", "Siyuan Zhuang", "Kyle Montgomery"],
       "year": 2025,
       "relevance": "Benchmark for evaluating LLM judges, directly relevant to using LLMs as judgment agents."
     },
     {
       "title": "LLM-TOPLA: Efficient LLM ensemble by maximising diversity",
       "authors": ["Selim Furkan Tekin", "Fatih Ilhan", "Tiansheng Huang"],
       "year": 2024,
       "relevance": "LLM ensemble method maximizing diversity, compared in Table 1 as a selection/regeneration baseline."
     },
     {
       "title": "Beyond majority voting: LLM aggregation by leveraging higher-order information",
       "authors": ["Rui Ai", "Yuqi Pan", "David Simchi-Levi"],
       "year": 2025,
       "arxiv_id": "2510.01499",
       "relevance": "Advanced LLM aggregation beyond majority voting, directly relevant to the ensemble aggregation challenge addressed in this paper."
     }
   ]
 }